Let's Encrypt script is getting ugly, but covers the edge cases now.

Needs refactor eventually.

workers/lets_encrypt_worker.rb

@@ -1,6 +1,7 @@
 class LetsEncryptWorker
   class NotAuthorizedYetError < StandardError; end
   class VerificationTimeoutError < StandardError; end
+  class InvalidAuthError < StandardError; end
   class VerifyNotFoundWithDomain < StandardError; end
   include Sidekiq::Worker
   sidekiq_options queue: :lets_encrypt_worker, retry: 5, backtrace: true
@@ -9,6 +10,9 @@ class LetsEncryptWorker
     1.hour.to_i
   end
 
+  # If you need to clear scheduled jobs:
+  # Sidekiq::ScheduledSet.new.select {|s| JSON.parse(s.value)['class'] == 'LetsEncryptWorker'}.length
+
   def letsencrypt
     Acme::Client.new(
       private_key: OpenSSL::PKey::RSA.new(File.read($config['letsencrypt_key'])),
@@ -28,53 +32,124 @@ class LetsEncryptWorker
     site = Site[site_id]
     return if site.domain.blank? || site.is_deleted || site.is_banned
 
-    domains = [site.domain, "www.#{site.domain}"]
+    return if site.values[:domain].match /\.neocities\.org$/i
+
+    domain_raw = site.values[:domain].gsub(/^www\./, '')
+
+    domains = [domain_raw, "www.#{domain_raw}"]
+
+    verified_domains = []
 
     domains.each_with_index do |domain, index|
-      auth = letsencrypt.authorize domain: domain
-      challenge = auth.http01
+      puts "verifying accessability of test file on #{domain}"
+      challenge_base_path = File.join '.well-known', 'acme-challenge'
+      testfile_name, testfile_key = "test#{UUIDTools::UUID.random_create}", SecureRandom.hex
+      testfile_fs_path = File.join site.base_files_path, challenge_base_path
 
-      FileUtils.mkdir_p File.join(site.base_files_path, File.dirname(challenge.filename)) if index == 0
-      File.write File.join(site.base_files_path, challenge.filename), challenge.file_content
+      begin
+        FileUtils.mkdir_p File.join(site.base_files_path, challenge_base_path)
+        File.write File.join(testfile_fs_path, testfile_name), testfile_key
+      rescue => e
+        puts "ERROR WRITING TO WELLKNOWN FILE, SKIPPING #{domain}: #{e.inspect}"
+        next
+      end
 
       # Ensure that both domains work before sending request. Let's Encrypt has a low
       # pending request limit, and it takes one week (!) to flush out.
-      sleep 2
-      challenge_url = "#{domain}/#{challenge.filename}"
-      ["http://#{challenge_url}", "http://www.#{challenge_url}"].each do |url|
-        res = HTTP.follow.get(url)
-        raise VerifyNotFoundWithDomain unless res.status == 200
+
+      challenge_url = "http://#{domain}/#{challenge_base_path}/#{testfile_name}"
+
+      puts "testing #{challenge_url}"
+
+      begin
+        res = HTTP.timeout(:global, write: 5, connect: 10, read: 10).follow.get(challenge_url)
+      rescue
+        puts "error with #{challenge_url}"
+        next
+      end
+
+      if res.status != 200 && res.body != testfile_key
+        puts "CONTENT DOWNLOADED DID NOT MATCH #{challenge_url}"
+        next
+      end
+
+      puts "test succeeded, sending challenge request verification"
+
+      begin
+        auth = letsencrypt.authorize domain: domain
+        challenge = auth.http01
+      rescue Acme::Client::Error::Malformed
+        puts "international domains not supported yet, quitting"
+        return
+      end
+
+      begin
+        FileUtils.mkdir_p File.join(site.base_files_path, File.dirname(challenge.filename))
+        File.write File.join(site.base_files_path, challenge.filename), challenge.file_content
+      rescue => e
+        put "FAILED TO WRITE CHALLENGE: #{site.domain} #{challenge.filename}"
+        # A verification needs to be made anyways, otherwise 300 of them will jam up the system for a week
       end
 
       challenge.request_verification
 
-      sleep 60
+      sleep 1
       attempts = 0
 
-      begin
-        puts "WAITING FOR #{domain} VALIDATION"
+      while true
+        result = challenge.verify_status
+        puts "#{domain} : #{result}"
+
+        if result == 'valid'
+          puts "VALIDATED: #{domain}"
+          clean_wellknown_turds site
+          verified_domains.push domain
+          break
+        end
 
         raise VerificationTimeoutError if attempts == 60
-        raise NotAuthorizedYetError if challenge.verify_status != 'valid'
-      rescue NotAuthorizedYetError
-        sleep 20
+
+        if result == 'invalid'
+          puts "returned invalid, walking away"
+          clean_wellknown_turds site
+          break
+        end
+
         attempts += 1
-        retry
-      ensure
-        clean_wellknown_turds site
+        sleep 2
       end
-      puts "DONE!"
     end
 
-    csr = Acme::Client::CertificateRequest.new names: domains
-    certificate = letsencrypt.new_certificate csr
-    site.ssl_key = certificate.request.private_key.to_pem
-    site.ssl_cert = certificate.fullchain_to_pem
-    site.save_changes validate: false
+    if verified_domains.empty?
+      puts "no verified domains, skipping"
+      return
+    end
+
+    puts "verified domains: #{verified_domains.inspect}"
+
     clean_wellknown_turds site
 
+    retries = 0
+    begin
+      csr = Acme::Client::CertificateRequest.new names: verified_domains
+      certificate = letsencrypt.new_certificate csr
+    rescue Acme::Client::Error => e
+      if retries == 2
+        puts "Failed to create cert, returning: #{e.message}"
+        return
+      end
+      retries += 1
+      retry
+    end
+
+    site.ssl_key = certificate.request.private_key.to_pem
+    site.ssl_cert = certificate.fullchain_to_pem
+    site.cert_updated_at = Time.now
+    site.save_changes validate: false
+
     # Refresh the cert periodically, current expire time is 90 days
-    LetsEncryptWorker.perform_in 60.days, site.id
+    # We're going for a cron task for this now, so this is commented out.
+    #LetsEncryptWorker.perform_in 60.days, site.id
   end
 
   def clean_wellknown_turds(site)