Better error handling for screenshot and thumbnail workers, hacky fix for lets encrypt cert issue that's temporary

Rakefile

@@ -30,24 +30,6 @@ HERE
 end
 =end
 
-desc "retry dead screenshots"
-task :retry_dead_screenshots => [:environment] do
-  ds = Sidekiq::DeadSet.new
-
-  ds.select { |job|
-    job.klass == 'ScreenshotWorker'
-  }.map(&:retry)
-end
-
-desc "delete dead screenshots"
-task :delete_dead_screenshots => [:environment] do
-  ds = Sidekiq::DeadSet.new
-
-  ds.select { |job|
-    job.klass == 'ScreenshotWorker'
-  }.map(&:delete)
-end
-
 desc "prune logs"
 task :prune_logs => [:environment] do
   Stat.prune!

workers/lets_encrypt_worker.rb

@@ -64,7 +64,11 @@ class LetsEncryptWorker
       puts "testing #{challenge_url}"
 
       begin
-        res = HTTP.timeout(connect: 10, write: 10, read: 10).follow.get(challenge_url)
+        # Some dumb letsencrypt related cert expiration issue hotfix
+        ctx = OpenSSL::SSL::SSLContext.new
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+        res = HTTP.timeout(connect: 10, write: 10, read: 10).follow.get(challenge_url, ssl_context: ctx)
       rescue => e
         puts e.inspect
         puts "error with #{challenge_url}"

workers/screenshot_worker.rb

@@ -6,7 +6,7 @@ class ScreenshotWorker
   HARD_TIMEOUT = 30.freeze
   PAGE_WAIT_TIME = 5.freeze # 3D/VR sites take a bit to render after loading usually.
   include Sidekiq::Worker
-  sidekiq_options queue: :screenshots, retry: 2, backtrace: true
+  sidekiq_options queue: :screenshots, retry: 10, backtrace: true
 
   def perform(username, path)
     site = Site[username: username]
@@ -47,13 +47,15 @@ class ScreenshotWorker
       base_image_tmpfile_path = "/tmp/#{SecureRandom.uuid}.png"
 
       http_resp = HTTP.basic_auth(user: api_user, pass: api_password).get(uri)
-      BlackBox.new(site, path).check_uri(http_resp.headers['X-URL']) if defined?(BlackBox)
+      BlackBox.new(site, path).check_uri(http_resp.headers['X-URL']) if defined?(BlackBox) && http_resp.headers['X-URL']
       File.write base_image_tmpfile_path, http_resp.to_s
 
       user_screenshots_path = File.join SCREENSHOTS_PATH, Site.sharding_dir(username), username
       screenshot_path = File.join user_screenshots_path, File.dirname(path_for_screenshot)
       FileUtils.mkdir_p screenshot_path unless Dir.exist?(screenshot_path)
 
+      FileUtils.cp base_image_tmpfile_path, File.join(user_screenshots_path, "#{path_for_screenshot}.png")
+
       Site::SCREENSHOT_RESOLUTIONS.each do |res|
         width, height = res.split('x').collect {|r| r.to_i}
 
@@ -69,6 +71,8 @@ class ScreenshotWorker
       end
 
       true
+    rescue WebP::EncoderError => e
+      puts "Failed: #{username} #{path} #{e.inspect}"
     rescue => e
       raise e
     ensure

workers/thumbnail_worker.rb

@@ -24,12 +24,21 @@ class ThumbnailWorker
       format = File.extname(path).gsub('.', '')
       full_thumbnail_path = File.join(user_thumbnails_path, "#{path}.#{res}.webp")
 
+      begin
         image = Rszr::Image.load site_file_path
+      rescue Rszr::LoadError
+        next
+      end
+
+      begin
         if image.width > image.height
           image.resize! width, :auto
         else
           image.resize! :auto, height
         end
+      rescue Rszr::TransformationError
+        next
+      end
 
       begin
         tmpfile = "/tmp/#{SecureRandom.uuid}.png"