diff --git a/Gemfile b/Gemfile index 2b92885d..dcf35e6f 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,7 @@ gem 'google-api-client', require: 'google/api_client' gem 'tilt' gem 'erubis' gem 'stripe', '1.15.0' #, source: 'https://code.stripe.com/' -gem 'screencap', '~> 0.1.4' +#gem 'screencap', '~> 0.1.4' gem 'cocaine' gem 'zipruby' gem 'sass', require: nil @@ -38,7 +38,7 @@ platform :mri, :rbx do gem 'pg' gem 'sequel_pg', require: nil gem 'hiredis' - gem 'rainbows', require: nil + gem 'posix-spawn' group :development, :test do gem 'pry' diff --git a/files/phantomjs_screenshot.js b/files/phantomjs_screenshot.js new file mode 100644 index 00000000..0ac8e24b --- /dev/null +++ b/files/phantomjs_screenshot.js @@ -0,0 +1,56 @@ +var page = require('webpage').create() +var system = require('system') + +var maxTimeout = 15000 + +if (system.args.length === 1) { + console.log('required args: '); + phantom.exit(1) +} + +var address = system.args[1] +var outputPath = system.args[2] + +page.viewportSize = { width: 1280, height: 960 } +page.clipRect = { top: 0, left: 0, width: 1280, height: 960} + +/* +In development, not working yet. + +page.settings.scriptTimeout = 1000 + +page.onLongRunningScript = function() { + page.stopJavaScript() + phantom.exit(4) +} +*/ + +var t = Date.now() + +console.log('Loading ' + address) + +setTimeout(function() { + console.log('timeout') + phantom.exit(62) +}, maxTimeout) + +page.settings.resourceTimeout = maxTimeout + +page.onResourceTimeout = function(e) { + console.log(e.errorCode) + console.log(e.errorString) + console.log(e.url) + phantom.exit(3) +} + +page.open(address, function(status) { + if(status !== 'success') { + console.log('failed') + phantom.exit(2) + } + + page.render(outputPath) + console.log('Loading time ' + (Date.now() - t) + ' msec'); + phantom.exit() +}) + diff --git a/workers/screenshot_worker.rb b/workers/screenshot_worker.rb index 16701df8..76f55255 100644 --- a/workers/screenshot_worker.rb +++ b/workers/screenshot_worker.rb @@ -4,32 +4,9 @@ require 'securerandom' require 'thread' require 'open3' -# Don't judge - Ruby handling of timeouts is a joke.. -module Phantomjs - def self.run(*args, &block) - pid = nil - stdin, stdout, stderr, wait_thr = nil - begin - Timeout::timeout(30) do - stdin, stdout, stderr, wait_thr = Open3.popen3(path, *args) - pid = wait_thr.pid - wait_thr.join - return stdout.read - end - # :nocov: - rescue Timeout::Error - stdin.close - stdout.close - stderr.close - Process.kill 'QUIT', pid - raise Timeout::Error - end - # :nocov: - end -end - class ScreenshotWorker SCREENSHOTS_PATH = Site::SCREENSHOTS_ROOT + HARD_TIMEOUT = 20.freeze include Sidekiq::Worker sidekiq_options queue: :screenshots, retry: 3, backtrace: true @@ -39,42 +16,43 @@ class ScreenshotWorker screenshot.close screenshot_output_path = screenshot.path+'.png' - begin - f = Screencap::Fetcher.new("http://#{username}.neocities.org#{path}") - f.fetch( - output: screenshot_output_path, - width: 1280, - height: 960, - maxRenderWait: 25000, - cutoffWait: 30000 - ) - rescue Timeout::Error - # :nocov: - puts "#{username}/#{path} is timing out, discontinuing" - site = Site[username: username] - site.is_crashing = true - site.save_changes validate: false + line = Cocaine::CommandLine.new( + "timeout #{HARD_TIMEOUT} phantomjs #{File.join DIR_ROOT, 'files', 'phantomjs_screenshot.js'}", ":url :output", + expected_outcodes: [0] + ) - # Don't enable until we know it works well. + begin + output = line.run( + url: "http://#{username}.neocities.org#{path}", + output: screenshot_output_path + ) + rescue Cocaine::ExitStatusError => e + if e.message && e.message.match(/returned 124/) + puts "#{username}/#{path} is timing out, discontinuing" + site = Site[username: username] + site.is_crashing = true + site.save_changes validate: false + return true =begin - if site.email - EmailWorker.perform_async({ - from: 'web@neocities.org', - to: site.email, - subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow", - body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+ - "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+ - "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+ - "so we're taking steps to inform you and see if you can resolve the issue. "+ - "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+ - "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+ - "a mistake."+ - "\n\nOur best,\n- Neocities" - }) - end + if site.email + EmailWorker.perform_async({ + from: 'web@neocities.org', + to: site.email, + subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow", + body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+ + "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+ + "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+ + "so we're taking steps to inform you and see if you can resolve the issue. "+ + "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+ + "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+ + "a mistake."+ + "\n\nOur best,\n- Neocities" + }) + end =end - return - # :nocov: + else + raise + end end img_list = Magick::ImageList.new @@ -99,7 +77,6 @@ class ScreenshotWorker else new_img = img.scale width, height end - new_img.write(File.join(user_screenshots_path, "#{path}.#{res}.jpg")) { self.quality = 90 }