From b488cf4347e30ad4eddeb93c7ea11b48e10d87d0 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Sun, 20 Apr 2014 19:04:41 -0700 Subject: [PATCH] refactor screenshots to work for all pages --- Gemfile | 2 +- Gemfile.lock | 12 ++----- app.rb | 11 ++---- models/site.rb | 67 ++++++++++++++++++++---------------- views/browse.erb | 2 +- views/dashboard.erb | 4 +-- workers/screenshot_worker.rb | 42 ++++++++++++---------- 7 files changed, 70 insertions(+), 70 deletions(-) diff --git a/Gemfile b/Gemfile index 5f3a88aa..74a4b86c 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,6 @@ gem 'puma', require: nil gem 'rubyzip', require: 'zip' gem 'rack-recaptcha', require: 'rack/recaptcha' gem 'rmagick', require: nil -gem 'selenium-webdriver', require: nil gem 'sidekiq' gem 'ago' gem 'mail' @@ -19,6 +18,7 @@ gem 'google-api-client', require: 'google/api_client' gem 'tilt' gem 'erubis' gem 'stripe', :git => 'https://github.com/stripe/stripe-ruby' +gem 'screencap' platform :mri do gem 'magic' # sudo apt-get install file, For OSX: brew install libmagic diff --git a/Gemfile.lock b/Gemfile.lock index 3bedd6ad..aae4ddd8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -30,8 +30,6 @@ GEM minitest (>= 2) celluloid (0.15.2) timers (~> 1.1.0) - childprocess (0.5.2) - ffi (~> 1.0, >= 1.0.11) cliver (0.3.2) coderay (1.1.0) columnize (0.3.6) @@ -136,11 +134,8 @@ GEM rmagick (2.13.2) rubyzip (1.1.2) safe_yaml (1.0.1) - selenium-webdriver (2.40.0) - childprocess (>= 0.5.0) - multi_json (~> 1.0) - rubyzip (~> 1.0) - websocket (~> 1.0.4) + screencap (0.1.1) + phantomjs sequel (4.8.0) sequel_pg (1.6.9) pg (>= 0.8.0) @@ -189,7 +184,6 @@ GEM webmock (1.17.4) addressable (>= 2.2.7) crack (>= 0.3.2) - websocket (1.0.7) websocket-driver (0.3.2) xpath (2.0.0) nokogiri (~> 1.3) @@ -230,7 +224,7 @@ DEPENDENCIES ruby-debug rubyzip sass - selenium-webdriver + screencap sequel sequel_pg shotgun diff --git a/app.rb b/app.rb index 7df87f6c..64258892 100644 --- a/app.rb +++ b/app.rb @@ -437,7 +437,7 @@ end post '/site_files/delete' do require_login - sanitized_filename = params[:filename].gsub(/[^a-zA-Z0-9_\-.]/, '') + sanitized_filename = Site.sanitize_filename params[:filename] current_site.delete_file(sanitized_filename) @@ -485,17 +485,10 @@ post '/site_files/save/:filename' do |filename| halt 'File is too large to fit in your space, it has NOT been saved. Please make a local copy and then try to reduce the size.' end - sanitized_filename = filename.gsub(/[^a-zA-Z0-9_\-.]/, '') + sanitized_filename = Site.sanitize_filename filename current_site.store_file sanitized_filename, tempfile - if sanitized_filename =~ /index\.html/ - ScreenshotWorker.perform_async current_site.username - current_site.update site_changed: true - end - - current_site.update changed_count: 1+current_site.changed_count, updated_at: Time.now - 'ok' end diff --git a/models/site.rb b/models/site.rb index cba4a26f..9998397f 100644 --- a/models/site.rb +++ b/models/site.rb @@ -24,7 +24,7 @@ class Site < Sequel::Model application/xml audio/midi } - VALID_EXTENSIONS = %w{ + VALID_EXTENSIONS = %w{ html htm txt text css js jpg jpeg png gif svg md markdown eot ttf woff json geojson csv tsv mf ico pdf asc key pgp xml mid midi } @@ -40,28 +40,30 @@ class Site < Sequel::Model VALID_HOSTNAME = /^[a-z0-9][a-z0-9-]+?[a-z0-9]$/i # http://tools.ietf.org/html/rfc1123 # FIXME smarter DIR_ROOT discovery - DIR_ROOT = './' - TEMPLATE_ROOT = File.join DIR_ROOT, 'views', 'templates' - PUBLIC_ROOT = File.join DIR_ROOT, 'public' - SITE_FILES_ROOT = File.join PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites') + DIR_ROOT = './' + TEMPLATE_ROOT = File.join DIR_ROOT, 'views', 'templates' + PUBLIC_ROOT = File.join DIR_ROOT, 'public' + SITE_FILES_ROOT = File.join PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites') + SCREENSHOTS_ROOT = File.join(PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'site_screenshots_test' : 'site_screenshots')) + SCREENSHOTS_URL_ROOT = '/site_screenshots' many_to_one :server - + many_to_many :tags - + one_to_many :follows one_to_many :followings, key: :actioning_site_id, class: :Follow - + one_to_many :tips one_to_many :tippings, key: :actioning_site_id, class: :Tip - + one_to_many :blocks one_to_many :blockings, key: :actioning_site_id, class: :Block - + one_to_many :stats - + one_to_many :events - + one_to_many :changes class << self @@ -128,6 +130,7 @@ class Site < Sequel::Model %w{index not_found}.each do |name| File.write file_path("#{name}.html"), render_template("#{name}.erb") + ScreenshotWorker.perform_async values[:username], "#{name}.html" end FileUtils.cp template_file_path('cat.png'), file_path('cat.png') @@ -162,7 +165,7 @@ class Site < Sequel::Model def self.valid_file_type?(uploaded_file) mime_type = Magic.guess_file_mime_type uploaded_file[:tempfile].path - return true if (Site::VALID_MIME_TYPES.include?(mime_type) || mime_type =~ /text/) && + return true if (Site::VALID_MIME_TYPES.include?(mime_type) || mime_type =~ /text/) && Site::VALID_EXTENSIONS.include?(File.extname(uploaded_file[:filename]).sub(/^./, '').downcase) false end @@ -171,13 +174,13 @@ class Site < Sequel::Model FileUtils.mv uploaded.path, file_path(filename) File.chmod(0640, file_path(filename)) - if filename =~ /index\.html/ - ScreenshotWorker.perform_async values[:username] - self.site_changed = true - save(validate: false) - end + ScreenshotWorker.perform_async values[:username], filename + + self.site_changed = true + self.changed_count += 1 + save(validate: false) end - + def increment_changed_count self.changed_count += 1 self.updated_at = Time.now @@ -250,14 +253,14 @@ class Site < Sequel::Model if new? && values[:username].length > 2 && !values[:username].match(VALID_HOSTNAME) errors.add :username, 'A valid user/site name is required.' end - + if new? && values[:username].length > 32 errors.add :username, 'User/site name cannot exceed 32 characters.' end - # Check for existing user + # Check for existing user user = self.class.select(:id, :username).filter(username: values[:username]).first - + if user if user.id != values[:id] errors.add :username, 'This username is already taken. Try using another one.' @@ -267,7 +270,7 @@ class Site < Sequel::Model if values[:password].nil? || (@password_length && @password_length < MINIMUM_PASSWORD_LENGTH) errors.add :password, "Password must be at least #{MINIMUM_PASSWORD_LENGTH} characters." end - + if !values[:domain].nil? && !values[:domain].empty? if !(values[:domain] =~ /^[a-zA-Z0-9.-]+\.[a-zA-Z0-9]+$/i) || values[:domain].length > 90 errors.add :domain, "Domain provided is not valid. Must take the form of domain.com" @@ -291,7 +294,7 @@ class Site < Sequel::Model def files_path(name=nil) File.join SITE_FILES_ROOT, (name || username) end - + def file_path(filename) File.join files_path, filename end @@ -309,7 +312,7 @@ class Site < Sequel::Model space = Dir.glob(File.join(files_path, '*')).collect {|p| File.size(p)}.inject {|sum,x| sum += x} space.nil? ? 0 : space end - + def used_space_in_megabytes (used_space_in_bytes.to_f / self.class::ONE_MEGABYTE_IN_BYTES).round(2) end @@ -318,7 +321,7 @@ class Site < Sequel::Model remaining = maximum_space_in_bytes - used_space_in_bytes remaining < 0 ? 0 : remaining end - + def available_space_in_megabytes (available_space_in_bytes.to_f / self.class::ONE_MEGABYTE_IN_BYTES).round(2) end @@ -339,18 +342,22 @@ class Site < Sequel::Model def supporter? !values[:stripe_customer_id].nil? end - + # This will return false if they have ended their support plan. def ended_supporter? values[:ended_plan] end - + def plan_name return 'Free Plan' if !supporter? || (supporter? && ended_supporter?) 'Supporter Plan' end - + def title values[:title] || values[:username] end -end + + def screenshot_url(filename, resolution) + "#{SCREENSHOTS_URL_ROOT}/#{values[:username]}/#{filename}.#{resolution}.jpg" + end +end \ No newline at end of file diff --git a/views/browse.erb b/views/browse.erb index f1e3ef0e..c639bbb8 100644 --- a/views/browse.erb +++ b/views/browse.erb @@ -49,7 +49,7 @@ <% @sites.each do |site| %>
  • - + <%= site.username %> diff --git a/views/dashboard.erb b/views/dashboard.erb index a1901068..7b0561e2 100644 --- a/views/dashboard.erb +++ b/views/dashboard.erb @@ -30,7 +30,7 @@ @@ -78,7 +78,7 @@
    <% if file.ext.match(/html|htm/) %>
    - +
    <% elsif file.ext.match(/jpg|png|bmp|gif/) %> diff --git a/workers/screenshot_worker.rb b/workers/screenshot_worker.rb index bc7d74d3..3663d58d 100644 --- a/workers/screenshot_worker.rb +++ b/workers/screenshot_worker.rb @@ -1,34 +1,40 @@ -require 'selenium-webdriver' require 'RMagick' class ScreenshotWorker + REQUIRED_RESOLUTIONS = ['235x141', '105x63', '270x162'] + SCREENSHOTS_PATH = File.join DIR_ROOT, 'public', 'site_screenshots' include Sidekiq::Worker sidekiq_options queue: :screenshots, retry: 3, backtrace: true - def perform(username) + def perform(username, filename) screenshot = Tempfile.new 'neocities_screenshot' screenshot.close + screenshot_output_path = screenshot.path+'.png' - caps = Selenium::WebDriver::Remote::Capabilities.htmlunit javascript_enabled: true, takesScreenshot: true + f = Screencap::Fetcher.new("http://#{username}.neocities.org/#{filename}") + f.fetch( + output: screenshot_output_path, + width: 1280, + height: 720 + ) - driver = Selenium::WebDriver.for :remote, url: $config['phantomjs_url'][rand($config['phantomjs_url'].length)], desired_capabilities: caps - driver.manage.window.resize_to 1280, 720 - - wait = Selenium::WebDriver::Wait.new(timeout: 10) # seconds - wait.until { - driver.navigate.to "http://#{username}.neocities.org" - driver.save_screenshot screenshot.path - } - - driver.quit img_list = Magick::ImageList.new - img_list.read screenshot.path + img_list.from_blob File.read(screenshot_output_path) + screenshot.unlink + File.unlink screenshot_output_path + img_list.new_image(img_list.first.columns, img_list.first.rows) { self.background_color = "white" } img = img_list.reverse.flatten_images - img.crop!(0, 0, 1280, 720) - img.resize! 208, 125 - img.write File.join(DIR_ROOT, 'public', 'site_screenshots', "#{username}.jpg") + + user_screenshots_path = File.join SCREENSHOTS_PATH, username + FileUtils.mkdir_p user_screenshots_path + + REQUIRED_RESOLUTIONS.each do |res| + img.scale(*res.split('x').collect {|r| r.to_i}).write(File.join(user_screenshots_path, "#{filename}.#{res}.jpg")) { + self.quality = 90 + } + end end -end +end \ No newline at end of file