image thumbnails, catch crashy sites with screenshots

.gitignore

@@ -19,3 +19,4 @@ tests/coverage
 config.yml
 .DS_Store
 public/assets/css/.sass-cache/
+public/site_thumbnails

environment.rb

@@ -45,6 +45,7 @@ Sidekiq.configure_client do |config|
   config.redis = { namespace: 'neocitiesworker' }
 end
 
+require File.join(DIR_ROOT, 'workers', 'thumbnail_worker.rb')
 require File.join(DIR_ROOT, 'workers', 'screenshot_worker.rb')
 require File.join(DIR_ROOT, 'workers', 'email_worker.rb')
 

migrations/021_add_site_is_crashing.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :is_crashing, :boolean, default: false
+  }
+
+  down {
+    DB.drop_column :sites, :is_crashing
+  }
+end

models/site.rb

@@ -45,7 +45,13 @@ class Site < Sequel::Model
   PUBLIC_ROOT          = File.join DIR_ROOT, 'public'
   SITE_FILES_ROOT      = File.join PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites')
   SCREENSHOTS_ROOT     = File.join(PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'site_screenshots_test' : 'site_screenshots'))
+  THUMBNAILS_ROOT      = File.join(PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'site_thumbnails_test' : 'site_thumbnails'))
   SCREENSHOTS_URL_ROOT = '/site_screenshots'
+  THUMBNAILS_URL_ROOT  = '/site_thumbnails'
+  IMAGE_REGEX          = /jpg|jpeg|png|bmp|gif/
+  LOSSLESS_IMAGE_REGEX = /png|bmp|gif/
+  LOSSY_IMAGE_REGEX    = /jpg|jpeg/
+  HTML_REGEX           = /htm|html/
 
   many_to_one :server
 
@@ -174,7 +180,13 @@ class Site < Sequel::Model
     FileUtils.mv uploaded.path, file_path(filename)
     File.chmod(0640, file_path(filename))
 
-    ScreenshotWorker.perform_async values[:username], filename
+    ext = File.extname(filename).gsub('.', '')
+
+    if ext.match HTML_REGEX
+      ScreenshotWorker.perform_async values[:username], filename
+    elsif ext.match IMAGE_REGEX
+      ThumbnailWorker.perform_async values[:username], filename
+    end
 
     self.site_changed = true
     self.changed_count += 1
@@ -357,7 +369,21 @@ class Site < Sequel::Model
     values[:title] || values[:username]
   end
 
+  def screenshot_exists?(filename, resolution)
+    File.exist? File.join(SCREENSHOTS_ROOT, values[:username], "#{filename}.#{resolution}.jpg")
+  end
+
   def screenshot_url(filename, resolution)
     "#{SCREENSHOTS_URL_ROOT}/#{values[:username]}/#{filename}.#{resolution}.jpg"
   end
+  
+  def thumbnail_exists?(filename, resolution)
+    ext = File.extname(filename).gsub('.', '').match(LOSSY_IMAGE_REGEX) ? 'jpg' : 'png'
+    File.exist?(File.join(THUMBNAILS_ROOT, values[:username], "#{filename}.#{resolution}.#{ext}"))
+  end
+  
+  def thumbnail_url(filename, resolution)
+    ext = File.extname(filename).gsub('.', '').match(LOSSY_IMAGE_REGEX) ? 'jpg' : 'png'
+    "#{THUMBNAILS_URL_ROOT}/#{values[:username]}/#{filename}.#{resolution}.#{ext}"
+  end
 end

views/dashboard.erb

@@ -60,6 +60,12 @@
   To get started, click on the <strong>index.html</strong> file below to edit it. It's your home page! You can add more files (such as images) from your computer by dragging them into the box below. Need help building web sites? Check out these <a href="/tutorials">tutorials</a>!
 </div>
 
+  <% if @error %>
+    <div class="alert alert-block alert-error">
+      <p><%= @error %></p>
+    </div>
+  <% end %>
+
 <div class="files">
   <div class="header">
     <div class="breadcrumbs">My Files</div> <!-- Should be Home when Folders are implemented -->
@@ -76,14 +82,14 @@
       <div class="upload-Boundary with-instruction"> <!--should remove instruction after at least one new file have been uploaded-->
         <% current_site.file_list.each do |file| %>
           <div class="file filehover">
-            <% if file.ext.match(/html|htm/) %>
+            <% if file.ext.match(Site::HTML_REGEX) && current_site.screenshot_exists?(file.filename, '105x63') %>
               <div class="html-thumbnail html fileimagehover">
                 <img src="<%= current_site.screenshot_url(file.filename, '105x63') %>">
                 <div class="overlay"></div>
               </div>
-            <% elsif file.ext.match(/jpg|png|bmp|gif/) %>
+            <% elsif file.ext.match(Site::IMAGE_REGEX) && current_site.thumbnail_exists?(file.filename, '105x63') %>
               <div class="html-thumbnail image fileimagehover">
-                <img src="https://neocities.org/assets/img/cat-larger.png" style="">
+                <img src="<%= current_site.thumbnail_url(file.filename, '105x63') %>" style="">
                 <div class="overlay"></div>
               </div>
             <% else %>

views/tutorials.erb

@@ -1,8 +1,12 @@
-<div class="content single-Col">
+<div class="header-Outro">
+  <div class="row content single-Col">
+  <h1>Learn How to Make Web Sites!</h1>
+  <h3 class="subtitle"></h3>
+  </div>
+</div>
 
-  <h2 class="alpha">
-    Learn to Code!
-  </h2>
+<div class="content single-Col">
+  <p>Learning how to make web sites is one of the most important things you can learn. It allows you to create any content you want, arrange that content in any way you like, and share that content with the world. Be it an <a href="http://sheldonbrown.com" target="_blank">information portal for teaching people how to repair bikes</a>, or an interesting graphics project, or your poetry. These are things you can't do with traditional social networks, but that you can do with HTML.</p>
 
   <p class="delta">
     We are currently building out our library of tutorials.

workers/screenshot_worker.rb

@@ -1,4 +1,30 @@
 require 'RMagick'
+require 'timeout'
+require 'securerandom'
+require 'thread'
+require 'open3'
+
+# Don't judge - Ruby handling of timeouts is a joke..
+module Phantomjs
+  def self.run(*args, &block)
+    pid = nil
+    stdin, stdout, stderr, wait_thr = nil
+    begin
+      Timeout::timeout(50) do
+        stdin, stdout, stderr, wait_thr = Open3.popen3(path, *args)
+        pid = wait_thr.pid
+        wait_thr.join
+        return stdout.read
+      end
+    rescue Timeout::Error
+      stdin.close
+      stdout.close
+      stderr.close
+      Process.kill 'QUIT', pid
+      raise Timeout::Error
+    end
+  end
+end
 
 class ScreenshotWorker
   REQUIRED_RESOLUTIONS = ['235x141', '105x63', '270x162']
@@ -11,13 +37,38 @@ class ScreenshotWorker
     screenshot.close
     screenshot_output_path = screenshot.path+'.png'
 
-    f = Screencap::Fetcher.new("http://#{username}.neocities.org/#{filename}")
-    f.fetch(
-      output: screenshot_output_path,
-      width: 1280,
-      height: 720
-    )
+    begin
+      f = Screencap::Fetcher.new("http://#{username}.neocities.org/#{filename}")
+      f.fetch(
+        output: screenshot_output_path,
+        width: 1280,
+        height: 720
+      )
+    rescue Timeout::Error
+      puts "#{username}/#{filename} is timing out, discontinuing"
+      site = Site[username: username]
+      site.update is_crashing: true
       
+      # Don't enable until we know it works well.
+=begin
+      if site.email
+        EmailWorker.perform_async({
+          from: 'web@neocities.org',
+          to: site.email,
+          subject: "[NeoCities] The web page \"#{filename}\" on your site (#{username}.neocities.org) is slow",
+          body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+
+                "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+
+                "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+
+                "so we're taking steps to inform you and see if you can resolve the issue. "+
+                "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+
+                "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+
+                "a mistake."+
+                "\n\nOur best,\n- Neocities"
+        })
+      end
+=end
+      return
+    end
 
     img_list = Magick::ImageList.new
     img_list.from_blob File.read(screenshot_output_path)

workers/thumbnail_worker.rb

@@ -0,0 +1,28 @@
+require 'RMagick'
+
+class ThumbnailWorker
+  REQUIRED_RESOLUTIONS = ['105x63']
+  THUMBNAILS_PATH = File.join DIR_ROOT, 'public', 'site_thumbnails'
+  include Sidekiq::Worker
+  sidekiq_options queue: :thumbnails, retry: 3, backtrace: true
+
+  def perform(username, filename)
+    img_list = Magick::ImageList.new
+    img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, filename))
+    img = img_list.first
+
+    user_thumbnails_path = File.join THUMBNAILS_PATH, username
+    FileUtils.mkdir_p user_thumbnails_path
+
+    REQUIRED_RESOLUTIONS.each do |res|
+      resimg = img.resize_to_fit(*res.split('x').collect {|r| r.to_i})
+      format = File.extname(filename).gsub('.', '')
+
+      save_ext = format.match(Site::LOSSY_IMAGE_REGEX) ? 'jpg' : 'png'
+
+      resimg.write(File.join(user_thumbnails_path, "#{filename}.#{res}.#{save_ext}")) {
+        self.quality = 90
+      }
+    end
+  end
+end