Merge branch 'v2' of github.com:neocities/neocities into v2

Gemfile

@@ -11,7 +11,6 @@ gem 'puma',               require: nil
 gem 'rubyzip',            require: 'zip'
 gem 'rack-recaptcha',     require: 'rack/recaptcha'
 gem 'rmagick',            require: nil
-gem 'selenium-webdriver', require: nil
 gem 'sidekiq'
 gem 'ago'
 gem 'mail'
@@ -19,6 +18,7 @@ gem 'google-api-client',  require: 'google/api_client'
 gem 'tilt'
 gem 'erubis'
 gem 'stripe', :git => 'https://github.com/stripe/stripe-ruby'
+gem 'screencap'
 
 platform :mri do
   gem 'magic' # sudo apt-get install file, For OSX: brew install libmagic

Gemfile.lock

@@ -30,8 +30,6 @@ GEM
       minitest (>= 2)
     celluloid (0.15.2)
       timers (~> 1.1.0)
-    childprocess (0.5.2)
-      ffi (~> 1.0, >= 1.0.11)
     cliver (0.3.2)
     coderay (1.1.0)
     columnize (0.3.6)
@@ -136,11 +134,8 @@ GEM
     rmagick (2.13.2)
     rubyzip (1.1.2)
     safe_yaml (1.0.1)
-    selenium-webdriver (2.40.0)
+    screencap (0.1.1)
-      childprocess (>= 0.5.0)
+      phantomjs
-      multi_json (~> 1.0)
-      rubyzip (~> 1.0)
-      websocket (~> 1.0.4)
     sequel (4.8.0)
     sequel_pg (1.6.9)
       pg (>= 0.8.0)
@@ -189,7 +184,6 @@ GEM
     webmock (1.17.4)
       addressable (>= 2.2.7)
       crack (>= 0.3.2)
-    websocket (1.0.7)
     websocket-driver (0.3.2)
     xpath (2.0.0)
       nokogiri (~> 1.3)
@@ -230,7 +224,7 @@ DEPENDENCIES
   ruby-debug
   rubyzip
   sass
-  selenium-webdriver
+  screencap
   sequel
   sequel_pg
   shotgun

app.rb

@@ -389,38 +389,55 @@ get '/site_files/upload' do
   slim :'site_files/upload'
 end
 
+def file_upload_response(error=nil)
+  http_error_code = 406
+
+  if params[:from_button]
+    if error
+      @error = error
+      halt 200, erb(:'dashboard')
+    else
+      redirect '/dashboard'
+    end
+  else
+    halt http_error_code, error if error
+    halt 200, 'File(s) successfully uploaded.'
+  end
+end
+
 post '/site_files/upload' do
   require_login
   @errors = []
   http_error_code = 406
 
-  if params[:newfile] == '' || params[:newfile].nil?
+  params[:files].each do |file|
-    @errors << 'You must select a file to upload.'
+    if current_site.file_size_too_large? file[:tempfile].size
-    halt http_error_code, 'Did not receive file upload.'
+      file_upload_response "#{file[:filename]} is too large, upload cancelled."
     end
 
-  if current_site.file_size_too_large? params[:newfile][:tempfile].size
+    if !Site.valid_file_type? file
-    @errors << 'File size must be smaller than available space.'
+      file_upload_response "#{file[:filename]}: file type is not allowed on Neocities, upload cancelled."
-    halt http_error_code, 'File size must be smaller than available space.'
+    end
   end
 
-  unless Site.valid_file_type? params[:newfile]
+  uploaded_size = params[:files].collect {|f| f[:tempfile].size}.inject{|sum,x| sum + x }
-    @errors << 'File must me one of the following: HTML, Text, Image (JPG PNG GIF JPEG SVG), JS, CSS, Markdown.'
+
-    halt http_error_code, 'File type is not supported.' # slim(:'site_files/new')
+  if current_site.file_size_too_large? uploaded_size
+    file_upload_response "File(s) do not fit in your available space, upload cancelled."
   end
 
-  sanitized_filename = params[:newfile][:filename].gsub(/[^a-zA-Z0-9_\-.]/, '')
+  params[:files].each do |file|
+    current_site.store_file Site.sanitize_filename(file[:filename]), file[:tempfile]
+  end
 
-  current_site.store_file sanitized_filename, params[:newfile][:tempfile]
   current_site.increment_changed_count
 
-  flash[:success] = "Successfully uploaded file #{sanitized_filename}."
+  file_upload_response
-  redirect '/dashboard'
 end
 
 post '/site_files/delete' do
   require_login
-  sanitized_filename = params[:filename].gsub(/[^a-zA-Z0-9_\-.]/, '')
+  sanitized_filename = Site.sanitize_filename params[:filename]
 
   current_site.delete_file(sanitized_filename)
 
@@ -468,17 +485,10 @@ post '/site_files/save/:filename' do |filename|
     halt 'File is too large to fit in your space, it has NOT been saved. Please make a local copy and then try to reduce the size.'
   end
 
-  sanitized_filename = filename.gsub(/[^a-zA-Z0-9_\-.]/, '')
+  sanitized_filename = Site.sanitize_filename filename
 
   current_site.store_file sanitized_filename, tempfile
 
-  if sanitized_filename =~ /index\.html/
-    ScreenshotWorker.perform_async current_site.username
-    current_site.update site_changed: true
-  end
-
-  current_site.update changed_count: 1+current_site.changed_count, updated_at: Time.now
-
   'ok'
 end
 

models/site.rb

@@ -44,6 +44,8 @@ class Site < Sequel::Model
   TEMPLATE_ROOT        = File.join DIR_ROOT, 'views', 'templates'
   PUBLIC_ROOT          = File.join DIR_ROOT, 'public'
   SITE_FILES_ROOT      = File.join PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites')
+  SCREENSHOTS_ROOT     = File.join(PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'site_screenshots_test' : 'site_screenshots'))
+  SCREENSHOTS_URL_ROOT = '/site_screenshots'
 
   many_to_one :server
 
@@ -128,6 +130,7 @@ class Site < Sequel::Model
 
     %w{index not_found}.each do |name|
       File.write file_path("#{name}.html"), render_template("#{name}.erb")
+      ScreenshotWorker.perform_async values[:username], "#{name}.html"
     end
 
     FileUtils.cp template_file_path('cat.png'), file_path('cat.png')
@@ -171,12 +174,12 @@ class Site < Sequel::Model
     FileUtils.mv uploaded.path, file_path(filename)
     File.chmod(0640, file_path(filename))
 
-    if filename =~ /index\.html/
+    ScreenshotWorker.perform_async values[:username], filename
-      ScreenshotWorker.perform_async values[:username]
+
     self.site_changed = true
+    self.changed_count += 1
     save(validate: false)
   end
-  end
 
   def increment_changed_count
     self.changed_count += 1
@@ -353,4 +356,8 @@ class Site < Sequel::Model
   def title
     values[:title] || values[:username]
   end
+
+  def screenshot_url(filename, resolution)
+    "#{SCREENSHOTS_URL_ROOT}/#{values[:username]}/#{filename}.#{resolution}.jpg"
+  end
 end

views/browse.erb

@@ -49,7 +49,7 @@
     <% @sites.each do |site| %>
       <li>
         <a href="http://<%= site.username %>.neocities.org" class="neo-Screen-Shot" target="_blank" title="Website of <%= site.username %>">
-          <span class="img-Holder" style="background:url(/site_screenshots/<%= site.username %>.jpg) no-repeat;">
+          <span class="img-Holder" style="background:url(<%= site.screenshot_url('index.html', '270x162') %>) no-repeat;">
             <img src="/img/placeholder.png" alt="<%= site.username %>" />
           </span>
         </a>

views/dashboard.erb

@@ -1,14 +1,28 @@
-<script type="text/javascript">
+<style>
-  function confirmFileDelete(name) {
+  .dz-default {
-    $('#deleteFileName').html(name);
+    display: none;
-    $('#deleteConfirmModal').modal();
   }
+  .dz-preview {
+    display: none;
+  }
+  .dz-processing {
+    display: none;
+  }
+  .dz-error {
+    display: none;
+  }
+  .dz-image-preview {
+    display: none;
+  }
+</style>
+<!--
+<form action="/site_files/upload" class="dropzone" id="uploads">
+  <input name="csrf_token" type="hidden" value="#{csrf_token}">
+</form>
 
-  function fileDelete() {
+
-    $('#deleteFilenameInput').val($('#deleteFileName').html());
+<div id="upload_status" style="font-size: 16pt"></div>
-    $('#deleteFilenameForm').submit();
+-->
-  }
-</script>
 
 <div class="header-Outro">
   <div class="row content wide" style="padding-top: 10px">
@@ -16,7 +30,7 @@
     <div class="col col-50 signup-Area" style="width: 289px;">
       <div class="signup-Form">
   	  <fieldset class="content">
-        <img class="screenshot" src="/site_screenshots/<%= current_site.username %>.jpg">
+        <img class="screenshot" src="<%= current_site.screenshot_url('index.html', '270x162') %>">
 	    </fieldset>
       </div>
     </div>
@@ -26,10 +40,10 @@
     <p style="font-size:19px; margin-top: -11px; margin-bottom: 8px;"><a href="http://<%= current_site.username %>.neocities.org" target="_blank">http://<%= current_site.username %>.neocities.org</a></p>
 	<ul>
 <% if current_site.updated_at %>
-	<li>Last updated <%= current_site.updated_at %></li>
+	<li>Last updated <%= current_site.updated_at.ago %></li>
 <% end %>
-	<li>Using <strong><%= current_site.space_percentage_used %>% of your <%= current_site.maximum_space_in_megabytes %> MB</strong>. <% if !current_site.supporter? %>Need more space? <a href="/plan">Become a Supporter!</a><% end %></li>
+	<li>Using <strong><%= current_site.space_percentage_used %>% (<%= current_site.used_space_in_megabytes %>MB) of your <%= current_site.maximum_space_in_megabytes %> MB</strong>. <% if !current_site.supporter? %>Need more space? <a href="/plan">Become a Supporter!</a><% end %></li>
-	<li><strong><%= current_site.hits.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse %></strong> visitors</li>
+	<li><strong><%= current_site.hits.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse %></strong> hits</li>
 	</ul>
 	</div>
 
@@ -52,17 +66,19 @@
     <div class="actions">
       <a href="/site_files/new_page" class="btn-Action new-Page"><span>New Page</span></a>
       <!--<a href="" class="btn-Action new-Folder"><span>New Folder</span></a>-->
-      <a href="/site_files/upload" class="btn-Action upload"><span>Upload</span></a>
+      <a href="#" class="btn-Action upload" onclick="clickUploadFiles(); return false"><span>Upload</span></a>
     </div>
   </div>
   <div class="list">
+    <form action="/site_files/upload" class="dropzone" id="uploads">
+      <div class="dz-message" style="display: none"></div>
+      <input name="csrf_token" type="hidden" value="<%= csrf_token %>">
       <div class="upload-Boundary with-instruction"> <!--should remove instruction after at least one new file have been uploaded-->
-      
         <% current_site.file_list.each do |file| %>
           <div class="file filehover">
             <% if file.ext.match(/html|htm/) %>
               <div class="html-thumbnail html fileimagehover">
-              <img src="https://neocities.org//site_screenshots/bigpig.jpg">
+                <img src="<%= current_site.screenshot_url(file.filename, '105x63') %>">
                 <div class="overlay"></div>
               </div>
             <% elsif file.ext.match(/jpg|png|bmp|gif/) %>
@@ -96,9 +112,9 @@
           </div>
         <% end %>
       </div>
+    </form>
   </div>
 </div>
-
       <form method="POST" action="/site_files/delete" id="deleteFilenameForm">
         <input name="csrf_token" type="hidden" value="<%= csrf_token %>">
         <input type="hidden" id="deleteFilenameInput" name="filename">
@@ -124,3 +140,49 @@
 
 </div>
 </main>
+
+<form id="uploadFilesButtonForm" method="POST" action="/site_files/upload" enctype=multipart/form-data style="display: none">
+  <input name="csrf_token" type="hidden" value="<%= csrf_token %>">
+  <input name="from_button" type="hidden" value="true">
+  <input id="uploadFiles" type="file" name="files[]" multiple onchange="$('#uploadFilesButtonForm').submit()">
+</form>
+
+<script src="/js/dropzone.min.js"></script>
+<script type="text/javascript">
+
+  function confirmFileDelete(name) {
+    $('#deleteFileName').html(name);
+    $('#deleteConfirmModal').modal();
+  }
+
+  function fileDelete() {
+    $('#deleteFilenameInput').val($('#deleteFileName').html());
+    $('#deleteFilenameForm').submit();
+  }
+
+  function clickUploadFiles() {
+    $("input[id='uploadFiles']").click()
+  }
+
+  Dropzone.options.uploads = {
+    paramName: 'files',
+    maxFilesize: <%= current_site.available_space_in_megabytes %>,
+    clickable: false,
+    addRemoveLinks: false,
+    dictDefaultMessage: '',
+    uploadMultiple: true,
+    init: function() {
+      this.on("successmultiple", function(file) {
+        location.reload()
+      })
+
+      this.on("error", function(file, errorMessage) {
+        alert('Failed: '+errorMessage)
+      })
+
+      this.on("uploadprogress", function(file, progress) {
+        // $('#upload_progress_bar').css('width', progress+'%');
+      })
+    }
+  }
+</script>

views/dashboard.slim

@@ -1,95 +0,0 @@
-css:
-  #files a {
-    color: #D00000;
-    text-decoration: none;
-  }
-
-javascript:
-  function confirmFileDelete(name) {
-    $('#deleteFileName').html(name);
-    $('#deleteConfirmModal').modal();
-  }
-
-  function fileDelete() {
-    $('#deleteFilenameInput').val($('#deleteFileName').html());
-    $('#deleteFilenameForm').submit();
-  }
-
-.content-Base
-  .row.content
-    .col.col-60
-
-      h1 Your Website
-      p These are the files on your site. <strong>index.html</strong> is the main page, it is the first page loaded when users visit your site. <strong>not_found.html</strong> is the page users see when they try to access a file on the site that's not there (it is also commonly referred to as the 404 page).
-      - current_site.file_list.each do |file|
-        hr style="margin: 12px 0px"
-        .row style="margin-bottom: 0px" id="files"
-          .col style="padding-left: 20px;"
-            a href="http://#{current_site.username}.neocities.org/#{file.filename}" title="View #{file.filename}" target="_blank"
-              - if file.ext.match(/jpg|png|bmp|gif/)
-                i class="icon-picture icon": span style="margin-left: 5px" #{file.filename}
-              - else
-                i class="icon-file-alt icon": span style="margin-left: 10px;" #{file.filename}
-            span class="float-Right"
-              - if file.ext.match(/html|htm|txt|js|css|md/)
-                a href="/site_files/text_editor/#{file.filename}" : i class="icon-edit" style="margin-right: 10px" title="Edit"
-              - else
-                span style="margin-left: 25px"
-
-              a href="/site_files/download/#{file.filename}": i class="icon-download" style="margin-right: 10px" title="Download"
-              - if file.filename != 'index.html'
-                a href="#" onclick="confirmFileDelete('#{file.filename}')": i class="icon-trash" style="margin-right: 10px" title="Delete"
-              - else
-                span style="margin-left: 22px"
-
-
-
-
-    .col.col-40
-
-      div.txt-Center.dash-SS
-        a href="http://#{current_site.username}.neocities.org" style="display:block" target="_blank": img src="/site_screenshots/#{current_site.username}.jpg" alt="screen shot"
-        br
-        h3 class="base" style="line-height:1": a href="http://#{current_site.username}.neocities.org" target="_blank" http://#{current_site.username}.neocities.org
-        p Your page has been accessed <b>#{current_site.hits.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}</b> times.
-        p: small
-          |
-            (access statistics updated hourly)
-
-      .progress.progress-info.progress-striped
-        .bar style="width: #{(current_site.total_space / Site::MAX_SPACE.to_f) * 100}%"
-
-      h4.base.txt-Center
-        p Using #{((current_site.total_space.to_f / Site::MAX_SPACE) * 100).round(1)}% (#{current_site.total_space_in_megabytes}MB) of your #{(Site::MAX_SPACE.to_f / 2**20).to_i}MB of free space.
-
-      div.txt-Center
-        a href="/site_files/upload" class="btn-Action" style="margin-bottom:10px" Upload New Files
-        br
-        a href="/site_files/new_page" class="btn-Action" style="margin-bottom:10px" Create New HTML Page
-        br
-        a href="/site_files/#{current_site.username}.zip" class="btn-Action" Download Entire Site
-
-      hr
-        
-      div
-        <i class="icon-question-sign icon-3x"></i>&nbsp;&nbsp; <span style="font-size: 20pt">Need to learn web design?</span>
-        br
-        i class="icon-globe" &nbsp;&nbsp;
-        a href="/tutorials" target="_blank" Web Design, HTML, CSS, JavaScript Tutorials
-              
-         
-          
-  
-  form method="POST" action="/site_files/delete" id="deleteFilenameForm"
-    input name="csrf_token" type="hidden" value="#{csrf_token}"
-    input type="hidden" id="deleteFilenameInput" name="filename"
-
-  .modal.hide.fade id="deleteConfirmModal" tabindex="-1" role="dialog" aria-labelledby="deleteConfirmModalLabel" aria-hidden="true"
-    .modal-header
-      button.close type="button" data-dismiss="modal" aria-hidden="true" x
-      h3 id="deleteConfirmModalLabel" Confirm delete of file
-    .modal-body
-      p You are about to delete the file <span id="deleteFileName"></span>. Are you sure?
-    .modal-footer
-      button.btn data-dismiss="modal" aria-hidden="true" Cancel
-      button.btn.btn-danger onclick="fileDelete()" Delete

workers/screenshot_worker.rb

@@ -1,34 +1,40 @@
-require 'selenium-webdriver'
 require 'RMagick'
 
 class ScreenshotWorker
+  REQUIRED_RESOLUTIONS = ['235x141', '105x63', '270x162']
+  SCREENSHOTS_PATH = File.join DIR_ROOT, 'public', 'site_screenshots'
   include Sidekiq::Worker
   sidekiq_options queue: :screenshots, retry: 3, backtrace: true
 
-  def perform(username)
+  def perform(username, filename)
     screenshot = Tempfile.new 'neocities_screenshot'
     screenshot.close
+    screenshot_output_path = screenshot.path+'.png'
 
-    caps = Selenium::WebDriver::Remote::Capabilities.htmlunit javascript_enabled: true, takesScreenshot: true
+    f = Screencap::Fetcher.new("http://#{username}.neocities.org/#{filename}")
+    f.fetch(
+      output: screenshot_output_path,
+      width: 1280,
+      height: 720
+    )
 
-    driver = Selenium::WebDriver.for :remote, url: $config['phantomjs_url'][rand($config['phantomjs_url'].length)], desired_capabilities: caps
-    driver.manage.window.resize_to 1280, 720
-
-    wait = Selenium::WebDriver::Wait.new(timeout: 10) # seconds
-    wait.until {
-      driver.navigate.to "http://#{username}.neocities.org"
-      driver.save_screenshot screenshot.path
-    }
-
-    driver.quit
 
     img_list = Magick::ImageList.new
-    img_list.read screenshot.path
+    img_list.from_blob File.read(screenshot_output_path)
+
     screenshot.unlink
+    File.unlink screenshot_output_path
+
     img_list.new_image(img_list.first.columns, img_list.first.rows) { self.background_color = "white" }
     img = img_list.reverse.flatten_images
-    img.crop!(0, 0, 1280, 720)
+
-    img.resize! 208, 125
+    user_screenshots_path = File.join SCREENSHOTS_PATH, username
-    img.write File.join(DIR_ROOT, 'public', 'site_screenshots', "#{username}.jpg")
+    FileUtils.mkdir_p user_screenshots_path
+
+    REQUIRED_RESOLUTIONS.each do |res|
+      img.scale(*res.split('x').collect {|r| r.to_i}).write(File.join(user_screenshots_path, "#{filename}.#{res}.jpg")) {
+        self.quality = 90
+      }
+    end
   end
 end