refactor file logic into models to prepare for API, minor touches.

Gemfile

@@ -15,6 +15,8 @@ gem 'selenium-webdriver', require: nil
 gem 'sidekiq'
 gem 'ago'
 gem 'mail'
+gem 'google-api-client',  require: 'google/api_client'
+gem 'tilt'
 
 platform :mri do
   gem 'magic' # sudo apt-get install file, For OSX: brew install libmagic

Gemfile.lock

@@ -4,6 +4,10 @@ GEM
     addressable (2.3.6)
     ago (0.1.5)
     ansi (1.4.3)
+    autoparse (0.3.3)
+      addressable (>= 2.3.1)
+      extlib (>= 0.9.15)
+      multi_json (>= 1.0.0)
     bcrypt (3.1.7)
     builder (3.2.2)
     capybara (2.2.1)
@@ -32,15 +36,33 @@ GEM
     debugger-linecache (1.2.0)
     debugger-ruby_core_source (1.3.2)
     docile (1.1.3)
+    extlib (0.9.16)
     fabrication (2.11.0)
     faker (1.3.0)
       i18n (~> 0.5)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.3)
+    google-api-client (0.7.1)
+      addressable (>= 2.3.2)
+      autoparse (>= 0.3.3)
+      extlib (>= 0.9.15)
+      faraday (>= 0.9.0)
+      jwt (>= 0.1.5)
+      launchy (>= 2.1.1)
+      multi_json (>= 1.0.0)
+      retriable (>= 1.4)
+      signet (>= 0.5.0)
+      uuidtools (>= 2.1.0)
     hashie (2.0.5)
     hiredis (0.5.0)
     i18n (0.6.9)
     json (1.8.1)
+    jwt (0.1.11)
+      multi_json (>= 1.5)
     kgio (2.9.2)
+    launchy (2.4.2)
+      addressable (~> 2.3)
     magic (0.2.6)
       ffi (>= 0.6.3)
     mail (2.5.4)
@@ -59,6 +81,7 @@ GEM
     mocha (1.0.0)
       metaclass (~> 0.0.1)
     multi_json (1.9.2)
+    multipart-post (2.0.0)
     nokogiri (1.6.1)
       mini_portile (~> 0.5.0)
     pg (0.17.1)
@@ -97,6 +120,7 @@ GEM
     redis (3.0.7)
     redis-namespace (1.4.1)
       redis (~> 3.0.4)
+    retriable (1.4.1)
     rmagick (2.13.2)
     rubyzip (1.1.2)
     safe_yaml (1.0.1)
@@ -117,6 +141,11 @@ GEM
       json
       redis (>= 3.0.6)
       redis-namespace (>= 1.3.1)
+    signet (0.5.0)
+      addressable (>= 2.2.3)
+      faraday (>= 0.9.0.rc5)
+      jwt (>= 0.1.5)
+      multi_json (>= 1.0.0)
     simplecov (0.8.2)
       docile (~> 1.1.0)
       multi_json
@@ -144,6 +173,7 @@ GEM
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
+    uuidtools (2.1.4)
     webmock (1.17.4)
       addressable (>= 2.2.7)
       crack (>= 0.3.2)
@@ -162,6 +192,7 @@ DEPENDENCIES
   capybara_minitest_spec
   fabrication
   faker
+  google-api-client
   hiredis
   jdbc-postgres
   jruby-openssl
@@ -195,4 +226,5 @@ DEPENDENCIES
   sinatra-flash
   sinatra-xsendfile
   slim
+  tilt
   webmock

app.rb

@@ -1,4 +1,6 @@
 require 'base64'
+require 'uri'
+require 'net/http'
 require './environment.rb'
 
 use Rack::Session::Cookie, key:          'neocities',
@@ -102,15 +104,13 @@ get '/donate' do
 end
 
 get '/blog' do
-  # expires 500, :public, :must_revalidate
+  expires 500, :public, :must_revalidate
-  return File.read File.join(Sites::SITE_FILES_ROOT, 'blog', 'index.html')
+  return Net::HTTP.get_response(URI('http://blog.neocities.org')).body
 end
 
 get '/blog/:article' do |article|
-  # expires 500, :public, :must_revalidate
+  expires 500, :public, :must_revalidate
-  path = File.join Sites::SITE_FILES_ROOT, 'blog', "#{article}.html"
+  return Net::HTTP.get_response(URI("http://blog.neocities.org/#{article}.html")).body
-  pass if !File.exist?(path)
-  File.read path
 end
 
 get '/new' do
@@ -149,17 +149,7 @@ post '/create' do
   recaptcha_is_valid = ENV['RACK_ENV'] == 'test' || recaptcha_valid?
 
   if @site.valid? && recaptcha_is_valid
-
+    @site.save
-    base_path = site_base_path @site.username
-
-    DB.transaction {
-      @site.save
-
-      FileUtils.mkdir_p base_path
-
-      File.write File.join(base_path, 'index.html'), slim(:'templates/index', pretty: true, layout: false)
-      File.write File.join(base_path, 'not_found.html'), slim(:'templates/not_found', pretty: true, layout: false)
-    }
 
     session[:id] = @site.id
     redirect '/dashboard'
@@ -232,19 +222,19 @@ end
 
 post '/change_name' do
   require_login
-  current_username = current_site.username
+  old_username = current_site.username
 
-  if current_site.username == params[:name]
+  if old_username == params[:name]
     flash[:error] = 'You already have this name.'
     redirect '/settings'
   end
-
+  
   current_site.username = params[:name]
 
   if current_site.valid?
     DB.transaction {
       current_site.save
-      FileUtils.mv site_base_path(current_username), site_base_path(current_site.username)
+      current_site.move_files_from old_username
     }
 
     flash[:success] = "Site/user name has been changed. You will need to use this name to login, <b>don't forget it</b>."
@@ -273,14 +263,13 @@ post '/site_files/create_page' do
   end
 
   name = "#{params[:pagefilename]}.html"
-  path = site_file_path name
 
-  if File.exist? path
+  if current_site.file_exists?(name)
     @errors << %{Web page "#{name}" already exists! Choose another name.}
     halt slim(:'site_files/new_page')
   end
 
-  File.write path, slim(:'templates/index', pretty: true, layout: false)
+  current_site.install_new_html_file name
 
   flash[:success] = %{#{name} was created! <a style="color: #FFFFFF; text-decoration: underline" href="/site_files/text_editor/#{name}">Click here to edit it</a>.}
 
@@ -304,12 +293,12 @@ post '/site_files/upload' do
 
   if params[:newfile] == '' || params[:newfile].nil?
     @errors << 'You must select a file to upload.'
-    halt http_error_code, 'Did not receive file upload.' # slim(:'site_files/new')
+    halt http_error_code, 'Did not receive file upload.'
   end
 
   if params[:newfile][:tempfile].size > Site::MAX_SPACE || (params[:newfile][:tempfile].size + current_site.total_space) > Site::MAX_SPACE
     @errors << 'File size must be smaller than available space.'
-    halt http_error_code, 'File size must be smaller than available space.' # slim(:'site_files/new')
+    halt http_error_code, 'File size must be smaller than available space.'
   end
 
   mime_type = Magic.guess_file_mime_type params[:newfile][:tempfile].path
@@ -320,10 +309,7 @@ post '/site_files/upload' do
   end
 
   sanitized_filename = params[:newfile][:filename].gsub(/[^a-zA-Z0-9_\-.]/, '')
-
+  current_site.store_file sanitized_filename, params[:newfile][:tempfile]
-  dest_path = File.join(site_base_path(current_site.username), sanitized_filename)
-  FileUtils.mv params[:newfile][:tempfile].path, dest_path
-  File.chmod(0640, dest_path) if self.class.production?
 
   if sanitized_filename =~ /index\.html/
     ScreenshotWorker.perform_async current_site.username
@@ -339,71 +325,56 @@ end
 post '/site_files/delete' do
   require_login
   sanitized_filename = params[:filename].gsub(/[^a-zA-Z0-9_\-.]/, '')
-  begin
+
-    FileUtils.rm File.join(site_base_path(current_site.username), sanitized_filename)
+  current_site.delete_file(sanitized_filename)
-  rescue Errno::ENOENT
+
-    flash[:error] = 'File was already deleted.'
-    redirect '/dashboard'
-  end
   flash[:success] = "Deleted file #{params[:filename]}."
   redirect '/dashboard'
 end
 
 get '/site_files/:username.zip' do |username|
   require_login
-  file_path = "/tmp/neocities-site-#{username}.zip"
+  zipfile = current_site.files_zip
-
-  Zip::File.open(file_path, Zip::File::CREATE) do |zipfile|
-    current_site.file_list.collect {|f| f.filename}.each do |filename|
-      zipfile.add filename, site_file_path(filename)
-    end
-  end
-
-  # I don't want to have to deal with cleaning up old tmpfiles
-  zipfile = File.read file_path
-  File.delete file_path
-
   content_type 'application/octet-stream'
   attachment   "#{current_site.username}.zip"
-
+  zipfile
-  return zipfile
 end
 
 get '/site_files/download/:filename' do |filename|
   require_login
-  send_file File.join(site_base_path(current_site.username), filename), filename: filename, type: 'Application/octet-stream'
+  content_type 'application/octet-stream'
+  attachment filename
+  current_site.get_file filename
 end
 
 get '/site_files/text_editor/:filename' do |filename|
   require_login
   begin
-    @file_data = File.read File.join(site_base_path(current_site.username), filename)
+    @file_data = current_site.get_file filename
   rescue Errno::ENOENT
     flash[:error] = 'We could not find the requested file.'
     redirect '/dashboard'
   end
-  slim :'site_files/text_editor'
+  slim :'site_files/text_editor', indent: false
 end
 
 post '/site_files/save/:filename' do |filename|
   require_login_ajax
 
-  tmpfile = Tempfile.new 'neocities_saving_file'
+  tempfile = Tempfile.new 'neocities_saving_file'
 
-  if (tmpfile.size + current_site.total_space) > Site::MAX_SPACE
+  if (tempfile.size + current_site.total_space) > Site::MAX_SPACE
     halt 'File is too large to fit in your space, it has NOT been saved. Please make a local copy and then try to reduce the size.'
   end
 
   input = request.body.read
-  tmpfile.set_encoding input.encoding
+  tempfile.set_encoding input.encoding
-  tmpfile.write input
+  tempfile.write input
-  tmpfile.close
+  tempfile.close
 
   sanitized_filename = filename.gsub(/[^a-zA-Z0-9_\-.]/, '')
-  dest_path = File.join site_base_path(current_site.username), sanitized_filename
 
-  FileUtils.mv tmpfile.path, dest_path
+  current_site.store_file sanitized_filename, tempfile
-  File.chmod(0640, dest_path) if self.class.production?
 
   if sanitized_filename =~ /index\.html/
     ScreenshotWorker.perform_async current_site.username
@@ -430,24 +401,6 @@ get '/admin' do
   slim :'admin'
 end
 
-def ban_site(username)
-  site = Site[username: username]
-  return false if site.nil?
-  return false if site.is_banned == true
-
-  DB.transaction {
-    FileUtils.mv site_base_path(site.username), File.join(settings.public_folder, 'banned_sites', site.username)
-    site.is_banned = true
-    site.save(validate: false)
-  }
-
-  if !['127.0.0.1', nil, ''].include? site.ip
-    `sudo ufw insert 1 deny from #{site.ip}`
-  end
-
-  true
-end
-
 post '/admin/banip' do
   require_admin
   site = Site[username: params[:username]]
@@ -462,8 +415,8 @@ post '/admin/banip' do
     redirect '/admin'
   end
 
-  sites = Site.filter(ip: site.ip).all
+  sites = Site.filter(ip: site.ip, is_banned: false).all
-  sites.each {|s| ban_site(s.username)}
+  sites.each {|s| s.ban!}
   flash[:error] = "#{sites.length} sites have been banned."
   redirect '/admin'
 end
@@ -483,7 +436,7 @@ post '/admin/banhammer' do
     redirect '/admin'
   end
 
-  ban_site params[:username]
+  site.ban!
 
   flash[:success] = 'MISSION ACCOMPLISHED'
   redirect '/admin'
@@ -584,18 +537,9 @@ post '/custom_domain' do
   require_login
   original_domain = current_site.domain
   current_site.domain = params[:domain]
+
   if current_site.valid?
-
+    current_site.save
-    DB.transaction do
-      current_site.save
-
-      if !params[:domain].empty? && !params[:domain].nil?
-        File.open(File.join(DIR_ROOT, 'domains', "#{current_site.username}.conf"), 'w') do |file|
-          file.write erb(:'templates/domain', layout: false)
-        end
-      end
-
-    end
     flash[:success] = 'The domain has been successfully updated.'
     redirect '/custom_domain'
   else
@@ -667,18 +611,6 @@ def current_site
   @site ||= Site[id: session[:id]]
 end
 
-def site_base_path(subname)
-  File.join Site::SITE_FILES_ROOT, subname
-end
-
-def site_file_path(filename)
-  File.join(site_base_path(current_site.username), filename)
-end
-
-def template_site_title(username)
-  "#{username.capitalize}#{username[username.length-1] == 's' ? "'" : "'s"} Site"
-end
-
 def encoding_fix(file)
   begin
     Rack::Utils.escape_html file
@@ -686,4 +618,4 @@ def encoding_fix(file)
     return Rack::Utils.escape_html(file.force_encoding('BINARY')) if e.message =~ /invalid byte sequence in UTF-8/
     fail
   end
-end
+end

models/site.rb

@@ -1,3 +1,5 @@
+require 'tilt'
+
 class Site < Sequel::Model
   # We might need to include fonts in here..
   VALID_MIME_TYPES = %w{
@@ -30,9 +32,13 @@ class Site < Sequel::Model
   MINIMUM_PASSWORD_LENGTH = 5
   BAD_USERNAME_REGEX = /[^\w-]/i
   VALID_HOSTNAME = /^[a-z0-9][a-z0-9-]+?[a-z0-9]$/i # http://tools.ietf.org/html/rfc1123
-  
+
-  SITE_FILES_ROOT = File.join(DIR_ROOT, 'public', (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites'))
+  # FIXME smarter DIR_ROOT discovery
-  
+  DIR_ROOT        = './'
+  TEMPLATE_ROOT   = File.join DIR_ROOT, 'views', 'templates'
+  PUBLIC_ROOT     = File.join DIR_ROOT, 'public'
+  SITE_FILES_ROOT = File.join PUBLIC_ROOT, (ENV['RACK_ENV'] == 'test' ? 'sites_test' : 'sites')
+
   many_to_one :server
   many_to_many :tags
 
@@ -78,6 +84,87 @@ class Site < Sequel::Model
     super
   end
 
+  def save(validate={})
+    DB.transaction do
+      is_new = new?
+      install_custom_domain if !domain.nil? && !domain.empty?
+      result = super(validate)
+      install_new_files if is_new
+      result
+    end
+  end
+
+  def install_custom_domain
+    File.open(File.join(DIR_ROOT, 'domains', "#{username}.conf"), 'w') do |file|
+      file.write render_template('domain.erb')
+    end
+  end
+
+  def install_new_files
+    FileUtils.mkdir_p files_path
+
+    %w{index not_found}.each do |name|
+      File.write file_path("#{name}.html"), render_template("#{name}.slim")
+    end
+  end
+
+  def get_file(filename)
+    File.read file_path(filename)
+  end
+
+  def ban!
+    DB.transaction {
+      FileUtils.mv files_path, File.join(PUBLIC_ROOT, 'banned_sites', username)
+      self.is_banned = true
+
+      if !['127.0.0.1', nil, ''].include? ip
+        `sudo ufw insert 1 deny from #{ip}`
+      end
+
+      save(validate: false)
+    }
+  end
+
+  def store_file(filename, uploaded)
+    FileUtils.mv uploaded.path, file_path(filename)
+    File.chmod(0640, file_path(filename))
+  end
+
+  def files_zip
+    file_path = "/tmp/neocities-site-#{username}.zip"
+
+    Zip::File.open(file_path, Zip::File::CREATE) do |zipfile|
+      file_list.collect {|f| f.filename}.each do |filename|
+        zipfile.add filename, file_path(filename)
+      end
+    end
+
+    # TODO Don't dump the zipfile into memory
+    zipfile = File.read file_path
+    File.delete file_path
+    zipfile
+  end
+
+  def delete_file(filename)
+    begin
+      FileUtils.rm file_path(filename)
+    rescue Errno::ENOENT
+      # File was probably already deleted
+    end
+  end
+
+  def move_files_from(oldusername)
+    FileUtils.mv files_path(oldusername), files_path
+  end
+
+  def install_new_html_file(name)
+    File.write file_path(name), render_template('index.slim')
+  end
+
+  def file_exists?(filename)
+    File.exist? file_path(filename)
+  end
+
   def after_save
     if @new_tag_strings
       @new_tag_strings.each do |new_tag_string|
@@ -137,16 +224,24 @@ class Site < Sequel::Model
     end
   end
 
-  def file_path
+  def render_template(name)
-    File.join SITE_FILES_ROOT, username
+    Tilt.new(File.join(TEMPLATE_ROOT, name), pretty: true).render self
+  end
+
+  def files_path(name=nil)
+    File.join SITE_FILES_ROOT, (name || username)
+  end
+  
+  def file_path(filename)
+    File.join files_path, filename
   end
 
   def file_list
-    Dir.glob(File.join(file_path, '*')).collect {|p| File.basename(p)}.sort.collect {|sitename| SiteFile.new sitename}
+    Dir.glob(File.join(files_path, '*')).collect {|p| File.basename(p)}.sort.collect {|sitename| SiteFile.new sitename}
   end
 
   def total_space
-    space = Dir.glob(File.join(file_path, '*')).collect {|p| File.size(p)}.inject {|sum,x| sum += x}
+    space = Dir.glob(File.join(files_path, '*')).collect {|p| File.size(p)}.inject {|sum,x| sum += x}
     space.nil? ? 0 : space
   end
   

views/dashboard.slim

@@ -20,8 +20,7 @@ javascript:
           span
             <i class="icon-file-alt icon-3x"></i>&nbsp;&nbsp; <span style="font-size: 20pt">#{file.filename}</span>
             - if file.filename == 'index.html'
-              p.tiny
+              p.tiny This is your index file! It is the "default file" that loads when you go to <a href="http://#{current_site.username}.neocities.org">#{current_site.username}.neocities.org</a>. In effect, it's your front page. If you want to change your front page, you need to edit (or overwrite) this file, which you should do right now if you just created your site. The default file is always named <b>index.html</b>, and you cannot delete it.
-                This is your index file! It is the "default file" that loads when you go to <a href="http://#{current_site.username}.neocities.org">#{current_site.username}.neocities.org</a>. In effect, it's your front page. If you want to change your front page, you need to edit (or overwrite) this file. The default file is always named <b>index.html</b>.
           
           div style="margin-bottom:30px"
             span
@@ -36,14 +35,23 @@ javascript:
             span
               i class="icon-edit" &nbsp;&nbsp;
               span: a href="/site_files/download/#{file.filename}" Download <br />
-            span
+
-              i class="icon-trash" &nbsp;&nbsp;
+            - if file.filename != 'index.html'
-              a href="#" onclick="confirmFileDelete('#{file.filename}')" Delete
+              span
+                i class="icon-trash" &nbsp;&nbsp;
+                a href="#" onclick="confirmFileDelete('#{file.filename}')" Delete
         - else
           <i class="icon-picture icon-3x"></i>&nbsp;&nbsp; <span style="font-size: 20pt">#{file.filename}</span>
-          div style="margin-top: 3px; margin-bottom:10px"
+          div style="margin-top: 3px; margin-bottom: 30px"
-              | To use in an HTML file, paste this text: <code class="tiny" style="margin:0">&lt;img src="/#{file.filename}"&gt;</code>
+            | To use in an HTML file, paste this text: <code class="tiny" style="margin:0">&lt;img src="/#{file.filename}"&gt;</code>
+            span
+              i class="icon-globe" &nbsp;&nbsp;
               a href="http://#{current_site.username}.neocities.org/#{file.filename}" target="_blank" View <br />
+            span
+              i class="icon-edit" &nbsp;&nbsp;
+              span: a href="/site_files/download/#{file.filename}" Download <br />
+            span
+              i class="icon-trash" &nbsp;&nbsp;
               a href="#" onclick="confirmFileDelete('#{file.filename}')" Delete
   
     .col.col-40

views/site_files/text_editor.slim

@@ -53,9 +53,7 @@ css:
           option value="ace/theme/twilight" Twilight
           option value="ace/theme/vibrant_ink" Vibrant Ink
 
-    div id="editor" style="width: 100%; height: 600px; position: relative; margin-bottom:25px"
+    <div id="editor" style="width: 100%; height: 600px; position: relative; margin-bottom:25px">#{{encoding_fix(@file_data)}}</div>
-      == encoding_fix @file_data
-    
     
     .row
       .col.col-33.txt-Center style="margin-bottom:10px"
@@ -125,4 +123,5 @@ css:
         editor.getSession().setUseWrapMode(true);
         editor.setFontSize(14);
         editor.setShowPrintMargin(false);
-      });
+        
+      });

views/templates/domain.erb

@@ -1,8 +1,8 @@
 server {
   listen 80;
-  server_name <%= current_site.domain %> *.<%= current_site.domain %>;
+  server_name <%= domain %> *.<%= domain %>;
   access_log /var/log/nginx/neocities-domains.log neocitiesdomain;
-  root /home/web/neocities-web/public/sites/<%= current_site.username %>;
+  root /home/web/neocities-web/public/sites/<%= username %>;
   index /index.html;
 
   error_page 404 = @notfound;

views/templates/index.slim

@@ -3,7 +3,7 @@ html
   head
     meta http-equiv="Content-Type" content="text/html; charset=UTF-8"
     
-    title #{template_site_title @site.username} - Front Page
+    title #{username}
     
     meta name="description" content=""
     meta name="keywords" content=""

views/templates/not_found.slim

@@ -3,7 +3,7 @@ html
   head
     meta http-equiv="Content-Type" content="text/html; charset=UTF-8"
     
-    title #{template_site_title @site.username} - Not Found
+    title #{username} - Page Not Found
     
     link href="//groundfloor.neocities.org/default.css" rel="stylesheet" type="text/css" media="all"