finish merge on cleaned up news feed

.gitignore

@@ -1,24 +1,9 @@
-*.gem
-*.rbc
-.bundle
 .config
-coverage
-InstalledFiles
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
-# YARD artifacts
-.yardoc
-_yardoc
-doc/
 tests/coverage
 config.yml
 .DS_Store
 public/css/neo.css
+public/css/neo.css.map
 public/site_thumbnails
 public/sites
 public/site_screenshots
@@ -26,12 +11,16 @@ public/site_screenshots_test
 public/site_thumbnails_test
 *.swp
 files/map.txt
+files/supporter-map.txt
+files/maps
 .sass-cache
 .sass-cache/*
 files/sslsites.zip
 .tm_properties
-./black_box.rb
 .vagrant
 public/banned_sites
 public/deleted_sites
-tests/stat_logs/*
+files/disposable_email_blacklist.conf
+files/letsencrypt.key
+files/tor.txt
+.bundle

.travis.yml

@@ -1,7 +1,9 @@
 language: ruby
 rvm:
-  - "2.1.1"
+  - "2.3.0"
 addons:
   postgresql: "9.3"
 before_script:
   - psql -c 'create database travis_ci_test;' -U postgres
+sudo: false
+bundler_args: --jobs=1

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 
 gem 'sinatra'
 gem 'redis'
+gem 'redis-namespace'
 gem 'sequel', '4.8.0'
 gem 'bcrypt'
 gem 'sinatra-flash',      require: 'sinatra/flash'
@@ -9,14 +10,13 @@ gem 'sinatra-xsendfile',  require: 'sinatra/xsendfile'
 gem 'puma',               require: nil
 gem 'rack-recaptcha',     require: 'rack/recaptcha'
 gem 'rmagick',            require: nil
-gem 'sidekiq'
+gem 'sidekiq', '~> 4.1.2'
 gem 'ago'
 gem 'mail'
-gem 'google-api-client',  require: 'google/api_client'
 gem 'tilt'
 gem 'erubis'
-gem 'stripe' #, source: 'https://code.stripe.com/'
+gem 'stripe', '1.15.0' #, source: 'https://code.stripe.com/'
-gem 'screencap'
+#gem 'screencap', '~> 0.1.4'
 gem 'cocaine'
 gem 'zipruby'
 gem 'sass', require: nil
@@ -25,18 +25,37 @@ gem 'filesize'
 gem 'thread'
 gem 'scrypt'
 gem 'rack-cache'
-gem 'rest-client'
+gem 'rest-client', require: 'rest_client'
+gem 'addressable', require: 'addressable/uri'
+gem 'paypal-recurring', require: 'paypal/recurring'
 gem 'geoip'
 gem 'io-extra', require: 'io/extra'
 gem 'rye'
-gem 'dnsruby'
+gem 'base32'
+gem 'coveralls', require: false
+gem 'sanitize'
+gem 'will_paginate'
+gem 'simpleidn'
+gem 'gandi'
+gem 'hoe', '3.14.2', require: nil
+gem 'msgpack'
+gem 'json-jwt', {
+  git: 'https://github.com/neocities/json-jwt.git',
+  branch: 'drop_activesupport'
+}
+gem 'acme-client', {
+  git: 'https://github.com/jhass/acme-client.git',
+  branch: 'no_activesupport'
+}
+gem 'http'
+gem 'htmlentities'
 
 platform :mri, :rbx do
   gem 'magic' # sudo apt-get install file, For OSX: brew install libmagic
   gem 'pg'
   gem 'sequel_pg', require: nil
   gem 'hiredis'
-  gem 'rainbows',  require: nil
+  gem 'posix-spawn'
 
   group :development, :test do
     gem 'pry'
@@ -61,6 +80,7 @@ end
 
 group :development do
   gem 'shotgun', require: nil
+  gem 'certified'
 end
 
 group :test do
@@ -73,10 +93,12 @@ group :test do
   gem 'rake',                  require: nil
   gem 'poltergeist'
   gem 'capybara_minitest_spec'
+  gem 'capybara', '2.6.2',     require: nil
   gem 'rack_session_access',   require: nil
   gem 'webmock',               require: nil
-  gem 'stripe-ruby-mock',      '~> 2.0.1', require: 'stripe_mock'
+  gem 'stripe-ruby-mock', '2.0.1',      require: 'stripe_mock'
   gem 'timecop'
+  gem 'mock_redis'
 
   platform :mri, :rbx do
     gem 'simplecov',        require: nil

Gemfile.lock

@@ -1,28 +1,46 @@
+GIT
+  remote: https://github.com/jhass/acme-client.git
+  revision: d0ced992bfe42908bb1fc25ac549ae3318386c97
+  branch: no_activesupport
+  specs:
+    acme-client (0.2.2)
+      faraday (~> 0.9, >= 0.9.1)
+      json-jwt (~> 1.2, >= 1.2.3)
+
+GIT
+  remote: https://github.com/neocities/json-jwt.git
+  revision: 00d9c3d34e6bfbab866a4a0405897182a4ed3833
+  branch: drop_activesupport
+  specs:
+    json-jwt (1.5.2)
+      bindata
+      hashery (~> 2.0)
+      multi_json (>= 1.3)
+      securecompare
+      url_safe_base64
+
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.1.4)
+    activesupport (4.2.6)
-      i18n (~> 0.6, >= 0.6.9)
+      i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
-      thread_safe (~> 0.1)
+      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.3.7)
+    addressable (2.4.0)
     ago (0.1.5)
     annoy (0.5.6)
       highline (>= 1.5.0)
-    ansi (1.4.3)
+    ansi (1.5.0)
-    autoparse (0.3.3)
+    base32 (0.3.2)
-      addressable (>= 2.3.1)
+    bcrypt (3.1.11)
-      extlib (>= 0.9.15)
+    bindata (2.3.1)
-      multi_json (>= 1.0.0)
-    bcrypt (3.1.7)
     blankslate (3.1.3)
     builder (3.2.2)
-    byebug (2.7.0)
+    byebug (8.2.4)
-      columnize (~> 0.3)
+    capybara (2.6.2)
-      debugger-linecache (~> 1.2)
+      addressable
-    capybara (2.4.4)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
@@ -31,123 +49,123 @@ GEM
     capybara_minitest_spec (1.0.5)
       capybara (>= 2)
       minitest (>= 4)
-    celluloid (0.15.2)
+    certified (1.0.0)
-      timers (~> 1.1.0)
     climate_control (0.0.3)
       activesupport (>= 3.0)
     cliver (0.3.2)
-    cocaine (0.5.4)
+    cocaine (0.5.8)
       climate_control (>= 0.0.3, < 1.0)
-    coderay (1.1.0)
+    coderay (1.1.1)
-    columnize (0.8.9)
+    concurrent-ruby (1.0.2)
-    connection_pool (2.0.0)
+    connection_pool (2.2.0)
-    crack (0.4.2)
+    coveralls (0.8.13)
+      json (~> 1.8)
+      simplecov (~> 0.11.0)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
+      tins (~> 1.6.0)
+    crack (0.4.3)
       safe_yaml (~> 1.0.0)
+    crass (1.0.2)
     dante (0.2.0)
     dav4rack (0.3.0)
       nokogiri (>= 1.4.2)
       rack (>= 1.1.0)
       uuidtools (~> 2.1.1)
-    debugger-linecache (1.2.0)
+    docile (1.1.5)
-    dnsruby (1.58.0)
+    domain_name (0.5.20160310)
-    docile (1.1.3)
-    domain_name (0.5.23)
       unf (>= 0.0.5, < 1.0.0)
     drydock (0.6.9)
     erubis (2.7.0)
-    extlib (0.9.16)
+    fabrication (2.15.0)
-    fabrication (2.11.0)
+    faker (1.6.3)
-    faker (1.3.0)
       i18n (~> 0.5)
-    faraday (0.9.0)
+    faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    ffi (1.9.6)
+    ffi (1.9.10)
     ffi-compiler (0.1.3)
       ffi (>= 1.0.0)
       rake
-    filesize (0.0.3)
+    filesize (0.1.1)
-    geoip (1.5.0)
+    gandi (2.1.3)
-    google-api-client (0.7.1)
+      hashie
-      addressable (>= 2.3.2)
+    geoip (1.6.1)
-      autoparse (>= 0.3.3)
+    hashdiff (0.3.0)
-      extlib (>= 0.9.15)
+    hashery (2.1.2)
-      faraday (>= 0.9.0)
+    hashie (3.4.3)
-      jwt (>= 0.1.5)
+    highline (1.7.8)
-      launchy (>= 2.1.1)
+    hiredis (0.6.1)
-      multi_json (>= 1.0.0)
+    hoe (3.14.2)
-      retriable (>= 1.4)
+      rake (>= 0.8, < 11.0)
-      signet (>= 0.5.0)
+    htmlentities (4.3.4)
-      uuidtools (>= 2.1.0)
+    http (2.0.1)
-    hashie (2.0.5)
+      addressable (~> 2.3)
-    highline (1.7.2)
+      http-cookie (~> 1.0)
-    hiredis (0.5.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
-    i18n (0.6.9)
+    http-form_data (1.0.1)
+    http_parser.rb (0.6.0)
+    i18n (0.7.0)
     io-extra (1.2.8)
     jimson-temp (0.9.5)
       blankslate (>= 3.1.2)
       multi_json (~> 1.0)
       rack (~> 1.4)
       rest-client (~> 1.0)
-    json (1.8.1)
+    json (1.8.3)
-    jwt (0.1.11)
+    m (1.4.2)
-      multi_json (>= 1.5)
-    kgio (2.9.2)
-    launchy (2.4.2)
-      addressable (~> 2.3)
-    m (1.3.4)
       method_source (>= 0.6.7)
       rake (>= 0.9.2.2)
-    magic (0.2.6)
+    magic (0.2.9)
       ffi (>= 0.6.3)
-    mail (2.5.4)
+    mail (2.6.4)
-      mime-types (~> 1.16)
+      mime-types (>= 1.16, < 4)
-      treetop (~> 1.4.8)
     metaclass (0.0.4)
     method_source (0.8.2)
-    mime-types (1.25.1)
+    mime-types (2.99.1)
-    mini_portile (0.6.2)
+    mini_portile2 (2.0.0)
-    minitest (5.6.1)
+    minitest (5.8.4)
-    minitest-reporters (1.0.2)
+    minitest-reporters (1.1.8)
       ansi
       builder
       minitest (>= 5.0)
-      powerbar
+      ruby-progressbar
-    mocha (1.0.0)
+    mocha (1.1.0)
       metaclass (~> 0.0.1)
-    multi_json (1.11.0)
+    mock_redis (0.16.1)
+    msgpack (0.7.5)
+    multi_json (1.11.2)
     multipart-post (2.0.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (2.9.2)
+    net-ssh (3.1.1)
-    netrc (0.10.3)
+    netrc (0.11.0)
-    nokogiri (1.6.6.2)
+    nokogiri (1.6.7.2)
-      mini_portile (~> 0.6.0)
+      mini_portile2 (~> 2.0.0.rc2)
-    pg (0.17.1)
+    nokogumbo (1.4.7)
-    phantomjs (1.9.7.1)
+      nokogiri
-    poltergeist (1.6.0)
+    paypal-recurring (1.1.0)
+    pg (0.18.4)
+    poltergeist (1.9.0)
       capybara (~> 2.1)
       cliver (~> 0.3.1)
       multi_json (~> 1.0)
       websocket-driver (>= 0.2.0)
-    polyglot (0.3.4)
+    posix-spawn (0.3.11)
-    powerbar (1.0.11)
+    pry (0.10.3)
-      ansi (~> 1.4.0)
+      coderay (~> 1.1.0)
-      hashie (>= 1.1.0)
+      method_source (~> 0.8.1)
-    pry (0.9.12.6)
-      coderay (~> 1.0)
-      method_source (~> 0.8)
       slop (~> 3.4)
-    pry-byebug (1.3.2)
+    pry-byebug (3.3.0)
-      byebug (~> 2.7)
+      byebug (~> 8.0)
-      pry (~> 0.9.12)
+      pry (~> 0.10)
-    puma (2.8.1)
+    puma (3.4.0)
-      rack (>= 1.1, < 2.0)
+    rack (1.6.4)
-    rack (1.6.0)
+    rack-cache (1.6.1)
-    rack-cache (1.2)
       rack (>= 0.4)
-    rack-protection (1.5.2)
+    rack-protection (1.5.3)
       rack
     rack-recaptcha (0.6.6)
       json
@@ -156,21 +174,16 @@ GEM
     rack_session_access (0.1.1)
       builder (>= 2.0.0)
       rack (>= 1.0.0)
-    rainbows (4.6.1)
+    rake (10.5.0)
-      kgio (~> 2.5)
+    redis (3.2.2)
-      rack (~> 1.1)
+    redis-namespace (1.5.2)
-      unicorn (~> 4.8)
+      redis (~> 3.0, >= 3.0.4)
-    raindrops (0.13.0)
-    rake (10.3.2)
-    redis (3.0.7)
-    redis-namespace (1.4.1)
-      redis (~> 3.0.4)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
-    retriable (1.4.1)
+    rmagick (2.15.4)
-    rmagick (2.15.0)
+    ruby-progressbar (1.7.5)
     rye (0.9.13)
       annoy
       docile (>= 1.0.1)
@@ -179,38 +192,35 @@ GEM
       net-ssh (>= 2.0.13)
       sysinfo (>= 0.8.1)
     safe_yaml (1.0.4)
-    sass (3.3.8)
+    sanitize (4.0.1)
-    screencap (0.1.1)
+      crass (~> 1.0.2)
-      phantomjs
+      nokogiri (>= 1.4.4)
-    scrypt (2.0.0)
+      nokogumbo (~> 1.4.1)
+    sass (3.4.22)
+    scrypt (2.1.1)
       ffi-compiler (>= 0.0.2)
       rake
+    securecompare (1.0.0)
     sequel (4.8.0)
-    sequel_pg (1.6.9)
+    sequel_pg (1.6.16)
       pg (>= 0.8.0)
-      sequel (>= 3.39.0)
+      sequel (>= 4.0.0)
-    shotgun (0.9)
+    shotgun (0.9.1)
       rack (>= 1.0)
-    sidekiq (3.0.0)
+    sidekiq (4.1.2)
-      celluloid (>= 0.15.2)
+      concurrent-ruby (~> 1.0)
-      connection_pool (>= 2.0.0)
+      connection_pool (~> 2.2, >= 2.2.0)
-      json
+      redis (~> 3.2, >= 3.2.1)
-      redis (>= 3.0.6)
+    simplecov (0.11.2)
-      redis-namespace (>= 1.3.1)
-    signet (0.5.0)
-      addressable (>= 2.2.3)
-      faraday (>= 0.9.0.rc5)
-      jwt (>= 0.1.5)
-      multi_json (>= 1.0.0)
-    simplecov (0.8.2)
       docile (~> 1.1.0)
-      multi_json
+      json (~> 1.8)
-      simplecov-html (~> 0.8.0)
+      simplecov-html (~> 0.10.0)
-    simplecov-html (0.8.0)
+    simplecov-html (0.10.0)
-    sinatra (1.4.4)
+    simpleidn (0.0.6)
-      rack (~> 1.4)
+    sinatra (1.4.7)
+      rack (~> 1.5)
       rack-protection (~> 1.4)
-      tilt (~> 1.3, >= 1.3.4)
+      tilt (>= 1.3, < 3)
     sinatra-flash (0.3.0)
       sinatra (>= 1.0.0)
     sinatra-xsendfile (0.4.2)
@@ -228,30 +238,29 @@ GEM
     sysinfo (0.8.1)
       drydock
       storable
-    thread (0.1.4)
+    term-ansicolor (1.3.2)
-    thread_safe (0.3.4)
+      tins (~> 1.0)
-    tilt (1.4.1)
+    thor (0.19.1)
-    timecop (0.7.4)
+    thread (0.2.2)
-    timers (1.1.0)
+    thread_safe (0.3.5)
-    treetop (1.4.15)
+    tilt (2.0.2)
-      polyglot
+    timecop (0.8.1)
-      polyglot (>= 0.3.1)
+    tins (1.6.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.6)
+    unf_ext (0.0.7.2)
-    unicorn (4.8.2)
+    url_safe_base64 (0.2.2)
-      kgio (~> 2.6)
+    uuidtools (2.1.5)
-      rack
+    webmock (1.24.2)
-      raindrops (~> 0.7)
+      addressable (>= 2.3.6)
-    uuidtools (2.1.4)
-    webmock (1.17.4)
-      addressable (>= 2.2.7)
       crack (>= 0.3.2)
-    websocket-driver (0.5.4)
+      hashdiff
+    websocket-driver (0.6.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.2)
+    will_paginate (3.1.0)
     xpath (2.0.0)
       nokogiri (~> 1.3)
     zipruby (0.3.6)
@@ -260,31 +269,44 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  acme-client!
+  addressable
   ago
+  base32
   bcrypt
+  capybara (= 2.6.2)
   capybara_minitest_spec
+  certified
   cocaine
+  coveralls
   dav4rack
-  dnsruby
   erubis
   fabrication
   faker
   filesize
+  gandi
   geoip
-  google-api-client
   hiredis
+  hoe (= 3.14.2)
+  htmlentities
+  http
   io-extra
   jdbc-postgres
   jruby-openssl
   json
+  json-jwt!
   m
   magic
   mail
   minitest
   minitest-reporters
   mocha
+  mock_redis
+  msgpack
+  paypal-recurring
   pg
   poltergeist
+  posix-spawn
   pry
   pry-byebug
   puma
@@ -292,31 +314,33 @@ DEPENDENCIES
   rack-recaptcha
   rack-test
   rack_session_access
-  rainbows
   rake
   redis
+  redis-namespace
   rest-client
   rmagick
   ruby-debug
   rye
+  sanitize
   sass
-  screencap
   scrypt
   sequel (= 4.8.0)
   sequel_pg
   shotgun
-  sidekiq
+  sidekiq (~> 4.1.2)
   simplecov
+  simpleidn
   sinatra
   sinatra-flash
   sinatra-xsendfile
-  stripe
+  stripe (= 1.15.0)
-  stripe-ruby-mock (~> 2.0.1)
+  stripe-ruby-mock (= 2.0.1)
   thread
   tilt
   timecop
   webmock
+  will_paginate
   zipruby
 
 BUNDLED WITH
-   1.10.2
+   1.12.1

README.md

@@ -1,6 +1,7 @@
 # Neocities.org
 
 [![Build Status](https://travis-ci.org/neocities/neocities.png?branch=master)](https://travis-ci.org/neocities/neocities)
+[![Coverage Status](https://coveralls.io/repos/neocities/neocities/badge.svg?branch=master&service=github)](https://coveralls.io/github/neocities/neocities?branch=master)
 
 The web site for Neocities! It's open source. Want a feature on the site? Send a pull request!
 

Rakefile

@@ -38,10 +38,17 @@ task :parse_logs => [:environment] do
   Stat.parse_logfiles $config['logs_path']
 end
 
+desc 'Update disposable email blacklist'
+task :update_disposable_email_blacklist => [:environment] do
+  uri = URI.parse('https://raw.githubusercontent.com/martenson/disposable-email-domains/master/disposable_email_blacklist.conf')
+
+  File.write(Site::DISPOSABLE_EMAIL_BLACKLIST_PATH, Net::HTTP.get(uri))
+end
+
 desc 'Update banned IPs list'
 task :update_blocked_ips => [:environment] do
   uri = URI.parse('http://www.stopforumspam.com/downloads/listed_ip_90.zip')
-  blocked_ips_zip = Tempfile.new('blockedipszip', Dir.tmpdir, 'wb')
+  blocked_ips_zip = Tempfile.new('blockedipszip', Dir.tmpdir)
   blocked_ips_zip.binmode
 
   Net::HTTP.start(uri.host, uri.port) do |http|
@@ -65,13 +72,65 @@ task :update_blocked_ips => [:environment] do
   end
 end
 
-desc 'Compile domain map for nginx'
+desc 'parse tor exits'
-task :compile_domain_map => [:environment] do
+task :parse_tor_exits => [:environment] do
-  File.open('./files/map.txt', 'w') do |file|
+  exit_ips = Net::HTTP.get(URI.parse('https://check.torproject.org/exit-addresses'))
-    Site.exclude(domain: nil).exclude(domain: '').select(:username,:domain).all.collect do |site|
+
-      file.write ".#{site.domain} #{site.username};\n"
+  exit_ips.split("\n").collect {|line|
+    line.match(/ExitAddress (\d+\.\d+\.\d+\.\d+)/)&.captures&.first
+  }.compact
+
+  # ^^ Array of ip addresses of known exit nodes
+end
+
+desc 'Compile nginx mapfiles'
+task :compile_nginx_mapfiles => [:environment] do
+  FileUtils.mkdir_p './files/maps'
+
+  File.open('./files/maps/domains.txt', 'w') do |file|
+    Site.exclude(domain: nil).exclude(domain: '').select(:username,:domain).all.each do |site|
+      file.write ".#{site.values[:domain]} #{site.username};\n"
     end
   end
+
+  File.open('./files/maps/supporters.txt', 'w') do |file|
+    Site.select(:username, :domain).exclude(plan_type: 'free').exclude(plan_type: nil).all.each do |parent_site|
+      sites = [parent_site] + parent_site.children
+      sites.each do |site|
+        file.write "#{site.username}.neocities.org 1;\n"
+        unless site.host.match(/\.neocities\.org$/)
+          file.write ".#{site.values[:domain]} 1;\n"
+        end
+      end
+    end
+  end
+
+  File.open('./files/maps/sandboxed.txt', 'w') do |file|
+    usernames = DB["select username from sites where created_at > ? and (plan_type is null or plan_type='free')", 1.week.ago].all.collect {|s| s[:username]}.each {|username| file.write "#{username} 1;\n"}
+  end
+
+  # Compile letsencrypt ssl keys
+  sites = DB[%{select username,ssl_key,ssl_cert,domain from sites where ssl_cert is not null and ssl_key is not null and (domain is not null or domain != '') and is_banned != 't' and is_deleted != 't'}].all
+
+  ssl_path = './files/maps/ssl'
+
+  FileUtils.mkdir_p ssl_path
+
+  sites.each do |site|
+    [site[:domain], "www.#{site[:domain]}"].each do |domain|
+      begin
+        key = OpenSSL::PKey::RSA.new site[:ssl_key]
+        crt = OpenSSL::X509::Certificate.new site[:ssl_cert]
+      rescue => e
+        puts "SSL ERROR: #{e.class} #{e.inspect}"
+        next
+      end
+
+      File.open(File.join(ssl_path, "#{domain}.key"), 'wb') {|f| f.write key.to_der}
+      File.open(File.join(ssl_path, "#{domain}.crt"), 'wb') {|f| f.write site[:ssl_cert]}
+    end
+  end
+
 end
 
 desc 'Produce SSL config package for proxy'
@@ -216,6 +275,73 @@ task :hash_ips => [:environment] do
   end
 end
 
+desc 'prime_site_files'
+task :prime_site_files => [:environment] do
+  Site.where(is_banned: false).where(is_deleted: false).select(:id, :username).all.each do |site|
+    Dir.glob(File.join(site.files_path, '**/*')).each do |file|
+      path = file.gsub(site.base_files_path, '').sub(/^\//, '')
+
+      site_file = site.site_files_dataset[path: path]
+
+      if site_file.nil?
+        mtime = File.mtime file
+
+        site_file_opts = {
+          path: path,
+          updated_at: mtime,
+          created_at: mtime
+        }
+
+        if File.directory? file
+          site_file_opts.merge! is_directory: true
+        else
+          site_file_opts.merge!(
+            size: File.size(file),
+            sha1_hash: Digest::SHA1.file(file).hexdigest
+          )
+        end
+
+        site.add_site_file site_file_opts
+      end
+    end
+  end
+end
+
+desc 'dedupe_follows'
+task :dedupe_follows => [:environment] do
+  follows = Follow.all
+  deduped_follows = Follow.all.uniq {|f| "#{f.site_id}_#{f.actioning_site_id}"}
+
+  follows.each do |follow|
+    unless deduped_follows.include?(follow)
+      puts "deleting dedupe: #{follow.inspect}"
+      follow.delete
+    end
+  end
+end
+
+desc 'flush_empty_index_sites'
+task :flush_empty_index_sites => [:environment] do
+  sites = Site.select(:id).all
+
+  counter = 0
+
+  sites.each do |site|
+    if site.empty_index?
+      counter += 1
+      site.site_changed = false
+      site.save_changes validate: false
+    end
+  end
+
+  puts "#{counter} sites set to not changed."
+end
+
+desc 'compute_scores'
+task :compute_scores => [:environment] do
+  Site.compute_scores
+end
+
 =begin
 desc 'Update screenshots'
 task :update_screenshots => [:environment] do
@@ -224,3 +350,56 @@ task :update_screenshots => [:environment] do
   }
 end
 =end
+
+desc 'prime_classifier'
+task :prime_classifier => [:environment] do
+  Site.select(:id, :username).where(is_banned: false, is_deleted: false).all.each do |site|
+    next if site.site_files_dataset.where(classifier: 'spam').count > 0
+    html_files = site.site_files_dataset.where(path: /\.html$/).all
+
+    html_files.each do |html_file|
+      print "training #{site.username}/#{html_file.path}..."
+      site.train html_file.path
+      print "done.\n"
+    end
+  end
+end
+
+desc 'train_spam'
+task :train_spam => [:environment] do
+  paths = File.read('./spam.txt')
+
+  paths.split("\n").each do |path|
+    username, site_file_path = path.match(/^([a-zA-Z0-9_\-]+)\/(.+)$/i).captures
+    site = Site[username: username]
+    next if site.nil?
+    site_file = site.site_files_dataset.where(path: site_file_path).first
+    next if site_file.nil?
+    site.train site_file_path, :spam
+    site.ban!
+    puts "Deleted #{site_file_path}, banned #{site.username}"
+  end
+end
+
+desc 'regenerate_ssl_certs'
+task :regenerate_ssl_certs => [:environment] do
+  sites = DB[%{select id from sites where (domain is not null or domain != '') and is_banned != 't' and is_deleted != 't'}].all
+
+  seconds = 2
+
+  sites.each do |site|
+    LetsEncryptWorker.perform_in seconds, site[:id]
+    seconds += 10
+  end
+
+  puts "#{sites.length.to_s} records are primed"
+end
+
+desc 'renew_ssl_certs'
+task :renew_ssl_certs => [:environment] do
+  delay = 0
+  DB[%{select id from sites where (domain is not null or domain != '') and is_banned != 't' and is_deleted != 't' and (cert_updated_at is null or cert_updated_at < ?)}, 60.days.ago].all.each do |site|
+    LetsEncryptWorker.perform_in delay.seconds, site[:id]
+    delay += 10
+  end
+end

app.rb

@@ -27,8 +27,10 @@ before do
   if request.path.match /^\/api\//i
     @api = true
     content_type :json
-  elsif request.path.match /^\/stripe_webhook$/
+  elsif request.path.match /^\/webhooks\//
-    # Skips the CSRF check for stripe web hooks
+    # Skips the CSRF/validation check for stripe web hooks
+  elsif email_not_validated? && !(request.path =~ /^\/site\/.+\/confirm_email|^\/settings\/change_email|^\/signout|^\/welcome|^\/plan/)
+    redirect "/site/#{current_site.username}/confirm_email"
   else
     content_type :html, 'charset' => 'utf-8'
     redirect '/' if request.post? && !csrf_safe?

app/admin.rb

@@ -11,11 +11,166 @@ get '/admin/reports' do
   erb :'admin/reports'
 end
 
+get '/admin/site/:username' do |username|
+  require_admin
+  @site = Site[username: username]
+  not_found if @site.nil?
+  @title = "Site Inspector - #{@site.username}"
+  erb :'admin/site'
+end
+
+post '/admin/reports' do
+
+end
+
+post '/admin/site_files/train' do
+  require_admin
+  site = Site[params[:site_id]]
+  site_file = site.site_files_dataset.where(path: params[:path]).first
+  not_found if site_file.nil?
+  site.untrain site_file.path
+  site.train site_file.path, params[:classifier]
+  'ok'
+end
+
+get '/admin/usage' do
+  require_admin
+  today = Date.today
+  current_month = Date.new today.year, today.month, 1
+
+  @monthly_stats = []
+
+  month = current_month
+
+  until month.year == 2016 && month.month == 2 do
+
+    stats = DB["select sum(views) as views, sum(hits) as hits,sum(bandwidth) as bandwidth from daily_site_stats where created_at::text LIKE '#{month.year}-#{month.strftime('%m')}-%'"].first
+
+    stats.keys.each do |key|
+      stats[key] ||= 0
+    end
+
+    stats.collect {|s| s == 0}.uniq
+
+    if stats[:views] != 0 && stats[:hits] != 0 && stats[:bandwidth] != 0
+      popular_sites = DB[
+        'select sum(bandwidth) as bandwidth,username from stats left join sites on sites.id=stats.site_id where stats.created_at >= ? and stats.created_at < ? group by username order by bandwidth desc limit 50',
+        month,
+        month.next_month
+      ].all
+
+      @monthly_stats.push stats.merge(date: month).merge(popular_sites: popular_sites)
+    end
+
+    month = month.prev_month
+  end
+
+  erb :'admin/usage'
+end
+
 get '/admin/email' do
   require_admin
   erb :'admin/email'
 end
 
+get '/admin/stats' do
+  require_admin
+  # expires 14400, :public, :must_revalidate if self.class.production? # 4 hours
+
+  @stats = {
+    total_hosted_site_hits: DB['SELECT SUM(hits) FROM sites'].first[:sum],
+    total_hosted_site_views: DB['SELECT SUM(views) FROM sites'].first[:sum],
+    total_site_changes: DB['select max(changed_count) from sites'].first[:max],
+    total_sites: Site.count
+  }
+
+  # Start with the date of the first created site
+
+  start = Site.select(:created_at).
+               exclude(created_at: nil).
+               order(:created_at).
+               first[:created_at].to_date
+
+  runner = start
+
+  monthly_stats = []
+
+  now = Date.today
+
+  while Date.new(runner.year, runner.month, 1) <= Date.new(now.year, now.month, 1)
+    monthly_stats.push(
+      date: runner,
+      sites_created: Site.where(created_at: runner..runner.next_month).count,
+      total_from_start: Site.where(created_at: start..runner.next_month).count,
+      supporters: Site.where(created_at: start..runner.next_month).exclude(stripe_customer_id: nil).count,
+    )
+
+    runner = runner.next_month
+  end
+
+  @stats[:monthly_stats] = monthly_stats
+
+  if $stripe_cache && Time.now < $stripe_cache[:time] + 14400
+    customers = $stripe_cache[:customers]
+  else
+    customers = Stripe::Customer.all limit: 100000
+    $stripe_cache = {
+      customers: customers,
+      time: Time.now
+    }
+  end
+
+  @stats[:monthly_revenue] = 0.0
+
+  subscriptions = []
+  @stats[:cancelled_subscriptions] = 0
+
+  customers.each do |customer|
+    sub = {created_at: Time.at(customer.created)}
+
+    if customer[:subscriptions][:data].empty?
+      @stats[:cancelled_subscriptions] += 1
+      next
+    end
+
+    next if customer[:subscriptions][:data].first[:plan][:amount] == 0
+
+    sub[:status] = 'active'
+    plan = customer[:subscriptions][:data].first[:plan]
+
+    sub[:amount_without_fees] = (plan[:amount] / 100.0).round(2)
+    sub[:percentage_fee] = (sub[:amount_without_fees]/(100/2.9)).ceil_to(2)
+    sub[:fixed_fee] = 0.30
+    sub[:amount] = sub[:amount_without_fees] - sub[:percentage_fee] - sub[:fixed_fee]
+
+    if(plan[:interval] == 'year')
+      sub[:amount] = (sub[:amount] / 12).round(2)
+    end
+
+    @stats[:monthly_revenue] += sub[:amount]
+
+    subscriptions.push sub
+  end
+
+  @stats[:subscriptions] = subscriptions
+
+  # Hotwired for now
+  @stats[:expenses] = 300.0 #/mo
+  @stats[:percent_until_profit] = (
+    (@stats[:monthly_revenue].to_f / @stats[:expenses]) * 100
+  )
+
+  @stats[:poverty_threshold] = 11_945
+  @stats[:poverty_threshold_percent] = (@stats[:monthly_revenue].to_f / ((@stats[:poverty_threshold]/12) + @stats[:expenses])) * 100
+
+  # http://en.wikipedia.org/wiki/Poverty_threshold
+
+  @stats[:average_developer_salary] = 93_280.00 # google "average developer salary"
+  @stats[:percent_until_developer_salary] = (@stats[:monthly_revenue].to_f / ((@stats[:average_developer_salary]/12) + @stats[:expenses])) * 100
+
+  erb :'admin/stats'
+end
+
 post '/admin/email' do
   require_admin
 
@@ -55,43 +210,43 @@ post '/admin/email' do
   redirect '/'
 end
 
-post '/admin/banip' do
-  require_admin
-  site = Site[username: params[:username]]
-
-  if site.nil?
-    flash[:error] = 'User not found'
-    redirect '/admin'
-  end
-
-  if site.ip.nil? || site.ip.empty?
-    flash[:error] = 'IP is blank, cannot continue'
-    redirect '/admin'
-  end
-  sites = Site.filter(ip: site.ip, is_banned: false).all
-  sites.each {|s| s.ban!}
-  flash[:error] = "#{sites.length} sites have been banned."
-  redirect '/admin'
-end
-
 post '/admin/banhammer' do
   require_admin
 
-  site = Site[username: params[:username]]
+  if params[:usernames].empty?
-
+    flash[:error] = 'no usernames provided'
-  if site.nil?
-    flash[:error] = 'User not found'
     redirect '/admin'
   end
 
-  if site.is_banned
+  usernames = params[:usernames].split("\n").collect {|u| u.strip}
-    flash[:error] = 'User is already banned'
+
-    redirect '/admin'
+  deleted_count = 0
+  ip_deleted_count = 0
+
+  usernames.each do |username|
+    next if username == ''
+    site = Site[username: username]
+    next if site.nil? || site.is_banned
+
+    if !params[:classifier].empty?
+      site.untrain 'index.html'
+      site.train 'index.html', params[:classifier]
+    end
+
+    site.ban!
+    deleted_count += 1
+
+    if !params[:ban_using_ips].empty? && !site.ip.empty?
+      sites = Site.filter(ip: site.ip, is_banned: false).all
+      sites.each do |s|
+        next if usernames.include?(s.username)
+        s.ban!
+      end
+      ip_deleted_count += 1
+    end
   end
 
-  site.ban!
+  flash[:success] = "#{ip_deleted_count + deleted_count} sites have been banned, including #{ip_deleted_count} matching IPs."
-
-  flash[:success] = 'MISSION ACCOMPLISHED'
   redirect '/admin'
 end
 
@@ -105,6 +260,7 @@ post '/admin/mark_nsfw' do
   end
 
   site.is_nsfw = true
+  site.admin_nsfw = true
   site.save_changes validate: false
 
   flash[:success] = 'MISSION ACCOMPLISHED'
@@ -126,6 +282,14 @@ post '/admin/feature' do
   redirect '/admin'
 end
 
+get '/admin/masquerade/:username' do
+  require_admin
+  site = Site[username: params[:username]]
+  not_found if site.nil?
+  session[:id] = site.id
+  redirect '/'
+end
+
 def require_admin
   redirect '/' unless signed_in? && current_site.is_admin
 end

app/api.rb

@@ -5,6 +5,31 @@ get '/api' do
   erb :'api'
 end
 
+get '/api/list' do
+  require_api_credentials
+
+  files = []
+
+  if params[:path].nil? || params[:path].empty?
+    file_list = current_site.site_files
+  else
+    file_list = current_site.file_list params[:path]
+  end
+
+  file_list.each do |file|
+    new_file = {}
+    new_file[:path] = file[:path]
+    new_file[:is_directory] = file[:is_directory]
+    new_file[:size] = file[:size] unless file[:is_directory]
+    new_file[:updated_at] = file[:updated_at].rfc2822
+    files << new_file
+  end
+
+  files.each {|f| f[:path].sub!(/^\//, '')}
+
+  api_success files: files
+end
+
 post '/api/upload' do
   require_api_credentials
 
@@ -77,7 +102,6 @@ end
 get '/api/info' do
   if params[:sitename]
     site = Site[username: params[:sitename]]
-
     api_error 400, 'site_not_found', "could not find site #{params[:sitename]}" if site.nil? || site.is_banned
     api_success api_info_for(site)
   else
@@ -95,7 +119,8 @@ def api_info_for(site)
       created_at: site.created_at.rfc2822,
       last_updated: site.site_updated_at ? site.site_updated_at.rfc2822 : nil,
       domain: site.domain,
-      tags: site.tags.collect {|t| t.name}
+      tags: site.tags.collect {|t| t.name},
+      latest_ipfs_hash: site.latest_archive ? site.latest_archive.ipfs_hash : nil
     }
   }
 end
@@ -113,6 +138,7 @@ end
 def require_api_credentials
   if !request.env['HTTP_AUTHORIZATION'].nil?
     init_api_credentials
+    api_error(403, 'email_not_validated', 'you need to validate your email address before using the API') if email_not_validated?
   else
     api_error_invalid_auth
   end

app/blog.rb

@@ -1,22 +1,7 @@
-require 'net/http'
-require 'uri'
-
 get '/blog/?' do
-  expires 60, :public, :must_revalidate
+  redirect 'https://blog.neocities.org', 301
-  return Net::HTTP.get_response(URI('http://blog.neocities.org')).body
 end
 
 get '/blog/:article' do |article|
-  expires 60, :public, :must_revalidate
+  redirect "https://blog.neocities.org/#{article}.html", 301
-
-  attempted = false
-
-  begin
-    return Net::HTTP.get_response(URI("http://blog.neocities.org/#{article}.html")).body
-  rescue => e
-    raise e if attempted
-    attempted = true
-    article = article.match(/^[a-zA-Z0-9-]+/).to_s
-    retry
-  end
 end

app/browse.rb

@@ -1,7 +1,7 @@
 get '/browse/?' do
-  @current_page = params[:current_page]
+  @surfmode = false
-  @current_page = @current_page.to_i
+  @page = params[:page].to_i
-  @current_page = 1 if @current_page == 0
+  @page = 1 if @page == 0
 
   params.delete 'tag' if params[:tag].nil? || params[:tag].strip.empty?
 
@@ -11,12 +11,14 @@ get '/browse/?' do
     site_dataset = browse_sites_dataset
   end
 
-  site_dataset = site_dataset.paginate @current_page, Site::BROWSE_PAGINATION_LENGTH
+  site_dataset = site_dataset.paginate @page, Site::BROWSE_PAGINATION_LENGTH
-  @page_count = site_dataset.page_count || 1
+  @pagination_dataset = site_dataset
   @sites = site_dataset.all
+
   if params[:tag]
     @title = "Sites tagged #{params[:tag]}"
   end
+
   erb :browse
 end
 
@@ -28,9 +30,12 @@ def education_sites_dataset
 end
 
 def browse_sites_dataset
-  site_dataset = Site.filter(is_deleted: false, is_banned: false, is_crashing: false).filter(site_changed: true)
+
+  site_dataset = Site.browse_dataset
 
   if current_site
+    site_dataset.or! sites__id: current_site.id
+
     if !current_site.blocking_site_ids.empty?
       site_dataset.where!(Sequel.~(Sequel.qualify(:sites, :id) => current_site.blocking_site_ids))
     end
@@ -43,6 +48,9 @@ def browse_sites_dataset
   end
 
   case params[:sort_by]
+    when 'special_sauce'
+      site_dataset.exclude! score: nil
+      site_dataset.order! :score.desc
     when 'followers'
       site_dataset = site_dataset.association_left_join :follows
       site_dataset.select_all! :sites
@@ -75,16 +83,12 @@ def browse_sites_dataset
       params[:sort_by] = 'last_updated'
       site_dataset.order!(:site_updated_at.desc, :views.desc)
     else
-      if params[:tag]
+      params[:sort_by] = 'followers'
-        params[:sort_by] = 'views'
+      site_dataset = site_dataset.association_left_join :follows
-        site_dataset.order!(:views.desc, :site_updated_at.desc)
+      site_dataset.select_all! :sites
-      else
+      site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
-        site_dataset = site_dataset.association_left_join :follows
+      site_dataset.group! :sites__id
-        site_dataset.select_all! :sites
+      site_dataset.order! :follow_count.desc, :views.desc, :updated_at.desc
-        site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
-        site_dataset.group! :sites__id
-        site_dataset.order! :follow_count.desc, :updated_at.desc
-      end
   end
 
   site_dataset.where! ['sites.is_nsfw = ?', (params[:is_nsfw] == 'true' ? true : false)]

app/create.rb

@@ -1,5 +1,5 @@
 def new_recaptcha_valid?
-  return session[:captcha_valid] = true if ENV['RACK_ENV'] == 'test'
+  return session[:captcha_valid] = true if ENV['RACK_ENV'] == 'test' || ENV['TRAVIS']
   resp = Net::HTTP.get URI(
     'https://www.google.com/recaptcha/api/siteverify?'+
     Rack::Utils.build_query(

app/dashboard.rb

@@ -1,6 +1,13 @@
 get '/dashboard' do
   require_login
   dashboard_init
+  dont_browser_cache
+
+  unless current_site.dashboard_accessed
+    current_site.dashboard_accessed = true
+    current_site.save_changes validate: false
+  end
+
   erb :'dashboard'
 end
 

app/dmca.rb

@@ -30,7 +30,7 @@ post '/dmca/contact' do
       no_footer: true
     })
 
-    flash[:success] = 'Your DCMA notification has been sent.'
+    flash[:success] = 'Your DMCA notification has been sent.'
     redirect '/'
   end
 end

app/domain.rb

@@ -0,0 +1,29 @@
+get '/domain/new' do
+  require_login
+  @title = 'Register a Domain'
+
+  erb :'domain/new'
+end
+
+post '/domain/check_availability.json' do
+  require_login
+  content_type :json
+
+  timer = Time.now.to_i
+
+  while true
+    if (Time.now.to_i - timer) > 60
+      api_error 200, :contact_fail, 'Error contacting domain server, please try again.'
+    end
+
+    begin
+      res = $gandi.domain.available([params[:domain]])[params[:domain]]
+    rescue => Gandi::DataError
+      api_error 200, :invalid_domain, 'Domain name was invalid, please try another.'
+    end
+
+    api_success res unless res == 'pending'
+    sleep 0.2
+  end
+
+end

app/index.rb

@@ -6,23 +6,23 @@ get '/?' do
 
     @suggestions = current_site.suggestions
 
-    @current_page = params[:current_page].to_i
+    @page = params[:page].to_i
-    @current_page = 1 if @current_page == 0
+    @page = 1 if @page == 0
 
     if params[:activity] == 'mine'
-      events_dataset = current_site.latest_events(@current_page, 10)
+      events_dataset = current_site.latest_events(@page, 10)
     elsif params[:event_id]
       event = Event.select(:id).where(id: params[:event_id]).first
       not_found if event.nil?
       not_found if event.is_deleted
       events_dataset = Event.where(id: params[:event_id]).paginate(1, 1)
     elsif params[:activity] == 'global'
-      events_dataset = Event.global_dataset @current_page
+      events_dataset = Event.global_dataset @page
     else
-      events_dataset = current_site.news_feed(@current_page, 10)
+      events_dataset = current_site.news_feed(@page, 10)
     end
 
-    @page_count = events_dataset.page_count || 1
+    @pagination_dataset = events_dataset
     @events = events_dataset.all
 
     current_site.events_dataset.update notification_seen: true
@@ -50,10 +50,6 @@ get '/education' do
   erb :education, layout: :index_layout
 end
 
-get '/tutorials' do
-  erb :'tutorials'
-end
-
 get '/donate' do
   erb :'donate'
 end
@@ -82,3 +78,46 @@ end
 get '/permanent-web' do
   erb :'permanent_web'
 end
+
+get '/thankyou' do
+  erb :'thankyou'
+end
+
+get '/forgot_username' do
+  erb :'forgot_username'
+end
+
+post '/forgot_username' do
+  if params[:email].blank?
+    flash[:error] = 'Cannot use an empty email address!'
+    redirect '/forgot_username'
+  end
+
+  sites = Site.where(email: params[:email]).all
+
+  sites.each do |site|
+    body = <<-EOT
+Hello! This is the Neocities cat, and I have received a username lookup request using this email address.
+
+Your username is #{site.username}
+
+If you didn't request this, you can ignore it. Or hide under a bed. Or take a nap. Your call.
+
+Meow,
+the Neocities Cat
+    EOT
+
+    body.strip!
+
+    EmailWorker.perform_async({
+      from: 'web@neocities.org',
+      to: params[:email],
+      subject: '[Neocities] Username lookup',
+      body: body
+    })
+
+  end
+
+  flash[:success] = 'If your email was valid, the Neocities Cat will send an e-mail with your username in it.'
+  redirect '/'
+end

app/mockup.rb

@@ -29,4 +29,9 @@ get '/stats_mockup' do
   require_login
   erb :'stats_mockup', locals: {site: current_site}
 end
+
+get '/tutorial_mockup_c1p2' do
+  require_login
+  erb :'tutorial_mockup_c1p2', locals: {site: current_site}
+end
 # :nocov:

app/password_reset.rb

@@ -1,39 +1,43 @@
 get '/password_reset' do
+  redirect '/' if signed_in?
   erb :'password_reset'
 end
 
 post '/send_password_reset' do
+  if params[:email].blank?
+    flash[:error] = 'You must enter a valid email address.'
+    redirect '/password_reset'
+  end
+
   sites = Site.filter(email: params[:email]).all
 
   if sites.length > 0
     token = SecureRandom.uuid.gsub('-', '')
     sites.each do |site|
+      next unless site.parent?
       site.update password_reset_token: token
-    end
 
-    body = <<-EOT
+      body = <<-EOT
-Hello! This is the Neocities cat, and I have received a password reset request for your e-mail address. Purrrr.
+Hello! This is the Neocities cat, and I have received a password reset request for your e-mail address.
 
-Go to this URL to reset your password: http://neocities.org/password_reset_confirm?token=#{token}
+Go to this URL to reset your password: http://neocities.org/password_reset_confirm?username=#{Rack::Utils.escape(site.username)}&token=#{token}
 
-After clicking on this link, your password for all the sites registered to this email address will be changed to this token.
+If you didn't request this password reset, you can ignore it. Or hide under a bed. Or take a nap. Your call.
-
-Token: #{token}
-
-If you didn't request this reset, you can ignore it. Or hide under a bed. Or take a nap. Your call.
 
 Meow,
 the Neocities Cat
     EOT
 
-    body.strip!
+      body.strip!
 
-    EmailWorker.perform_async({
+      EmailWorker.perform_async({
-      from: 'web@neocities.org',
+        from: 'web@neocities.org',
-      to: params[:email],
+        to: params[:email],
-      subject: '[Neocities] Password Reset',
+        subject: '[Neocities] Password Reset',
-      body: body
+        body: body
-    })
+      })
+
+    end
   end
 
   flash[:success] = 'If your email was valid (and used by a site), the Neocities Cat will send an e-mail to your account with password reset instructions.'
@@ -42,29 +46,22 @@ end
 
 get '/password_reset_confirm' do
   if params[:token].nil? || params[:token].strip.empty?
-    flash[:error] = 'Could not find a site with this token.'
+    flash[:error] = 'Token cannot be empty.'
     redirect '/'
   end
 
-  reset_site = Site[password_reset_token: params[:token]]
+  reset_site = Site.where(username: params[:username], password_reset_token: params[:token]).first
 
   if reset_site.nil?
-    flash[:error] = 'Could not find a site with this token.'
+    flash[:error] = 'Could not find a site with this username and token.'
     redirect '/'
   end
 
-  sites = Site.filter(email: reset_site.email).all
+  reset_site.password_reset_token = nil
+  reset_site.password_reset_confirmed = true
+  reset_site.save_changes
 
-  if sites.length > 0
+  session[:id] = reset_site.id
-    sites.each do |site|
-      site.password = reset_site.password_reset_token
-      site.save_changes
-    end
 
-    flash[:success] = 'Your password for all sites with your email address has been changed to the token sent in your e-mail. Please login and change your password as soon as possible.'
+  redirect '/settings#password'
-  else
-    flash[:error] = 'Could not find a site with this token.'
-  end
-
-  redirect '/'
 end

app/plan.rb

@@ -100,11 +100,103 @@ get '/plan/thanks' do
   erb :'plan/thanks'
 end
 
+get '/plan/bitcoin/?' do
+  erb :'plan/bitcoin'
+end
+
+get '/plan/alternate/?' do
+  redirect '/plan/bitcoin'
+end
+
+def paypal_recurring_hash
+  {
+    ipn_url:     "https://neocities.org/webhooks/paypal",
+    description: 'Neocities Supporter - Monthly',
+    amount:      Site::PLAN_FEATURES[:supporter][:price].to_s,
+    currency:    'USD'
+  }
+end
+
+def paypal_recurring_authorization_hash
+  paypal_recurring_hash.merge(
+    return_url:  "https://neocities.org/plan/paypal/return",
+    cancel_url:  "https://neocities.org/plan",
+    ipn_url:     "https://neocities.org/webhooks/paypal"
+  )
+end
+
+get '/plan/paypal' do
+  require_login
+  redirect '/plan' if parent_site.supporter?
+
+  hash = paypal_recurring_authorization_hash
+
+  if current_site.paypal_token
+    hash.merge! token: current_site.paypal_token
+  end
+
+  ppr = PayPal::Recurring.new hash
+
+  paypal_response = ppr.checkout
+
+  redirect paypal_response.checkout_url if paypal_response.valid?
+end
+
+get '/plan/paypal/return' do
+  require_login
+
+  if params[:token].nil? || params[:PayerID].nil?
+    flash[:error] = 'Unknown error, could not complete the request. Please contact Neocities support.'
+  end
+
+  ppr = PayPal::Recurring.new(paypal_recurring_hash.merge(
+    token:    params[:token],
+    payer_id: params[:PayerID]
+  ))
+
+  paypal_response = ppr.request_payment
+  unless paypal_response.approved? && paypal_response.completed?
+    flash[:error] = 'Unknown error, could not complete the request. Please contact Neocities support.'
+    redirect '/plan'
+  end
+
+  ppr = PayPal::Recurring.new(paypal_recurring_authorization_hash.merge(
+    frequency:   1,
+    token:       params[:token],
+    period:      :monthly,
+    reference:   current_site.id.to_s,
+    payer_id:    params[:PayerID],
+    start_at:    1.month.from_now,
+    failed:      3,
+    outstanding: :next_billing
+  ))
+
+  paypal_response = ppr.create_recurring_profile
+
+  current_site.paypal_token = params[:token]
+  current_site.paypal_profile_id = paypal_response.profile_id
+  current_site.paypal_active = true
+  current_site.plan_type = 'supporter'
+  current_site.save_changes validate: false
+
+  redirect '/plan/thanks-paypal'
+end
+
 get '/plan/thanks-paypal' do
   require_login
   erb :'plan/thanks-paypal'
 end
 
-get '/plan/alternate/?' do
+get '/plan/paypal/cancel' do
-  erb :'/plan/alternate'
+  require_login
+  redirect '/plan' unless parent_site.paypal_active
+  ppr = PayPal::Recurring.new profile_id: parent_site.paypal_profile_id
+  ppr.cancel
+
+  parent_site.plan_type = nil
+  parent_site.paypal_active = false
+  parent_site.paypal_profile_id = nil
+  parent_site.paypal_token = nil
+  parent_site.save_changes validate: false
+  redirect '/plan'
 end

app/settings.rb

@@ -66,6 +66,7 @@ post '/settings/:username/profile' do
   redirect "/settings/#{@site.username}#profile"
 end
 
+=begin
 post '/settings/:username/ssl' do
   require_login
   require_ownership_for_settings
@@ -167,6 +168,7 @@ post '/settings/:username/ssl' do
   flash[:success] = 'Updated SSL key/certificate.'
   redirect "/settings/#{@site.username}#custom_domain"
 end
+=end
 
 post '/settings/:username/change_name' do
   require_login
@@ -179,13 +181,13 @@ post '/settings/:username/change_name' do
     redirect "/settings/#{@site.username}#username"
   end
 
-  if old_username == params[:name]
+  if old_username.downcase == params[:name].downcase
     flash[:error] = 'You already have this name.'
     redirect "/settings/#{@site.username}#username"
   end
 
   old_host = @site.host
-  old_file_paths = @site.file_list.collect {|f| f[:path]}
+  old_site_file_paths = @site.site_files.collect {|site_file| site_file.path}
 
   @site.username = params[:name]
 
@@ -195,11 +197,11 @@ post '/settings/:username/change_name' do
       @site.move_files_from old_username
     }
 
-    old_file_paths.each do |file_path|
+    old_site_file_paths.each do |site_file_path|
-      @site.purge_cache file_path
+      @site.delete_cache site_file_path
     end
 
-    flash[:success] = "Site/user name has been changed. You will need to use this name to login, <b>don't forget it</b>."
+    flash[:success] = "Site/user name has been changed. You will need to use this name to login, <b>don't forget it!</b>"
     redirect "/settings/#{@site.username}#username"
   else
     flash[:error] = @site.errors.first.last.first
@@ -211,6 +213,8 @@ post '/settings/:username/change_nsfw' do
   require_login
   require_ownership_for_settings
 
+  redirect "/settings/#{@site.username}" if @site.admin_nsfw == true
+
   @site.is_nsfw = params[:is_nsfw]
   @site.save_changes validate: false
   flash[:success] = @site.is_nsfw ? 'Marked 18+' : 'Unmarked 18+'
@@ -221,10 +225,30 @@ post '/settings/:username/custom_domain' do
   require_login
   require_ownership_for_settings
 
+  original_domain = @site.domain
   @site.domain = params[:domain]
 
+  begin
+    Socket.gethostbyname @site.values[:domain]
+  rescue SocketError => e
+    if e.message =~ /name or service not known/i
+      flash[:error] = 'Domain needs to be valid and already registered.'
+      redirect "/settings/#{@site.username}#custom_domain"
+    elsif e.message =~ /No address associated with hostname/i
+      flash[:error] = "The domain isn't setup to use Neocities yet, cannot add. Please make the A and CNAME record changes where you registered your domain."
+      redirect "/settings/#{@site.username}#custom_domain"
+    end
+
+    raise e
+  end
+
   if @site.valid?
     @site.save_changes
+
+    if @site.domain != original_domain
+      LetsEncryptWorker.perform_async @site.id
+    end
+
     flash[:success] = 'The domain has been successfully updated.'
     redirect "/settings/#{@site.username}#custom_domain"
   else
@@ -236,7 +260,7 @@ end
 post '/settings/change_password' do
   require_login
 
-  if !Site.valid_login?(parent_site.username, params[:current_password])
+  if !current_site.password_reset_confirmed && !Site.valid_login?(parent_site.username, params[:current_password])
     flash[:error] = 'Your provided password does not match the current one.'
     redirect "/settings#password"
   end
@@ -248,6 +272,9 @@ post '/settings/change_password' do
     parent_site.errors.add :password, 'New passwords do not match.'
   end
 
+  parent_site.password_reset_token = nil
+  parent_site.password_reset_confirmed = false
+
   if parent_site.errors.empty?
     parent_site.save_changes
     flash[:success] = 'Successfully changed password.'
@@ -261,9 +288,15 @@ end
 post '/settings/change_email' do
   require_login
 
+  if params[:from_confirm]
+    redirect_url = "/site/#{parent_site.username}/confirm_email"
+  else
+    redirect_url = '/settings#email'
+  end
+
   if params[:email] == parent_site.email
     flash[:error] = 'You are already using this email address for this account.'
-    redirect '/settings#email'
+    redirect redirect_url
   end
 
   parent_site.email = params[:email]
@@ -273,12 +306,17 @@ post '/settings/change_email' do
   if parent_site.valid?
     parent_site.save_changes
     send_confirmation_email
-    flash[:success] = 'Successfully changed email. We have sent a confirmation email, please use it to confirm your email address.'
+    if !parent_site.supporter?
-    redirect '/settings#email'
+      session[:fromsettings] = true
+      redirect "/site/#{parent_site.email}/confirm_email"
+    else
+      flash[:success] = 'Email address changed.'
+      redirect '/settings#email'
+    end
   end
 
   flash[:error] = parent_site.errors.first.last.first
-  redirect '/settings#email'
+  redirect redirect_url
 end
 
 post '/settings/change_email_notification' do
@@ -332,3 +370,29 @@ get '/settings/unsubscribe_email/?' do
   end
   erb :'settings/account/unsubscribe'
 end
+
+post '/settings/update_card' do
+  require_login
+
+  customer = Stripe::Customer.retrieve current_site.stripe_customer_id
+
+  old_card_ids = customer.sources.collect {|s| s.id}
+
+  begin
+    customer.sources.create source: params[:stripe_token]
+  rescue Stripe::InvalidRequestError => e
+    if  e.message.match /cannot use a.+token more than once/
+      flash[:error] = 'Card is already being used.'
+      redirect '/settings#billing'
+    else
+      raise e
+    end
+  end
+
+  old_card_ids.each do |card_id|
+    customer.sources.retrieve(card_id).delete
+  end
+
+  flash[:success] = 'Card information updated.'
+  redirect '/settings#billing'
+end

app/site.rb

@@ -13,9 +13,9 @@ get '/site/:username/?' do |username|
 
   @title = site.title
 
-  @current_page = params[:current_page]
+  @page = params[:page]
-  @current_page = @current_page.to_i
+  @page = @page.to_i
-  @current_page = 1 if @current_page == 0
+  @page = 1 if @page == 0
 
   if params[:event_id]
     not_found unless params[:event_id].is_integer?
@@ -23,10 +23,11 @@ get '/site/:username/?' do |username|
     not_found if event.nil?
     events_dataset = Event.where(id: params[:event_id]).paginate(1, 1)
   else
-    events_dataset = site.latest_events(@current_page, 10)
+    events_dataset = site.latest_events(@page, 10)
   end
 
   @page_count = events_dataset.page_count || 1
+  @pagination_dataset = events_dataset
   @latest_events = events_dataset.all
 
   erb :'site', locals: {site: site, is_current_site: site == current_site}
@@ -161,7 +162,6 @@ post '/site/create_directory' do
   require_login
 
   path = "#{params[:dir] || ''}/#{params[:name]}"
-
   result = current_site.create_directory path
 
   if result != true
@@ -172,8 +172,23 @@ post '/site/create_directory' do
 end
 
 get '/site/:username/confirm_email/:token' do
+  if current_site && current_site.email_confirmed
+    return erb(:'site_email_confirmed')
+  end
+
   site = Site[username: params[:username]]
-  if !site.nil? && site.email_confirmation_token == params[:token]
+
+  if site.nil?
+    return erb(:'site_email_not_confirmed')
+  end
+
+  if site.email_confirmed
+    return erb(:'site_email_confirmed')
+  end
+
+  if site.email_confirmation_token == params[:token]
+    site.email_confirmation_token = nil
+    site.email_confirmation_count = 0
     site.email_confirmed = true
     site.save_changes
 
@@ -183,6 +198,47 @@ get '/site/:username/confirm_email/:token' do
   end
 end
 
+get '/site/:username/confirm_email' do
+  require_login
+  @fromsettings = session[:fromsettings]
+  redirect '/' if current_site.username != params[:username] || !current_site.parent? || current_site.email_confirmed
+  erb :'site/confirm_email'
+end
+
+post '/site/:username/confirm_email' do
+  require_login
+
+  redirect '/' if current_site.username != params[:username] || !current_site.parent? || current_site.email_confirmed
+
+  # Update email, resend token
+  if params[:email]
+    send_confirmation_email @site
+  end
+
+  if params[:token].blank?
+    flash[:error] = 'You must enter a valid token.'
+    redirect "/site/#{current_site.username}/confirm_email"
+  end
+
+  if current_site.email_confirmation_token == params[:token]
+    current_site.email_confirmation_token = nil
+    current_site.email_confirmation_count = 0
+    current_site.email_confirmed = true
+    current_site.save_changes
+
+    if session[:fromsettings]
+      session[:fromsettings] = nil
+      flash[:success] = 'Email address changed.'
+      redirect '/settings#email'
+    end
+
+    redirect '/tutorial'
+  else
+    flash[:error] = 'You must enter a valid token.'
+    redirect "/site/#{current_site.username}/confirm_email"
+  end
+end
+
 post '/site/:username/report' do |username|
   site = Site[username: username]
 

app/site_files.rb

@@ -134,10 +134,11 @@ end
 
 post '/site_files/delete' do
   require_login
-  current_site.delete_file params[:filename]
+  path = HTMLEntities.new.decode params[:filename]
-  flash[:success] = "Deleted #{params[:filename]}."
+  current_site.delete_file path
+  flash[:success] = "Deleted #{params[:filename]}. Please note it can take up to 30 minutes for deleted files to stop being viewable on your site."
 
-  dirname = Pathname(params[:filename]).dirname
+  dirname = Pathname(path).dirname
   dir_query = dirname.nil? || dirname.to_s == '.' ? '' : "?dir=#{Rack::Utils.escape dirname}"
 
   redirect "/dashboard#{dir_query}"
@@ -151,15 +152,18 @@ get '/site_files/:username.zip' do |username|
   send_file zipfile_path
 end
 
-get '/site_files/download/:filename' do |filename|
+get %r{\/site_files\/download\/(.+)} do
   require_login
-  content_type 'application/octet-stream'
+  not_found if params[:captures].nil? || params[:captures].length != 1
+  filename = params[:captures].first
   attachment filename
-  current_site.get_file filename
+  send_file current_site.current_files_path(filename)
 end
 
 get %r{\/site_files\/text_editor\/(.+)} do
   require_login
+  dont_browser_cache
+
   @filename = params[:captures].first
   extname = File.extname @filename
   @ace_mode = case extname
@@ -171,15 +175,20 @@ get %r{\/site_files\/text_editor\/(.+)} do
       nil
   end
 
-  begin
+  file_path = current_site.current_files_path @filename
-    @file_data = current_site.get_file @filename
+
-  rescue Errno::ENOENT
+  if File.directory? file_path
-    flash[:error] = 'We could not find the requested file.'
-    redirect '/dashboard'
-  rescue Errno::EISDIR
     flash[:error] = 'Cannot edit a directory.'
     redirect '/dashboard'
   end
+
+  if !File.exist?(file_path)
+    flash[:error] = 'We could not find the requested file.'
+    redirect '/dashboard'
+  end
+
+  @title = "Editing #{@filename}"
+
   erb :'site_files/text_editor'
 end
 
@@ -207,6 +216,10 @@ get '/site_files/allowed_types' do
   erb :'site_files/allowed_types'
 end
 
+get '/site_files/hotlinking' do
+  erb :'site_files/hotlinking'
+end
+
 get '/site_files/mount_info' do
   erb :'site_files/mount_info'
 end

app/stats.rb

@@ -1,100 +0,0 @@
-get '/stats/?' do
-  # expires 14400, :public, :must_revalidate if self.class.production? # 4 hours
-
-  @stats = {
-    total_hosted_site_hits: DB['SELECT SUM(hits) FROM sites'].first[:sum],
-    total_hosted_site_views: DB['SELECT SUM(views) FROM sites'].first[:sum],
-    total_sites: Site.count,
-    total_unbanned_sites: Site.where(is_banned: false).count,
-    total_banned_sites: Site.where(is_banned: true).count,
-    total_nsfw_sites: Site.where(is_nsfw: true).count,
-    total_unbanned_nsfw_sites: Site.where(is_banned: false, is_nsfw: true).count,
-    total_banned_nsfw_sites: Site.where(is_banned: true, is_nsfw: true).count
-  }
-
-  # Start with the date of the first created site
-
-  start = Site.select(:created_at).
-               exclude(created_at: nil).
-               order(:created_at).
-               first[:created_at].to_date
-
-  runner = start
-
-  monthly_stats = []
-
-  now = Date.today
-
-  while Date.new(runner.year, runner.month, 1) <= Date.new(now.year, now.month, 1)
-    monthly_stats.push(
-      date: runner,
-      sites_created: Site.where(created_at: runner..runner.next_month).count,
-      total_from_start: Site.where(created_at: start..runner.next_month).count,
-      supporters: Site.where(created_at: start..runner.next_month).exclude(stripe_customer_id: nil).count,
-    )
-
-    runner = runner.next_month
-  end
-
-  @stats[:monthly_stats] = monthly_stats
-
-  if $stripe_cache && Time.now < $stripe_cache[:time] + 14400
-    customers = $stripe_cache[:customers]
-  else
-    customers = Stripe::Customer.all limit: 100000
-    $stripe_cache = {
-      customers: customers,
-      time: Time.now
-    }
-  end
-
-  @stats[:monthly_revenue] = 0.0
-
-  subscriptions = []
-  @stats[:cancelled_subscriptions] = 0
-
-  customers.each do |customer|
-    sub = {created_at: Time.at(customer.created)}
-
-    if customer[:subscriptions][:data].empty?
-      @stats[:cancelled_subscriptions] += 1
-      next
-    end
-
-    next if customer[:subscriptions][:data].first[:plan][:amount] == 0
-
-    sub[:status] = 'active'
-    plan = customer[:subscriptions][:data].first[:plan]
-
-    sub[:amount_without_fees] = (plan[:amount] / 100.0).round(2)
-    sub[:percentage_fee] = (sub[:amount_without_fees]/(100/2.9)).ceil_to(2)
-    sub[:fixed_fee] = 0.30
-    sub[:amount] = sub[:amount_without_fees] - sub[:percentage_fee] - sub[:fixed_fee]
-
-    if(plan[:interval] == 'year')
-      sub[:amount] = (sub[:amount] / 12).round(2)
-    end
-
-    @stats[:monthly_revenue] += sub[:amount]
-
-    subscriptions.push sub
-  end
-
-  @stats[:subscriptions] = subscriptions
-
-  # Hotwired for now
-  @stats[:expenses] = 300.0 #/mo
-  @stats[:percent_until_profit] = (
-    (@stats[:monthly_revenue].to_f / @stats[:expenses]) * 100
-  )
-
-  @stats[:poverty_threshold] = 11_945
-  @stats[:poverty_threshold_percent] = (@stats[:monthly_revenue].to_f / ((@stats[:poverty_threshold]/12) + @stats[:expenses])) * 100
-
-  # http://en.wikipedia.org/wiki/Poverty_threshold
-
-  @stats[:average_developer_salary] = 93_280.00 # google "average developer salary"
-  @stats[:percent_until_developer_salary] = (@stats[:monthly_revenue].to_f / ((@stats[:average_developer_salary]/12) + @stats[:expenses])) * 100
-
-  erb :'stats'
-end

app/surf.rb

@@ -1,8 +1,8 @@
 get '/surf/?' do
-  @current_page = params[:current_page].to_i || 1
+  @page = params[:page].to_i || 1
   params.delete 'tag' if params[:tag].nil? || params[:tag].strip.empty?
   site_dataset = browse_sites_dataset
-  site_dataset = site_dataset.paginate @current_page, 1
+  site_dataset = site_dataset.paginate @page, 1
   @page_count = site_dataset.page_count || 1
   @site = site_dataset.first
   redirect "/browse?#{Rack::Utils.build_query params}" if @site.nil?

app/tags.rb

@@ -16,7 +16,10 @@ post '/tags/remove' do
 
   if params[:tags].is_a?(Array)
     DB.transaction {
-      params[:tags].each {|tag| current_site.remove_tag Tag[name: tag]}
+      params[:tags].each do |tag|
+        tag_to_remove = current_site.tags.select {|t| t.name == tag}.first
+        current_site.remove_tag(tag_to_remove) if tag_to_remove
+      end
     }
   end
 

app/tutorial.rb

@@ -0,0 +1,41 @@
+def default_tutorial_html
+  <<-EOT.strip
+<!DOCTYPE html>
+<html>
+  <body>
+
+
+    Hello World!
+
+
+  </body>
+</html>
+EOT
+end
+
+get '/tutorials' do
+  erb :'tutorials'
+end
+
+get '/tutorial/?' do
+  require_login
+  erb :'tutorial/index'
+end
+
+get '/tutorial/:section/?' do
+  require_login
+  redirect "/tutorial/#{params[:section]}/1"
+end
+
+get '/tutorial/:section/:page/?' do
+  require_login
+  @page = params[:page]
+  not_found if @page.to_i == 0
+  not_found unless %w{html css js}.include?(params[:section])
+
+  @section = params[:section]
+
+  @title = "#{params[:section].upcase} Tutorial - #{@page}/10"
+
+  erb "tutorial/layout".to_sym
+end

app/webhooks.rb

@@ -1,24 +1,16 @@
-def stripe_get_site_from_event(event)
+post '/webhooks/paypal' do
-  customer_id = event['data']['object']['customer']
+  EmailWorker.perform_async({
-  customer = Stripe::Customer.retrieve customer_id
+    from: 'web@neocities.org',
+    to: 'errors@neocities.org',
+    subject: "[Neocities Paypal Webhook] Received a Webhook from Paypal",
+    body: params.inspect,
+    no_footer: true
+  })
 
-  # Some old accounts only have a username for the desc
+  'ok'
-  desc_split = customer.description.split(' - ')
-
-  if desc_split.length == 1
-    site_where = {username: desc_split.first}
-  end
-
-  if desc_split.last.to_i == 0
-    site_where = {username: desc_split.first}
-  else
-    site_where = {id: desc_split.last}
-  end
-
-  Site.where(site_where).first
 end
 
-post '/stripe_webhook' do
+post '/webhooks/stripe' do
   event = JSON.parse request.body.read
   if event['type'] == 'customer.created'
     username  = event['data']['object']['description'].split(' - ').first
@@ -60,3 +52,23 @@ post '/stripe_webhook' do
 
   'ok'
 end
+
+def stripe_get_site_from_event(event)
+  customer_id = event['data']['object']['customer']
+  customer = Stripe::Customer.retrieve customer_id
+
+  # Some old accounts only have a username for the desc
+  desc_split = customer.description.split(' - ')
+
+  if desc_split.length == 1
+    site_where = {username: desc_split.first}
+  end
+
+  if desc_split.last.to_i == 0
+    site_where = {username: desc_split.first}
+  else
+    site_where = {id: desc_split.last}
+  end
+
+  Site.where(site_where).first
+end

app_helpers.rb

@@ -1,13 +1,3 @@
-def kickstarter_days_remaining
-  ending = Time.parse('Sat, Jul 25 2015 3:05 PM PDT')
-  today = Time.now
-
-  remaining = ending - today
-  return 0 if remaining < 0
-
-  ((ending - today) / 86400).to_i
-end
-
 def dashboard_if_signed_in
   redirect '/dashboard' if signed_in?
 end
@@ -82,12 +72,19 @@ def encoding_fix(file)
 end
 
 def send_confirmation_email(site=current_site)
+  if site.email_confirmation_count > Site::MAXIMUM_EMAIL_CONFIRMATIONS
+    flash[:error] = 'You sent too many email confirmation requests, cannot continue.'
+    redirect request.referrer
+  end
+
+  DB['UPDATE sites set email_confirmation_count=email_confirmation_count+1 WHERE id=?', site.id].first
+
   EmailWorker.perform_async({
     from: 'web@neocities.org',
     reply_to: 'contact@neocities.org',
     to: site.email,
     subject: "[Neocities] Confirm your email address",
-    body: Tilt.new('./views/templates/email_confirm.erb', pretty: true).render(self, site: site)
+    body: Tilt.new('./views/templates/email/confirm.erb', pretty: true).render(self, site: site)
   })
 end
 
@@ -115,6 +112,20 @@ def plan_pricing_button(plan_type)
       button_title = parent_site.plan_type == 'free' ? 'Upgrade' : 'Change'
     end
 
+    if button_title == 'Change' && parent_site && parent_site.paypal_active
+      return %{<a href="/plan/paypal/cancel" onclick="return confirm('This will end your supporter plan.')" class="btn-Action">Change</a>}
+    end
+
     %{<a data-plan_name="#{Site::PLAN_FEATURES[plan_type.to_sym][:name]}" data-plan_type="#{plan_type}" data-plan_price="#{plan_price}" onclick="card = new Skeuocard($('#skeuocard')); return false" class="btn-Action planPricingButton">#{button_title}</a>}
   end
 end
+
+def dont_browser_cache
+  @dont_browser_cache = true
+end
+
+def email_not_validated?
+  return false if current_site && current_site.created_at < Site::EMAIL_VALIDATION_CUTOFF_DATE
+
+  current_site && current_site.parent? && !current_site.is_education && !current_site.email_confirmed && !current_site.supporter?
+end

code-of-conduct.txt

@@ -0,0 +1,9 @@
+These guidelines apply to Neocities contributor communications, including 
+messages in code repositories, collaboration tools, outreach channels, etc.
+
+* Language and actions must be free of personal attacks, harassment, 
+  discrimination, and NSFW content.
+
+Instances of unacceptable behavior can be reported by contacting the project 
+team at abuse@neocities.org. All complaints will be investigated and reporter 
+confidentiality will be maintained.

config.ru

@@ -1,11 +1,6 @@
 require 'rubygems'
 require './app.rb'
 require 'sidekiq/web'
-require 'unicorn/preread_input'
-
-if defined?(Unicorn)
-  use Unicorn::PrereadInput
-end
 
 map('/') do
   use(Rack::Cache,
@@ -35,13 +30,18 @@ map '/webdav' do
       end
 
       if Site.valid_file_type?(filename: path, tempfile: tmpfile)
-        site.store_file path, tmpfile
+        site.store_files [{filename: path, tempfile: tmpfile}]
         return [201, {}, ['']]
       else
         return [415, {}, ['']]
       end
     end
 
+    if env['REQUEST_METHOD'] == 'MKCOL'
+      site.create_directory env['PATH_INFO']
+      return [201, {}, ['']]
+    end
+
     if env['REQUEST_METHOD'] == 'MOVE'
       tmpfile = Tempfile.new 'moved_file'
       tmpfile.close
@@ -51,7 +51,7 @@ map '/webdav' do
       FileUtils.cp site.files_path(env['PATH_INFO']), tmpfile.path
 
       DB.transaction do
-        site.store_file destination, tmpfile
+        site.store_files [{filename: destination, tempfile: tmpfile}]
         site.delete_file env['PATH_INFO']
       end
 

config.yml.template

@@ -12,6 +12,11 @@ development:
   proxy_pass: 'somethinglongandrandom'
   email_unsubscribe_token: 'somethingrandom'
   logs_path: /path/to/nginx/logs
+  paypal_api_username: derp
+  paypal_api_password: ing
+  paypal_api_signature: tonz
+  letsencrypt_key: ./tests/files/letsencrypt.key
+  letsencrypt_endpoint: https://acme-staging.api.letsencrypt.org/
 test:
   database: 'postgres://neocities@localhost/neocities_test'
   database_pool: 1
@@ -25,3 +30,8 @@ test:
   ip_hash_salt: "400$8$1$fc21863da5d531c1"
   proxy_pass: 'somethinglongandrandom'
   email_unsubscribe_token: 'somethingrandom'
+  paypal_api_username: derp
+  paypal_api_password: ing
+  paypal_api_signature: tonz
+  letsencrypt_key: ./tests/files/letsencrypt.key
+  letsencrypt_endpoint: https://acme-staging.api.letsencrypt.org/

config.yml.travis

@@ -7,4 +7,9 @@ phantomjs_url:
   - http://localhost:8910
 ip_hash_salt: "400$8$1$fc21863da5d531c1"
 email_unsubscribe_token: "somethingrandomderrrrp"
+paypal_api_username: derp
+paypal_api_password: ing
+paypal_api_signature: tonz
 logs_path: "/tmp/neocitiestestlogs"
+letsencrypt_key: ./tests/files/letsencrypt.key
+letsencrypt_endpoint: https://acme-staging.api.letsencrypt.org/

environment.rb

@@ -11,6 +11,8 @@ require 'logger'
 Bundler.require
 Bundler.require :development if ENV['RACK_ENV'] == 'development'
 
+require 'tilt/erubis'
+
 Dir['./ext/**/*.rb'].each {|f| require f}
 
 # :nocov:
@@ -31,6 +33,8 @@ raise 'hash_ip_salt is required' unless $config['ip_hash_salt']
 DB = Sequel.connect $config['database'], sslmode: 'disable', max_connections: $config['database_pool']
 DB.extension :pagination
 
+require 'will_paginate/sequel'
+
 # :nocov:
 =begin
 if defined?(Pry)
@@ -58,6 +62,14 @@ Sidekiq.configure_client do |config|
   config.redis = sidekiq_redis_config
 end
 
+if ENV['RACK_ENV'] == 'test'
+  $redis = MockRedis.new
+else
+  $redis = Redis.new
+end
+
+$redis_cache = Redis::Namespace.new :cache, redis: $redis
+
 # :nocov:
 if ENV['RACK_ENV'] == 'development'
   # Run async jobs immediately in development.
@@ -122,6 +134,13 @@ if ENV['RACK_ENV'] != 'development'
   Sass::Plugin.options[:full_exception] = false
 end
 
+PayPal::Recurring.configure do |config|
+  config.sandbox = false
+  config.username = $config['paypal_api_username']
+  config.password = $config['paypal_api_password']
+  config.signature = $config['paypal_api_signature']
+end
+
 require 'csv'
 
 $country_codes = {}
@@ -129,3 +148,7 @@ $country_codes = {}
 CSV.foreach("./files/country_codes.csv") do |row|
   $country_codes[row.last] = row.first
 end
+
+gandi_opts = {}
+gandi_opts[:env] = :test unless ENV['RACK_ENV'] == 'production'
+$gandi = Gandi::Session.new $config['gandi_api_key'], gandi_opts

ext/NilClass.rb

@@ -2,4 +2,8 @@ class NilClass
   def empty?
     true
   end
+
+  def blank?
+    true
+  end
 end

ext/base58.rb

@@ -0,0 +1,41 @@
+module Base58
+  class << self
+    def int_to_base58(int_val, leading_zero_bytes=0)
+      alpha = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+      base58_val, base = '', alpha.size
+      while int_val > 0
+        int_val, remainder = int_val.divmod(base)
+        base58_val = alpha[remainder] + base58_val
+      end
+      base58_val
+    end
+
+    def base58_to_int(base58_val)
+      alpha = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+      int_val, base = 0, alpha.size
+      base58_val.reverse.each_char.with_index do |char,index|
+        raise ArgumentError, 'Value not a valid Base58 String.' unless char_index = alpha.index(char)
+        int_val += char_index*(base**index)
+      end
+      int_val
+    end
+
+    def base58_to_bytestring(base58_val)
+      [Base58.decode_base58(base58_val)].pack('H*')
+    end
+
+    def encode_base58(hex)
+      leading_zero_bytes  = (hex.match(/^([0]+)/) ? $1 : '').size / 2
+      ("1"*leading_zero_bytes) + int_to_base58( hex.to_i(16) )
+    end
+
+    def decode_base58(base58_val)
+      s = base58_to_int(base58_val).to_s(16); s = (s.bytesize.odd? ? '0'+s : s)
+      s = '' if s == '00'
+      leading_zero_bytes = (base58_val.match(/^([1]+)/) ? $1 : '').size
+      s = ("00"*leading_zero_bytes) + s  if leading_zero_bytes > 0
+      s
+    end
+    alias_method :base58_to_hex, :decode_base58
+  end
+end

ext/numeric.rb

@@ -31,7 +31,7 @@ class Numeric
   end
 
   def to_comma_separated
-    self.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(",").reverse
+    self.to_i.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(",").reverse
   end
 
   def format_large_number

ext/string.rb

@@ -15,4 +15,9 @@ class String
   def is_integer?
     true if Integer(self) rescue false
   end
+
+  def blank?
+    return true if self == ''
+    false
+  end
 end

files/phantomjs_screenshot.js

@@ -0,0 +1,56 @@
+var page   = require('webpage').create()
+var system = require('system')
+
+var maxTimeout = 15000
+
+if (system.args.length === 1) {
+  console.log('required args: <siteURL> <outputFilePath>');
+  phantom.exit(1)
+}
+
+var address = system.args[1]
+var outputPath = system.args[2]
+
+page.viewportSize = { width: 1280, height: 960 }
+page.clipRect = { top: 0, left: 0, width: 1280, height: 960}
+
+/*
+In development, not working yet.
+
+page.settings.scriptTimeout = 1000
+
+page.onLongRunningScript = function() {
+  page.stopJavaScript()
+  phantom.exit(4)
+}
+*/
+
+var t = Date.now()
+
+console.log('Loading ' + address)
+
+setTimeout(function() {
+  console.log('timeout')
+  phantom.exit(62)
+}, maxTimeout)
+
+page.settings.resourceTimeout = maxTimeout
+
+page.onResourceTimeout = function(e) {
+  console.log(e.errorCode)
+  console.log(e.errorString)
+  console.log(e.url)
+  phantom.exit(3)
+}
+
+page.open(address, function(status) {
+  if(status !== 'success') {
+    console.log('failed')
+    phantom.exit(2)
+  }
+
+  page.render(outputPath)
+  console.log('Loading time ' + (Date.now() - t) + ' msec');
+  phantom.exit()
+})
+

migrations/070_add_admin_nsfw.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :admin_nsfw, :boolean
+  }
+
+  down {
+    DB.drop_column :sites, :admin_nsfw
+  }
+end

migrations/071_banned_referrers.rb

@@ -0,0 +1,12 @@
+Sequel.migration do
+  up {
+    DB.create_table! :banned_referrers do
+      primary_key :id
+      String   :name
+    end
+  }
+
+  down {
+    DB.drop_table :banned_referrers
+  }
+end

migrations/072_banned_commenting.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :commenting_banned, :boolean, default: false
+  }
+
+  down {
+    DB.drop_column :sites, :commenting_banned
+  }
+end

migrations/073_add_follow_uniqueness.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB['alter table follows add constraint one_follow_per_site unique (site_id, actioning_site_id)'].first
+  }
+
+  down {
+    DB['alter table follows drop constraint one_follow_per_site'].first
+  }
+end

migrations/074_add_custom_max_space.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :custom_max_space, :bigint, default: 0
+  }
+
+  down {
+    DB.drop_column :sites, :custom_max_space
+  }
+end

migrations/075_special_sauce.rb

@@ -0,0 +1,11 @@
+# IT'S MADE OUT OF FUCKING PEOPLE
+
+Sequel.migration do
+  up {
+    DB.add_column :sites, :score, :integer
+  }
+
+  down {
+    DB.drop_column :sites, :score
+  }
+end

migrations/076_decimal_sauce.rb

@@ -0,0 +1,13 @@
+# IT'S MADE OUT OF FUCKING DECIMAL PEOPLE
+
+Sequel.migration do
+  up {
+    DB.drop_column :sites, :score
+    DB.add_column :sites, :score, :decimal, default: 0
+  }
+
+  down {
+    DB.drop_column :sites, :score
+    DB.add_column :sites, :score, :integer
+  }
+end

migrations/077_decimal_sauce_index.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_index :sites, :score
+  }
+
+  down {
+    DB.drop_index :sites, :score
+  }
+end

migrations/078_total_stat.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  up {
+    DB.create_table! :daily_site_stats do
+      primary_key :id
+      Date        :created_at,   index: true
+      Integer     :hits,         default: 0
+      Integer     :views,        default: 0
+      Integer     :comments,     default: 0
+      Integer     :follows,      default: 0
+      Integer     :site_updates, default: 0
+    end
+  }
+
+  down {
+    DB.drop_table :daily_site_stats
+  }
+end

migrations/079_add_total_stat_bandwidth.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :daily_site_stats, :bandwidth, :integer
+  }
+
+  down {
+    DB.drop_column :daily_site_stats, :bandwidth
+  }
+end

migrations/080_fix_total_stat_bandwidth.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up {
+    DB.drop_column :daily_site_stats, :bandwidth
+    DB.add_column :daily_site_stats, :bandwidth, :integer, default: 0
+  }
+
+  down {
+    DB.drop_column :daily_site_stats, :bandwidth
+    DB.add_column :daily_site_stats, :bandwidth, :integer
+  }
+end

migrations/081_stats_bigint.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up {
+    alter_table(:stats) do
+      set_column_type :hits, Bignum
+      set_column_type :views, Bignum
+    end
+  }
+
+  down {
+    alter_table(:stats) do
+      set_column_type :hits, Integer
+      set_column_type :views, Integer
+    end
+  }
+end

migrations/082_daily_stats_bigint.rb

@@ -0,0 +1,19 @@
+Sequel.migration do
+  up {
+    alter_table(:daily_site_stats) do
+      set_column_type :hits, Bignum
+      set_column_type :views, Bignum
+      set_column_type :bandwidth, Bignum
+      set_column_type :site_updates, Bignum
+    end
+  }
+
+  down {
+    alter_table(:daily_site_stats) do
+      set_column_type :hits, Integer
+      set_column_type :views, Integer
+      set_column_type :bandwidth, Integer
+      set_column_type :site_updates, Integer
+    end
+  }
+end

migrations/083_add_classifiers.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :site_files, :classifier, :text, default: nil, index: true
+  }
+
+  down {
+    DB.drop_column :site_files, :classifier
+  }
+end

migrations/084_dashboard_accessed.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :dashboard_accessed, :boolean, default: false
+  }
+
+  down {
+    DB.drop_column :sites, :dashboard_accessed
+  }
+end

migrations/085_gandi_contact_handle.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :gandi_handle, :text, index: true
+
+    # This is not as horrible as it looks.
+    # It basically serves as a temp password when account is released from reseller account.
+    DB.add_column :sites, :gandi_password, :text
+  }
+
+  down {
+    DB.drop_column :sites, :gandi_handle
+    DB.drop_column :sites, :gandi_password
+  }
+end

migrations/086_create_domains.rb

@@ -0,0 +1,23 @@
+Sequel.migration do
+  up {
+    DB.drop_column :sites, :gandi_handle
+    DB.drop_column :sites, :gandi_password
+
+    DB.create_table! :domains do
+      primary_key :id
+      Integer  :site_id, index: true
+      String   :gandi_handle
+      String   :gandi_password
+      String   :gandi_domain_id
+      String   :name
+      DateTime :created_at
+      DateTime :released_at
+    end
+  }
+
+  down {
+    DB.drop_table :domains
+    DB.add_column :sites, :gandi_handle, :text, index: true
+    DB.add_column :sites, :gandi_password, :text
+  }
+end

migrations/087_password_reset_confirmed.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :password_reset_confirmed, :boolean, default: false
+  }
+
+  down {
+    DB.drop_column :sites, :password_reset_confirmed
+  }
+end

migrations/088_site_change_files_indexes.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up {
+    DB.add_index :site_change_files, :site_change_id
+    DB.add_index :site_change_files, :site_id
+  }
+
+  down {
+    DB.drop_index :site_change_files, :site_change_id
+    DB.drop_index :site_change_files, :site_id
+  }
+end

migrations/089_maximum_email_confirmations.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :email_confirmation_count, :integer, default: 0
+
+  }
+
+  down {
+    DB.drop_column :sites, :email_confirmation_count
+  }
+end

migrations/090_add_cert_updated_at.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :cert_updated_at, Time
+  }
+
+  down {
+    DB.drop_column :sites, :cert_updated_at
+  }
+end

migrations/091_site_banned_at.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :banned_at, Time
+  }
+
+  down {
+    DB.drop_column :sites, :banned_at
+  }
+end

models/archive.rb

@@ -1,9 +1,47 @@
+require 'base32'
+
 class Archive < Sequel::Model
   many_to_one :site
   set_primary_key [:site_id, :ipfs_hash]
   unrestrict_primary_key
+  MAXIMUM_ARCHIVES_PER_SITE = 100
+  ARCHIVE_WAIT_TIME = 1.hour
+
+  def before_destroy
+    unpin
+    super
+  end
+
+  def self.base58_to_hshca(base58)
+    Base32.encode(Base58.base58_to_bytestring(base58)).gsub('=', '').downcase
+  end
+
+  def hshca_hash
+    self.class.base58_to_hshca ipfs_hash
+  end
+
+  def unpin
+    # Not ideal. An SoA version is in progress.
+    if ENV['RACK_ENV'] == 'production' && $config['ipfs_ssh_host'] && $config['ipfs_ssh_user']
+      rbox = Rye::Box.new $config['ipfs_ssh_host'], :user => $config['ipfs_ssh_user']
+      rbox.disable_safe_mode
+      begin
+        response = rbox.execute "ipfs pin rm #{ipfs_hash}"
+        output_array = response
+      rescue => e
+        return true if e.message =~ /indirect pins cannot be removed directly/
+      ensure
+        rbox.disconnect
+      end
+    else
+      line = Cocaine::CommandLine.new('ipfs', 'pin rm :ipfs_hash')
+      binding.pry
+      response = line.run ipfs_hash: ipfs_hash
+      output_array = response.to_s.split("\n")
+    end
+  end
 
   def url
-    "https://#{ipfs_hash}.ipfs.neocities.org"
+    "http://#{hshca_hash}.ipfs.neocitiesops.net"
   end
 end

models/daily_site_stat.rb

@@ -0,0 +1,2 @@
+class DailySiteStat < Sequel::Model
+end

models/event.rb

@@ -8,7 +8,7 @@ class Event < Sequel::Model
   many_to_one :site_change
   many_to_one :profile_comment
   one_to_many :likes
-  one_to_many :comments
+  one_to_many :comments, order: :created_at
   many_to_one :site
   many_to_one :actioning_site, key: :actioning_site_id, class: :Site
 

models/site.rb

@@ -30,14 +30,15 @@ class Site < Sequel::Model
     application/xml
     audio/midi
     text/cache-manifest
+    application/rss+xml
   }
 
   VALID_EXTENSIONS = %w{
-    html htm txt text css js jpg jpeg png gif svg md markdown eot ttf woff woff2 json geojson csv tsv mf ico pdf asc key pgp xml mid midi manifest otf webapp
+    html htm txt text css js jpg jpeg png gif svg md markdown eot ttf woff woff2 json geojson csv tsv mf ico pdf asc key pgp xml mid midi manifest otf webapp less sass rss kml dae obj mtl
   }
 
   VALID_EDITABLE_EXTENSIONS = %w{
-    html htm txt js css md manifest
+    html htm txt js css md manifest less
   }
 
   MINIMUM_PASSWORD_LENGTH = 5
@@ -74,6 +75,8 @@ class Site < Sequel::Model
     /PHP\.Hide/
   ]
 
+  EMPTY_FILE_HASH = Digest::SHA1.hexdigest ''
+
   PHISHING_FORM_REGEX = /www.formbuddy.com\/cgi-bin\/form.pl/i
   SPAM_MATCH_REGEX = ENV['RACK_ENV'] == 'test' ? /pillz/ : /#{$config['spam_smart_filter'].join('|')}/i
   EMAIL_SANITY_REGEX = /.+@.+\..+/i
@@ -85,7 +88,7 @@ class Site < Sequel::Model
 
   SUGGESTIONS_LIMIT = 30
   SUGGESTIONS_VIEWS_MIN = 500
-  CHILD_SITES_MAX = 100
+  CHILD_SITES_MAX = 30
 
   IP_CREATE_LIMIT = 1000
   TOTAL_IP_CREATE_LIMIT = 10000
@@ -103,7 +106,7 @@ class Site < Sequel::Model
     custom_ssl_certificates: true,
     no_file_restrictions: true,
     custom_domains: true,
-    maximum_site_files: 25000
+    maximum_site_files: 50000
   }
 
   PLAN_FEATURES[:free] = PLAN_FEATURES[:supporter].merge(
@@ -115,9 +118,12 @@ class Site < Sequel::Model
     custom_ssl_certificates: false,
     no_file_restrictions: false,
     custom_domains: false,
-    maximum_site_files: 1000
+    maximum_site_files: 2000
   )
 
+  EMAIL_VALIDATION_CUTOFF_DATE = Time.parse('May 16, 2016')
+  DISPOSABLE_EMAIL_BLACKLIST_PATH = File.join(DIR_ROOT, 'files', 'disposable_email_blacklist.conf')
+
   def self.newsletter_sites
      Site.select(:email).
        exclude(email: 'nil').exclude(is_banned: true).
@@ -157,6 +163,8 @@ class Site < Sequel::Model
     EMAIL_BLAST_MAXIMUM_PER_DAY = 1000
   end
 
+  MAXIMUM_EMAIL_CONFIRMATIONS = 20
+
   many_to_many :tags
 
   one_to_many :profile_comments
@@ -319,7 +327,7 @@ class Site < Sequel::Model
   end
 
   def is_following?(site)
-    followings_dataset.select(:id).filter(site_id: site.id).first ? true : false
+    followings_dataset.select(:follows__id).filter(site_id: site.id).first ? true : false
   end
 
   def toggle_follow(site)
@@ -409,16 +417,16 @@ class Site < Sequel::Model
     FileUtils.cp template_file_path('style.css'), tmpfile.path
     files << {filename: 'style.css', tempfile: tmpfile}
 
-    tmpfile = Tempfile.new 'cat.png'
+    tmpfile = Tempfile.new 'neocities.png'
     tmpfile.close
-    FileUtils.cp template_file_path('cat.png'), tmpfile.path
+    FileUtils.cp template_file_path('neocities.png'), tmpfile.path
-    files << {filename: 'cat.png', tempfile: tmpfile}
+    files << {filename: 'neocities.png', tempfile: tmpfile}
 
     store_files files, new_install: true
   end
 
   def get_file(path)
-    File.read files_path(path)
+    File.read current_files_path(path)
   end
 
   def before_destroy
@@ -447,7 +455,7 @@ class Site < Sequel::Model
 
     DB.transaction {
       self.is_banned = true
-      self.updated_at = Time.now
+      self.banned_at = Time.now
       save(validate: false)
 
       if !Dir.exist? BANNED_SITES_ROOT
@@ -457,8 +465,8 @@ class Site < Sequel::Model
       FileUtils.mv files_path, File.join(BANNED_SITES_ROOT, username)
     }
 
-    file_list.each do |path|
+    site_files.each do |site_file|
-      purge_cache path
+      delete_cache site_file.path
     end
   end
 
@@ -468,6 +476,16 @@ class Site < Sequel::Model
     }
   end
 
+  # Who this site follows
+  def followings_dataset
+    super.select_all(:follows).inner_join(:sites, :id=>:site_id).exclude(:sites__is_deleted => true).exclude(:sites__is_banned => true)
+  end
+
+  # Who this site is following
+  def follows_dataset
+    super.select_all(:follows).inner_join(:sites, :id=>:actioning_site_id).exclude(:sites__is_deleted => true).exclude(:sites__is_banned => true)
+  end
+
 =begin
   def follows_dataset
     super.where(Sequel.~(site_id: blocking_site_ids))
@@ -486,6 +504,7 @@ class Site < Sequel::Model
 =end
 
   def commenting_allowed?
+    return false if owner.commenting_banned == true
     return true if owner.commenting_allowed
 
     if owner.supporter?
@@ -522,6 +541,21 @@ class Site < Sequel::Model
     !username.empty? && username.match(/^[a-zA-Z0-9_\-]+$/i)
   end
 
+  def self.disposable_email?(email)
+    return false unless File.exist?(DISPOSABLE_EMAIL_BLACKLIST_PATH)
+    return false if email.blank?
+
+    email.strip!
+
+    disposable_email_domains = File.readlines DISPOSABLE_EMAIL_BLACKLIST_PATH
+
+    disposable_email_domains.each do |disposable_email_domain|
+      return true if email.match disposable_email_domain.strip
+    end
+
+    false
+  end
+
   def okay_to_upload?(uploaded_file)
     return true if [:supporter].include?(plan_type.to_sym)
     return false if self.class.possible_phishing?(uploaded_file)
@@ -580,16 +614,41 @@ class Site < Sequel::Model
     # We gotta flush the dirname too if it's an index file.
     if relative_path != '' && relative_path.match(/\/$|index\.html?$/i)
       PurgeCacheOrderWorker.perform_async username, relative_path
-      PurgeCacheOrderWorker.perform_async username, Pathname(relative_path).dirname.to_s
+
+      purge_file_path = Pathname(relative_path).dirname.to_s
+
+      PurgeCacheOrderWorker.perform_async username, '/?surf=1' if purge_file_path == '/'
+      PurgeCacheOrderWorker.perform_async username, purge_file_path
     else
       PurgeCacheOrderWorker.perform_async username, relative_path
     end
   end
 
+  # TODO DRY this up
+
+  def delete_cache(path)
+    relative_path = path.gsub base_files_path, ''
+
+    DeleteCacheOrderWorker.perform_async username, relative_path
+
+    # We gotta flush the dirname too if it's an index file.
+    if relative_path != '' && relative_path.match(/\/$|index\.html?$/i)
+      purge_file_path = Pathname(relative_path).dirname.to_s
+
+      DeleteCacheOrderWorker.perform_async username, '/?surf=1' if purge_file_path == '/'
+      DeleteCacheOrderWorker.perform_async username, purge_file_path
+    end
+  end
+
   Rye::Cmd.add_command :ipfs, nil, 'add', :r
 
   def add_to_ipfs
     # Not ideal. An SoA version is in progress.
+
+    if archives_dataset.count > Archive::MAXIMUM_ARCHIVES_PER_SITE
+      archives_dataset.order(:updated_at).first.destroy
+    end
+
     if $config['ipfs_ssh_host'] && $config['ipfs_ssh_user']
       rbox = Rye::Box.new $config['ipfs_ssh_host'], :user => $config['ipfs_ssh_user']
       begin
@@ -607,6 +666,12 @@ class Site < Sequel::Model
     output_array.last.split(' ')[1]
   end
 
+  def purge_old_archives
+    archives_dataset.order(:updated_at).offset(Archive::MAXIMUM_ARCHIVES_PER_SITE).all.each do |archive|
+      archive.destroy
+    end
+  end
+
   def archive!
     #if ENV["RACK_ENV"] == 'test'
     #  ipfs_hash = "QmcKi2ae3uGb1kBg1yBpsuwoVqfmcByNdMiZ2pukxyLWD8"
@@ -638,6 +703,32 @@ class Site < Sequel::Model
       return 'Directory (or file) already exists.'
     end
 
+    path_dirs = path.to_s.split('/').select {|p| ![nil, '.', ''].include?(p) }
+
+    path_site_file = ''
+
+    until path_dirs.empty?
+      if path_site_file == ''
+        path_site_file += path_dirs.shift
+      else
+        path_site_file += '/' + path_dirs.shift
+      end
+
+      raise ArgumentError, 'directory name cannot be empty' if path_site_file == ''
+
+      site_file = SiteFile.where(site_id: self.id, path: path_site_file).first
+
+      if site_file.nil?
+        SiteFile.create(
+          site_id: self.id,
+          path: path_site_file,
+          is_directory: true,
+          created_at: Time.now,
+          updated_at: Time.now
+        )
+      end
+    end
+
     FileUtils.mkdir_p relative_path
     true
   end
@@ -650,10 +741,12 @@ class Site < Sequel::Model
 
     Zip::Archive.open(tmpfile.path, Zip::CREATE) do |ar|
       ar.add_dir(zip_name)
+    end
 
-      Dir.glob("#{base_files_path}/**/*").each do |path|
+    Dir.glob("#{base_files_path}/**/*").each do |path|
-        relative_path = path.gsub(base_files_path+'/', '')
+      relative_path = path.gsub(base_files_path+'/', '')
 
+      Zip::Archive.open(tmpfile.path, Zip::CREATE) do |ar|
         if File.directory?(path)
           ar.add_dir(zip_name+'/'+relative_path)
         else
@@ -741,6 +834,14 @@ class Site < Sequel::Model
 #    super
 #  end
 
+  def domain=(domain)
+    super SimpleIDN.to_ascii(domain)
+  end
+
+  def domain
+    SimpleIDN.to_unicode values[:domain]
+  end
+
   def validate
     super
 
@@ -764,6 +865,14 @@ class Site < Sequel::Model
       errors.add :email, 'An email address is required.'
     end
 
+    if parent? && values[:email] =~ /@neocities.org/
+      errors.add :email, 'Cannot use this email address.'
+    end
+
+    if parent? && new? && self.class.disposable_email?(values[:email])
+      errors.add :email, 'Cannot use a disposable email address.'
+    end
+
     # Check for existing email if new or changing email.
     if new? || @original_email
       email_check = self.class.select(:id).filter(email: values[:email])
@@ -793,15 +902,10 @@ class Site < Sequel::Model
     end
 
     if !values[:domain].nil? && !values[:domain].empty?
-
       if values[:domain] =~ /neocities\.org/ || values[:domain] =~ /neocitiesops\.net/
         errors.add :domain, "Domain is already being used.. by Neocities."
       end
 
-      if !(values[:domain] =~ /^[a-zA-Z0-9.-]+\.[a-zA-Z0-9]+$/i) || values[:domain].length > 90
-        errors.add :domain, "Domain provided is not valid. Must take the form of domain.com"
-      end
-
       site = Site[domain: values[:domain]]
       if !site.nil? && site.id != self.id
         errors.add :domain, "Domain provided is already being used by another site, please choose another."
@@ -871,6 +975,12 @@ class Site < Sequel::Model
     File.join TEMPLATE_ROOT, name
   end
 
+  def current_base_files_path(name=username)
+    raise 'username missing' if name.nil? || name.empty?
+    return File.join BANNED_SITES_ROOT, name if is_banned
+    base_files_path name
+  end
+
   def base_files_path(name=username)
     raise 'username missing' if name.nil? || name.empty?
     File.join SITE_FILES_ROOT, name
@@ -881,7 +991,7 @@ class Site < Sequel::Model
     path ||= ''
     clean = []
 
-    parts = path.split '/'
+    parts = path.to_s.split '/'
 
     parts.each do |part|
       next if part.empty? || part == '.'
@@ -891,6 +1001,10 @@ class Site < Sequel::Model
     clean.join '/'
   end
 
+  def current_files_path(path='')
+    File.join current_base_files_path, scrubbed_path(path)
+  end
+
   def files_path(path='')
     File.join base_files_path, scrubbed_path(path)
   end
@@ -929,8 +1043,16 @@ class Site < Sequel::Model
   end
 
   def actual_space_used
-    space = Dir.glob(File.join(files_path, '*')).collect {|p| File.size(p)}.inject {|sum,x| sum += x}
+    space = 0
-    space.nil? ? 0 : space
+
+    files = Dir.glob File.join(files_path, '**', '*')
+
+    files.each do |file|
+      next if File.directory? file
+      space += File.size file
+    end
+
+    space
   end
 
   def total_space_used
@@ -945,14 +1067,18 @@ class Site < Sequel::Model
   end
 
   def maximum_space
-    PLAN_FEATURES[(parent? ? self : parent).plan_type.to_sym][:space]
+    plan_space = PLAN_FEATURES[(parent? ? self : parent).plan_type.to_sym][:space]
+
+    return custom_max_space if custom_max_space > plan_space
+
+    plan_space
   end
 
   def space_percentage_used
     ((total_space_used.to_f / maximum_space) * 100).round(1)
   end
 
-  # Note: Change Stat#prune! if you change this business logic.
+  # Note: Change Stat#prune! and the nginx map compiler if you change this business logic.
   def supporter?
     owner.plan_type != 'free'
   end
@@ -965,6 +1091,10 @@ class Site < Sequel::Model
     PLAN_FEATURES[plan_type.to_sym][:name]
   end
 
+  def stripe_paying_supporter?
+    stripe_customer_id && !plan_ended && values[:plan_type].match(/free|special/).nil?
+  end
+
   def unconverted_legacy_supporter?
     stripe_customer_id && !plan_ended && values[:plan_type].nil? && stripe_subscription_id.nil?
   end
@@ -974,8 +1104,9 @@ class Site < Sequel::Model
     !values[:plan_type].match(/plan_/).nil?
   end
 
-  # Note: Change Stat#prune! if you change this business logic.
+  # Note: Change Stat#prune! and the nginx map compiler if you change this business logic.
   def plan_type
+    return 'supporter' if owner.values[:paypal_active] == true
     return 'free' if owner.values[:plan_type].nil?
     return 'supporter' if owner.values[:plan_type].match /^plan_/
     return 'supporter' if owner.values[:plan_type] == 'special'
@@ -1037,13 +1168,70 @@ class Site < Sequel::Model
     end
   end
 
+  def self.compute_scores
+    select(:id, :username, :created_at, :updated_at, :views, :featured_at, :changed_count, :api_calls).exclude(is_banned: true).exclude(is_crashing: true).exclude(is_nsfw: true).exclude(updated_at: nil).where(site_changed: true).all.each do |s|
+      s.score = s.compute_score
+      s.save_changes validate: false
+    end
+  end
+
+  SCORE_GRAVITY = 1.8
+
+  def compute_score
+    points = 0
+    points += follows_dataset.count * 30
+    points += profile_comments_dataset.count * 1
+    points += views / 1000
+    points += 20 if !featured_at.nil?
+
+    # penalties
+    points = 0 if changed_count < 2
+    points = 0 if api_calls && api_calls > 1000
+
+    (points / ((Time.now - updated_at) / 7.days)**SCORE_GRAVITY).round(4)
+  end
+
+=begin
+  def compute_score
+    score = 0
+    score += (Time.now - created_at) / 1.day
+    score -= ((Time.now - updated_at) / 1.day) * 2
+    score += 500 if (updated_at > 1.week.ago)
+    score -= 1000 if
+    follow_count = follows_dataset.count
+    score -= 1000 if follow_count == 0
+    score += follow_count * 100
+    score += profile_comments_dataset.count * 5
+    score += profile_commentings_dataset.count
+    score.to_i
+  end
+=end
+
+  def self.browse_dataset
+    dataset.where is_deleted: false, is_banned: false, is_crashing: false, site_changed: true
+  end
+
   def suggestions(limit=SUGGESTIONS_LIMIT, offset=0)
     suggestions_dataset = Site.exclude(id: id).exclude(is_banned: true).exclude(is_nsfw: true).order(:views.desc, :updated_at.desc)
     suggestions = suggestions_dataset.where(tags: tags).limit(limit, offset).all
 
     return suggestions if suggestions.length == limit
 
-    suggestions += suggestions_dataset.where("views >= #{SUGGESTIONS_VIEWS_MIN}").limit(limit-suggestions.length).order(Sequel.lit('RANDOM()')).all
+    # Old.
+    #suggestions += suggestions_dataset.where("views >= #{SUGGESTIONS_VIEWS_MIN}").limit(limit-suggestions.length).order(Sequel.lit('RANDOM()')).all
+
+    # New:
+
+    site_dataset = self.class.browse_dataset.association_left_join :follows
+    site_dataset.select_all! :sites
+    site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
+    site_dataset.group! :sites__id
+    site_dataset.order! :follow_count.desc, :updated_at.desc
+    site_dataset.where! "views >= #{SUGGESTIONS_VIEWS_MIN}"
+    site_dataset.limit! limit-suggestions.length
+    #site_dataset.order! Sequel.lit('RANDOM()')
+
+    suggestions += site_dataset.all
   end
 
   def screenshot_path(path, resolution)
@@ -1099,11 +1287,53 @@ class Site < Sequel::Model
     end
   end
 
+  def empty_index?
+    !site_files_dataset.where(path: /^\/?index.html$/).where(sha1_hash: EMPTY_FILE_HASH).first.nil?
+  end
+
+  def classify(path)
+    return nil unless classification_allowed? path
+    #$classifier.classify process_for_classification(path)
+  end
+
+  def classification_scores(path)
+    return nil unless classification_allowed? path
+    #$classifier.classification_scores process_for_classification(path)
+  end
+
+  def train(path, category='ham')
+    return nil unless classification_allowed? path
+    # $trainer.train(category, process_for_classification(path))
+    site_file = site_files_dataset.where(path: path).first
+    site_file.classifier = category
+    site_file.save_changes validate: false
+  end
+
+  def untrain(path, category='ham')
+    return nil unless classification_allowed? path
+    # $trainer.untrain(category, process_for_classification(path))
+    site_file = site_files_dataset.where(path: path).first
+    site_file.classifier = category
+    site_file.save_changes validate: false
+  end
+
+  def classification_allowed?(path)
+    site_file = site_files_dataset.where(path: path).first
+    return false if site_file.is_directory
+    return false if site_file.size > SiteFile::CLASSIFIER_LIMIT
+    return false if !path.match(/\.html$/)
+    true
+  end
+
+  def process_for_classification(path)
+    sanitized = Sanitize.fragment get_file(path)
+    sanitized.gsub(/(http|https):\/\//, '').gsub(/[^\w\s]/, '').downcase.split.uniq.select{|v| v.length < SiteFile::CLASSIFIER_WORD_LIMIT}.join(' ')
+  end
+
   # array of hashes: filename, tempfile, opts.
   def store_files(files, opts={})
     results = []
     new_size = 0
-    html_uploaded = false
 
     if too_many_files?(files.length)
       results << false
@@ -1111,35 +1341,53 @@ class Site < Sequel::Model
     end
 
     files.each do |file|
-      html_uploaded = true if file[:filename].match HTML_REGEX
-
       existing_size = 0
+
       site_file = site_files_dataset.where(path: scrubbed_path(file[:filename])).first
+
       if site_file
         existing_size = site_file.size
       end
 
       res = store_file(file[:filename], file[:tempfile], file[:opts] || opts)
+
       if res == true
         new_size -= existing_size
         new_size += file[:tempfile].size
       end
+
       results << res
     end
 
-    if results.include? true && opts[:new_install] != true
+    if results.include? true
-      time = Time.now
+
-      sql = DB["update sites set site_changed=?, site_updated_at=?, updated_at=?, changed_count=changed_count+1, space_used=space_used#{new_size < 0 ? new_size.to_s : '+'+new_size.to_s} where id=?",
+      DB["update sites set space_used=space_used#{new_size < 0 ? new_size.to_s : '+'+new_size.to_s} where id=?", self.id].first
-        true,
+
-        time,
+      if opts[:new_install] != true
-        time,
+        if files.select {|f| f[:filename] =~ /^\/?index.html$/}.length > 0 || site_changed == true
-        self.id
+          index_changed = true
-      ]
+        else
-      sql.first
+          index_changed = false
+        end
+
+        index_changed = false if empty_index?
+
+        time = Time.now
+
+        sql = DB["update sites set site_changed=?, site_updated_at=?, updated_at=?, changed_count=changed_count+1 where id=?",
+          index_changed,
+          time,
+          time,
+          self.id
+        ]
+        sql.first
+
+        ArchiveWorker.perform_in Archive::ARCHIVE_WAIT_TIME, self.id
+      end
+
       reload
 
       #SiteChange.record self, relative_path unless opts[:new_install]
-      ArchiveWorker.perform_async self.id
     end
 
     results
@@ -1147,36 +1395,9 @@ class Site < Sequel::Model
 
   def delete_file(path)
     return false if files_path(path) == files_path
-    begin
+    path = scrubbed_path path
-      FileUtils.rm files_path(path)
+    site_file = site_files_dataset.where(path: path).first
-    rescue Errno::EISDIR
+    site_file.destroy if site_file
-      site_files.each do |site_file|
-        if site_file.path.match /^#{path}\//
-          site_file.destroy
-        end
-      end
-      FileUtils.remove_dir files_path(path), true
-    rescue Errno::ENOENT
-    end
-
-    purge_cache path
-
-    ext = File.extname(path).gsub(/^./, '')
-
-    screenshots_delete(path) if ext.match HTML_REGEX
-    thumbnails_delete(path) if ext.match IMAGE_REGEX
-
-    path = path[1..path.length] if path[0] == '/'
-
-    DB.transaction do
-      site_file = site_files_dataset.where(path: path).first
-      if site_file
-        DB['update sites set space_used=space_used-? where id=?', site_file.size, self.id].first
-        site_file.delete
-      end
-      SiteChangeFile.filter(site_id: self.id, filename: path).delete
-    end
-
     true
   end
 
@@ -1195,6 +1416,15 @@ class Site < Sequel::Model
       return false
     end
 
+    if pathname.extname.match HTML_REGEX
+      # SPAM and phishing checking code goes here
+    end
+
+    relative_path_dir = Pathname(relative_path).dirname
+    create_directory relative_path_dir unless relative_path_dir == '.'
+
+    uploaded_size = uploaded.size
+
     if relative_path == 'index.html'
       begin
         new_title = Nokogiri::HTML(File.read(uploaded.path)).css('title').first.text
@@ -1207,18 +1437,6 @@ class Site < Sequel::Model
       end
     end
 
-    if pathname.extname.match HTML_REGEX
-      # SPAM and phishing checking code goes here
-    end
-
-    dirname = pathname.dirname.to_s
-
-    if !File.exists? dirname
-      FileUtils.mkdir_p dirname
-    end
-
-    uploaded_size = uploaded.size
-
     FileUtils.cp uploaded.path, path
     File.chmod 0640, path
 
@@ -1236,11 +1454,11 @@ class Site < Sequel::Model
     purge_cache path
 
     if pathname.extname.match HTML_REGEX
-      ScreenshotWorker.perform_async values[:username], relative_path
+      ScreenshotWorker.perform_in 1.minute, values[:username], relative_path
     elsif pathname.extname.match IMAGE_REGEX
       ThumbnailWorker.perform_async values[:username], relative_path
     end
+
     true
   end
-
 end

models/site_file.rb

@@ -1,5 +1,52 @@
+require 'sanitize'
+
 class SiteFile < Sequel::Model
+  CLASSIFIER_LIMIT = 1_000_000.freeze
+  CLASSIFIER_WORD_LIMIT = 25.freeze
   unrestrict_primary_key
   plugin :update_primary_key
   many_to_one :site
+
+  def before_destroy
+    if is_directory
+      site.site_files_dataset.where(path: /^#{Regexp.quote path}\//, is_directory: true).all.each do |site_file|
+        begin
+          site_file.destroy
+        rescue Sequel::NoExistingObject
+        end
+      end
+
+      site.site_files_dataset.where(path: /^#{Regexp.quote path}\//, is_directory: false).all.each do |site_file|
+        site_file.destroy
+      end
+
+      begin
+        FileUtils.remove_dir site.files_path(path)
+      rescue Errno::ENOENT
+      end
+
+    else
+
+      begin
+        FileUtils.rm site.files_path(path)
+      rescue Errno::ENOENT
+      end
+
+      ext = File.extname(path).gsub(/^./, '')
+      site.screenshots_delete(path) if ext.match Site::HTML_REGEX
+      site.thumbnails_delete(path) if ext.match Site::IMAGE_REGEX
+    end
+
+    super
+  end
+
+  def after_destroy
+    super
+    unless is_directory
+      DB['update sites set space_used=space_used-? where id=?', size, site_id].first
+    end
+
+    site.delete_cache site.files_path(path)
+    SiteChangeFile.filter(site_id: site_id, filename: path).delete
+  end
 end

models/stat.rb

@@ -15,8 +15,11 @@ class Stat < Sequel::Model
     end
 
     def parse_logfiles(path)
+      total_site_stats = {}
+
       Dir["#{path}/*.log"].each do |log_path|
         site_logs = {}
+
         logfile = File.open log_path, 'r'
 
         while hit = logfile.gets
@@ -26,9 +29,13 @@ class Stat < Sequel::Model
 
           time, username, size, path, ip, referrer = hit_array
 
+          log_time = Time.parse time
+
           next if !referrer.nil? && referrer.match(/bot/i)
 
-          site_logs[username] = {
+          site_logs[log_time] = {} unless site_logs[log_time]
+
+          site_logs[log_time][username] = {
             hits: 0,
             views: 0,
             bandwidth: 0,
@@ -36,78 +43,111 @@ class Stat < Sequel::Model
             ips: [],
             referrers: {},
             paths: {}
-          } unless site_logs[username]
+          } unless site_logs[log_time][username]
 
-          site_logs[username][:hits] += 1
+          total_site_stats[log_time] = {
-          site_logs[username][:bandwidth] += size.to_i
+            hits: 0,
+            views: 0,
+            bandwidth: 0
+          } unless total_site_stats[log_time]
 
-          unless site_logs[username][:view_ips].include?(ip)
+          site_logs[log_time][username][:hits] += 1
-            site_logs[username][:views] += 1
+          site_logs[log_time][username][:bandwidth] += size.to_i
-            site_logs[username][:view_ips] << ip
+
+          total_site_stats[log_time][:hits] += 1
+          total_site_stats[log_time][:bandwidth] += size.to_i
+
+          unless site_logs[log_time][username][:view_ips].include?(ip)
+            site_logs[log_time][username][:views] += 1
+
+            total_site_stats[log_time][:views] += 1
+
+            site_logs[log_time][username][:view_ips] << ip
 
             if referrer != '-' && !referrer.nil?
-              site_logs[username][:referrers][referrer] ||= 0
+              site_logs[log_time][username][:referrers][referrer] ||= 0
-              site_logs[username][:referrers][referrer] += 1
+              site_logs[log_time][username][:referrers][referrer] += 1
             end
           end
 
-          site_logs[username][:paths][path] ||= 0
+          site_logs[log_time][username][:paths][path] ||= 0
-          site_logs[username][:paths][path] += 1
+          site_logs[log_time][username][:paths][path] += 1
         end
 
         logfile.close
 
-        current_time = Time.now.utc
-        current_day_string = current_time.to_date.to_s
-
-        Site.select(:id, :username).where(username: site_logs.keys).all.each do |site|
-          site_logs[site.username][:id] = site.id
-        end
-
         DB.transaction do
-          site_logs.each do |username, site_log|
+          site_logs.each do |log_time, usernames|
-            DB['update sites set hits=hits+?, views=views+? where username=?',
+            Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
-               site_log[:hits],
+              site_logs[log_time][site.username][:id] = site.id
-               site_log[:views],
+            end
-               username
+
+            usernames.each do |username, site_log|
+              DB['update sites set hits=hits+?, views=views+? where username=?',
+                site_log[:hits],
+                site_log[:views],
+                username
               ].first
 
-            opts = {site_id: site_log[:id], created_at: current_day_string}
+              opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s}
 
-            stat = Stat.select(:id).where(opts).first
+              stat = nil
-            DB[:stats].lock('EXCLUSIVE') { stat = Stat.create opts } if stat.nil?
 
-            DB[
+              DB[:stats].lock('EXCLUSIVE') {
-              'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                stat = Stat.select(:id).where(opts).first
-              site_log[:hits],
+                stat = Stat.create opts if stat.nil?
-              site_log[:views],
+              }
-              site_log[:bandwidth],
+
-              stat.id
+              DB[
-            ].first
+                'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                site_log[:hits],
+                site_log[:views],
+                site_log[:bandwidth],
+                stat.id
+              ].first
 
 =begin
-            site_log[:referrers].each do |referrer, views|
+              site_log[:referrers].each do |referrer, views|
-              stat_referrer = StatReferrer.create_or_get site_log[:id], referrer
+                stat_referrer = StatReferrer.create_or_get site_log[:id], referrer
-              DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first
+                DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first
-            end
+              end
 
-            site_log[:view_ips].each do |ip|
+              site_log[:view_ips].each do |ip|
-              site_location = StatLocation.create_or_get site_log[:id], ip
+                site_location = StatLocation.create_or_get site_log[:id], ip
-              next if site_location.nil?
+                next if site_location.nil?
-              DB['update stat_locations set views=views+1 where id=?', site_location.id].first
+                DB['update stat_locations set views=views+1 where id=?', site_location.id].first
-            end
+              end
 
-            site_log[:paths].each do |path, views|
+              site_log[:paths].each do |path, views|
-              site_path = StatPath.create_or_get site_log[:id], path
+                site_path = StatPath.create_or_get site_log[:id], path
-              next if site_path.nil?
+                next if site_path.nil?
-              DB['update stat_paths set views=views+? where id=?', views, site_path.id].first
+                DB['update stat_paths set views=views+? where id=?', views, site_path.id].first
-            end
+              end
 =end
+            end
           end
         end
 
         FileUtils.rm log_path
       end
+
+      total_site_stats.each do |time, stats|
+        opts = {created_at: time.to_date.to_s}
+
+        DB[:stats].lock('EXCLUSIVE') {
+          stat = DailySiteStat.select(:id).where(opts).first
+          stat = DailySiteStat.create opts if stat.nil?
+        }
+
+        DB[
+          'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?',
+          stats[:hits],
+          stats[:views],
+          stats[:bandwidth],
+          time.to_date
+        ].first
+      end
+
     end
   end
 end

models/tag.rb

@@ -23,6 +23,14 @@ class Tag < Sequel::Model
   end
 
   def self.popular_names(limit=10)
-    DB["select tags.name,count(*) as c from sites_tags inner join tags on tags.id=sites_tags.tag_id where tags.name != '' and tags.is_nsfw='f' group by tags.name having count(*) > 1 order by c desc LIMIT ?", limit].all
+    cache = $redis_cache['tag_popular_names']
+    if cache.nil?
+      res = DB["select tags.name,count(*) as c from sites_tags inner join tags on tags.id=sites_tags.tag_id where tags.name != '' and tags.is_nsfw='f' group by tags.name having count(*) > 1 order by c desc LIMIT ?", limit].all
+      $redis_cache.set :tag_popular_names, res.to_msgpack
+      $redis_cache.expire :tag_popular_names, 86400 # 24 hours
+    else
+      res = MessagePack.unpack cache, symbolize_keys: true
+    end
+    res
   end
 end

public/css/highlight/styles/sunburst.css

@@ -0,0 +1,102 @@
+/*
+
+Sunburst-like style (c) Vasily Polovnyov <vast@whiteants.net>
+
+*/
+
+.hljs {
+  display: block;
+  overflow-x: auto;
+  padding: 0.5em;
+  background: #000;
+  color: #f8f8f8;
+}
+
+.hljs-comment,
+.hljs-quote {
+  color: #aeaeae;
+  font-style: italic;
+}
+
+.hljs-keyword,
+.hljs-selector-tag,
+.hljs-type {
+  color: #e28964;
+}
+
+.hljs-string {
+  color: #65b042;
+}
+
+.hljs-subst {
+  color: #daefa3;
+}
+
+.hljs-regexp,
+.hljs-link {
+  color: #e9c062;
+}
+
+.hljs-title,
+.hljs-section,
+.hljs-tag,
+.hljs-name {
+  color: #89bdff;
+}
+
+.hljs-class .hljs-title,
+.hljs-doctag {
+  text-decoration: underline;
+}
+
+.hljs-symbol,
+.hljs-bullet,
+.hljs-number {
+  color: #3387cc;
+}
+
+.hljs-params,
+.hljs-variable,
+.hljs-template-variable {
+  color: #3e87e3;
+}
+
+.hljs-attribute {
+  color: #cda869;
+}
+
+.hljs-meta {
+  color: #8996a8;
+}
+
+.hljs-formula {
+  background-color: #0e2231;
+  color: #f8f8f8;
+  font-style: italic;
+}
+
+.hljs-addition {
+  background-color: #253b22;
+  color: #f8f8f8;
+}
+
+.hljs-deletion {
+  background-color: #420e09;
+  color: #f8f8f8;
+}
+
+.hljs-selector-class {
+  color: #9b703f;
+}
+
+.hljs-selector-id {
+  color: #8b98ab;
+}
+
+.hljs-emphasis {
+  font-style: italic;
+}
+
+.hljs-strong {
+  font-weight: bold;
+}

public/css/highlight/styles/tomorrow-night.css

@@ -0,0 +1,75 @@
+/* Tomorrow Night Theme */
+/* http://jmblog.github.com/color-themes-for-google-code-highlightjs */
+/* Original theme - https://github.com/chriskempson/tomorrow-theme */
+/* http://jmblog.github.com/color-themes-for-google-code-highlightjs */
+
+/* Tomorrow Comment */
+.hljs-comment,
+.hljs-quote {
+  color: #969896;
+}
+
+/* Tomorrow Red */
+.hljs-variable,
+.hljs-template-variable,
+.hljs-tag,
+.hljs-name,
+.hljs-selector-id,
+.hljs-selector-class,
+.hljs-regexp,
+.hljs-deletion {
+  color: #cc6666;
+}
+
+/* Tomorrow Orange */
+.hljs-number,
+.hljs-built_in,
+.hljs-builtin-name,
+.hljs-literal,
+.hljs-type,
+.hljs-params,
+.hljs-meta,
+.hljs-link {
+  color: #de935f;
+}
+
+/* Tomorrow Yellow */
+.hljs-attribute {
+  color: #f0c674;
+}
+
+/* Tomorrow Green */
+.hljs-string,
+.hljs-symbol,
+.hljs-bullet,
+.hljs-addition {
+  color: #b5bd68;
+}
+
+/* Tomorrow Blue */
+.hljs-title,
+.hljs-section {
+  color: #81a2be;
+}
+
+/* Tomorrow Purple */
+.hljs-keyword,
+.hljs-selector-tag {
+  color: #b294bb;
+}
+
+.hljs {
+  display: block;
+  overflow-x: auto;
+  background: #1d1f21;
+  color: #c5c8c6;
+  padding: 0.5em;
+}
+
+.hljs-emphasis {
+  font-style: italic;
+}
+
+.hljs-strong {
+  font-weight: bold;
+}

public/js/keyboard.js

@@ -1,961 +0,0 @@
-/**
- * Title: KeyboardJS
- * Version: v0.4.1
- * Description: KeyboardJS is a flexible and easy to use keyboard binding
- * library.
- * Author: Robert Hurst.
- *
- * Copyright 2011, Robert William Hurst
- * Licenced under the BSD License.
- * See https://raw.github.com/RobertWHurst/KeyboardJS/master/license.txt
- */
-(function(context, factory) {
-
-  //INDEXOF POLLYFILL
-  [].indexOf||(Array.prototype.indexOf=function(a,b,c){for(c=this.length,b=(c+~~b)%c;b<c&&(!(b in this)||this[b]!==a);b++);return b^c?b:-1;});
-
-  //AMD
-  if(typeof define === 'function' && define.amd) { define(constructAMD); }
-
-  //CommonJS
-  else if(typeof module !== 'undefined') {constructCommonJS()}
-
-  //GLOBAL
-  else { constructGlobal(); }
-
-  /**
-   * Construct AMD version of the library
-   */
-  function constructAMD() {
-
-    //create a library instance
-    return init(context);
-
-    //spawns a library instance
-    function init(context) {
-      var library;
-      library = factory(context, 'amd');
-      library.fork = init;
-      return library;
-    }
-  }
-
-  /**
-   * Construct CommonJS version of the library
-   */
-  function constructCommonJS() {
-
-    //create a library instance
-    module.exports = init(context);
-
-    return;
-
-    //spawns a library instance
-    function init(context) {
-      var library;
-      library = factory(context, 'CommonJS');
-      library.fork = init;
-      return library;
-
-    }
-
-  }
-
-  /**
-   * Construct a Global version of the library
-   */
-  function constructGlobal() {
-    var library;
-
-    //create a library instance
-    library = init(context);
-
-    //spawns a library instance
-    function init(context) {
-      var library, namespaces = [], previousValues = {};
-
-      library = factory(context, 'global');
-      library.fork = init;
-      library.noConflict = noConflict;
-      library.noConflict('KeyboardJS', 'k');
-      return library;
-
-      //sets library namespaces
-      function noConflict(    ) {
-        var args, nI, newNamespaces;
-
-        newNamespaces = Array.prototype.slice.apply(arguments);
-
-        for(nI = 0; nI < namespaces.length; nI += 1) {
-          if(typeof previousValues[namespaces[nI]] === 'undefined') {
-            delete context[namespaces[nI]];
-          } else {
-            context[namespaces[nI]] = previousValues[namespaces[nI]];
-          }
-        }
-
-        previousValues = {};
-
-        for(nI = 0; nI < newNamespaces.length; nI += 1) {
-          if(typeof newNamespaces[nI] !== 'string') {
-            throw new Error('Cannot replace namespaces. All new namespaces must be strings.');
-          }
-          previousValues[newNamespaces[nI]] = context[newNamespaces[nI]];
-          context[newNamespaces[nI]] = library;
-        }
-
-        namespaces = newNamespaces;
-
-        return namespaces;
-      }
-    }
-  }
-
-})(this, function(targetWindow, env) {
-  var KeyboardJS = {}, locales = {}, locale, map, macros, activeKeys = [], bindings = [], activeBindings = [],
-  activeMacros = [], aI, usLocale;
-  targetWindow = targetWindow || window;
-
-  ///////////////////////
-  // DEFAULT US LOCALE //
-  ///////////////////////
-
-  //define US locale
-  //If you create a new locale please submit it as a pull request or post
-  // it in the issue tracker at
-  // http://github.com/RobertWhurst/KeyboardJS/issues/
-  usLocale = {
-    "map": {
-
-      //general
-      "3": ["cancel"],
-      "8": ["backspace"],
-      "9": ["tab"],
-      "12": ["clear"],
-      "13": ["enter"],
-      "16": ["shift"],
-      "17": ["ctrl"],
-      "18": ["alt", "menu"],
-      "19": ["pause", "break"],
-      "20": ["capslock"],
-      "27": ["escape", "esc"],
-      "32": ["space", "spacebar"],
-      "33": ["pageup"],
-      "34": ["pagedown"],
-      "35": ["end"],
-      "36": ["home"],
-      "37": ["left"],
-      "38": ["up"],
-      "39": ["right"],
-      "40": ["down"],
-      "41": ["select"],
-      "42": ["printscreen"],
-      "43": ["execute"],
-      "44": ["snapshot"],
-      "45": ["insert", "ins"],
-      "46": ["delete", "del"],
-      "47": ["help"],
-      "91": ["command", "windows", "win", "super", "leftcommand", "leftwindows", "leftwin", "leftsuper"],
-      "92": ["command", "windows", "win", "super", "rightcommand", "rightwindows", "rightwin", "rightsuper"],
-      "145": ["scrolllock", "scroll"],
-      "186": ["semicolon", ";"],
-      "187": ["equal", "equalsign", "="],
-      "188": ["comma", ","],
-      "189": ["dash", "-"],
-      "190": ["period", "."],
-      "191": ["slash", "forwardslash", "/"],
-      "192": ["graveaccent", "`"],
-      "219": ["openbracket", "["],
-      "220": ["backslash", "\\"],
-      "221": ["closebracket", "]"],
-      "222": ["apostrophe", "'"],
-
-      //0-9
-      "48": ["zero", "0"],
-      "49": ["one", "1"],
-      "50": ["two", "2"],
-      "51": ["three", "3"],
-      "52": ["four", "4"],
-      "53": ["five", "5"],
-      "54": ["six", "6"],
-      "55": ["seven", "7"],
-      "56": ["eight", "8"],
-      "57": ["nine", "9"],
-
-      //numpad
-      "96": ["numzero", "num0"],
-      "97": ["numone", "num1"],
-      "98": ["numtwo", "num2"],
-      "99": ["numthree", "num3"],
-      "100": ["numfour", "num4"],
-      "101": ["numfive", "num5"],
-      "102": ["numsix", "num6"],
-      "103": ["numseven", "num7"],
-      "104": ["numeight", "num8"],
-      "105": ["numnine", "num9"],
-      "106": ["nummultiply", "num*"],
-      "107": ["numadd", "num+"],
-      "108": ["numenter"],
-      "109": ["numsubtract", "num-"],
-      "110": ["numdecimal", "num."],
-      "111": ["numdivide", "num/"],
-      "144": ["numlock", "num"],
-
-      //function keys
-      "112": ["f1"],
-      "113": ["f2"],
-      "114": ["f3"],
-      "115": ["f4"],
-      "116": ["f5"],
-      "117": ["f6"],
-      "118": ["f7"],
-      "119": ["f8"],
-      "120": ["f9"],
-      "121": ["f10"],
-      "122": ["f11"],
-      "123": ["f12"]
-    },
-    "macros": [
-
-      //secondary key symbols
-      ['shift + `', ["tilde", "~"]],
-      ['shift + 1', ["exclamation", "exclamationpoint", "!"]],
-      ['shift + 2', ["at", "@"]],
-      ['shift + 3', ["number", "#"]],
-      ['shift + 4', ["dollar", "dollars", "dollarsign", "$"]],
-      ['shift + 5', ["percent", "%"]],
-      ['shift + 6', ["caret", "^"]],
-      ['shift + 7', ["ampersand", "and", "&"]],
-      ['shift + 8', ["asterisk", "*"]],
-      ['shift + 9', ["openparen", "("]],
-      ['shift + 0', ["closeparen", ")"]],
-      ['shift + -', ["underscore", "_"]],
-      ['shift + =', ["plus", "+"]],
-      ['shift + (', ["opencurlybrace", "opencurlybracket", "{"]],
-      ['shift + )', ["closecurlybrace", "closecurlybracket", "}"]],
-      ['shift + \\', ["verticalbar", "|"]],
-      ['shift + ;', ["colon", ":"]],
-      ['shift + \'', ["quotationmark", "\""]],
-      ['shift + !,', ["openanglebracket", "<"]],
-      ['shift + .', ["closeanglebracket", ">"]],
-      ['shift + /', ["questionmark", "?"]]
-    ]
-  };
-  //a-z and A-Z
-  for (aI = 65; aI <= 90; aI += 1) {
-    usLocale.map[aI] = String.fromCharCode(aI + 32);
-    usLocale.macros.push(['shift + ' + String.fromCharCode(aI + 32) + ', capslock + ' + String.fromCharCode(aI + 32), [String.fromCharCode(aI)]]);
-  }
-  registerLocale('us', usLocale);
-  getSetLocale('us');
-
-
-  //////////
-  // INIT //
-  //////////
-
-  //enable the library
-  enable();
-
-
-  /////////
-  // API //
-  /////////
-
-  //assemble the library and return it
-  KeyboardJS.enable = enable;
-  KeyboardJS.disable = disable;
-  KeyboardJS.activeKeys = getActiveKeys;
-  KeyboardJS.releaseKey = removeActiveKey;
-  KeyboardJS.pressKey = addActiveKey;
-  KeyboardJS.on = createBinding;
-  KeyboardJS.clear = removeBindingByKeyCombo;
-  KeyboardJS.clear.key = removeBindingByKeyName;
-  KeyboardJS.locale = getSetLocale;
-  KeyboardJS.locale.register = registerLocale;
-  KeyboardJS.macro = createMacro;
-  KeyboardJS.macro.remove = removeMacro;
-  KeyboardJS.key = {};
-  KeyboardJS.key.name = getKeyName;
-  KeyboardJS.key.code = getKeyCode;
-  KeyboardJS.combo = {};
-  KeyboardJS.combo.active = isSatisfiedCombo;
-  KeyboardJS.combo.parse = parseKeyCombo;
-  KeyboardJS.combo.stringify = stringifyKeyCombo;
-  return KeyboardJS;
-
-
-  //////////////////////
-  // INSTANCE METHODS //
-  //////////////////////
-
-  /**
-   * Enables KeyboardJS
-   */
-  function enable() {
-    if(targetWindow.addEventListener) {
-      targetWindow.document.addEventListener('keydown', keydown, false);
-      targetWindow.document.addEventListener('keyup', keyup, false);
-      targetWindow.addEventListener('blur', reset, false);
-      targetWindow.addEventListener('webkitfullscreenchange', reset, false);
-      targetWindow.addEventListener('mozfullscreenchange', reset, false);
-    } else if(targetWindow.attachEvent) {
-      targetWindow.document.attachEvent('onkeydown', keydown);
-      targetWindow.document.attachEvent('onkeyup', keyup);
-      targetWindow.attachEvent('onblur', reset);
-    }
-  }
-
-  /**
-   * Exits all active bindings and disables KeyboardJS
-   */
-  function disable() {
-    reset();
-    if(targetWindow.removeEventListener) {
-      targetWindow.document.removeEventListener('keydown', keydown, false);
-      targetWindow.document.removeEventListener('keyup', keyup, false);
-      targetWindow.removeEventListener('blur', reset, false);
-      targetWindow.removeEventListener('webkitfullscreenchange', reset, false);
-      targetWindow.removeEventListener('mozfullscreenchange', reset, false);
-    } else if(targetWindow.detachEvent) {
-      targetWindow.document.detachEvent('onkeydown', keydown);
-      targetWindow.document.detachEvent('onkeyup', keyup);
-      targetWindow.detachEvent('onblur', reset);
-    }
-  }
-
-
-  ////////////////////
-  // EVENT HANDLERS //
-  ////////////////////
-
-  /**
-   * Exits all active bindings. Optionally passes an event to all binding
-   *  handlers.
-   * @param  {KeyboardEvent}  event [Optional]
-   */
-  function reset(event) {
-    activeKeys = [];
-    pruneMacros();
-    pruneBindings(event);
-  }
-
-  /**
-   * Key down event handler.
-   * @param  {KeyboardEvent}  event
-   */
-  function keydown(event) {
-    var keyNames, keyName, kI;
-    keyNames = getKeyName(event.keyCode);
-    if(keyNames.length < 1) { return; }
-    event.isRepeat = false;
-    for(kI = 0; kI < keyNames.length; kI += 1) {
-        keyName = keyNames[kI];
-        if (getActiveKeys().indexOf(keyName) != -1)
-            event.isRepeat = true;
-      addActiveKey(keyName);
-    }
-    executeMacros();
-    executeBindings(event);
-  }
-
-  /**
-   * Key up event handler.
-   * @param  {KeyboardEvent} event
-   */
-  function keyup(event) {
-    var keyNames, kI;
-    keyNames = getKeyName(event.keyCode);
-    if(keyNames.length < 1) { return; }
-    for(kI = 0; kI < keyNames.length; kI += 1) {
-      removeActiveKey(keyNames[kI]);
-    }
-    pruneMacros();
-    pruneBindings(event);
-  }
-
-  /**
-   * Accepts a key code and returns the key names defined by the current
-   *  locale.
-   * @param  {Number} keyCode
-   * @return {Array}  keyNames  An array of key names defined for the key
-   *  code as defined by the current locale.
-   */
-  function getKeyName(keyCode) {
-    return map[keyCode] || [];
-  }
-
-  /**
-   * Accepts a key name and returns the key code defined by the current
-   *  locale.
-   * @param  {Number} keyName
-   * @return {Number|false}
-   */
-  function getKeyCode(keyName) {
-    var keyCode;
-    for(keyCode in map) {
-      if(!map.hasOwnProperty(keyCode)) { continue; }
-      if(map[keyCode].indexOf(keyName) > -1) { return keyCode; }
-    }
-    return false;
-  }
-
-
-  ////////////
-  // MACROS //
-  ////////////
-
-  /**
-   * Accepts a key combo and an array of key names to inject once the key
-   *  combo is satisfied.
-   * @param  {String} combo
-   * @param  {Array}  injectedKeys
-   */
-  function createMacro(combo, injectedKeys) {
-    if(typeof combo !== 'string' && (typeof combo !== 'object' || typeof combo.push !== 'function')) {
-      throw new Error("Cannot create macro. The combo must be a string or array.");
-    }
-    if(typeof injectedKeys !== 'object' || typeof injectedKeys.push !== 'function') {
-      throw new Error("Cannot create macro. The injectedKeys must be an array.");
-    }
-    macros.push([combo, injectedKeys]);
-  }
-
-  /**
-   * Accepts a key combo and clears any and all macros bound to that key
-   * combo.
-   * @param  {String} combo
-   */
-  function removeMacro(combo) {
-    var macro;
-    if(typeof combo !== 'string' && (typeof combo !== 'object' || typeof combo.push !== 'function')) { throw new Error("Cannot remove macro. The combo must be a string or array."); }
-    for(mI = 0; mI < macros.length; mI += 1) {
-      macro = macros[mI];
-      if(compareCombos(combo, macro[0])) {
-        removeActiveKey(macro[1]);
-        macros.splice(mI, 1);
-        break;
-      }
-    }
-  }
-
-  /**
-   * Executes macros against the active keys. Each macro's key combo is
-   *  checked and if found to be satisfied, the macro's key names are injected
-   *  into active keys.
-   */
-  function executeMacros() {
-    var mI, combo, kI;
-    for(mI = 0; mI < macros.length; mI += 1) {
-      combo = parseKeyCombo(macros[mI][0]);
-      if(activeMacros.indexOf(macros[mI]) === -1 && isSatisfiedCombo(combo)) {
-        activeMacros.push(macros[mI]);
-        for(kI = 0; kI < macros[mI][1].length; kI += 1) {
-          addActiveKey(macros[mI][1][kI]);
-        }
-      }
-    }
-  }
-
-  /**
-   * Prunes active macros. Checks each active macro's key combo and if found
-   *  to no longer to be satisfied, each of the macro's key names are removed
-   *  from active keys.
-   */
-  function pruneMacros() {
-    var mI, combo, kI;
-    for(mI = 0; mI < activeMacros.length; mI += 1) {
-      combo = parseKeyCombo(activeMacros[mI][0]);
-      if(isSatisfiedCombo(combo) === false) {
-        for(kI = 0; kI < activeMacros[mI][1].length; kI += 1) {
-          removeActiveKey(activeMacros[mI][1][kI]);
-        }
-        activeMacros.splice(mI, 1);
-        mI -= 1;
-      }
-    }
-  }
-
-
-  //////////////
-  // BINDINGS //
-  //////////////
-
-  /**
-   * Creates a binding object, and, if provided, binds a key down hander and
-   *  a key up handler. Returns a binding object that emits keyup and
-   *  keydown events.
-   * @param  {String}   keyCombo
-   * @param  {Function} keyDownCallback [Optional]
-   * @param  {Function} keyUpCallback [Optional]
-   * @return {Object}   binding
-   */
-  function createBinding(keyCombo, keyDownCallback, keyUpCallback) {
-    var api = {}, binding, subBindings = [], bindingApi = {}, kI,
-    subCombo;
-
-    //break the combo down into a combo array
-    if(typeof keyCombo === 'string') {
-      keyCombo = parseKeyCombo(keyCombo);
-    }
-
-    //bind each sub combo contained within the combo string
-    for(kI = 0; kI < keyCombo.length; kI += 1) {
-      binding = {};
-
-      //stringify the combo again
-      subCombo = stringifyKeyCombo([keyCombo[kI]]);
-
-      //validate the sub combo
-      if(typeof subCombo !== 'string') { throw new Error('Failed to bind key combo. The key combo must be string.'); }
-
-      //create the binding
-      binding.keyCombo = subCombo;
-      binding.keyDownCallback = [];
-      binding.keyUpCallback = [];
-
-      //inject the key down and key up callbacks if given
-      if(keyDownCallback) { binding.keyDownCallback.push(keyDownCallback); }
-      if(keyUpCallback) { binding.keyUpCallback.push(keyUpCallback); }
-
-      //stash the new binding
-      bindings.push(binding);
-      subBindings.push(binding);
-    }
-
-    //build the binding api
-    api.clear = clear;
-    api.on = on;
-    return api;
-
-    /**
-     * Clears the binding
-     */
-    function clear() {
-      var bI;
-      for(bI = 0; bI < subBindings.length; bI += 1) {
-        bindings.splice(bindings.indexOf(subBindings[bI]), 1);
-      }
-    }
-
-    /**
-     * Accepts an event name. and any number of callbacks. When the event is
-     *  emitted, all callbacks are executed. Available events are key up and
-     *  key down.
-     * @param  {String} eventName
-     * @return {Object} subBinding
-     */
-    function on(eventName    ) {
-      var api = {}, callbacks, cI, bI;
-
-      //validate event name
-      if(typeof eventName !== 'string') { throw new Error('Cannot bind callback. The event name must be a string.'); }
-      if(eventName !== 'keyup' && eventName !== 'keydown') { throw new Error('Cannot bind callback. The event name must be a "keyup" or "keydown".'); }
-
-      //gather the callbacks
-      callbacks = Array.prototype.slice.apply(arguments, [1]);
-
-      //stash each the new binding
-      for(cI = 0; cI < callbacks.length; cI += 1) {
-        if(typeof callbacks[cI] === 'function') {
-          if(eventName === 'keyup') {
-            for(bI = 0; bI < subBindings.length; bI += 1) {
-              subBindings[bI].keyUpCallback.push(callbacks[cI]);
-            }
-          } else if(eventName === 'keydown') {
-            for(bI = 0; bI < subBindings.length; bI += 1) {
-              subBindings[bI].keyDownCallback.push(callbacks[cI]);
-            }
-          }
-        }
-      }
-
-      //construct and return the sub binding api
-      api.clear = clear;
-      return api;
-
-      /**
-       * Clears the binding
-       */
-      function clear() {
-        var cI, bI;
-        for(cI = 0; cI < callbacks.length; cI += 1) {
-          if(typeof callbacks[cI] === 'function') {
-            if(eventName === 'keyup') {
-              for(bI = 0; bI < subBindings.length; bI += 1) {
-                subBindings[bI].keyUpCallback.splice(subBindings[bI].keyUpCallback.indexOf(callbacks[cI]), 1);
-              }
-            } else {
-              for(bI = 0; bI < subBindings.length; bI += 1) {
-                subBindings[bI].keyDownCallback.splice(subBindings[bI].keyDownCallback.indexOf(callbacks[cI]), 1);
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Clears all binding attached to a given key combo. Key name order does not
-   * matter as long as the key combos equate.
-   * @param  {String} keyCombo
-   */
-  function removeBindingByKeyCombo(keyCombo) {
-    var bI, binding, keyName;
-    for(bI = 0; bI < bindings.length; bI += 1) {
-      binding = bindings[bI];
-      if(compareCombos(keyCombo, binding.keyCombo)) {
-        bindings.splice(bI, 1); bI -= 1;
-      }
-    }
-  }
-
-  /**
-   * Clears all binding attached to key combos containing a given key name.
-   * @param  {String} keyName
-   */
-  function removeBindingByKeyName(keyName) {
-    var bI, kI, binding;
-    if(keyName) {
-      for(bI = 0; bI < bindings.length; bI += 1) {
-        binding = bindings[bI];
-        for(kI = 0; kI < binding.keyCombo.length; kI += 1) {
-          if(binding.keyCombo[kI].indexOf(keyName) > -1) {
-            bindings.splice(bI, 1); bI -= 1;
-            break;
-          }
-        }
-      }
-    } else {
-      bindings = [];
-    }
-  }
-
-  /**
-   * Executes bindings that are active. Only allows the keys to be used once
-   *  as to prevent binding overlap.
-   * @param  {KeyboardEvent}  event The keyboard event.
-   */
-  function executeBindings(event) {
-    var bI, sBI, binding, bindingKeys, remainingKeys, cI, killEventBubble, kI, bindingKeysSatisfied,
-    index, sortedBindings = [], bindingWeight;
-
-    remainingKeys = [].concat(activeKeys);
-    for(bI = 0; bI < bindings.length; bI += 1) {
-      bindingWeight = extractComboKeys(bindings[bI].keyCombo).length;
-      if(!sortedBindings[bindingWeight]) { sortedBindings[bindingWeight] = []; }
-      sortedBindings[bindingWeight].push(bindings[bI]);
-    }
-    for(sBI = sortedBindings.length - 1; sBI >= 0; sBI -= 1) {
-      if(!sortedBindings[sBI]) { continue; }
-      for(bI = 0; bI < sortedBindings[sBI].length; bI += 1) {
-        binding = sortedBindings[sBI][bI];
-        bindingKeys = extractComboKeys(binding.keyCombo);
-        bindingKeysSatisfied = true;
-        for(kI = 0; kI < bindingKeys.length; kI += 1) {
-          if(remainingKeys.indexOf(bindingKeys[kI]) === -1) {
-            bindingKeysSatisfied = false;
-            break;
-          }
-        }
-        if(bindingKeysSatisfied && isSatisfiedCombo(binding.keyCombo)) {
-          activeBindings.push(binding);
-          for(kI = 0; kI < bindingKeys.length; kI += 1) {
-            index = remainingKeys.indexOf(bindingKeys[kI]);
-            if(index > -1) {
-              remainingKeys.splice(index, 1);
-              kI -= 1;
-            }
-          }
-          for(cI = 0; cI < binding.keyDownCallback.length; cI += 1) {
-            if (binding.keyDownCallback[cI](event, getActiveKeys(), binding.keyCombo) === false) {
-              killEventBubble = true;
-            }
-          }
-          if(killEventBubble === true) {
-            event.preventDefault();
-            event.stopPropagation();
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Removes bindings that are no longer satisfied by the active keys. Also
-   *  fires the key up callbacks.
-   * @param  {KeyboardEvent}  event
-   */
-  function pruneBindings(event) {
-    var bI, cI, binding, killEventBubble;
-    for(bI = 0; bI < activeBindings.length; bI += 1) {
-      binding = activeBindings[bI];
-      if(isSatisfiedCombo(binding.keyCombo) === false) {
-        for(cI = 0; cI < binding.keyUpCallback.length; cI += 1) {
-          if (binding.keyUpCallback[cI](event, getActiveKeys(), binding.keyCombo) === false) {
-            killEventBubble = true;
-          }
-        }
-        if(killEventBubble === true) {
-          event.preventDefault();
-          event.stopPropagation();
-        }
-        activeBindings.splice(bI, 1);
-        bI -= 1;
-      }
-    }
-  }
-
-
-  ///////////////////
-  // COMBO STRINGS //
-  ///////////////////
-
-  /**
-   * Compares two key combos returning true when they are functionally
-   *  equivalent.
-   * @param  {String} keyComboArrayA keyCombo A key combo string or array.
-   * @param  {String} keyComboArrayB keyCombo A key combo string or array.
-   * @return {Boolean}
-   */
-  function compareCombos(keyComboArrayA, keyComboArrayB) {
-    var cI, sI, kI;
-    keyComboArrayA = parseKeyCombo(keyComboArrayA);
-    keyComboArrayB = parseKeyCombo(keyComboArrayB);
-    if(keyComboArrayA.length !== keyComboArrayB.length) { return false; }
-    for(cI = 0; cI < keyComboArrayA.length; cI += 1) {
-      if(keyComboArrayA[cI].length !== keyComboArrayB[cI].length) { return false; }
-      for(sI = 0; sI < keyComboArrayA[cI].length; sI += 1) {
-        if(keyComboArrayA[cI][sI].length !== keyComboArrayB[cI][sI].length) { return false; }
-        for(kI = 0; kI < keyComboArrayA[cI][sI].length; kI += 1) {
-          if(keyComboArrayB[cI][sI].indexOf(keyComboArrayA[cI][sI][kI]) === -1) { return false; }
-        }
-      }
-    }
-    return true;
-  }
-
-  /**
-   * Checks to see if a key combo string or key array is satisfied by the
-   *  currently active keys. It does not take into account spent keys.
-   * @param  {String} keyCombo  A key combo string or array.
-   * @return {Boolean}
-   */
-  function isSatisfiedCombo(keyCombo) {
-    var cI, sI, stage, kI, stageOffset = 0, index, comboMatches;
-    keyCombo = parseKeyCombo(keyCombo);
-    for(cI = 0; cI < keyCombo.length; cI += 1) {
-      comboMatches = true;
-      stageOffset = 0;
-      for(sI = 0; sI < keyCombo[cI].length; sI += 1) {
-        stage = [].concat(keyCombo[cI][sI]);
-        for(kI = stageOffset; kI < activeKeys.length; kI += 1) {
-          index = stage.indexOf(activeKeys[kI]);
-          if(index > -1) {
-            stage.splice(index, 1);
-            stageOffset = kI;
-          }
-        }
-        if(stage.length !== 0) { comboMatches = false; break; }
-      }
-      if(comboMatches) { return true; }
-    }
-    return false;
-  }
-
-  /**
-   * Accepts a key combo array or string and returns a flat array containing all keys referenced by
-   * the key combo.
-   * @param  {String} keyCombo  A key combo string or array.
-   * @return {Array}
-   */
-  function extractComboKeys(keyCombo) {
-    var cI, sI, kI, keys = [];
-    keyCombo = parseKeyCombo(keyCombo);
-    for(cI = 0; cI < keyCombo.length; cI += 1) {
-      for(sI = 0; sI < keyCombo[cI].length; sI += 1) {
-        keys = keys.concat(keyCombo[cI][sI]);
-      }
-    }
-    return keys;
-  }
-
-  /**
-   * Parses a key combo string into a 3 dimensional array.
-   * - Level 1 - sub combos.
-   * - Level 2 - combo stages. A stage is a set of key name pairs that must
-   *  be satisfied in the order they are defined.
-   * - Level 3 - each key name to the stage.
-   * @param  {String|Array} keyCombo  A key combo string.
-   * @return {Array}
-   */
-  function parseKeyCombo(keyCombo) {
-    var s = keyCombo, i = 0, op = 0, ws = false, nc = false, combos = [], combo = [], stage = [], key = '';
-
-    if(typeof keyCombo === 'object' && typeof keyCombo.push === 'function') { return keyCombo; }
-    if(typeof keyCombo !== 'string') { throw new Error('Cannot parse "keyCombo" because its type is "' + (typeof keyCombo) + '". It must be a "string".'); }
-
-    //remove leading whitespace
-    while(s.charAt(i) === ' ') { i += 1; }
-    while(true) {
-      if(s.charAt(i) === ' ') {
-        //white space & next combo op
-        while(s.charAt(i) === ' ') { i += 1; }
-        ws = true;
-      } else if(s.charAt(i) === ',') {
-        if(op || nc) { throw new Error('Failed to parse key combo. Unexpected , at character index ' + i + '.'); }
-        nc = true;
-        i += 1;
-      } else if(s.charAt(i) === '+') {
-        //next key
-        if(key.length) { stage.push(key); key = ''; }
-        if(op || nc) { throw new Error('Failed to parse key combo. Unexpected + at character index ' + i + '.'); }
-        op = true;
-        i += 1;
-      } else if(s.charAt(i) === '>') {
-        //next stage op
-        if(key.length) { stage.push(key); key = ''; }
-        if(stage.length) { combo.push(stage); stage = []; }
-        if(op || nc) { throw new Error('Failed to parse key combo. Unexpected > at character index ' + i + '.'); }
-        op = true;
-        i += 1;
-      } else if(i < s.length - 1 && s.charAt(i) === '!' && (s.charAt(i + 1) === '>' || s.charAt(i + 1) === ',' || s.charAt(i + 1) === '+')) {
-        key += s.charAt(i + 1);
-        op = false;
-        ws = false;
-        nc = false;
-        i += 2;
-      } else if(i < s.length && s.charAt(i) !== '+' && s.charAt(i) !== '>' && s.charAt(i) !== ',' && s.charAt(i) !== ' ') {
-        //end combo
-        if(op === false && ws === true || nc === true) {
-          if(key.length) { stage.push(key); key = ''; }
-          if(stage.length) { combo.push(stage); stage = []; }
-          if(combo.length) { combos.push(combo); combo = []; }
-        }
-        op = false;
-        ws = false;
-        nc = false;
-        //key
-        while(i < s.length && s.charAt(i) !== '+' && s.charAt(i) !== '>' && s.charAt(i) !== ',' && s.charAt(i) !== ' ') {
-          key += s.charAt(i);
-          i += 1;
-        }
-      } else {
-        //unknown char
-        i += 1;
-        continue;
-      }
-      //end of combos string
-      if(i >= s.length) {
-        if(key.length) { stage.push(key); key = ''; }
-        if(stage.length) { combo.push(stage); stage = []; }
-        if(combo.length) { combos.push(combo); combo = []; }
-        break;
-      }
-    }
-    return combos;
-  }
-
-  /**
-   * Stringifys a key combo.
-   * @param  {Array|String} keyComboArray A key combo array. If a key
-   *  combo string is given it will be returned.
-   * @return {String}
-   */
-  function stringifyKeyCombo(keyComboArray) {
-    var cI, ccI, output = [];
-    if(typeof keyComboArray === 'string') { return keyComboArray; }
-    if(typeof keyComboArray !== 'object' || typeof keyComboArray.push !== 'function') { throw new Error('Cannot stringify key combo.'); }
-    for(cI = 0; cI < keyComboArray.length; cI += 1) {
-      output[cI] = [];
-      for(ccI = 0; ccI < keyComboArray[cI].length; ccI += 1) {
-        output[cI][ccI] = keyComboArray[cI][ccI].join(' + ');
-      }
-      output[cI] = output[cI].join(' > ');
-    }
-    return output.join(' ');
-  }
-
-
-  /////////////////
-  // ACTIVE KEYS //
-  /////////////////
-
-  /**
-   * Returns the a copy of the active keys array.
-   * @return {Array}
-   */
-  function getActiveKeys() {
-    return [].concat(activeKeys);
-  }
-
-  /**
-   * Adds a key to the active keys array, but only if it has not already been
-   *  added.
-   * @param {String}  keyName The key name string.
-   */
-  function addActiveKey(keyName) {
-    if(keyName.match(/\s/)) { throw new Error('Cannot add key name ' + keyName + ' to active keys because it contains whitespace.'); }
-    if(activeKeys.indexOf(keyName) > -1) { return; }
-    activeKeys.push(keyName);
-  }
-
-  /**
-   * Removes a key from the active keys array.
-   * @param  {String} keyNames  The key name string.
-   */
-  function removeActiveKey(keyName) {
-    var keyCode = getKeyCode(keyName);
-    if(keyCode === '91' || keyCode === '92') { activeKeys = []; } //remove all key on release of super.
-    else { activeKeys.splice(activeKeys.indexOf(keyName), 1); }
-  }
-
-
-  /////////////
-  // LOCALES //
-  /////////////
-
-  /**
-   * Registers a new locale. This is useful if you would like to add support for a new keyboard layout. It could also be useful for
-   * alternative key names. For example if you program games you could create a locale for your key mappings. Instead of key 65 mapped
-   * to 'a' you could map it to 'jump'.
-   * @param  {String} localeName  The name of the new locale.
-   * @param  {Object} localeMap The locale map.
-   */
-  function registerLocale(localeName, localeMap) {
-
-    //validate arguments
-    if(typeof localeName !== 'string') { throw new Error('Cannot register new locale. The locale name must be a string.'); }
-    if(typeof localeMap !== 'object') { throw new Error('Cannot register ' + localeName + ' locale. The locale map must be an object.'); }
-    if(typeof localeMap.map !== 'object') { throw new Error('Cannot register ' + localeName + ' locale. The locale map is invalid.'); }
-
-    //stash the locale
-    if(!localeMap.macros) { localeMap.macros = []; }
-    locales[localeName] = localeMap;
-  }
-
-  /**
-   * Swaps the current locale.
-   * @param  {String} localeName  The locale to activate.
-   * @return {Object}
-   */
-  function getSetLocale(localeName) {
-
-    //if a new locale is given then set it
-    if(localeName) {
-      if(typeof localeName !== 'string') { throw new Error('Cannot set locale. The locale name must be a string.'); }
-      if(!locales[localeName]) { throw new Error('Cannot set locale to ' + localeName + ' because it does not exist. If you would like to submit a ' + localeName + ' locale map for KeyboardJS please submit it at https://github.com/RobertWHurst/KeyboardJS/issues.'); }
-
-      //set the current map and macros
-      map = locales[localeName].map;
-      macros = locales[localeName].macros;
-
-      //set the current locale
-      locale = localeName;
-    }
-
-    //return the current locale
-    return locale;
-  }
-});

puma_config.rb

@@ -0,0 +1,28 @@
+def processor_count
+  case RbConfig::CONFIG['host_os']
+  when /darwin9/
+    `hwprefs cpu_count`.to_i
+  when /darwin/
+    ((`which hwprefs` != '') ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
+  when /linux/
+    `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+  when /freebsd/
+    `sysctl -n hw.ncpu`.to_i
+  when /mswin|mingw/
+    require 'win32ole'
+    wmi = WIN32OLE.connect("winmgmts://")
+    cpu = wmi.ExecQuery("select NumberOfCores from Win32_Processor") # TODO count hyper-threaded in this
+    cpu.to_enum.first.NumberOfCores
+  end
+end
+
+environment 'production'
+daemonize
+pidfile '/var/run/neocities/neocities.pid'
+stdout_redirect '/var/log/neocities/neocities.log', '/var/log/neocities/neocities-errors.log', true
+quiet
+workers processor_count
+worker_timeout 600
+preload_app!
+on_worker_boot { DB.disconnect }
+bind 'unix:/var/run/neocities/neocities.sock?backlog=2048'

sass/_project-sass/_project-Header.scss

@@ -14,7 +14,7 @@
 }
 
 .row.header-Content.content {
-  padding-bottom:27px;
+    padding-bottom: 31px;
 }
 
 .header-Content.content{

sass/_project-sass/_project-Main.scss

@@ -187,7 +187,7 @@
 
   @media (max-device-width:480px), screen and (max-width:800px) {
     width: 100%;
-    height: 300px;
+    height: 200px;
   }
 }
 .interior .header-Outro .screenshot.dashboard {
@@ -276,9 +276,6 @@
       background: #77ABB8;
       @include box-shadow(0 0 5px rgba(0, 0, 0, 0.2));
 
-      &:hover {
-        background: #83B3C0;
-      }
       &:focus, &.active {
         outline: 0;
         background: #4F727B;
@@ -541,7 +538,7 @@
     }
     .title {
       margin: 0;
-      margin-left: 7px;
+      margin-left: 9px;
       margin-top: 2px;
       float: left;
       font-size: 14px;
@@ -562,7 +559,6 @@
   }
   .html-thumbnail, .misc-icon {
     margin: 0;
-    margin-left: 4px;
     float: left;
     width: 23px;
     height: 23px;
@@ -756,10 +752,9 @@
     }
   }
   @media (max-device-width:480px), screen and (max-width:800px) {
-    width: 10em!important;
+    position: absolute;
-    float: right;
+    top: 46px;
-    padding: 0 0 18px 0;
+    right: -8px;
-    margin-top: -77px;
   }
 }
 .interior .header-Outro.with-columns .col-66 {
@@ -820,6 +815,7 @@
 
   @media (max-device-width:480px), screen and (max-width:800px) {
     width: 60%;
+    height: 160px;
   }
 }
 .site-portrait {
@@ -989,12 +985,12 @@ a.tag:hover {
     width: 102px;
   }
   &:first-child .html-thumbnail.html {
-    width: 322px;
+    width: 540px;
-    height: 100px;
+    height: 405px;
   }
   &:first-child .html-thumbnail.html img {
-    width: 322px;
+    width: 540px;
-    height: 200px;
+    height: 405px;
   }
 }
 .news-item .file .image-container {
@@ -1020,6 +1016,7 @@ a.tag:hover {
 .news-item .comments .actions, .news-item .comments p {
   margin-left: 47px;
 }
+
 .news-item .comments p {
   margin-bottom: .4em;
   margin-top: .15em;
@@ -1867,7 +1864,7 @@ a.tag:hover {
 
   padding-top: 0;
   background: #4F7E89;
-  padding-bottom: 7em;
+  padding-bottom: 5em;
 
   a {
     color: white;
@@ -2137,43 +2134,71 @@ table#latest-visitors {
     width: 100%;
   }
 }
-.intro-List.kickstarter .col {
+.section.thankyou {
-  padding-top: 1em;
+  text-align: center;
-  padding-bottom: .8em;
+  color: #4F7E89;
-  margin-left: 0;
+  padding: 6.5em 8% 7em;
 
-  &:first-child{
+  a {
-    padding-left: 2px; 
+    color: #4F7E89;
+    text-decoration: underline;
   }
-  
+  img {
-  .title {
+    margin-bottom: 1em;
-    margin-top: 1%;
-  }
-  .title a {
-    color: white;
-    font-weight: bold;
-    text-decoration: none;
   }
   p {
-    margin-top: 15px;
+    font-size: 1em;
+    margin-bottom: .5em;
+    margin-top: .5em;
   }
-}
+  p:first-child {
-.welcome.kickstarter {
+    font-size: 120%;
-  background: #daeea5 url(/img/tutorialthumbnail.png) no-repeat;
-  background-position:  right center;
-  background-size: auto 100%;
-  padding: 15px 100px 4px 23px;
-  margin-bottom: 13px;
-  font-size: 95%;
-  @media (max-device-width:480px), screen and (max-width:800px) {
-    background-size: 32%;
-    background-position: right top;
-  }
-  
-  h4 {
     margin-bottom: .2em;
-    a {
-      color: #2c3e50!important;
-    }
   }
 }
+ul.thankyou {
+  list-style: none;
+  margin-top: 1.5em;
+  clear: both;
+
+  li {
+    display: inline-block;
+    width: 32%;
+  }
+}
+
+pre, code {
+  background: #1d1f21;
+  color: #FFFFFF;
+}
+.welcoming-cat {
+  width: 200px;
+  float: right;
+  margin-top: -40px;
+}
+.section.tutorial-welcome {
+  padding: 80px 18%;
+
+  .option {
+    margin: 3.5em 0;
+  }
+  h3 {
+    margin-top: 0em;
+    font-size: 1.3em;
+  }
+  p {
+    font-size: 1.1em;
+  }
+}
+
+.tagcloud .tag10 { font-size: 0.6em; font-weight: 90; }
+.tagcloud .tag9 { font-size: 0.7em; font-weight: 100; }
+.tagcloud .tag8 { font-size: 0.8em; font-weight: 200; }
+.tagcloud .tag7 { font-size: 0.9em; font-weight: 300; }
+.tagcloud .tag6 { font-size: 1.0em; font-weight: 400; }
+.tagcloud .tag5 { font-size: 1.2em; font-weight: 500; }
+.tagcloud .tag4 { font-size: 1.4em; font-weight: 600; }
+.tagcloud .tag3 { font-size: 1.6em; font-weight: 700; }
+.tagcloud .tag2 { font-size: 1.8em; font-weight: 800; }
+.tagcloud .tag1 { font-size: 2.2em; font-weight: 900; }
+.tagcloud .tag0 { font-size: 2.5em; font-weight: 900; }