Switch to directory sharding model for site storage

Rakefile

@@ -407,3 +407,21 @@ task :purge_tmp_turds => [:environment] do
     Dir.glob("/tmp/#{target}").select {|filename| File::Stat.new(filename).ctime < (Time.now - 3600)}.each {|filename| FileUtils.rm(filename)}
   end
 end
+
+desc 'shard_migration'
+task :shard_migration => [:environment] do
+  #Site.exclude(is_deleted: true).exclude(is_banned: true).select(:username).each do |site|
+  #  FileUtils.mkdir_p File.join('public', 'testsites', site.username)
+  #end
+  #exit
+  Dir.chdir('./public/testsites')
+  Dir.glob('*').each do |dir|
+    sharding_dir = Site.sharding_dir(dir)
+    FileUtils.mkdir_p File.join('..', 'newtestsites', sharding_dir)
+    FileUtils.mv dir, File.join('..', 'newtestsites', sharding_dir)
+  end
+  sleep 1
+  FileUtils.rmdir './public/testsites'
+  sleep 1
+  FileUtils.mv './public/newtestsites', './public/testsites'
+end

models/site.rb

@@ -2,6 +2,7 @@ require 'tilt'
 require 'rss'
 require 'nokogiri'
 require 'pathname'
+require 'zlib'
 
 class Site < Sequel::Model
   include Sequel::ParanoidDelete
@@ -436,7 +437,8 @@ class Site < Sequel::Model
         FileUtils.mkdir DELETED_SITES_ROOT
       end
 
-      FileUtils.mv files_path, File.join(DELETED_SITES_ROOT, username)
+      FileUtils.mkdir_p File.join(DELETED_SITES_ROOT, self.class.sharding_dir(username))
+      FileUtils.mv files_path, File.join(DELETED_SITES_ROOT, self.class.sharding_dir(username), '/')
       remove_all_tags
       #remove_all_events
       #Event.where(actioning_site_id: id).destroy
@@ -784,6 +786,7 @@ class Site < Sequel::Model
   end
 
   def move_files_from(oldusername)
+    FileUtils.mkdir_p self.class.sharding_base_path(username)
     FileUtils.mv base_files_path(oldusername), base_files_path
   end
 
@@ -1010,13 +1013,22 @@ class Site < Sequel::Model
 
   def current_base_files_path(name=username)
     raise 'username missing' if name.nil? || name.empty?
-    return File.join DELETED_SITES_ROOT, name if is_deleted
+    return File.join DELETED_SITES_ROOT, self.class.sharding_dir(name), name if is_deleted
     base_files_path name
   end
 
   def base_files_path(name=username)
     raise 'username missing' if name.nil? || name.empty?
-    File.join SITE_FILES_ROOT, name
+    File.join SITE_FILES_ROOT, self.class.sharding_dir(name), name
+  end
+
+  def self.sharding_base_path(name)
+    File.join SITE_FILES_ROOT, sharding_dir(name)
+  end
+
+  def self.sharding_dir(name)
+    chksum = Zlib::crc32(name).to_s
+    File.join(chksum[0..1], chksum[2..3])
   end
 
   # https://practicingruby.com/articles/implementing-an-http-file-server?u=dc2ab0f9bb
@@ -1304,7 +1316,7 @@ class Site < Sequel::Model
   end
 
   def screenshot_path(path, resolution)
-    File.join(SCREENSHOTS_ROOT, values[:username], "#{path}.#{resolution}.jpg")
+    File.join(SCREENSHOTS_ROOT, self.class.sharding_dir(values[:username]), values[:username], "#{path}.#{resolution}.jpg")
   end
 
   def screenshot_exists?(path, resolution)
@@ -1315,9 +1327,13 @@ class Site < Sequel::Model
     "#{SCREENSHOTS_URL_ROOT}/#{values[:username]}/#{path}.#{resolution}.jpg"
   end
 
+  def base_thumbnails_path
+    File.join THUMBNAILS_ROOT, self.class.sharding_dir(values[:username]), values[:username]
+  end
+
   def thumbnail_path(path, resolution)
     ext = File.extname(path).gsub('.', '').match(LOSSY_IMAGE_REGEX) ? 'jpg' : 'png'
-    File.join THUMBNAILS_ROOT, values[:username], "#{path}.#{resolution}.#{ext}"
+    File.join base_thumbnails_path, "#{path}.#{resolution}.#{ext}"
   end
 
   def thumbnail_exists?(path, resolution)

tests/acceptance/education_tests.rb

@@ -38,7 +38,7 @@ describe 'signup' do
     site = Site[username: @site[:username]]
     site.site_files.length.must_equal 4
     site.site_changed.must_equal false
-    site.site_updated_at.must_equal nil
+    site.site_updated_at.must_be_nil
     site.is_education.must_equal true
     site.tags.length.must_equal 1
     site.tags.first.name.must_equal @class_tag

tests/acceptance/password_reset_tests.rb

@@ -83,7 +83,7 @@ describe '/password_reset' do
     page.must_have_content 'Successfully changed password'
     Site.valid_login?(@site.username, 'n3wp4s$').must_equal true
     page.get_rack_session['id'].must_equal @site.id
-    @site.reload.password_reset_token.must_equal nil
+    @site.reload.password_reset_token.must_be_nil
     @site.password_reset_confirmed.must_equal false
   end
 

tests/acceptance/settings/account_tests.rb

@@ -29,7 +29,7 @@ describe 'site/settings' do
 
       @site.reload
       @site.email.must_equal @new_email
-      @site.password_reset_token.must_equal nil
+      @site.password_reset_token.must_be_nil
 
       EmailWorker.jobs.length.must_equal 2
 

tests/acceptance/settings/site_tests.rb

@@ -103,103 +103,6 @@ describe 'site/settings' do
     end
   end
 
-=begin
-  describe 'ssl' do
-    include Capybara::DSL
-
-    before do
-      @domain = SecureRandom.uuid.gsub('-', '')+'.com'
-      @site = Fabricate :site, domain: @domain
-      page.set_rack_session id: @site.id
-    end
-
-    it 'fails without domain set' do
-      @site = Fabricate :site
-      page.set_rack_session id: @site.id
-      visit "/settings/#{@site.username}#custom_domain"
-      page.must_have_content /Cannot upload SSL certificate until domain is added/i
-    end
-
-    it 'fails with expired key' do
-      @ssl = generate_ssl_certs domain: @domain, expired: true
-      visit "/settings/#{@site.username}#custom_domain"
-      attach_file 'key', @ssl[:key_path]
-      attach_file 'cert', @ssl[:combined_cert_path]
-      click_button 'Upload SSL Key and Certificate'
-      page.must_have_content /ssl certificate has expired/i
-    end
-
-    it 'works with valid key and unified cert' do
-      @ssl = generate_ssl_certs domain: @domain
-      visit "/settings/#{@site.username}#custom_domain"
-      key = File.read @ssl[:key_path]
-      combined_cert = File.read @ssl[:combined_cert_path]
-      page.must_have_content /status: inactive/i
-      attach_file 'key', @ssl[:key_path]
-      attach_file 'cert', @ssl[:combined_cert_path]
-      click_button 'Upload SSL Key and Certificate'
-      page.current_path.must_equal "/settings/#{@site.username}"
-      page.must_have_content /Updated SSL/
-      page.must_have_content /status: installed/i
-      @site.reload
-      @site.ssl_key.must_equal key
-      @site.ssl_cert.must_equal combined_cert
-    end
-
-    it 'fails with no uploads' do
-      visit "/settings/#{@site.username}#custom_domain"
-      click_button 'Upload SSL Key and Certificate'
-      page.current_path.must_equal "/settings/#{@site.username}"
-      page.must_have_content /ssl key.+certificate.+required/i
-      @site.reload
-      @site.ssl_key.must_equal nil
-      @site.ssl_cert.must_equal nil
-    end
-
-    it 'fails gracefully with encrypted key' do
-      @ssl = generate_ssl_certs domain: @domain
-      visit "/settings/#{@site.username}#custom_domain"
-      attach_file 'key', './tests/files/ssl/derpie.com-encrypted.key'
-      attach_file 'cert', @ssl[:cert_path]
-      click_button 'Upload SSL Key and Certificate'
-      page.current_path.must_equal "/settings/#{@site.username}"
-      page.must_have_content /could not process ssl key/i
-    end
-
-    it 'fails with junk key' do
-      @ssl = generate_ssl_certs domain: @domain
-      visit "/settings/#{@site.username}#custom_domain"
-      attach_file 'key', './tests/files/index.html'
-      attach_file 'cert', @ssl[:cert_path]
-      click_button 'Upload SSL Key and Certificate'
-      page.current_path.must_equal "/settings/#{@site.username}"
-      page.must_have_content /could not process ssl key/i
-    end
-
-    it 'fails with junk cert' do
-      @ssl = generate_ssl_certs domain: @domain
-      visit "/settings/#{@site.username}#custom_domain"
-      attach_file 'key', @ssl[:key_path]
-      attach_file 'cert', './tests/files/index.html'
-      click_button 'Upload SSL Key and Certificate'
-      page.current_path.must_equal "/settings/#{@site.username}"
-      page.must_have_content /could not process ssl certificate/i
-    end
-
-    if ENV['TRAVIS'] != 'true'
-      it 'fails with bad cert chain' do
-        @ssl = generate_ssl_certs domain: @domain
-        visit "/settings/#{@site.username}#custom_domain"
-        attach_file 'key', @ssl[:key_path]
-        attach_file 'cert', @ssl[:bad_combined_cert_path]
-        click_button 'Upload SSL Key and Certificate'
-        page.current_path.must_equal "/settings/#{@site.username}"
-        page.must_have_content /there is something wrong with your certificate/i
-      end
-    end
-  end
-=end
-
   describe 'changing username' do
     include Capybara::DSL
 
@@ -218,14 +121,14 @@ describe 'site/settings' do
       fill_in 'name', with: ''
       click_button 'Change Name'
       page.must_have_content /cannot be blank/i
-      Site[username: ''].must_equal nil
+      Site[username: ''].must_be_nil
     end
 
     it 'fails for subdir periods' do
       fill_in 'name', with: '../hack'
       click_button 'Change Name'
       page.must_have_content /Usernames can only contain/i
-      Site[username: '../hack'].must_equal nil
+      Site[username: '../hack'].must_be_nil
     end
 
     it 'fails for same username' do
@@ -279,7 +182,7 @@ describe 'delete' do
     File.exist?(@site.files_path('./index.html')).must_equal false
     Dir.exist?(@site.files_path).must_equal false
 
-    path = File.join Site::DELETED_SITES_ROOT, @site.username
+    path = File.join Site::DELETED_SITES_ROOT, Site.sharding_dir(@site.username), @site.username
     Dir.exist?(path).must_equal true
     File.exist?(File.join(path, 'index.html')).must_equal true
 
@@ -314,7 +217,7 @@ describe 'delete' do
 
     Stripe::Customer.retrieve(@site.stripe_customer_id).subscriptions.count.must_equal 0
     @site.reload
-    @site.stripe_subscription_id.must_equal nil
+    @site.stripe_subscription_id.must_be_nil
     @site.is_deleted.must_equal true
   end
 

tests/acceptance/signup_tests.rb

@@ -51,13 +51,13 @@ describe 'signup' do
     current_path.must_equal '/tutorial'
     page.must_have_content /Let's Get Started/
 
-    index_file_path = File.join Site::SITE_FILES_ROOT, @site[:username], 'index.html'
+    index_file_path = File.join Site::SITE_FILES_ROOT, Site.sharding_dir(@site[:username]), @site[:username], 'index.html'
     File.exist?(index_file_path).must_equal true
 
     site = Site[username: @site[:username]]
     site.site_files.length.must_equal 4
     site.site_changed.must_equal false
-    site.site_updated_at.must_equal nil
+    site.site_updated_at.must_be_nil
     site.is_education.must_equal false
 
     site.ip.must_equal '127.0.0.1'

tests/api_tests.rb

@@ -91,7 +91,7 @@ describe 'api info' do
     res[:info][:sitename].must_equal @site.username
     res[:info][:hits].must_equal 31337
     res[:info][:created_at].must_equal @site.created_at.rfc2822
-    res[:info][:last_updated].must_equal nil
+    res[:info][:last_updated].must_be_nil
     res[:info][:domain].must_equal 'derp.com'
     res[:info][:tags].must_equal ['derpie', 'man']
     res[:info][:latest_ipfs_hash].must_equal 'QmXGTaGWTT1uUtfSb2sBAvArMEVLK4rQEcQg5bv7wwdzwU'
@@ -101,7 +101,7 @@ describe 'api info' do
   it 'shows latest ipfs hash as nil when not present' do
     create_site
     get '/api/info', sitename: @user
-    res[:info][:latest_ipfs_hash].must_equal nil
+    res[:info][:latest_ipfs_hash].must_be_nil
   end
 
   it 'fails for bad auth' do
@@ -237,7 +237,7 @@ describe 'api upload' do
       '../lol.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
     }
     res[:result].must_equal 'success'
-    File.exist?(File.join(Site::SITE_FILES_ROOT, @site.username, 'lol.jpg')).must_equal true
+    File.exist?(File.join(Site::SITE_FILES_ROOT, Site.sharding_dir(@site.username), @site.username, 'lol.jpg')).must_equal true
     @site.reload.api_calls.must_equal 1
   end
 
@@ -248,7 +248,7 @@ describe 'api upload' do
       '/lol.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
     }
     res[:result].must_equal 'success'
-    File.exist?(File.join(Site::SITE_FILES_ROOT, @site.username, 'lol.jpg')).must_equal true
+    File.exist?(File.join(Site::SITE_FILES_ROOT, Site.sharding_dir(@site.username), @site.username, 'lol.jpg')).must_equal true
   end
 
   it 'fails for missing file name' do

tests/site_tests.rb

@@ -10,7 +10,7 @@ describe Site do
       site = Fabricate :site
       site.ban!
       File.exist?(site.current_files_path('index.html')).must_equal true
-      site.current_files_path('index.html').must_equal File.join(Site::DELETED_SITES_ROOT, site.username, 'index.html')
+      site.current_files_path('index.html').must_equal File.join(Site::DELETED_SITES_ROOT, Site.sharding_dir(site.username), site.username, 'index.html')
     end
   end
 

tests/webhook_tests.rb

@@ -61,7 +61,7 @@ describe 'tipping' do
     post '/webhooks/paypal/tipping_notify', paypal_hash
 
     @site.tips.length.must_equal 1
-    @site.tips.first.actioning_site_id.must_equal nil
+    @site.tips.first.actioning_site_id.must_be_nil
   end
 end
 

workers/screenshot_worker.rb

@@ -49,7 +49,7 @@ class ScreenshotWorker
     img = img_list.reverse.flatten_images
     img_list.destroy!
 
-    user_screenshots_path = File.join SCREENSHOTS_PATH, username
+    user_screenshots_path = File.join SCREENSHOTS_PATH, Site.sharding_dir(username), username
     screenshot_path = File.join user_screenshots_path, File.dirname(path)
 
     FileUtils.mkdir_p screenshot_path unless Dir.exists?(screenshot_path)
@@ -109,4 +109,3 @@ class ScreenshotWorker
 =end
   end
 end
-

workers/thumbnail_worker.rb

@@ -6,17 +6,19 @@ class ThumbnailWorker
   sidekiq_options queue: :thumbnails, retry: 3, backtrace: true
 
   def perform(username, path)
+    site = Site[username: username]
+
     img_list = Magick::ImageList.new
 
     begin
-      img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path))
+      img_list.from_blob File.read(site.files_path(path))
     rescue Errno::ENOENT => e # Not found, skip
       return
     end
 
     img = img_list.first
 
-    user_thumbnails_path = File.join THUMBNAILS_PATH, username
+    user_thumbnails_path = site.base_thumbnails_path
     FileUtils.mkdir_p user_thumbnails_path
     FileUtils.mkdir_p File.join(user_thumbnails_path, File.dirname(path))