diff --git a/Rakefile b/Rakefile index 37850bed..93a4ee60 100644 --- a/Rakefile +++ b/Rakefile @@ -407,3 +407,21 @@ task :purge_tmp_turds => [:environment] do Dir.glob("/tmp/#{target}").select {|filename| File::Stat.new(filename).ctime < (Time.now - 3600)}.each {|filename| FileUtils.rm(filename)} end end + +desc 'shard_migration' +task :shard_migration => [:environment] do + #Site.exclude(is_deleted: true).exclude(is_banned: true).select(:username).each do |site| + # FileUtils.mkdir_p File.join('public', 'testsites', site.username) + #end + #exit + Dir.chdir('./public/testsites') + Dir.glob('*').each do |dir| + sharding_dir = Site.sharding_dir(dir) + FileUtils.mkdir_p File.join('..', 'newtestsites', sharding_dir) + FileUtils.mv dir, File.join('..', 'newtestsites', sharding_dir) + end + sleep 1 + FileUtils.rmdir './public/testsites' + sleep 1 + FileUtils.mv './public/newtestsites', './public/testsites' +end diff --git a/models/site.rb b/models/site.rb index 97200cec..c3c532e9 100644 --- a/models/site.rb +++ b/models/site.rb @@ -2,6 +2,7 @@ require 'tilt' require 'rss' require 'nokogiri' require 'pathname' +require 'zlib' class Site < Sequel::Model include Sequel::ParanoidDelete @@ -436,7 +437,8 @@ class Site < Sequel::Model FileUtils.mkdir DELETED_SITES_ROOT end - FileUtils.mv files_path, File.join(DELETED_SITES_ROOT, username) + FileUtils.mkdir_p File.join(DELETED_SITES_ROOT, self.class.sharding_dir(username)) + FileUtils.mv files_path, File.join(DELETED_SITES_ROOT, self.class.sharding_dir(username), '/') remove_all_tags #remove_all_events #Event.where(actioning_site_id: id).destroy @@ -784,6 +786,7 @@ class Site < Sequel::Model end def move_files_from(oldusername) + FileUtils.mkdir_p self.class.sharding_base_path(username) FileUtils.mv base_files_path(oldusername), base_files_path end @@ -1010,13 +1013,22 @@ class Site < Sequel::Model def current_base_files_path(name=username) raise 'username missing' if name.nil? || name.empty? - return File.join DELETED_SITES_ROOT, name if is_deleted + return File.join DELETED_SITES_ROOT, self.class.sharding_dir(name), name if is_deleted base_files_path name end def base_files_path(name=username) raise 'username missing' if name.nil? || name.empty? - File.join SITE_FILES_ROOT, name + File.join SITE_FILES_ROOT, self.class.sharding_dir(name), name + end + + def self.sharding_base_path(name) + File.join SITE_FILES_ROOT, sharding_dir(name) + end + + def self.sharding_dir(name) + chksum = Zlib::crc32(name).to_s + File.join(chksum[0..1], chksum[2..3]) end # https://practicingruby.com/articles/implementing-an-http-file-server?u=dc2ab0f9bb @@ -1304,7 +1316,7 @@ class Site < Sequel::Model end def screenshot_path(path, resolution) - File.join(SCREENSHOTS_ROOT, values[:username], "#{path}.#{resolution}.jpg") + File.join(SCREENSHOTS_ROOT, self.class.sharding_dir(values[:username]), values[:username], "#{path}.#{resolution}.jpg") end def screenshot_exists?(path, resolution) @@ -1315,9 +1327,13 @@ class Site < Sequel::Model "#{SCREENSHOTS_URL_ROOT}/#{values[:username]}/#{path}.#{resolution}.jpg" end + def base_thumbnails_path + File.join THUMBNAILS_ROOT, self.class.sharding_dir(values[:username]), values[:username] + end + def thumbnail_path(path, resolution) ext = File.extname(path).gsub('.', '').match(LOSSY_IMAGE_REGEX) ? 'jpg' : 'png' - File.join THUMBNAILS_ROOT, values[:username], "#{path}.#{resolution}.#{ext}" + File.join base_thumbnails_path, "#{path}.#{resolution}.#{ext}" end def thumbnail_exists?(path, resolution) diff --git a/tests/acceptance/education_tests.rb b/tests/acceptance/education_tests.rb index 9059d5ce..aaa3b8a3 100644 --- a/tests/acceptance/education_tests.rb +++ b/tests/acceptance/education_tests.rb @@ -38,7 +38,7 @@ describe 'signup' do site = Site[username: @site[:username]] site.site_files.length.must_equal 4 site.site_changed.must_equal false - site.site_updated_at.must_equal nil + site.site_updated_at.must_be_nil site.is_education.must_equal true site.tags.length.must_equal 1 site.tags.first.name.must_equal @class_tag diff --git a/tests/acceptance/password_reset_tests.rb b/tests/acceptance/password_reset_tests.rb index 81e4d10a..91075249 100644 --- a/tests/acceptance/password_reset_tests.rb +++ b/tests/acceptance/password_reset_tests.rb @@ -83,7 +83,7 @@ describe '/password_reset' do page.must_have_content 'Successfully changed password' Site.valid_login?(@site.username, 'n3wp4s$').must_equal true page.get_rack_session['id'].must_equal @site.id - @site.reload.password_reset_token.must_equal nil + @site.reload.password_reset_token.must_be_nil @site.password_reset_confirmed.must_equal false end diff --git a/tests/acceptance/settings/account_tests.rb b/tests/acceptance/settings/account_tests.rb index 3934576d..2e6d0329 100644 --- a/tests/acceptance/settings/account_tests.rb +++ b/tests/acceptance/settings/account_tests.rb @@ -29,7 +29,7 @@ describe 'site/settings' do @site.reload @site.email.must_equal @new_email - @site.password_reset_token.must_equal nil + @site.password_reset_token.must_be_nil EmailWorker.jobs.length.must_equal 2 diff --git a/tests/acceptance/settings/site_tests.rb b/tests/acceptance/settings/site_tests.rb index 665d56c5..ec5cf693 100644 --- a/tests/acceptance/settings/site_tests.rb +++ b/tests/acceptance/settings/site_tests.rb @@ -103,103 +103,6 @@ describe 'site/settings' do end end -=begin - describe 'ssl' do - include Capybara::DSL - - before do - @domain = SecureRandom.uuid.gsub('-', '')+'.com' - @site = Fabricate :site, domain: @domain - page.set_rack_session id: @site.id - end - - it 'fails without domain set' do - @site = Fabricate :site - page.set_rack_session id: @site.id - visit "/settings/#{@site.username}#custom_domain" - page.must_have_content /Cannot upload SSL certificate until domain is added/i - end - - it 'fails with expired key' do - @ssl = generate_ssl_certs domain: @domain, expired: true - visit "/settings/#{@site.username}#custom_domain" - attach_file 'key', @ssl[:key_path] - attach_file 'cert', @ssl[:combined_cert_path] - click_button 'Upload SSL Key and Certificate' - page.must_have_content /ssl certificate has expired/i - end - - it 'works with valid key and unified cert' do - @ssl = generate_ssl_certs domain: @domain - visit "/settings/#{@site.username}#custom_domain" - key = File.read @ssl[:key_path] - combined_cert = File.read @ssl[:combined_cert_path] - page.must_have_content /status: inactive/i - attach_file 'key', @ssl[:key_path] - attach_file 'cert', @ssl[:combined_cert_path] - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /Updated SSL/ - page.must_have_content /status: installed/i - @site.reload - @site.ssl_key.must_equal key - @site.ssl_cert.must_equal combined_cert - end - - it 'fails with no uploads' do - visit "/settings/#{@site.username}#custom_domain" - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /ssl key.+certificate.+required/i - @site.reload - @site.ssl_key.must_equal nil - @site.ssl_cert.must_equal nil - end - - it 'fails gracefully with encrypted key' do - @ssl = generate_ssl_certs domain: @domain - visit "/settings/#{@site.username}#custom_domain" - attach_file 'key', './tests/files/ssl/derpie.com-encrypted.key' - attach_file 'cert', @ssl[:cert_path] - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /could not process ssl key/i - end - - it 'fails with junk key' do - @ssl = generate_ssl_certs domain: @domain - visit "/settings/#{@site.username}#custom_domain" - attach_file 'key', './tests/files/index.html' - attach_file 'cert', @ssl[:cert_path] - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /could not process ssl key/i - end - - it 'fails with junk cert' do - @ssl = generate_ssl_certs domain: @domain - visit "/settings/#{@site.username}#custom_domain" - attach_file 'key', @ssl[:key_path] - attach_file 'cert', './tests/files/index.html' - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /could not process ssl certificate/i - end - - if ENV['TRAVIS'] != 'true' - it 'fails with bad cert chain' do - @ssl = generate_ssl_certs domain: @domain - visit "/settings/#{@site.username}#custom_domain" - attach_file 'key', @ssl[:key_path] - attach_file 'cert', @ssl[:bad_combined_cert_path] - click_button 'Upload SSL Key and Certificate' - page.current_path.must_equal "/settings/#{@site.username}" - page.must_have_content /there is something wrong with your certificate/i - end - end - end -=end - describe 'changing username' do include Capybara::DSL @@ -218,14 +121,14 @@ describe 'site/settings' do fill_in 'name', with: '' click_button 'Change Name' page.must_have_content /cannot be blank/i - Site[username: ''].must_equal nil + Site[username: ''].must_be_nil end it 'fails for subdir periods' do fill_in 'name', with: '../hack' click_button 'Change Name' page.must_have_content /Usernames can only contain/i - Site[username: '../hack'].must_equal nil + Site[username: '../hack'].must_be_nil end it 'fails for same username' do @@ -279,7 +182,7 @@ describe 'delete' do File.exist?(@site.files_path('./index.html')).must_equal false Dir.exist?(@site.files_path).must_equal false - path = File.join Site::DELETED_SITES_ROOT, @site.username + path = File.join Site::DELETED_SITES_ROOT, Site.sharding_dir(@site.username), @site.username Dir.exist?(path).must_equal true File.exist?(File.join(path, 'index.html')).must_equal true @@ -314,7 +217,7 @@ describe 'delete' do Stripe::Customer.retrieve(@site.stripe_customer_id).subscriptions.count.must_equal 0 @site.reload - @site.stripe_subscription_id.must_equal nil + @site.stripe_subscription_id.must_be_nil @site.is_deleted.must_equal true end diff --git a/tests/acceptance/signup_tests.rb b/tests/acceptance/signup_tests.rb index e8e123ed..d43bf794 100644 --- a/tests/acceptance/signup_tests.rb +++ b/tests/acceptance/signup_tests.rb @@ -51,13 +51,13 @@ describe 'signup' do current_path.must_equal '/tutorial' page.must_have_content /Let's Get Started/ - index_file_path = File.join Site::SITE_FILES_ROOT, @site[:username], 'index.html' + index_file_path = File.join Site::SITE_FILES_ROOT, Site.sharding_dir(@site[:username]), @site[:username], 'index.html' File.exist?(index_file_path).must_equal true site = Site[username: @site[:username]] site.site_files.length.must_equal 4 site.site_changed.must_equal false - site.site_updated_at.must_equal nil + site.site_updated_at.must_be_nil site.is_education.must_equal false site.ip.must_equal '127.0.0.1' diff --git a/tests/api_tests.rb b/tests/api_tests.rb index b4f83821..b575a566 100644 --- a/tests/api_tests.rb +++ b/tests/api_tests.rb @@ -91,7 +91,7 @@ describe 'api info' do res[:info][:sitename].must_equal @site.username res[:info][:hits].must_equal 31337 res[:info][:created_at].must_equal @site.created_at.rfc2822 - res[:info][:last_updated].must_equal nil + res[:info][:last_updated].must_be_nil res[:info][:domain].must_equal 'derp.com' res[:info][:tags].must_equal ['derpie', 'man'] res[:info][:latest_ipfs_hash].must_equal 'QmXGTaGWTT1uUtfSb2sBAvArMEVLK4rQEcQg5bv7wwdzwU' @@ -101,7 +101,7 @@ describe 'api info' do it 'shows latest ipfs hash as nil when not present' do create_site get '/api/info', sitename: @user - res[:info][:latest_ipfs_hash].must_equal nil + res[:info][:latest_ipfs_hash].must_be_nil end it 'fails for bad auth' do @@ -237,7 +237,7 @@ describe 'api upload' do '../lol.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') } res[:result].must_equal 'success' - File.exist?(File.join(Site::SITE_FILES_ROOT, @site.username, 'lol.jpg')).must_equal true + File.exist?(File.join(Site::SITE_FILES_ROOT, Site.sharding_dir(@site.username), @site.username, 'lol.jpg')).must_equal true @site.reload.api_calls.must_equal 1 end @@ -248,7 +248,7 @@ describe 'api upload' do '/lol.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') } res[:result].must_equal 'success' - File.exist?(File.join(Site::SITE_FILES_ROOT, @site.username, 'lol.jpg')).must_equal true + File.exist?(File.join(Site::SITE_FILES_ROOT, Site.sharding_dir(@site.username), @site.username, 'lol.jpg')).must_equal true end it 'fails for missing file name' do diff --git a/tests/site_tests.rb b/tests/site_tests.rb index 2dbf3770..23f5cda0 100644 --- a/tests/site_tests.rb +++ b/tests/site_tests.rb @@ -10,7 +10,7 @@ describe Site do site = Fabricate :site site.ban! File.exist?(site.current_files_path('index.html')).must_equal true - site.current_files_path('index.html').must_equal File.join(Site::DELETED_SITES_ROOT, site.username, 'index.html') + site.current_files_path('index.html').must_equal File.join(Site::DELETED_SITES_ROOT, Site.sharding_dir(site.username), site.username, 'index.html') end end diff --git a/tests/webhook_tests.rb b/tests/webhook_tests.rb index c0dccd38..36fce9b1 100644 --- a/tests/webhook_tests.rb +++ b/tests/webhook_tests.rb @@ -61,7 +61,7 @@ describe 'tipping' do post '/webhooks/paypal/tipping_notify', paypal_hash @site.tips.length.must_equal 1 - @site.tips.first.actioning_site_id.must_equal nil + @site.tips.first.actioning_site_id.must_be_nil end end diff --git a/workers/screenshot_worker.rb b/workers/screenshot_worker.rb index 9432a14f..130fbe32 100644 --- a/workers/screenshot_worker.rb +++ b/workers/screenshot_worker.rb @@ -49,7 +49,7 @@ class ScreenshotWorker img = img_list.reverse.flatten_images img_list.destroy! - user_screenshots_path = File.join SCREENSHOTS_PATH, username + user_screenshots_path = File.join SCREENSHOTS_PATH, Site.sharding_dir(username), username screenshot_path = File.join user_screenshots_path, File.dirname(path) FileUtils.mkdir_p screenshot_path unless Dir.exists?(screenshot_path) @@ -109,4 +109,3 @@ class ScreenshotWorker =end end end - diff --git a/workers/thumbnail_worker.rb b/workers/thumbnail_worker.rb index a4b4849c..493609e9 100644 --- a/workers/thumbnail_worker.rb +++ b/workers/thumbnail_worker.rb @@ -6,17 +6,19 @@ class ThumbnailWorker sidekiq_options queue: :thumbnails, retry: 3, backtrace: true def perform(username, path) + site = Site[username: username] + img_list = Magick::ImageList.new begin - img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path)) + img_list.from_blob File.read(site.files_path(path)) rescue Errno::ENOENT => e # Not found, skip return end img = img_list.first - user_thumbnails_path = File.join THUMBNAILS_PATH, username + user_thumbnails_path = site.base_thumbnails_path FileUtils.mkdir_p user_thumbnails_path FileUtils.mkdir_p File.join(user_thumbnails_path, File.dirname(path))