diff --git a/migrations/052_site_files_composite_keys.rb b/migrations/052_site_files_composite_keys.rb new file mode 100644 index 00000000..d93edce8 --- /dev/null +++ b/migrations/052_site_files_composite_keys.rb @@ -0,0 +1,30 @@ +Sequel.migration do + up { + DB.drop_table :site_files + + DB.create_table! :site_files do + Integer :site_id + String :path + Bigint :size + String :sha1_hash + Boolean :is_directory, default: false + DateTime :created_at + DateTime :updated_at + primary_key [:site_id, :path], :name => :site_files_pk + end + } + + down { + DB.drop_table :site_files + + DB.create_table! :site_files do + Integer :site_id, index: true + String :path + Bigint :size + String :sha1_hash + Boolean :is_directory, default: false + DateTime :created_at + DateTime :updated_at + end + } +end \ No newline at end of file diff --git a/models/site.rb b/models/site.rb index d28b43ec..21db0433 100644 --- a/models/site.rb +++ b/models/site.rb @@ -144,6 +144,8 @@ class Site < Sequel::Model many_to_one :parent, :key => :parent_site_id, :class => self one_to_many :children, :key => :parent_site_id, :class => self + one_to_many :site_files + def account_sites_dataset Site.where(Sequel.|({id: owner.id}, {parent_site_id: owner.id})).order(:parent_site_id.desc, :username) end @@ -453,8 +455,11 @@ class Site < Sequel::Model relative_path = scrubbed_path path path = files_path path - if File.exist?(path) && - Digest::SHA1.file(path).digest == Digest::SHA1.file(uploaded.path).digest + site_file = site_files_dataset.where(path: relative_path).first + + uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest + + if site_file && site_file.sha1_hash == uploaded_sha1 return false end @@ -504,9 +509,20 @@ class Site < Sequel::Model FileUtils.mkdir_p dirname end + uploaded_size = uploaded.size + FileUtils.mv uploaded.path, path File.chmod 0640, path + site_file ||= SiteFile.new site_id: self.id, path: relative_path + + site_file.set_all( + size: uploaded_size, + sha1_hash: uploaded_sha1, + updated_at: Time.now + ) + site_file.save + purge_cache path ext = File.extname(path).gsub(/^./, '') @@ -581,6 +597,7 @@ class Site < Sequel::Model path = path[1..path.length] if path[0] == '/' + site_files_dataset.where(path: path).delete SiteChangeFile.filter(site_id: self.id, filename: path).delete true diff --git a/models/site_file.rb b/models/site_file.rb new file mode 100644 index 00000000..7e714808 --- /dev/null +++ b/models/site_file.rb @@ -0,0 +1,6 @@ +class SiteFile < Sequel::Model + + unrestrict_primary_key + plugin :update_primary_key + many_to_one :site +end \ No newline at end of file diff --git a/tests/files/img/test.jpg b/tests/files/img/test.jpg new file mode 100644 index 00000000..ff7f01a5 Binary files /dev/null and b/tests/files/img/test.jpg differ diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index afb06acf..76e0a2ed 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -1,62 +1,88 @@ require_relative './environment.rb' +include Rack::Test::Methods + +def app + Sinatra::Application +end + +def upload(hash) + post '/site_files/upload', hash.merge(csrf_token: 'abcd'), {'rack.session' => { 'id' => @site.id, '_csrf_token' => 'abcd' }} +end + +def delete_file(hash) + post '/site_files/delete', hash.merge(csrf_token: 'abcd'), {'rack.session' => { 'id' => @site.id, '_csrf_token' => 'abcd' }} +end + describe 'site_files' do + before do + @site = Fabricate :site + ThumbnailWorker.jobs.clear + PurgeCacheWorker.jobs.clear + ScreenshotWorker.jobs.clear + end + + describe 'delete' do + it 'works' do + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + file_path = @site.files_path 'test.jpg' + File.exists?(file_path).must_equal true + delete_file filename: 'test.jpg' + File.exists?(file_path).must_equal false + SiteFile[site_id: @site.id, path: 'test.jpg'].must_be_nil + end + end + describe 'upload' do it 'succeeds with index.html file' do - site = Fabricate :site - site.site_changed.must_equal false - PurgeCacheWorker.jobs.clear - ScreenshotWorker.jobs.clear - - post '/site_files/upload', { - 'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html'), - 'csrf_token' => 'abcd' - }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }} + @site.site_changed.must_equal false + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html') last_response.body.must_match /successfully uploaded/i - File.exists?(site.files_path('index.html')).must_equal true + File.exists?(@site.files_path('index.html')).must_equal true args = ScreenshotWorker.jobs.first['args'] - args.first.must_equal site.username + args.first.must_equal @site.username args.last.must_equal 'index.html' - site.reload.site_changed.must_equal true + @site.reload.site_changed.must_equal true end it 'succeeds with valid file' do - site = Fabricate :site - PurgeCacheWorker.jobs.clear - ThumbnailWorker.jobs.clear - post '/site_files/upload', { - 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg'), - 'csrf_token' => 'abcd' - }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }} + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') last_response.body.must_match /successfully uploaded/i - File.exists?(site.files_path('test.jpg')).must_equal true + File.exists?(@site.files_path('test.jpg')).must_equal true queue_args = PurgeCacheWorker.jobs.first['args'].first - queue_args['site'].must_equal site.username + queue_args['site'].must_equal @site.username queue_args['path'].must_equal '/test.jpg' ThumbnailWorker.jobs.length.must_equal 1 ThumbnailWorker.drain Site::THUMBNAIL_RESOLUTIONS.each do |resolution| - File.exists?(site.thumbnail_path('test.jpg', resolution)).must_equal true + File.exists?(@site.thumbnail_path('test.jpg', resolution)).must_equal true end - site.site_changed.must_equal false + @site.site_changed.must_equal false + end + + it 'overwrites existing file with new file' do + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + last_response.body.must_match /successfully uploaded/i + digest = @site.reload.site_files.first.sha1_hash + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/img/test.jpg', 'image/jpeg') + last_response.body.must_match /successfully uploaded/i + @site.reload.changed_count.must_equal 2 + @site.site_files.count.must_equal 1 + digest.wont_equal @site.reload.site_files.first.sha1_hash end it 'works with directory path' do - site = Fabricate :site - ThumbnailWorker.jobs.clear - PurgeCacheWorker.jobs.clear - post '/site_files/upload', { + upload( 'dir' => 'derpie/derptest', - 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg'), - 'csrf_token' => 'abcd' - }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }} + 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + ) last_response.body.must_match /successfully uploaded/i - File.exists?(site.files_path('derpie/derptest/test.jpg')).must_equal true + File.exists?(@site.files_path('derpie/derptest/test.jpg')).must_equal true PurgeCacheWorker.jobs.length.must_equal 1 queue_args = PurgeCacheWorker.jobs.first['args'].first @@ -66,11 +92,28 @@ describe 'site_files' do ThumbnailWorker.drain Site::THUMBNAIL_RESOLUTIONS.each do |resolution| - File.exists?(site.thumbnail_path('derpie/derptest/test.jpg', resolution)).must_equal true - site.thumbnail_url('derpie/derptest/test.jpg', resolution).must_equal( - File.join "#{Site::THUMBNAILS_URL_ROOT}", site.username, "/derpie/derptest/test.jpg.#{resolution}.jpg" + File.exists?(@site.thumbnail_path('derpie/derptest/test.jpg', resolution)).must_equal true + @site.thumbnail_url('derpie/derptest/test.jpg', resolution).must_equal( + File.join "#{Site::THUMBNAILS_URL_ROOT}", @site.username, "/derpie/derptest/test.jpg.#{resolution}.jpg" ) end end + + it 'does not store new file if hash matches' do + upload( + 'dir' => 'derpie/derptest', + 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + ) + @site.reload.changed_count.must_equal 1 + + upload( + 'dir' => 'derpie/derptest', + 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + ) + @site.reload.changed_count.must_equal 1 + + upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html') + @site.reload.changed_count.must_equal 2 + end end end \ No newline at end of file