start tracking files in our database

migrations/052_site_files_composite_keys.rb

@@ -0,0 +1,30 @@
+Sequel.migration do
+  up {
+    DB.drop_table :site_files
+
+    DB.create_table! :site_files do
+      Integer  :site_id
+      String   :path
+      Bigint   :size
+      String   :sha1_hash
+      Boolean  :is_directory, default: false
+      DateTime :created_at
+      DateTime :updated_at
+      primary_key [:site_id, :path], :name => :site_files_pk
+    end
+  }
+
+  down {
+    DB.drop_table :site_files
+
+    DB.create_table! :site_files do
+      Integer  :site_id, index: true
+      String   :path
+      Bigint   :size
+      String   :sha1_hash
+      Boolean  :is_directory, default: false
+      DateTime :created_at
+      DateTime :updated_at
+    end
+  }
+end

models/site.rb

@@ -144,6 +144,8 @@ class Site < Sequel::Model
   many_to_one :parent, :key => :parent_site_id, :class => self
   one_to_many :children, :key => :parent_site_id, :class => self
 
+  one_to_many :site_files
+
   def account_sites_dataset
     Site.where(Sequel.|({id: owner.id}, {parent_site_id: owner.id})).order(:parent_site_id.desc, :username)
   end
@@ -453,8 +455,11 @@ class Site < Sequel::Model
     relative_path = scrubbed_path path
     path = files_path path
 
-    if File.exist?(path) &&
+    site_file = site_files_dataset.where(path: relative_path).first
-       Digest::SHA1.file(path).digest == Digest::SHA1.file(uploaded.path).digest
+
+    uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest
+
+    if site_file && site_file.sha1_hash == uploaded_sha1
       return false
     end
 
@@ -504,9 +509,20 @@ class Site < Sequel::Model
       FileUtils.mkdir_p dirname
     end
 
+    uploaded_size = uploaded.size
+
     FileUtils.mv uploaded.path, path
     File.chmod 0640, path
 
+    site_file ||= SiteFile.new site_id: self.id, path: relative_path
+
+    site_file.set_all(
+      size: uploaded_size,
+      sha1_hash: uploaded_sha1,
+      updated_at: Time.now
+    )
+    site_file.save
+
     purge_cache path
 
     ext = File.extname(path).gsub(/^./, '')
@@ -581,6 +597,7 @@ class Site < Sequel::Model
 
     path = path[1..path.length] if path[0] == '/'
 
+    site_files_dataset.where(path: path).delete
     SiteChangeFile.filter(site_id: self.id, filename: path).delete
 
     true

models/site_file.rb

@@ -0,0 +1,6 @@
+class SiteFile < Sequel::Model
+
+  unrestrict_primary_key
+  plugin :update_primary_key
+  many_to_one :site
+end

tests/site_file_tests.rb

@@ -1,62 +1,88 @@
 require_relative './environment.rb'
 
+include Rack::Test::Methods
+
+def app
+  Sinatra::Application
+end
+
+def upload(hash)
+  post '/site_files/upload', hash.merge(csrf_token: 'abcd'), {'rack.session' => { 'id' => @site.id, '_csrf_token' => 'abcd' }}
+end
+
+def delete_file(hash)
+  post '/site_files/delete', hash.merge(csrf_token: 'abcd'), {'rack.session' => { 'id' => @site.id, '_csrf_token' => 'abcd' }}
+end
+
 describe 'site_files' do
-  describe 'upload' do
+  before do
-    it 'succeeds with index.html file' do
+    @site = Fabricate :site
-      site = Fabricate :site
+    ThumbnailWorker.jobs.clear
-      site.site_changed.must_equal false
     PurgeCacheWorker.jobs.clear
     ScreenshotWorker.jobs.clear
+  end
 
-      post '/site_files/upload', {
+  describe 'delete' do
-        'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html'),
+    it 'works' do
-        'csrf_token' => 'abcd'
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
-      }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }}
+      file_path = @site.files_path 'test.jpg'
+      File.exists?(file_path).must_equal true
+      delete_file filename: 'test.jpg'
+      File.exists?(file_path).must_equal false
+      SiteFile[site_id: @site.id, path: 'test.jpg'].must_be_nil
+    end
+  end
+
+  describe 'upload' do
+    it 'succeeds with index.html file' do
+      @site.site_changed.must_equal false
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html')
       last_response.body.must_match /successfully uploaded/i
-      File.exists?(site.files_path('index.html')).must_equal true
+      File.exists?(@site.files_path('index.html')).must_equal true
 
       args = ScreenshotWorker.jobs.first['args']
-      args.first.must_equal site.username
+      args.first.must_equal @site.username
       args.last.must_equal 'index.html'
-      site.reload.site_changed.must_equal true
+      @site.reload.site_changed.must_equal true
     end
 
     it 'succeeds with valid file' do
-      site = Fabricate :site
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
-      PurgeCacheWorker.jobs.clear
-      ThumbnailWorker.jobs.clear
-      post '/site_files/upload', {
-        'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg'),
-        'csrf_token' => 'abcd'
-      }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }}
       last_response.body.must_match /successfully uploaded/i
-      File.exists?(site.files_path('test.jpg')).must_equal true
+      File.exists?(@site.files_path('test.jpg')).must_equal true
 
       queue_args = PurgeCacheWorker.jobs.first['args'].first
-      queue_args['site'].must_equal site.username
+      queue_args['site'].must_equal @site.username
       queue_args['path'].must_equal '/test.jpg'
 
       ThumbnailWorker.jobs.length.must_equal 1
       ThumbnailWorker.drain
 
       Site::THUMBNAIL_RESOLUTIONS.each do |resolution|
-        File.exists?(site.thumbnail_path('test.jpg', resolution)).must_equal true
+        File.exists?(@site.thumbnail_path('test.jpg', resolution)).must_equal true
       end
 
-      site.site_changed.must_equal false
+      @site.site_changed.must_equal false
+    end
+
+    it 'overwrites existing file with new file' do
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+      last_response.body.must_match /successfully uploaded/i
+      digest = @site.reload.site_files.first.sha1_hash
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/img/test.jpg', 'image/jpeg')
+      last_response.body.must_match /successfully uploaded/i
+      @site.reload.changed_count.must_equal 2
+      @site.site_files.count.must_equal 1
+      digest.wont_equal @site.reload.site_files.first.sha1_hash
     end
 
     it 'works with directory path' do
-      site = Fabricate :site
+      upload(
-      ThumbnailWorker.jobs.clear
-      PurgeCacheWorker.jobs.clear
-      post '/site_files/upload', {
         'dir' => 'derpie/derptest',
-        'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg'),
+        'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
-        'csrf_token' => 'abcd'
+      )
-      }, {'rack.session' => { 'id' => site.id, '_csrf_token' => 'abcd' }}
       last_response.body.must_match /successfully uploaded/i
-      File.exists?(site.files_path('derpie/derptest/test.jpg')).must_equal true
+      File.exists?(@site.files_path('derpie/derptest/test.jpg')).must_equal true
 
       PurgeCacheWorker.jobs.length.must_equal 1
       queue_args = PurgeCacheWorker.jobs.first['args'].first
@@ -66,11 +92,28 @@ describe 'site_files' do
       ThumbnailWorker.drain
 
       Site::THUMBNAIL_RESOLUTIONS.each do |resolution|
-        File.exists?(site.thumbnail_path('derpie/derptest/test.jpg', resolution)).must_equal true
+        File.exists?(@site.thumbnail_path('derpie/derptest/test.jpg', resolution)).must_equal true
-        site.thumbnail_url('derpie/derptest/test.jpg', resolution).must_equal(
+        @site.thumbnail_url('derpie/derptest/test.jpg', resolution).must_equal(
-          File.join "#{Site::THUMBNAILS_URL_ROOT}", site.username, "/derpie/derptest/test.jpg.#{resolution}.jpg"
+          File.join "#{Site::THUMBNAILS_URL_ROOT}", @site.username, "/derpie/derptest/test.jpg.#{resolution}.jpg"
         )
       end
     end
+
+    it 'does not store new file if hash matches' do
+      upload(
+        'dir' => 'derpie/derptest',
+        'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+      )
+      @site.reload.changed_count.must_equal 1
+
+      upload(
+        'dir' => 'derpie/derptest',
+        'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+      )
+      @site.reload.changed_count.must_equal 1
+
+      upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html')
+      @site.reload.changed_count.must_equal 2
+    end
   end
 end