Implement IPFS archiving (locally). Refactor store_file.

app/api.rb

@@ -33,13 +33,7 @@ post '/api/upload' do
     end
   end
 
-  results = []
-  files.each do |file|
-    results << current_site.store_file(file[:filename], file[:tempfile])
-  end
-
-  current_site.increment_changed_count if results.include?(true)
-
+  results = current_site.store_files files
   api_success 'your file(s) have been successfully uploaded'
 end
 

app/site_files.rb

@@ -124,12 +124,7 @@ post '/site_files/upload' do
     file_upload_response "File(s) do not fit in your available space, upload cancelled."
   end
 
-  results = []
-  params[:files].each do |file|
-    results << current_site.store_file(file[:filename], file[:tempfile])
-  end
-  current_site.increment_changed_count if results.include?(true)
-
+  results = current_site.store_files params[:files]
   file_upload_response
 end
 
@@ -199,7 +194,7 @@ post %r{\/site_files\/save\/(.+)} do
     halt 'File is too large to fit in your space, it has NOT been saved. You will need to reduce the size or upgrade to a new plan.'
   end
 
-  current_site.store_file filename, tempfile
+  current_site.store_files [{filename: filename, tempfile: tempfile}]
 
   'ok'
 end

migrations/065_add_ipfs.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up {
+    DB.create_table! :archives do
+      Integer  :site_id, index: true
+      String   :ipfs_hash
+      DateTime :updated_at, index: true
+      unique [:site_id, :ipfs_hash]
+    end
+  }
+
+  down {
+    DB.drop_table :archives
+  }
+end

models/archive.rb

@@ -0,0 +1,5 @@
+class Archive < Sequel::Model
+  many_to_one :site
+  set_primary_key [:site_id, :ipfs_hash]
+  unrestrict_primary_key
+end

models/site.rb

@@ -170,6 +170,8 @@ class Site < Sequel::Model
   one_to_many :stat_locations
   one_to_many :stat_paths
 
+  one_to_many :archives
+
   def account_sites_dataset
     Site.where(Sequel.|({id: owner.id}, {parent_site_id: owner.id})).order(:parent_site_id.desc, :username)
   end
@@ -369,25 +371,26 @@ class Site < Sequel::Model
   def install_new_files
     FileUtils.mkdir_p files_path
 
+    files = []
+
     %w{index not_found}.each do |name|
       tmpfile = Tempfile.new "newinstall-#{name}"
       tmpfile.write render_template("#{name}.erb")
       tmpfile.close
-
-      store_file "#{name}.html", tmpfile, new_install: true
-      purge_cache "/#{name}.html"
-      ScreenshotWorker.perform_async values[:username], "#{name}.html"
+      files << {filename: "#{name}.html", tempfile: tmpfile}
     end
 
     tmpfile = Tempfile.new 'style.css'
     tmpfile.close
     FileUtils.cp template_file_path('style.css'), tmpfile.path
-    store_file 'style.css', tmpfile, new_install: true
+    files << {filename: 'style.css', tempfile: tmpfile}
 
     tmpfile = Tempfile.new 'cat.png'
     tmpfile.close
     FileUtils.cp template_file_path('cat.png'), tmpfile.path
-    store_file 'cat.png', tmpfile, new_install: true
+    files << {filename: 'cat.png', tempfile: tmpfile}
+
+    store_files files, new_install: true
   end
 
   def get_file(path)
@@ -554,71 +557,26 @@ class Site < Sequel::Model
     PurgeCacheWorker.perform_async payload
   end
 
-  def store_file(path, uploaded, opts={})
-    relative_path = scrubbed_path path
-    path = files_path path
-    pathname = Pathname(path)
+  def add_to_ipfs
+    line = Cocaine::CommandLine.new('ipfs', 'add -r :path')
+    response = line.run path: files_path
+    ipfs_hash = response.split("\n").last.split(' ')[1]
+    ipfs_hash
+  end
 
-    site_file = site_files_dataset.where(path: relative_path).first
+  def archive!
+    #if ENV["RACK_ENV"] == 'test'
+    #  ipfs_hash = "QmcKi2ae3uGb1kBg1yBpsuwoVqfmcByNdMiZ2pukxyLWD8"
+    #else
+    #end
 
-    uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest
-
-    if site_file && site_file.sha1_hash == uploaded_sha1
-      return false
+    archive = archives_dataset.where(ipfs_hash: ipfs_hash).first
+    if archive
+      archive.updated_at = Time.now
+      archive.save_changes
+    else
+      add_archive ipfs_hash: ipfs_hash, updated_at: Time.now
     end
-
-    if relative_path == 'index.html' && opts[:new_install] != true
-      begin
-        new_title = Nokogiri::HTML(File.read(uploaded.path)).css('title').first.text
-      rescue NoMethodError => e
-      else
-        if new_title.length < TITLE_MAX
-          self.title = new_title
-        end
-      end
-
-      self.site_changed = true
-      self.site_updated_at = Time.now
-      self.updated_at = Time.now
-
-      save_changes(validate: false)
-    end
-
-    if pathname.extname.match HTML_REGEX
-      # SPAM and phishing checking code goes here
-    end
-
-    dirname = pathname.dirname.to_s
-
-    if !File.exists? dirname
-      FileUtils.mkdir_p dirname
-    end
-
-    uploaded_size = uploaded.size
-
-    FileUtils.cp uploaded.path, path
-    File.chmod 0640, path
-
-    site_file ||= SiteFile.new site_id: self.id, path: relative_path
-
-    site_file.set_all(
-      size: uploaded_size,
-      sha1_hash: uploaded_sha1,
-      updated_at: Time.now
-    )
-    site_file.save
-
-    purge_cache path
-
-    if pathname.extname.match HTML_REGEX
-      ScreenshotWorker.perform_async values[:username], relative_path
-    elsif pathname.extname.match IMAGE_REGEX
-      ThumbnailWorker.perform_async values[:username], relative_path
-    end
-
-    SiteChange.record self, relative_path unless opts[:new_install]
-
-    true
   end
 
   def is_directory?(path)
@@ -699,7 +657,7 @@ class Site < Sequel::Model
     tmpfile = Tempfile.new 'neocities_html_template'
     tmpfile.write render_template('index.erb')
     tmpfile.close
-    store_file path, tmpfile
+    store_files [{filename: path, tempfile: tmpfile}]
     purge_cache path
     tmpfile.unlink
   end
@@ -1116,4 +1074,89 @@ class Site < Sequel::Model
       end
     end
   end
+
+  # array of hashes: filename, tempfile, opts.
+  def store_files(files, opts={})
+    results = []
+    files.each do |file|
+      results << store_file(file[:filename], file[:tempfile], file[:opts] || opts)
+    end
+
+    if results.include? true && opts[:new_install] != true
+      self.site_changed = true
+      self.site_updated_at = Time.now
+      self.updated_at = Time.now
+      save_changes validate: false
+      increment_changed_count
+      archive!
+      #SiteChange.record self, relative_path unless opts[:new_install]
+      ArchiveWorker.perform_async self.id
+    end
+
+    results
+  end
+
+  private
+
+  def store_file(path, uploaded, opts={})
+    relative_path = scrubbed_path path
+    path = files_path path
+    pathname = Pathname(path)
+
+    site_file = site_files_dataset.where(path: relative_path).first
+
+    uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest
+
+    if site_file && site_file.sha1_hash == uploaded_sha1
+      return false
+    end
+
+    if relative_path == 'index.html'
+      begin
+        new_title = Nokogiri::HTML(File.read(uploaded.path)).css('title').first.text
+      rescue NoMethodError => e
+      else
+        if new_title.length < TITLE_MAX
+          self.title = new_title
+          save_changes validate: false
+        end
+      end
+    end
+
+    if pathname.extname.match HTML_REGEX
+      # SPAM and phishing checking code goes here
+    end
+
+    dirname = pathname.dirname.to_s
+
+    if !File.exists? dirname
+      FileUtils.mkdir_p dirname
+    end
+
+    uploaded_size = uploaded.size
+
+    FileUtils.cp uploaded.path, path
+    File.chmod 0640, path
+
+    SiteChange.record self, relative_path unless opts[:new_install]
+
+    site_file ||= SiteFile.new site_id: self.id, path: relative_path
+
+    site_file.set_all(
+      size: uploaded_size,
+      sha1_hash: uploaded_sha1,
+      updated_at: Time.now
+    )
+    site_file.save
+
+    purge_cache path
+
+    if pathname.extname.match HTML_REGEX
+      ScreenshotWorker.perform_async values[:username], relative_path
+    elsif pathname.extname.match IMAGE_REGEX
+      ThumbnailWorker.perform_async values[:username], relative_path
+    end
+    true
+  end
+
 end

tests/acceptance/site_tests.rb

@@ -21,7 +21,7 @@ describe 'site page' do
       click_button 'Post'
       @site.profile_comments.count.must_equal 1
       profile_comment = @site.profile_comments.first
-      profile_comment.actioning_site.must_equal @commenting_site
+      profile_comment.actioning_site.id.must_equal @commenting_site.id
       profile_comment.message.must_equal 'I love your site!'
     end
 

tests/api_tests.rb

@@ -89,7 +89,7 @@ describe 'api delete' do
   it 'succeeds with weird filenames' do
     create_site
     basic_authorize @user, @pass
-    @site.store_file 't$st.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    @site.store_files [{filename: 't$st.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}]
     post '/api/delete', filenames: ['t$st.jpg']
     res[:result].must_equal 'success'
 
@@ -102,7 +102,7 @@ describe 'api delete' do
   it 'fails with missing files' do
     create_site
     basic_authorize @user, @pass
-    @site.store_file 'test.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    @site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}]
     post '/api/delete', filenames: ['doesntexist.jpg']
     res[:error_type].must_equal 'missing_files'
   end
@@ -110,8 +110,8 @@ describe 'api delete' do
   it 'succeeds with valid filenames' do
     create_site
     basic_authorize @user, @pass
-    @site.store_file 'test.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
-    @site.store_file 'test2.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    @site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}]
+    @site.store_files [{filename: 'test2.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}]
     post '/api/delete', filenames: ['test.jpg', 'test2.jpg']
     res[:result].must_equal 'success'
     site_file_exists?('test.jpg').must_equal false

tests/site_file_tests.rb

@@ -97,7 +97,7 @@ describe 'site_files' do
       args = ScreenshotWorker.jobs.first['args']
       args.first.must_equal @site.username
       args.last.must_equal 'index.html'
-      @site.title.must_equal "#{@site.username}.neocities.org"
+      @site.title.must_equal "The web site of #{@site.username}"
       @site.reload
       @site.site_changed.must_equal true
       @site.title.must_equal 'Hello?'
@@ -112,7 +112,6 @@ describe 'site_files' do
       @site.reload.title.must_equal title
     end
 
-
     it 'succeeds with valid file' do
       upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
       last_response.body.must_match /successfully uploaded/i

tests/workers/archive_worker_tests.rb

@@ -0,0 +1,28 @@
+require_relative '../environment.rb'
+
+describe ArchiveWorker do
+  it 'stores an IPFS archive' do
+    return if ENV['TRAVIS']
+    site = Fabricate :site
+    ipfs_hash = site.add_to_ipfs
+    ArchiveWorker.new.perform site.id
+    site.archives.length.must_equal 1
+    archive_one = site.archives.first
+    archive_one.ipfs_hash.must_equal ipfs_hash
+    archive_one.updated_at.wont_be_nil
+
+    new_updated_at = Time.now - 500
+    archive_one.update updated_at: new_updated_at
+
+    ArchiveWorker.new.perform site.id
+    archive_one.reload.updated_at.wont_equal new_updated_at
+
+    site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}]
+    ArchiveWorker.new.perform site.id
+
+    site.reload
+    site.archives.length.must_equal 2
+    archive_two = site.archives_dataset.exclude(ipfs_hash: archive_one.ipfs_hash).first
+    archive_two.ipfs_hash.wont_be_nil
+  end
+end

workers/archive_worker.rb

@@ -0,0 +1,8 @@
+class ArchiveWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :archive, retry: 10, backtrace: true
+
+  def perform(site_id)
+    Site[site_id].archive!
+  end
+end