From 8424cc02e8fc09f4447884f858feb070cc3e5983 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Thu, 21 May 2015 23:10:59 -0700 Subject: [PATCH] Implement IPFS archiving (locally). Refactor store_file. --- app/api.rb | 8 +- app/site_files.rb | 9 +- migrations/065_add_ipfs.rb | 14 ++ models/archive.rb | 5 + models/site.rb | 181 ++++++++++++++++---------- tests/acceptance/site_tests.rb | 2 +- tests/api_tests.rb | 8 +- tests/site_file_tests.rb | 3 +- tests/workers/archive_worker_tests.rb | 28 ++++ workers/archive_worker.rb | 8 ++ 10 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 migrations/065_add_ipfs.rb create mode 100644 models/archive.rb create mode 100644 tests/workers/archive_worker_tests.rb create mode 100644 workers/archive_worker.rb diff --git a/app/api.rb b/app/api.rb index 9a10751c..1143cee3 100644 --- a/app/api.rb +++ b/app/api.rb @@ -33,13 +33,7 @@ post '/api/upload' do end end - results = [] - files.each do |file| - results << current_site.store_file(file[:filename], file[:tempfile]) - end - - current_site.increment_changed_count if results.include?(true) - + results = current_site.store_files files api_success 'your file(s) have been successfully uploaded' end diff --git a/app/site_files.rb b/app/site_files.rb index baa52c79..80ca6130 100644 --- a/app/site_files.rb +++ b/app/site_files.rb @@ -124,12 +124,7 @@ post '/site_files/upload' do file_upload_response "File(s) do not fit in your available space, upload cancelled." end - results = [] - params[:files].each do |file| - results << current_site.store_file(file[:filename], file[:tempfile]) - end - current_site.increment_changed_count if results.include?(true) - + results = current_site.store_files params[:files] file_upload_response end @@ -199,7 +194,7 @@ post %r{\/site_files\/save\/(.+)} do halt 'File is too large to fit in your space, it has NOT been saved. You will need to reduce the size or upgrade to a new plan.' end - current_site.store_file filename, tempfile + current_site.store_files [{filename: filename, tempfile: tempfile}] 'ok' end diff --git a/migrations/065_add_ipfs.rb b/migrations/065_add_ipfs.rb new file mode 100644 index 00000000..68c285d2 --- /dev/null +++ b/migrations/065_add_ipfs.rb @@ -0,0 +1,14 @@ +Sequel.migration do + up { + DB.create_table! :archives do + Integer :site_id, index: true + String :ipfs_hash + DateTime :updated_at, index: true + unique [:site_id, :ipfs_hash] + end + } + + down { + DB.drop_table :archives + } +end diff --git a/models/archive.rb b/models/archive.rb new file mode 100644 index 00000000..0df7e093 --- /dev/null +++ b/models/archive.rb @@ -0,0 +1,5 @@ +class Archive < Sequel::Model + many_to_one :site + set_primary_key [:site_id, :ipfs_hash] + unrestrict_primary_key +end diff --git a/models/site.rb b/models/site.rb index f8dc9043..faf81cff 100644 --- a/models/site.rb +++ b/models/site.rb @@ -170,6 +170,8 @@ class Site < Sequel::Model one_to_many :stat_locations one_to_many :stat_paths + one_to_many :archives + def account_sites_dataset Site.where(Sequel.|({id: owner.id}, {parent_site_id: owner.id})).order(:parent_site_id.desc, :username) end @@ -369,25 +371,26 @@ class Site < Sequel::Model def install_new_files FileUtils.mkdir_p files_path + files = [] + %w{index not_found}.each do |name| tmpfile = Tempfile.new "newinstall-#{name}" tmpfile.write render_template("#{name}.erb") tmpfile.close - - store_file "#{name}.html", tmpfile, new_install: true - purge_cache "/#{name}.html" - ScreenshotWorker.perform_async values[:username], "#{name}.html" + files << {filename: "#{name}.html", tempfile: tmpfile} end tmpfile = Tempfile.new 'style.css' tmpfile.close FileUtils.cp template_file_path('style.css'), tmpfile.path - store_file 'style.css', tmpfile, new_install: true + files << {filename: 'style.css', tempfile: tmpfile} tmpfile = Tempfile.new 'cat.png' tmpfile.close FileUtils.cp template_file_path('cat.png'), tmpfile.path - store_file 'cat.png', tmpfile, new_install: true + files << {filename: 'cat.png', tempfile: tmpfile} + + store_files files, new_install: true end def get_file(path) @@ -554,71 +557,26 @@ class Site < Sequel::Model PurgeCacheWorker.perform_async payload end - def store_file(path, uploaded, opts={}) - relative_path = scrubbed_path path - path = files_path path - pathname = Pathname(path) + def add_to_ipfs + line = Cocaine::CommandLine.new('ipfs', 'add -r :path') + response = line.run path: files_path + ipfs_hash = response.split("\n").last.split(' ')[1] + ipfs_hash + end - site_file = site_files_dataset.where(path: relative_path).first + def archive! + #if ENV["RACK_ENV"] == 'test' + # ipfs_hash = "QmcKi2ae3uGb1kBg1yBpsuwoVqfmcByNdMiZ2pukxyLWD8" + #else + #end - uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest - - if site_file && site_file.sha1_hash == uploaded_sha1 - return false + archive = archives_dataset.where(ipfs_hash: ipfs_hash).first + if archive + archive.updated_at = Time.now + archive.save_changes + else + add_archive ipfs_hash: ipfs_hash, updated_at: Time.now end - - if relative_path == 'index.html' && opts[:new_install] != true - begin - new_title = Nokogiri::HTML(File.read(uploaded.path)).css('title').first.text - rescue NoMethodError => e - else - if new_title.length < TITLE_MAX - self.title = new_title - end - end - - self.site_changed = true - self.site_updated_at = Time.now - self.updated_at = Time.now - - save_changes(validate: false) - end - - if pathname.extname.match HTML_REGEX - # SPAM and phishing checking code goes here - end - - dirname = pathname.dirname.to_s - - if !File.exists? dirname - FileUtils.mkdir_p dirname - end - - uploaded_size = uploaded.size - - FileUtils.cp uploaded.path, path - File.chmod 0640, path - - site_file ||= SiteFile.new site_id: self.id, path: relative_path - - site_file.set_all( - size: uploaded_size, - sha1_hash: uploaded_sha1, - updated_at: Time.now - ) - site_file.save - - purge_cache path - - if pathname.extname.match HTML_REGEX - ScreenshotWorker.perform_async values[:username], relative_path - elsif pathname.extname.match IMAGE_REGEX - ThumbnailWorker.perform_async values[:username], relative_path - end - - SiteChange.record self, relative_path unless opts[:new_install] - - true end def is_directory?(path) @@ -699,7 +657,7 @@ class Site < Sequel::Model tmpfile = Tempfile.new 'neocities_html_template' tmpfile.write render_template('index.erb') tmpfile.close - store_file path, tmpfile + store_files [{filename: path, tempfile: tmpfile}] purge_cache path tmpfile.unlink end @@ -1116,4 +1074,89 @@ class Site < Sequel::Model end end end + + # array of hashes: filename, tempfile, opts. + def store_files(files, opts={}) + results = [] + files.each do |file| + results << store_file(file[:filename], file[:tempfile], file[:opts] || opts) + end + + if results.include? true && opts[:new_install] != true + self.site_changed = true + self.site_updated_at = Time.now + self.updated_at = Time.now + save_changes validate: false + increment_changed_count + archive! + #SiteChange.record self, relative_path unless opts[:new_install] + ArchiveWorker.perform_async self.id + end + + results + end + + private + + def store_file(path, uploaded, opts={}) + relative_path = scrubbed_path path + path = files_path path + pathname = Pathname(path) + + site_file = site_files_dataset.where(path: relative_path).first + + uploaded_sha1 = Digest::SHA1.file(uploaded.path).hexdigest + + if site_file && site_file.sha1_hash == uploaded_sha1 + return false + end + + if relative_path == 'index.html' + begin + new_title = Nokogiri::HTML(File.read(uploaded.path)).css('title').first.text + rescue NoMethodError => e + else + if new_title.length < TITLE_MAX + self.title = new_title + save_changes validate: false + end + end + end + + if pathname.extname.match HTML_REGEX + # SPAM and phishing checking code goes here + end + + dirname = pathname.dirname.to_s + + if !File.exists? dirname + FileUtils.mkdir_p dirname + end + + uploaded_size = uploaded.size + + FileUtils.cp uploaded.path, path + File.chmod 0640, path + + SiteChange.record self, relative_path unless opts[:new_install] + + site_file ||= SiteFile.new site_id: self.id, path: relative_path + + site_file.set_all( + size: uploaded_size, + sha1_hash: uploaded_sha1, + updated_at: Time.now + ) + site_file.save + + purge_cache path + + if pathname.extname.match HTML_REGEX + ScreenshotWorker.perform_async values[:username], relative_path + elsif pathname.extname.match IMAGE_REGEX + ThumbnailWorker.perform_async values[:username], relative_path + end + true + end + end diff --git a/tests/acceptance/site_tests.rb b/tests/acceptance/site_tests.rb index fd517412..60a75261 100644 --- a/tests/acceptance/site_tests.rb +++ b/tests/acceptance/site_tests.rb @@ -21,7 +21,7 @@ describe 'site page' do click_button 'Post' @site.profile_comments.count.must_equal 1 profile_comment = @site.profile_comments.first - profile_comment.actioning_site.must_equal @commenting_site + profile_comment.actioning_site.id.must_equal @commenting_site.id profile_comment.message.must_equal 'I love your site!' end diff --git a/tests/api_tests.rb b/tests/api_tests.rb index 6eaedc66..ca368d75 100644 --- a/tests/api_tests.rb +++ b/tests/api_tests.rb @@ -89,7 +89,7 @@ describe 'api delete' do it 'succeeds with weird filenames' do create_site basic_authorize @user, @pass - @site.store_file 't$st.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + @site.store_files [{filename: 't$st.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}] post '/api/delete', filenames: ['t$st.jpg'] res[:result].must_equal 'success' @@ -102,7 +102,7 @@ describe 'api delete' do it 'fails with missing files' do create_site basic_authorize @user, @pass - @site.store_file 'test.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + @site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}] post '/api/delete', filenames: ['doesntexist.jpg'] res[:error_type].must_equal 'missing_files' end @@ -110,8 +110,8 @@ describe 'api delete' do it 'succeeds with valid filenames' do create_site basic_authorize @user, @pass - @site.store_file 'test.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') - @site.store_file 'test2.jpg', Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') + @site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}] + @site.store_files [{filename: 'test2.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}] post '/api/delete', filenames: ['test.jpg', 'test2.jpg'] res[:result].must_equal 'success' site_file_exists?('test.jpg').must_equal false diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index cf6a1241..bd2052ab 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -97,7 +97,7 @@ describe 'site_files' do args = ScreenshotWorker.jobs.first['args'] args.first.must_equal @site.username args.last.must_equal 'index.html' - @site.title.must_equal "#{@site.username}.neocities.org" + @site.title.must_equal "The web site of #{@site.username}" @site.reload @site.site_changed.must_equal true @site.title.must_equal 'Hello?' @@ -112,7 +112,6 @@ describe 'site_files' do @site.reload.title.must_equal title end - it 'succeeds with valid file' do upload 'files[]' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') last_response.body.must_match /successfully uploaded/i diff --git a/tests/workers/archive_worker_tests.rb b/tests/workers/archive_worker_tests.rb new file mode 100644 index 00000000..90a748e2 --- /dev/null +++ b/tests/workers/archive_worker_tests.rb @@ -0,0 +1,28 @@ +require_relative '../environment.rb' + +describe ArchiveWorker do + it 'stores an IPFS archive' do + return if ENV['TRAVIS'] + site = Fabricate :site + ipfs_hash = site.add_to_ipfs + ArchiveWorker.new.perform site.id + site.archives.length.must_equal 1 + archive_one = site.archives.first + archive_one.ipfs_hash.must_equal ipfs_hash + archive_one.updated_at.wont_be_nil + + new_updated_at = Time.now - 500 + archive_one.update updated_at: new_updated_at + + ArchiveWorker.new.perform site.id + archive_one.reload.updated_at.wont_equal new_updated_at + + site.store_files [{filename: 'test.jpg', tempfile: Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')}] + ArchiveWorker.new.perform site.id + + site.reload + site.archives.length.must_equal 2 + archive_two = site.archives_dataset.exclude(ipfs_hash: archive_one.ipfs_hash).first + archive_two.ipfs_hash.wont_be_nil + end +end diff --git a/workers/archive_worker.rb b/workers/archive_worker.rb new file mode 100644 index 00000000..63df8a07 --- /dev/null +++ b/workers/archive_worker.rb @@ -0,0 +1,8 @@ +class ArchiveWorker + include Sidekiq::Worker + sidekiq_options queue: :archive, retry: 10, backtrace: true + + def perform(site_id) + Site[site_id].archive! + end +end