From 985a2f8b60d4e6f9cb20e738ab8ffa2c1ca1269c Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Wed, 26 Aug 2015 16:23:21 -0700 Subject: [PATCH] purge for delete and purge for refreshing --- app/settings.rb | 2 +- models/site.rb | 20 +++++- tests/site_file_tests.rb | 23 +++++++ .../delete_cache_order_worker_tests.rb | 21 ++++++ tests/workers/delete_cache_worker_tests.rb | 64 +++++++++++++++++++ workers/delete_cache_order_worker.rb | 23 +++++++ workers/delete_cache_worker.rb | 34 ++++++++++ 7 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/workers/delete_cache_order_worker_tests.rb create mode 100644 tests/workers/delete_cache_worker_tests.rb create mode 100644 workers/delete_cache_order_worker.rb create mode 100644 workers/delete_cache_worker.rb diff --git a/app/settings.rb b/app/settings.rb index 2d91c3e1..796ca220 100644 --- a/app/settings.rb +++ b/app/settings.rb @@ -196,7 +196,7 @@ post '/settings/:username/change_name' do } old_file_paths.each do |file_path| - @site.purge_cache file_path + @site.delete_cache file_path end flash[:success] = "Site/user name has been changed. You will need to use this name to login, don't forget it." diff --git a/models/site.rb b/models/site.rb index f7df9aa1..4fafb54b 100644 --- a/models/site.rb +++ b/models/site.rb @@ -458,7 +458,7 @@ class Site < Sequel::Model } file_list.each do |path| - purge_cache path + delete_cache path end end @@ -601,6 +601,22 @@ class Site < Sequel::Model end end + # TODO DRY this up + + def delete_cache(path) + relative_path = path.gsub base_files_path, '' + + DeleteCacheOrderWorker.perform_async username, relative_path + + # We gotta flush the dirname too if it's an index file. + if relative_path != '' && relative_path.match(/\/$|index\.html?$/i) + purge_file_path = Pathname(relative_path).dirname.to_s + + DeleteCacheOrderWorker.perform_async username, '/?surf=1' if purge_file_path == '/' + DeleteCacheOrderWorker.perform_async username, purge_file_path + end + end + Rye::Cmd.add_command :ipfs, nil, 'add', :r def add_to_ipfs @@ -1182,7 +1198,7 @@ class Site < Sequel::Model rescue Errno::ENOENT end - purge_cache path + delete_cache path ext = File.extname(path).gsub(/^./, '') diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index 28eb95b7..94e7c991 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -24,16 +24,39 @@ describe 'site_files' do end describe 'delete' do + before do + DeleteCacheWorker.jobs.clear + DeleteCacheOrderWorker.jobs.clear + end + it 'works' do uploaded_file = Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg') upload 'files[]' => uploaded_file + + PurgeCacheOrderWorker.jobs.clear + @site.reload.space_used.must_equal uploaded_file.size file_path = @site.files_path 'test.jpg' File.exists?(file_path).must_equal true delete_file filename: 'test.jpg' + File.exists?(file_path).must_equal false SiteFile[site_id: @site.id, path: 'test.jpg'].must_be_nil @site.reload.space_used.must_equal 0 + + PurgeCacheOrderWorker.jobs.length.must_equal 0 + DeleteCacheOrderWorker.jobs.length.must_equal 1 + args = DeleteCacheOrderWorker.jobs.first['args'] + args[0].must_equal @site.username + args[1].must_equal 'test.jpg' + end + + it 'flushes surf for index.html' do + uploaded_file = Rack::Test::UploadedFile.new('./tests/files/index.html', 'text/html') + upload 'files[]' => uploaded_file + delete_file filename: '/index.html' + DeleteCacheOrderWorker.jobs.length.must_equal 3 + DeleteCacheOrderWorker.jobs.collect {|j| j['args'].last}.must_equal ['/index.html', '/?surf=1', '/'] end it 'deletes a directory and all files in it' do diff --git a/tests/workers/delete_cache_order_worker_tests.rb b/tests/workers/delete_cache_order_worker_tests.rb new file mode 100644 index 00000000..9810fcd8 --- /dev/null +++ b/tests/workers/delete_cache_order_worker_tests.rb @@ -0,0 +1,21 @@ +require_relative '../environment.rb' + +describe DeleteCacheWorker do + before do + PurgeCacheOrderWorker.jobs.clear + PurgeCacheWorker.jobs.clear + end + + it 'queues up purges' do + DeleteCacheOrderWorker.new.perform('kyledrake', '/test.jpg') + + job_one_args = DeleteCacheWorker.jobs.first['args'] + job_two_args = DeleteCacheWorker.jobs.last['args'] + job_one_args[0].must_equal '10.0.0.1' + job_one_args[1].must_equal 'kyledrake' + job_one_args[2].must_equal '/test.jpg' + job_two_args[0].must_equal '10.0.0.2' + job_two_args[1].must_equal 'kyledrake' + job_two_args[2].must_equal '/test.jpg' + end +end diff --git a/tests/workers/delete_cache_worker_tests.rb b/tests/workers/delete_cache_worker_tests.rb new file mode 100644 index 00000000..70f6f690 --- /dev/null +++ b/tests/workers/delete_cache_worker_tests.rb @@ -0,0 +1,64 @@ +require_relative '../environment.rb' + +describe DeleteCacheWorker do + before do + @test_ip = '10.0.0.1' + end + + it 'throws exception without 200 or 404 http status' do + stub_request(:get, "http://#{@test_ip}/:cache/purge/test.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 503) + + worker = DeleteCacheWorker.new + + proc { + worker.perform @test_ip, 'kyledrake', '/test.jpg' + }.must_raise RestClient::ServiceUnavailable + end + + it 'handles 404 without exception' do + stub_request(:get, "http://#{@test_ip}/:cache/purge/test.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 404) + + worker = DeleteCacheWorker.new + worker.perform @test_ip, 'kyledrake', '/test.jpg' + end + + it 'sends a purge request' do + stub_request(:get, "http://#{@test_ip}/:cache/purge/test.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 200) + + worker = DeleteCacheWorker.new + worker.perform @test_ip, 'kyledrake', '/test.jpg' + + assert_requested :get, "http://#{@test_ip}/:cache/purge/test.jpg" + end + + it 'handles spaces correctly' do + stub_request(:get, "http://#{@test_ip}/:cache/purge/te st.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 200) + + url = Addressable::URI.encode_component( + "http://#{@test_ip}/:cache/purge/te st.jpg", + Addressable::URI::CharacterClasses::QUERY + ) + + worker = DeleteCacheWorker.new + worker.perform @test_ip, 'kyledrake', '/te st.jpg' + + assert_requested :get, url + end + + it 'works without forward slash' do + stub_request(:get, "http://#{@test_ip}/:cache/purge/test.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 200) + + worker = DeleteCacheWorker.new + worker.perform @test_ip, 'kyledrake', 'test.jpg' + end +end diff --git a/workers/delete_cache_order_worker.rb b/workers/delete_cache_order_worker.rb new file mode 100644 index 00000000..0e19b13c --- /dev/null +++ b/workers/delete_cache_order_worker.rb @@ -0,0 +1,23 @@ +class DeleteCacheOrderWorker + include Sidekiq::Worker + sidekiq_options queue: :deletecacheorder, retry: 1000, backtrace: true, average_scheduled_poll_interval: 1 + + sidekiq_retry_in do |count| + return 10 if count < 10 + 180 + end + + RESOLVER = Dnsruby::Resolver.new + + def perform(username, path) + if ENV['RACK_ENV'] == 'test' + proxy_ips = ['10.0.0.1', '10.0.0.2'] + else + proxy_ips = RESOLVER.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s} + end + + proxy_ips.each do |proxy_ip| + DeleteCacheWorker.perform_async proxy_ip, username, path + end + end +end diff --git a/workers/delete_cache_worker.rb b/workers/delete_cache_worker.rb new file mode 100644 index 00000000..3b0dc35e --- /dev/null +++ b/workers/delete_cache_worker.rb @@ -0,0 +1,34 @@ +require 'open-uri' + +# PurgeCacheWorker refreshes the cache, this actually deletes it. +# This is because when the file is 404ing the PurgeCacheWorker +# will just sit on the stale cache, even though it's not supposed to. +# It's some nginx bug. I'm not going to deal with it. + +class DeleteCacheWorker + HTTP_TIMEOUT = 5 + include Sidekiq::Worker + sidekiq_options queue: :deletecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1 + + sidekiq_retry_in do |count| + return 10 if count < 10 + 180 + end + + def perform(proxy_ip, username, path) + # Must always have a forward slash + path = '/' + path if path[0] != '/' + + url = Addressable::URI.encode_component( + "http://#{proxy_ip}/:cache/purge#{path}", + Addressable::URI::CharacterClasses::QUERY + ) + begin + RestClient::Request.execute method: :get, url: url, timeout: HTTP_TIMEOUT, headers: { + host: URI::encode("#{username}.neocities.org") + } + rescue RestClient::ResourceNotFound + rescue RestClient::Forbidden + end + end +end