From 4b87c818e7fe67d116bba7a40af5e1bd491f1cf3 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Thu, 2 Jul 2015 17:21:19 -0700 Subject: [PATCH] cache purge improvements, index path flusher --- models/site.rb | 13 +++++-- tests/site_file_tests.rb | 35 ++++++++++++++--- .../workers/purge_cache_order_worker_tests.rb | 21 ++++++++++ tests/workers/purge_cache_worker_tests.rb | 38 ++++++++----------- workers/purge_cache_order_worker.rb | 23 +++++++++++ workers/purge_cache_worker.rb | 26 +++++-------- 6 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 tests/workers/purge_cache_order_worker_tests.rb create mode 100644 workers/purge_cache_order_worker.rb diff --git a/models/site.rb b/models/site.rb index 5955646d..3da82121 100644 --- a/models/site.rb +++ b/models/site.rb @@ -574,10 +574,15 @@ class Site < Sequel::Model end def purge_cache(path) - relative_path = path.gsub(base_files_path, '') - payload = {site: username, path: relative_path} - payload[:domain] = domain if !domain.empty? - PurgeCacheWorker.perform_async payload + relative_path = path.gsub base_files_path, '' + + # We gotta flush the dirname too if it's an index file. + if relative_path != '' && relative_path.match(/\/$|index\.html?$/i) + PurgeCacheOrderWorker.perform_async username, relative_path + PurgeCacheOrderWorker.perform_async username, Pathname(relative_path).dirname.to_s + else + PurgeCacheOrderWorker.perform_async username, relative_path + end end Rye::Cmd.add_command :ipfs, nil, 'add', :r diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index 482027da..f2aee5ee 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -18,6 +18,7 @@ describe 'site_files' do before do @site = Fabricate :site ThumbnailWorker.jobs.clear + PurgeCacheOrderWorker.jobs.clear PurgeCacheWorker.jobs.clear ScreenshotWorker.jobs.clear end @@ -104,6 +105,19 @@ describe 'site_files' do @site.reload @site.site_changed.must_equal true @site.title.must_equal 'Hello?' + + # Purge cache needs to flush / and index.html for either scenario. + PurgeCacheOrderWorker.jobs.length.must_equal 2 + first_purge = PurgeCacheOrderWorker.jobs.first + dirname_purge = PurgeCacheOrderWorker.jobs.last + + username, pathname = first_purge['args'] + username.must_equal @site.username + pathname.must_equal '/index.html' + username, pathame = nil + username, pathname = dirname_purge['args'] + username.must_equal @site.username + pathname.must_equal '/' end it 'provides the correct space used after overwriting an existing file' do @@ -129,9 +143,9 @@ describe 'site_files' do last_response.body.must_match /successfully uploaded/i File.exists?(@site.files_path('test.jpg')).must_equal true - queue_args = PurgeCacheWorker.jobs.first['args'].first - queue_args['site'].must_equal @site.username - queue_args['path'].must_equal '/test.jpg' + username, path = PurgeCacheOrderWorker.jobs.first['args'] + username.must_equal @site.username + path.must_equal '/test.jpg' @site.reload @site.space_used.wont_equal 0 @@ -183,9 +197,18 @@ describe 'site_files' do last_response.body.must_match /successfully uploaded/i File.exists?(@site.files_path('derpie/derptest/test.jpg')).must_equal true - PurgeCacheWorker.jobs.length.must_equal 1 - queue_args = PurgeCacheWorker.jobs.first['args'].first - queue_args['path'].must_equal '/derpie/derptest/test.jpg' + PurgeCacheOrderWorker.jobs.length.must_equal 1 + username, path = PurgeCacheOrderWorker.jobs.first['args'] + path.must_equal '/derpie/derptest/test.jpg' + + PurgeCacheOrderWorker.drain + + PurgeCacheWorker.jobs.length.must_equal 2 + ip, username, path = PurgeCacheWorker.jobs.first['args'] + ip.must_equal '10.0.0.1' + username.must_equal @site.username + path.must_equal '/derpie/derptest/test.jpg' + PurgeCacheWorker.jobs.last['args'].first.must_equal '10.0.0.2' ThumbnailWorker.jobs.length.must_equal 1 ThumbnailWorker.drain diff --git a/tests/workers/purge_cache_order_worker_tests.rb b/tests/workers/purge_cache_order_worker_tests.rb new file mode 100644 index 00000000..39582f71 --- /dev/null +++ b/tests/workers/purge_cache_order_worker_tests.rb @@ -0,0 +1,21 @@ +require_relative '../environment.rb' + +describe PurgeCacheWorker do + before do + PurgeCacheOrderWorker.jobs.clear + PurgeCacheWorker.jobs.clear + end + + it 'queues up purges' do + PurgeCacheOrderWorker.new.perform('kyledrake', '/test.jpg') + + job_one_args = PurgeCacheWorker.jobs.first['args'] + job_two_args = PurgeCacheWorker.jobs.last['args'] + job_one_args[0].must_equal '10.0.0.1' + job_one_args[1].must_equal 'kyledrake' + job_one_args[2].must_equal '/test.jpg' + job_two_args[0].must_equal '10.0.0.2' + job_two_args[1].must_equal 'kyledrake' + job_two_args[2].must_equal '/test.jpg' + end +end diff --git a/tests/workers/purge_cache_worker_tests.rb b/tests/workers/purge_cache_worker_tests.rb index f6ce6a39..834261bb 100644 --- a/tests/workers/purge_cache_worker_tests.rb +++ b/tests/workers/purge_cache_worker_tests.rb @@ -2,46 +2,38 @@ require_relative '../environment.rb' describe PurgeCacheWorker do before do - @test_ips = ['10.0.0.1', '10.0.0.2'] + @test_ip = '10.0.0.1' end it 'throws exception without 200 or 404 http status' do - @test_ips.each do |ip| - stub_request(:get, "https://#{ip}/:cache/purgetest.jpg"). - with(headers: {'Host' => 'kyledrake.neocities.org'}) - .to_return(status: 503) - end + stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 503) worker = PurgeCacheWorker.new proc { - worker.perform 'kyledrake', 'test.jpg' + worker.perform @test_ip, 'kyledrake', 'test.jpg' }.must_raise RestClient::ServiceUnavailable end it 'handles 404 without exception' do - @test_ips.each do |ip| - stub_request(:get, "https://#{ip}/:cache/purgetest.jpg"). - with(headers: {'Host' => 'kyledrake.neocities.org'}) - .to_return(status: 404) - end + stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 404) worker = PurgeCacheWorker.new - worker.perform 'kyledrake', 'test.jpg' + worker.perform @test_ip, 'kyledrake', 'test.jpg' end - it 'sends a purge to each dns ip' do - @test_ips.each do |ip| - stub_request(:get, "https://#{ip}/:cache/purgetest.jpg"). - with(headers: {'Host' => 'kyledrake.neocities.org'}) - .to_return(status: 200) - end + it 'sends a purge request' do + stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg"). + with(headers: {'Host' => 'kyledrake.neocities.org'}) + .to_return(status: 200) worker = PurgeCacheWorker.new - worker.perform 'kyledrake', 'test.jpg' + worker.perform @test_ip, 'kyledrake', 'test.jpg' - @test_ips.each do |ip| - assert_requested :get, "https://#{ip}/:cache/purgetest.jpg" - end + assert_requested :get, "http://#{@test_ip}/:cache/purgetest.jpg" end end diff --git a/workers/purge_cache_order_worker.rb b/workers/purge_cache_order_worker.rb new file mode 100644 index 00000000..0c079eb4 --- /dev/null +++ b/workers/purge_cache_order_worker.rb @@ -0,0 +1,23 @@ +class PurgeCacheOrderWorker + include Sidekiq::Worker + sidekiq_options queue: :purgecacheorder, retry: 1000, backtrace: true, average_scheduled_poll_interval: 1 + + sidekiq_retry_in do |count| + return 10 if count < 10 + 180 + end + + RESOLVER = Dnsruby::Resolver.new + + def perform(username, path) + if ENV['RACK_ENV'] == 'test' + proxy_ips = ['10.0.0.1', '10.0.0.2'] + else + proxy_ips = RESOLVER.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s} + end + + proxy_ips.each do |proxy_ip| + PurgeCacheWorker.perform_async proxy_ip, username, path + end + end +end diff --git a/workers/purge_cache_worker.rb b/workers/purge_cache_worker.rb index 79499b9a..2d42b0b4 100644 --- a/workers/purge_cache_worker.rb +++ b/workers/purge_cache_worker.rb @@ -1,23 +1,17 @@ class PurgeCacheWorker include Sidekiq::Worker - sidekiq_options queue: :purgecache, retry: 10, backtrace: true + sidekiq_options queue: :purgecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1 - def perform(subdomain, path) - res = Dnsruby::Resolver.new + sidekiq_retry_in do |count| + return 10 if count < 10 + 180 + end - if ENV['RACK_ENV'] == 'test' - proxy_ips = ['10.0.0.1', '10.0.0.2'] - else - proxy_ips = res.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s} - end - - proxy_ips.each do |proxy_ip| - url = "http://#{proxy_ip}/:cache/purge#{path}" - - begin - RestClient.get(url, host: "#{subdomain}.neocities.org") - rescue RestClient::ResourceNotFound - end + def perform(proxy_ip, username, path) + url = "http://#{proxy_ip}/:cache/purge#{path}" + begin + RestClient.get(url, host: "#{username}.neocities.org") + rescue RestClient::ResourceNotFound end end end