cache purge improvements, index path flusher

models/site.rb

@@ -574,10 +574,15 @@ class Site < Sequel::Model
   end
 
   def purge_cache(path)
-    relative_path = path.gsub(base_files_path, '')
+    relative_path = path.gsub base_files_path, ''
-    payload = {site: username, path: relative_path}
+
-    payload[:domain] = domain if !domain.empty?
+    # We gotta flush the dirname too if it's an index file.
-    PurgeCacheWorker.perform_async payload
+    if relative_path != '' && relative_path.match(/\/$|index\.html?$/i)
+      PurgeCacheOrderWorker.perform_async username, relative_path
+      PurgeCacheOrderWorker.perform_async username, Pathname(relative_path).dirname.to_s
+    else
+      PurgeCacheOrderWorker.perform_async username, relative_path
+    end
   end
 
   Rye::Cmd.add_command :ipfs, nil, 'add', :r

tests/site_file_tests.rb

@@ -18,6 +18,7 @@ describe 'site_files' do
   before do
     @site = Fabricate :site
     ThumbnailWorker.jobs.clear
+    PurgeCacheOrderWorker.jobs.clear
     PurgeCacheWorker.jobs.clear
     ScreenshotWorker.jobs.clear
   end
@@ -104,6 +105,19 @@ describe 'site_files' do
       @site.reload
       @site.site_changed.must_equal true
       @site.title.must_equal 'Hello?'
+
+      # Purge cache needs to flush / and index.html for either scenario.
+      PurgeCacheOrderWorker.jobs.length.must_equal 2
+      first_purge = PurgeCacheOrderWorker.jobs.first
+      dirname_purge = PurgeCacheOrderWorker.jobs.last
+
+      username, pathname = first_purge['args']
+      username.must_equal @site.username
+      pathname.must_equal '/index.html'
+      username, pathame = nil
+      username, pathname = dirname_purge['args']
+      username.must_equal @site.username
+      pathname.must_equal '/'
     end
 
     it 'provides the correct space used after overwriting an existing file' do
@@ -129,9 +143,9 @@ describe 'site_files' do
       last_response.body.must_match /successfully uploaded/i
       File.exists?(@site.files_path('test.jpg')).must_equal true
 
-      queue_args = PurgeCacheWorker.jobs.first['args'].first
+      username, path = PurgeCacheOrderWorker.jobs.first['args']
-      queue_args['site'].must_equal @site.username
+      username.must_equal @site.username
-      queue_args['path'].must_equal '/test.jpg'
+      path.must_equal '/test.jpg'
 
       @site.reload
       @site.space_used.wont_equal 0
@@ -183,9 +197,18 @@ describe 'site_files' do
       last_response.body.must_match /successfully uploaded/i
       File.exists?(@site.files_path('derpie/derptest/test.jpg')).must_equal true
 
-      PurgeCacheWorker.jobs.length.must_equal 1
+      PurgeCacheOrderWorker.jobs.length.must_equal 1
-      queue_args = PurgeCacheWorker.jobs.first['args'].first
+      username, path = PurgeCacheOrderWorker.jobs.first['args']
-      queue_args['path'].must_equal '/derpie/derptest/test.jpg'
+      path.must_equal '/derpie/derptest/test.jpg'
+
+      PurgeCacheOrderWorker.drain
+
+      PurgeCacheWorker.jobs.length.must_equal 2
+      ip, username, path = PurgeCacheWorker.jobs.first['args']
+      ip.must_equal '10.0.0.1'
+      username.must_equal @site.username
+      path.must_equal '/derpie/derptest/test.jpg'
+      PurgeCacheWorker.jobs.last['args'].first.must_equal '10.0.0.2'
 
       ThumbnailWorker.jobs.length.must_equal 1
       ThumbnailWorker.drain

tests/workers/purge_cache_order_worker_tests.rb

@@ -0,0 +1,21 @@
+require_relative '../environment.rb'
+
+describe PurgeCacheWorker do
+  before do
+    PurgeCacheOrderWorker.jobs.clear
+    PurgeCacheWorker.jobs.clear
+  end
+
+  it 'queues up purges' do
+    PurgeCacheOrderWorker.new.perform('kyledrake', '/test.jpg')
+
+    job_one_args = PurgeCacheWorker.jobs.first['args']
+    job_two_args = PurgeCacheWorker.jobs.last['args']
+    job_one_args[0].must_equal '10.0.0.1'
+    job_one_args[1].must_equal 'kyledrake'
+    job_one_args[2].must_equal '/test.jpg'
+    job_two_args[0].must_equal '10.0.0.2'
+    job_two_args[1].must_equal 'kyledrake'
+    job_two_args[2].must_equal '/test.jpg'
+  end
+end

tests/workers/purge_cache_worker_tests.rb

@@ -2,46 +2,38 @@ require_relative '../environment.rb'
 
 describe PurgeCacheWorker do
   before do
-    @test_ips = ['10.0.0.1', '10.0.0.2']
+    @test_ip = '10.0.0.1'
   end
 
   it 'throws exception without 200 or 404 http status' do
-    @test_ips.each do |ip|
+    stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg").
-      stub_request(:get, "https://#{ip}/:cache/purgetest.jpg").
+      with(headers: {'Host' => 'kyledrake.neocities.org'})
-        with(headers: {'Host' => 'kyledrake.neocities.org'})
+      .to_return(status: 503)
-        .to_return(status: 503)
-    end
 
     worker = PurgeCacheWorker.new
 
     proc {
-      worker.perform 'kyledrake', 'test.jpg'
+      worker.perform @test_ip, 'kyledrake', 'test.jpg'
     }.must_raise RestClient::ServiceUnavailable
   end
 
   it 'handles 404 without exception' do
-    @test_ips.each do |ip|
+    stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg").
-      stub_request(:get, "https://#{ip}/:cache/purgetest.jpg").
+      with(headers: {'Host' => 'kyledrake.neocities.org'})
-        with(headers: {'Host' => 'kyledrake.neocities.org'})
+      .to_return(status: 404)
-        .to_return(status: 404)
-    end
 
     worker = PurgeCacheWorker.new
-    worker.perform 'kyledrake', 'test.jpg'
+    worker.perform @test_ip, 'kyledrake', 'test.jpg'
   end
 
-  it 'sends a purge to each dns ip' do
+  it 'sends a purge request' do
-    @test_ips.each do |ip|
+    stub_request(:get, "http://#{@test_ip}/:cache/purgetest.jpg").
-      stub_request(:get, "https://#{ip}/:cache/purgetest.jpg").
+      with(headers: {'Host' => 'kyledrake.neocities.org'})
-        with(headers: {'Host' => 'kyledrake.neocities.org'})
+      .to_return(status: 200)
-        .to_return(status: 200)
-    end
 
     worker = PurgeCacheWorker.new
-    worker.perform 'kyledrake', 'test.jpg'
+    worker.perform @test_ip, 'kyledrake', 'test.jpg'
 
-    @test_ips.each do |ip|
+    assert_requested :get, "http://#{@test_ip}/:cache/purgetest.jpg"
-      assert_requested :get, "https://#{ip}/:cache/purgetest.jpg"
-    end
   end
 end

workers/purge_cache_order_worker.rb

@@ -0,0 +1,23 @@
+class PurgeCacheOrderWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :purgecacheorder, retry: 1000, backtrace: true, average_scheduled_poll_interval: 1
+
+  sidekiq_retry_in do |count|
+    return 10 if count < 10
+    180
+  end
+
+  RESOLVER = Dnsruby::Resolver.new
+
+  def perform(username, path)
+    if ENV['RACK_ENV'] == 'test'
+      proxy_ips = ['10.0.0.1', '10.0.0.2']
+    else
+      proxy_ips = RESOLVER.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s}
+    end
+
+    proxy_ips.each do |proxy_ip|
+      PurgeCacheWorker.perform_async proxy_ip, username, path
+    end
+  end
+end

workers/purge_cache_worker.rb

@@ -1,23 +1,17 @@
 class PurgeCacheWorker
   include Sidekiq::Worker
-  sidekiq_options queue: :purgecache, retry: 10, backtrace: true
+  sidekiq_options queue: :purgecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1
 
-  def perform(subdomain, path)
+  sidekiq_retry_in do |count|
-    res = Dnsruby::Resolver.new
+    return 10 if count < 10
+    180
+  end
 
-    if ENV['RACK_ENV'] == 'test'
+  def perform(proxy_ip, username, path)
-      proxy_ips = ['10.0.0.1', '10.0.0.2']
+    url = "http://#{proxy_ip}/:cache/purge#{path}"
-    else
+    begin
-      proxy_ips = res.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s}
+      RestClient.get(url, host: "#{username}.neocities.org")
-    end
+    rescue RestClient::ResourceNotFound
-
-    proxy_ips.each do |proxy_ip|
-      url = "http://#{proxy_ip}/:cache/purge#{path}"
-
-      begin
-        RestClient.get(url, host: "#{subdomain}.neocities.org")
-      rescue RestClient::ResourceNotFound
-      end
     end
   end
 end