- <% end %>
+ Support Us
+
\ No newline at end of file
diff --git a/views/_news.erb b/views/_news.erb
index 2641c68d..7ca1b3c5 100644
--- a/views/_news.erb
+++ b/views/_news.erb
@@ -126,26 +126,7 @@
<% if params[:activity] != 'global' %>
- <% if @page_count > 1 %>
-
- <% if @current_page != 1 %>
-
- <% end %>
-
- <% 1.upto(@page_count) do |num| %>
- <% if num == @current_page %>
- <%= num %>
- <% else %>
- <%= num %>
- <% end %>
-
- <% end %>
-
- <% if @current_page != @page_count %>
-
- <% end %>
-
diff --git a/workers/delete_cache_order_worker.rb b/workers/delete_cache_order_worker.rb
new file mode 100644
index 00000000..0e19b13c
--- /dev/null
+++ b/workers/delete_cache_order_worker.rb
@@ -0,0 +1,23 @@
+class DeleteCacheOrderWorker
+ include Sidekiq::Worker
+ sidekiq_options queue: :deletecacheorder, retry: 1000, backtrace: true, average_scheduled_poll_interval: 1
+
+ sidekiq_retry_in do |count|
+ return 10 if count < 10
+ 180
+ end
+
+ RESOLVER = Dnsruby::Resolver.new
+
+ def perform(username, path)
+ if ENV['RACK_ENV'] == 'test'
+ proxy_ips = ['10.0.0.1', '10.0.0.2']
+ else
+ proxy_ips = RESOLVER.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s}
+ end
+
+ proxy_ips.each do |proxy_ip|
+ DeleteCacheWorker.perform_async proxy_ip, username, path
+ end
+ end
+end
diff --git a/workers/delete_cache_worker.rb b/workers/delete_cache_worker.rb
new file mode 100644
index 00000000..3b0dc35e
--- /dev/null
+++ b/workers/delete_cache_worker.rb
@@ -0,0 +1,34 @@
+require 'open-uri'
+
+# PurgeCacheWorker refreshes the cache, this actually deletes it.
+# This is because when the file is 404ing the PurgeCacheWorker
+# will just sit on the stale cache, even though it's not supposed to.
+# It's some nginx bug. I'm not going to deal with it.
+
+class DeleteCacheWorker
+ HTTP_TIMEOUT = 5
+ include Sidekiq::Worker
+ sidekiq_options queue: :deletecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1
+
+ sidekiq_retry_in do |count|
+ return 10 if count < 10
+ 180
+ end
+
+ def perform(proxy_ip, username, path)
+ # Must always have a forward slash
+ path = '/' + path if path[0] != '/'
+
+ url = Addressable::URI.encode_component(
+ "http://#{proxy_ip}/:cache/purge#{path}",
+ Addressable::URI::CharacterClasses::QUERY
+ )
+ begin
+ RestClient::Request.execute method: :get, url: url, timeout: HTTP_TIMEOUT, headers: {
+ host: URI::encode("#{username}.neocities.org")
+ }
+ rescue RestClient::ResourceNotFound
+ rescue RestClient::Forbidden
+ end
+ end
+end
diff --git a/workers/email_worker.rb b/workers/email_worker.rb
index b86a3829..078f4861 100644
--- a/workers/email_worker.rb
+++ b/workers/email_worker.rb
@@ -3,7 +3,7 @@ class EmailWorker
sidekiq_options queue: :emails, retry: 10, backtrace: true
def perform(args={})
- raise 'no' if ENV['RACK_ENV'].nil? || ENV['RACK_ENV'] == 'development'
+ return if ENV['RACK_ENV'].nil? || ENV['RACK_ENV'] == 'development'
unsubscribe_token = Site.email_unsubscribe_token args['to']
if args['no_footer']
diff --git a/workers/purge_cache_worker.rb b/workers/purge_cache_worker.rb
index 836b8a81..1ce5a260 100644
--- a/workers/purge_cache_worker.rb
+++ b/workers/purge_cache_worker.rb
@@ -1,6 +1,7 @@
require 'open-uri'
class PurgeCacheWorker
+ HTTP_TIMEOUT = 5
include Sidekiq::Worker
sidekiq_options queue: :purgecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1
@@ -14,12 +15,24 @@ class PurgeCacheWorker
path = '/' + path if path[0] != '/'
url = Addressable::URI.encode_component(
- "http://#{proxy_ip}/:cache/purge#{path}",
+ "http://#{proxy_ip}#{path}",
Addressable::URI::CharacterClasses::QUERY
)
+
+ retry_encoded = false
+
begin
- RestClient.get(url, host: URI::encode("#{username}.neocities.org"))
+ RestClient::Request.execute method: :head, url: url, timeout: HTTP_TIMEOUT, headers: {
+ host: URI::encode("#{username}.neocities.org"),
+ cache_purge: '1'
+ }
+ rescue URI::InvalidURIError
+ raise if retry_encoded == true
+ url = URI.encode url
+ retry_encoded = true
+ retry
rescue RestClient::ResourceNotFound
+ rescue RestClient::Forbidden
end
end
end
diff --git a/workers/screenshot_worker.rb b/workers/screenshot_worker.rb
index 8831a1d4..53e84c10 100644
--- a/workers/screenshot_worker.rb
+++ b/workers/screenshot_worker.rb
@@ -4,32 +4,9 @@ require 'securerandom'
require 'thread'
require 'open3'
-# Don't judge - Ruby handling of timeouts is a joke..
-module Phantomjs
- def self.run(*args, &block)
- pid = nil
- stdin, stdout, stderr, wait_thr = nil
- begin
- Timeout::timeout(50) do
- stdin, stdout, stderr, wait_thr = Open3.popen3(path, *args)
- pid = wait_thr.pid
- wait_thr.join
- return stdout.read
- end
- # :nocov:
- rescue Timeout::Error
- stdin.close
- stdout.close
- stderr.close
- Process.kill 'QUIT', pid
- raise Timeout::Error
- end
- # :nocov:
- end
-end
-
class ScreenshotWorker
SCREENSHOTS_PATH = Site::SCREENSHOTS_ROOT
+ HARD_TIMEOUT = 30.freeze
include Sidekiq::Worker
sidekiq_options queue: :screenshots, retry: 3, backtrace: true
@@ -39,40 +16,31 @@ class ScreenshotWorker
screenshot.close
screenshot_output_path = screenshot.path+'.png'
- begin
- f = Screencap::Fetcher.new("http://#{username}.neocities.org#{path}")
- f.fetch(
- output: screenshot_output_path,
- width: 1280,
- height: 960
- )
- rescue Timeout::Error
- # :nocov:
- puts "#{username}/#{path} is timing out, discontinuing"
- site = Site[username: username]
- site.is_crashing = true
- site.save_changes validate: false
+ line = Cocaine::CommandLine.new(
+ "timeout #{HARD_TIMEOUT} phantomjs #{File.join DIR_ROOT, 'files', 'phantomjs_screenshot.js'}", ":url :output",
+ expected_outcodes: [0]
+ )
- # Don't enable until we know it works well.
-=begin
- if site.email
- EmailWorker.perform_async({
- from: 'web@neocities.org',
- to: site.email,
- subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow",
- body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+
- "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+
- "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+
- "so we're taking steps to inform you and see if you can resolve the issue. "+
- "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+
- "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+
- "a mistake."+
- "\n\nOur best,\n- Neocities"
- })
- end
-=end
- return
- # :nocov:
+ begin
+ output = line.run(
+ url: "http://#{username}.neocities.org#{path}",
+ output: screenshot_output_path
+ )
+ rescue Cocaine::ExitStatusError => e
+ raise e
+
+ # We set is_crashing after retries now, but use this code to go back to instant:
+
+ #if e.message && e.message.match(/returned 124/)
+ # puts "#{username}/#{path} is timing out, discontinuing"
+ # site = Site[username: username]
+ # site.is_crashing = true
+ # site.save_changes validate: false
+ # return true
+ #
+ #else
+ # raise
+ #end
end
img_list = Magick::ImageList.new
@@ -97,10 +65,34 @@ class ScreenshotWorker
else
new_img = img.scale width, height
end
-
new_img.write(File.join(user_screenshots_path, "#{path}.#{res}.jpg")) {
self.quality = 90
}
end
end
+
+ sidekiq_retries_exhausted do |msg|
+ username, path = msg['args']
+ site = Site[username: username]
+ site.is_crashing = true
+ site.save_changes validate: false
+
+=begin
+ if site.email
+ EmailWorker.perform_async({
+ from: 'web@neocities.org',
+ to: site.email,
+ subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow",
+ body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+
+ "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+
+ "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+
+ "so we're taking steps to inform you and see if you can resolve the issue. "+
+ "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+
+ "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+
+ "a mistake."+
+ "\n\nOur best,\n- Neocities"
+ })
+ end
+=end
+ end
end
diff --git a/workers/thumbnail_worker.rb b/workers/thumbnail_worker.rb
index 181b2183..ad6a3e06 100644
--- a/workers/thumbnail_worker.rb
+++ b/workers/thumbnail_worker.rb
@@ -7,7 +7,13 @@ class ThumbnailWorker
def perform(username, path)
img_list = Magick::ImageList.new
- img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path))
+
+ begin
+ img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path))
+ rescue Errno::ENOENT => e # Not found, skip
+ return
+ end
+
img = img_list.first
user_thumbnails_path = File.join THUMBNAILS_PATH, username