<% event_site = event.site_dataset.select(:id, :username, :title, :domain).first %>
<% if current_site && event_site.id == current_site.id %>
- Your site was updated.
+ You updated your site.
<% else %>
- <%= event_site.title.shorten(45) %> has been updated.
+ <%= event_site.title.shorten(45) %> was updated.
<% end %>
-
\ No newline at end of file
+
diff --git a/views/admin.erb b/views/admin.erb
index f2ac59b1..578485a0 100644
--- a/views/admin.erb
+++ b/views/admin.erb
@@ -9,7 +9,8 @@
@@ -96,7 +96,7 @@
Upgrade for $<%= Site::PLAN_FEATURES[:supporter][:price] %>/mo
- or pay with PayPal or Bitcoin
+ or pay with PayPal or Bitcoin
diff --git a/workers/archive_worker.rb b/workers/archive_worker.rb
index 51067e14..e69e6cca 100644
--- a/workers/archive_worker.rb
+++ b/workers/archive_worker.rb
@@ -1,4 +1,5 @@
require 'sidekiq/api'
+require 'redis-namespace'
class ArchiveWorker
include Sidekiq::Worker
@@ -17,11 +18,21 @@ class ArchiveWorker
logger.info "JOB ID: #{jid} #{site_id.inspect}"
queue.each do |job|
if job.args == [site_id] && job.jid != jid
- logger.info "DELETING #{job.jid} #{job.args.inspect}"
+ logger.info "DELETING #{job.jid} for site_id #{site_id}"
job.delete
end
end
+ scheduled_jobs = Sidekiq::ScheduledSet.new.select do |scheduled_job|
+ scheduled_job.klass == 'ArchiveWorker' &&
+ scheduled_job.args[0] == site_id
+ end
+
+ scheduled_jobs.each do |scheduled_job|
+ logger.info "DELETING scheduled job #{scheduled_job.jid} for site_id #{site_id}"
+ scheduled_job.delete
+ end
+
site.archive!
end
end
diff --git a/workers/delete_cache_order_worker.rb b/workers/delete_cache_order_worker.rb
new file mode 100644
index 00000000..cc9bacf5
--- /dev/null
+++ b/workers/delete_cache_order_worker.rb
@@ -0,0 +1,22 @@
+class DeleteCacheOrderWorker
+ include Sidekiq::Worker
+ sidekiq_options queue: :deletecacheorder, retry: 1000, backtrace: true, average_scheduled_poll_interval: 1
+
+ sidekiq_retry_in do |count|
+ return 10 if count < 10
+ 180
+ end
+
+ def perform(username, path)
+ if ENV['RACK_ENV'] == 'test'
+ proxy_ips = ['10.0.0.1', '10.0.0.2']
+ else
+ #proxy_ips = Resolv.getaddresses($config['cache_purge_ips_uri'])
+ proxy_ips = Resolv.getaddresses($config['cache_purge_ips_uri']).keep_if {|r| !r.match(/:/)}
+ end
+
+ proxy_ips.each do |proxy_ip|
+ DeleteCacheWorker.perform_async proxy_ip, username, path
+ end
+ end
+end
diff --git a/workers/delete_cache_worker.rb b/workers/delete_cache_worker.rb
new file mode 100644
index 00000000..3b0dc35e
--- /dev/null
+++ b/workers/delete_cache_worker.rb
@@ -0,0 +1,34 @@
+require 'open-uri'
+
+# PurgeCacheWorker refreshes the cache, this actually deletes it.
+# This is because when the file is 404ing the PurgeCacheWorker
+# will just sit on the stale cache, even though it's not supposed to.
+# It's some nginx bug. I'm not going to deal with it.
+
+class DeleteCacheWorker
+ HTTP_TIMEOUT = 5
+ include Sidekiq::Worker
+ sidekiq_options queue: :deletecache, retry: 1000, backtrace: false, average_scheduled_poll_interval: 1
+
+ sidekiq_retry_in do |count|
+ return 10 if count < 10
+ 180
+ end
+
+ def perform(proxy_ip, username, path)
+ # Must always have a forward slash
+ path = '/' + path if path[0] != '/'
+
+ url = Addressable::URI.encode_component(
+ "http://#{proxy_ip}/:cache/purge#{path}",
+ Addressable::URI::CharacterClasses::QUERY
+ )
+ begin
+ RestClient::Request.execute method: :get, url: url, timeout: HTTP_TIMEOUT, headers: {
+ host: URI::encode("#{username}.neocities.org")
+ }
+ rescue RestClient::ResourceNotFound
+ rescue RestClient::Forbidden
+ end
+ end
+end
diff --git a/workers/lets_encrypt_worker.rb b/workers/lets_encrypt_worker.rb
new file mode 100644
index 00000000..b16fef5a
--- /dev/null
+++ b/workers/lets_encrypt_worker.rb
@@ -0,0 +1,162 @@
+class LetsEncryptWorker
+ class NotAuthorizedYetError < StandardError; end
+ class VerificationTimeoutError < StandardError; end
+ class InvalidAuthError < StandardError; end
+ class VerifyNotFoundWithDomain < StandardError; end
+ include Sidekiq::Worker
+ sidekiq_options queue: :lets_encrypt_worker, retry: 5, backtrace: true
+
+ sidekiq_retry_in do |count|
+ 1.hour.to_i
+ end
+
+ # If you need to clear scheduled jobs:
+ # Sidekiq::ScheduledSet.new.select {|s| JSON.parse(s.value)['class'] == 'LetsEncryptWorker'}.each {|j| j.delete}
+
+ def letsencrypt
+ Acme::Client.new(
+ private_key: OpenSSL::PKey::RSA.new(File.read($config['letsencrypt_key'])),
+ endpoint: $config['letsencrypt_endpoint']
+ )
+ end
+
+ def perform(site_id)
+ # Dispose of dupes
+ queue = Sidekiq::Queue.new self.class.sidekiq_options_hash['queue']
+ queue.each do |job|
+ if job.args == [site_id] && job.jid != jid
+ job.delete
+ end
+ end
+
+ site = Site[site_id]
+ return if site.domain.blank? || site.is_deleted || site.is_banned
+
+ return if site.values[:domain].match /\.neocities\.org$/i
+
+ domain_raw = site.values[:domain].gsub(/^www\./, '')
+
+ domains = [domain_raw, "www.#{domain_raw}"]
+
+ verified_domains = []
+
+ domains.each_with_index do |domain, index|
+ puts "verifying accessability of test file on #{domain}"
+ challenge_base_path = File.join '.well-known', 'acme-challenge'
+ testfile_name, testfile_key = "test#{UUIDTools::UUID.random_create}", SecureRandom.hex
+ testfile_fs_path = File.join site.base_files_path, challenge_base_path
+
+ begin
+ FileUtils.mkdir_p File.join(site.base_files_path, challenge_base_path)
+ File.write File.join(testfile_fs_path, testfile_name), testfile_key
+ rescue => e
+ puts "ERROR WRITING TO WELLKNOWN FILE, SKIPPING #{domain}: #{e.inspect}"
+ next
+ end
+
+ # Ensure that both domains work before sending request. Let's Encrypt has a low
+ # pending request limit, and it takes one week (!) to flush out.
+
+ challenge_url = "http://#{domain}/#{challenge_base_path}/#{testfile_name}"
+
+ puts "testing #{challenge_url}"
+
+ begin
+ res = HTTP.timeout(:global, write: 5, connect: 10, read: 10).follow.get(challenge_url)
+ rescue
+ puts "error with #{challenge_url}"
+ next
+ end
+
+ if res.status != 200 && res.body != testfile_key
+ puts "CONTENT DOWNLOADED DID NOT MATCH #{challenge_url}"
+ next
+ end
+
+ puts "test succeeded, sending challenge request verification"
+
+ begin
+ auth = letsencrypt.authorize domain: domain
+ challenge = auth.http01
+ rescue Acme::Client::Error::Malformed
+ puts "international domains not supported yet, quitting"
+ return
+ end
+
+ begin
+ FileUtils.mkdir_p File.join(site.base_files_path, File.dirname(challenge.filename))
+ File.write File.join(site.base_files_path, challenge.filename), challenge.file_content
+ rescue => e
+ put "FAILED TO WRITE CHALLENGE: #{site.domain} #{challenge.filename}"
+ # A verification needs to be made anyways, otherwise 300 of them will jam up the system for a week
+ end
+
+ challenge.request_verification
+
+ sleep 1
+ attempts = 0
+
+ while true
+ result = challenge.verify_status
+ puts "#{domain} : #{result}"
+
+ if result == 'valid'
+ puts "VALIDATED: #{domain}"
+ clean_wellknown_turds site
+ verified_domains.push domain
+ break
+ end
+
+ raise VerificationTimeoutError if attempts == 60
+
+ if result == 'invalid'
+ puts "returned invalid, walking away"
+ clean_wellknown_turds site
+ break
+ end
+
+ attempts += 1
+ sleep 2
+ end
+ end
+
+ if verified_domains.empty?
+ puts "no verified domains, skipping"
+ return
+ end
+
+ puts "verified domains: #{verified_domains.inspect}"
+
+ clean_wellknown_turds site
+
+ retries = 0
+ begin
+ csr = Acme::Client::CertificateRequest.new names: verified_domains
+ certificate = letsencrypt.new_certificate csr
+ rescue Acme::Client::Error => e
+ if retries == 2
+ puts "Failed to create cert, returning: #{e.message}"
+ return
+ end
+ retries += 1
+ retry
+ end
+
+ site.ssl_key = certificate.request.private_key.to_pem
+ site.ssl_cert = certificate.fullchain_to_pem
+ site.cert_updated_at = Time.now
+ site.save_changes validate: false
+
+ # Refresh the cert periodically, current expire time is 90 days
+ # We're going for a cron task for this now, so this is commented out.
+ #LetsEncryptWorker.perform_in 60.days, site.id
+ end
+
+ def clean_wellknown_turds(site)
+ wellknown_path = File.join(site.base_files_path, '.well-known')
+
+ if File.exist?(wellknown_path)
+ FileUtils.rm_rf wellknown_path
+ end
+ end
+end
diff --git a/workers/purge_cache_order_worker.rb b/workers/purge_cache_order_worker.rb
index 0c079eb4..2f78e516 100644
--- a/workers/purge_cache_order_worker.rb
+++ b/workers/purge_cache_order_worker.rb
@@ -7,13 +7,12 @@ class PurgeCacheOrderWorker
180
end
- RESOLVER = Dnsruby::Resolver.new
-
def perform(username, path)
if ENV['RACK_ENV'] == 'test'
proxy_ips = ['10.0.0.1', '10.0.0.2']
else
- proxy_ips = RESOLVER.query($config['cache_purge_ips_uri']).answer.collect {|a| a.address.to_s}
+ #proxy_ips = Resolv.getaddresses($config['cache_purge_ips_uri'])
+ proxy_ips = Resolv.getaddresses($config['cache_purge_ips_uri']).keep_if {|r| !r.match(/:/)}
end
proxy_ips.each do |proxy_ip|
diff --git a/workers/purge_cache_worker.rb b/workers/purge_cache_worker.rb
index eaf123c9..1ce5a260 100644
--- a/workers/purge_cache_worker.rb
+++ b/workers/purge_cache_worker.rb
@@ -15,14 +15,24 @@ class PurgeCacheWorker
path = '/' + path if path[0] != '/'
url = Addressable::URI.encode_component(
- "http://#{proxy_ip}/:cache/purge#{path}",
+ "http://#{proxy_ip}#{path}",
Addressable::URI::CharacterClasses::QUERY
)
+
+ retry_encoded = false
+
begin
- RestClient::Request.execute method: :get, url: url, timeout: HTTP_TIMEOUT, headers: {
- host: URI::encode("#{username}.neocities.org")
+ RestClient::Request.execute method: :head, url: url, timeout: HTTP_TIMEOUT, headers: {
+ host: URI::encode("#{username}.neocities.org"),
+ cache_purge: '1'
}
+ rescue URI::InvalidURIError
+ raise if retry_encoded == true
+ url = URI.encode url
+ retry_encoded = true
+ retry
rescue RestClient::ResourceNotFound
+ rescue RestClient::Forbidden
end
end
end
diff --git a/workers/screenshot_worker.rb b/workers/screenshot_worker.rb
index 8831a1d4..53e84c10 100644
--- a/workers/screenshot_worker.rb
+++ b/workers/screenshot_worker.rb
@@ -4,32 +4,9 @@ require 'securerandom'
require 'thread'
require 'open3'
-# Don't judge - Ruby handling of timeouts is a joke..
-module Phantomjs
- def self.run(*args, &block)
- pid = nil
- stdin, stdout, stderr, wait_thr = nil
- begin
- Timeout::timeout(50) do
- stdin, stdout, stderr, wait_thr = Open3.popen3(path, *args)
- pid = wait_thr.pid
- wait_thr.join
- return stdout.read
- end
- # :nocov:
- rescue Timeout::Error
- stdin.close
- stdout.close
- stderr.close
- Process.kill 'QUIT', pid
- raise Timeout::Error
- end
- # :nocov:
- end
-end
-
class ScreenshotWorker
SCREENSHOTS_PATH = Site::SCREENSHOTS_ROOT
+ HARD_TIMEOUT = 30.freeze
include Sidekiq::Worker
sidekiq_options queue: :screenshots, retry: 3, backtrace: true
@@ -39,40 +16,31 @@ class ScreenshotWorker
screenshot.close
screenshot_output_path = screenshot.path+'.png'
- begin
- f = Screencap::Fetcher.new("http://#{username}.neocities.org#{path}")
- f.fetch(
- output: screenshot_output_path,
- width: 1280,
- height: 960
- )
- rescue Timeout::Error
- # :nocov:
- puts "#{username}/#{path} is timing out, discontinuing"
- site = Site[username: username]
- site.is_crashing = true
- site.save_changes validate: false
+ line = Cocaine::CommandLine.new(
+ "timeout #{HARD_TIMEOUT} phantomjs #{File.join DIR_ROOT, 'files', 'phantomjs_screenshot.js'}", ":url :output",
+ expected_outcodes: [0]
+ )
- # Don't enable until we know it works well.
-=begin
- if site.email
- EmailWorker.perform_async({
- from: 'web@neocities.org',
- to: site.email,
- subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow",
- body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+
- "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+
- "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+
- "so we're taking steps to inform you and see if you can resolve the issue. "+
- "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+
- "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+
- "a mistake."+
- "\n\nOur best,\n- Neocities"
- })
- end
-=end
- return
- # :nocov:
+ begin
+ output = line.run(
+ url: "http://#{username}.neocities.org#{path}",
+ output: screenshot_output_path
+ )
+ rescue Cocaine::ExitStatusError => e
+ raise e
+
+ # We set is_crashing after retries now, but use this code to go back to instant:
+
+ #if e.message && e.message.match(/returned 124/)
+ # puts "#{username}/#{path} is timing out, discontinuing"
+ # site = Site[username: username]
+ # site.is_crashing = true
+ # site.save_changes validate: false
+ # return true
+ #
+ #else
+ # raise
+ #end
end
img_list = Magick::ImageList.new
@@ -97,10 +65,34 @@ class ScreenshotWorker
else
new_img = img.scale width, height
end
-
new_img.write(File.join(user_screenshots_path, "#{path}.#{res}.jpg")) {
self.quality = 90
}
end
end
+
+ sidekiq_retries_exhausted do |msg|
+ username, path = msg['args']
+ site = Site[username: username]
+ site.is_crashing = true
+ site.save_changes validate: false
+
+=begin
+ if site.email
+ EmailWorker.perform_async({
+ from: 'web@neocities.org',
+ to: site.email,
+ subject: "[NeoCities] The web page \"#{path}\" on your site (#{username}.neocities.org) is slow",
+ body: "Hi there! This is an automated email to inform you that we're having issues loading your site to take a "+
+ "screenshot. It is possible that this is an error specific to our screenshot program, but it is much more "+
+ "likely that your site is too slow to be used with browsers. We don't want Neocities sites crashing browsers, "+
+ "so we're taking steps to inform you and see if you can resolve the issue. "+
+ "We may have to de-list your web site from being viewable in our browse page if it is not resolved shortly. "+
+ "We will review the site manually before taking this step, so don't worry if your site is fine and we made "+
+ "a mistake."+
+ "\n\nOur best,\n- Neocities"
+ })
+ end
+=end
+ end
end
diff --git a/workers/thumbnail_worker.rb b/workers/thumbnail_worker.rb
index 181b2183..ad6a3e06 100644
--- a/workers/thumbnail_worker.rb
+++ b/workers/thumbnail_worker.rb
@@ -7,7 +7,13 @@ class ThumbnailWorker
def perform(username, path)
img_list = Magick::ImageList.new
- img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path))
+
+ begin
+ img_list.from_blob File.read(File.join(Site::SITE_FILES_ROOT, username, path))
+ rescue Errno::ENOENT => e # Not found, skip
+ return
+ end
+
img = img_list.first
user_thumbnails_path = File.join THUMBNAILS_PATH, username
The Neocities Team
Kyle Drake
Victoria Wang
- Scott O'Hara -
Follow us on Twitter or Facebook
-+ Site Bandwidth Usage
Ban Site
Ban by IP
- -Upgrade to Supporter
Feature Site
-- <%= (Site::PLAN_FEATURES[:supporter][:space] / (10**6)).to_comma_separated %> MB storage
- - <%= Site::PLAN_FEATURES[:supporter][:bandwidth].to_bytes_pretty %> bandwidth
+ - <%= (Site::PLAN_FEATURES[:supporter][:space] / (10**6)).to_comma_separated %> MB storage
+ - <%= Site::PLAN_FEATURES[:supporter][:bandwidth].to_bytes_pretty %> bandwidth
-- No File Upload Type Restrictions
+ - No File Upload Type Restrictions
- Unlimited Site Creation
- - Full Domain Support
+ - Custom Domain Support
- Remote Filesystem Support