diff --git a/Gemfile b/Gemfile index ec146b7e..1f7824d9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'sinatra', '2.0.5' +gem 'sinatra', '2.2.0' gem 'redis' gem 'redis-namespace' gem 'sequel' @@ -53,7 +53,7 @@ gem 'activesupport' gem 'facter', require: nil gem 'maxmind-db' gem 'json', '>= 2.3.0' -gem 'nokogiri', '= 1.11.4' +gem 'nokogiri', '1.13.6' group :development, :test do gem 'pry' diff --git a/Gemfile.lock b/Gemfile.lock index 142cca59..826116cc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,7 +143,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2019.0904) mini_mime (1.1.2) - mini_portile2 (2.5.3) + mini_portile2 (2.8.0) minitest (5.11.3) minitest-reporters (1.3.8) ansi @@ -160,16 +160,17 @@ GEM msgpack (1.3.1) multi_json (1.13.1) multipart-post (2.1.1) - mustermann (1.0.3) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) net-ssh (5.2.0) netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.11.4) - mini_portile2 (~> 2.5.0) + nokogiri (1.13.6) + mini_portile2 (~> 2.8.0) racc (~> 1.4) - nokogiri (1.11.4-x86_64-linux) + nokogiri (1.13.6-x86_64-linux) racc (~> 1.4) nokogumbo (2.0.2) nokogiri (~> 1.8, >= 1.8.4) @@ -184,13 +185,13 @@ GEM byebug (~> 11.0) pry (~> 0.10) public_suffix (4.0.6) - puma (5.5.1) + puma (5.6.4) nio4r (~> 2.0) racc (1.6.0) - rack (2.2.3) + rack (2.2.3.1) rack-cache (1.9.0) rack (>= 0.4) - rack-protection (2.0.5) + rack-protection (2.2.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) @@ -213,6 +214,7 @@ GEM rinku (2.0.6) rmagick (4.1.2) ruby-progressbar (1.10.1) + ruby2_keywords (0.0.5) rye (0.9.13) annoy docile (>= 1.0.1) @@ -248,10 +250,10 @@ GEM simplecov-html (0.10.2) simpleidn (0.1.1) unf (~> 0.1.4) - sinatra (2.0.5) + sinatra (2.2.0) mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.5) + rack (~> 2.2) + rack-protection (= 2.2.0) tilt (~> 2.0) sinatra-flash (0.3.0) sinatra (>= 1.0.0) @@ -338,7 +340,7 @@ DEPENDENCIES mock_redis monetize msgpack - nokogiri (= 1.11.4) + nokogiri (= 1.13.6) paypal-recurring pg pry @@ -362,7 +364,7 @@ DEPENDENCIES sidekiq (~> 5.2.0) simplecov simpleidn - sinatra (= 2.0.5) + sinatra (= 2.2.0) sinatra-flash sinatra-xsendfile stripe (~> 5.17.0) diff --git a/Rakefile b/Rakefile index d74be430..718793f6 100644 --- a/Rakefile +++ b/Rakefile @@ -30,12 +30,16 @@ HERE end =end -desc "parse logs" -task :parse_logs => [:environment] do +desc "prune logs" +task :prune_logs => [:environment] do Stat.prune! StatLocation.prune! StatReferrer.prune! StatPath.prune! +end + +desc "parse logs" +task :parse_logs => [:environment] do Stat.parse_logfiles $config['logs_path'] end @@ -473,7 +477,7 @@ end desc 'ml_screenshots_list_dump' task :ml_screenshots_list_dump => [:environment] do ['phishing', 'spam', 'ham', nil].each do |classifier| - File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp| + File.open("./files/screenshot-urls#{classifier.nil? ? '' : '-'+classifier.to_s}.txt", 'w') do |fp| SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file| begin fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n" @@ -585,19 +589,4 @@ task :generate_sitemap => [:environment] do end gz.write %{} end - - desc 'ml_screenshots_list_dump' - task :ml_screenshots_list_dump => [:environment] do - ['phishing', 'spam', 'ham', nil].each do |classifier| - File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp| - SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file| - begin - fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n" - rescue NoMethodError - end - end - end - end - end - end diff --git a/app.rb b/app.rb index 2d7f7331..f5a9d9b1 100644 --- a/app.rb +++ b/app.rb @@ -58,6 +58,7 @@ GEOCITIES_NEIGHBORHOODS = %w{ televisioncity tokyo vienna + westhollywood yosemite }.freeze diff --git a/app_helpers.rb b/app_helpers.rb index acbbbd53..c09f59f7 100644 --- a/app_helpers.rb +++ b/app_helpers.rb @@ -40,7 +40,7 @@ def deleted? end def banned?(ip_check=false) - return true if session[:banned] + #return true if session[:banned] return true if current_site && (current_site.is_banned || parent_site.is_banned) return true if ip_check && Site.banned_ip?(request.ip) @@ -137,4 +137,4 @@ def hcaptcha_valid? else false end -end \ No newline at end of file +end diff --git a/models/archive.rb b/models/archive.rb index 37108e59..49aeacac 100644 --- a/models/archive.rb +++ b/models/archive.rb @@ -4,7 +4,7 @@ class Archive < Sequel::Model many_to_one :site set_primary_key [:site_id, :ipfs_hash] unrestrict_primary_key - MAXIMUM_ARCHIVES_PER_SITE = 10 + MAXIMUM_ARCHIVES_PER_SITE = 5 ARCHIVE_WAIT_TIME = 1.minute def before_destroy diff --git a/models/site.rb b/models/site.rb index cb3e14a1..cb46d0fb 100644 --- a/models/site.rb +++ b/models/site.rb @@ -72,6 +72,7 @@ class Site < Sequel::Model INDEX_HTML_REGEX = /\/?index.html$/ ROOT_INDEX_HTML_REGEX = /^\/?index.html$/ MAX_COMMENT_SIZE = 420 # Used to be the limit for Facebook.. no comment (PUN NOT INTENDED). + MAX_FOLLOWS = 1000 SCREENSHOT_DELAY_SECONDS = 30 SCREENSHOT_RESOLUTIONS = ['540x405', '210x158', '100x100', '50x50'] @@ -160,7 +161,7 @@ class Site < Sequel::Model MAXIMUM_EMAIL_CONFIRMATIONS = 20 MAX_COMMENTS_PER_DAY = 5 - SANDBOX_TIME = 2.days + SANDBOX_TIME = 14.days many_to_many :tags @@ -380,6 +381,7 @@ class Site < Sequel::Model end def toggle_follow(site) + return false if followings_dataset.count > MAX_FOLLOWS if is_following? site DB.transaction do follow = followings_dataset.filter(site_id: site.id).first @@ -508,6 +510,7 @@ class Site < Sequel::Model } delete_all_cache + update_redis_proxy_record true end @@ -986,35 +989,47 @@ class Site < Sequel::Model end def update_redis_proxy_record - user_record = {} - domain_record = {} + u_key = "u-#{username}" - unless values[:domain].blank? - domain_record[:username] = username + if supporter? + $redis_proxy.hset u_key, 'is_supporter', '1' + else + $redis_proxy.hdel u_key, 'is_supporter' + end + + if sandboxed? + $redis_proxy.hset u_key, 'is_sandboxed', '1' + else + $redis_proxy.hdel u_key, 'is_sandboxed' + end + + if values[:domain] + d_root_key = "d-#{values[:domain]}" + d_www_key = "d-www.#{values[:domain]}" + + $redis_proxy.hset u_key, 'domain', values[:domain] + $redis_proxy.hset d_root_key, 'username', username + $redis_proxy.hset d_www_key, 'username', username if ssl_installed? - domain_record[:ssl_cert] = ssl_cert - domain_record[:ssl_key] = ssl_key + $redis_proxy.hset d_root_key, 'ssl_cert', ssl_cert + $redis_proxy.hset d_root_key, 'ssl_key', ssl_key + $redis_proxy.hset d_www_key, 'ssl_cert', ssl_cert + $redis_proxy.hset d_www_key, 'ssl_key', ssl_key end - end - - user_record[:is_sandboxed] = '1' if sandboxed? - user_record[:is_supporter] = '1' if supporter? - - unless user_record.empty? - user_record[:domain] = values[:domain] - $redis_proxy.mapped_hmset "u-#{username}", user_record - end - - unless domain_record.empty? - $redis_proxy.mapped_hmset "d-#{values[:domain]}", domain_record - $redis_proxy.mapped_hmset "d-www.#{values[:domain]}", domain_record + else + $redis_proxy.hdel u_key, 'domain' end $redis_proxy.del "u-#{@old_username}" if @old_username $redis_proxy.del "d-#{@old_domain}" if @old_domain $redis_proxy.del "d-www.#{@old_domain}" if @old_domain - $redis_proxy.del "u-#{username}" if is_deleted + + if is_deleted + $redis_proxy.del u_key + $redis_proxy.del d_root_key + $redis_proxy.del d_www_key + end true end @@ -1819,3 +1834,4 @@ class Site < Sequel::Model true end end + diff --git a/models/site_file.rb b/models/site_file.rb index 6faded0f..28136ed1 100644 --- a/models/site_file.rb +++ b/models/site_file.rb @@ -44,6 +44,10 @@ class SiteFile < Sequel::Model current_path = self.path new_path = site.scrubbed_path new_path + if new_path == '' + return false, 'cannot rename to empty path' + end + if current_path == 'index.html' return false, 'cannot rename or move root index.html' end diff --git a/puma_config.rb b/puma_config.rb index 3671990e..2bb939dc 100644 --- a/puma_config.rb +++ b/puma_config.rb @@ -1,12 +1,14 @@ require 'facter' -threads 5, 5 +threads 1, 1 +#threads 5, 5 environment 'production' #daemonize pidfile '/var/run/neocities/neocities.pid' stdout_redirect '/var/log/neocities/neocities.stdout.log', '/var/log/neocities/neocities.stderr.log', true quiet -workers Facter.value('processors')['count'] +workers 8 +#workers Facter.value('processors')['count'] preload_app! on_worker_boot { DB.disconnect } bind 'unix:/var/run/neocities/neocities.sock?backlog=2048' diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index fad72060..77e1b5fd 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -81,6 +81,17 @@ describe 'site_files' do PurgeCacheWorker.jobs.last['args'].last.must_equal 'dasharezone' end + it 'wont set an empty directory' do + @site.create_directory 'dirone' + @site.site_files.select {|sf| sf.path == 'dirone'}.length.must_equal 1 + + dirone = @site.site_files_dataset.where(path: 'dirone').first + res = dirone.rename('') + @site.site_files_dataset.where(path: '').count.must_equal 0 + res.must_equal [false, 'cannot rename to empty path'] + @site.site_files_dataset.where(path: '').count.wont_equal 1 + end + it 'changes path of files and dirs within directory when changed' do upload( 'dir' => 'test', diff --git a/workers/ban_worker.rb b/workers/ban_worker.rb index 02bf85bc..0f728b13 100644 --- a/workers/ban_worker.rb +++ b/workers/ban_worker.rb @@ -3,6 +3,7 @@ class BanWorker sidekiq_options queue: :ban, retry: 10, backtrace: true def perform(site_id) - Site[site_id].ban! + site = Site[site_id] + site.ban! unless site.supporter? end end diff --git a/workers/black_box_worker.rb b/workers/black_box_worker.rb index 37eccb48..84d2dc43 100644 --- a/workers/black_box_worker.rb +++ b/workers/black_box_worker.rb @@ -5,9 +5,9 @@ class BlackBoxWorker def perform(site_id, path) site = Site[site_id] return true if site.nil? || site.is_banned? || site.is_deleted - BlackBox.tos_violation_check site, path + BlackBox.new(site, path).tos_violation_check! end end -# BlackBox.tos_violation_check self, uploaded \ No newline at end of file +# BlackBox.tos_violation_check self, uploaded