From e049ccc1a2c98a7d94f8a1f4dfda4f026f60001f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 21:56:51 +0000 Subject: [PATCH 01/14] Bump puma from 5.5.1 to 5.6.4 Bumps [puma](https://github.com/puma/puma) from 5.5.1 to 5.6.4. - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v5.5.1...v5.6.4) --- updated-dependencies: - dependency-name: puma dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 142cca59..f76572a3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -184,7 +184,7 @@ GEM byebug (~> 11.0) pry (~> 0.10) public_suffix (4.0.6) - puma (5.5.1) + puma (5.6.4) nio4r (~> 2.0) racc (1.6.0) rack (2.2.3) From d43fa676435ac7b0003a819717fb9e2cd7e0c544 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 May 2022 21:17:29 +0000 Subject: [PATCH 02/14] Bump sinatra from 2.0.5 to 2.2.0 Bumps [sinatra](https://github.com/sinatra/sinatra) from 2.0.5 to 2.2.0. - [Release notes](https://github.com/sinatra/sinatra/releases) - [Changelog](https://github.com/sinatra/sinatra/blob/master/CHANGELOG.md) - [Commits](https://github.com/sinatra/sinatra/compare/v2.0.5...v2.2.0) --- updated-dependencies: - dependency-name: sinatra dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Gemfile | 2 +- Gemfile.lock | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index ec146b7e..a17c0fcf 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'sinatra', '2.0.5' +gem 'sinatra', '2.2.0' gem 'redis' gem 'redis-namespace' gem 'sequel' diff --git a/Gemfile.lock b/Gemfile.lock index 142cca59..f8d5bf37 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -160,7 +160,8 @@ GEM msgpack (1.3.1) multi_json (1.13.1) multipart-post (2.1.1) - mustermann (1.0.3) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) net-ssh (5.2.0) @@ -190,7 +191,7 @@ GEM rack (2.2.3) rack-cache (1.9.0) rack (>= 0.4) - rack-protection (2.0.5) + rack-protection (2.2.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) @@ -213,6 +214,7 @@ GEM rinku (2.0.6) rmagick (4.1.2) ruby-progressbar (1.10.1) + ruby2_keywords (0.0.5) rye (0.9.13) annoy docile (>= 1.0.1) @@ -248,10 +250,10 @@ GEM simplecov-html (0.10.2) simpleidn (0.1.1) unf (~> 0.1.4) - sinatra (2.0.5) + sinatra (2.2.0) mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.5) + rack (~> 2.2) + rack-protection (= 2.2.0) tilt (~> 2.0) sinatra-flash (0.3.0) sinatra (>= 1.0.0) @@ -362,7 +364,7 @@ DEPENDENCIES sidekiq (~> 5.2.0) simplecov simpleidn - sinatra (= 2.0.5) + sinatra (= 2.2.0) sinatra-flash sinatra-xsendfile stripe (~> 5.17.0) From 985a21d3475607ee125996e6b3a5f3d5b83de996 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:17:11 +0000 Subject: [PATCH 03/14] Revised how black box works --- workers/black_box_worker.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers/black_box_worker.rb b/workers/black_box_worker.rb index 37eccb48..84d2dc43 100644 --- a/workers/black_box_worker.rb +++ b/workers/black_box_worker.rb @@ -5,9 +5,9 @@ class BlackBoxWorker def perform(site_id, path) site = Site[site_id] return true if site.nil? || site.is_banned? || site.is_deleted - BlackBox.tos_violation_check site, path + BlackBox.new(site, path).tos_violation_check! end end -# BlackBox.tos_violation_check self, uploaded \ No newline at end of file +# BlackBox.tos_violation_check self, uploaded From e61de5b0d65ca9281b6d605b1343ed81ec0c56c2 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:18:16 +0000 Subject: [PATCH 04/14] manual review for supporter violations --- workers/ban_worker.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workers/ban_worker.rb b/workers/ban_worker.rb index 02bf85bc..0f728b13 100644 --- a/workers/ban_worker.rb +++ b/workers/ban_worker.rb @@ -3,6 +3,7 @@ class BanWorker sidekiq_options queue: :ban, retry: 10, backtrace: true def perform(site_id) - Site[site_id].ban! + site = Site[site_id] + site.ban! unless site.supporter? end end From f468b261dd160862cc24e93eab05d11b827fb226 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:19:19 +0000 Subject: [PATCH 05/14] No threading for the moment to address bugs, set CPU manually --- puma_config.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/puma_config.rb b/puma_config.rb index 3671990e..2bb939dc 100644 --- a/puma_config.rb +++ b/puma_config.rb @@ -1,12 +1,14 @@ require 'facter' -threads 5, 5 +threads 1, 1 +#threads 5, 5 environment 'production' #daemonize pidfile '/var/run/neocities/neocities.pid' stdout_redirect '/var/log/neocities/neocities.stdout.log', '/var/log/neocities/neocities.stderr.log', true quiet -workers Facter.value('processors')['count'] +workers 8 +#workers Facter.value('processors')['count'] preload_app! on_worker_boot { DB.disconnect } bind 'unix:/var/run/neocities/neocities.sock?backlog=2048' From 558edb442ee1abebfd9063a93d8bdeee87f4318a Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:19:59 +0000 Subject: [PATCH 06/14] Checks to prevent follow abuse - fixes for redis proxy settings --- models/site.rb | 58 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/models/site.rb b/models/site.rb index cb3e14a1..cb46d0fb 100644 --- a/models/site.rb +++ b/models/site.rb @@ -72,6 +72,7 @@ class Site < Sequel::Model INDEX_HTML_REGEX = /\/?index.html$/ ROOT_INDEX_HTML_REGEX = /^\/?index.html$/ MAX_COMMENT_SIZE = 420 # Used to be the limit for Facebook.. no comment (PUN NOT INTENDED). + MAX_FOLLOWS = 1000 SCREENSHOT_DELAY_SECONDS = 30 SCREENSHOT_RESOLUTIONS = ['540x405', '210x158', '100x100', '50x50'] @@ -160,7 +161,7 @@ class Site < Sequel::Model MAXIMUM_EMAIL_CONFIRMATIONS = 20 MAX_COMMENTS_PER_DAY = 5 - SANDBOX_TIME = 2.days + SANDBOX_TIME = 14.days many_to_many :tags @@ -380,6 +381,7 @@ class Site < Sequel::Model end def toggle_follow(site) + return false if followings_dataset.count > MAX_FOLLOWS if is_following? site DB.transaction do follow = followings_dataset.filter(site_id: site.id).first @@ -508,6 +510,7 @@ class Site < Sequel::Model } delete_all_cache + update_redis_proxy_record true end @@ -986,35 +989,47 @@ class Site < Sequel::Model end def update_redis_proxy_record - user_record = {} - domain_record = {} + u_key = "u-#{username}" - unless values[:domain].blank? - domain_record[:username] = username + if supporter? + $redis_proxy.hset u_key, 'is_supporter', '1' + else + $redis_proxy.hdel u_key, 'is_supporter' + end + + if sandboxed? + $redis_proxy.hset u_key, 'is_sandboxed', '1' + else + $redis_proxy.hdel u_key, 'is_sandboxed' + end + + if values[:domain] + d_root_key = "d-#{values[:domain]}" + d_www_key = "d-www.#{values[:domain]}" + + $redis_proxy.hset u_key, 'domain', values[:domain] + $redis_proxy.hset d_root_key, 'username', username + $redis_proxy.hset d_www_key, 'username', username if ssl_installed? - domain_record[:ssl_cert] = ssl_cert - domain_record[:ssl_key] = ssl_key + $redis_proxy.hset d_root_key, 'ssl_cert', ssl_cert + $redis_proxy.hset d_root_key, 'ssl_key', ssl_key + $redis_proxy.hset d_www_key, 'ssl_cert', ssl_cert + $redis_proxy.hset d_www_key, 'ssl_key', ssl_key end - end - - user_record[:is_sandboxed] = '1' if sandboxed? - user_record[:is_supporter] = '1' if supporter? - - unless user_record.empty? - user_record[:domain] = values[:domain] - $redis_proxy.mapped_hmset "u-#{username}", user_record - end - - unless domain_record.empty? - $redis_proxy.mapped_hmset "d-#{values[:domain]}", domain_record - $redis_proxy.mapped_hmset "d-www.#{values[:domain]}", domain_record + else + $redis_proxy.hdel u_key, 'domain' end $redis_proxy.del "u-#{@old_username}" if @old_username $redis_proxy.del "d-#{@old_domain}" if @old_domain $redis_proxy.del "d-www.#{@old_domain}" if @old_domain - $redis_proxy.del "u-#{username}" if is_deleted + + if is_deleted + $redis_proxy.del u_key + $redis_proxy.del d_root_key + $redis_proxy.del d_www_key + end true end @@ -1819,3 +1834,4 @@ class Site < Sequel::Model true end end + From 08a29c8768669cf6a3dfaa4827fbc6a2857a3d2e Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:20:34 +0000 Subject: [PATCH 07/14] Reduce archives --- models/archive.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/archive.rb b/models/archive.rb index 37108e59..49aeacac 100644 --- a/models/archive.rb +++ b/models/archive.rb @@ -4,7 +4,7 @@ class Archive < Sequel::Model many_to_one :site set_primary_key [:site_id, :ipfs_hash] unrestrict_primary_key - MAXIMUM_ARCHIVES_PER_SITE = 10 + MAXIMUM_ARCHIVES_PER_SITE = 5 ARCHIVE_WAIT_TIME = 1.minute def before_destroy From 322201332f04bc9639ca01ace59e0c5497fe5740 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:20:58 +0000 Subject: [PATCH 08/14] No session ban --- app_helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app_helpers.rb b/app_helpers.rb index acbbbd53..c09f59f7 100644 --- a/app_helpers.rb +++ b/app_helpers.rb @@ -40,7 +40,7 @@ def deleted? end def banned?(ip_check=false) - return true if session[:banned] + #return true if session[:banned] return true if current_site && (current_site.is_banned || parent_site.is_banned) return true if ip_check && Site.banned_ip?(request.ip) @@ -137,4 +137,4 @@ def hcaptcha_valid? else false end -end \ No newline at end of file +end From 671b29bc9fad10469c0dac7fa97cd46f30a85558 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 10 May 2022 00:21:44 +0000 Subject: [PATCH 09/14] Little rearrangin --- Rakefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index d74be430..9717e5e5 100644 --- a/Rakefile +++ b/Rakefile @@ -30,12 +30,16 @@ HERE end =end -desc "parse logs" -task :parse_logs => [:environment] do +desc "prune logs" +task :prune_logs => [:environment] do Stat.prune! StatLocation.prune! StatReferrer.prune! StatPath.prune! +end + +desc "parse logs" +task :parse_logs => [:environment] do Stat.parse_logfiles $config['logs_path'] end From 0b7a05487d9ff62f63da57455cb0c46337b148a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 May 2022 00:27:07 +0000 Subject: [PATCH 10/14] Bump nokogiri from 1.11.4 to 1.13.6 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.11.4 to 1.13.6. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.11.4...v1.13.6) --- updated-dependencies: - dependency-name: nokogiri dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Gemfile | 2 +- Gemfile.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index ec146b7e..b546b88a 100644 --- a/Gemfile +++ b/Gemfile @@ -53,7 +53,7 @@ gem 'activesupport' gem 'facter', require: nil gem 'maxmind-db' gem 'json', '>= 2.3.0' -gem 'nokogiri', '= 1.11.4' +gem 'nokogiri', '1.13.6' group :development, :test do gem 'pry' diff --git a/Gemfile.lock b/Gemfile.lock index 142cca59..4c666aac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,7 +143,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2019.0904) mini_mime (1.1.2) - mini_portile2 (2.5.3) + mini_portile2 (2.8.0) minitest (5.11.3) minitest-reporters (1.3.8) ansi @@ -166,10 +166,10 @@ GEM net-ssh (5.2.0) netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.11.4) - mini_portile2 (~> 2.5.0) + nokogiri (1.13.6) + mini_portile2 (~> 2.8.0) racc (~> 1.4) - nokogiri (1.11.4-x86_64-linux) + nokogiri (1.13.6-x86_64-linux) racc (~> 1.4) nokogumbo (2.0.2) nokogiri (~> 1.8, >= 1.8.4) @@ -338,7 +338,7 @@ DEPENDENCIES mock_redis monetize msgpack - nokogiri (= 1.11.4) + nokogiri (= 1.13.6) paypal-recurring pg pry From 4cf91506c664c424d90835b55a5f866bf5435087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 May 2022 16:44:44 +0000 Subject: [PATCH 11/14] Bump rack from 2.2.3 to 2.2.3.1 Bumps [rack](https://github.com/rack/rack) from 2.2.3 to 2.2.3.1. - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/2.2.3...2.2.3.1) --- updated-dependencies: - dependency-name: rack dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 142cca59..6649279f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -187,7 +187,7 @@ GEM puma (5.5.1) nio4r (~> 2.0) racc (1.6.0) - rack (2.2.3) + rack (2.2.3.1) rack-cache (1.9.0) rack (>= 0.4) rack-protection (2.0.5) From 75f225fd880fc677fecc9ab08b8aba09c013016e Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Wed, 22 Jun 2022 18:29:58 -0500 Subject: [PATCH 12/14] prevent creation of empty path with rename --- models/site_file.rb | 4 ++++ tests/site_file_tests.rb | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/models/site_file.rb b/models/site_file.rb index 6faded0f..28136ed1 100644 --- a/models/site_file.rb +++ b/models/site_file.rb @@ -44,6 +44,10 @@ class SiteFile < Sequel::Model current_path = self.path new_path = site.scrubbed_path new_path + if new_path == '' + return false, 'cannot rename to empty path' + end + if current_path == 'index.html' return false, 'cannot rename or move root index.html' end diff --git a/tests/site_file_tests.rb b/tests/site_file_tests.rb index fad72060..77e1b5fd 100644 --- a/tests/site_file_tests.rb +++ b/tests/site_file_tests.rb @@ -81,6 +81,17 @@ describe 'site_files' do PurgeCacheWorker.jobs.last['args'].last.must_equal 'dasharezone' end + it 'wont set an empty directory' do + @site.create_directory 'dirone' + @site.site_files.select {|sf| sf.path == 'dirone'}.length.must_equal 1 + + dirone = @site.site_files_dataset.where(path: 'dirone').first + res = dirone.rename('') + @site.site_files_dataset.where(path: '').count.must_equal 0 + res.must_equal [false, 'cannot rename to empty path'] + @site.site_files_dataset.where(path: '').count.wont_equal 1 + end + it 'changes path of files and dirs within directory when changed' do upload( 'dir' => 'test', From 5670ef1ad8cfb6be00a910c76e38aedc6699f01b Mon Sep 17 00:00:00 2001 From: Kiril Misnikov <56405352+lime360@users.noreply.github.com> Date: Mon, 18 Jul 2022 18:51:31 +0300 Subject: [PATCH 13/14] you forgot west hollywood kyle --- app.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app.rb b/app.rb index 2d7f7331..f5a9d9b1 100644 --- a/app.rb +++ b/app.rb @@ -58,6 +58,7 @@ GEOCITIES_NEIGHBORHOODS = %w{ televisioncity tokyo vienna + westhollywood yosemite }.freeze From fcdbf2f55c1d47881ba34f3ee15fcb0e24329763 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Wed, 10 Aug 2022 18:55:37 +0000 Subject: [PATCH 14/14] Fix hyphen on normal screenshot urls, remove dupe code --- Rakefile | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/Rakefile b/Rakefile index 9717e5e5..718793f6 100644 --- a/Rakefile +++ b/Rakefile @@ -477,7 +477,7 @@ end desc 'ml_screenshots_list_dump' task :ml_screenshots_list_dump => [:environment] do ['phishing', 'spam', 'ham', nil].each do |classifier| - File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp| + File.open("./files/screenshot-urls#{classifier.nil? ? '' : '-'+classifier.to_s}.txt", 'w') do |fp| SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file| begin fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n" @@ -589,19 +589,4 @@ task :generate_sitemap => [:environment] do end gz.write %{} end - - desc 'ml_screenshots_list_dump' - task :ml_screenshots_list_dump => [:environment] do - ['phishing', 'spam', 'ham', nil].each do |classifier| - File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp| - SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file| - begin - fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n" - rescue NoMethodError - end - end - end - end - end - end