Merge branch 'master' of github.com:neocities/neocities

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'sinatra', '2.0.5'
+gem 'sinatra', '2.2.0'
 gem 'redis'
 gem 'redis-namespace'
 gem 'sequel'
@@ -53,7 +53,7 @@ gem 'activesupport'
 gem 'facter', require: nil
 gem 'maxmind-db'
 gem 'json', '>= 2.3.0'
-gem 'nokogiri', '= 1.11.4'
+gem 'nokogiri', '1.13.6'
 
 group :development, :test do
   gem 'pry'

Gemfile.lock

@@ -143,7 +143,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2019.0904)
     mini_mime (1.1.2)
-    mini_portile2 (2.5.3)
+    mini_portile2 (2.8.0)
     minitest (5.11.3)
     minitest-reporters (1.3.8)
       ansi
@@ -160,16 +160,17 @@ GEM
     msgpack (1.3.1)
     multi_json (1.13.1)
     multipart-post (2.1.1)
-    mustermann (1.0.3)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     net-scp (2.0.0)
       net-ssh (>= 2.6.5, < 6.0.0)
     net-ssh (5.2.0)
     netrc (0.11.0)
     nio4r (2.5.8)
-    nokogiri (1.11.4)
+    nokogiri (1.13.6)
-      mini_portile2 (~> 2.5.0)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    nokogiri (1.11.4-x86_64-linux)
+    nokogiri (1.13.6-x86_64-linux)
       racc (~> 1.4)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
@@ -184,13 +185,13 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.10)
     public_suffix (4.0.6)
-    puma (5.5.1)
+    puma (5.6.4)
       nio4r (~> 2.0)
     racc (1.6.0)
-    rack (2.2.3)
+    rack (2.2.3.1)
     rack-cache (1.9.0)
       rack (>= 0.4)
-    rack-protection (2.0.5)
+    rack-protection (2.2.0)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -213,6 +214,7 @@ GEM
     rinku (2.0.6)
     rmagick (4.1.2)
     ruby-progressbar (1.10.1)
+    ruby2_keywords (0.0.5)
     rye (0.9.13)
       annoy
       docile (>= 1.0.1)
@@ -248,10 +250,10 @@ GEM
     simplecov-html (0.10.2)
     simpleidn (0.1.1)
       unf (~> 0.1.4)
-    sinatra (2.0.5)
+    sinatra (2.2.0)
       mustermann (~> 1.0)
-      rack (~> 2.0)
+      rack (~> 2.2)
-      rack-protection (= 2.0.5)
+      rack-protection (= 2.2.0)
       tilt (~> 2.0)
     sinatra-flash (0.3.0)
       sinatra (>= 1.0.0)
@@ -338,7 +340,7 @@ DEPENDENCIES
   mock_redis
   monetize
   msgpack
-  nokogiri (= 1.11.4)
+  nokogiri (= 1.13.6)
   paypal-recurring
   pg
   pry
@@ -362,7 +364,7 @@ DEPENDENCIES
   sidekiq (~> 5.2.0)
   simplecov
   simpleidn
-  sinatra (= 2.0.5)
+  sinatra (= 2.2.0)
   sinatra-flash
   sinatra-xsendfile
   stripe (~> 5.17.0)

Rakefile

@@ -30,12 +30,16 @@ HERE
 end
 =end
 
-desc "parse logs"
+desc "prune logs"
-task :parse_logs => [:environment] do
+task :prune_logs => [:environment] do
   Stat.prune!
   StatLocation.prune!
   StatReferrer.prune!
   StatPath.prune!
+end
+
+desc "parse logs"
+task :parse_logs => [:environment] do
   Stat.parse_logfiles $config['logs_path']
 end
 
@@ -473,7 +477,7 @@ end
 desc 'ml_screenshots_list_dump'
 task :ml_screenshots_list_dump => [:environment] do
   ['phishing', 'spam', 'ham', nil].each do |classifier|
-    File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp|
+    File.open("./files/screenshot-urls#{classifier.nil? ? '' : '-'+classifier.to_s}.txt", 'w') do |fp|
       SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file|
         begin
           fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n"
@@ -585,19 +589,4 @@ task :generate_sitemap => [:environment] do
     end
     gz.write %{</sitemapindex>}
   end
-
-  desc 'ml_screenshots_list_dump'
-  task :ml_screenshots_list_dump => [:environment] do
-    ['phishing', 'spam', 'ham', nil].each do |classifier|
-      File.open("./files/screenshot-urls-#{classifier.to_s}.txt", 'w') do |fp|
-        SiteFile.where(classifier: classifier).where(path: 'index.html').each do |site_file|
-          begin
-            fp.write "#{site_file.site.screenshot_url('index.html', Site::SCREENSHOT_RESOLUTIONS.first)}\n"
-          rescue NoMethodError
-          end
-        end
-      end
-    end
-  end
-
 end

app.rb

@@ -58,6 +58,7 @@ GEOCITIES_NEIGHBORHOODS = %w{
   televisioncity
   tokyo
   vienna
+  westhollywood
   yosemite
 }.freeze
 

app_helpers.rb

@@ -40,7 +40,7 @@ def deleted?
 end
 
 def banned?(ip_check=false)
-  return true if session[:banned]
+  #return true if session[:banned]
   return true if current_site && (current_site.is_banned || parent_site.is_banned)
 
   return true if ip_check && Site.banned_ip?(request.ip)
@@ -137,4 +137,4 @@ def hcaptcha_valid?
   else
     false
   end
-end
+end

models/archive.rb

@@ -4,7 +4,7 @@ class Archive < Sequel::Model
   many_to_one :site
   set_primary_key [:site_id, :ipfs_hash]
   unrestrict_primary_key
-  MAXIMUM_ARCHIVES_PER_SITE = 10
+  MAXIMUM_ARCHIVES_PER_SITE = 5
   ARCHIVE_WAIT_TIME = 1.minute
 
   def before_destroy

models/site.rb

@@ -72,6 +72,7 @@ class Site < Sequel::Model
   INDEX_HTML_REGEX       = /\/?index.html$/
   ROOT_INDEX_HTML_REGEX  = /^\/?index.html$/
   MAX_COMMENT_SIZE       = 420 # Used to be the limit for Facebook.. no comment (PUN NOT INTENDED).
+  MAX_FOLLOWS            = 1000
 
   SCREENSHOT_DELAY_SECONDS = 30
   SCREENSHOT_RESOLUTIONS   = ['540x405', '210x158', '100x100', '50x50']
@@ -160,7 +161,7 @@ class Site < Sequel::Model
 
   MAXIMUM_EMAIL_CONFIRMATIONS = 20
   MAX_COMMENTS_PER_DAY = 5
-  SANDBOX_TIME = 2.days
+  SANDBOX_TIME = 14.days
 
   many_to_many :tags
 
@@ -380,6 +381,7 @@ class Site < Sequel::Model
   end
 
   def toggle_follow(site)
+    return false if followings_dataset.count > MAX_FOLLOWS
     if is_following? site
       DB.transaction do
         follow = followings_dataset.filter(site_id: site.id).first
@@ -508,6 +510,7 @@ class Site < Sequel::Model
     }
 
     delete_all_cache
+    update_redis_proxy_record
     true
   end
 
@@ -986,35 +989,47 @@ class Site < Sequel::Model
   end
 
   def update_redis_proxy_record
-    user_record = {}
+    u_key = "u-#{username}"
-    domain_record = {}
 
-    unless values[:domain].blank?
+    if supporter?
-      domain_record[:username] = username
+      $redis_proxy.hset u_key, 'is_supporter', '1'
+    else
+      $redis_proxy.hdel u_key, 'is_supporter'
+    end
+
+    if sandboxed?
+      $redis_proxy.hset u_key, 'is_sandboxed', '1'
+    else
+      $redis_proxy.hdel u_key, 'is_sandboxed'
+    end
+
+    if values[:domain]
+      d_root_key = "d-#{values[:domain]}"
+      d_www_key  = "d-www.#{values[:domain]}"
+
+      $redis_proxy.hset u_key, 'domain', values[:domain]
+      $redis_proxy.hset d_root_key, 'username', username
+      $redis_proxy.hset d_www_key, 'username', username
 
       if ssl_installed?
-        domain_record[:ssl_cert] = ssl_cert
+        $redis_proxy.hset d_root_key, 'ssl_cert', ssl_cert
-        domain_record[:ssl_key]  = ssl_key
+        $redis_proxy.hset d_root_key, 'ssl_key',  ssl_key
+        $redis_proxy.hset d_www_key,  'ssl_cert', ssl_cert
+        $redis_proxy.hset d_www_key,  'ssl_key',  ssl_key
       end
-    end
+    else
-
+      $redis_proxy.hdel u_key, 'domain'
-    user_record[:is_sandboxed] = '1' if sandboxed?
-    user_record[:is_supporter] = '1' if supporter?
-
-    unless user_record.empty?
-      user_record[:domain] = values[:domain]
-      $redis_proxy.mapped_hmset "u-#{username}", user_record
-    end
-
-    unless domain_record.empty?
-      $redis_proxy.mapped_hmset "d-#{values[:domain]}", domain_record
-      $redis_proxy.mapped_hmset "d-www.#{values[:domain]}", domain_record
     end
 
     $redis_proxy.del "u-#{@old_username}" if @old_username
     $redis_proxy.del "d-#{@old_domain}" if @old_domain
     $redis_proxy.del "d-www.#{@old_domain}" if @old_domain
-    $redis_proxy.del "u-#{username}" if is_deleted
+
+    if is_deleted
+      $redis_proxy.del u_key
+      $redis_proxy.del d_root_key
+      $redis_proxy.del d_www_key
+    end
 
     true
   end
@@ -1819,3 +1834,4 @@ class Site < Sequel::Model
     true
   end
 end
+

models/site_file.rb

@@ -44,6 +44,10 @@ class SiteFile < Sequel::Model
     current_path = self.path
     new_path = site.scrubbed_path new_path
 
+    if new_path == ''
+      return false, 'cannot rename to empty path'
+    end
+
     if current_path == 'index.html'
       return false, 'cannot rename or move root index.html'
     end

puma_config.rb

@@ -1,12 +1,14 @@
 require 'facter'
 
-threads 5, 5
+threads 1, 1
+#threads 5, 5
 environment 'production'
 #daemonize
 pidfile '/var/run/neocities/neocities.pid'
 stdout_redirect '/var/log/neocities/neocities.stdout.log', '/var/log/neocities/neocities.stderr.log', true
 quiet
-workers Facter.value('processors')['count']
+workers 8
+#workers Facter.value('processors')['count']
 preload_app!
 on_worker_boot { DB.disconnect }
 bind 'unix:/var/run/neocities/neocities.sock?backlog=2048'

tests/site_file_tests.rb

@@ -81,6 +81,17 @@ describe 'site_files' do
       PurgeCacheWorker.jobs.last['args'].last.must_equal 'dasharezone'
     end
 
+    it 'wont set an empty directory' do
+      @site.create_directory 'dirone'
+      @site.site_files.select {|sf| sf.path == 'dirone'}.length.must_equal 1
+
+      dirone = @site.site_files_dataset.where(path: 'dirone').first
+      res = dirone.rename('')
+      @site.site_files_dataset.where(path: '').count.must_equal 0
+      res.must_equal [false, 'cannot rename to empty path']
+      @site.site_files_dataset.where(path: '').count.wont_equal 1
+    end
+
     it 'changes path of files and dirs within directory when changed' do
       upload(
         'dir' => 'test',

workers/ban_worker.rb

@@ -3,6 +3,7 @@ class BanWorker
   sidekiq_options queue: :ban, retry: 10, backtrace: true
 
   def perform(site_id)
-    Site[site_id].ban!
+    site = Site[site_id]
+    site.ban! unless site.supporter?
   end
 end

workers/black_box_worker.rb

@@ -5,9 +5,9 @@ class BlackBoxWorker
   def perform(site_id, path)
     site = Site[site_id]
     return true if site.nil? || site.is_banned? || site.is_deleted
-    BlackBox.tos_violation_check site, path
+    BlackBox.new(site, path).tos_violation_check!
   end
 end
 
 
-# BlackBox.tos_violation_check self, uploaded
+# BlackBox.tos_violation_check self, uploaded