performance: denormalize count of followers

Rakefile

@@ -425,3 +425,13 @@ task :shard_migration => [:environment] do
   sleep 1
   FileUtils.mv './public/newtestsites', './public/testsites'
 end
+
+desc 'prime_follow_count'
+task :prime_follow_count => [:environment] do
+  DB['update sites set follow_count=0'].first
+  Site.select(:id,:username).all.each do |site|
+    count = site.follows_dataset.count
+    next if count == 0
+    DB['update sites set follow_count=? where id=?', count, site.id].first
+  end
+end

app/browse.rb

@@ -52,10 +52,6 @@ def browse_sites_dataset
       site_dataset.exclude! score: nil
       site_dataset.order! :score.desc
     when 'followers'
-      site_dataset = site_dataset.association_left_join :follows
-      site_dataset.select_all! :sites
-      site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
-      site_dataset.group! :sites__id
       site_dataset.order! :follow_count.desc, :updated_at.desc
     when 'supporters'
       site_dataset.exclude! plan_type: nil
@@ -86,18 +82,12 @@ def browse_sites_dataset
     when 'tipping_enabled'
       site_dataset.where! tipping_enabled: true
       site_dataset.where!("(tipping_paypal is not null and tipping_paypal != '') or (tipping_bitcoin is not null and tipping_bitcoin != '')")
-      site_dataset = site_dataset.association_left_join :follows
-      site_dataset.select_all! :sites
-      site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
       site_dataset.where!{views > 10_000}
       site_dataset.group! :sites__id
       site_dataset.order! :follow_count.desc, :views.desc, :updated_at.desc
     else
       params[:sort_by] = 'followers'
-      site_dataset = site_dataset.association_left_join :follows
       site_dataset.select_all! :sites
-      site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
-      site_dataset.group! :sites__id
       site_dataset.order! :follow_count.desc, :views.desc, :updated_at.desc
   end
 

migrations/097_add_follow_count_to_sites.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :follow_count, :integer, default: 0
+  }
+
+  down {
+    DB.drop_column :sites, :follow_count
+  }
+end

migrations/098_add_follow_count_index.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_index :sites, :follow_count
+  }
+
+  down {
+    DB.drop_index :sites, :follow_count
+  }
+end

models/site.rb

@@ -342,15 +342,19 @@ class Site < Sequel::Model
 
   def toggle_follow(site)
     if is_following? site
-      follow = followings_dataset.filter(site_id: site.id).first
+      DB.transaction do
-      site.events_dataset.filter(follow_id: follow.id).delete
+        follow = followings_dataset.filter(site_id: site.id).first
-      follow.delete
+        site.events_dataset.filter(follow_id: follow.id).delete
+        follow.delete
+        DB['update sites set follow_count=follow_count-1 where id=?', site.id].first
+      end
       false
     else
       return false if site.id == self.id # Do not follow yourself
 
       DB.transaction do
         follow = add_following site_id: site.id
+        DB['update sites set follow_count=follow_count+1 where id=?', site.id].first
         Event.create site_id: site.id, actioning_site_id: self.id, follow_id: follow.id
       end
 
@@ -1265,7 +1269,7 @@ class Site < Sequel::Model
 
   def compute_score
     points = 0
-    points += follows_dataset.count * 30
+    points += follow_count * 30
     points += profile_comments_dataset.count * 1
     points += views / 1000
     points += 20 if !featured_at.nil?
@@ -1284,7 +1288,6 @@ class Site < Sequel::Model
     score -= ((Time.now - updated_at) / 1.day) * 2
     score += 500 if (updated_at > 1.week.ago)
     score -= 1000 if
-    follow_count = follows_dataset.count
     score -= 1000 if follow_count == 0
     score += follow_count * 100
     score += profile_comments_dataset.count * 5
@@ -1308,10 +1311,8 @@ class Site < Sequel::Model
 
     # New:
 
-    site_dataset = self.class.browse_dataset.association_left_join :follows
+    site_dataset = self.class.browse_dataset
     site_dataset.select_all! :sites
-    site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count")
-    site_dataset.group! :sites__id
     site_dataset.order! :follow_count.desc, :updated_at.desc
     site_dataset.where! "views >= #{SUGGESTIONS_VIEWS_MIN}"
     site_dataset.limit! limit-suggestions.length
@@ -1329,7 +1330,9 @@ class Site < Sequel::Model
   end
 
   def screenshot_url(path, resolution)
-    "#{SCREENSHOTS_URL_ROOT}/#{self.class.sharding_dir(values[:username])}/#{values[:username]}/#{path}.#{resolution}.jpg"
+    out = ''
+    out = 'https://neocities.org/' if ENV['RACK_ENV'] == 'development'
+    out+"#{SCREENSHOTS_URL_ROOT}/#{self.class.sharding_dir(values[:username])}/#{values[:username]}/#{path}.#{resolution}.jpg"
   end
 
   def base_thumbnails_path

views/home.erb

@@ -99,8 +99,7 @@
             </div>
             <div class="col col-50">
               <div><strong><%= site.views.format_large_number %></strong> views</div>
-              <% follows_count = site.follows_dataset.count %>
+              <div><strong><%= site.follow_count.format_large_number %></strong> follower<%= site.follow_count == 1 ? '' : 's' %></div>
-              <div><strong><%= follows_count.format_large_number %></strong> follower<%= follows_count == 1 ? '' : 's' %></div>
             </div>
           </div>
         </div>

views/site.erb

@@ -28,8 +28,7 @@
       -->
       <div class="stats">
         <div class="stat"><strong><%= site.views.format_large_number %></strong> <span>view<%= site.views == 1 ? '' : 's' %></span></div>
-        <% follows_count = site.follows_dataset.count %>
+        <div class="stat"><strong><%= site.follow_count.format_large_number %></strong> <span>follower<%= site.follow_count == 1 ? '' : 's' %></span></div>
-        <div class="stat"><strong><%= follows_count.format_large_number %></strong> <span>follower<%= follows_count == 1 ? '' : 's' %></span></div>
         <div class="stat"><strong><%= site.changed_count.format_large_number %></strong> <span>update<%= site.changed_count == 1 ? '' : 's' %></span></div>
         <div class="stat"><strong><%= site.tips_dataset.count %></strong> <span>tips</span></div>
       </div>

views/site/stats.erb

@@ -239,8 +239,7 @@
             </div>
             <div class="col col-50">
               <div><strong><%= site.views.format_large_number %></strong> views</div>
-              <% follows_count = site.follows_dataset.count %>
+              <div><strong><%= site.follow_count.format_large_number %></strong> follower<%= site.follow_count == 1 ? '' : 's' %></div>
-              <div><strong><%= follows_count.format_large_number %></strong> follower<%= follows_count == 1 ? '' : 's' %></div>
             </div>
           </div>
         </div>