From 04af230f8d5fa961c4474f9e2507342287b5256d Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Sun, 2 Apr 2017 20:47:06 -0700 Subject: [PATCH] performance: denormalize count of followers --- Rakefile | 10 ++++++++++ app/browse.rb | 10 ---------- migrations/097_add_follow_count_to_sites.rb | 9 +++++++++ migrations/098_add_follow_count_index.rb | 9 +++++++++ models/site.rb | 21 ++++++++++++--------- views/home.erb | 3 +-- views/site.erb | 3 +-- views/site/stats.erb | 3 +-- 8 files changed, 43 insertions(+), 25 deletions(-) create mode 100644 migrations/097_add_follow_count_to_sites.rb create mode 100644 migrations/098_add_follow_count_index.rb diff --git a/Rakefile b/Rakefile index 93a4ee60..3522cf4e 100644 --- a/Rakefile +++ b/Rakefile @@ -425,3 +425,13 @@ task :shard_migration => [:environment] do sleep 1 FileUtils.mv './public/newtestsites', './public/testsites' end + +desc 'prime_follow_count' +task :prime_follow_count => [:environment] do + DB['update sites set follow_count=0'].first + Site.select(:id,:username).all.each do |site| + count = site.follows_dataset.count + next if count == 0 + DB['update sites set follow_count=? where id=?', count, site.id].first + end +end diff --git a/app/browse.rb b/app/browse.rb index f0212bab..91ead72b 100644 --- a/app/browse.rb +++ b/app/browse.rb @@ -52,10 +52,6 @@ def browse_sites_dataset site_dataset.exclude! score: nil site_dataset.order! :score.desc when 'followers' - site_dataset = site_dataset.association_left_join :follows - site_dataset.select_all! :sites - site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count") - site_dataset.group! :sites__id site_dataset.order! :follow_count.desc, :updated_at.desc when 'supporters' site_dataset.exclude! plan_type: nil @@ -86,18 +82,12 @@ def browse_sites_dataset when 'tipping_enabled' site_dataset.where! tipping_enabled: true site_dataset.where!("(tipping_paypal is not null and tipping_paypal != '') or (tipping_bitcoin is not null and tipping_bitcoin != '')") - site_dataset = site_dataset.association_left_join :follows - site_dataset.select_all! :sites - site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count") site_dataset.where!{views > 10_000} site_dataset.group! :sites__id site_dataset.order! :follow_count.desc, :views.desc, :updated_at.desc else params[:sort_by] = 'followers' - site_dataset = site_dataset.association_left_join :follows site_dataset.select_all! :sites - site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count") - site_dataset.group! :sites__id site_dataset.order! :follow_count.desc, :views.desc, :updated_at.desc end diff --git a/migrations/097_add_follow_count_to_sites.rb b/migrations/097_add_follow_count_to_sites.rb new file mode 100644 index 00000000..a75a42e3 --- /dev/null +++ b/migrations/097_add_follow_count_to_sites.rb @@ -0,0 +1,9 @@ +Sequel.migration do + up { + DB.add_column :sites, :follow_count, :integer, default: 0 + } + + down { + DB.drop_column :sites, :follow_count + } +end diff --git a/migrations/098_add_follow_count_index.rb b/migrations/098_add_follow_count_index.rb new file mode 100644 index 00000000..1b3d2434 --- /dev/null +++ b/migrations/098_add_follow_count_index.rb @@ -0,0 +1,9 @@ +Sequel.migration do + up { + DB.add_index :sites, :follow_count + } + + down { + DB.drop_index :sites, :follow_count + } +end diff --git a/models/site.rb b/models/site.rb index d64bd889..c1484c6f 100644 --- a/models/site.rb +++ b/models/site.rb @@ -342,15 +342,19 @@ class Site < Sequel::Model def toggle_follow(site) if is_following? site - follow = followings_dataset.filter(site_id: site.id).first - site.events_dataset.filter(follow_id: follow.id).delete - follow.delete + DB.transaction do + follow = followings_dataset.filter(site_id: site.id).first + site.events_dataset.filter(follow_id: follow.id).delete + follow.delete + DB['update sites set follow_count=follow_count-1 where id=?', site.id].first + end false else return false if site.id == self.id # Do not follow yourself DB.transaction do follow = add_following site_id: site.id + DB['update sites set follow_count=follow_count+1 where id=?', site.id].first Event.create site_id: site.id, actioning_site_id: self.id, follow_id: follow.id end @@ -1265,7 +1269,7 @@ class Site < Sequel::Model def compute_score points = 0 - points += follows_dataset.count * 30 + points += follow_count * 30 points += profile_comments_dataset.count * 1 points += views / 1000 points += 20 if !featured_at.nil? @@ -1284,7 +1288,6 @@ class Site < Sequel::Model score -= ((Time.now - updated_at) / 1.day) * 2 score += 500 if (updated_at > 1.week.ago) score -= 1000 if - follow_count = follows_dataset.count score -= 1000 if follow_count == 0 score += follow_count * 100 score += profile_comments_dataset.count * 5 @@ -1308,10 +1311,8 @@ class Site < Sequel::Model # New: - site_dataset = self.class.browse_dataset.association_left_join :follows + site_dataset = self.class.browse_dataset site_dataset.select_all! :sites - site_dataset.select_append! Sequel.lit("count(follows.site_id) AS follow_count") - site_dataset.group! :sites__id site_dataset.order! :follow_count.desc, :updated_at.desc site_dataset.where! "views >= #{SUGGESTIONS_VIEWS_MIN}" site_dataset.limit! limit-suggestions.length @@ -1329,7 +1330,9 @@ class Site < Sequel::Model end def screenshot_url(path, resolution) - "#{SCREENSHOTS_URL_ROOT}/#{self.class.sharding_dir(values[:username])}/#{values[:username]}/#{path}.#{resolution}.jpg" + out = '' + out = 'https://neocities.org/' if ENV['RACK_ENV'] == 'development' + out+"#{SCREENSHOTS_URL_ROOT}/#{self.class.sharding_dir(values[:username])}/#{values[:username]}/#{path}.#{resolution}.jpg" end def base_thumbnails_path diff --git a/views/home.erb b/views/home.erb index 6bfdea6d..f76e44c1 100644 --- a/views/home.erb +++ b/views/home.erb @@ -99,8 +99,7 @@
<%= site.views.format_large_number %> views
- <% follows_count = site.follows_dataset.count %> -
<%= follows_count.format_large_number %> follower<%= follows_count == 1 ? '' : 's' %>
+
<%= site.follow_count.format_large_number %> follower<%= site.follow_count == 1 ? '' : 's' %>
diff --git a/views/site.erb b/views/site.erb index 4e518933..177d0f6d 100644 --- a/views/site.erb +++ b/views/site.erb @@ -28,8 +28,7 @@ -->
<%= site.views.format_large_number %> view<%= site.views == 1 ? '' : 's' %>
- <% follows_count = site.follows_dataset.count %> -
<%= follows_count.format_large_number %> follower<%= follows_count == 1 ? '' : 's' %>
+
<%= site.follow_count.format_large_number %> follower<%= site.follow_count == 1 ? '' : 's' %>
<%= site.changed_count.format_large_number %> update<%= site.changed_count == 1 ? '' : 's' %>
<%= site.tips_dataset.count %> tips
diff --git a/views/site/stats.erb b/views/site/stats.erb index 3a924854..72392804 100644 --- a/views/site/stats.erb +++ b/views/site/stats.erb @@ -239,8 +239,7 @@
<%= site.views.format_large_number %> views
- <% follows_count = site.follows_dataset.count %> -
<%= follows_count.format_large_number %> follower<%= follows_count == 1 ? '' : 's' %>
+
<%= site.follow_count.format_large_number %> follower<%= site.follow_count == 1 ? '' : 's' %>