From a67588a1c036cf6174481b7418dd62547316e6c1 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Mon, 26 Aug 2024 21:53:10 -0500 Subject: [PATCH] speed up and improve global activity --- app/activity.rb | 73 +++---------------------------------------------- app/index.rb | 2 -- models/event.rb | 51 +++++++++------------------------- views/_news.erb | 8 ++---- 4 files changed, 20 insertions(+), 114 deletions(-) diff --git a/app/activity.rb b/app/activity.rb index 73935767..201ebbc3 100644 --- a/app/activity.rb +++ b/app/activity.rb @@ -1,74 +1,9 @@ get '/activity' do - #expires 7200, :public, :must_revalidate if self.class.production? # 2 hours - params[:activity] = 'global' # FIXME this is a bad hack + expires 7200, :public, :must_revalidate if self.class.production? # 2 hours - global_dataset = Event.global_dataset - - if params[:event_id] - global_dataset = global_dataset.where Sequel.qualify(:events, :id) => params[:event_id] - end - -=begin - - initial_events = global_dataset.all - events = [] - - initial_events.each do |event| - site = Site.select(:id).where(id: event.site_id).first - actioning_site = Site.select(:id).where(id: event.actioning_site_id).first - - disclude_event = false - disclude_event = true if site.is_a_jerk? - - if event.tip_id - disclude_event = true if actioning_site && actioning_site.is_a_jerk? - else - disclude_event = true if actioning_site && (actioning_site.is_a_jerk? || actioning_site.follows_dataset.count < 2) - end - - events.push(event) unless disclude_event - end - - initial_site_change_events = Event.global_site_changes_dataset.limit(100).all - site_change_events = [] - - initial_site_change_events.each do |event| - site = Site.select(:id).where(id: event.site_id).first - site_change_events.push(event) if !site.is_a_jerk? && site.follows_dataset.count > 1 - end - - @events = [] - - events.each do |event| - unless site_change_events.empty? - until site_change_events.first.created_at < event.created_at - @events << site_change_events.shift - break if site_change_events.empty? - end - end - @events << event - end - -=end - - - if SimpleCache.expired?(:activity_event_ids) - - initial_events = Event.global_site_changes_dataset.limit(500).all - @events = [] - initial_events.each do |event| - event_site = event.site - next if @events.select {|e| e.site_id == event.site_id}.count >= 1 - next if event_site.changed_count > 50_000 # Likely a bot - next if event_site.is_a_jerk? - next unless event_site.follows_dataset.count > 1 - @events.push event - end - - SimpleCache.store :activity_event_ids, @events.collect {|e| e.id}, 60.minutes - else - @events = Event.where(id: SimpleCache.get(:activity_event_ids)).order(:created_at.desc).all - end + @page = params[:page] || 1 + @pagination_dataset = Event.global_dataset.paginate(@page.to_i, Event::GLOBAL_PAGINATION_LENGTH) + @events = @pagination_dataset.all erb :'activity' end diff --git a/app/index.rb b/app/index.rb index 679201cc..ef3f2f59 100644 --- a/app/index.rb +++ b/app/index.rb @@ -14,8 +14,6 @@ get '/?' do not_found if event.nil? not_found if event.is_deleted events_dataset = Event.where(id: params[:event_id]).paginate(1, 1) - elsif params[:activity] == 'global' - events_dataset = Event.global_dataset @page else events_dataset = current_site.news_feed(@page, 10) end diff --git a/models/event.rb b/models/event.rb index f59fb4e8..2309e8fc 100644 --- a/models/event.rb +++ b/models/event.rb @@ -13,9 +13,9 @@ class Event < Sequel::Model many_to_one :site many_to_one :actioning_site, key: :actioning_site_id, class: :Site - DEFAULT_GLOBAL_LIMIT = 300 - GLOBAL_VIEWS_MINIMUM = 5 - GLOBAL_VIEWS_SITE_CHANGE_MINIMUM = 3_000 + PAGINATION_LENGTH = 10 + GLOBAL_PAGINATION_LENGTH = 20 + GLOBAL_SCORE_LIMIT = 3 def undeleted_comments_count comments_dataset.exclude(is_deleted: true).count @@ -26,43 +26,18 @@ class Event < Sequel::Model end def self.news_feed_default_dataset - if SimpleCache.expired?(:excluded_actioning_site_ids) - res = DB[%{select distinct(actioning_site_id) from events join sites on actioning_site_id=sites.id where sites.is_deleted='t'}].all.collect {|r| r[:actioning_site_id]} - excluded_actioning_site_ids = SimpleCache.store :excluded_actioning_site_ids, res, 2.hours - else - excluded_actioning_site_ids = SimpleCache.get :excluded_actioning_site_ids - end - - ds = select_all(:events). - order(:created_at.desc). - join_table(:inner, :sites, id: :site_id). - where(follow_id: nil). - exclude(Sequel.qualify(:sites, :is_deleted) => true). - exclude(Sequel.qualify(:events, :is_deleted) => true). - exclude(is_banned: true) - - unless excluded_actioning_site_ids.empty? - return ds.where("actioning_site_id is null or actioning_site_id not in ?", excluded_actioning_site_ids) - end - - ds + select(:events.*). + join(:sites, id: :site_id). + order(:created_at.desc). + exclude(events__is_deleted: true). + exclude(sites__is_deleted: true). + exclude(is_nsfw: true). + exclude(is_crashing: true). + where(follow_id: nil) end - def self.global_dataset(current_page=1, limit=DEFAULT_GLOBAL_LIMIT) - news_feed_default_dataset. - paginate(current_page.to_i, 100). - exclude(is_nsfw: true). - exclude(is_crashing: true). - where{views > GLOBAL_VIEWS_MINIMUM}. - where(site_change_id: nil) - end - - def self.global_site_changes_dataset - news_feed_default_dataset. - where{views > GLOBAL_VIEWS_SITE_CHANGE_MINIMUM}. - exclude(is_nsfw: true). - exclude(is_crashing: true). - exclude(site_change_id: nil) + def self.global_dataset + news_feed_default_dataset.where{score > GLOBAL_SCORE_LIMIT} end def created_by?(site) diff --git a/views/_news.erb b/views/_news.erb index ec8d5091..d05961e6 100644 --- a/views/_news.erb +++ b/views/_news.erb @@ -165,10 +165,8 @@ <% end %> -<% if params[:activity] != 'global' %> -
- <%== erb :_pagination, layout: false %> -
-<% end %> +
+ <%== erb :_pagination, layout: false %> +
<%== erb :'_news_templates', layout: false %>