speed up and improve global activity

app/activity.rb

@@ -1,74 +1,9 @@
 get '/activity' do
-  #expires 7200, :public, :must_revalidate if self.class.production? # 2 hours
+  expires 7200, :public, :must_revalidate if self.class.production? # 2 hours
-  params[:activity] = 'global' # FIXME this is a bad hack
 
-  global_dataset = Event.global_dataset
+  @page = params[:page] || 1
-
+  @pagination_dataset = Event.global_dataset.paginate(@page.to_i, Event::GLOBAL_PAGINATION_LENGTH)
-  if params[:event_id]
+  @events = @pagination_dataset.all
-    global_dataset = global_dataset.where Sequel.qualify(:events, :id) => params[:event_id]
-  end
-
-=begin
-
-  initial_events = global_dataset.all
-  events = []
-
-  initial_events.each do |event|
-    site = Site.select(:id).where(id: event.site_id).first
-    actioning_site = Site.select(:id).where(id: event.actioning_site_id).first
-
-    disclude_event = false
-    disclude_event = true if site.is_a_jerk?
-
-    if event.tip_id
-      disclude_event = true if actioning_site && actioning_site.is_a_jerk?
-    else
-      disclude_event = true if actioning_site && (actioning_site.is_a_jerk? || actioning_site.follows_dataset.count < 2)
-    end
-
-    events.push(event) unless disclude_event
-  end
-
-  initial_site_change_events = Event.global_site_changes_dataset.limit(100).all
-  site_change_events = []
-
-  initial_site_change_events.each do |event|
-    site = Site.select(:id).where(id: event.site_id).first
-    site_change_events.push(event) if !site.is_a_jerk? && site.follows_dataset.count > 1
-  end
-
-  @events = []
-
-  events.each do |event|
-    unless site_change_events.empty?
-      until site_change_events.first.created_at < event.created_at
-        @events << site_change_events.shift
-        break if site_change_events.empty?
-      end
-    end
-    @events << event
-  end
-
-=end
-
-
-  if SimpleCache.expired?(:activity_event_ids)
-
-    initial_events = Event.global_site_changes_dataset.limit(500).all
-    @events = []
-    initial_events.each do |event|
-      event_site = event.site
-      next if @events.select {|e| e.site_id == event.site_id}.count >= 1
-      next if event_site.changed_count > 50_000 # Likely a bot
-      next if event_site.is_a_jerk?
-      next unless event_site.follows_dataset.count > 1
-      @events.push event
-    end
-
-    SimpleCache.store :activity_event_ids, @events.collect {|e| e.id}, 60.minutes
-  else
-    @events = Event.where(id: SimpleCache.get(:activity_event_ids)).order(:created_at.desc).all
-  end
 
   erb :'activity'
 end

app/index.rb

@@ -14,8 +14,6 @@ get '/?' do
       not_found if event.nil?
       not_found if event.is_deleted
       events_dataset = Event.where(id: params[:event_id]).paginate(1, 1)
-    elsif params[:activity] == 'global'
-      events_dataset = Event.global_dataset @page
     else
       events_dataset = current_site.news_feed(@page, 10)
     end

models/event.rb

@@ -13,9 +13,9 @@ class Event < Sequel::Model
   many_to_one :site
   many_to_one :actioning_site, key: :actioning_site_id, class: :Site
 
-  DEFAULT_GLOBAL_LIMIT = 300
+  PAGINATION_LENGTH = 10
-  GLOBAL_VIEWS_MINIMUM = 5
+  GLOBAL_PAGINATION_LENGTH = 20
-  GLOBAL_VIEWS_SITE_CHANGE_MINIMUM = 3_000
+  GLOBAL_SCORE_LIMIT = 3
 
   def undeleted_comments_count
     comments_dataset.exclude(is_deleted: true).count
@@ -26,43 +26,18 @@ class Event < Sequel::Model
   end
 
   def self.news_feed_default_dataset
-    if SimpleCache.expired?(:excluded_actioning_site_ids)
+    select(:events.*).
-      res = DB[%{select distinct(actioning_site_id) from events join sites on actioning_site_id=sites.id where sites.is_deleted='t'}].all.collect {|r| r[:actioning_site_id]}
+    join(:sites, id: :site_id).
-      excluded_actioning_site_ids = SimpleCache.store :excluded_actioning_site_ids, res, 2.hours
-    else
-      excluded_actioning_site_ids = SimpleCache.get :excluded_actioning_site_ids
-    end
-
-    ds = select_all(:events).
     order(:created_at.desc).
-      join_table(:inner, :sites, id: :site_id).
+    exclude(events__is_deleted: true).
-      where(follow_id: nil).
+    exclude(sites__is_deleted: true).
-      exclude(Sequel.qualify(:sites, :is_deleted) => true).
-      exclude(Sequel.qualify(:events, :is_deleted) => true).
-      exclude(is_banned: true)
-
-    unless excluded_actioning_site_ids.empty?
-      return ds.where("actioning_site_id is null or actioning_site_id not in ?", excluded_actioning_site_ids)
-    end
-
-    ds
-  end
-
-  def self.global_dataset(current_page=1, limit=DEFAULT_GLOBAL_LIMIT)
-    news_feed_default_dataset.
-      paginate(current_page.to_i, 100).
     exclude(is_nsfw: true).
     exclude(is_crashing: true).
-      where{views > GLOBAL_VIEWS_MINIMUM}.
+    where(follow_id: nil)
-      where(site_change_id: nil)
   end
 
-  def self.global_site_changes_dataset
+  def self.global_dataset
-    news_feed_default_dataset.
+    news_feed_default_dataset.where{score > GLOBAL_SCORE_LIMIT}
-      where{views > GLOBAL_VIEWS_SITE_CHANGE_MINIMUM}.
-      exclude(is_nsfw: true).
-      exclude(is_crashing: true).
-      exclude(site_change_id: nil)
   end
 
   def created_by?(site)

views/_news.erb

@@ -165,10 +165,8 @@
 
 <% end %>
 
-<% if params[:activity] != 'global' %>
 <div class="content">
   <%== erb :_pagination, layout: false %>
 </div>
-<% end %>
 
 <%== erb :'_news_templates', layout: false %>