major improvements to special sauce, site browsing, move front page to use non-featured, and fix bug when title missing

app/browse.rb

@@ -63,40 +63,43 @@ def browse_sites_dataset
     return ds
   end
 
+  params[:sort_by] ||= 'followers'
+
   case params[:sort_by]
     when 'special_sauce'
-      ds = ds.exclude score: nil
+      ds = ds.where{score > 1}
-      ds = ds.order :score.desc
+      ds = ds.order(:score.desc, :follow_count.desc, :views.desc, :site_updated_at.desc)
-    when 'followers'
-      ds = ds.order :follow_count.desc, :updated_at.desc
     when 'supporters'
       ds = ds.where sites__id: Site.supporter_ids
       ds = ds.order :follow_count.desc, :views.desc, :site_updated_at.desc
-    when 'featured'
+    #when 'featured'
-      ds = ds.exclude featured_at: nil
+    #  ds = ds.exclude featured_at: nil
-      ds = ds.order :featured_at.desc
+    #  ds = ds.order :featured_at.desc
-    when 'hits'
+    #when 'hits'
-      ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
+    #  ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
-      ds = ds.order(:hits.desc, :site_updated_at.desc)
+    #  ds = ds.order(:hits.desc, :site_updated_at.desc)
-    when 'most_views'
+    #when 'most_views'
-      ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
+    #  ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
-      ds = ds.order(:views.desc, :site_updated_at.desc)
+    #  ds = ds.order(:views.desc, :site_updated_at.desc)
-    when 'least_views'
+    #when 'least_views'
-      ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
+    #  ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
-      ds = ds.order(:views.asc, :site_updated_at.desc)
+    #  ds = ds.order(:views.asc, :site_updated_at.desc)
     when 'newest'
       ds = ds.order(:created_at.desc, :views.desc)
-    when 'oldest'
+    #when 'oldest'
-      ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
+    #  ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
-      ds = ds.order(:created_at, :views.desc)
+    #  ds = ds.order(:created_at, :views.desc)
     when 'random'
       ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
-      ds = ds.where 'random() < 0.01'
+      ds = ds.where{score > 5}
+      ds = ds.order(Sequel.lit('RANDOM()'))
+      #ds = ds.where 'random() < 0.01'
     when 'last_updated'
+      ds = ds.where{score > 5}
       ds = ds.where{views > Site::BROWSE_MINIMUM_VIEWS}
       params[:sort_by] = 'last_updated'
       ds = ds.exclude(site_updated_at: nil)
-      ds = ds.order(:site_updated_at.desc, :views.desc)
+      ds = ds.order(:site_updated_at.desc)
     when 'tipping_enabled'
       ds = ds.where tipping_enabled: true
       ds = ds.where("(tipping_paypal is not null and tipping_paypal != '') or (tipping_bitcoin is not null and tipping_bitcoin != '')")
@@ -109,10 +112,11 @@ def browse_sites_dataset
       ds = ds.inner_join :blocks, :site_id => :id
       ds = ds.group :sites__id
       ds = ds.order :total.desc
-    else
+    when 'followers'
       params[:sort_by] = 'followers'
-      ds = ds.where{views > Site::BROWSE_MINIMUM_FOLLOWER_VIEWS}
+      ds = ds.where{follow_count > 0}
-      ds = ds.order :follow_count.desc, :views.desc, :updated_at.desc
+      ds = ds.where{updated_at > 9.months.ago}
+      ds = ds.order :follow_count.desc, :score.desc, :updated_at.desc
   end
 
   ds = ds.where ['sites.is_nsfw = ?', (params[:is_nsfw] == 'true' ? true : false)]

migrations/126_sites_score_real.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+    up {
+      alter_table(:sites) do
+        set_column_type :score, :real
+      end
+    }
+  
+    down {
+      alter_table(:sites) do
+        set_column_type :score, :decimal
+      end
+    }
+  end

models/site.rb

@@ -1449,43 +1449,144 @@ class Site < Sequel::Model
   end
 
   def self.compute_scores
-    select(:id, :username, :created_at, :updated_at, :views, :featured_at, :changed_count, :api_calls).exclude(is_banned: true).exclude(is_crashing: true).exclude(is_nsfw: true).exclude(updated_at: nil).where(site_changed: true).all.each do |s|
+    sites = select(:id, :username, :created_at, :updated_at, :views, :featured_at, :changed_count, :api_calls, :follow_count, :score)
-      s.score = s.compute_score
+            .exclude(is_deleted: true)
-      s.save_changes validate: false
+            .exclude(is_crashing: true)
+            .exclude(updated_at: nil)
+            .where(site_changed: true)
+            #.all
+            #.where { changed_count > 25 }
+            #.where { created_at < 1.week.ago }
+            #.where { follow_count > 4 }
+            #.where { views > 10_000 }
+            .all
+
+    site_ids = sites.map(&:id)
+
+    likes_counts = DB[:events]
+                   .join(:likes, event_id: :id)
+                   .where(site_id: site_ids)
+                   .group_and_count(:site_id)
+                   .map { |r| [r[:site_id], r[:count]] }.to_h
+
+    profile_comments_counts = DB[:events]
+                              .exclude(profile_comment_id: nil)
+                              .exclude(is_deleted: true)
+                              .where(site_id: site_ids)
+                              .group_and_count(:site_id)
+                              .map { |r| [r[:site_id], r[:count]] }.to_h
+
+    comment_counts = DB[:comments]
+                     .where(actioning_site_id: site_ids)
+                     .exclude(is_deleted: true)
+                     .group_and_count(:actioning_site_id)
+                     .map { |r| [r[:actioning_site_id], r[:count]] }.to_h
+
+    blocks_counts = DB[:blocks]
+                    .where(site_id: site_ids)
+                    .group_and_count(:site_id)
+                    .map { |r| [r[:site_id], r[:count]] }.to_h
+
+    follow_counts = DB[:follows]
+                    .join(:sites, id: :actioning_site_id  )
+                    .where(site_id: site_ids)
+                    .exclude(Sequel[:sites][:is_deleted] => true)
+                    .exclude(Sequel[:sites][:profile_enabled] => false)
+                    .group_and_count(:site_id)
+                    .map { |r| [r[:site_id], r[:count]] }.to_h
+
+    followings_counts = DB[:follows]
+                    .join(:sites, id: :site_id)
+                    .where(actioning_site_id: site_ids)
+                    .exclude(Sequel[:sites][:is_deleted] => true)
+                    .exclude(Sequel[:sites][:profile_enabled] => false)
+                    .group_and_count(:actioning_site_id)
+                    .map { |r| [r[:actioning_site_id], r[:count]] }.to_h
+
+    site_files_counts = DB[:site_files]
+                        .where(site_id: site_ids)
+                        .group_and_count(:site_id)
+                        .map { |r| [r[:site_id], r[:count]] }.to_h
+
+    updates = []
+
+    max_points = 100
+
+    follow_count_weight     = 30
+    views_weight            = 20
+    feature_bonus_weight    = 20
+    likes_weight            = 20
+    profile_comments_weight = 10
+
+    follow_count_factor = 0.1
+    views_factor = 0.005
+    profile_comments_factor = 0.01
+    likes_factor = 0.01
+
+    sites.each do |site|
+      points = 0
+
+      #$debug = true if site.username == 'username'
+
+      points += [(site.follow_count || 0) * follow_count_factor, follow_count_weight].min
+      puts "follows #{points}" if $debug
+      points += [(site.views * views_factor).to_i, views_weight].min
+      puts "views #{points}" if $debug
+      points += feature_bonus_weight if !site.featured_at.nil?
+      puts "featured #{points}" if $debug
+
+      likes_count = likes_counts[site.id] || 0
+      points += [likes_count * likes_factor, likes_weight].min
+      puts "likes #{points}" if $debug
+
+      profile_comments_count = profile_comments_counts[site.id] || 0
+      points += [profile_comments_count * profile_comments_factor, profile_comments_weight].min
+      puts "profile_comments #{points}" if $debug
+
+      blocks_count = blocks_counts[site.id] || 0
+      follow_count = follow_counts[site.id] || 0
+      followings_count = followings_counts[site.id] || 0
+      site_files_count = site_files_counts[site.id] || 0
+
+      if (site.api_calls && site.api_calls > 500_000) || follow_count == 0 && blocks_count > 5 || ((blocks_count / follow_count.to_f) > 0.06)
+        points *= 0.1
+      end
+
+      puts "api_call, blocks #{points}" if $debug
+
+      if followings_count < 5 || site_files_count < 10
+        points *= 0.5
+      end
+
+      comment_count = comment_counts[site.id] || 0
+      points *= 0.5 if comment_count < 20
+      puts "comment_count #{points}" if $debug
+
+      time_score_gravity = 0.3
+      time_factor = ((Time.now - site.updated_at) / 1.days)
+
+      points = (points / (time_factor**time_score_gravity)) if time_factor > 0
+      puts "time #{points}" if $debug
+
+      points = [points, max_points].min
+      puts "max_points #{points}" if $debug
+
+      score = [0, points].max
+
+      exit if $debug
+
+      if score != site.score
+        updates << { id: site.id, score: score }
+      end
+    end
+
+    DB.synchronize do |conn|
+      updates.each do |update|
+        conn.async_exec_params("UPDATE sites SET score = $1 WHERE id = $2", [update[:score], update[:id]])
+      end
     end
   end
 
-  SCORE_GRAVITY = 1.8
-
-  def compute_score
-    points = 0
-    points += (follow_count || 0) * 30
-    points += profile_comments_dataset.count * 1
-    points += views / 1000
-    points += 20 if !featured_at.nil?
-
-    # penalties
-    points = 0 if changed_count < 2
-    points = 0 if api_calls && api_calls > 1000
-
-    (points / ((Time.now - updated_at) / 7.days)**SCORE_GRAVITY).round(4)
-  end
-
-=begin
-  def compute_score
-    score = 0
-    score += (Time.now - created_at) / 1.day
-    score -= ((Time.now - updated_at) / 1.day) * 2
-    score += 500 if (updated_at > 1.week.ago)
-    score -= 1000 if
-    score -= 1000 if follow_count == 0
-    score += follow_count * 100
-    score += profile_comments_dataset.count * 5
-    score += profile_commentings_dataset.count
-    score.to_i
-  end
-=end
-
   def self.browse_dataset
     dataset.select(:id,:username,:hits,:views,:created_at,:plan_type,:parent_site_id,:domain,:score,:title).
       where(is_deleted: false, is_banned: false, is_crashing: false, site_changed: true)

sass/_project-sass/_project-Website-Gallery.scss

@@ -155,7 +155,7 @@
   li {
     width: 14.5%;
     margin-right: 1.5%;
-    margin-bottom: 0;
+    margin-bottom: 10px;
 
     @media (max-device-width:480px), screen and (max-width:800px){
       width:45%;

views/browse.erb

@@ -15,18 +15,18 @@
             <label class="text-Label" for="sort_by">Sort by:</label>
             <div class="select-Container">
               <select name="sort_by" id="sort_by" class="input-Select">
-                <option value="followers" <%= params[:sort_by] == 'followers' ? 'selected' : '' %>>Most Followed</option>
+                <option value="followers" <%= params[:sort_by] == 'followers' ? 'selected' : '' %>>Most Followed</option>\
-                <option value="special_sauce" <%= params[:sort_by] == 'special_sauce' ? 'selected' : '' %>>Special Sauce (alpha)</option>
+                <option value="special_sauce" <%= params[:sort_by] == 'special_sauce' ? 'selected' : '' %>>Special Sauce</option>
+                <option value="random" <%= params[:sort_by] == 'random' ? 'selected' : '' %>>Random</option>
                 <option value="last_updated" <%= params[:sort_by] == 'last_updated' ? 'selected' : '' %>>Last Updated</option>
                 <option value="supporters" <%= params[:sort_by] == 'supporters' ? 'selected' : '' %>>Neocities Supporters</option>
-                <option value="featured" <%= params[:sort_by] == 'featured' ? 'selected' : '' %>>Featured</option>
+                <!-- <option value="featured" <%= params[:sort_by] == 'featured' ? 'selected' : '' %>>Featured</option> -->
                 <option value="tipping_enabled" <%= params[:sort_by] == 'tipping_enabled' ? 'selected' : '' %>>Accepting Tips</option>
-                <option value="most_views" <%= params[:sort_by] == 'most_views' ? 'selected' : '' %>>Most Views</option>
+                <!-- <option value="most_views" <%= params[:sort_by] == 'most_views' ? 'selected' : '' %>>Most Views</option> -->
-                <option value="least_views" <%= params[:sort_by] == 'least_views' ? 'selected' : '' %>>Least Views</option>
+                <!-- <option value="least_views" <%= params[:sort_by] == 'least_views' ? 'selected' : '' %>>Least Views</option> -->
-                <option value="hits" <%= params[:sort_by] == 'hits' ? 'selected' : '' %>>Most Hits</option>
+                <!-- <option value="hits" <%= params[:sort_by] == 'hits' ? 'selected' : '' %>>Most Hits</option> -->
                 <option value="newest" <%= params[:sort_by] == 'newest' ? 'selected' : '' %>>Newest</option>
-                <option value="oldest" <%= params[:sort_by] == 'oldest' ? 'selected' : '' %>>Oldest</option>
+                <!-- <option value="oldest" <%= params[:sort_by] == 'oldest' ? 'selected' : '' %>>Oldest</option> -->
-                <option value="random" <%= params[:sort_by] == 'random' ? 'selected' : '' %>>Random</option>
               </select>
             </div>
 
@@ -79,7 +79,7 @@
         <div class="title">
           <a href="<%= site.uri %>"
              title="<%= site.title %>"
-          ><%= site.title.shorten(30) %></a>
+          ><%= site.title.shorten(30) %></a>&nbsp;
         </div>
         <div class="site-info">
           <div class="username">
@@ -136,6 +136,10 @@
 
 <% if params[:sort_by] != 'random' %>
   <%== erb :'_pagination', layout: false %>
+<% else %>
+  <div class="txt-Center eps pagination" style="margin-top: 0"  >
+    <a href="/browse?sort_by=random&tag=<%= Rack::Utils.escape params[:tag] %>" class="btn-Action txt-Center" style="margin: 0 auto;">More</a>
+  </div>
 <% end %>
 
 <% unless is_education? %>

views/index.erb

@@ -164,10 +164,10 @@
 <div class="nav prev"></div>
 -->
 <ul class="website-Gallery hp-Gallery">
-  <% Site.featured.each do |site| %>
+  <% Site.order(:score.desc).limit(12).all.shuffle.each do |site| %>
     <li>
       <a href="<%= site.uri %>" title="<%= site.title %>" target="_blank">
-        <img src="<%= site.screenshot_url 'index.html', '210x158' %>" class="neo-SS" alt="<%= site.title %>" />
+        <img src="<%= site.screenshot_url 'index.html', '540x405' %>" class="neo-SS" alt="<%= site.title %>" />
       </a>
     </li>
   <% end %>
@@ -175,7 +175,7 @@
 <!--
 <div class="nav next"></div>
 -->
-<a href="/browse" class="btn-Action float-Right">Browse all sites</a>
+<a href="/browse" class="btn-Action float-Right">Browse more sites</a>
 </div>
 
 <div class="section previews">