speed up stats parsing

models/stat.rb

@@ -92,27 +92,25 @@ class Stat < Sequel::Model
 
         logfile.close
 
+        DB[:stats].lock('EXCLUSIVE') do
+          DB.transaction do
             site_logs.each do |log_time, usernames|
               Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
-            site_logs[log_time][site.username][:id] = site.id
+                usernames[site.username][:id] = site.id
               end
 
               usernames.each do |username, site_log|
-            DB.transaction do
+                next unless site_log[:id]
-              DB['update sites set hits=hits+?, views=views+? where username=?',
-                site_log[:hits],
-                site_log[:views],
-                username
-              ].first
 
                 opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s}
-
-              stat = nil
-
-              DB[:stats].lock('EXCLUSIVE') {
                 stat = Stat.select(:id).where(opts).first
                 stat = Stat.create opts if stat.nil?
-              }
+
+                DB['update sites set hits=hits+?, views=views+? where id=?',
+                  site_log[:hits],
+                  site_log[:views],
+                  site_log[:id]
+                ].first
 
                 DB[
                   'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
@@ -124,17 +122,16 @@ class Stat < Sequel::Model
               end
             end
           end
-
+        end
         FileUtils.rm log_path
       end
 
+      DB[:daily_site_stats].lock('EXCLUSIVE') do
+        DB.transaction do
           total_site_stats.each do |time, stats|
             opts = {created_at: time.to_date.to_s}
-
-        DB[:stats].lock('EXCLUSIVE') {
               stat = DailySiteStat.select(:id).where(opts).first
               stat = DailySiteStat.create opts if stat.nil?
-        }
 
               DB[
                 'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?',
@@ -144,7 +141,8 @@ class Stat < Sequel::Model
                 time.to_date
               ].first
           end
-
+        end
+      end
     end
   end
 end