speed up stats parsing

models/stat.rb

@@ -92,59 +92,57 @@ class Stat < Sequel::Model
 
         logfile.close
 
-        site_logs.each do |log_time, usernames|
+        DB[:stats].lock('EXCLUSIVE') do
-          Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
+          DB.transaction do
-            site_logs[log_time][site.username][:id] = site.id
+            site_logs.each do |log_time, usernames|
-          end
+              Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
+                usernames[site.username][:id] = site.id
+              end
 
-          usernames.each do |username, site_log|
+              usernames.each do |username, site_log|
-            DB.transaction do
+                next unless site_log[:id]
-              DB['update sites set hits=hits+?, views=views+? where username=?',
-                site_log[:hits],
-                site_log[:views],
-                username
-              ].first
 
-              opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s}
+                opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s}
-
-              stat = nil
-
-              DB[:stats].lock('EXCLUSIVE') {
                 stat = Stat.select(:id).where(opts).first
                 stat = Stat.create opts if stat.nil?
-              }
 
-              DB[
+                DB['update sites set hits=hits+?, views=views+? where id=?',
-                'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                  site_log[:hits],
-                site_log[:hits],
+                  site_log[:views],
-                site_log[:views],
+                  site_log[:id]
-                site_log[:bandwidth],
+                ].first
-                stat.id
+
-              ].first
+                DB[
+                  'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                  site_log[:hits],
+                  site_log[:views],
+                  site_log[:bandwidth],
+                  stat.id
+                ].first
+              end
             end
           end
         end
-
         FileUtils.rm log_path
       end
 
-      total_site_stats.each do |time, stats|
+      DB[:daily_site_stats].lock('EXCLUSIVE') do
-        opts = {created_at: time.to_date.to_s}
+        DB.transaction do
+          total_site_stats.each do |time, stats|
+            opts = {created_at: time.to_date.to_s}
+              stat = DailySiteStat.select(:id).where(opts).first
+              stat = DailySiteStat.create opts if stat.nil?
 
-        DB[:stats].lock('EXCLUSIVE') {
+              DB[
-          stat = DailySiteStat.select(:id).where(opts).first
+                'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?',
-          stat = DailySiteStat.create opts if stat.nil?
+                stats[:hits],
-        }
+                stats[:views],
-
+                stats[:bandwidth],
-        DB[
+                time.to_date
-          'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?',
+              ].first
-          stats[:hits],
+          end
-          stats[:views],
+        end
-          stats[:bandwidth],
-          time.to_date
-        ].first
       end
-
     end
   end
 end