Log stats for log date, rather than processing time. Add daily total

stats.

migrations/078_total_stat.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  up {
+    DB.create_table! :daily_site_stats do
+      primary_key :id
+      Date        :created_at,   index: true
+      Integer     :hits,         default: 0
+      Integer     :views,        default: 0
+      Integer     :comments,     default: 0
+      Integer     :follows,      default: 0
+      Integer     :site_updates, default: 0
+    end
+  }
+
+  down {
+    DB.drop_table :daily_site_stats
+  }
+end

migrations/079_add_total_stat_bandwidth.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :daily_site_stats, :bandwidth, :integer
+  }
+
+  down {
+    DB.drop_column :daily_site_stats, :bandwidth
+  }
+end

migrations/080_fix_total_stat_bandwidth.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up {
+    DB.drop_column :daily_site_stats, :bandwidth
+    DB.add_column :daily_site_stats, :bandwidth, :integer, default: 0
+  }
+
+  down {
+    DB.drop_column :daily_site_stats, :bandwidth
+    DB.add_column :daily_site_stats, :bandwidth, :integer
+  }
+end

models/daily_site_stat.rb

@@ -0,0 +1,2 @@
+class DailySiteStat < Sequel::Model
+end

models/stat.rb

@@ -15,8 +15,11 @@ class Stat < Sequel::Model
     end
 
     def parse_logfiles(path)
+      total_site_stats = {}
+
       Dir["#{path}/*.log"].each do |log_path|
         site_logs = {}
+
         logfile = File.open log_path, 'r'
 
         while hit = logfile.gets
@@ -26,9 +29,13 @@ class Stat < Sequel::Model
 
           time, username, size, path, ip, referrer = hit_array
 
+          log_time = Time.parse time
+
           next if !referrer.nil? && referrer.match(/bot/i)
 
-          site_logs[username] = {
+          site_logs[log_time] = {} unless site_logs[log_time]
+
+          site_logs[log_time][username] = {
             hits: 0,
             views: 0,
             bandwidth: 0,
@@ -36,78 +43,111 @@ class Stat < Sequel::Model
             ips: [],
             referrers: {},
             paths: {}
-          } unless site_logs[username]
+          } unless site_logs[log_time][username]
 
-          site_logs[username][:hits] += 1
+          total_site_stats[log_time] = {
-          site_logs[username][:bandwidth] += size.to_i
+            hits: 0,
+            views: 0,
+            bandwidth: 0
+          } unless total_site_stats[log_time]
 
-          unless site_logs[username][:view_ips].include?(ip)
+          site_logs[log_time][username][:hits] += 1
-            site_logs[username][:views] += 1
+          site_logs[log_time][username][:bandwidth] += size.to_i
-            site_logs[username][:view_ips] << ip
+
+          total_site_stats[log_time][:hits] += 1
+          total_site_stats[log_time][:bandwidth] += size.to_i
+
+          unless site_logs[log_time][username][:view_ips].include?(ip)
+            site_logs[log_time][username][:views] += 1
+
+            total_site_stats[log_time][:views] += 1
+
+            site_logs[log_time][username][:view_ips] << ip
 
             if referrer != '-' && !referrer.nil?
-              site_logs[username][:referrers][referrer] ||= 0
+              site_logs[log_time][username][:referrers][referrer] ||= 0
-              site_logs[username][:referrers][referrer] += 1
+              site_logs[log_time][username][:referrers][referrer] += 1
             end
           end
 
-          site_logs[username][:paths][path] ||= 0
+          site_logs[log_time][username][:paths][path] ||= 0
-          site_logs[username][:paths][path] += 1
+          site_logs[log_time][username][:paths][path] += 1
         end
 
         logfile.close
 
-        current_time = Time.now.utc
-        current_day_string = current_time.to_date.to_s
-
-        Site.select(:id, :username).where(username: site_logs.keys).all.each do |site|
-          site_logs[site.username][:id] = site.id
-        end
-
         DB.transaction do
-          site_logs.each do |username, site_log|
+          site_logs.each do |log_time, usernames|
-            DB['update sites set hits=hits+?, views=views+? where username=?',
+            Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
-               site_log[:hits],
+              site_logs[log_time][site.username][:id] = site.id
-               site_log[:views],
+            end
-               username
+
+            usernames.each do |username, site_log|
+              DB['update sites set hits=hits+?, views=views+? where username=?',
+                site_log[:hits],
+                site_log[:views],
+                username
               ].first
 
-            opts = {site_id: site_log[:id], created_at: current_day_string}
+              opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s}
 
-            stat = Stat.select(:id).where(opts).first
+              stat = nil
-            DB[:stats].lock('EXCLUSIVE') { stat = Stat.create opts } if stat.nil?
 
-            DB[
+              DB[:stats].lock('EXCLUSIVE') {
-              'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                stat = Stat.select(:id).where(opts).first
-              site_log[:hits],
+                stat = Stat.create opts if stat.nil?
-              site_log[:views],
+              }
-              site_log[:bandwidth],
+
-              stat.id
+              DB[
-            ].first
+                'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?',
+                site_log[:hits],
+                site_log[:views],
+                site_log[:bandwidth],
+                stat.id
+              ].first
 
 =begin
-            site_log[:referrers].each do |referrer, views|
+              site_log[:referrers].each do |referrer, views|
-              stat_referrer = StatReferrer.create_or_get site_log[:id], referrer
+                stat_referrer = StatReferrer.create_or_get site_log[:id], referrer
-              DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first
+                DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first
-            end
+              end
 
-            site_log[:view_ips].each do |ip|
+              site_log[:view_ips].each do |ip|
-              site_location = StatLocation.create_or_get site_log[:id], ip
+                site_location = StatLocation.create_or_get site_log[:id], ip
-              next if site_location.nil?
+                next if site_location.nil?
-              DB['update stat_locations set views=views+1 where id=?', site_location.id].first
+                DB['update stat_locations set views=views+1 where id=?', site_location.id].first
-            end
+              end
 
-            site_log[:paths].each do |path, views|
+              site_log[:paths].each do |path, views|
-              site_path = StatPath.create_or_get site_log[:id], path
+                site_path = StatPath.create_or_get site_log[:id], path
-              next if site_path.nil?
+                next if site_path.nil?
-              DB['update stat_paths set views=views+? where id=?', views, site_path.id].first
+                DB['update stat_paths set views=views+? where id=?', views, site_path.id].first
-            end
+              end
 =end
+            end
           end
         end
 
         FileUtils.rm log_path
       end
+
+      total_site_stats.each do |time, stats|
+        opts = {created_at: time.to_date.to_s}
+
+        DB[:stats].lock('EXCLUSIVE') {
+          stat = DailySiteStat.select(:id).where(opts).first
+          stat = DailySiteStat.create opts if stat.nil?
+        }
+
+        DB[
+          'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?',
+          stats[:hits],
+          stats[:views],
+          stats[:bandwidth],
+          time.to_date
+        ].first
+      end
+
     end
   end
 end

tests/stat_tests.rb

@@ -35,11 +35,34 @@ describe 'stats' do
     end
 
     Stat.parse_logfiles STAT_LOGS_PATH
+
     @site.stats.first.bandwidth.must_equal 612917*2
     #@site.stat_referrers.first.url.must_equal 'http://derp.com'
     #@site.stat_locations.first.city_name.must_equal 'Menlo Park'
   end
 
+  it 'takes accout for log hit time' do
+    @site = Fabricate :site
+    File.open("tests/stat_logs/#{SecureRandom.uuid}.log", 'w') do |file|
+      file.write "2015-05-01T21:16:35+00:00\t#{@site.username}\t612917\t/images/derpie space.png\t67.180.75.140\thttp://derp.com\n"
+      file.write "2015-05-02T21:16:35+00:00\t#{@site.username}\t612917\t/images/derpie space.png\t67.180.75.140\thttp://derp.com\n"
+    end
+
+    Stat.parse_logfiles STAT_LOGS_PATH
+
+    @site.stats.length.must_equal 2
+
+    [Date.new(2015, 5, 2), Date.new(2015, 5, 1)].each do |date|
+      stats = @site.stats.select {|stat| stat.created_at == date}
+      stats.length.must_equal 1
+      stat = stats.first
+      stat.hits.must_equal 1
+      stat.views.must_equal 1
+      stat.bandwidth.must_equal 612917
+    end
+
+  end
+
   it 'deals with spaces in referrer' do
     @site = Fabricate :site
     File.open("tests/stat_logs/#{SecureRandom.uuid}.log", 'w') do |file|
@@ -103,6 +126,7 @@ describe 'stats' do
   end
 
   it 'parses logfile' do
+    DB[:daily_site_stats].delete
     Stat.parse_logfiles STAT_LOGS_PATH
 
     @site_one.reload
@@ -149,5 +173,14 @@ describe 'stats' do
     #stat_paths.last.name.must_equal '/derp.html'
 
     # [geoip.city('67.180.75.140'), geoip.city('172.56.16.152')]
+
+    # Saves to daily_site_stats
+
+    DailySiteStat.count.must_equal 1
+    d = DailySiteStat.first
+    d.created_at.must_equal Date.new(@time.year, @time.month, @time.day)
+    d.hits.must_equal 7
+    d.views.must_equal 5
+    d.bandwidth.must_equal 35000
   end
 end