From 7d8ad81fd0393080b2aa66002a3d2252bbe26358 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Tue, 22 Dec 2015 04:37:16 -0600 Subject: [PATCH] Log stats for log date, rather than processing time. Add daily total stats. --- migrations/078_total_stat.rb | 17 +++ migrations/079_add_total_stat_bandwidth.rb | 9 ++ migrations/080_fix_total_stat_bandwidth.rb | 11 ++ models/daily_site_stat.rb | 2 + models/stat.rb | 134 +++++++++++++-------- tests/stat_tests.rb | 33 +++++ 6 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 migrations/078_total_stat.rb create mode 100644 migrations/079_add_total_stat_bandwidth.rb create mode 100644 migrations/080_fix_total_stat_bandwidth.rb create mode 100644 models/daily_site_stat.rb diff --git a/migrations/078_total_stat.rb b/migrations/078_total_stat.rb new file mode 100644 index 00000000..6698e2a9 --- /dev/null +++ b/migrations/078_total_stat.rb @@ -0,0 +1,17 @@ +Sequel.migration do + up { + DB.create_table! :daily_site_stats do + primary_key :id + Date :created_at, index: true + Integer :hits, default: 0 + Integer :views, default: 0 + Integer :comments, default: 0 + Integer :follows, default: 0 + Integer :site_updates, default: 0 + end + } + + down { + DB.drop_table :daily_site_stats + } +end diff --git a/migrations/079_add_total_stat_bandwidth.rb b/migrations/079_add_total_stat_bandwidth.rb new file mode 100644 index 00000000..a3d30feb --- /dev/null +++ b/migrations/079_add_total_stat_bandwidth.rb @@ -0,0 +1,9 @@ +Sequel.migration do + up { + DB.add_column :daily_site_stats, :bandwidth, :integer + } + + down { + DB.drop_column :daily_site_stats, :bandwidth + } +end diff --git a/migrations/080_fix_total_stat_bandwidth.rb b/migrations/080_fix_total_stat_bandwidth.rb new file mode 100644 index 00000000..3c87e95a --- /dev/null +++ b/migrations/080_fix_total_stat_bandwidth.rb @@ -0,0 +1,11 @@ +Sequel.migration do + up { + DB.drop_column :daily_site_stats, :bandwidth + DB.add_column :daily_site_stats, :bandwidth, :integer, default: 0 + } + + down { + DB.drop_column :daily_site_stats, :bandwidth + DB.add_column :daily_site_stats, :bandwidth, :integer + } +end diff --git a/models/daily_site_stat.rb b/models/daily_site_stat.rb new file mode 100644 index 00000000..cdcc3622 --- /dev/null +++ b/models/daily_site_stat.rb @@ -0,0 +1,2 @@ +class DailySiteStat < Sequel::Model +end diff --git a/models/stat.rb b/models/stat.rb index 687997ee..fe2ea15c 100644 --- a/models/stat.rb +++ b/models/stat.rb @@ -15,8 +15,11 @@ class Stat < Sequel::Model end def parse_logfiles(path) + total_site_stats = {} + Dir["#{path}/*.log"].each do |log_path| site_logs = {} + logfile = File.open log_path, 'r' while hit = logfile.gets @@ -26,9 +29,13 @@ class Stat < Sequel::Model time, username, size, path, ip, referrer = hit_array + log_time = Time.parse time + next if !referrer.nil? && referrer.match(/bot/i) - site_logs[username] = { + site_logs[log_time] = {} unless site_logs[log_time] + + site_logs[log_time][username] = { hits: 0, views: 0, bandwidth: 0, @@ -36,78 +43,111 @@ class Stat < Sequel::Model ips: [], referrers: {}, paths: {} - } unless site_logs[username] + } unless site_logs[log_time][username] - site_logs[username][:hits] += 1 - site_logs[username][:bandwidth] += size.to_i + total_site_stats[log_time] = { + hits: 0, + views: 0, + bandwidth: 0 + } unless total_site_stats[log_time] - unless site_logs[username][:view_ips].include?(ip) - site_logs[username][:views] += 1 - site_logs[username][:view_ips] << ip + site_logs[log_time][username][:hits] += 1 + site_logs[log_time][username][:bandwidth] += size.to_i + + total_site_stats[log_time][:hits] += 1 + total_site_stats[log_time][:bandwidth] += size.to_i + + unless site_logs[log_time][username][:view_ips].include?(ip) + site_logs[log_time][username][:views] += 1 + + total_site_stats[log_time][:views] += 1 + + site_logs[log_time][username][:view_ips] << ip if referrer != '-' && !referrer.nil? - site_logs[username][:referrers][referrer] ||= 0 - site_logs[username][:referrers][referrer] += 1 + site_logs[log_time][username][:referrers][referrer] ||= 0 + site_logs[log_time][username][:referrers][referrer] += 1 end end - site_logs[username][:paths][path] ||= 0 - site_logs[username][:paths][path] += 1 + site_logs[log_time][username][:paths][path] ||= 0 + site_logs[log_time][username][:paths][path] += 1 end logfile.close - current_time = Time.now.utc - current_day_string = current_time.to_date.to_s - - Site.select(:id, :username).where(username: site_logs.keys).all.each do |site| - site_logs[site.username][:id] = site.id - end - DB.transaction do - site_logs.each do |username, site_log| - DB['update sites set hits=hits+?, views=views+? where username=?', - site_log[:hits], - site_log[:views], - username + site_logs.each do |log_time, usernames| + Site.select(:id, :username).where(username: usernames.keys).all.each do |site| + site_logs[log_time][site.username][:id] = site.id + end + + usernames.each do |username, site_log| + DB['update sites set hits=hits+?, views=views+? where username=?', + site_log[:hits], + site_log[:views], + username ].first - opts = {site_id: site_log[:id], created_at: current_day_string} + opts = {site_id: site_log[:id], created_at: log_time.to_date.to_s} - stat = Stat.select(:id).where(opts).first - DB[:stats].lock('EXCLUSIVE') { stat = Stat.create opts } if stat.nil? + stat = nil - DB[ - 'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?', - site_log[:hits], - site_log[:views], - site_log[:bandwidth], - stat.id - ].first + DB[:stats].lock('EXCLUSIVE') { + stat = Stat.select(:id).where(opts).first + stat = Stat.create opts if stat.nil? + } + + DB[ + 'update stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where id=?', + site_log[:hits], + site_log[:views], + site_log[:bandwidth], + stat.id + ].first =begin - site_log[:referrers].each do |referrer, views| - stat_referrer = StatReferrer.create_or_get site_log[:id], referrer - DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first - end + site_log[:referrers].each do |referrer, views| + stat_referrer = StatReferrer.create_or_get site_log[:id], referrer + DB['update stat_referrers set views=views+? where site_id=?', views, site_log[:id]].first + end - site_log[:view_ips].each do |ip| - site_location = StatLocation.create_or_get site_log[:id], ip - next if site_location.nil? - DB['update stat_locations set views=views+1 where id=?', site_location.id].first - end + site_log[:view_ips].each do |ip| + site_location = StatLocation.create_or_get site_log[:id], ip + next if site_location.nil? + DB['update stat_locations set views=views+1 where id=?', site_location.id].first + end - site_log[:paths].each do |path, views| - site_path = StatPath.create_or_get site_log[:id], path - next if site_path.nil? - DB['update stat_paths set views=views+? where id=?', views, site_path.id].first - end + site_log[:paths].each do |path, views| + site_path = StatPath.create_or_get site_log[:id], path + next if site_path.nil? + DB['update stat_paths set views=views+? where id=?', views, site_path.id].first + end =end + end end end FileUtils.rm log_path end + + total_site_stats.each do |time, stats| + opts = {created_at: time.to_date.to_s} + + DB[:stats].lock('EXCLUSIVE') { + stat = DailySiteStat.select(:id).where(opts).first + stat = DailySiteStat.create opts if stat.nil? + } + + DB[ + 'update daily_site_stats set hits=hits+?, views=views+?, bandwidth=bandwidth+? where created_at=?', + stats[:hits], + stats[:views], + stats[:bandwidth], + time.to_date + ].first + end + end end end diff --git a/tests/stat_tests.rb b/tests/stat_tests.rb index 4216d39e..3c22fdeb 100644 --- a/tests/stat_tests.rb +++ b/tests/stat_tests.rb @@ -35,11 +35,34 @@ describe 'stats' do end Stat.parse_logfiles STAT_LOGS_PATH + @site.stats.first.bandwidth.must_equal 612917*2 #@site.stat_referrers.first.url.must_equal 'http://derp.com' #@site.stat_locations.first.city_name.must_equal 'Menlo Park' end + it 'takes accout for log hit time' do + @site = Fabricate :site + File.open("tests/stat_logs/#{SecureRandom.uuid}.log", 'w') do |file| + file.write "2015-05-01T21:16:35+00:00\t#{@site.username}\t612917\t/images/derpie space.png\t67.180.75.140\thttp://derp.com\n" + file.write "2015-05-02T21:16:35+00:00\t#{@site.username}\t612917\t/images/derpie space.png\t67.180.75.140\thttp://derp.com\n" + end + + Stat.parse_logfiles STAT_LOGS_PATH + + @site.stats.length.must_equal 2 + + [Date.new(2015, 5, 2), Date.new(2015, 5, 1)].each do |date| + stats = @site.stats.select {|stat| stat.created_at == date} + stats.length.must_equal 1 + stat = stats.first + stat.hits.must_equal 1 + stat.views.must_equal 1 + stat.bandwidth.must_equal 612917 + end + + end + it 'deals with spaces in referrer' do @site = Fabricate :site File.open("tests/stat_logs/#{SecureRandom.uuid}.log", 'w') do |file| @@ -103,6 +126,7 @@ describe 'stats' do end it 'parses logfile' do + DB[:daily_site_stats].delete Stat.parse_logfiles STAT_LOGS_PATH @site_one.reload @@ -149,5 +173,14 @@ describe 'stats' do #stat_paths.last.name.must_equal '/derp.html' # [geoip.city('67.180.75.140'), geoip.city('172.56.16.152')] + + # Saves to daily_site_stats + + DailySiteStat.count.must_equal 1 + d = DailySiteStat.first + d.created_at.must_equal Date.new(@time.year, @time.month, @time.day) + d.hits.must_equal 7 + d.views.must_equal 5 + d.bandwidth.must_equal 35000 end end