mirror of
https://github.com/neocities/neocities.git
synced 2025-07-03 01:23:22 +02:00
Refactor logs, granularity to one day, with testing
This commit is contained in:
parent
434deee366
commit
3bca5e8839
9 changed files with 353 additions and 37 deletions
208
models/stat.rb
208
models/stat.rb
|
@ -1,3 +1,209 @@
|
|||
class Stat < Sequel::Model
|
||||
GEOCITY_PATH = './files/GeoLiteCity.dat'
|
||||
|
||||
many_to_one :site
|
||||
end
|
||||
one_to_many :stat_referrers
|
||||
one_to_many :stat_locations
|
||||
one_to_many :stat_paths
|
||||
|
||||
class << self
|
||||
def parse_logfiles(path)
|
||||
Dir["#{path}/*.log"].each do |log_path|
|
||||
site_logs = {}
|
||||
logfile = File.open log_path, 'r'
|
||||
|
||||
while hit = logfile.gets
|
||||
time, username, size, path, ip, referrer = hit.split ' '
|
||||
|
||||
next if referrer.match /bot/i
|
||||
|
||||
site_logs[username] = {
|
||||
hits: 0,
|
||||
views: 0,
|
||||
view_ips: []
|
||||
} unless site_logs[username]
|
||||
|
||||
site_logs[username][:hits] += 1
|
||||
|
||||
unless site_logs[username][:view_ips].include?(ip)
|
||||
site_logs[username][:views] += 1
|
||||
site_logs[username][:view_ips] << ip
|
||||
end
|
||||
end
|
||||
|
||||
logfile.close
|
||||
|
||||
current_time = Time.now.utc
|
||||
current_day_string = current_time.to_date.to_s
|
||||
|
||||
Site.select(:id, :username).where(username: site_logs.keys).all.each do |site|
|
||||
site_logs[site.username][:id] = site.id
|
||||
end
|
||||
|
||||
DB.transaction do
|
||||
site_logs.each do |username, site_log|
|
||||
DB['update sites set hits=hits+?, views=views+? where username=?',
|
||||
site_log[:hits],
|
||||
site_log[:views],
|
||||
username
|
||||
].first
|
||||
|
||||
opts = {site_id: site_log[:id], created_at: current_day_string}
|
||||
|
||||
stat = Stat.select(:id).where(opts).first
|
||||
DB[:stats].lock('EXCLUSIVE') { stat = Stat.create opts } if stat.nil?
|
||||
|
||||
DB[
|
||||
'update stats set hits=hits+?, views=views+? where site_id=?',
|
||||
site_log[:hits],
|
||||
site_log[:views],
|
||||
site_log[:id]
|
||||
].first
|
||||
end
|
||||
end
|
||||
|
||||
FileUtils.rm log_path
|
||||
end
|
||||
end
|
||||
|
||||
def get_or_create
|
||||
DB[:stats].lock 'EXCLUSIVE' do
|
||||
stat = Stat.where(opts).first
|
||||
stat ||= Stat.new opts
|
||||
stat.hits += site_log[:hits]
|
||||
stat.views += site_log[:views]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
=begin
|
||||
require 'io/extra'
|
||||
require 'geoip'
|
||||
|
||||
# Note: This isn't really a class right now.
|
||||
module Stat
|
||||
|
||||
|
||||
class << self
|
||||
def parse_logfiles(path)
|
||||
Dir["#{path}/*.log"].each do |logfile_path|
|
||||
parse_logfile logfile_path
|
||||
FileUtils.rm logfile_path
|
||||
end
|
||||
end
|
||||
|
||||
def parse_logfile(path)
|
||||
geoip = GeoIP.new GEOCITY_PATH
|
||||
logfile = File.open path, 'r'
|
||||
|
||||
hits = []
|
||||
|
||||
while hit = logfile.gets
|
||||
time, username, size, path, ip, referrer = hit.split ' '
|
||||
|
||||
site = Site.select(:id).where(username: username).first
|
||||
next unless site
|
||||
|
||||
paths_dataset = StatsDB[:paths]
|
||||
path_record = paths_dataset[name: path]
|
||||
path_id = path_record ? path_record[:id] : paths_dataset.insert(name: path)
|
||||
|
||||
referrers_dataset = StatsDB[:referrers]
|
||||
referrer_record = referrers_dataset[name: referrer]
|
||||
referrer_id = referrer_record ? referrer_record[:id] : referrers_dataset.insert(name: referrer)
|
||||
|
||||
location_id = nil
|
||||
|
||||
if city = geoip.city(ip)
|
||||
locations_dataset = StatsDB[:locations].select(:id)
|
||||
location_hash = {country_code2: city.country_code2, region_name: city.region_name, city_name: city.city_name}
|
||||
|
||||
location = locations_dataset.where(location_hash).first
|
||||
location_id = location ? location[:id] : locations_dataset.insert(location_hash)
|
||||
end
|
||||
|
||||
hits << [site.id, referrer_id, path_id, location_id, size, time]
|
||||
end
|
||||
|
||||
StatsDB[:hits].import(
|
||||
[:site_id, :referrer_id, :path_id, :location_id, :bytes_sent, :logged_at],
|
||||
hits
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
=begin
|
||||
def parse_logfile(path)
|
||||
hits = {}
|
||||
visits = {}
|
||||
visit_ips = {}
|
||||
|
||||
logfile = File.open path, 'r'
|
||||
|
||||
while hit = logfile.gets
|
||||
time, username, size, path, ip, referrer = hit.split ' '
|
||||
|
||||
hits[username] ||= 0
|
||||
hits[username] += 1
|
||||
visit_ips[username] = [] if !visit_ips[username]
|
||||
|
||||
unless visit_ips[username].include? ip
|
||||
visits[username] ||= 0
|
||||
visits[username] += 1
|
||||
visit_ips[username] << ip
|
||||
end
|
||||
end
|
||||
|
||||
logfile.close
|
||||
|
||||
|
||||
hits.each do |username,hitcount|
|
||||
DB['update sites set hits=hits+? where username=?', hitcount, username].first
|
||||
end
|
||||
|
||||
visits.each do |username,visitcount|
|
||||
DB['update sites set views=views+? where username=?', visitcount, username].first
|
||||
end
|
||||
end
|
||||
end
|
||||
=end
|
||||
|
||||
=begin
|
||||
def self.parse(logfile_path)
|
||||
hits = {}
|
||||
visits = {}
|
||||
visit_ips = {}
|
||||
|
||||
logfile = File.open logfile_path, 'r'
|
||||
|
||||
while hit = logfile.gets
|
||||
time, username, size, path, ip = hit.split ' '
|
||||
|
||||
hits[username] ||= 0
|
||||
hits[username] += 1
|
||||
|
||||
visit_ips[username] = [] if !visit_ips[username]
|
||||
|
||||
unless visit_ips[username].include?(ip)
|
||||
visits[username] ||= 0
|
||||
visits[username] += 1
|
||||
visit_ips[username] << ip
|
||||
end
|
||||
end
|
||||
|
||||
logfile.close
|
||||
|
||||
hits.each do |username,hitcount|
|
||||
DB['update sites set hits=hits+? where username=?', hitcount, username].first
|
||||
end
|
||||
|
||||
visits.each do |username,visitcount|
|
||||
DB['update sites set views=views+? where username=?', visitcount, username].first
|
||||
end
|
||||
end
|
||||
=end
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue