diff --git a/Rakefile b/Rakefile index d15b9229..c7fc9814 100644 --- a/Rakefile +++ b/Rakefile @@ -31,10 +31,8 @@ end desc "parse logs" task :parse_logs => [:environment] do - Dir[File.join($config['logs_path'], '*.log')].each do |logfile_path| - Stat.parse logfile_path - FileUtils.rm logfile_path - end + Stat.parse_logfiles $config['logs_path'] + Stat.prune! end desc 'Update banned IPs list' diff --git a/config.yml.travis b/config.yml.travis index 627467e9..876179cf 100644 --- a/config.yml.travis +++ b/config.yml.travis @@ -6,4 +6,5 @@ recaptcha_private_key: '5678' phantomjs_url: - http://localhost:8910 ip_hash_salt: "400$8$1$fc21863da5d531c1" -email_unsubscribe_token: "somethingrandomderrrrp" \ No newline at end of file +email_unsubscribe_token: "somethingrandomderrrrp" +logs_path: "/tmp/neocitiestestlogs" diff --git a/files/GeoLiteCity.dat b/files/GeoLiteCity.dat new file mode 100644 index 00000000..b9841afb Binary files /dev/null and b/files/GeoLiteCity.dat differ diff --git a/models/stat.rb b/models/stat.rb index c6540427..540bee9d 100644 --- a/models/stat.rb +++ b/models/stat.rb @@ -1,5 +1,6 @@ class Stat < Sequel::Model GEOCITY_PATH = './files/GeoLiteCity.dat' + FREE_RETAINMENT_DAYS = 7 many_to_one :site one_to_many :stat_referrers @@ -7,6 +8,13 @@ class Stat < Sequel::Model one_to_many :stat_paths class << self + def prune! + DB[ + "DELETE FROM stats WHERE created_at < ? AND site_id NOT IN (SELECT id FROM sites WHERE plan_type IS NOT NULL OR plan_type != 'free')", + (FREE_RETAINMENT_DAYS-1).days.ago.to_date.to_s + ].first + end + def parse_logfiles(path) Dir["#{path}/*.log"].each do |log_path| site_logs = {} diff --git a/tests/stat_tests.rb b/tests/stat_tests.rb index da579e3e..3a487b6c 100644 --- a/tests/stat_tests.rb +++ b/tests/stat_tests.rb @@ -24,6 +24,27 @@ describe 'stats' do @s2u = @site_two.username end + it 'prunes logs for free sites' do + @free_site = Fabricate :site + @supporter_site = Fabricate :site, plan_type: 'supporter' + + day = Date.today + (Stat::FREE_RETAINMENT_DAYS+1).times do |i| + [@free_site, @supporter_site].each do |site| + Stat.create site_id: site.id, created_at: day + end + day = day - 1 + end + + count_site_ids = [@free_site.id, @supporter_site.id] + expected_stat_count = (Stat::FREE_RETAINMENT_DAYS+1)*2 + + Stat.where(site_id: count_site_ids).count.must_equal expected_stat_count + Stat.prune! + Stat.where(site_id: count_site_ids).count.must_equal expected_stat_count-1 + Stat.where(site_id: @supporter_site.id).count.must_equal expected_stat_count/2 + end + it 'parses multiple sets of logs' do geoip = GeoIP.new Stat::GEOCITY_PATH