Add GeoLiteCity, and stats pruning code

Rakefile

@@ -31,10 +31,8 @@ end
 
 desc "parse logs"
 task :parse_logs => [:environment] do
-  Dir[File.join($config['logs_path'], '*.log')].each do |logfile_path|
+  Stat.parse_logfiles $config['logs_path']
-    Stat.parse logfile_path
+  Stat.prune!
-    FileUtils.rm logfile_path
-  end
 end
 
 desc 'Update banned IPs list'

config.yml.travis

@@ -6,4 +6,5 @@ recaptcha_private_key: '5678'
 phantomjs_url:
   - http://localhost:8910
 ip_hash_salt: "400$8$1$fc21863da5d531c1"
-email_unsubscribe_token: "somethingrandomderrrrp"
+email_unsubscribe_token: "somethingrandomderrrrp"
+logs_path: "/tmp/neocitiestestlogs"

models/stat.rb

@@ -1,5 +1,6 @@
 class Stat < Sequel::Model
   GEOCITY_PATH = './files/GeoLiteCity.dat'
+  FREE_RETAINMENT_DAYS = 7
 
   many_to_one :site
   one_to_many :stat_referrers
@@ -7,6 +8,13 @@ class Stat < Sequel::Model
   one_to_many :stat_paths
 
   class << self
+    def prune!
+      DB[
+        "DELETE FROM stats WHERE created_at < ? AND site_id NOT IN (SELECT id FROM sites WHERE plan_type IS NOT NULL OR plan_type != 'free')",
+        (FREE_RETAINMENT_DAYS-1).days.ago.to_date.to_s
+      ].first
+    end
+
     def parse_logfiles(path)
       Dir["#{path}/*.log"].each do |log_path|
         site_logs = {}

tests/stat_tests.rb

@@ -24,6 +24,27 @@ describe 'stats' do
     @s2u = @site_two.username
   end
 
+  it 'prunes logs for free sites' do
+    @free_site = Fabricate :site
+    @supporter_site = Fabricate :site, plan_type: 'supporter'
+
+    day = Date.today
+    (Stat::FREE_RETAINMENT_DAYS+1).times do |i|
+      [@free_site, @supporter_site].each do |site|
+        Stat.create site_id: site.id, created_at: day
+      end
+      day = day - 1
+    end
+
+    count_site_ids = [@free_site.id, @supporter_site.id]
+    expected_stat_count = (Stat::FREE_RETAINMENT_DAYS+1)*2
+
+    Stat.where(site_id: count_site_ids).count.must_equal expected_stat_count
+    Stat.prune!
+    Stat.where(site_id: count_site_ids).count.must_equal expected_stat_count-1
+    Stat.where(site_id: @supporter_site.id).count.must_equal expected_stat_count/2
+  end
+
   it 'parses multiple sets of logs' do
     geoip = GeoIP.new Stat::GEOCITY_PATH