clump logs into single hash before importing to database

models/stat.rb

@@ -23,9 +23,9 @@ class Stat < Sequel::Model
 
       cache_control_ips = $config['cache_control_ips']
 
-      Dir["#{path}/*.log.gz"].each do |log_path|
       site_logs = {}
 
+      Dir["#{path}/*.log.gz"].each do |log_path|
         gzfile = File.open log_path, 'r'
         logfile = Zlib::GzipReader.new gzfile
 
@@ -83,17 +83,18 @@ class Stat < Sequel::Model
             site_logs[log_time][username][:paths][path] ||= 0
             site_logs[log_time][username][:paths][path] += 1
           end
+
+          logfile.close
+          FileUtils.rm log_path
         rescue => e
           puts "Log parse exception: #{e.inspect}"
           logfile.close
           FileUtils.mv log_path, log_path.gsub('.log', '.brokenlog')
           next
         end
+        #FileUtils.rm log_path
+      end
 
-        logfile.close
-
-        DB[:stats].lock('EXCLUSIVE') do
-          DB.transaction do
       site_logs.each do |log_time, usernames|
         Site.select(:id, :username).where(username: usernames.keys).all.each do |site|
           usernames[site.username][:id] = site.id
@@ -121,13 +122,7 @@ class Stat < Sequel::Model
           ].first
         end
       end
-          end
-        end
-        FileUtils.rm log_path
-      end
 
-      DB[:daily_site_stats].lock('EXCLUSIVE') do
-        DB.transaction do
       total_site_stats.each do |time, stats|
         opts = {created_at: time.to_date.to_s}
           stat = DailySiteStat.select(:id).where(opts).first
@@ -143,8 +138,6 @@ class Stat < Sequel::Model
       end
     end
   end
-    end
-  end
 end
 
 =begin

tests/stat_tests.rb

@@ -12,7 +12,7 @@ describe 'stats' do
     @time = Time.now
     @time_iso8601 = @time.iso8601
 
-    log = [
+    @log = [
       "#{@time_iso8601}\t#{@site_one.username}\t5000\t/\t67.180.75.140\thttp://example.com",
       "#{@time_iso8601}\t#{@site_one.username}\t5000\t/\t67.180.75.140\thttp://example.com",
       "#{@time_iso8601}\t#{@site_one.username}\t5000\t/\t172.56.16.152\thttp://example.com",
@@ -23,10 +23,21 @@ describe 'stats' do
     ]
 
     Zlib::GzipWriter.open("tests/stat_logs/#{SecureRandom.uuid}.log.gz") do |gz|
-      gz.write log.join("\n")
+      gz.write @log.join("\n")
     end
   end
 
+  it 'works with two logfiles' do
+    Zlib::GzipWriter.open("tests/stat_logs/#{SecureRandom.uuid}.log.gz") do |gz|
+      gz.write @log.join("\n")
+    end
+    Stat.parse_logfiles STAT_LOGS_PATH
+    stat = @site_one.stats.first
+    stat.hits.must_equal 8
+    stat.bandwidth.must_equal 40000
+    stat.views.must_equal 2
+  end
+
   it 'deals with spaces in paths' do
     @site = Fabricate :site