Automatically cut off database-readable logs by id & job_id age

lib/travis/logs/config.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/integer'
+require 'active_support/core_ext/numeric'
+require 'active_support/core_ext/time'
 
 require 'travis/config'
 
@@ -30,18 +32,19 @@ class Config < Travis::Config
             min_accepted_id: 0,
             min_accepted_job_id: 0
           },
-          cache_size_bytes: 10_000_000,
+          cache_size_bytes: 10.megabytes,
+          drain_threads: 4,
           drain_batch_size: 100,
           drain_consumer_count: 10,
           drain_execution_interval: 3,
           drain_loop_sleep_interval: 10,
           drain_timeout_interval: 3,
           intervals: {
-            aggregate: 60,
-            force: 3 * 60 * 60,
-            purge: 6,
-            regular: 3 * 60,
-            sweeper: 10 * 60
+            aggregate: 1.minute,
+            force: 3.hours,
+            purge: 6.seconds,
+            regular: 3.minutes,
+            sweeper: 10.minutes
           },
           maintenance_expiry: 5.minutes,
           maintenance_initial_sleep: 30.seconds,
@@ -50,13 +53,15 @@ class Config < Travis::Config
           sidekiq_error_retry_pause: 3.seconds
         },
         logs_database: {
+          min_readable_cutoff_age: 6.months,
           sql_logging: false,
           url: ENV.fetch(
             'LOGS_DATABASE_URL',
             "postgres://localhost/travis_logs_#{env}"
           )
         },
         logs_readonly_database: {
+          min_readable_cutoff_age: 6.months,
           sql_logging: false,
           url: ENV.fetch(
             'LOGS_READONLY_DATABASE_URL',

lib/travis/logs/database.rb

@@ -72,6 +72,28 @@ def initialize(config: Travis.config.logs_database.to_h,
           object_id: object_id,
           max_size: db.pool.max_size
         )
+        @min_readable_cutoff_age = config[:min_readable_cutoff_age]
+      end
+
+      attr_reader :db, :min_readable_cutoff_age
+      private :min_readable_cutoff_age
+
+      def log_id_min_readable
+        min_readable_settings[:id] || 0
+      end
+
+      def job_id_min_readable
+        min_readable_settings[:job_id] || 0
+      end
+
+      private def min_readable_settings
+        cutoff = (Time.now.utc - min_readable_cutoff_age).strftime('%Y-%m-%d')
+        db[:logs]
+          .select(:id, :job_id)
+          .where { archived_at < cutoff }
+          .reverse(:archived_at)
+          .limit(1)
+          .first
       end
 
       attr_reader :db, :cache, :maint

lib/travis/logs/services/fetch_log.rb

@@ -20,13 +20,13 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true)
           if job_id
             if ignored_job_id?(job_id)
               return temporarily_unavailable_log(job_id: job_id)
-            elsif job_id < min_accepted_job_id
+            elsif job_id < database.job_id_min_readable
               return spoofed_archived_log(job_id: job_id)
             end
           elsif id
             if ignored_log_id?(id)
               return temporarily_unavailable_log(id: id)
-            elsif id < min_accepted_id
+            elsif id < database.log_id_min_readable
               return spoofed_archived_log(id: id)
             end
           end
@@ -59,14 +59,6 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true)
           )
         end
 
-        private def min_accepted_job_id
-          Travis.config.logs.archive_spoofing.min_accepted_job_id
-        end
-
-        private def min_accepted_id
-          Travis.config.logs.archive_spoofing.min_accepted_id
-        end
-
         private def ignored_job_id?(job_id)
           Travis::Logs.redis.sismember('logs:ignored-job-ids', job_id.to_s)
         end

lib/travis/logs/services/fetch_log_parts.rb

@@ -15,13 +15,13 @@ def run(log_id: nil, job_id: nil, after: nil, part_numbers: [])
           if job_id
             if ignored_job_id?(job_id)
               return temporarily_unavailable_log_parts
-            elsif job_id < min_accepted_job_id
+            elsif job_id < database.job_id_min_readable
               return []
             end
           elsif log_id
             if ignored_log_id?(log_id)
               return temporarily_unavailable_log_parts(log_id: log_id)
-            elsif log_id < min_accepted_id
+            elsif log_id < database.log_id_min_readable
               return []
             end
           end
@@ -42,14 +42,6 @@ def run(log_id: nil, job_id: nil, after: nil, part_numbers: [])
           database.log_parts(log_id, after: after, part_numbers: part_numbers)
         end
 
-        private def min_accepted_job_id
-          Travis.config.logs.archive_spoofing.min_accepted_job_id
-        end
-
-        private def min_accepted_id
-          Travis.config.logs.archive_spoofing.min_accepted_id
-        end
-
         private def ignored_job_id?(job_id)
           Travis::Logs.redis.sismember('logs:ignored-job-ids', job_id.to_s)
         end