Automatically cut off database-readable logs by id & job_id age

lib/travis/logs/config.rb

@@ -52,6 +52,7 @@ class Config < Travis::Config
           log_parts_autovacuum_vacuum_scale_factor: 0.001,
           log_parts_autovacuum_vacuum_threshold: 0,
           min_messages: 'warning',
+          min_readable_cutoff_age: 60 * 60 * 24 * 180,
           sql_logging: false,
           url: ENV.fetch(
             'LOGS_DATABASE_URL',

lib/travis/logs/database.rb

@@ -10,9 +10,7 @@ module Travis
   module Logs
     class Database
       class << self
-        def create_sequel
-          config = Travis.config.logs_database.to_h
-
+        def create_sequel(config: Travis.config.logs_database.to_h)
           Sequel.default_timezone = :utc
           conn = Sequel.connect(
             config[:url],
@@ -64,16 +62,36 @@ def connect
         end
       end
 
-      def initialize
-        @db = self.class.create_sequel
+      def initialize(config: Travis.config.logs_database.to_h)
+        @db = self.class.create_sequel(config: config)
         Travis.logger.info(
           'new database connection',
           object_id: object_id,
           max_size: db.pool.max_size
         )
+        @min_readable_cutoff_age = config[:min_readable_cutoff_age]
+      end
+
+      attr_reader :db, :min_readable_cutoff_age
+      private :min_readable_cutoff_age
+
+      def log_id_min_readable
+        min_readable_settings[:id] || 0
       end
 
-      attr_reader :db
+      def job_id_min_readable
+        min_readable_settings[:job_id] || 0
+      end
+
+      private def min_readable_settings
+        cutoff = (Time.now.utc - min_readable_cutoff_age).strftime('%Y-%m-%d')
+        db[:logs]
+          .select(:id, :job_id)
+          .where { archived_at < cutoff }
+          .reverse(:archived_at)
+          .limit(1)
+          .first
+      end
 
       def connect
         db.test_connection

lib/travis/logs/services/fetch_log.rb

@@ -17,11 +17,13 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true)
             raise ArgumentError, 'only one of job_id or id allowed'
           end
 
-          if job_id && job_id < min_accepted_job_id
+          if job_id && job_id < database.job_id_min_readable
             return spoofed_archived_log(job_id: job_id)
           end
 
-          return spoofed_archived_log(id: id) if id && id < min_accepted_id
+          if id && id < database.log_id_min_readable
+            return spoofed_archived_log(id: id)
+          end
 
           fetch(
             job_id: job_id,
@@ -51,14 +53,6 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true)
           )
         end
 
-        private def min_accepted_job_id
-          Travis.config.logs.archive_spoofing.min_accepted_job_id
-        end
-
-        private def min_accepted_id
-          Travis.config.logs.archive_spoofing.min_accepted_id
-        end
-
         private def spoofed_archived_log(job_id: nil, id: nil)
           {
             aggregated_at: Time.now - 300,

lib/travis/logs/services/fetch_log_parts.rb

@@ -12,8 +12,8 @@ def initialize(database: nil)
         private :database
 
         def run(log_id: nil, job_id: nil, after: nil, part_numbers: [])
-          return [] if job_id && job_id < min_accepted_job_id
-          return [] if log_id && log_id < min_accepted_id
+          return [] if job_id && job_id < database.job_id_min_readable
+          return [] if log_id && log_id < database.log_id_min_readable
 
           fetch(
             log_id: log_id,
@@ -30,14 +30,6 @@ def run(log_id: nil, job_id: nil, after: nil, part_numbers: [])
           return nil if log_id.nil?
           database.log_parts(log_id, after: after, part_numbers: part_numbers)
         end
-
-        private def min_accepted_job_id
-          Travis.config.logs.archive_spoofing.min_accepted_job_id
-        end
-
-        private def min_accepted_id
-          Travis.config.logs.archive_spoofing.min_accepted_id
-        end
       end
     end
   end