diff --git a/lib/travis/logs/config.rb b/lib/travis/logs/config.rb index 2bb5fc29..d12b0166 100644 --- a/lib/travis/logs/config.rb +++ b/lib/travis/logs/config.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true -require 'active_support/core_ext/numeric/time' +require 'active_support/core_ext/integer' +require 'active_support/core_ext/numeric' +require 'active_support/core_ext/time' require 'travis/config' @@ -30,18 +32,19 @@ class Config < Travis::Config min_accepted_id: 0, min_accepted_job_id: 0 }, - cache_size_bytes: 10_000_000, + cache_size_bytes: 10.megabytes, + drain_threads: 4, drain_batch_size: 100, drain_consumer_count: 10, drain_execution_interval: 3, drain_loop_sleep_interval: 10, drain_timeout_interval: 3, intervals: { - aggregate: 60, - force: 3 * 60 * 60, - purge: 6, - regular: 3 * 60, - sweeper: 10 * 60 + aggregate: 1.minute, + force: 3.hours, + purge: 6.seconds, + regular: 3.minutes, + sweeper: 10.minutes }, maintenance_expiry: 5.minutes, maintenance_initial_sleep: 30.seconds, @@ -50,6 +53,7 @@ class Config < Travis::Config sidekiq_error_retry_pause: 3.seconds }, logs_database: { + min_readable_cutoff_age: 6.months, sql_logging: false, url: ENV.fetch( 'LOGS_DATABASE_URL', @@ -57,6 +61,7 @@ class Config < Travis::Config ) }, logs_readonly_database: { + min_readable_cutoff_age: 6.months, sql_logging: false, url: ENV.fetch( 'LOGS_READONLY_DATABASE_URL', diff --git a/lib/travis/logs/database.rb b/lib/travis/logs/database.rb index 14eb64c3..1e077957 100644 --- a/lib/travis/logs/database.rb +++ b/lib/travis/logs/database.rb @@ -72,6 +72,28 @@ def initialize(config: Travis.config.logs_database.to_h, object_id: object_id, max_size: db.pool.max_size ) + @min_readable_cutoff_age = config[:min_readable_cutoff_age] + end + + attr_reader :db, :min_readable_cutoff_age + private :min_readable_cutoff_age + + def log_id_min_readable + min_readable_settings[:id] || 0 + end + + def job_id_min_readable + min_readable_settings[:job_id] || 0 + end + + private def min_readable_settings + cutoff = (Time.now.utc - min_readable_cutoff_age).strftime('%Y-%m-%d') + db[:logs] + .select(:id, :job_id) + .where { archived_at < cutoff } + .reverse(:archived_at) + .limit(1) + .first end attr_reader :db, :cache, :maint diff --git a/lib/travis/logs/services/fetch_log.rb b/lib/travis/logs/services/fetch_log.rb index bf1240fb..d4b3c07e 100644 --- a/lib/travis/logs/services/fetch_log.rb +++ b/lib/travis/logs/services/fetch_log.rb @@ -20,13 +20,13 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true) if job_id if ignored_job_id?(job_id) return temporarily_unavailable_log(job_id: job_id) - elsif job_id < min_accepted_job_id + elsif job_id < database.job_id_min_readable return spoofed_archived_log(job_id: job_id) end elsif id if ignored_log_id?(id) return temporarily_unavailable_log(id: id) - elsif id < min_accepted_id + elsif id < database.log_id_min_readable return spoofed_archived_log(id: id) end end @@ -59,14 +59,6 @@ def run(job_id: nil, id: nil, aggregate_on_demand: true) ) end - private def min_accepted_job_id - Travis.config.logs.archive_spoofing.min_accepted_job_id - end - - private def min_accepted_id - Travis.config.logs.archive_spoofing.min_accepted_id - end - private def ignored_job_id?(job_id) Travis::Logs.redis.sismember('logs:ignored-job-ids', job_id.to_s) end diff --git a/lib/travis/logs/services/fetch_log_parts.rb b/lib/travis/logs/services/fetch_log_parts.rb index 57d8df2e..a38b9e8b 100644 --- a/lib/travis/logs/services/fetch_log_parts.rb +++ b/lib/travis/logs/services/fetch_log_parts.rb @@ -15,13 +15,13 @@ def run(log_id: nil, job_id: nil, after: nil, part_numbers: []) if job_id if ignored_job_id?(job_id) return temporarily_unavailable_log_parts - elsif job_id < min_accepted_job_id + elsif job_id < database.job_id_min_readable return [] end elsif log_id if ignored_log_id?(log_id) return temporarily_unavailable_log_parts(log_id: log_id) - elsif log_id < min_accepted_id + elsif log_id < database.log_id_min_readable return [] end end @@ -42,14 +42,6 @@ def run(log_id: nil, job_id: nil, after: nil, part_numbers: []) database.log_parts(log_id, after: after, part_numbers: part_numbers) end - private def min_accepted_job_id - Travis.config.logs.archive_spoofing.min_accepted_job_id - end - - private def min_accepted_id - Travis.config.logs.archive_spoofing.min_accepted_id - end - private def ignored_job_id?(job_id) Travis::Logs.redis.sismember('logs:ignored-job-ids', job_id.to_s) end