From ae97c5ca117054152bb5718972868244ac8a74d8 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sat, 14 Aug 2021 20:58:26 +0200 Subject: [PATCH 01/84] GCE backup and GCE files purging remved --- README.md | 5 +---- config/settings.yml | 6 ------ lib/backup.rb | 32 +------------------------------- lib/config.rb | 7 +------ 4 files changed, 3 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 0ebd57d..ec19808 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # README *travis-backup* is a cron application, which export builds and it's correspoonding jobs -to json files and sends them to GCE. +to json files. * Ruby version @@ -17,9 +17,6 @@ to json files and sends them to GCE. `BACKUP_HOUSEKEEPING_PERIOD` `LOGS_URL` `DATABASE_URL` -`GCE_PROJECT` -`GCE_CREDENTIALS` -`GCE_BUCKET` `REDIS_URL` * How to run the test suite diff --git a/config/settings.yml b/config/settings.yml index 0a00e4c..5687c5a 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,12 +7,6 @@ backup: housekeeping_period: 150 # logs URL (the same as via UI) logs_url: https://api.travis-ci.org/v3/job - -# GCE Settings -gce: - project: travis-ci-prod-services-1 - credentials: config/secrets/travis-ci-prod-services-travis-backup.json - bucket: travis-backup-staging redis: url: redis://127.0.0.1:6379 \ No newline at end of file diff --git a/lib/backup.rb b/lib/backup.rb index 6964cc8..6d3e883 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -12,40 +12,22 @@ class Backup def initialize @config = Config.new - connect_gce connect_db connect_redis end def run export - purge end def connect_db ActiveRecord::Base.establish_connection(@config.database_url) end - def connect_gce - return unless @config.gce_project && @config.gce_credentials - - storage = Google::Cloud::Storage.new( - project_id: @config.gce_project, - credentials: @config.gce_credentials - ) - @bucket = storage.bucket(@config.gce_bucket) - end - def connect_redis @redis = Redis.new(url: @config.redis_url) end - def purge - BuildBackup.where('created_at < ?', @config.housekeeping_period.to_i.days.ago.to_datetime) do |backup| - purge_backup(backup) - end - end - def export(owner_id = nil) if owner_id Repository.where('owner_id = ?', owner_id).order(:id).each do |repository| @@ -74,15 +56,6 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe end end - def purge_backup(backup) - begin - @bucket.file(backup.file_name).delete - rescue - print "Unable to remove file #{backup.file_name}\n" - end - backup.destroy - end - private def upload(file_name, content) # rubocop:disable Metrics/MethodLength @@ -91,13 +64,10 @@ def upload(file_name, content) # rubocop:disable Metrics/MethodLength File.open(file_name, 'w') do |file| file.write(content) file.close - remote_file = @bucket.create_file(file_name, file_name) - uploaded = remote_file.name == file_name + uploaded = true end rescue => e print "Failed to save #{file_name}, error: #{e.inspect}\n" - ensure - File.delete(file_name) end uploaded end diff --git a/lib/config.rb b/lib/config.rb index c8a1247..7ad184c 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,8 +2,7 @@ # Config for travis-backup class Config - attr_reader :limit, :delay, :housekeeping_period, :database_url, :logs_url, :gce_project, - :gce_credentials, :gce_bucket, :redis_url + attr_reader :limit, :delay, :housekeeping_period, :database_url, :logs_url, :redis_url def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) @@ -14,10 +13,6 @@ def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, @logs_url = ENV['LOGS_URL'] || config['backup']['logs_url'] @housekeeping_period = ENV['BACKUP_HOUSEKEEPING_PERIOD'] || config['backup']['housekeeping_period'] @database_url = ENV['DATABASE_URL'] || connection_details['development'] - @gce_project = ENV['GCE_PROJECT'] || config['gce']['project'] - credentials = ENV['GCE_CREDENTIALS'] || (File.exist?(config['gce']['credentials']) ? File.read(config['gce']['credentials']) : nil) - @gce_credentials = credentials ? JSON.parse(credentials) : nil - @gce_bucket = ENV['GCE_BUCKET'] || config['gce']['bucket'] @redis = ENV['REDIS_URL'] || config['redis']['url'] end end From dd32eb6a2e14fa8a92cb465b5a648b3e5d16e38b Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sat, 14 Aug 2021 21:26:31 +0200 Subject: [PATCH 02/84] .gitignore file and saving files to dump folder --- .gitignore | 6 ++++++ Gemfile | 2 +- dump/.keep | 0 lib/backup.rb | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 dump/.keep diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d37aeb --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +tmp/* +log/* +dump/* +!tmp/.keep +!log/.keep +!dump/.keep \ No newline at end of file diff --git a/Gemfile b/Gemfile index ecbc13b..b7eaaa0 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.2' +ruby '2.7.0' gem 'activerecord' gem 'google-cloud-storage', '~> 1.8', require: false diff --git a/dump/.keep b/dump/.keep new file mode 100644 index 0000000..e69de29 diff --git a/lib/backup.rb b/lib/backup.rb index 6d3e883..5de49f5 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -61,7 +61,7 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe def upload(file_name, content) # rubocop:disable Metrics/MethodLength uploaded = false begin - File.open(file_name, 'w') do |file| + File.open("dump/#{file_name}", 'w') do |file| file.write(content) file.close uploaded = true From 2dc78940a3d0d3aa9bdff338d45d39db97eed6c2 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sat, 14 Aug 2021 22:29:32 +0200 Subject: [PATCH 03/84] deletion of redis and tokens for private repositories --- README.md | 1 - config/settings.yml | 3 --- lib/backup.rb | 14 -------------- lib/config.rb | 3 +-- spec/backup_spec.rb | 6 ------ 5 files changed, 1 insertion(+), 26 deletions(-) diff --git a/README.md b/README.md index ec19808..e3fb13f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ to json files. `BACKUP_HOUSEKEEPING_PERIOD` `LOGS_URL` `DATABASE_URL` -`REDIS_URL` * How to run the test suite diff --git a/config/settings.yml b/config/settings.yml index 5687c5a..98bea1a 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,6 +7,3 @@ backup: housekeeping_period: 150 # logs URL (the same as via UI) logs_url: https://api.travis-ci.org/v3/job - -redis: - url: redis://127.0.0.1:6379 \ No newline at end of file diff --git a/lib/backup.rb b/lib/backup.rb index 5de49f5..81342ae 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -6,14 +6,12 @@ require 'google/cloud/storage' require 'models/build_backup' require 'models/repository' -require 'redis' # main travis-backup class class Backup def initialize @config = Config.new connect_db - connect_redis end def run @@ -24,10 +22,6 @@ def connect_db ActiveRecord::Base.establish_connection(@config.database_url) end - def connect_redis - @redis = Redis.new(url: @config.redis_url) - end - def export(owner_id = nil) if owner_id Repository.where('owner_id = ?', owner_id).order(:id).each do |repository| @@ -72,13 +66,6 @@ def upload(file_name, content) # rubocop:disable Metrics/MethodLength uploaded end - def generate_log_token(job_id) - token = SecureRandom.urlsafe_base64(16) - @redis.set("l:#{token}", job_id) - @redis.expire("l:#{token}", @config.housekeeping_period.to_i * 86400) - token - end - def export_builds(builds) builds.map do |build| build_export = build.attributes @@ -94,7 +81,6 @@ def export_jobs(jobs) job_export = job.attributes job_export[:job_config] = job.job_config&.attributes job_export[:log_url] = "#{@config.logs_url}/#{job.id}/log.txt" - job_export[:log_url] += "?log.token=#{generate_log_token(job.id)}" if job.repository&.private? job_export end diff --git a/lib/config.rb b/lib/config.rb index 7ad184c..a8dde72 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,7 @@ # Config for travis-backup class Config - attr_reader :limit, :delay, :housekeeping_period, :database_url, :logs_url, :redis_url + attr_reader :limit, :delay, :housekeeping_period, :database_url, :logs_url def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) @@ -13,6 +13,5 @@ def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, @logs_url = ENV['LOGS_URL'] || config['backup']['logs_url'] @housekeeping_period = ENV['BACKUP_HOUSEKEEPING_PERIOD'] || config['backup']['housekeeping_period'] @database_url = ENV['DATABASE_URL'] || connection_details['development'] - @redis = ENV['REDIS_URL'] || config['redis']['url'] end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 31e440f..45e3db8 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -187,10 +187,4 @@ build_export.first.first[:jobs].first[:updated_at] = datetime expect(build_export.to_json).to eq(exported_object.to_json) end - - it 'for private repository should prepare proper JSON export with token for log urls' do - build_export = backup.process_repo(private_repository) - log_url = build_export.first.first[:jobs].first[:log_url] - expect(log_url).to include('?log.token=') - end end From 45eb47885b8bdbf573290987c1fa9e542f03efc6 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sat, 14 Aug 2021 22:38:31 +0200 Subject: [PATCH 04/84] BuildBackup removed --- lib/backup.rb | 2 -- lib/models/build_backup.rb | 11 ----------- lib/models/repository.rb | 1 - 3 files changed, 14 deletions(-) delete mode 100644 lib/models/build_backup.rb diff --git a/lib/backup.rb b/lib/backup.rb index 81342ae..b7569e3 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -4,7 +4,6 @@ require 'active_support/time' require 'config' require 'google/cloud/storage' -require 'models/build_backup' require 'models/repository' # main travis-backup class @@ -42,7 +41,6 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe file_name = "repository_#{repository.id}_builds_#{builds.first.id}-#{builds.last.id}.json" pretty_json = JSON.pretty_generate(builds_export) if upload(file_name, pretty_json) - BuildBackup.new(repository_id: repository.id, file_name: file_name).save! builds.each(&:destroy) end builds_export diff --git a/lib/models/build_backup.rb b/lib/models/build_backup.rb deleted file mode 100644 index e3acad8..0000000 --- a/lib/models/build_backup.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -require 'models/model' -require 'models/repository' - -# Build model -class BuildBackup < Model - belongs_to :repository - - self.table_name = 'build_backups' -end diff --git a/lib/models/repository.rb b/lib/models/repository.rb index 0bdc048..6022e54 100644 --- a/lib/models/repository.rb +++ b/lib/models/repository.rb @@ -6,7 +6,6 @@ # Repository model class Repository < Model has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build' - has_many :build_backup self.table_name = 'repositories' end From 90083240b0dba37bc67ad73e4d293503b0a25e69 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sat, 14 Aug 2021 23:34:06 +0200 Subject: [PATCH 05/84] problem with current_build_id solved --- lib/backup.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/backup.rb b/lib/backup.rb index b7569e3..d7fc158 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -34,7 +34,8 @@ def export(owner_id = nil) end def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - repository.builds.where('created_at < ?', @config.delay.to_i.months.ago.to_datetime) + delay = @config.delay.to_i.months.ago.to_datetime + repository.builds.where('created_at < ? and id != ?', delay, repository.current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds| if builds.count == @config.limit.to_i builds_export = export_builds(builds) From fa2b43514d4e5ccf0499fb0581e2fa08593a2c9e Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 15 Aug 2021 11:26:55 +0200 Subject: [PATCH 06/84] housekeeping period for files removed from settings --- config/settings.yml | 2 -- lib/config.rb | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/config/settings.yml b/config/settings.yml index 98bea1a..bd2b407 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -3,7 +3,5 @@ backup: limit: 1000 # delay in months delay: 6 - # how long (in days) expoorted files should be kept in storage - housekeeping_period: 150 # logs URL (the same as via UI) logs_url: https://api.travis-ci.org/v3/job diff --git a/lib/config.rb b/lib/config.rb index a8dde72..05df5c5 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,7 @@ # Config for travis-backup class Config - attr_reader :limit, :delay, :housekeeping_period, :database_url, :logs_url + attr_reader :limit, :delay, :database_url, :logs_url def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) @@ -11,7 +11,6 @@ def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, @limit = ENV['BACKUP_LIMIT'] || config['backup']['limit'] @delay = ENV['BACKUP_DELAY'] || config['backup']['delay'] @logs_url = ENV['LOGS_URL'] || config['backup']['logs_url'] - @housekeeping_period = ENV['BACKUP_HOUSEKEEPING_PERIOD'] || config['backup']['housekeeping_period'] @database_url = ENV['DATABASE_URL'] || connection_details['development'] end end From fe269fcb37b5d3966e8c68b3dc27b91260780f79 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 15 Aug 2021 11:29:21 +0200 Subject: [PATCH 07/84] logs_url removed --- config/settings.yml | 2 -- lib/backup.rb | 1 - lib/config.rb | 3 +-- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/settings.yml b/config/settings.yml index bd2b407..99a7a23 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -3,5 +3,3 @@ backup: limit: 1000 # delay in months delay: 6 - # logs URL (the same as via UI) - logs_url: https://api.travis-ci.org/v3/job diff --git a/lib/backup.rb b/lib/backup.rb index d7fc158..609b762 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -79,7 +79,6 @@ def export_jobs(jobs) jobs.map do |job| job_export = job.attributes job_export[:job_config] = job.job_config&.attributes - job_export[:log_url] = "#{@config.logs_url}/#{job.id}/log.txt" job_export end diff --git a/lib/config.rb b/lib/config.rb index 05df5c5..6830aad 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,7 @@ # Config for travis-backup class Config - attr_reader :limit, :delay, :database_url, :logs_url + attr_reader :limit, :delay, :database_url def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) @@ -10,7 +10,6 @@ def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, @limit = ENV['BACKUP_LIMIT'] || config['backup']['limit'] @delay = ENV['BACKUP_DELAY'] || config['backup']['delay'] - @logs_url = ENV['LOGS_URL'] || config['backup']['logs_url'] @database_url = ENV['DATABASE_URL'] || connection_details['development'] end end From 8585d353180a4408340bdc5db89f73f984a64dac Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 15 Aug 2021 12:03:42 +0200 Subject: [PATCH 08/84] build_config and job_config models removed --- lib/backup.rb | 2 -- lib/models/build.rb | 2 -- lib/models/build_config.rb | 8 ------ lib/models/job.rb | 2 -- lib/models/job_config.rb | 8 ------ spec/backup_spec.rb | 56 +++----------------------------------- spec/support/factories.rb | 2 -- 7 files changed, 4 insertions(+), 76 deletions(-) delete mode 100644 lib/models/build_config.rb delete mode 100644 lib/models/job_config.rb diff --git a/lib/backup.rb b/lib/backup.rb index 609b762..ee3b00d 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -68,7 +68,6 @@ def upload(file_name, content) # rubocop:disable Metrics/MethodLength def export_builds(builds) builds.map do |build| build_export = build.attributes - build_export[:build_config] = build.build_config&.attributes build_export[:jobs] = export_jobs(build.jobs) build_export @@ -78,7 +77,6 @@ def export_builds(builds) def export_jobs(jobs) jobs.map do |job| job_export = job.attributes - job_export[:job_config] = job.job_config&.attributes job_export end diff --git a/lib/models/build.rb b/lib/models/build.rb index 5741cc2..bfec5f6 100644 --- a/lib/models/build.rb +++ b/lib/models/build.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'models/build_config' require 'models/job' require 'models/model' require 'models/repository' @@ -8,7 +7,6 @@ # Build model class Build < Model belongs_to :repository - belongs_to :build_config, foreign_key: :config_id, dependent: :delete has_many :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :delete_all, class_name: 'Job' self.table_name = 'builds' diff --git a/lib/models/build_config.rb b/lib/models/build_config.rb deleted file mode 100644 index fca1389..0000000 --- a/lib/models/build_config.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -require 'models/model' - -# BuildConfig model -class BuildConfig < Model - self.table_name = 'build_configs' -end diff --git a/lib/models/job.rb b/lib/models/job.rb index 423f46a..7b11f88 100644 --- a/lib/models/job.rb +++ b/lib/models/job.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'models/job_config' require 'models/model' require 'models/repository' @@ -9,7 +8,6 @@ class Job < Model self.inheritance_column = :_type_disabled belongs_to :repository - belongs_to :job_config, foreign_key: :config_id, dependent: :delete self.table_name = 'jobs' end diff --git a/lib/models/job_config.rb b/lib/models/job_config.rb deleted file mode 100644 index e263013..0000000 --- a/lib/models/job_config.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -require 'models/model' - -# JobConfig model -class JobConfig < Model - self.table_name = 'job_configs' -end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 45e3db8..a4e84d1 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -23,33 +23,12 @@ private: true ) } - let(:build_config) { - FactoryBot.create( - :build_config, - repository_id: repository.id, - key: '', - org_id: org_id, - com_id: com_id, - config: '' - ) - } - let(:private_build_config) { - FactoryBot.create( - :build_config, - repository_id: private_repository.id, - key: '', - org_id: private_org_id, - com_id: private_com_id, - config: '' - ) - } let(:build) { FactoryBot.create( :build, created_at: datetime, updated_at: datetime, - repository: repository, - build_config: build_config + repository: repository ) } let(:private_build) { @@ -57,28 +36,7 @@ :build, created_at: datetime, updated_at: datetime, - repository: private_repository, - build_config: private_build_config - ) - } - let(:job_config) { - FactoryBot.create( - :job_config, - repository_id: repository.id, - key: '', - org_id: org_id, - com_id: com_id, - config: '' - ) - } - let(:private_job_config) { - FactoryBot.create( - :job_config, - repository_id: private_repository.id, - key: '', - org_id: private_org_id, - com_id: private_com_id, - config: '' + repository: private_repository ) } let(:job) { @@ -88,8 +46,7 @@ updated_at: datetime, source_id: build.id, source_type: 'Build', - repository: repository, - job_config: job_config + repository: repository ) } let(:private_job) { @@ -99,8 +56,7 @@ updated_at: datetime, source_id: private_build.id, source_type: 'Build', - repository: private_repository, - job_config: private_job_config + repository: private_repository ) } let(:exported_object) { @@ -133,10 +89,8 @@ "sender_type"=>nil, "org_id"=>nil, "com_id"=>nil, - "config_id"=>build_config.id, "restarted_at"=>nil, "unique_number"=>nil, - :build_config=>{"id"=>build_config.id, "repository_id"=>repository.id, "key"=>"", "org_id"=>org_id, "com_id"=>com_id, "config"=>""}, :jobs=> [{"id"=>job.id, "repository_id"=>repository.id, @@ -167,10 +121,8 @@ "stage_id"=>nil, "org_id"=>nil, "com_id"=>nil, - "config_id"=>job_config.id, "restarted_at"=>nil, "priority"=>nil, - :job_config=>{"id"=>job_config.id, "repository_id"=>repository.id, "key"=>"", "org_id"=>org_id, "com_id"=>com_id, "config"=>""}, :log_url=>"https://api.travis-ci.org/v3/job/#{job.id}/log.txt"}]}]] } diff --git a/spec/support/factories.rb b/spec/support/factories.rb index ad1e927..fa43f59 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -5,7 +5,5 @@ FactoryBot.define do factory :repository, class: Repository factory :build, class: Build - factory :build_config, class: BuildConfig factory :job, class: Job - factory :job_config, class: JobConfig end From 1198b476fc12e5da5f67fa1b6ce4a015b2046541 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 15 Aug 2021 12:04:22 +0200 Subject: [PATCH 09/84] README updated --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e3fb13f..c5da738 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,6 @@ to json files. `config/settinigs.yml` or env vars like: `BACKUP_LIMIT` `BACKUP_DELAY` -`BACKUP_HOUSEKEEPING_PERIOD` -`LOGS_URL` `DATABASE_URL` * How to run the test suite From 188e298e5425d451353c2d7f4f7dd73680d5d93e Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 15 Aug 2021 12:24:33 +0200 Subject: [PATCH 10/84] Gemfile updated --- Gemfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Gemfile b/Gemfile index b7eaaa0..be0e6b0 100644 --- a/Gemfile +++ b/Gemfile @@ -6,11 +6,9 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby '2.7.0' gem 'activerecord' -gem 'google-cloud-storage', '~> 1.8', require: false gem 'pg' gem 'pry' gem 'rails', '~> 6.1.3.1' -gem 'redis' gem 'bootsnap', require: false From 42c012a9e17d3c1ecb86dd4e43343a6228751658 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 16 Aug 2021 12:39:44 +0200 Subject: [PATCH 11/84] if_backup flag, small fix --- Gemfile.lock | 72 +-------------------------------------------- README.md | 1 + config/settings.yml | 2 ++ lib/backup.rb | 23 ++++++++------- lib/config.rb | 3 +- 5 files changed, 19 insertions(+), 82 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5b6034d..ba978b9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -60,8 +60,6 @@ GEM minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) ast (2.4.2) bootsnap (1.7.3) msgpack (~> 1.0) @@ -71,62 +69,16 @@ GEM coderay (1.1.3) concurrent-ruby (1.1.8) crass (1.0.6) - declarative (0.0.20) - declarative-option (0.1.0) diff-lcs (1.4.4) - digest-crc (0.6.3) - rake (>= 12.0.0, < 14.0.0) erubi (1.10.0) factory_bot (6.1.0) activesupport (>= 5.0.0) - faraday (1.3.0) - faraday-net_http (~> 1.0) - multipart-post (>= 1.2, < 3) - ruby2_keywords - faraday-net_http (1.0.1) ffi (1.15.0) globalid (0.4.2) activesupport (>= 4.2.0) - google-apis-core (0.3.0) - addressable (~> 2.5, >= 2.5.1) - googleauth (~> 0.14) - httpclient (>= 2.8.1, < 3.0) - mini_mime (~> 1.0) - representable (~> 3.0) - retriable (>= 2.0, < 4.0) - rexml - signet (~> 0.14) - webrick - google-apis-iamcredentials_v1 (0.2.0) - google-apis-core (~> 0.1) - google-apis-storage_v1 (0.3.0) - google-apis-core (~> 0.1) - google-cloud-core (1.6.0) - google-cloud-env (~> 1.0) - google-cloud-errors (~> 1.0) - google-cloud-env (1.5.0) - faraday (>= 0.17.3, < 2.0) - google-cloud-errors (1.1.0) - google-cloud-storage (1.31.0) - addressable (~> 2.5) - digest-crc (~> 0.4) - google-apis-iamcredentials_v1 (~> 0.1) - google-apis-storage_v1 (~> 0.1) - google-cloud-core (~> 1.2) - googleauth (~> 0.9) - mini_mime (~> 1.0) - googleauth (0.16.0) - faraday (>= 0.17.3, < 2.0) - jwt (>= 1.4, < 3.0) - memoist (~> 0.16) - multi_json (~> 1.11) - os (>= 0.9, < 2.0) - signet (~> 0.14) - httpclient (2.8.3) i18n (1.8.9) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) - jwt (2.2.2) listen (3.5.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -136,21 +88,17 @@ GEM mail (2.7.1) mini_mime (>= 0.1.1) marcel (1.0.0) - memoist (0.16.2) method_source (1.0.0) mini_mime (1.0.3) mini_portile2 (2.5.0) minitest (5.14.4) msgpack (1.4.2) - multi_json (1.15.0) - multipart-post (2.1.1) nio4r (2.5.7) nokogiri (1.11.2) mini_portile2 (~> 2.5.0) racc (~> 1.4) nokogiri (1.11.2-x86_64-darwin) racc (~> 1.4) - os (1.1.1) parallel (1.20.1) parser (3.0.0.0) ast (~> 2.4.1) @@ -158,7 +106,6 @@ GEM pry (0.14.0) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (4.0.6) racc (1.5.2) rack (2.2.3) rack-test (1.1.0) @@ -194,13 +141,6 @@ GEM rb-fsevent (0.10.4) rb-inotify (0.10.1) ffi (~> 1.0) - redis (4.2.5) - representable (3.0.4) - declarative (< 0.1.0) - declarative-option (< 0.2.0) - uber (< 0.2.0) - retriable (3.1.2) - rexml (3.2.4) rspec-core (3.10.1) rspec-support (~> 3.10.0) rspec-expectations (3.10.1) @@ -228,12 +168,6 @@ GEM rubocop-rspec (1.41.0) rubocop (>= 0.68.1) ruby-progressbar (1.11.0) - ruby2_keywords (0.0.4) - signet (0.15.0) - addressable (~> 2.3) - faraday (>= 0.17.3, < 2.0) - jwt (>= 1.5, < 3.0) - multi_json (~> 1.10) sprockets (4.0.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -244,9 +178,7 @@ GEM thor (1.1.0) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - uber (0.1.0) unicode-display_width (1.6.1) - webrick (1.7.0) websocket-driver (0.7.3) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -262,19 +194,17 @@ DEPENDENCIES brakeman byebug factory_bot - google-cloud-storage (~> 1.8) listen pg pry rails (~> 6.1.3.1) - redis rspec-rails rubocop (~> 0.75.1) rubocop-rspec tzinfo-data RUBY VERSION - ruby 2.7.2p137 + ruby 2.7.0p0 BUNDLED WITH 2.2.7 diff --git a/README.md b/README.md index c5da738..d9ea8b1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ to json files. * Configuration `config/settinigs.yml` or env vars like: +`IF_BACKUP` `BACKUP_LIMIT` `BACKUP_DELAY` `DATABASE_URL` diff --git a/config/settings.yml b/config/settings.yml index 99a7a23..0646c21 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1,4 +1,6 @@ backup: + # when false, removes data without saving it to file + if_backup: true # builds limit in file limit: 1000 # delay in months diff --git a/lib/backup.rb b/lib/backup.rb index ee3b00d..0c358d8 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -3,7 +3,6 @@ require 'active_support/core_ext/array' require 'active_support/time' require 'config' -require 'google/cloud/storage' require 'models/repository' # main travis-backup class @@ -36,21 +35,25 @@ def export(owner_id = nil) def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength delay = @config.delay.to_i.months.ago.to_datetime repository.builds.where('created_at < ? and id != ?', delay, repository.current_build_id) - .in_groups_of(@config.limit.to_i, false).map do |builds| - if builds.count == @config.limit.to_i - builds_export = export_builds(builds) - file_name = "repository_#{repository.id}_builds_#{builds.first.id}-#{builds.last.id}.json" - pretty_json = JSON.pretty_generate(builds_export) - if upload(file_name, pretty_json) - builds.each(&:destroy) - end - builds_export + .in_groups_of(@config.limit.to_i, false).map do |builds_batch| + if builds_batch.count == @config.limit.to_i + @config.if_backup ? save_batch(builds_batch, repository) : builds.each(&:destroy) end end end private + def save_batch(builds_batch, repository) + builds_export = export_builds(builds_batch) + file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" + pretty_json = JSON.pretty_generate(builds_export) + if upload(file_name, pretty_json) + builds_batch.each(&:destroy) + end + builds_export + end + def upload(file_name, content) # rubocop:disable Metrics/MethodLength uploaded = false begin diff --git a/lib/config.rb b/lib/config.rb index 6830aad..34d2e50 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,12 +2,13 @@ # Config for travis-backup class Config - attr_reader :limit, :delay, :database_url + attr_reader :if_backup, :limit, :delay, :database_url def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) connection_details = YAML.load(File.open('config/database.yml')) + @if_backup = ENV['IF_BACKUP'] || config['backup']['if_backup'] @limit = ENV['BACKUP_LIMIT'] || config['backup']['limit'] @delay = ENV['BACKUP_DELAY'] || config['backup']['delay'] @database_url = ENV['DATABASE_URL'] || connection_details['development'] From 61ceb619abf1d9029e7870bea53608f3a85b457e Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 16 Aug 2021 23:11:40 +0200 Subject: [PATCH 12/84] problem with current_build_id fixed --- lib/backup.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/backup.rb b/lib/backup.rb index 0c358d8..a8aefe5 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -34,7 +34,8 @@ def export(owner_id = nil) def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength delay = @config.delay.to_i.months.ago.to_datetime - repository.builds.where('created_at < ? and id != ?', delay, repository.current_build_id) + current_build_id = repository.current_build_id || -1 + repository.builds.where('created_at < ? and id != ?', delay, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| if builds_batch.count == @config.limit.to_i @config.if_backup ? save_batch(builds_batch, repository) : builds.each(&:destroy) From 504943fbf36582c5662cd299172ee744fd489d90 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 17 Aug 2021 00:12:52 +0200 Subject: [PATCH 13/84] json in backup_spec updated --- spec/backup_spec.rb | 66 +++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index a4e84d1..a778cf4 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -67,6 +67,7 @@ "finished_at"=>nil, "created_at"=>datetime, "updated_at"=>datetime, + "config"=>nil, "commit_id"=>nil, "request_id"=>nil, "state"=>nil, @@ -87,43 +88,36 @@ "tag_id"=>nil, "sender_id"=>nil, "sender_type"=>nil, - "org_id"=>nil, - "com_id"=>nil, - "restarted_at"=>nil, - "unique_number"=>nil, :jobs=> - [{"id"=>job.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>build.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "job_state_id"=>nil, - "stage_number"=>nil, - "stage_id"=>nil, - "org_id"=>nil, - "com_id"=>nil, - "restarted_at"=>nil, - "priority"=>nil, - :log_url=>"https://api.travis-ci.org/v3/job/#{job.id}/log.txt"}]}]] + [{"id"=>job.id, + "repository_id"=>repository.id, + "commit_id"=>nil, + "source_id"=>build.id, + "source_type"=>"Build", + "queue"=>nil, + "type"=>nil, + "state"=>nil, + "number"=>nil, + "config"=>nil, + "worker"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "tags"=>nil, + "allow_failure"=>false, + "owner_id"=>nil, + "owner_type"=>nil, + "result"=>nil, + "queued_at"=>nil, + "canceled_at"=>nil, + "received_at"=>nil, + "debug_options"=>nil, + "private"=>nil, + "stage_number"=>nil, + "stage_id"=>nil + }] + }]] } before do From 97935869c0dd4e35374bbc92c504e6c33070b29f Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 17 Aug 2021 00:19:12 +0200 Subject: [PATCH 14/84] bug with URI fixed --- spec/backup_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index a778cf4..47417b1 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -1,4 +1,5 @@ $: << 'lib' +require 'uri' require 'backup' require 'models/repository' require 'support/factories' From e133d53ee3a6a121310775c68bb786d7f84b179f Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 17 Aug 2021 01:39:29 +0200 Subject: [PATCH 15/84] new tests and fixes --- lib/backup.rb | 6 +- lib/config.rb | 17 +++-- spec/backup_spec.rb | 181 +++++++++++++++++++++++++++----------------- 3 files changed, 125 insertions(+), 79 deletions(-) diff --git a/lib/backup.rb b/lib/backup.rb index a8aefe5..f91871b 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -7,8 +7,8 @@ # main travis-backup class class Backup - def initialize - @config = Config.new + def initialize(config_args={}) + @config = Config.new(config_args) connect_db end @@ -38,7 +38,7 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe repository.builds.where('created_at < ? and id != ?', delay, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| if builds_batch.count == @config.limit.to_i - @config.if_backup ? save_batch(builds_batch, repository) : builds.each(&:destroy) + @config.if_backup ? save_batch(builds_batch, repository) : builds_batch.each(&:destroy) end end end diff --git a/lib/config.rb b/lib/config.rb index 34d2e50..f987471 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -4,13 +4,20 @@ class Config attr_reader :if_backup, :limit, :delay, :database_url - def initialize # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength + def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) connection_details = YAML.load(File.open('config/database.yml')) - @if_backup = ENV['IF_BACKUP'] || config['backup']['if_backup'] - @limit = ENV['BACKUP_LIMIT'] || config['backup']['limit'] - @delay = ENV['BACKUP_DELAY'] || config['backup']['delay'] - @database_url = ENV['DATABASE_URL'] || connection_details['development'] + if !args[:if_backup].nil? + @if_backup = args[:if_backup] + elsif !ENV['IF_BACKUP'].nil? + @if_backup = ENV['IF_BACKUP'] + else + @if_backup = config['backup']['if_backup'] + end + + @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config['backup']['limit'] + @delay = args[:delay] || ENV['BACKUP_DELAY'] || config['backup']['delay'] + @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details['development'] end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 47417b1..6b394ad 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -2,6 +2,8 @@ require 'uri' require 'backup' require 'models/repository' +require 'models/build' +require 'models/job' require 'support/factories' require 'pry' @@ -60,78 +62,115 @@ repository: private_repository ) } - let(:exported_object) { - [[{"id"=>build.id, - "repository_id"=>repository.id, - "number"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "config"=>nil, - "commit_id"=>nil, - "request_id"=>nil, - "state"=>nil, - "duration"=>nil, - "owner_id"=>nil, - "owner_type"=>nil, - "event_type"=>nil, - "previous_state"=>nil, - "pull_request_title"=>nil, - "pull_request_number"=>nil, - "branch"=>nil, - "canceled_at"=>nil, - "cached_matrix_ids"=>nil, - "received_at"=>nil, - "private"=>nil, - "pull_request_id"=>nil, - "branch_id"=>nil, - "tag_id"=>nil, - "sender_id"=>nil, - "sender_type"=>nil, - :jobs=> - [{"id"=>job.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>build.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "config"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "stage_number"=>nil, - "stage_id"=>nil - }] - }]] - } - before do - build.jobs = [job] - repository.builds = [build] - private_build.jobs = [private_job] - private_repository.builds = [private_build] - end + describe 'process_repo' do + let(:exported_object) { + [[{"id"=>build.id, + "repository_id"=>repository.id, + "number"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "config"=>nil, + "commit_id"=>nil, + "request_id"=>nil, + "state"=>nil, + "duration"=>nil, + "owner_id"=>nil, + "owner_type"=>nil, + "event_type"=>nil, + "previous_state"=>nil, + "pull_request_title"=>nil, + "pull_request_number"=>nil, + "branch"=>nil, + "canceled_at"=>nil, + "cached_matrix_ids"=>nil, + "received_at"=>nil, + "private"=>nil, + "pull_request_id"=>nil, + "branch_id"=>nil, + "tag_id"=>nil, + "sender_id"=>nil, + "sender_type"=>nil, + :jobs=> + [{"id"=>job.id, + "repository_id"=>repository.id, + "commit_id"=>nil, + "source_id"=>build.id, + "source_type"=>"Build", + "queue"=>nil, + "type"=>nil, + "state"=>nil, + "number"=>nil, + "config"=>nil, + "worker"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "tags"=>nil, + "allow_failure"=>false, + "owner_id"=>nil, + "owner_type"=>nil, + "result"=>nil, + "queued_at"=>nil, + "canceled_at"=>nil, + "received_at"=>nil, + "debug_options"=>nil, + "private"=>nil, + "stage_number"=>nil, + "stage_id"=>nil + }] + }]] + } + + before do + build.jobs = [job] + repository.builds = [build] + private_build.jobs = [private_job] + private_repository.builds = [private_build] + end + + shared_context 'removing builds and jobs' do + it 'should delete build' do + expect do + backup.process_repo(repository) + end.to change { Build.count }.by(-1) + end + + it 'should delete job' do + expect do + backup.process_repo(repository) + end.to change { Job.count }.by(-1) + end + end + + context 'when if_backup config is set to true' do + it 'should prepare proper JSON export' do + build_export = backup.process_repo(repository) + build_export.first.first[:updated_at] = datetime + build_export.first.first[:jobs].first[:updated_at] = datetime + expect(build_export.to_json).to eq(exported_object.to_json) + end + + it 'should save JSON to file' do + expect(File).to receive(:open).once + backup.process_repo(repository) + end + + it_behaves_like 'removing builds and jobs' + end + + context 'when if_backup config is set to false' do + let!(:backup) { Backup.new(if_backup: false) } + + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo(repository) + end - it 'should prepare proper JSON export' do - build_export = backup.process_repo(repository) - build_export.first.first[:updated_at] = datetime - build_export.first.first[:jobs].first[:updated_at] = datetime - expect(build_export.to_json).to eq(exported_object.to_json) + it_behaves_like 'removing builds and jobs' + end end end From be8da5186736a241b983ea8072e4c0c658976df4 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 18 Aug 2021 00:10:44 +0200 Subject: [PATCH 16/84] configurable backup files path --- README.md | 1 + config/settings.yml | 4 +++- lib/backup.rb | 6 +++++- lib/config.rb | 3 ++- spec/backup_spec.rb | 17 ++++++++++++++--- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d9ea8b1..24ad840 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ to json files. `IF_BACKUP` `BACKUP_LIMIT` `BACKUP_DELAY` +`BACKUP_FILES_LOCATION` `DATABASE_URL` * How to run the test suite diff --git a/config/settings.yml b/config/settings.yml index 0646c21..af36b3a 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -3,5 +3,7 @@ backup: if_backup: true # builds limit in file limit: 1000 - # delay in months + # number of months - data younger than this time won't be backuped delay: 6 + # path of the folder in which backup files will be placed + files_location: './dump' \ No newline at end of file diff --git a/lib/backup.rb b/lib/backup.rb index f91871b..615f4ed 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -58,7 +58,11 @@ def save_batch(builds_batch, repository) def upload(file_name, content) # rubocop:disable Metrics/MethodLength uploaded = false begin - File.open("dump/#{file_name}", 'w') do |file| + unless File.directory?(@config.files_location) + FileUtils.mkdir_p(@config.files_location) + end + + File.open("#{@config.files_location}/#{file_name}", 'w') do |file| file.write(content) file.close uploaded = true diff --git a/lib/config.rb b/lib/config.rb index f987471..fe4cd00 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,7 @@ # Config for travis-backup class Config - attr_reader :if_backup, :limit, :delay, :database_url + attr_reader :if_backup, :limit, :delay, :files_location, :database_url def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = YAML.load(File.open('config/settings.yml')) @@ -18,6 +18,7 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config['backup']['limit'] @delay = args[:delay] || ENV['BACKUP_DELAY'] || config['backup']['delay'] + @files_location = args[:files_location] || ENV['BACKUP_FILES_LOCATION'] || config['backup']['files_location'] @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details['development'] end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 6b394ad..0d035cd 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -9,7 +9,8 @@ describe Backup do let!(:config) { Config.new } - let!(:backup) { Backup.new } + let(:files_location) { "dump/tests" } + let!(:backup) { Backup.new(files_location: files_location) } let(:datetime) { (config.delay + 1).months.ago.to_time.utc } let(:org_id) { rand(100000) } let(:com_id) { rand(100000) } @@ -154,11 +155,21 @@ expect(build_export.to_json).to eq(exported_object.to_json) end - it 'should save JSON to file' do - expect(File).to receive(:open).once + it 'should save JSON to file at proper path' do + expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') backup.process_repo(repository) end + context 'when path with nonexistent folders is given' do + let(:random_files_location) { "dump/tests/#{rand(100000)}" } + let!(:backup) { Backup.new(files_location: random_files_location) } + + it 'should create needed folders' do + expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original + backup.process_repo(repository) + end + end + it_behaves_like 'removing builds and jobs' end From dac885f944762c1ef3a4389964c34da294e65b37 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 18 Aug 2021 11:33:12 +0200 Subject: [PATCH 17/84] resetting database before tests --- db/schema.sql | 3502 +++++++++++++++++++++++++++++++++++++++++++ spec/backup_spec.rb | 4 + 2 files changed, 3506 insertions(+) create mode 100644 db/schema.sql diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 0000000..74b07fd --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,3502 @@ +DROP SCHEMA public CASCADE; +DROP SCHEMA sqitch CASCADE; +CREATE SCHEMA public; + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; + +-- +-- Name: sqitch; Type: SCHEMA; Schema: -; Owner: postgres +-- + +CREATE SCHEMA sqitch; + + +ALTER SCHEMA sqitch OWNER TO postgres; + +-- +-- Name: SCHEMA sqitch; Type: COMMENT; Schema: -; Owner: postgres +-- + +COMMENT ON SCHEMA sqitch IS 'Sqitch database deployment metadata v1.1.'; + + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +-- +-- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; + + +-- +-- Name: EXTENSION pg_trgm; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION pg_trgm IS 'text similarity measurement and index searching based on trigrams'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +SET search_path = public, pg_catalog; + +-- +-- Name: source_type; Type: TYPE; Schema: public; Owner: postgres +-- + +CREATE TYPE source_type AS ENUM ( + 'manual', + 'stripe', + 'github', + 'unknown' +); + + +ALTER TYPE public.source_type OWNER TO postgres; + +-- +-- Name: set_updated_at(); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION set_updated_at() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF TG_OP = 'INSERT' OR + (TG_OP = 'UPDATE' AND NEW.* IS DISTINCT FROM OLD.*) THEN + NEW.updated_at := statement_timestamp(); + END IF; + RETURN NEW; + END; + $$; + + +ALTER FUNCTION public.set_updated_at() OWNER TO postgres; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: abuses; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE abuses ( + id integer NOT NULL, + owner_id integer, + owner_type character varying, + request_id integer, + level integer NOT NULL, + reason character varying NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.abuses OWNER TO postgres; + +-- +-- Name: abuses_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE abuses_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.abuses_id_seq OWNER TO postgres; + +-- +-- Name: abuses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE abuses_id_seq OWNED BY abuses.id; + + +-- +-- Name: annotation_providers; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE annotation_providers ( + id integer NOT NULL, + name character varying, + api_username character varying, + api_key character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.annotation_providers OWNER TO postgres; + +-- +-- Name: annotation_providers_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE annotation_providers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.annotation_providers_id_seq OWNER TO postgres; + +-- +-- Name: annotation_providers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE annotation_providers_id_seq OWNED BY annotation_providers.id; + + +-- +-- Name: annotations; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE annotations ( + id integer NOT NULL, + job_id integer NOT NULL, + url character varying, + description text NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + annotation_provider_id integer NOT NULL, + status character varying +); + + +ALTER TABLE public.annotations OWNER TO postgres; + +-- +-- Name: annotations_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE annotations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.annotations_id_seq OWNER TO postgres; + +-- +-- Name: annotations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE annotations_id_seq OWNED BY annotations.id; + + +-- +-- Name: beta_features; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE beta_features ( + id integer NOT NULL, + name character varying, + description text, + feedback_url character varying, + staff_only boolean, + default_enabled boolean, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +ALTER TABLE public.beta_features OWNER TO postgres; + +-- +-- Name: beta_features_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE beta_features_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.beta_features_id_seq OWNER TO postgres; + +-- +-- Name: beta_features_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE beta_features_id_seq OWNED BY beta_features.id; + + +-- +-- Name: branches; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE branches ( + id integer NOT NULL, + repository_id integer NOT NULL, + last_build_id integer, + name character varying NOT NULL, + exists_on_github boolean DEFAULT true NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.branches OWNER TO postgres; + +-- +-- Name: branches_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE branches_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.branches_id_seq OWNER TO postgres; + +-- +-- Name: branches_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE branches_id_seq OWNED BY branches.id; + + +-- +-- Name: broadcasts; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE broadcasts ( + id integer NOT NULL, + recipient_id integer, + recipient_type character varying, + kind character varying, + message character varying, + expired boolean, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + category character varying +); + + +ALTER TABLE public.broadcasts OWNER TO postgres; + +-- +-- Name: broadcasts_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE broadcasts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.broadcasts_id_seq OWNER TO postgres; + +-- +-- Name: broadcasts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE broadcasts_id_seq OWNED BY broadcasts.id; + + +-- +-- Name: shared_builds_tasks_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE shared_builds_tasks_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.shared_builds_tasks_seq OWNER TO postgres; + +-- +-- Name: builds; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE builds ( + id bigint DEFAULT nextval('shared_builds_tasks_seq'::regclass) NOT NULL, + repository_id integer, + number character varying, + started_at timestamp without time zone, + finished_at timestamp without time zone, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + config text, + commit_id integer, + request_id integer, + state character varying, + duration integer, + owner_id integer, + owner_type character varying, + event_type character varying, + previous_state character varying, + pull_request_title text, + pull_request_number integer, + branch character varying, + canceled_at timestamp without time zone, + cached_matrix_ids integer[], + received_at timestamp without time zone, + private boolean, + pull_request_id integer, + branch_id integer, + tag_id integer, + sender_id integer, + sender_type character varying +); + + +ALTER TABLE public.builds OWNER TO postgres; + +-- +-- Name: builds_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE builds_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.builds_id_seq OWNER TO postgres; + +-- +-- Name: builds_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE builds_id_seq OWNED BY builds.id; + + +-- +-- Name: commits; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE commits ( + id integer NOT NULL, + repository_id integer, + commit character varying, + ref character varying, + branch character varying, + message text, + compare_url character varying, + committed_at timestamp without time zone, + committer_name character varying, + committer_email character varying, + author_name character varying, + author_email character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + branch_id integer, + tag_id integer +); + + +ALTER TABLE public.commits OWNER TO postgres; + +-- +-- Name: commits_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE commits_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.commits_id_seq OWNER TO postgres; + +-- +-- Name: commits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE commits_id_seq OWNED BY commits.id; + + +-- +-- Name: coupons; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE coupons ( + id integer NOT NULL, + percent_off integer, + coupon_id character varying, + redeem_by timestamp without time zone, + amount_off integer, + duration character varying, + duration_in_months integer, + max_redemptions integer, + redemptions integer +); + + +ALTER TABLE public.coupons OWNER TO postgres; + +-- +-- Name: coupons_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE coupons_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.coupons_id_seq OWNER TO postgres; + +-- +-- Name: coupons_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE coupons_id_seq OWNED BY coupons.id; + + +-- +-- Name: crons; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE crons ( + id integer NOT NULL, + branch_id integer, + "interval" character varying NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + next_run timestamp without time zone, + last_run timestamp without time zone, + dont_run_if_recent_build_exists boolean DEFAULT false +); + + +ALTER TABLE public.crons OWNER TO postgres; + +-- +-- Name: crons_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE crons_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.crons_id_seq OWNER TO postgres; + +-- +-- Name: crons_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE crons_id_seq OWNED BY crons.id; + + +-- +-- Name: emails; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE emails ( + id integer NOT NULL, + user_id integer, + email character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.emails OWNER TO postgres; + +-- +-- Name: emails_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE emails_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.emails_id_seq OWNER TO postgres; + +-- +-- Name: emails_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE emails_id_seq OWNED BY emails.id; + + +-- +-- Name: invoices; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE invoices ( + id integer NOT NULL, + object text, + created_at timestamp without time zone, + updated_at timestamp without time zone, + subscription_id integer, + invoice_id character varying, + stripe_id character varying, + cc_last_digits character varying +); + + +ALTER TABLE public.invoices OWNER TO postgres; + +-- +-- Name: invoices_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE invoices_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.invoices_id_seq OWNER TO postgres; + +-- +-- Name: invoices_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE invoices_id_seq OWNED BY invoices.id; + + +-- +-- Name: jobs; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE jobs ( + id bigint DEFAULT nextval('shared_builds_tasks_seq'::regclass) NOT NULL, + repository_id integer, + commit_id integer, + source_id integer, + source_type character varying, + queue character varying, + type character varying, + state character varying, + number character varying, + config text, + worker character varying, + started_at timestamp without time zone, + finished_at timestamp without time zone, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + tags text, + allow_failure boolean DEFAULT false, + owner_id integer, + owner_type character varying, + result integer, + queued_at timestamp without time zone, + canceled_at timestamp without time zone, + received_at timestamp without time zone, + debug_options text, + private boolean, + stage_number character varying, + stage_id integer +); + + +ALTER TABLE public.jobs OWNER TO postgres; + +-- +-- Name: jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE jobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.jobs_id_seq OWNER TO postgres; + +-- +-- Name: jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE jobs_id_seq OWNED BY jobs.id; + + +-- +-- Name: log_parts; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE log_parts ( + id bigint NOT NULL, + log_id integer NOT NULL, + content text, + number integer, + final boolean, + created_at timestamp without time zone DEFAULT '2000-01-01 00:00:00'::timestamp without time zone NOT NULL +) +WITH (autovacuum_vacuum_threshold='0', autovacuum_vacuum_scale_factor='0.001'); + + +ALTER TABLE public.log_parts OWNER TO postgres; + +-- +-- Name: log_parts_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE log_parts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.log_parts_id_seq OWNER TO postgres; + +-- +-- Name: log_parts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE log_parts_id_seq OWNED BY log_parts.id; + + +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE logs ( + id integer NOT NULL, + job_id integer, + content text, + removed_by integer, + created_at timestamp without time zone, + updated_at timestamp without time zone, + aggregated_at timestamp without time zone, + archived_at timestamp without time zone, + purged_at timestamp without time zone, + removed_at timestamp without time zone, + archiving boolean, + archive_verified boolean +); + + +ALTER TABLE public.logs OWNER TO postgres; + +-- +-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE logs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.logs_id_seq OWNER TO postgres; + +-- +-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE logs_id_seq OWNED BY logs.id; + + +-- +-- Name: memberships; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE memberships ( + id integer NOT NULL, + organization_id integer, + user_id integer, + role character varying +); + + +ALTER TABLE public.memberships OWNER TO postgres; + +-- +-- Name: memberships_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE memberships_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.memberships_id_seq OWNER TO postgres; + +-- +-- Name: memberships_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE memberships_id_seq OWNED BY memberships.id; + + +-- +-- Name: messages; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE messages ( + id integer NOT NULL, + subject_id integer, + subject_type character varying, + level character varying, + key character varying, + code character varying, + args json, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.messages OWNER TO postgres; + +-- +-- Name: messages_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE messages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.messages_id_seq OWNER TO postgres; + +-- +-- Name: messages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE messages_id_seq OWNED BY messages.id; + + +-- +-- Name: organizations; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE organizations ( + id integer NOT NULL, + name character varying, + login character varying, + github_id integer, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + avatar_url character varying, + location character varying, + email character varying, + company character varying, + homepage character varying, + billing_admin_only boolean +); + + +ALTER TABLE public.organizations OWNER TO postgres; + +-- +-- Name: organizations_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE organizations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.organizations_id_seq OWNER TO postgres; + +-- +-- Name: organizations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE organizations_id_seq OWNED BY organizations.id; + + +-- +-- Name: owner_groups; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE owner_groups ( + id integer NOT NULL, + uuid character varying, + owner_id integer, + owner_type character varying, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +ALTER TABLE public.owner_groups OWNER TO postgres; + +-- +-- Name: owner_groups_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE owner_groups_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.owner_groups_id_seq OWNER TO postgres; + +-- +-- Name: owner_groups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE owner_groups_id_seq OWNED BY owner_groups.id; + + +-- +-- Name: permissions; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE permissions ( + id integer NOT NULL, + user_id integer, + repository_id integer, + admin boolean DEFAULT false, + push boolean DEFAULT false, + pull boolean DEFAULT false +); + + +ALTER TABLE public.permissions OWNER TO postgres; + +-- +-- Name: permissions_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE permissions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.permissions_id_seq OWNER TO postgres; + +-- +-- Name: permissions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE permissions_id_seq OWNED BY permissions.id; + + +-- +-- Name: pull_requests; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE pull_requests ( + id integer NOT NULL, + repository_id integer, + number integer, + title character varying, + state character varying, + head_repo_github_id integer, + head_repo_slug character varying, + head_ref character varying, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +ALTER TABLE public.pull_requests OWNER TO postgres; + +-- +-- Name: pull_requests_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE pull_requests_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.pull_requests_id_seq OWNER TO postgres; + +-- +-- Name: pull_requests_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE pull_requests_id_seq OWNED BY pull_requests.id; + + +-- +-- Name: queueable_jobs; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE queueable_jobs ( + id integer NOT NULL, + job_id integer +); + + +ALTER TABLE public.queueable_jobs OWNER TO postgres; + +-- +-- Name: queueable_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE queueable_jobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.queueable_jobs_id_seq OWNER TO postgres; + +-- +-- Name: queueable_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE queueable_jobs_id_seq OWNED BY queueable_jobs.id; + + +-- +-- Name: repositories; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE repositories ( + id integer NOT NULL, + name character varying, + url character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + last_build_id integer, + last_build_number character varying, + last_build_started_at timestamp without time zone, + last_build_finished_at timestamp without time zone, + owner_name character varying, + owner_email text, + active boolean, + description text, + last_build_duration integer, + owner_id integer, + owner_type character varying, + private boolean DEFAULT false, + last_build_state character varying, + github_id integer, + default_branch character varying, + github_language character varying, + settings json, + next_build_number integer, + invalidated_at timestamp without time zone, + current_build_id bigint +); + + +ALTER TABLE public.repositories OWNER TO postgres; + +-- +-- Name: repositories_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE repositories_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.repositories_id_seq OWNER TO postgres; + +-- +-- Name: repositories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE repositories_id_seq OWNED BY repositories.id; + + +-- +-- Name: requests; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE requests ( + id integer NOT NULL, + repository_id integer, + commit_id integer, + state character varying, + source character varying, + payload text, + token character varying, + config text, + started_at timestamp without time zone, + finished_at timestamp without time zone, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + event_type character varying, + comments_url character varying, + base_commit character varying, + head_commit character varying, + owner_id integer, + owner_type character varying, + result character varying, + message character varying, + private boolean, + pull_request_id integer, + branch_id integer, + tag_id integer, + sender_id integer, + sender_type character varying +); + + +ALTER TABLE public.requests OWNER TO postgres; + +-- +-- Name: requests_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE requests_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.requests_id_seq OWNER TO postgres; + +-- +-- Name: requests_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE requests_id_seq OWNED BY requests.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE schema_migrations ( + version character varying NOT NULL +); + + +ALTER TABLE public.schema_migrations OWNER TO postgres; + +-- +-- Name: ssl_keys; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE ssl_keys ( + id integer NOT NULL, + repository_id integer, + public_key text, + private_key text, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.ssl_keys OWNER TO postgres; + +-- +-- Name: ssl_keys_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE ssl_keys_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.ssl_keys_id_seq OWNER TO postgres; + +-- +-- Name: ssl_keys_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE ssl_keys_id_seq OWNED BY ssl_keys.id; + + +-- +-- Name: stages; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE stages ( + id integer NOT NULL, + build_id integer, + number integer, + name character varying, + state character varying, + started_at timestamp without time zone, + finished_at timestamp without time zone +); + + +ALTER TABLE public.stages OWNER TO postgres; + +-- +-- Name: stages_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE stages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.stages_id_seq OWNER TO postgres; + +-- +-- Name: stages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE stages_id_seq OWNED BY stages.id; + + +-- +-- Name: stars; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE stars ( + id integer NOT NULL, + repository_id integer, + user_id integer, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.stars OWNER TO postgres; + +-- +-- Name: stars_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE stars_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.stars_id_seq OWNER TO postgres; + +-- +-- Name: stars_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE stars_id_seq OWNED BY stars.id; + + +-- +-- Name: stripe_events; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE stripe_events ( + id integer NOT NULL, + created_at timestamp without time zone, + updated_at timestamp without time zone, + event_object text, + event_type character varying, + date timestamp without time zone, + event_id character varying +); + + +ALTER TABLE public.stripe_events OWNER TO postgres; + +-- +-- Name: stripe_events_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE stripe_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.stripe_events_id_seq OWNER TO postgres; + +-- +-- Name: stripe_events_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE stripe_events_id_seq OWNED BY stripe_events.id; + + +-- +-- Name: subscriptions; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE subscriptions ( + id integer NOT NULL, + cc_token character varying, + valid_to timestamp without time zone, + owner_id integer, + owner_type character varying, + first_name character varying, + last_name character varying, + company character varying, + zip_code character varying, + address character varying, + address2 character varying, + city character varying, + state character varying, + country character varying, + vat_id character varying, + customer_id character varying, + created_at timestamp without time zone, + updated_at timestamp without time zone, + cc_owner character varying, + cc_last_digits character varying, + cc_expiration_date character varying, + billing_email character varying, + selected_plan character varying, + coupon character varying, + contact_id integer, + canceled_at timestamp without time zone, + canceled_by_id integer, + status character varying, + source source_type DEFAULT 'unknown'::source_type NOT NULL, + concurrency integer +); + + +ALTER TABLE public.subscriptions OWNER TO postgres; + +-- +-- Name: subscriptions_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE subscriptions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.subscriptions_id_seq OWNER TO postgres; + +-- +-- Name: subscriptions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE subscriptions_id_seq OWNED BY subscriptions.id; + + +-- +-- Name: tags; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE tags ( + id integer NOT NULL, + repository_id integer, + name character varying, + last_build_id integer, + exists_on_github boolean, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +ALTER TABLE public.tags OWNER TO postgres; + +-- +-- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE tags_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.tags_id_seq OWNER TO postgres; + +-- +-- Name: tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE tags_id_seq OWNED BY tags.id; + + +-- +-- Name: tokens; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE tokens ( + id integer NOT NULL, + user_id integer, + token character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.tokens OWNER TO postgres; + +-- +-- Name: tokens_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.tokens_id_seq OWNER TO postgres; + +-- +-- Name: tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE tokens_id_seq OWNED BY tokens.id; + + +-- +-- Name: trial_allowances; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE trial_allowances ( + id integer NOT NULL, + trial_id integer, + creator_id integer, + creator_type character varying, + builds_allowed integer, + builds_remaining integer, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +ALTER TABLE public.trial_allowances OWNER TO postgres; + +-- +-- Name: trial_allowances_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE trial_allowances_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.trial_allowances_id_seq OWNER TO postgres; + +-- +-- Name: trial_allowances_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE trial_allowances_id_seq OWNED BY trial_allowances.id; + + +-- +-- Name: trials; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE trials ( + id integer NOT NULL, + owner_id integer, + owner_type character varying, + chartmogul_customer_uuids text[] DEFAULT '{}'::text[], + status character varying DEFAULT 'new'::character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.trials OWNER TO postgres; + +-- +-- Name: trials_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE trials_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.trials_id_seq OWNER TO postgres; + +-- +-- Name: trials_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE trials_id_seq OWNED BY trials.id; + + +-- +-- Name: urls; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE urls ( + id integer NOT NULL, + url character varying, + code character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +ALTER TABLE public.urls OWNER TO postgres; + +-- +-- Name: urls_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE urls_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.urls_id_seq OWNER TO postgres; + +-- +-- Name: urls_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE urls_id_seq OWNED BY urls.id; + + +-- +-- Name: user_beta_features; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE user_beta_features ( + id integer NOT NULL, + user_id integer, + beta_feature_id integer, + enabled boolean, + last_deactivated_at timestamp without time zone, + last_activated_at timestamp without time zone +); + + +ALTER TABLE public.user_beta_features OWNER TO postgres; + +-- +-- Name: user_beta_features_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE user_beta_features_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.user_beta_features_id_seq OWNER TO postgres; + +-- +-- Name: user_beta_features_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE user_beta_features_id_seq OWNED BY user_beta_features.id; + + +-- +-- Name: users; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE users ( + id integer NOT NULL, + name character varying, + login character varying, + email character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + is_admin boolean DEFAULT false, + github_id integer, + github_oauth_token character varying, + gravatar_id character varying, + locale character varying, + is_syncing boolean, + synced_at timestamp without time zone, + github_scopes text, + education boolean, + first_logged_in_at timestamp without time zone, + avatar_url character varying, + suspended boolean DEFAULT false, + suspended_at timestamp without time zone +); + + +ALTER TABLE public.users OWNER TO postgres; + +-- +-- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE users_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.users_id_seq OWNER TO postgres; + +-- +-- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE users_id_seq OWNED BY users.id; + + +SET search_path = sqitch, pg_catalog; + +-- +-- Name: changes; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE changes ( + change_id text NOT NULL, + script_hash text, + change text NOT NULL, + project text NOT NULL, + note text DEFAULT ''::text NOT NULL, + committed_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL, + committer_name text NOT NULL, + committer_email text NOT NULL, + planned_at timestamp with time zone NOT NULL, + planner_name text NOT NULL, + planner_email text NOT NULL +); + + +ALTER TABLE sqitch.changes OWNER TO postgres; + +-- +-- Name: TABLE changes; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE changes IS 'Tracks the changes currently deployed to the database.'; + + +-- +-- Name: COLUMN changes.change_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.change_id IS 'Change primary key.'; + + +-- +-- Name: COLUMN changes.script_hash; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.script_hash IS 'Deploy script SHA-1 hash.'; + + +-- +-- Name: COLUMN changes.change; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.change IS 'Name of a deployed change.'; + + +-- +-- Name: COLUMN changes.project; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.project IS 'Name of the Sqitch project to which the change belongs.'; + + +-- +-- Name: COLUMN changes.note; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.note IS 'Description of the change.'; + + +-- +-- Name: COLUMN changes.committed_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.committed_at IS 'Date the change was deployed.'; + + +-- +-- Name: COLUMN changes.committer_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.committer_name IS 'Name of the user who deployed the change.'; + + +-- +-- Name: COLUMN changes.committer_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.committer_email IS 'Email address of the user who deployed the change.'; + + +-- +-- Name: COLUMN changes.planned_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.planned_at IS 'Date the change was added to the plan.'; + + +-- +-- Name: COLUMN changes.planner_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.planner_name IS 'Name of the user who planed the change.'; + + +-- +-- Name: COLUMN changes.planner_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN changes.planner_email IS 'Email address of the user who planned the change.'; + + +-- +-- Name: dependencies; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE dependencies ( + change_id text NOT NULL, + type text NOT NULL, + dependency text NOT NULL, + dependency_id text, + CONSTRAINT dependencies_check CHECK ((((type = 'require'::text) AND (dependency_id IS NOT NULL)) OR ((type = 'conflict'::text) AND (dependency_id IS NULL)))) +); + + +ALTER TABLE sqitch.dependencies OWNER TO postgres; + +-- +-- Name: TABLE dependencies; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE dependencies IS 'Tracks the currently satisfied dependencies.'; + + +-- +-- Name: COLUMN dependencies.change_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN dependencies.change_id IS 'ID of the depending change.'; + + +-- +-- Name: COLUMN dependencies.type; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN dependencies.type IS 'Type of dependency.'; + + +-- +-- Name: COLUMN dependencies.dependency; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN dependencies.dependency IS 'Dependency name.'; + + +-- +-- Name: COLUMN dependencies.dependency_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN dependencies.dependency_id IS 'Change ID the dependency resolves to.'; + + +-- +-- Name: events; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE events ( + event text NOT NULL, + change_id text NOT NULL, + change text NOT NULL, + project text NOT NULL, + note text DEFAULT ''::text NOT NULL, + requires text[] DEFAULT '{}'::text[] NOT NULL, + conflicts text[] DEFAULT '{}'::text[] NOT NULL, + tags text[] DEFAULT '{}'::text[] NOT NULL, + committed_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL, + committer_name text NOT NULL, + committer_email text NOT NULL, + planned_at timestamp with time zone NOT NULL, + planner_name text NOT NULL, + planner_email text NOT NULL, + CONSTRAINT events_event_check CHECK ((event = ANY (ARRAY['deploy'::text, 'revert'::text, 'fail'::text, 'merge'::text]))) +); + + +ALTER TABLE sqitch.events OWNER TO postgres; + +-- +-- Name: TABLE events; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE events IS 'Contains full history of all deployment events.'; + + +-- +-- Name: COLUMN events.event; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.event IS 'Type of event.'; + + +-- +-- Name: COLUMN events.change_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.change_id IS 'Change ID.'; + + +-- +-- Name: COLUMN events.change; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.change IS 'Change name.'; + + +-- +-- Name: COLUMN events.project; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.project IS 'Name of the Sqitch project to which the change belongs.'; + + +-- +-- Name: COLUMN events.note; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.note IS 'Description of the change.'; + + +-- +-- Name: COLUMN events.requires; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.requires IS 'Array of the names of required changes.'; + + +-- +-- Name: COLUMN events.conflicts; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.conflicts IS 'Array of the names of conflicting changes.'; + + +-- +-- Name: COLUMN events.tags; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.tags IS 'Tags associated with the change.'; + + +-- +-- Name: COLUMN events.committed_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.committed_at IS 'Date the event was committed.'; + + +-- +-- Name: COLUMN events.committer_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.committer_name IS 'Name of the user who committed the event.'; + + +-- +-- Name: COLUMN events.committer_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.committer_email IS 'Email address of the user who committed the event.'; + + +-- +-- Name: COLUMN events.planned_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.planned_at IS 'Date the event was added to the plan.'; + + +-- +-- Name: COLUMN events.planner_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.planner_name IS 'Name of the user who planed the change.'; + + +-- +-- Name: COLUMN events.planner_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN events.planner_email IS 'Email address of the user who plan planned the change.'; + + +-- +-- Name: projects; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE projects ( + project text NOT NULL, + uri text, + created_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL, + creator_name text NOT NULL, + creator_email text NOT NULL +); + + +ALTER TABLE sqitch.projects OWNER TO postgres; + +-- +-- Name: TABLE projects; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE projects IS 'Sqitch projects deployed to this database.'; + + +-- +-- Name: COLUMN projects.project; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN projects.project IS 'Unique Name of a project.'; + + +-- +-- Name: COLUMN projects.uri; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN projects.uri IS 'Optional project URI'; + + +-- +-- Name: COLUMN projects.created_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN projects.created_at IS 'Date the project was added to the database.'; + + +-- +-- Name: COLUMN projects.creator_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN projects.creator_name IS 'Name of the user who added the project.'; + + +-- +-- Name: COLUMN projects.creator_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN projects.creator_email IS 'Email address of the user who added the project.'; + + +-- +-- Name: releases; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE releases ( + version real NOT NULL, + installed_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL, + installer_name text NOT NULL, + installer_email text NOT NULL +); + + +ALTER TABLE sqitch.releases OWNER TO postgres; + +-- +-- Name: TABLE releases; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE releases IS 'Sqitch registry releases.'; + + +-- +-- Name: COLUMN releases.version; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN releases.version IS 'Version of the Sqitch registry.'; + + +-- +-- Name: COLUMN releases.installed_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN releases.installed_at IS 'Date the registry release was installed.'; + + +-- +-- Name: COLUMN releases.installer_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN releases.installer_name IS 'Name of the user who installed the registry release.'; + + +-- +-- Name: COLUMN releases.installer_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN releases.installer_email IS 'Email address of the user who installed the registry release.'; + + +-- +-- Name: tags; Type: TABLE; Schema: sqitch; Owner: postgres; Tablespace: +-- + +CREATE TABLE tags ( + tag_id text NOT NULL, + tag text NOT NULL, + project text NOT NULL, + change_id text NOT NULL, + note text DEFAULT ''::text NOT NULL, + committed_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL, + committer_name text NOT NULL, + committer_email text NOT NULL, + planned_at timestamp with time zone NOT NULL, + planner_name text NOT NULL, + planner_email text NOT NULL +); + + +ALTER TABLE sqitch.tags OWNER TO postgres; + +-- +-- Name: TABLE tags; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON TABLE tags IS 'Tracks the tags currently applied to the database.'; + + +-- +-- Name: COLUMN tags.tag_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.tag_id IS 'Tag primary key.'; + + +-- +-- Name: COLUMN tags.tag; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.tag IS 'Project-unique tag name.'; + + +-- +-- Name: COLUMN tags.project; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.project IS 'Name of the Sqitch project to which the tag belongs.'; + + +-- +-- Name: COLUMN tags.change_id; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.change_id IS 'ID of last change deployed before the tag was applied.'; + + +-- +-- Name: COLUMN tags.note; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.note IS 'Description of the tag.'; + + +-- +-- Name: COLUMN tags.committed_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.committed_at IS 'Date the tag was applied to the database.'; + + +-- +-- Name: COLUMN tags.committer_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.committer_name IS 'Name of the user who applied the tag.'; + + +-- +-- Name: COLUMN tags.committer_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.committer_email IS 'Email address of the user who applied the tag.'; + + +-- +-- Name: COLUMN tags.planned_at; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.planned_at IS 'Date the tag was added to the plan.'; + + +-- +-- Name: COLUMN tags.planner_name; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.planner_name IS 'Name of the user who planed the tag.'; + + +-- +-- Name: COLUMN tags.planner_email; Type: COMMENT; Schema: sqitch; Owner: postgres +-- + +COMMENT ON COLUMN tags.planner_email IS 'Email address of the user who planned the tag.'; + + +SET search_path = public, pg_catalog; + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY abuses ALTER COLUMN id SET DEFAULT nextval('abuses_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY annotation_providers ALTER COLUMN id SET DEFAULT nextval('annotation_providers_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY annotations ALTER COLUMN id SET DEFAULT nextval('annotations_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY beta_features ALTER COLUMN id SET DEFAULT nextval('beta_features_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY branches ALTER COLUMN id SET DEFAULT nextval('branches_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY broadcasts ALTER COLUMN id SET DEFAULT nextval('broadcasts_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY commits ALTER COLUMN id SET DEFAULT nextval('commits_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY coupons ALTER COLUMN id SET DEFAULT nextval('coupons_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY crons ALTER COLUMN id SET DEFAULT nextval('crons_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY emails ALTER COLUMN id SET DEFAULT nextval('emails_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY invoices ALTER COLUMN id SET DEFAULT nextval('invoices_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY log_parts ALTER COLUMN id SET DEFAULT nextval('log_parts_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY memberships ALTER COLUMN id SET DEFAULT nextval('memberships_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY messages ALTER COLUMN id SET DEFAULT nextval('messages_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY organizations ALTER COLUMN id SET DEFAULT nextval('organizations_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY owner_groups ALTER COLUMN id SET DEFAULT nextval('owner_groups_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY permissions ALTER COLUMN id SET DEFAULT nextval('permissions_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY pull_requests ALTER COLUMN id SET DEFAULT nextval('pull_requests_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY queueable_jobs ALTER COLUMN id SET DEFAULT nextval('queueable_jobs_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY repositories ALTER COLUMN id SET DEFAULT nextval('repositories_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY requests ALTER COLUMN id SET DEFAULT nextval('requests_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY ssl_keys ALTER COLUMN id SET DEFAULT nextval('ssl_keys_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY stages ALTER COLUMN id SET DEFAULT nextval('stages_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY stars ALTER COLUMN id SET DEFAULT nextval('stars_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY stripe_events ALTER COLUMN id SET DEFAULT nextval('stripe_events_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY subscriptions ALTER COLUMN id SET DEFAULT nextval('subscriptions_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY tags ALTER COLUMN id SET DEFAULT nextval('tags_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY tokens ALTER COLUMN id SET DEFAULT nextval('tokens_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY trial_allowances ALTER COLUMN id SET DEFAULT nextval('trial_allowances_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY trials ALTER COLUMN id SET DEFAULT nextval('trials_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY urls ALTER COLUMN id SET DEFAULT nextval('urls_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY user_beta_features ALTER COLUMN id SET DEFAULT nextval('user_beta_features_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY users ALTER COLUMN id SET DEFAULT nextval('users_id_seq'::regclass); + +-- +-- Name: abuses_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('abuses_id_seq', 1, false); + +-- +-- Name: annotation_providers_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('annotation_providers_id_seq', 1, false); + +-- +-- Name: annotations_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('annotations_id_seq', 1, false); + +-- +-- Name: beta_features_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('beta_features_id_seq', 1, false); + +-- +-- Name: branches_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('branches_id_seq', 72, true); + +-- +-- Name: broadcasts_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('broadcasts_id_seq', 1, false); + +-- +-- Name: builds_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('builds_id_seq', 1, false); + +-- +-- Name: commits_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('commits_id_seq', 210, true); + +-- +-- Name: coupons_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('coupons_id_seq', 1, false); + +-- +-- Name: crons_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('crons_id_seq', 1, false); + +-- +-- Name: emails_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('emails_id_seq', 8, true); + +-- +-- Name: invoices_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('invoices_id_seq', 1, false); + +-- +-- Name: jobs_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('jobs_id_seq', 1, true); + +-- +-- Name: log_parts_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('log_parts_id_seq', 7609, true); + +-- +-- Name: ssl_keys_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('ssl_keys_id_seq', 30, true); + + +-- +-- Name: stages_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('stages_id_seq', 19, true); + +-- +-- Name: stars_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('stars_id_seq', 1, false); + +-- +-- Name: stripe_events_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('stripe_events_id_seq', 1, false); + +-- +-- Name: subscriptions_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('subscriptions_id_seq', 1, false); + +-- +-- Name: tags_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('tags_id_seq', 1, false); + +-- +-- Name: tokens_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('tokens_id_seq', 8, true); + +-- +-- Name: trial_allowances_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('trial_allowances_id_seq', 1, false); + +-- +-- Name: trials_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('trials_id_seq', 1, false); + +-- +-- Name: urls_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('urls_id_seq', 1, false); + +-- +-- Name: user_beta_features_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('user_beta_features_id_seq', 1, false); + +-- +-- Name: users_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('users_id_seq', 8, true); + + +SET search_path = sqitch, pg_catalog; + + +SET search_path = public, pg_catalog; + +-- +-- Name: abuses_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY abuses + ADD CONSTRAINT abuses_pkey PRIMARY KEY (id); + + +-- +-- Name: annotation_providers_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY annotation_providers + ADD CONSTRAINT annotation_providers_pkey PRIMARY KEY (id); + + +-- +-- Name: annotations_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY annotations + ADD CONSTRAINT annotations_pkey PRIMARY KEY (id); + + +-- +-- Name: beta_features_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY beta_features + ADD CONSTRAINT beta_features_pkey PRIMARY KEY (id); + + +-- +-- Name: branches_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY branches + ADD CONSTRAINT branches_pkey PRIMARY KEY (id); + + +-- +-- Name: broadcasts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY broadcasts + ADD CONSTRAINT broadcasts_pkey PRIMARY KEY (id); + + +-- +-- Name: builds_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY builds + ADD CONSTRAINT builds_pkey PRIMARY KEY (id); + + +-- +-- Name: commits_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY commits + ADD CONSTRAINT commits_pkey PRIMARY KEY (id); + + +-- +-- Name: coupons_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY coupons + ADD CONSTRAINT coupons_pkey PRIMARY KEY (id); + + +-- +-- Name: crons_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY crons + ADD CONSTRAINT crons_pkey PRIMARY KEY (id); + + +-- +-- Name: emails_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY emails + ADD CONSTRAINT emails_pkey PRIMARY KEY (id); + + +-- +-- Name: invoices_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY invoices + ADD CONSTRAINT invoices_pkey PRIMARY KEY (id); + + +-- +-- Name: jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY jobs + ADD CONSTRAINT jobs_pkey PRIMARY KEY (id); + + +-- +-- Name: log_parts_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY log_parts + ADD CONSTRAINT log_parts_pkey PRIMARY KEY (id); + + +-- +-- Name: logs_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT logs_pkey PRIMARY KEY (id); + + +-- +-- Name: memberships_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY memberships + ADD CONSTRAINT memberships_pkey PRIMARY KEY (id); + + +-- +-- Name: messages_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY messages + ADD CONSTRAINT messages_pkey PRIMARY KEY (id); + + +-- +-- Name: organizations_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY organizations + ADD CONSTRAINT organizations_pkey PRIMARY KEY (id); + + +-- +-- Name: owner_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY owner_groups + ADD CONSTRAINT owner_groups_pkey PRIMARY KEY (id); + + +-- +-- Name: permissions_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY permissions + ADD CONSTRAINT permissions_pkey PRIMARY KEY (id); + + +-- +-- Name: pull_requests_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY pull_requests + ADD CONSTRAINT pull_requests_pkey PRIMARY KEY (id); + + +-- +-- Name: queueable_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY queueable_jobs + ADD CONSTRAINT queueable_jobs_pkey PRIMARY KEY (id); + + +-- +-- Name: repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY repositories + ADD CONSTRAINT repositories_pkey PRIMARY KEY (id); + + +-- +-- Name: requests_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY requests + ADD CONSTRAINT requests_pkey PRIMARY KEY (id); + + +-- +-- Name: ssl_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY ssl_keys + ADD CONSTRAINT ssl_keys_pkey PRIMARY KEY (id); + + +-- +-- Name: stages_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY stages + ADD CONSTRAINT stages_pkey PRIMARY KEY (id); + + +-- +-- Name: stars_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY stars + ADD CONSTRAINT stars_pkey PRIMARY KEY (id); + + +-- +-- Name: stripe_events_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY stripe_events + ADD CONSTRAINT stripe_events_pkey PRIMARY KEY (id); + + +-- +-- Name: subscriptions_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY subscriptions + ADD CONSTRAINT subscriptions_pkey PRIMARY KEY (id); + + +-- +-- Name: tags_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (id); + + +-- +-- Name: tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY tokens + ADD CONSTRAINT tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: trial_allowances_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY trial_allowances + ADD CONSTRAINT trial_allowances_pkey PRIMARY KEY (id); + + +-- +-- Name: trials_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY trials + ADD CONSTRAINT trials_pkey PRIMARY KEY (id); + + +-- +-- Name: urls_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY urls + ADD CONSTRAINT urls_pkey PRIMARY KEY (id); + + +-- +-- Name: user_beta_features_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY user_beta_features + ADD CONSTRAINT user_beta_features_pkey PRIMARY KEY (id); + + +-- +-- Name: users_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +SET search_path = sqitch, pg_catalog; + +-- +-- Name: changes_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY changes + ADD CONSTRAINT changes_pkey PRIMARY KEY (change_id); + + +-- +-- Name: changes_project_script_hash_key; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY changes + ADD CONSTRAINT changes_project_script_hash_key UNIQUE (project, script_hash); + + +-- +-- Name: dependencies_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY dependencies + ADD CONSTRAINT dependencies_pkey PRIMARY KEY (change_id, dependency); + + +-- +-- Name: events_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY events + ADD CONSTRAINT events_pkey PRIMARY KEY (change_id, committed_at); + + +-- +-- Name: projects_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY projects + ADD CONSTRAINT projects_pkey PRIMARY KEY (project); + + +-- +-- Name: projects_uri_key; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY projects + ADD CONSTRAINT projects_uri_key UNIQUE (uri); + + +-- +-- Name: releases_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY releases + ADD CONSTRAINT releases_pkey PRIMARY KEY (version); + + +-- +-- Name: tags_pkey; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY tags + ADD CONSTRAINT tags_pkey PRIMARY KEY (tag_id); + + +-- +-- Name: tags_project_tag_key; Type: CONSTRAINT; Schema: sqitch; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY tags + ADD CONSTRAINT tags_project_tag_key UNIQUE (project, tag); + + +SET search_path = public, pg_catalog; + +-- +-- Name: index_abuses_on_owner; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_abuses_on_owner ON abuses USING btree (owner_id); + + +-- +-- Name: index_abuses_on_owner_id_and_owner_type_and_level; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_abuses_on_owner_id_and_owner_type_and_level ON abuses USING btree (owner_id, owner_type, level); + + +-- +-- Name: index_annotations_on_job_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_annotations_on_job_id ON annotations USING btree (job_id); + + +-- +-- Name: index_branches_on_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_branches_on_repository_id ON branches USING btree (repository_id); + + +-- +-- Name: index_branches_on_repository_id_and_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_branches_on_repository_id_and_name ON branches USING btree (repository_id, name); + + +-- +-- Name: index_broadcasts_on_recipient_id_and_recipient_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_broadcasts_on_recipient_id_and_recipient_type ON broadcasts USING btree (recipient_id, recipient_type); + + +-- +-- Name: index_builds_on_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id ON builds USING btree (repository_id); + + +-- +-- Name: index_builds_on_repository_id_and_branch_and_event_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id_and_branch_and_event_type ON builds USING btree (repository_id, branch, event_type) WHERE ((state)::text = ANY ((ARRAY['created'::character varying, 'queued'::character varying, 'received'::character varying])::text[])); + + +-- +-- Name: index_builds_on_repository_id_and_branch_and_event_type_and_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id_and_branch_and_event_type_and_id ON builds USING btree (repository_id, branch, event_type, id); + + +-- +-- Name: index_builds_on_repository_id_and_branch_and_id_desc; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id_and_branch_and_id_desc ON builds USING btree (repository_id, branch, id DESC); + + +-- +-- Name: index_builds_on_repository_id_and_number; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id_and_number ON builds USING btree (repository_id, ((number)::integer)); + + +-- +-- Name: index_builds_on_repository_id_and_number_and_event_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_repository_id_and_number_and_event_type ON builds USING btree (repository_id, number, event_type); + + +-- +-- Name: index_builds_on_request_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_request_id ON builds USING btree (request_id); + + +-- +-- Name: index_builds_on_sender_type_and_sender_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_sender_type_and_sender_id ON builds USING btree (sender_type, sender_id); + + +-- +-- Name: index_builds_on_state; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_builds_on_state ON builds USING btree (state); + + +-- +-- Name: index_emails_on_email; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_emails_on_email ON emails USING btree (email); + + +-- +-- Name: index_emails_on_user_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_emails_on_user_id ON emails USING btree (user_id); + + +-- +-- Name: index_invoices_on_stripe_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_invoices_on_stripe_id ON invoices USING btree (stripe_id); + + +-- +-- Name: index_jobs_on_created_at; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_created_at ON jobs USING btree (created_at); + + +-- +-- Name: index_jobs_on_owner_id_and_owner_type_and_state; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_owner_id_and_owner_type_and_state ON jobs USING btree (owner_id, owner_type, state); + + +-- +-- Name: index_jobs_on_source_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_source_id ON jobs USING btree (source_id); + + +-- +-- Name: index_jobs_on_stage_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_stage_id ON jobs USING btree (stage_id); + + +-- +-- Name: index_jobs_on_state; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_state ON jobs USING btree (state); + + +-- +-- Name: index_jobs_on_type_and_source_id_and_source_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_type_and_source_id_and_source_type ON jobs USING btree (type, source_id, source_type); + + +-- +-- Name: index_jobs_on_updated_at; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_jobs_on_updated_at ON jobs USING btree (updated_at); + + +-- +-- Name: index_log_parts_on_created_at; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_log_parts_on_created_at ON log_parts USING btree (created_at); + + +-- +-- Name: index_log_parts_on_log_id_and_number; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_log_parts_on_log_id_and_number ON log_parts USING btree (log_id, number); + + +-- +-- Name: index_logs_on_archive_verified; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_logs_on_archive_verified ON logs USING btree (archive_verified); + + +-- +-- Name: index_logs_on_archived_at; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_logs_on_archived_at ON logs USING btree (archived_at); + + +-- +-- Name: index_logs_on_job_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_logs_on_job_id ON logs USING btree (job_id); + + +-- +-- Name: index_memberships_on_user_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_memberships_on_user_id ON memberships USING btree (user_id); + + +-- +-- Name: index_messages_on_subject_type_and_subject_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_messages_on_subject_type_and_subject_id ON messages USING btree (subject_type, subject_id); + + +-- +-- Name: index_organizations_on_github_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_organizations_on_github_id ON organizations USING btree (github_id); + + +-- +-- Name: index_organizations_on_login; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_organizations_on_login ON organizations USING btree (login); + + +-- +-- Name: index_organizations_on_lower_login; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_organizations_on_lower_login ON organizations USING btree (lower((login)::text)); + + +-- +-- Name: index_owner_groups_on_owner_type_and_owner_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_owner_groups_on_owner_type_and_owner_id ON owner_groups USING btree (owner_type, owner_id); + + +-- +-- Name: index_owner_groups_on_uuid; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_owner_groups_on_uuid ON owner_groups USING btree (uuid); + + +-- +-- Name: index_permissions_on_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_permissions_on_repository_id ON permissions USING btree (repository_id); + + +-- +-- Name: index_permissions_on_user_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_permissions_on_user_id ON permissions USING btree (user_id); + + +-- +-- Name: index_permissions_on_user_id_and_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_permissions_on_user_id_and_repository_id ON permissions USING btree (user_id, repository_id); + + +-- +-- Name: index_pull_requests_on_repository_id_and_number; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_pull_requests_on_repository_id_and_number ON pull_requests USING btree (repository_id, number); + + +-- +-- Name: index_queueable_jobs_on_job_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_queueable_jobs_on_job_id ON queueable_jobs USING btree (job_id); + + +-- +-- Name: index_repositories_on_active; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_active ON repositories USING btree (active); + + +-- +-- Name: index_repositories_on_github_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_repositories_on_github_id ON repositories USING btree (github_id); + + +-- +-- Name: index_repositories_on_lower_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_lower_name ON repositories USING btree (lower((name)::text)); + + +-- +-- Name: index_repositories_on_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_name ON repositories USING btree (name); + + +-- +-- Name: index_repositories_on_owner_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_owner_id ON repositories USING btree (owner_id); + + +-- +-- Name: index_repositories_on_owner_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_owner_name ON repositories USING btree (owner_name); + + +-- +-- Name: index_repositories_on_slug; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_repositories_on_slug ON repositories USING gin (((((owner_name)::text || '/'::text) || (name)::text)) gin_trgm_ops); + + +-- +-- Name: index_requests_on_commit_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_requests_on_commit_id ON requests USING btree (commit_id); + + +-- +-- Name: index_requests_on_created_at; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_requests_on_created_at ON requests USING btree (created_at); + + +-- +-- Name: index_requests_on_head_commit; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_requests_on_head_commit ON requests USING btree (head_commit); + + +-- +-- Name: index_requests_on_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_requests_on_repository_id ON requests USING btree (repository_id); + + +-- +-- Name: index_requests_on_repository_id_and_id_desc; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_requests_on_repository_id_and_id_desc ON requests USING btree (repository_id, id DESC); + + +-- +-- Name: index_ssl_key_on_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_ssl_key_on_repository_id ON ssl_keys USING btree (repository_id); + + +-- +-- Name: index_stages_on_build_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_stages_on_build_id ON stages USING btree (build_id); + + +-- +-- Name: index_stars_on_user_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_stars_on_user_id ON stars USING btree (user_id); + + +-- +-- Name: index_stars_on_user_id_and_repository_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_stars_on_user_id_and_repository_id ON stars USING btree (user_id, repository_id); + + +-- +-- Name: index_stripe_events_on_date; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_stripe_events_on_date ON stripe_events USING btree (date); + + +-- +-- Name: index_stripe_events_on_event_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_stripe_events_on_event_id ON stripe_events USING btree (event_id); + + +-- +-- Name: index_stripe_events_on_event_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_stripe_events_on_event_type ON stripe_events USING btree (event_type); + + +-- +-- Name: index_tags_on_repository_id_and_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_tags_on_repository_id_and_name ON tags USING btree (repository_id, name); + + +-- +-- Name: index_tokens_on_token; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_tokens_on_token ON tokens USING btree (token); + + +-- +-- Name: index_tokens_on_user_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_tokens_on_user_id ON tokens USING btree (user_id); + + +-- +-- Name: index_trial_allowances_on_creator_id_and_creator_type; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_trial_allowances_on_creator_id_and_creator_type ON trial_allowances USING btree (creator_id, creator_type); + + +-- +-- Name: index_trial_allowances_on_trial_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_trial_allowances_on_trial_id ON trial_allowances USING btree (trial_id); + + +-- +-- Name: index_trials_on_owner; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_trials_on_owner ON trials USING btree (owner_id, owner_type); + + +-- +-- Name: index_user_beta_features_on_user_id_and_beta_feature_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_user_beta_features_on_user_id_and_beta_feature_id ON user_beta_features USING btree (user_id, beta_feature_id); + + +-- +-- Name: index_users_on_github_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_users_on_github_id ON users USING btree (github_id); + + +-- +-- Name: index_users_on_github_oauth_token; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX index_users_on_github_oauth_token ON users USING btree (github_oauth_token); + + +-- +-- Name: index_users_on_login; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_users_on_login ON users USING btree (login); + + +-- +-- Name: index_users_on_lower_login; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX index_users_on_lower_login ON users USING btree (lower((login)::text)); + + +-- +-- Name: subscriptions_owner; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX subscriptions_owner ON subscriptions USING btree (owner_id, owner_type) WHERE ((status)::text = 'subscribed'::text); + + +-- +-- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX unique_schema_migrations ON schema_migrations USING btree (version); + + +-- +-- Name: set_updated_at_on_builds; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER set_updated_at_on_builds BEFORE INSERT OR UPDATE ON builds FOR EACH ROW EXECUTE PROCEDURE set_updated_at(); + + +-- +-- Name: set_updated_at_on_jobs; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER set_updated_at_on_jobs BEFORE INSERT OR UPDATE ON jobs FOR EACH ROW EXECUTE PROCEDURE set_updated_at(); + + +-- +-- Name: fk_repositories_current_build_id; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY repositories + ADD CONSTRAINT fk_repositories_current_build_id FOREIGN KEY (current_build_id) REFERENCES builds(id); + + +SET search_path = sqitch, pg_catalog; + +-- +-- Name: changes_project_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY changes + ADD CONSTRAINT changes_project_fkey FOREIGN KEY (project) REFERENCES projects(project) ON UPDATE CASCADE; + + +-- +-- Name: dependencies_change_id_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY dependencies + ADD CONSTRAINT dependencies_change_id_fkey FOREIGN KEY (change_id) REFERENCES changes(change_id) ON UPDATE CASCADE ON DELETE CASCADE; + + +-- +-- Name: dependencies_dependency_id_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY dependencies + ADD CONSTRAINT dependencies_dependency_id_fkey FOREIGN KEY (dependency_id) REFERENCES changes(change_id) ON UPDATE CASCADE; + + +-- +-- Name: events_project_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY events + ADD CONSTRAINT events_project_fkey FOREIGN KEY (project) REFERENCES projects(project) ON UPDATE CASCADE; + + +-- +-- Name: tags_change_id_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY tags + ADD CONSTRAINT tags_change_id_fkey FOREIGN KEY (change_id) REFERENCES changes(change_id) ON UPDATE CASCADE; + + +-- +-- Name: tags_project_fkey; Type: FK CONSTRAINT; Schema: sqitch; Owner: postgres +-- + +ALTER TABLE ONLY tags + ADD CONSTRAINT tags_project_fkey FOREIGN KEY (project) REFERENCES projects(project) ON UPDATE CASCADE; + + +-- +-- Name: public; Type: ACL; Schema: -; Owner: postgres +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 0d035cd..f1731b6 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -8,6 +8,10 @@ require 'pry' describe Backup do + before(:all) do + system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") + end + let!(:config) { Config.new } let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location) } From 9e4d44d73e1af89db687acfe47ad2e8af69d4b18 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 18 Aug 2021 13:16:33 +0200 Subject: [PATCH 18/84] private repos, builds and jobs in spec removed --- spec/backup_spec.rb | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index f1731b6..ea63e1e 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -25,12 +25,6 @@ :repository ) } - let(:private_repository) { - FactoryBot.create( - :repository, - private: true - ) - } let(:build) { FactoryBot.create( :build, @@ -39,14 +33,6 @@ repository: repository ) } - let(:private_build) { - FactoryBot.create( - :build, - created_at: datetime, - updated_at: datetime, - repository: private_repository - ) - } let(:job) { FactoryBot.create( :job, @@ -57,16 +43,6 @@ repository: repository ) } - let(:private_job) { - FactoryBot.create( - :job, - created_at: datetime, - updated_at: datetime, - source_id: private_build.id, - source_type: 'Build', - repository: private_repository - ) - } describe 'process_repo' do let(:exported_object) { @@ -133,8 +109,6 @@ before do build.jobs = [job] repository.builds = [build] - private_build.jobs = [private_job] - private_repository.builds = [private_build] end shared_context 'removing builds and jobs' do From 99b806b4a1c851197d5ecce4f7d2df2b9d1bea2d Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 18 Aug 2021 15:31:47 +0200 Subject: [PATCH 19/84] joining builds and jobs with repositories using FactoryBot --- spec/backup_spec.rb | 39 +++++++++------------------------------ spec/support/factories.rb | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index ea63e1e..6cf41ed 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -20,33 +20,17 @@ let(:com_id) { rand(100000) } let(:private_org_id) { rand(100000) } let(:private_com_id) { rand(100000) } - let(:repository) { + let!(:repository) { FactoryBot.create( - :repository - ) - } - let(:build) { - FactoryBot.create( - :build, + :repository_with_builds, created_at: datetime, - updated_at: datetime, - repository: repository - ) - } - let(:job) { - FactoryBot.create( - :job, - created_at: datetime, - updated_at: datetime, - source_id: build.id, - source_type: 'Build', - repository: repository + updated_at: datetime ) } describe 'process_repo' do - let(:exported_object) { - [[{"id"=>build.id, + let!(:exported_object) { + [[{"id"=>repository.builds.first.id, "repository_id"=>repository.id, "number"=>nil, "started_at"=>nil, @@ -75,10 +59,10 @@ "sender_id"=>nil, "sender_type"=>nil, :jobs=> - [{"id"=>job.id, + [{"id"=>repository.builds.first.jobs.first.id, "repository_id"=>repository.id, "commit_id"=>nil, - "source_id"=>build.id, + "source_id"=>repository.builds.first.id, "source_type"=>"Build", "queue"=>nil, "type"=>nil, @@ -106,11 +90,6 @@ }]] } - before do - build.jobs = [job] - repository.builds = [build] - end - shared_context 'removing builds and jobs' do it 'should delete build' do expect do @@ -128,8 +107,8 @@ context 'when if_backup config is set to true' do it 'should prepare proper JSON export' do build_export = backup.process_repo(repository) - build_export.first.first[:updated_at] = datetime - build_export.first.first[:jobs].first[:updated_at] = datetime + build_export.first.first['updated_at'] = datetime + build_export.first.first[:jobs].first['updated_at'] = datetime expect(build_export.to_json).to eq(exported_object.to_json) end diff --git a/spec/support/factories.rb b/spec/support/factories.rb index fa43f59..86a6c5f 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -3,7 +3,40 @@ require 'factory_bot' FactoryBot.define do - factory :repository, class: Repository - factory :build, class: Build - factory :job, class: Job + factory :repository do + factory :repository_with_builds do + transient do + builds_count { 1 } + end + after(:create) do |repository, evaluator| + create_list( + :build_with_jobs, + evaluator.builds_count, + repository: repository, + created_at: repository.created_at, + updated_at: repository.updated_at + ) + end + end + end + + factory :build do + factory :build_with_jobs do + transient do + jobs_count { 1 } + end + after(:create) do |build, evaluator| + create_list( + :job, + evaluator.jobs_count, + repository: build.repository, + source_type: 'Build', + source_id: build.id, + created_at: build.created_at, + updated_at: build.updated_at + ) + end + end + end + factory :job end From 9b77b63aa60f840f1bf9dc0dc88cb5951c750479 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 19 Aug 2021 09:31:01 +0200 Subject: [PATCH 20/84] tests improved --- lib/backup.rb | 2 +- spec/backup_spec.rb | 222 ++++++++++++++++++++++++++++++-------- spec/support/factories.rb | 4 +- 3 files changed, 179 insertions(+), 49 deletions(-) diff --git a/lib/backup.rb b/lib/backup.rb index 615f4ed..46f435e 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -40,7 +40,7 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe if builds_batch.count == @config.limit.to_i @config.if_backup ? save_batch(builds_batch, repository) : builds_batch.each(&:destroy) end - end + end.compact end private diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 6cf41ed..7d6c768 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -12,9 +12,15 @@ system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") end + after(:each) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + end + let!(:config) { Config.new } let(:files_location) { "dump/tests" } - let!(:backup) { Backup.new(files_location: files_location) } + let!(:backup) { Backup.new(files_location: files_location, limit: 2) } let(:datetime) { (config.delay + 1).months.ago.to_time.utc } let(:org_id) { rand(100000) } let(:com_id) { rand(100000) } @@ -30,36 +36,38 @@ describe 'process_repo' do let!(:exported_object) { - [[{"id"=>repository.builds.first.id, - "repository_id"=>repository.id, - "number"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "config"=>nil, - "commit_id"=>nil, - "request_id"=>nil, - "state"=>nil, - "duration"=>nil, - "owner_id"=>nil, - "owner_type"=>nil, - "event_type"=>nil, - "previous_state"=>nil, - "pull_request_title"=>nil, - "pull_request_number"=>nil, - "branch"=>nil, - "canceled_at"=>nil, - "cached_matrix_ids"=>nil, - "received_at"=>nil, - "private"=>nil, - "pull_request_id"=>nil, - "branch_id"=>nil, - "tag_id"=>nil, - "sender_id"=>nil, - "sender_type"=>nil, - :jobs=> - [{"id"=>repository.builds.first.jobs.first.id, + [[ + { + "id"=>repository.builds.first.id, + "repository_id"=>repository.id, + "number"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "config"=>nil, + "commit_id"=>nil, + "request_id"=>nil, + "state"=>nil, + "duration"=>nil, + "owner_id"=>nil, + "owner_type"=>nil, + "event_type"=>nil, + "previous_state"=>nil, + "pull_request_title"=>nil, + "pull_request_number"=>nil, + "branch"=>nil, + "canceled_at"=>nil, + "cached_matrix_ids"=>nil, + "received_at"=>nil, + "private"=>nil, + "pull_request_id"=>nil, + "branch_id"=>nil, + "tag_id"=>nil, + "sender_id"=>nil, + "sender_type"=>nil, + :jobs=>[{ + "id"=>repository.builds.first.jobs.first.id, "repository_id"=>repository.id, "commit_id"=>nil, "source_id"=>repository.builds.first.id, @@ -86,21 +94,138 @@ "private"=>nil, "stage_number"=>nil, "stage_id"=>nil + }, + { + "id"=>repository.builds.first.jobs.second.id, + "repository_id"=>repository.id, + "commit_id"=>nil, + "source_id"=>repository.builds.first.id, + "source_type"=>"Build", + "queue"=>nil, + "type"=>nil, + "state"=>nil, + "number"=>nil, + "config"=>nil, + "worker"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "tags"=>nil, + "allow_failure"=>false, + "owner_id"=>nil, + "owner_type"=>nil, + "result"=>nil, + "queued_at"=>nil, + "canceled_at"=>nil, + "received_at"=>nil, + "debug_options"=>nil, + "private"=>nil, + "stage_number"=>nil, + "stage_id"=>nil + }] + }, + { + "id"=>repository.builds.second.id, + "repository_id"=>repository.id, + "number"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "config"=>nil, + "commit_id"=>nil, + "request_id"=>nil, + "state"=>nil, + "duration"=>nil, + "owner_id"=>nil, + "owner_type"=>nil, + "event_type"=>nil, + "previous_state"=>nil, + "pull_request_title"=>nil, + "pull_request_number"=>nil, + "branch"=>nil, + "canceled_at"=>nil, + "cached_matrix_ids"=>nil, + "received_at"=>nil, + "private"=>nil, + "pull_request_id"=>nil, + "branch_id"=>nil, + "tag_id"=>nil, + "sender_id"=>nil, + "sender_type"=>nil, + :jobs=>[{ + "id"=>repository.builds.second.jobs.first.id, + "repository_id"=>repository.id, + "commit_id"=>nil, + "source_id"=>repository.builds.second.id, + "source_type"=>"Build", + "queue"=>nil, + "type"=>nil, + "state"=>nil, + "number"=>nil, + "config"=>nil, + "worker"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "tags"=>nil, + "allow_failure"=>false, + "owner_id"=>nil, + "owner_type"=>nil, + "result"=>nil, + "queued_at"=>nil, + "canceled_at"=>nil, + "received_at"=>nil, + "debug_options"=>nil, + "private"=>nil, + "stage_number"=>nil, + "stage_id"=>nil + }, + { + "id"=>repository.builds.second.jobs.second.id, + "repository_id"=>repository.id, + "commit_id"=>nil, + "source_id"=>repository.builds.second.id, + "source_type"=>"Build", + "queue"=>nil, + "type"=>nil, + "state"=>nil, + "number"=>nil, + "config"=>nil, + "worker"=>nil, + "started_at"=>nil, + "finished_at"=>nil, + "created_at"=>datetime, + "updated_at"=>datetime, + "tags"=>nil, + "allow_failure"=>false, + "owner_id"=>nil, + "owner_type"=>nil, + "result"=>nil, + "queued_at"=>nil, + "canceled_at"=>nil, + "received_at"=>nil, + "debug_options"=>nil, + "private"=>nil, + "stage_number"=>nil, + "stage_id"=>nil }] - }]] + } + ]] } - shared_context 'removing builds and jobs' do - it 'should delete build' do - expect do - backup.process_repo(repository) - end.to change { Build.count }.by(-1) + shared_context 'removing builds and jobs in batches' do + it 'should delete builds fitting to batches' do + backup.process_repo(repository) + expect(Build.all.size).to eq(1) end - it 'should delete job' do - expect do - backup.process_repo(repository) - end.to change { Job.count }.by(-1) + it 'should delete all jobs of removed builds and leave the rest' do + backup.process_repo(repository) + build_id = Build.first.id + expect(Job.all.map(&:source_id)).to eq([build_id, build_id]) end end @@ -108,7 +233,12 @@ it 'should prepare proper JSON export' do build_export = backup.process_repo(repository) build_export.first.first['updated_at'] = datetime + build_export.first.second['updated_at'] = datetime build_export.first.first[:jobs].first['updated_at'] = datetime + build_export.first.first[:jobs].second['updated_at'] = datetime + build_export.first.second[:jobs].first['updated_at'] = datetime + build_export.first.second[:jobs].second['updated_at'] = datetime + expect(build_export.to_json).to eq(exported_object.to_json) end @@ -117,28 +247,28 @@ backup.process_repo(repository) end + it_behaves_like 'removing builds and jobs in batches' + context 'when path with nonexistent folders is given' do let(:random_files_location) { "dump/tests/#{rand(100000)}" } - let!(:backup) { Backup.new(files_location: random_files_location) } + let!(:backup) { Backup.new(files_location: random_files_location, limit: 2) } it 'should create needed folders' do expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original backup.process_repo(repository) end end - - it_behaves_like 'removing builds and jobs' end context 'when if_backup config is set to false' do - let!(:backup) { Backup.new(if_backup: false) } + let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } it 'should not save JSON to file' do expect(File).not_to receive(:open) backup.process_repo(repository) end - it_behaves_like 'removing builds and jobs' + it_behaves_like 'removing builds and jobs in batches' end end end diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 86a6c5f..15860e1 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -6,7 +6,7 @@ factory :repository do factory :repository_with_builds do transient do - builds_count { 1 } + builds_count { 3 } end after(:create) do |repository, evaluator| create_list( @@ -23,7 +23,7 @@ factory :build do factory :build_with_jobs do transient do - jobs_count { 1 } + jobs_count { 2 } end after(:create) do |build, evaluator| create_list( From 940ebaf2dff2de70a0d3ed8e716d7eae1f325d08 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 19 Aug 2021 09:39:31 +0200 Subject: [PATCH 21/84] README improved --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 24ad840..78648aa 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,32 @@ # README *travis-backup* is a cron application, which export builds and it's correspoonding jobs -to json files. +to json files and sends them to GCE. -* Ruby version +#### Ruby version -2.7.2 +2.7.0 -* System dependencies - -* Configuration +#### Configuration `config/settinigs.yml` or env vars like: -`IF_BACKUP` `BACKUP_LIMIT` `BACKUP_DELAY` -`BACKUP_FILES_LOCATION` +`BACKUP_HOUSEKEEPING_PERIOD` +`LOGS_URL` `DATABASE_URL` +`GCE_PROJECT` +`GCE_CREDENTIALS` +`GCE_BUCKET` +`REDIS_URL` -* How to run the test suite +#### How to run the test suite `bundle exec rspec` -* How to run appication +To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`. + +#### How to run appication `bundle exec bin/run_backup` From 81ff4b65acda79d29d686b3feed2133a1105a3bf Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 19 Aug 2021 10:15:23 +0200 Subject: [PATCH 22/84] Travis fixes --- .travis.yml | 27 ++++++++++++++++++++------- Gemfile | 2 +- README.md | 2 +- db/schema.sql | 4 ++-- lib/config.rb | 2 +- spec/backup_spec.rb | 2 +- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5b14c3..11831e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,9 @@ rvm: 2.7.2 cache: bundler -import: - - travis-ci/build-configs:db-setup.yml - services: - redis -before_install: - - gem install bundler - env: global: - PATH=/snap/bin:$PATH @@ -17,4 +11,23 @@ env: jobs: include: - stage: "testing time" - script: bundle exec rspec -e test \ No newline at end of file + script: bundle exec rspec + +dist: xenial + +before_install: + - gem install bundler + - sudo apt-get install -yq --no-install-suggests --no-install-recommends postgresql-common + - sudo service postgresql stop + - sudo apt install -yq --no-install-suggests --no-install-recommends postgresql-11 postgresql-client-11 + - sed -e 's/^port.*/port = 5432/' /etc/postgresql/11/main/postgresql.conf > postgresql.conf + - sudo chown postgres postgresql.conf + - sudo mv postgresql.conf /etc/postgresql/11/main + - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf + - sudo service postgresql start 11 + +before_script: + - psql --version + - psql -c 'CREATE DATABASE travis_test;' -U postgres + - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres + - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test \ No newline at end of file diff --git a/Gemfile b/Gemfile index be0e6b0..f0c28a3 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.0' +ruby '2.7.2' gem 'activerecord' gem 'pg' diff --git a/README.md b/README.md index 78648aa..4744a26 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ to json files and sends them to GCE. #### Ruby version -2.7.0 +2.7.2 #### Configuration diff --git a/db/schema.sql b/db/schema.sql index 74b07fd..aa9ca08 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,5 +1,5 @@ -DROP SCHEMA public CASCADE; -DROP SCHEMA sqitch CASCADE; +DROP SCHEMA IF EXISTS public CASCADE; +DROP SCHEMA IF EXISTS sqitch CASCADE; CREATE SCHEMA public; SET statement_timeout = 0; diff --git a/lib/config.rb b/lib/config.rb index fe4cd00..4bc190f 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -19,6 +19,6 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config['backup']['limit'] @delay = args[:delay] || ENV['BACKUP_DELAY'] || config['backup']['delay'] @files_location = args[:files_location] || ENV['BACKUP_FILES_LOCATION'] || config['backup']['files_location'] - @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details['development'] + @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details[ENV['RAILS_ENV']] end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 7d6c768..0ebe865 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -9,7 +9,7 @@ describe Backup do before(:all) do - system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") + system("psql '#{Config.new.database_url}' -f db/schema.sql") end after(:each) do From c395c287ab1e89f5181f46f593d3804ebca9ac7b Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 19 Aug 2021 18:48:54 +0200 Subject: [PATCH 23/84] export for given user, organization or repo id --- lib/backup.rb | 17 +++++- lib/models/organization.rb | 7 +++ lib/models/user.rb | 7 +++ spec/backup_spec.rb | 105 ++++++++++++++++++++++++++++++------- spec/support/factories.rb | 32 +++++++++++ 5 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 lib/models/organization.rb create mode 100644 lib/models/user.rb diff --git a/lib/backup.rb b/lib/backup.rb index 46f435e..39d0e27 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -20,11 +20,24 @@ def connect_db ActiveRecord::Base.establish_connection(@config.database_url) end - def export(owner_id = nil) + def export(args={}) + if args[:user_id] + owner_id = args[:user_id] + owner_type = 'User' + elsif args[:org_id] + owner_id = args[:org_id] + owner_type = 'Organization' + elsif args[:repo_id] + repo_id = args[:repo_id] + end + if owner_id - Repository.where('owner_id = ?', owner_id).order(:id).each do |repository| + Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| process_repo(repository) end + elsif repo_id + repository = Repository.find(repo_id) + process_repo(repository) else Repository.order(:id).each do |repository| process_repo(repository) diff --git a/lib/models/organization.rb b/lib/models/organization.rb new file mode 100644 index 0000000..edbd92c --- /dev/null +++ b/lib/models/organization.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Organization < Model + self.table_name = 'organizations' +end diff --git a/lib/models/user.rb b/lib/models/user.rb new file mode 100644 index 0000000..380619f --- /dev/null +++ b/lib/models/user.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class User < Model + self.table_name = 'users' +end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 0ebe865..a1f2789 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -4,37 +4,102 @@ require 'models/repository' require 'models/build' require 'models/job' +require 'models/organization' +require 'models/user' require 'support/factories' require 'pry' describe Backup do before(:all) do - system("psql '#{Config.new.database_url}' -f db/schema.sql") + system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") end - after(:each) do - Repository.destroy_all - Build.destroy_all - Job.destroy_all - end - - let!(:config) { Config.new } let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location, limit: 2) } - let(:datetime) { (config.delay + 1).months.ago.to_time.utc } - let(:org_id) { rand(100000) } - let(:com_id) { rand(100000) } - let(:private_org_id) { rand(100000) } - let(:private_com_id) { rand(100000) } - let!(:repository) { - FactoryBot.create( - :repository_with_builds, - created_at: datetime, - updated_at: datetime - ) - } + + describe 'export' do + let!(:unassigned_repositories) { + (1..3).to_a.map do + FactoryBot.create(:repository) + end + } + let!(:user1) { + FactoryBot.create(:user_with_repos) + } + let!(:user2) { + FactoryBot.create(:user_with_repos) + } + let!(:organization1) { + FactoryBot.create(:organization_with_repos) + } + let!(:organization2) { + FactoryBot.create(:organization_with_repos) + } + + context 'when no arguments are given' do + it 'processes every repository' do + Repository.all.each do |repository| + expect(backup).to receive(:process_repo).once.with(repository) + end + backup.export + end + end + context 'when user_id is given' do + it 'processes only the repositories of the given user' do + processed_repos_ids = [] + allow(backup).to receive(:process_repo) {|repo| processed_repos_ids.push(repo.id)} + user_repos_ids = Repository.where( + 'owner_id = ? and owner_type = ?', + user1.id, + 'User' + ).map(&:id) + backup.export(user_id: user1.id) + expect(processed_repos_ids).to match_array(user_repos_ids) + end + end + context 'when org_id is given' do + it 'processes only the repositories of the given organization' do + processed_repos_ids = [] + allow(backup).to receive(:process_repo) {|repo| processed_repos_ids.push(repo.id)} + organization_repos_ids = Repository.where( + 'owner_id = ? and owner_type = ?', + organization1.id, + 'Organization' + ).map(&:id) + backup.export(org_id: organization1.id) + expect(processed_repos_ids).to match_array(organization_repos_ids) + end + end + context 'when repo_id is given' do + it 'processes only the repository with the given id' do + repo = Repository.first + expect(backup).to receive(:process_repo).once.with(repo) + backup.export(repo_id: repo.id) + end + end + end describe 'process_repo' do + after(:each) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + end + + let!(:config) { Config.new } + let(:datetime) { (config.delay + 1).months.ago.to_time.utc } + let(:org_id) { rand(100000) } + let(:com_id) { rand(100000) } + let(:private_org_id) { rand(100000) } + let(:private_com_id) { rand(100000) } + let!(:repository) { + FactoryBot.create( + :repository_with_builds, + created_at: datetime, + updated_at: datetime + ) + } + let!(:exported_object) { [[ { diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 15860e1..c119dfd 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -3,6 +3,38 @@ require 'factory_bot' FactoryBot.define do + factory :organization do + factory :organization_with_repos do + transient do + repos_count { 3 } + end + after(:create) do |organization, evaluator| + create_list( + :repository, + evaluator.repos_count, + owner_id: organization.id, + owner_type: 'Organization' + ) + end + end + end + + factory :user do + factory :user_with_repos do + transient do + repos_count { 3 } + end + after(:create) do |user, evaluator| + create_list( + :repository, + evaluator.repos_count, + owner_id: user.id, + owner_type: 'User' + ) + end + end + end + factory :repository do factory :repository_with_builds do transient do From b84c268fa34822e137e84eacfcdc469485b0b319 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 20 Aug 2021 12:49:24 +0200 Subject: [PATCH 24/84] gem wip --- Gemfile | 25 +------------------------ Rakefile | 2 +- bin/run_backup | 2 +- lib/{backup.rb => tci-backup.rb} | 0 spec/backup_spec.rb | 2 +- tci-backup.gemspec | 30 ++++++++++++++++++++++++++++++ 6 files changed, 34 insertions(+), 27 deletions(-) rename lib/{backup.rb => tci-backup.rb} (100%) create mode 100644 tci-backup.gemspec diff --git a/Gemfile b/Gemfile index f0c28a3..65903ed 100644 --- a/Gemfile +++ b/Gemfile @@ -3,27 +3,4 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.2' - -gem 'activerecord' -gem 'pg' -gem 'pry' -gem 'rails', '~> 6.1.3.1' - -gem 'bootsnap', require: false - -group :development, :test do - gem 'brakeman' - gem 'byebug', platforms: %i[mri mingw x64_mingw] - gem 'factory_bot' - gem 'rspec-rails' - gem 'listen' -end - -group :development do - gem 'rubocop', '~> 0.75.1', require: false - gem 'rubocop-rspec' -end - -# Windows does not include zoneinfo files, so bundle the tzinfo-data gem -gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gemspec \ No newline at end of file diff --git a/Rakefile b/Rakefile index 43ed54a..38ed1f9 100644 --- a/Rakefile +++ b/Rakefile @@ -2,7 +2,7 @@ namespace :backup do desc 'Backup all daily outdated build/job' task :cron do $: << 'lib' - require 'backup' + require 'tci-backup' Backup.new.run end diff --git a/bin/run_backup b/bin/run_backup index 344f0f5..cfdbbd3 100755 --- a/bin/run_backup +++ b/bin/run_backup @@ -2,6 +2,6 @@ $: << 'lib' -require 'backup' +require 'tci-backup' Backup.new.run diff --git a/lib/backup.rb b/lib/tci-backup.rb similarity index 100% rename from lib/backup.rb rename to lib/tci-backup.rb diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index a1f2789..33afc8f 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -1,6 +1,6 @@ $: << 'lib' require 'uri' -require 'backup' +require 'tci-backup' require 'models/repository' require 'models/build' require 'models/job' diff --git a/tci-backup.gemspec b/tci-backup.gemspec new file mode 100644 index 0000000..7eff21e --- /dev/null +++ b/tci-backup.gemspec @@ -0,0 +1,30 @@ +Gem::Specification.new do |s| + s.name = 'tci-backup' + s.version = '0.0.1' + s.summary = 'Travis CI backup tool' + s.authors = ['Karol Selak'] + s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") + s.files = Dir.chdir(File.expand_path('..', __FILE__)) do + `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + end + s.require_paths = ["."] + + s.add_dependency 'activerecord' + s.add_dependency 'pg' + s.add_dependency 'pry' + s.add_dependency 'rails' + + s.add_dependency 'bootsnap'#, require: false + + # Windows does not include zoneinfo files, so bundle the tzinfo-data s.add_dependency + s.add_dependency 'tzinfo-data'#, platforms: [:mingw, :mswin, :x64_mingw, :jruby] + + + s.add_development_dependency 'brakeman' + s.add_development_dependency 'byebug'#, platforms: %i[mri mingw x64_mingw] + s.add_development_dependency 'factory_bot' + s.add_development_dependency 'rspec-rails' + s.add_development_dependency 'listen' + s.add_development_dependency 'rubocop', '~> 0.75.1'#, require: false + s.add_development_dependency 'rubocop-rspec' +end \ No newline at end of file From 0c40046c622c4fe67567d8f91b669c769c28b347 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 20 Aug 2021 14:28:04 +0200 Subject: [PATCH 25/84] gem configuration working, config improved --- .gitignore | 3 +- Gemfile.lock | 184 +++++++++++++++++++++++---------------------- lib/config.rb | 26 +++++-- tci-backup.gemspec | 2 +- 4 files changed, 115 insertions(+), 100 deletions(-) diff --git a/.gitignore b/.gitignore index 9d37aeb..09f42e1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ log/* dump/* !tmp/.keep !log/.keep -!dump/.keep \ No newline at end of file +!dump/.keep +*.gem diff --git a/Gemfile.lock b/Gemfile.lock index ba978b9..5520076 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,144 +1,153 @@ +PATH + remote: . + specs: + tci-backup (0.0.1) + activerecord + bootsnap + pg + pry + rails + tzinfo-data + GEM remote: https://rubygems.org/ specs: - actioncable (6.1.3.1) - actionpack (= 6.1.3.1) - activesupport (= 6.1.3.1) + actioncable (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.3.1) - actionpack (= 6.1.3.1) - activejob (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionmailbox (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (>= 2.7.1) - actionmailer (6.1.3.1) - actionpack (= 6.1.3.1) - actionview (= 6.1.3.1) - activejob (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionmailer (6.1.4.1) + actionpack (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.3.1) - actionview (= 6.1.3.1) - activesupport (= 6.1.3.1) + actionpack (6.1.4.1) + actionview (= 6.1.4.1) + activesupport (= 6.1.4.1) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.3.1) - actionpack (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + actiontext (6.1.4.1) + actionpack (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) nokogiri (>= 1.8.5) - actionview (6.1.3.1) - activesupport (= 6.1.3.1) + actionview (6.1.4.1) + activesupport (= 6.1.4.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.3.1) - activesupport (= 6.1.3.1) + activejob (6.1.4.1) + activesupport (= 6.1.4.1) globalid (>= 0.3.6) - activemodel (6.1.3.1) - activesupport (= 6.1.3.1) - activerecord (6.1.3.1) - activemodel (= 6.1.3.1) - activesupport (= 6.1.3.1) - activestorage (6.1.3.1) - actionpack (= 6.1.3.1) - activejob (= 6.1.3.1) - activerecord (= 6.1.3.1) - activesupport (= 6.1.3.1) + activemodel (6.1.4.1) + activesupport (= 6.1.4.1) + activerecord (6.1.4.1) + activemodel (= 6.1.4.1) + activesupport (= 6.1.4.1) + activestorage (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activesupport (= 6.1.4.1) marcel (~> 1.0.0) - mini_mime (~> 1.0.2) - activesupport (6.1.3.1) + mini_mime (>= 1.1.0) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) ast (2.4.2) - bootsnap (1.7.3) + bootsnap (1.7.7) msgpack (~> 1.0) - brakeman (5.0.0) + brakeman (5.1.1) builder (3.2.4) byebug (11.1.3) coderay (1.1.3) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) crass (1.0.6) diff-lcs (1.4.4) erubi (1.10.0) - factory_bot (6.1.0) + factory_bot (6.2.0) activesupport (>= 5.0.0) - ffi (1.15.0) - globalid (0.4.2) - activesupport (>= 4.2.0) - i18n (1.8.9) + ffi (1.15.3) + globalid (0.5.2) + activesupport (>= 5.0) + i18n (1.8.10) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) - listen (3.5.0) + listen (3.7.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.9.0) + loofah (2.12.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) - marcel (1.0.0) + marcel (1.0.1) method_source (1.0.0) - mini_mime (1.0.3) - mini_portile2 (2.5.0) + mini_mime (1.1.0) + mini_portile2 (2.6.1) minitest (5.14.4) msgpack (1.4.2) - nio4r (2.5.7) - nokogiri (1.11.2) - mini_portile2 (~> 2.5.0) - racc (~> 1.4) - nokogiri (1.11.2-x86_64-darwin) + nio4r (2.5.8) + nokogiri (1.12.3) + mini_portile2 (~> 2.6.1) racc (~> 1.4) parallel (1.20.1) - parser (3.0.0.0) + parser (3.0.2.0) ast (~> 2.4.1) pg (1.2.3) - pry (0.14.0) + pry (0.14.1) coderay (~> 1.1) method_source (~> 1.0) racc (1.5.2) rack (2.2.3) rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.3.1) - actioncable (= 6.1.3.1) - actionmailbox (= 6.1.3.1) - actionmailer (= 6.1.3.1) - actionpack (= 6.1.3.1) - actiontext (= 6.1.3.1) - actionview (= 6.1.3.1) - activejob (= 6.1.3.1) - activemodel (= 6.1.3.1) - activerecord (= 6.1.3.1) - activestorage (= 6.1.3.1) - activesupport (= 6.1.3.1) + rails (6.1.4.1) + actioncable (= 6.1.4.1) + actionmailbox (= 6.1.4.1) + actionmailer (= 6.1.4.1) + actionpack (= 6.1.4.1) + actiontext (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activemodel (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) bundler (>= 1.15.0) - railties (= 6.1.3.1) + railties (= 6.1.4.1) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) + rails-html-sanitizer (1.4.1) loofah (~> 2.3) - railties (6.1.3.1) - actionpack (= 6.1.3.1) - activesupport (= 6.1.3.1) + railties (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) method_source - rake (>= 0.8.7) + rake (>= 0.13) thor (~> 1.0) rainbow (3.0.0) - rake (13.0.3) - rb-fsevent (0.10.4) + rake (13.0.6) + rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) rspec-core (3.10.1) @@ -149,7 +158,7 @@ GEM rspec-mocks (3.10.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.10.0) - rspec-rails (5.0.1) + rspec-rails (5.0.2) actionpack (>= 5.2) activesupport (>= 5.2) railties (>= 5.2) @@ -178,33 +187,26 @@ GEM thor (1.1.0) tzinfo (2.0.4) concurrent-ruby (~> 1.0) + tzinfo-data (1.2021.1) + tzinfo (>= 1.0.0) unicode-display_width (1.6.1) - websocket-driver (0.7.3) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) zeitwerk (2.4.2) PLATFORMS ruby - x86_64-darwin-20 DEPENDENCIES - activerecord - bootsnap brakeman byebug factory_bot listen - pg - pry - rails (~> 6.1.3.1) rspec-rails rubocop (~> 0.75.1) rubocop-rspec - tzinfo-data - -RUBY VERSION - ruby 2.7.0p0 + tci-backup! BUNDLED WITH - 2.2.7 + 2.1.4 diff --git a/lib/config.rb b/lib/config.rb index 4bc190f..f830727 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -5,20 +5,32 @@ class Config attr_reader :if_backup, :limit, :delay, :files_location, :database_url def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength - config = YAML.load(File.open('config/settings.yml')) - connection_details = YAML.load(File.open('config/database.yml')) + begin + config = YAML.load(File.open('config/settings.yml')) + rescue => e + config = {} + end + + begin + connection_details = YAML.load(File.open('config/database.yml')) + rescue => e + connection_details = {} + end + if !args[:if_backup].nil? @if_backup = args[:if_backup] elsif !ENV['IF_BACKUP'].nil? @if_backup = ENV['IF_BACKUP'] + elsif !config.dig('backup', 'if_backup').nil? + @if_backup = config.dig('backup', 'if_backup') else - @if_backup = config['backup']['if_backup'] + @if_backup = true end - @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config['backup']['limit'] - @delay = args[:delay] || ENV['BACKUP_DELAY'] || config['backup']['delay'] - @files_location = args[:files_location] || ENV['BACKUP_FILES_LOCATION'] || config['backup']['files_location'] - @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details[ENV['RAILS_ENV']] + @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config.dig('backup', 'limit') || 1000 + @delay = args[:delay] || ENV['BACKUP_DELAY'] || config.dig('backup', 'delay') || 6 + @files_location = args[:files_location] || ENV['BACKUP_FILES_LOCATION'] || config.dig('backup', 'files_location') || './dump' + @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details.dig(ENV['RAILS_ENV']) end end diff --git a/tci-backup.gemspec b/tci-backup.gemspec index 7eff21e..6bbd647 100644 --- a/tci-backup.gemspec +++ b/tci-backup.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |s| s.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - s.require_paths = ["."] + s.require_paths = ["lib"] s.add_dependency 'activerecord' s.add_dependency 'pg' From 6717b516dc35341107314081ee464d1c0a7ad268 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 20 Aug 2021 20:44:25 +0200 Subject: [PATCH 26/84] README improved and small refactoring done --- README.md | 102 ++++++++++++++++++++++++++++++++++---------- config/settings.yml | 4 +- lib/tci-backup.rb | 6 +-- spec/backup_spec.rb | 10 ++--- 4 files changed, 87 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 4744a26..55da0a2 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,93 @@ # README -*travis-backup* is a cron application, which export builds and it's correspoonding jobs -to json files and sends them to GCE. +*travis-backup* is an application that removes builds and their corresponding jobs +and exports them (optionally) to json files. -#### Ruby version +### Installation and run -2.7.2 +You can install the gem using + +`gem install tci-backup` + +Next you can run it in your app like + +``` +require 'tci-backup' + +backup = Backup.new +backup.run + +# or with configs: + +backup = Backup.new( + if_backup: true, + limit: 500, + delay: 12, + files_location: './my_folder/dump', + database_url: 'postgresql://postgres:pass@localhost:5432/my_db' +) +backup.run +``` + +You can also run backup only for given user, organisation or repository: + +``` +backup = Backup.new +backup.run(user_id: 1) +# or +backup.run(org_id: 1) +# or +backup.run(repo_id: 1) +``` + +### Configuration -#### Configuration +One of the ways you can configure your export is a file `config/settinigs.yml` that you should place in your app's main directory. The gem uses the properties in following format: -`config/settinigs.yml` or env vars like: -`BACKUP_LIMIT` -`BACKUP_DELAY` -`BACKUP_HOUSEKEEPING_PERIOD` -`LOGS_URL` -`DATABASE_URL` -`GCE_PROJECT` -`GCE_CREDENTIALS` -`GCE_BUCKET` -`REDIS_URL` +``` +backup: + if_backup: true # when false, removes data without saving it to file + limit: 1000 # builds limit for one backup file + delay: 6 # number of months from now - data younger than this time won't be backuped + files_location: './dump' # path of the folder in which backup files will be placed +``` -#### How to run the test suite +You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_LIMIT`, `BACKUP_DELAY`, `FILES_LOCATION`. -`bundle exec rspec` +You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema. + +### How to run the test suite + +You can run the test after cloning this repository. Next you should call + +``` +bundle install +``` + +and + +``` +bundle exec rspec +``` To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`. -#### How to run appication +**Warning: this database will be cleaned during tests, so ensure that it includes no important data.** -`bundle exec bin/run_backup` +#### Using as standalone application -It's also possibe to run console +After cloning this repo you can also run it as a standalone app using -`bundle exec bin/console` -and then run export for single user/organization -`Backup.new.export(owner_id)` \ No newline at end of file +``` +bundle exec bin/run_backup +``` + +or customize your run after opening the console: + +``` +bundle exec bin/console +``` + +### Ruby version + +2.7.2 diff --git a/config/settings.yml b/config/settings.yml index af36b3a..569d6d6 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1,9 +1,9 @@ backup: # when false, removes data without saving it to file if_backup: true - # builds limit in file + # builds limit for one backup file limit: 1000 - # number of months - data younger than this time won't be backuped + # number of months from now - data younger than this time won't be backuped delay: 6 # path of the folder in which backup files will be placed files_location: './dump' \ No newline at end of file diff --git a/lib/tci-backup.rb b/lib/tci-backup.rb index 39d0e27..d3ad261 100644 --- a/lib/tci-backup.rb +++ b/lib/tci-backup.rb @@ -12,15 +12,11 @@ def initialize(config_args={}) connect_db end - def run - export - end - def connect_db ActiveRecord::Base.establish_connection(@config.database_url) end - def export(args={}) + def run(args={}) if args[:user_id] owner_id = args[:user_id] owner_type = 'User' diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 33afc8f..6fd1bf3 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -17,7 +17,7 @@ let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location, limit: 2) } - describe 'export' do + describe 'run' do let!(:unassigned_repositories) { (1..3).to_a.map do FactoryBot.create(:repository) @@ -41,7 +41,7 @@ Repository.all.each do |repository| expect(backup).to receive(:process_repo).once.with(repository) end - backup.export + backup.run end end context 'when user_id is given' do @@ -53,7 +53,7 @@ user1.id, 'User' ).map(&:id) - backup.export(user_id: user1.id) + backup.run(user_id: user1.id) expect(processed_repos_ids).to match_array(user_repos_ids) end end @@ -66,7 +66,7 @@ organization1.id, 'Organization' ).map(&:id) - backup.export(org_id: organization1.id) + backup.run(org_id: organization1.id) expect(processed_repos_ids).to match_array(organization_repos_ids) end end @@ -74,7 +74,7 @@ it 'processes only the repository with the given id' do repo = Repository.first expect(backup).to receive(:process_repo).once.with(repo) - backup.export(repo_id: repo.id) + backup.run(repo_id: repo.id) end end end From 81dddf8742d7d30a73b663a24835caf1f6eed22e Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 20 Aug 2021 21:12:52 +0200 Subject: [PATCH 27/84] remove unused code --- app/assets/config/manifest.js | 2 -- app/assets/images/.keep | 0 app/assets/stylesheets/application.css | 15 --------------- app/channels/application_cable/channel.rb | 4 ---- app/channels/application_cable/connection.rb | 4 ---- app/controllers/application_controller.rb | 2 -- app/controllers/concerns/.keep | 0 app/helpers/application_helper.rb | 2 -- app/javascript/channels/consumer.js | 6 ------ app/javascript/channels/index.js | 5 ----- app/javascript/packs/application.js | 13 ------------- app/jobs/application_job.rb | 7 ------- app/mailers/application_mailer.rb | 4 ---- app/models/application_record.rb | 3 --- app/models/concerns/.keep | 0 app/views/layouts/application.html.erb | 16 ---------------- app/views/layouts/mailer.html.erb | 13 ------------- app/views/layouts/mailer.text.erb | 1 - test/application_system_test_case.rb | 5 ----- .../application_cable/connection_test.rb | 11 ----------- test/controllers/.keep | 0 test/fixtures/files/.keep | 0 test/helpers/.keep | 0 test/integration/.keep | 0 test/mailers/.keep | 0 test/models/.keep | 0 test/system/.keep | 0 test/test_helper.rb | 13 ------------- 28 files changed, 126 deletions(-) delete mode 100644 app/assets/config/manifest.js delete mode 100644 app/assets/images/.keep delete mode 100644 app/assets/stylesheets/application.css delete mode 100644 app/channels/application_cable/channel.rb delete mode 100644 app/channels/application_cable/connection.rb delete mode 100644 app/controllers/application_controller.rb delete mode 100644 app/controllers/concerns/.keep delete mode 100644 app/helpers/application_helper.rb delete mode 100644 app/javascript/channels/consumer.js delete mode 100644 app/javascript/channels/index.js delete mode 100644 app/javascript/packs/application.js delete mode 100644 app/jobs/application_job.rb delete mode 100644 app/mailers/application_mailer.rb delete mode 100644 app/models/application_record.rb delete mode 100644 app/models/concerns/.keep delete mode 100644 app/views/layouts/application.html.erb delete mode 100644 app/views/layouts/mailer.html.erb delete mode 100644 app/views/layouts/mailer.text.erb delete mode 100644 test/application_system_test_case.rb delete mode 100644 test/channels/application_cable/connection_test.rb delete mode 100644 test/controllers/.keep delete mode 100644 test/fixtures/files/.keep delete mode 100644 test/helpers/.keep delete mode 100644 test/integration/.keep delete mode 100644 test/mailers/.keep delete mode 100644 test/models/.keep delete mode 100644 test/system/.keep delete mode 100644 test/test_helper.rb diff --git a/app/assets/config/manifest.js b/app/assets/config/manifest.js deleted file mode 100644 index 5918193..0000000 --- a/app/assets/config/manifest.js +++ /dev/null @@ -1,2 +0,0 @@ -//= link_tree ../images -//= link_directory ../stylesheets .css diff --git a/app/assets/images/.keep b/app/assets/images/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/app/assets/stylesheets/application.css b/app/assets/stylesheets/application.css deleted file mode 100644 index d05ea0f..0000000 --- a/app/assets/stylesheets/application.css +++ /dev/null @@ -1,15 +0,0 @@ -/* - * This is a manifest file that'll be compiled into application.css, which will include all the files - * listed below. - * - * Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's - * vendor/assets/stylesheets directory can be referenced here using a relative path. - * - * You're free to add application-wide styles to this file and they'll appear at the bottom of the - * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS - * files in this directory. Styles in this file should be added after the last require_* statement. - * It is generally better to create a new file per style scope. - * - *= require_tree . - *= require_self - */ diff --git a/app/channels/application_cable/channel.rb b/app/channels/application_cable/channel.rb deleted file mode 100644 index d672697..0000000 --- a/app/channels/application_cable/channel.rb +++ /dev/null @@ -1,4 +0,0 @@ -module ApplicationCable - class Channel < ActionCable::Channel::Base - end -end diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb deleted file mode 100644 index 0ff5442..0000000 --- a/app/channels/application_cable/connection.rb +++ /dev/null @@ -1,4 +0,0 @@ -module ApplicationCable - class Connection < ActionCable::Connection::Base - end -end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb deleted file mode 100644 index 09705d1..0000000 --- a/app/controllers/application_controller.rb +++ /dev/null @@ -1,2 +0,0 @@ -class ApplicationController < ActionController::Base -end diff --git a/app/controllers/concerns/.keep b/app/controllers/concerns/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb deleted file mode 100644 index de6be79..0000000 --- a/app/helpers/application_helper.rb +++ /dev/null @@ -1,2 +0,0 @@ -module ApplicationHelper -end diff --git a/app/javascript/channels/consumer.js b/app/javascript/channels/consumer.js deleted file mode 100644 index 8ec3aad..0000000 --- a/app/javascript/channels/consumer.js +++ /dev/null @@ -1,6 +0,0 @@ -// Action Cable provides the framework to deal with WebSockets in Rails. -// You can generate new channels where WebSocket features live using the `bin/rails generate channel` command. - -import { createConsumer } from "@rails/actioncable" - -export default createConsumer() diff --git a/app/javascript/channels/index.js b/app/javascript/channels/index.js deleted file mode 100644 index 0cfcf74..0000000 --- a/app/javascript/channels/index.js +++ /dev/null @@ -1,5 +0,0 @@ -// Load all the channels within this directory and all subdirectories. -// Channel files must be named *_channel.js. - -const channels = require.context('.', true, /_channel\.js$/) -channels.keys().forEach(channels) diff --git a/app/javascript/packs/application.js b/app/javascript/packs/application.js deleted file mode 100644 index f710851..0000000 --- a/app/javascript/packs/application.js +++ /dev/null @@ -1,13 +0,0 @@ -// This file is automatically compiled by Webpack, along with any other files -// present in this directory. You're encouraged to place your actual application logic in -// a relevant structure within app/javascript and only use these pack files to reference -// that code so it'll be compiled. - -import Rails from "@rails/ujs" -import Turbolinks from "turbolinks" -import * as ActiveStorage from "@rails/activestorage" -import "channels" - -Rails.start() -Turbolinks.start() -ActiveStorage.start() diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb deleted file mode 100644 index d394c3d..0000000 --- a/app/jobs/application_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -class ApplicationJob < ActiveJob::Base - # Automatically retry jobs that encountered a deadlock - # retry_on ActiveRecord::Deadlocked - - # Most jobs are safe to ignore if the underlying records are no longer available - # discard_on ActiveJob::DeserializationError -end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb deleted file mode 100644 index 286b223..0000000 --- a/app/mailers/application_mailer.rb +++ /dev/null @@ -1,4 +0,0 @@ -class ApplicationMailer < ActionMailer::Base - default from: 'from@example.com' - layout 'mailer' -end diff --git a/app/models/application_record.rb b/app/models/application_record.rb deleted file mode 100644 index 10a4cba..0000000 --- a/app/models/application_record.rb +++ /dev/null @@ -1,3 +0,0 @@ -class ApplicationRecord < ActiveRecord::Base - self.abstract_class = true -end diff --git a/app/models/concerns/.keep b/app/models/concerns/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb deleted file mode 100644 index 39fd3c2..0000000 --- a/app/views/layouts/application.html.erb +++ /dev/null @@ -1,16 +0,0 @@ - - - - TravisBackup - - <%= csrf_meta_tags %> - <%= csp_meta_tag %> - - <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> - <%= javascript_pack_tag 'application', 'data-turbolinks-track': 'reload' %> - - - - <%= yield %> - - diff --git a/app/views/layouts/mailer.html.erb b/app/views/layouts/mailer.html.erb deleted file mode 100644 index cbd34d2..0000000 --- a/app/views/layouts/mailer.html.erb +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - <%= yield %> - - diff --git a/app/views/layouts/mailer.text.erb b/app/views/layouts/mailer.text.erb deleted file mode 100644 index 37f0bdd..0000000 --- a/app/views/layouts/mailer.text.erb +++ /dev/null @@ -1 +0,0 @@ -<%= yield %> diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb deleted file mode 100644 index d19212a..0000000 --- a/test/application_system_test_case.rb +++ /dev/null @@ -1,5 +0,0 @@ -require "test_helper" - -class ApplicationSystemTestCase < ActionDispatch::SystemTestCase - driven_by :selenium, using: :chrome, screen_size: [1400, 1400] -end diff --git a/test/channels/application_cable/connection_test.rb b/test/channels/application_cable/connection_test.rb deleted file mode 100644 index 800405f..0000000 --- a/test/channels/application_cable/connection_test.rb +++ /dev/null @@ -1,11 +0,0 @@ -require "test_helper" - -class ApplicationCable::ConnectionTest < ActionCable::Connection::TestCase - # test "connects with cookies" do - # cookies.signed[:user_id] = 42 - # - # connect - # - # assert_equal connection.user_id, "42" - # end -end diff --git a/test/controllers/.keep b/test/controllers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/fixtures/files/.keep b/test/fixtures/files/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/helpers/.keep b/test/helpers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/integration/.keep b/test/integration/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/mailers/.keep b/test/mailers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/models/.keep b/test/models/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/system/.keep b/test/system/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/test_helper.rb b/test/test_helper.rb deleted file mode 100644 index a4a491d..0000000 --- a/test/test_helper.rb +++ /dev/null @@ -1,13 +0,0 @@ -ENV['RAILS_ENV'] = 'test' -require_relative "../config/environment" -require "rails/test_help" - -class ActiveSupport::TestCase - # Run tests in parallel with specified workers - parallelize(workers: :number_of_processors) - - # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. - fixtures :all - - # Add more helper methods to be used by all tests here... -end From 847959896efd5c75cffbbab172d7e87ae4f3789d Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 23 Aug 2021 13:15:32 +0200 Subject: [PATCH 28/84] configuration improved, refactoring --- README.md | 20 ++++++-- bin/run_backup | 16 ++++++- config/environments/production.rb | 6 +-- config/settings.yml | 2 +- lib/config.rb | 80 +++++++++++++++++++++---------- lib/tci-backup.rb | 30 +++++++----- spec/backup_spec.rb | 2 +- 7 files changed, 108 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 55da0a2..669c09b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ backup.run backup = Backup.new( if_backup: true, limit: 500, - delay: 12, + threshold: 12, files_location: './my_folder/dump', database_url: 'postgresql://postgres:pass@localhost:5432/my_db' ) @@ -48,11 +48,14 @@ One of the ways you can configure your export is a file `config/settinigs.yml` t backup: if_backup: true # when false, removes data without saving it to file limit: 1000 # builds limit for one backup file - delay: 6 # number of months from now - data younger than this time won't be backuped + threshold: 6 # number of months from now - data younger than this time won't be backuped files_location: './dump' # path of the folder in which backup files will be placed + user_id # run only for given user + org_id # run only for given organization + repo_id # run only for given repository ``` -You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_LIMIT`, `BACKUP_DELAY`, `FILES_LOCATION`. +You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_LIMIT`, `BACKUP_DELAY`, `FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema. @@ -82,10 +85,17 @@ After cloning this repo you can also run it as a standalone app using bundle exec bin/run_backup ``` -or customize your run after opening the console: +You can also pass arguments: ``` -bundle exec bin/console + first argument, no flag # database url + -b, --if_backup # when not present, removes data without saving it to file + -l, --limit LIMIT # builds limit for one backup file + -t, --threshold MONTHS # number of months from now - data younger than this time won't be backuped + -f, --files_location PATH # path of the folder in which backup files will be placed + -u, --user_id ID # run only for given user + -o, --org_id ID # run only for given organization + -r, --repo_id ID # run only for given repository ``` ### Ruby version diff --git a/bin/run_backup b/bin/run_backup index cfdbbd3..84d284b 100755 --- a/bin/run_backup +++ b/bin/run_backup @@ -3,5 +3,19 @@ $: << 'lib' require 'tci-backup' +require 'optparse' -Backup.new.run +options = {} +OptionParser.new do |opt| + opt.on('-b', '--if_backup') { |o| options[:if_backup] = o } + opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i } + opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i } + opt.on('-f', '--files_location X') { |o| options[:files_location] = o } + opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i } + opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i } + opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i } +end.parse! + +options[:database_url] = ARGV[0] if ARGV[0] + +Backup.new(options).run diff --git a/config/environments/production.rb b/config/environments/production.rb index be36373..a0a0af1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -99,11 +99,11 @@ # Inserts middleware to perform automatic connection switching. # The `database_selector` hash is used to pass options to the DatabaseSelector - # middleware. The `delay` is used to determine how long to wait after a write + # middleware. The `threshold` is used to determine how long to wait after a write # to send a subsequent read to the primary. # # The `database_resolver` class is used by the middleware to determine which - # database is appropriate to use based on the time delay. + # database is appropriate to use based on the time threshold. # # The `database_resolver_context` class is used by the middleware to set # timestamps for the last write to the primary. The resolver uses the context @@ -114,7 +114,7 @@ # DatabaseSelector middleware is designed as such you can define your own # strategy for connection switching and pass that into the middleware through # these configuration options. - # config.active_record.database_selector = { delay: 2.seconds } + # config.active_record.database_selector = { threshold: 2.seconds } # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session end diff --git a/config/settings.yml b/config/settings.yml index 569d6d6..2e2c81f 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -4,6 +4,6 @@ backup: # builds limit for one backup file limit: 1000 # number of months from now - data younger than this time won't be backuped - delay: 6 + threshold: 6 # path of the folder in which backup files will be placed files_location: './dump' \ No newline at end of file diff --git a/lib/config.rb b/lib/config.rb index f830727..d813bc9 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,35 +2,67 @@ # Config for travis-backup class Config - attr_reader :if_backup, :limit, :delay, :files_location, :database_url + attr_reader :if_backup, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength - begin - config = YAML.load(File.open('config/settings.yml')) - rescue => e - config = {} - end + config = yaml_load('config/settings.yml') + connection_details = yaml_load('config/database.yml') - begin - connection_details = YAML.load(File.open('config/database.yml')) - rescue => e - connection_details = {} - end + @if_backup = first_not_nil( + args[:if_backup], + ENV['IF_BACKUP'], + config.dig('backup', 'if_backup'), + true + ) + @limit = first_not_nil( + args[:limit], + ENV['BACKUP_LIMIT'], + config.dig('backup', 'limit'), + 1000 + ) + @threshold = first_not_nil( + args[:threshold], + ENV['BACKUP_DELAY'], + config.dig('backup', 'threshold'), + 6 + ) + @files_location = first_not_nil( + args[:files_location], + ENV['BACKUP_FILES_LOCATION'], + config.dig('backup', 'files_location'), + './dump' + ) + @database_url = first_not_nil( + args[:database_url], + ENV['DATABASE_URL'], + connection_details.dig(ENV['RAILS_ENV']) + ) + @user_id = first_not_nil( + args[:user_id], + ENV['USER_ID'], + config.dig('backup', 'user_id') + ) + @repo_id = first_not_nil( + args[:repo_id], + ENV['REPO_ID'], + config.dig('backup', 'repo_id') + ) + @org_id = first_not_nil( + args[:org_id], + ENV['ORG_ID'], + config.dig('backup', 'org_id') + ) + end + def first_not_nil(*arr) + arr.compact.first + end - if !args[:if_backup].nil? - @if_backup = args[:if_backup] - elsif !ENV['IF_BACKUP'].nil? - @if_backup = ENV['IF_BACKUP'] - elsif !config.dig('backup', 'if_backup').nil? - @if_backup = config.dig('backup', 'if_backup') - else - @if_backup = true + def yaml_load(url) + begin + YAML.load(File.open(url)) + rescue => e + {} end - - @limit = args[:limit] || ENV['BACKUP_LIMIT'] || config.dig('backup', 'limit') || 1000 - @delay = args[:delay] || ENV['BACKUP_DELAY'] || config.dig('backup', 'delay') || 6 - @files_location = args[:files_location] || ENV['BACKUP_FILES_LOCATION'] || config.dig('backup', 'files_location') || './dump' - @database_url = args[:database_url] || ENV['DATABASE_URL'] || connection_details.dig(ENV['RAILS_ENV']) end end diff --git a/lib/tci-backup.rb b/lib/tci-backup.rb index d3ad261..b4723fd 100644 --- a/lib/tci-backup.rb +++ b/lib/tci-backup.rb @@ -17,14 +17,18 @@ def connect_db end def run(args={}) - if args[:user_id] - owner_id = args[:user_id] + user_id = args[:user_id] || @config.user_id + repo_id = args[:repo_id] || @config.repo_id + org_id = args[:org_id] || @config.org_id + + if user_id + owner_id = user_id owner_type = 'User' - elsif args[:org_id] - owner_id = args[:org_id] + elsif org_id + owner_id = org_id owner_type = 'Organization' - elsif args[:repo_id] - repo_id = args[:repo_id] + elsif repo_id + repo_id = repo_id end if owner_id @@ -42,9 +46,9 @@ def run(args={}) end def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - delay = @config.delay.to_i.months.ago.to_datetime + threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 - repository.builds.where('created_at < ? and id != ?', delay, current_build_id) + repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| if builds_batch.count == @config.limit.to_i @config.if_backup ? save_batch(builds_batch, repository) : builds_batch.each(&:destroy) @@ -58,14 +62,14 @@ def save_batch(builds_batch, repository) builds_export = export_builds(builds_batch) file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" pretty_json = JSON.pretty_generate(builds_export) - if upload(file_name, pretty_json) + if save_file(file_name, pretty_json) builds_batch.each(&:destroy) end builds_export end - def upload(file_name, content) # rubocop:disable Metrics/MethodLength - uploaded = false + def save_file(file_name, content) # rubocop:disable Metrics/MethodLength + saved = false begin unless File.directory?(@config.files_location) FileUtils.mkdir_p(@config.files_location) @@ -74,12 +78,12 @@ def upload(file_name, content) # rubocop:disable Metrics/MethodLength File.open("#{@config.files_location}/#{file_name}", 'w') do |file| file.write(content) file.close - uploaded = true + saved = true end rescue => e print "Failed to save #{file_name}, error: #{e.inspect}\n" end - uploaded + saved end def export_builds(builds) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 6fd1bf3..78fe4eb 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -87,7 +87,7 @@ end let!(:config) { Config.new } - let(:datetime) { (config.delay + 1).months.ago.to_time.utc } + let(:datetime) { (config.threshold + 1).months.ago.to_time.utc } let(:org_id) { rand(100000) } let(:com_id) { rand(100000) } let(:private_org_id) { rand(100000) } From 5b4e3a742607248618609605187f65813036e108 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 23 Aug 2021 14:36:02 +0200 Subject: [PATCH 29/84] dry run --- README.md | 7 ++++++- bin/run_backup | 3 ++- lib/config.rb | 8 +++++++- lib/tci-backup.rb | 38 ++++++++++++++++++++++++++++++++++---- spec/backup_spec.rb | 25 +++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 669c09b..4d59c53 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,18 @@ You can also pass arguments: ``` first argument, no flag # database url - -b, --if_backup # when not present, removes data without saving it to file + -b, --backup # when not present, removes data without saving it to file + -d, --dry_run # only prints in console what data will be backuped and deleted -l, --limit LIMIT # builds limit for one backup file -t, --threshold MONTHS # number of months from now - data younger than this time won't be backuped -f, --files_location PATH # path of the folder in which backup files will be placed -u, --user_id ID # run only for given user -o, --org_id ID # run only for given organization -r, --repo_id ID # run only for given repository + + # example: + + bundle exec bin/run_backup 'postgres://user:pass@localhost:5432/my_db' -b ``` ### Ruby version diff --git a/bin/run_backup b/bin/run_backup index 84d284b..be412eb 100755 --- a/bin/run_backup +++ b/bin/run_backup @@ -7,7 +7,8 @@ require 'optparse' options = {} OptionParser.new do |opt| - opt.on('-b', '--if_backup') { |o| options[:if_backup] = o } + opt.on('-b', '--backup') { |o| options[:if_backup] = o } + opt.on('-d', '--dry_run') { |o| options[:dry_run] = o } opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i } opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i } opt.on('-f', '--files_location X') { |o| options[:files_location] = o } diff --git a/lib/config.rb b/lib/config.rb index d813bc9..03c45fa 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,7 @@ # Config for travis-backup class Config - attr_reader :if_backup, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id + attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = yaml_load('config/settings.yml') @@ -14,6 +14,12 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom config.dig('backup', 'if_backup'), true ) + @dry_run = first_not_nil( + args[:dry_run], + ENV['DRY_RUN'], + config.dig('backup', 'dry_run'), + false + ) @limit = first_not_nil( args[:limit], ENV['BACKUP_LIMIT'], diff --git a/lib/tci-backup.rb b/lib/tci-backup.rb index b4723fd..d122e6d 100644 --- a/lib/tci-backup.rb +++ b/lib/tci-backup.rb @@ -51,31 +51,50 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| if builds_batch.count == @config.limit.to_i - @config.if_backup ? save_batch(builds_batch, repository) : builds_batch.each(&:destroy) + @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch) end end.compact end private - def save_batch(builds_batch, repository) + def save_and_destroy_batch(builds_batch, repository) builds_export = export_builds(builds_batch) file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" pretty_json = JSON.pretty_generate(builds_export) if save_file(file_name, pretty_json) - builds_batch.each(&:destroy) + destroy_batch(builds_batch) end builds_export end + def destroy_batch(builds_batch) + return destroy_batch_dry(builds_batch) if @config.dry_run + + builds_batch.each(&:destroy) + end + + def destroy_batch_dry(builds_batch) + puts 'Dry mode: this code would destroy the following data in normal mode:' + puts ' - builds: ' + builds_batch.map(&:id).to_json + + jobs = builds_batch.map do |build| + build.jobs.map(&:id) || [] + end.flatten.to_json + + puts " - jobs: #{jobs}" + end + def save_file(file_name, content) # rubocop:disable Metrics/MethodLength + return save_file_dry(file_name, content) if @config.dry_run + saved = false begin unless File.directory?(@config.files_location) FileUtils.mkdir_p(@config.files_location) end - File.open("#{@config.files_location}/#{file_name}", 'w') do |file| + File.open(file_path(file_name), 'w') do |file| file.write(content) file.close saved = true @@ -86,6 +105,17 @@ def save_file(file_name, content) # rubocop:disable Metrics/MethodLength saved end + def file_path(file_name) + "#{@config.files_location}/#{file_name}" + end + + def save_file_dry(file_name, content) + puts 'Dry mode: this code would generate the following files in normal mode:' + puts "\n#{file_path(file_name)}:\n\n" + puts content + return true + end + def export_builds(builds) builds.map do |build| build_export = build.attributes diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 78fe4eb..c52ff02 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -335,5 +335,30 @@ it_behaves_like 'removing builds and jobs in batches' end + + context 'when dry_run config is set to true' do + let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } + + before do + allow_any_instance_of(IO).to receive(:puts) + end + + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo(repository) + end + + it 'should not delete builds' do + expect { + backup.process_repo(repository) + }.not_to change { Build.all.size } + end + + it 'should not delete jobs' do + expect { + backup.process_repo(repository) + }.not_to change { Job.all.size } + end + end end end From dedf231d2dd2c1b15507e44faf05056b27290a88 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 23 Aug 2021 14:52:20 +0200 Subject: [PATCH 30/84] README and config.rb improved --- README.md | 5 +++-- lib/config.rb | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4d59c53..d930098 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,16 @@ One of the ways you can configure your export is a file `config/settinigs.yml` t ``` backup: if_backup: true # when false, removes data without saving it to file + dry_run: false # when true, only prints in console what data should be backuped and deleted limit: 1000 # builds limit for one backup file - threshold: 6 # number of months from now - data younger than this time won't be backuped + threshold: 6 # number of months from now - data younger than this time won't be backuped files_location: './dump' # path of the folder in which backup files will be placed user_id # run only for given user org_id # run only for given organization repo_id # run only for given repository ``` -You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_LIMIT`, `BACKUP_DELAY`, `FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. +You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema. diff --git a/lib/config.rb b/lib/config.rb index 03c45fa..63c655d 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -16,7 +16,7 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom ) @dry_run = first_not_nil( args[:dry_run], - ENV['DRY_RUN'], + ENV['BACKUP_DRY_RUN'], config.dig('backup', 'dry_run'), false ) @@ -28,13 +28,13 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom ) @threshold = first_not_nil( args[:threshold], - ENV['BACKUP_DELAY'], + ENV['BACKUP_THRESHOLD'], config.dig('backup', 'threshold'), 6 ) @files_location = first_not_nil( args[:files_location], - ENV['BACKUP_FILES_LOCATION'], + ENV['BACKUP_BACKUP_FILES_LOCATION'], config.dig('backup', 'files_location'), './dump' ) From e850f192214ee1802268933197445a1699c00a65 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 23 Aug 2021 23:48:35 +0200 Subject: [PATCH 31/84] files renamed, license, argv options in config, small fix --- Gemfile.lock | 6 ++--- README.md | 4 ++-- Rakefile | 2 +- bin/run_backup | 19 ++-------------- lib/config.rb | 30 +++++++++++++++++++++++-- lib/{tci-backup.rb => travis-backup.rb} | 0 spec/backup_spec.rb | 2 +- tci-backup.gemspec | 30 ------------------------- travis-backup.gemspec | 28 +++++++++++++++++++++++ 9 files changed, 65 insertions(+), 56 deletions(-) rename lib/{tci-backup.rb => travis-backup.rb} (100%) delete mode 100644 tci-backup.gemspec create mode 100644 travis-backup.gemspec diff --git a/Gemfile.lock b/Gemfile.lock index 5520076..e184761 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - tci-backup (0.0.1) + travis-backup (0.0.1) activerecord bootsnap pg @@ -100,7 +100,7 @@ GEM mini_mime (>= 0.1.1) marcel (1.0.1) method_source (1.0.0) - mini_mime (1.1.0) + mini_mime (1.1.1) mini_portile2 (2.6.1) minitest (5.14.4) msgpack (1.4.2) @@ -206,7 +206,7 @@ DEPENDENCIES rspec-rails rubocop (~> 0.75.1) rubocop-rspec - tci-backup! + travis-backup! BUNDLED WITH 2.1.4 diff --git a/README.md b/README.md index d930098..ace6067 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,12 @@ and exports them (optionally) to json files. You can install the gem using -`gem install tci-backup` +`gem install travis-backup` Next you can run it in your app like ``` -require 'tci-backup' +require 'travis-backup' backup = Backup.new backup.run diff --git a/Rakefile b/Rakefile index 38ed1f9..e85148f 100644 --- a/Rakefile +++ b/Rakefile @@ -2,7 +2,7 @@ namespace :backup do desc 'Backup all daily outdated build/job' task :cron do $: << 'lib' - require 'tci-backup' + require 'travis-backup' Backup.new.run end diff --git a/bin/run_backup b/bin/run_backup index be412eb..35d6d55 100755 --- a/bin/run_backup +++ b/bin/run_backup @@ -2,21 +2,6 @@ $: << 'lib' -require 'tci-backup' -require 'optparse' +require 'travis-backup' -options = {} -OptionParser.new do |opt| - opt.on('-b', '--backup') { |o| options[:if_backup] = o } - opt.on('-d', '--dry_run') { |o| options[:dry_run] = o } - opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i } - opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i } - opt.on('-f', '--files_location X') { |o| options[:files_location] = o } - opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i } - opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i } - opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i } -end.parse! - -options[:database_url] = ARGV[0] if ARGV[0] - -Backup.new(options).run +Backup.new.run diff --git a/lib/config.rb b/lib/config.rb index 63c655d..c78ce13 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -1,65 +1,91 @@ # frozen_string_literal: true +require 'optparse' -# Config for travis-backup class Config attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength config = yaml_load('config/settings.yml') connection_details = yaml_load('config/database.yml') - + argv_opts = argv_options @if_backup = first_not_nil( args[:if_backup], + argv_opts[:if_backup], ENV['IF_BACKUP'], config.dig('backup', 'if_backup'), true ) @dry_run = first_not_nil( args[:dry_run], + argv_opts[:dry_run], ENV['BACKUP_DRY_RUN'], config.dig('backup', 'dry_run'), false ) @limit = first_not_nil( args[:limit], + argv_opts[:limit], ENV['BACKUP_LIMIT'], config.dig('backup', 'limit'), 1000 ) @threshold = first_not_nil( args[:threshold], + argv_opts[:threshold], ENV['BACKUP_THRESHOLD'], config.dig('backup', 'threshold'), 6 ) @files_location = first_not_nil( args[:files_location], + argv_opts[:files_location], ENV['BACKUP_BACKUP_FILES_LOCATION'], config.dig('backup', 'files_location'), './dump' ) @database_url = first_not_nil( args[:database_url], + argv_opts[:database_url], ENV['DATABASE_URL'], connection_details.dig(ENV['RAILS_ENV']) ) @user_id = first_not_nil( args[:user_id], + argv_opts[:user_id], ENV['USER_ID'], config.dig('backup', 'user_id') ) @repo_id = first_not_nil( args[:repo_id], + argv_opts[:repo_id], ENV['REPO_ID'], config.dig('backup', 'repo_id') ) @org_id = first_not_nil( args[:org_id], + argv_opts[:org_id], ENV['ORG_ID'], config.dig('backup', 'org_id') ) end + def argv_options + options = {} + OptionParser.new do |opt| + opt.on('-b', '--backup') { |o| options[:if_backup] = o } + opt.on('-d', '--dry_run') { |o| options[:dry_run] = o } + opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i } + opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i } + opt.on('-f', '--files_location X') { |o| options[:files_location] = o } + opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i } + opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i } + opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i } + end.parse! + + options[:database_url] = ARGV[0] if ARGV[0] + options + end + def first_not_nil(*arr) arr.compact.first end diff --git a/lib/tci-backup.rb b/lib/travis-backup.rb similarity index 100% rename from lib/tci-backup.rb rename to lib/travis-backup.rb diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index c52ff02..57e726e 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -1,6 +1,6 @@ $: << 'lib' require 'uri' -require 'tci-backup' +require 'travis-backup' require 'models/repository' require 'models/build' require 'models/job' diff --git a/tci-backup.gemspec b/tci-backup.gemspec deleted file mode 100644 index 6bbd647..0000000 --- a/tci-backup.gemspec +++ /dev/null @@ -1,30 +0,0 @@ -Gem::Specification.new do |s| - s.name = 'tci-backup' - s.version = '0.0.1' - s.summary = 'Travis CI backup tool' - s.authors = ['Karol Selak'] - s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") - s.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - s.require_paths = ["lib"] - - s.add_dependency 'activerecord' - s.add_dependency 'pg' - s.add_dependency 'pry' - s.add_dependency 'rails' - - s.add_dependency 'bootsnap'#, require: false - - # Windows does not include zoneinfo files, so bundle the tzinfo-data s.add_dependency - s.add_dependency 'tzinfo-data'#, platforms: [:mingw, :mswin, :x64_mingw, :jruby] - - - s.add_development_dependency 'brakeman' - s.add_development_dependency 'byebug'#, platforms: %i[mri mingw x64_mingw] - s.add_development_dependency 'factory_bot' - s.add_development_dependency 'rspec-rails' - s.add_development_dependency 'listen' - s.add_development_dependency 'rubocop', '~> 0.75.1'#, require: false - s.add_development_dependency 'rubocop-rspec' -end \ No newline at end of file diff --git a/travis-backup.gemspec b/travis-backup.gemspec new file mode 100644 index 0000000..cf2e4a7 --- /dev/null +++ b/travis-backup.gemspec @@ -0,0 +1,28 @@ +Gem::Specification.new do |s| + s.name = 'travis-backup' + s.version = '0.0.1' + s.summary = 'Travis CI backup tool' + s.authors = ['Karol Selak'] + s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") + s.files = Dir.chdir(File.expand_path('..', __FILE__)) do + `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + end + s.require_paths = ["lib"] + s.license = 'Beerware' + + s.add_dependency 'activerecord' + s.add_dependency 'pg' + s.add_dependency 'pry' + s.add_dependency 'rails' + + s.add_dependency 'bootsnap' + s.add_dependency 'tzinfo-data' + + s.add_development_dependency 'brakeman' + s.add_development_dependency 'byebug' + s.add_development_dependency 'factory_bot' + s.add_development_dependency 'rspec-rails' + s.add_development_dependency 'listen' + s.add_development_dependency 'rubocop', '~> 0.75.1' + s.add_development_dependency 'rubocop-rspec' +end \ No newline at end of file From b5b29a99a91b1629acb2b66ca7299883deb32ca3 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 13:42:41 +0200 Subject: [PATCH 32/84] removing data not fitting in limit, dry_run logs changed --- lib/travis-backup.rb | 33 ++++++++++++++++----------------- spec/backup_spec.rb | 20 +++++++++++++++----- spec/support/factories.rb | 2 +- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index d122e6d..3b874c0 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -9,6 +9,11 @@ class Backup def initialize(config_args={}) @config = Config.new(config_args) + + if @config.dry_run + @dry_run_removed = {builds: [], jobs: []} + end + connect_db end @@ -43,6 +48,12 @@ def run(args={}) process_repo(repository) end end + + if @config.dry_run + puts 'Dry run active. The following data would be removed in normal mode:' + puts " - builds: #{@dry_run_removed[:builds].to_json}" + puts " - jobs: #{@dry_run_removed[:jobs].to_json}" + end end def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength @@ -50,9 +61,7 @@ def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe current_build_id = repository.current_build_id || -1 repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - if builds_batch.count == @config.limit.to_i - @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch) - end + @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch) end.compact end @@ -75,18 +84,15 @@ def destroy_batch(builds_batch) end def destroy_batch_dry(builds_batch) - puts 'Dry mode: this code would destroy the following data in normal mode:' - puts ' - builds: ' + builds_batch.map(&:id).to_json - + @dry_run_removed[:builds].concat(builds_batch.map(&:id)) jobs = builds_batch.map do |build| build.jobs.map(&:id) || [] - end.flatten.to_json - - puts " - jobs: #{jobs}" + end.flatten + @dry_run_removed[:jobs].concat(jobs) end def save_file(file_name, content) # rubocop:disable Metrics/MethodLength - return save_file_dry(file_name, content) if @config.dry_run + return true if @config.dry_run saved = false begin @@ -109,13 +115,6 @@ def file_path(file_name) "#{@config.files_location}/#{file_name}" end - def save_file_dry(file_name, content) - puts 'Dry mode: this code would generate the following files in normal mode:' - puts "\n#{file_path(file_name)}:\n\n" - puts content - return true - end - def export_builds(builds) builds.map do |build| build_export = build.attributes diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 57e726e..0ea4e60 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -11,6 +11,7 @@ describe Backup do before(:all) do + ARGV = [] system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") end @@ -99,6 +100,15 @@ updated_at: datetime ) } + let!(:repository2) { + FactoryBot.create( + :repository_with_builds, + created_at: datetime, + updated_at: datetime, + builds_count: 1 + ) + } + let!(:exported_object) { [[ @@ -281,10 +291,10 @@ ]] } - shared_context 'removing builds and jobs in batches' do - it 'should delete builds fitting to batches' do + shared_context 'removing builds and jobs' do + it 'should delete all builds of the repository' do backup.process_repo(repository) - expect(Build.all.size).to eq(1) + expect(Build.all.map(&:repository_id)).to eq([repository2.id]) end it 'should delete all jobs of removed builds and leave the rest' do @@ -312,7 +322,7 @@ backup.process_repo(repository) end - it_behaves_like 'removing builds and jobs in batches' + it_behaves_like 'removing builds and jobs' context 'when path with nonexistent folders is given' do let(:random_files_location) { "dump/tests/#{rand(100000)}" } @@ -333,7 +343,7 @@ backup.process_repo(repository) end - it_behaves_like 'removing builds and jobs in batches' + it_behaves_like 'removing builds and jobs' end context 'when dry_run config is set to true' do diff --git a/spec/support/factories.rb b/spec/support/factories.rb index c119dfd..169d3fd 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -38,7 +38,7 @@ factory :repository do factory :repository_with_builds do transient do - builds_count { 3 } + builds_count { 2 } end after(:create) do |repository, evaluator| create_list( From 8f0c2f021f74971b289c7e627003625b5f8f68a6 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 13:43:35 +0200 Subject: [PATCH 33/84] executables in gemspec --- travis-backup.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/travis-backup.gemspec b/travis-backup.gemspec index cf2e4a7..be16983 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -7,6 +7,7 @@ Gem::Specification.new do |s| s.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end + s.executables = Dir.glob('bin/*').map { |f| File.basename(f) } s.require_paths = ["lib"] s.license = 'Beerware' From b5ae20520bbed961801ea33a1c767fae01792b37 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 13:57:44 +0200 Subject: [PATCH 34/84] run_backup -> travis_backup --- README.md | 4 ++-- bin/{run_backup => travis_backup} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename bin/{run_backup => travis_backup} (100%) diff --git a/README.md b/README.md index ace6067..50e04de 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ To make tests working properly you should also ensure the database connection st After cloning this repo you can also run it as a standalone app using ``` -bundle exec bin/run_backup +bundle exec bin/travis_backup ``` You can also pass arguments: @@ -101,7 +101,7 @@ You can also pass arguments: # example: - bundle exec bin/run_backup 'postgres://user:pass@localhost:5432/my_db' -b + bundle exec bin/travis_backup 'postgres://user:pass@localhost:5432/my_db' -b ``` ### Ruby version diff --git a/bin/run_backup b/bin/travis_backup similarity index 100% rename from bin/run_backup rename to bin/travis_backup From 5f180a36798f516af5f71776551a45caacfc101b Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 14:07:28 +0200 Subject: [PATCH 35/84] rescuing AdapterNotSpecified error --- lib/travis-backup.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 3b874c0..82b761c 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -18,7 +18,13 @@ def initialize(config_args={}) end def connect_db - ActiveRecord::Base.establish_connection(@config.database_url) + begin + ActiveRecord::Base.establish_connection(@config.database_url) + rescue ActiveRecord::AdapterNotSpecified => e + raise "Please provide proper database URL. You can set it as first argument running the application:\n" + + "\n $ bin/travis_backup 'postgres://my_database_url'\n" + + "\nor setting it using environment variables or config/database.yml file.\n" + end end def run(args={}) From 2f298cc322b12d737ccc4554c2c3593190dab6ca Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 14:27:09 +0200 Subject: [PATCH 36/84] requiring threshold --- config/settings.yml | 2 -- lib/config.rb | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/settings.yml b/config/settings.yml index 2e2c81f..61e3a29 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -3,7 +3,5 @@ backup: if_backup: true # builds limit for one backup file limit: 1000 - # number of months from now - data younger than this time won't be backuped - threshold: 6 # path of the folder in which backup files will be placed files_location: './dump' \ No newline at end of file diff --git a/lib/config.rb b/lib/config.rb index c78ce13..347d4da 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -33,8 +33,7 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom args[:threshold], argv_opts[:threshold], ENV['BACKUP_THRESHOLD'], - config.dig('backup', 'threshold'), - 6 + config.dig('backup', 'threshold') ) @files_location = first_not_nil( args[:files_location], @@ -67,6 +66,11 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom ENV['ORG_ID'], config.dig('backup', 'org_id') ) + if !@threshold + raise 'Please provide the threshold argument. Data younger than it will be omitted.' + + "Threshold defines number of months from now. It can be set like:\n\n"+ + " $ bin/travis_backup --threshold 6\n" + end end def argv_options From a672f6189b4e8b995d1d76a302fb3bbe30909631 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 14:51:12 +0200 Subject: [PATCH 37/84] fix with ARGV --- lib/config.rb | 6 +++++- spec/backup_spec.rb | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index 347d4da..7d06712 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -74,6 +74,7 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom end def argv_options + argv_copy = ARGV.clone options = {} OptionParser.new do |opt| opt.on('-b', '--backup') { |o| options[:if_backup] = o } @@ -86,7 +87,10 @@ def argv_options opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i } end.parse! - options[:database_url] = ARGV[0] if ARGV[0] + options[:database_url] = ARGV.shift if ARGV[0] + argv_copy.each do |arg| + ARGV.push(arg) + end options end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 0ea4e60..5f8e63b 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -11,7 +11,7 @@ describe Backup do before(:all) do - ARGV = [] + ARGV = ['-t', '6'] system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") end From 11bb1e75344ca66bd43bb1b24461ab10b2a59ed0 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 15:04:40 +0200 Subject: [PATCH 38/84] abort instead of raise, better messages --- lib/config.rb | 7 ++++--- lib/travis-backup.rb | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index 7d06712..fdc28fe 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -67,9 +67,10 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom config.dig('backup', 'org_id') ) if !@threshold - raise 'Please provide the threshold argument. Data younger than it will be omitted.' + - "Threshold defines number of months from now. It can be set like:\n\n"+ - " $ bin/travis_backup --threshold 6\n" + abort "\nPlease provide the threshold argument. Data younger than it will be omitted. " + + "Threshold defines number of months from now. Example usage:\n"+ + "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + + "\nYou can also set it using environment variables or config/database.yml file.\n" end end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 82b761c..df03630 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -21,9 +21,9 @@ def connect_db begin ActiveRecord::Base.establish_connection(@config.database_url) rescue ActiveRecord::AdapterNotSpecified => e - raise "Please provide proper database URL. You can set it as first argument running the application:\n" + - "\n $ bin/travis_backup 'postgres://my_database_url'\n" + - "\nor setting it using environment variables or config/database.yml file.\n" + abort "\nPlease provide proper database URL. Example usage:\n" + + "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + + "\nYou can also set it using environment variables or config/database.yml file.\n" end end From 6bfbd60685dd09f3cf0e56d9e42cb1837ff92195 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 15:12:31 +0200 Subject: [PATCH 39/84] refactoring --- lib/config.rb | 26 ++++++++++++++++++++++---- lib/travis-backup.rb | 8 +------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index fdc28fe..3ddffdf 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -5,6 +5,11 @@ class Config attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength + set_values(args) + check_values + end + + def set_values(args) config = yaml_load('config/settings.yml') connection_details = yaml_load('config/database.yml') argv_opts = argv_options @@ -66,12 +71,25 @@ def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCom ENV['ORG_ID'], config.dig('backup', 'org_id') ) + end + + def check_values if !@threshold - abort "\nPlease provide the threshold argument. Data younger than it will be omitted. " + - "Threshold defines number of months from now. Example usage:\n"+ - "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + - "\nYou can also set it using environment variables or config/database.yml file.\n" + message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " + + "Threshold defines number of months from now.") + abort message end + + if !@database_url + message = abort_message("Please provide proper database URL.") + abort message + end + end + + def abort_message(intro) + "\n#{intro} Example usage:\n"+ + "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + + "\nYou can also set it using environment variables or config/database.yml file.\n" end def argv_options diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index df03630..3b874c0 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -18,13 +18,7 @@ def initialize(config_args={}) end def connect_db - begin - ActiveRecord::Base.establish_connection(@config.database_url) - rescue ActiveRecord::AdapterNotSpecified => e - abort "\nPlease provide proper database URL. Example usage:\n" + - "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + - "\nYou can also set it using environment variables or config/database.yml file.\n" - end + ActiveRecord::Base.establish_connection(@config.database_url) end def run(args={}) From 98317b8ffdba70f594389ba6f8309b64b28176f6 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 16:24:36 +0200 Subject: [PATCH 40/84] README improved --- README.md | 56 ++++++++++++++++++++------------------------------- lib/config.rb | 2 ++ 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 50e04de..026a6e9 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,30 @@ You can install the gem using `gem install travis-backup` -Next you can run it in your app like +Next you can run it like: ``` -require 'travis-backup' +travis_backup 'postgres://user:pass@localhost:5432/my_db' --threshold 6 +``` -backup = Backup.new -backup.run +All arguments: -# or with configs: +``` + first argument, no flag # database url + -b, --backup # when not present, removes data without saving it to file + -d, --dry_run # only prints in console what data will be backuped and deleted + -l, --limit LIMIT # builds limit for one backup file + -t, --threshold MONTHS # number of months from now - data younger than this time won't be backuped + -f, --files_location PATH # path of the folder in which backup files will be placed + -u, --user_id ID # run only for given user + -o, --org_id ID # run only for given organization + -r, --repo_id ID # run only for given repository +``` + +Or inside your app: + +``` +require 'travis-backup' backup = Backup.new( if_backup: true, @@ -32,7 +47,6 @@ backup.run You can also run backup only for given user, organisation or repository: ``` -backup = Backup.new backup.run(user_id: 1) # or backup.run(org_id: 1) @@ -42,7 +56,7 @@ backup.run(repo_id: 1) ### Configuration -One of the ways you can configure your export is a file `config/settinigs.yml` that you should place in your app's main directory. The gem uses the properties in following format: +Despite of command line arguments, one of the ways you can configure your export is a file `config/settinigs.yml` that you can place in your app's main directory. The gem expects properties in the following format: ``` backup: @@ -56,7 +70,7 @@ backup: repo_id # run only for given repository ``` -You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. +You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema. @@ -78,32 +92,6 @@ To make tests working properly you should also ensure the database connection st **Warning: this database will be cleaned during tests, so ensure that it includes no important data.** -#### Using as standalone application - -After cloning this repo you can also run it as a standalone app using - -``` -bundle exec bin/travis_backup -``` - -You can also pass arguments: - -``` - first argument, no flag # database url - -b, --backup # when not present, removes data without saving it to file - -d, --dry_run # only prints in console what data will be backuped and deleted - -l, --limit LIMIT # builds limit for one backup file - -t, --threshold MONTHS # number of months from now - data younger than this time won't be backuped - -f, --files_location PATH # path of the folder in which backup files will be placed - -u, --user_id ID # run only for given user - -o, --org_id ID # run only for given organization - -r, --repo_id ID # run only for given repository - - # example: - - bundle exec bin/travis_backup 'postgres://user:pass@localhost:5432/my_db' -b -``` - ### Ruby version 2.7.2 diff --git a/lib/config.rb b/lib/config.rb index 3ddffdf..7341f63 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -89,6 +89,8 @@ def check_values def abort_message(intro) "\n#{intro} Example usage:\n"+ "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + + "\nor using in code:\n" + + "\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" + "\nYou can also set it using environment variables or config/database.yml file.\n" end From 79e60e2e3126cf179ef5cdd4b92e28ba831815d2 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 19:49:30 +0200 Subject: [PATCH 41/84] v0.0.2 --- travis-backup.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-backup.gemspec b/travis-backup.gemspec index be16983..22203a1 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.0.1' + s.version = '0.0.2' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From b78f4c03066456109df60ea298894cd4e952dcf9 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 24 Aug 2021 20:03:37 +0200 Subject: [PATCH 42/84] fix --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e184761..f8b61a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.0.1) + travis-backup (0.0.2) activerecord bootsnap pg From 155da2fb672550c8dfc31284f23f89c4c3950d76 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 14:30:41 +0200 Subject: [PATCH 43/84] moving logs --- lib/config.rb | 39 ++++++++++++++++++++-- lib/models/log.rb | 7 ++++ lib/travis-backup.rb | 26 +++++++++++++-- spec/backup_spec.rb | 70 +++++++++++++++++++++++++++++++++++++-- spec/support/factories.rb | 9 +++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 lib/models/log.rb diff --git a/lib/config.rb b/lib/config.rb index 7341f63..92a24ac 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -2,7 +2,17 @@ require 'optparse' class Config - attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id + attr_reader :if_backup, + :dry_run, + :limit, + :threshold, + :files_location, + :database_url, + :user_id, + :repo_id, + :org_id, + :move_logs, + :destination_db_url def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength set_values(args) @@ -43,7 +53,7 @@ def set_values(args) @files_location = first_not_nil( args[:files_location], argv_opts[:files_location], - ENV['BACKUP_BACKUP_FILES_LOCATION'], + ENV['BACKUP_FILES_LOCATION'], config.dig('backup', 'files_location'), './dump' ) @@ -71,6 +81,19 @@ def set_values(args) ENV['ORG_ID'], config.dig('backup', 'org_id') ) + @move_logs = first_not_nil( + args[:move_logs], + argv_opts[:move_logs], + ENV['BACKUP_MOVE_LOGS'], + config.dig('backup', 'move_logs'), + false + ) + @destination_db_url = first_not_nil( + args[:destination_db_url], + argv_opts[:destination_db_url], + ENV['BACKUP_DESTINATION_DB_URL'], + config.dig('backup', 'destination_db_url') + ) end def check_values @@ -84,6 +107,14 @@ def check_values message = abort_message("Please provide proper database URL.") abort message end + + if (@move_logs && !@destination_db_url) + abort "\nFor moving logs you need to specify your destination database. Example usage:\n" + + "\n $ bin/travis_backup 'postgres://source_url' --move_logs --destination_db_url 'postgres://destination_url'\n" + + "\nor using in code:\n" + + "\n Backup.new(database_url: 'postgres://source_url', destination_db_url: 'postgres://destination_url', move_logs: true)\n" + + "\nYou can also set it using environment variables or configuration files.\n" + end end def abort_message(intro) @@ -91,7 +122,7 @@ def abort_message(intro) "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" + "\nor using in code:\n" + "\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" + - "\nYou can also set it using environment variables or config/database.yml file.\n" + "\nYou can also set it using environment variables or configuration files.\n" end def argv_options @@ -106,6 +137,8 @@ def argv_options opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i } opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i } opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i } + opt.on('--move_logs') { |o| options[:move_logs] = o } + opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o } end.parse! options[:database_url] = ARGV.shift if ARGV[0] diff --git a/lib/models/log.rb b/lib/models/log.rb new file mode 100644 index 0000000..165f223 --- /dev/null +++ b/lib/models/log.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Log < Model + self.table_name = 'logs' +end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 3b874c0..6613fab 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -4,9 +4,12 @@ require 'active_support/time' require 'config' require 'models/repository' +require 'models/log' # main travis-backup class class Backup + attr_accessor :config + def initialize(config_args={}) @config = Config.new(config_args) @@ -17,11 +20,13 @@ def initialize(config_args={}) connect_db end - def connect_db - ActiveRecord::Base.establish_connection(@config.database_url) + def connect_db(url=@config.database_url) + ActiveRecord::Base.establish_connection(url) end def run(args={}) + return move_logs if @config.move_logs + user_id = args[:user_id] || @config.user_id repo_id = args[:repo_id] || @config.repo_id org_id = args[:org_id] || @config.org_id @@ -56,6 +61,23 @@ def run(args={}) end end + def move_logs + connect_db(@config.database_url) + Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch| + log_hashes = logs_batch.as_json + connect_db(@config.destination_db_url) + + log_hashes.each do |log_hash| + new_log = Log.new(log_hash) + new_log.save! + end + + connect_db(@config.database_url) + + logs_batch.each(&:destroy) + end + end + def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 5f8e63b..af86b77 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -12,11 +12,58 @@ describe Backup do before(:all) do ARGV = ['-t', '6'] - system("psql '#{Config.new.database_url}' -f db/schema.sql > /dev/null") + config = Config.new + system("psql '#{config.database_url}' -f db/schema.sql > /dev/null") + system("psql '#{config.destination_db_url}' -f db/schema.sql > /dev/null") if config.destination_db_url end let(:files_location) { "dump/tests" } - let!(:backup) { Backup.new(files_location: files_location, limit: 2) } + let!(:backup) { Backup.new(files_location: files_location, limit: 5) } + + describe 'move_logs' do + let!(:logs) { + (1..10).to_a.map do + FactoryBot.create(:log) + end + } + + def connect_db(url) + ActiveRecord::Base.establish_connection(url) + end + + def do_in_destination_db(config) + connect_db(config.destination_db_url) + result = yield + connect_db(config.database_url) + result + end + + it 'copies logs to destination database' do + def destination_logs_size + do_in_destination_db(backup.config) do + Log.all.size + end + end + + source_db_logs = Log.all.as_json + + expect { + backup.move_logs + }.to change { destination_logs_size }.by 10 + + destination_db_logs = do_in_destination_db(backup.config) do + Log.all.as_json + end + + expect(destination_db_logs).to eql(source_db_logs) + end + + it 'removes copied logs from source database' do + expect { + backup.move_logs + }.to change { Log.all.size }.by -10 + end + end describe 'run' do let!(:unassigned_repositories) { @@ -45,6 +92,7 @@ backup.run end end + context 'when user_id is given' do it 'processes only the repositories of the given user' do processed_repos_ids = [] @@ -58,6 +106,7 @@ expect(processed_repos_ids).to match_array(user_repos_ids) end end + context 'when org_id is given' do it 'processes only the repositories of the given organization' do processed_repos_ids = [] @@ -71,6 +120,7 @@ expect(processed_repos_ids).to match_array(organization_repos_ids) end end + context 'when repo_id is given' do it 'processes only the repository with the given id' do repo = Repository.first @@ -78,6 +128,22 @@ backup.run(repo_id: repo.id) end end + + context 'when move logs mode is on' do + let!(:backup) { Backup.new(files_location: files_location, limit: 5, move_logs: true) } + + it 'does not process repositories' do + repo = Repository.first + expect(backup).not_to receive(:process_repo) + backup.run + end + + it 'moves logs' do + repo = Repository.first + expect(backup).to receive(:move_logs).once + backup.run + end + end end describe 'process_repo' do diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 169d3fd..a33fcbb 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -70,5 +70,14 @@ end end end + factory :job + + factory :log do + job_id { 1 } + content { 'some log content' } + removed_by { 1 } + archiving { false } + archive_verified { true } + end end From d74f3083962bcee7c12f8a5f901db1daa718049e Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 14:42:05 +0200 Subject: [PATCH 44/84] travis fix --- .travis.yml | 5 ++++- config/database.yml | 3 +++ lib/config.rb | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11831e1..104c991 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,4 +30,7 @@ before_script: - psql --version - psql -c 'CREATE DATABASE travis_test;' -U postgres - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres - - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test \ No newline at end of file + - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test + - psql -c 'CREATE DATABASE travis_test_destination;' -U postgres + - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres + - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test_destination \ No newline at end of file diff --git a/config/database.yml b/config/database.yml index d9a9f06..1fd47de 100644 --- a/config/database.yml +++ b/config/database.yml @@ -18,3 +18,6 @@ test: <<: *default url: 'postgresql://localhost/travis_test' eager_load: false + destination: + url: 'postgresql://localhost/travis_test_destination' + eager_load: false diff --git a/lib/config.rb b/lib/config.rb index 92a24ac..b1a1349 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -92,7 +92,7 @@ def set_values(args) args[:destination_db_url], argv_opts[:destination_db_url], ENV['BACKUP_DESTINATION_DB_URL'], - config.dig('backup', 'destination_db_url') + connection_details.dig(ENV['RAILS_ENV'], 'destination') ) end From 8492d27ea465cf7f9b9862ce6810eff33451eaf3 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 15:02:57 +0200 Subject: [PATCH 45/84] small fix and v0.0.3 defined --- lib/config.rb | 2 +- travis-backup.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index b1a1349..7d1cf8c 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -97,7 +97,7 @@ def set_values(args) end def check_values - if !@threshold + if !@move_logs && !@threshold message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " + "Threshold defines number of months from now.") abort message diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 22203a1..516e34c 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.0.2' + s.version = '0.0.3' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From 0bcfdc04bfcfd050ae858389cc67fd7f8ee829ae Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 15:08:54 +0200 Subject: [PATCH 46/84] fix --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f8b61a5..dc75f20 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.0.2) + travis-backup (0.0.3) activerecord bootsnap pg From 386d23b9700a5927a9d2eff35a37bce58633ab28 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 21:07:00 +0200 Subject: [PATCH 47/84] models for removing orphans --- lib/models/branch.rb | 7 +++++++ lib/models/commit.rb | 7 +++++++ lib/models/cron.rb | 7 +++++++ lib/models/pull_request.rb | 7 +++++++ lib/models/request.rb | 7 +++++++ lib/models/ssl_keys.rb | 7 +++++++ lib/models/stage.rb | 7 +++++++ lib/models/tag.rb | 7 +++++++ 8 files changed, 56 insertions(+) create mode 100644 lib/models/branch.rb create mode 100644 lib/models/commit.rb create mode 100644 lib/models/cron.rb create mode 100644 lib/models/pull_request.rb create mode 100644 lib/models/request.rb create mode 100644 lib/models/ssl_keys.rb create mode 100644 lib/models/stage.rb create mode 100644 lib/models/tag.rb diff --git a/lib/models/branch.rb b/lib/models/branch.rb new file mode 100644 index 0000000..ed099f7 --- /dev/null +++ b/lib/models/branch.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Branch < Model + self.table_name = 'branches' +end diff --git a/lib/models/commit.rb b/lib/models/commit.rb new file mode 100644 index 0000000..0cf4b0d --- /dev/null +++ b/lib/models/commit.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Commit < Model + self.table_name = 'commits' +end diff --git a/lib/models/cron.rb b/lib/models/cron.rb new file mode 100644 index 0000000..cc8ef31 --- /dev/null +++ b/lib/models/cron.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Cron < Model + self.table_name = 'crons' +end diff --git a/lib/models/pull_request.rb b/lib/models/pull_request.rb new file mode 100644 index 0000000..a9b5553 --- /dev/null +++ b/lib/models/pull_request.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class PullRequest < Model + self.table_name = 'pull_requests' +end diff --git a/lib/models/request.rb b/lib/models/request.rb new file mode 100644 index 0000000..9ed9d97 --- /dev/null +++ b/lib/models/request.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Request < Model + self.table_name = 'requests' +end diff --git a/lib/models/ssl_keys.rb b/lib/models/ssl_keys.rb new file mode 100644 index 0000000..0e0c7bb --- /dev/null +++ b/lib/models/ssl_keys.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class SslKey < Model + self.table_name = 'ssl_keys' +end diff --git a/lib/models/stage.rb b/lib/models/stage.rb new file mode 100644 index 0000000..38a675e --- /dev/null +++ b/lib/models/stage.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Stage < Model + self.table_name = 'stages' +end diff --git a/lib/models/tag.rb b/lib/models/tag.rb new file mode 100644 index 0000000..9d6f9e7 --- /dev/null +++ b/lib/models/tag.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'models/model' + +class Tag < Model + self.table_name = 'tags' +end From 4d4d7cd0af8ac0bd7bf21aec4e8c7fdc2034bfbd Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 31 Aug 2021 22:05:46 +0200 Subject: [PATCH 48/84] remove_orphans for repositories without current_build_id - first naive solution --- lib/travis-backup.rb | 7 +++++++ spec/backup_spec.rb | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 6613fab..b80c9b6 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -78,6 +78,13 @@ def move_logs end end + def remove_orphans + repositories = Repository.find_by_sql( + 'select * from repositories where current_build_id is not null and current_build_id not in (select id from builds);' + ) + repositories.each(&:delete) + end + def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index af86b77..707ee82 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -65,6 +65,37 @@ def destination_logs_size end end + describe 'remove_orphans' do + let!(:repositories) { + 4000.times do + FactoryBot.create(:repository) + end + } + let!(:orphan_repositories) { + ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') + 4000.times do + FactoryBot.create( + :repository, + current_build_id: 2000000 + ) + end + ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + } + let!(:builds) { + 4000.times do + FactoryBot.create(:build) + end + } + it 'removes orphaned repositories' do + expect { + start = Time.now + backup.remove_orphans + removing_time = Time.now - start + puts removing_time + }.to change { Repository.all.size }.by -4000 + end + end + describe 'run' do let!(:unassigned_repositories) { (1..3).to_a.map do From 68947b06d8fde5ea459e93cd2a9f0e24cfc49639 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 1 Sep 2021 13:15:41 +0200 Subject: [PATCH 49/84] time test --- lib/travis-backup.rb | 33 ++++++++++++++++++++++++-- spec/backup_spec.rb | 50 +++++++++++++++++++++++++++------------ spec/support/factories.rb | 4 ++++ 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index b80c9b6..91e3653 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -80,9 +80,38 @@ def move_logs def remove_orphans repositories = Repository.find_by_sql( - 'select * from repositories where current_build_id is not null and current_build_id not in (select id from builds);' + "select r.* from repositories r left join builds b on r.current_build_id = b.id where (r.current_build_id is not null) and (b.id is null);" ) - repositories.each(&:delete) + for_delete = repositories.map(&:id) + Repository.where(id: for_delete).delete_all + end + + def remove_orphans2 + build_ids = {} + Build.pluck(:id).each do |id| + build_ids[id] = true + end + + repositories = Repository.find_by_sql( + "select * from repositories where current_build_id is not null;" + ) + for_delete = [] + repositories.each do |repo| + for_delete.push(repo.id) if !build_ids[repo.current_build_id] + end + Repository.where(id: for_delete).delete_all + end + + def remove_orphans3 + repositories = Repository.find_by_sql( + "select * from repositories where current_build_id is not null and current_build_id not in (select id from builds);" + ) + for_delete = repositories.map(&:id) + Repository.where(id: for_delete).delete_all + end + + def remove_orphans4 + ActiveRecord::Base.connection.execute("delete from repositories where current_build_id is not null and current_build_id not in (select id from builds);") end def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 707ee82..1cf61a8 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -67,32 +67,52 @@ def destination_logs_size describe 'remove_orphans' do let!(:repositories) { - 4000.times do - FactoryBot.create(:repository) - end + FactoryBot.create_list(:repository, 100_000) + } + let!(:repositories_with_builds) { + FactoryBot.create_list(:repository_with_builds, 100_000) } let!(:orphan_repositories) { ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - 4000.times do - FactoryBot.create( - :repository, - current_build_id: 2000000 - ) - end + FactoryBot.create_list(:orphan_repository, 10000) ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') } - let!(:builds) { - 4000.times do - FactoryBot.create(:build) - end - } it 'removes orphaned repositories' do expect { start = Time.now backup.remove_orphans removing_time = Time.now - start + puts 1 + puts removing_time + + ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') + FactoryBot.create_list(:orphan_repository, 10000) + ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + start = Time.now + backup.remove_orphans2 + removing_time = Time.now - start + puts 2 puts removing_time - }.to change { Repository.all.size }.by -4000 + + ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') + FactoryBot.create_list(:orphan_repository, 10000) + ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + start = Time.now + backup.remove_orphans3 + removing_time = Time.now - start + puts 3 + puts removing_time + + ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') + FactoryBot.create_list(:orphan_repository, 10000) + ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + start = Time.now + backup.remove_orphans4 + removing_time = Time.now - start + puts 4 + puts removing_time + + }.to change { Repository.all.size }.by -10000 end end diff --git a/spec/support/factories.rb b/spec/support/factories.rb index a33fcbb..afe4ee9 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -50,6 +50,10 @@ ) end end + + factory :orphan_repository do + current_build_id { 2_000_000_000 } + end end factory :build do From 7af0ec5eb5cbde221ab8e0a02543d18e922e8607 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 1 Sep 2021 13:17:49 +0200 Subject: [PATCH 50/84] remove orphans optimized --- lib/travis-backup.rb | 28 ---------------------------- spec/backup_spec.rb | 40 ++++------------------------------------ 2 files changed, 4 insertions(+), 64 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 91e3653..dec3e54 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -86,34 +86,6 @@ def remove_orphans Repository.where(id: for_delete).delete_all end - def remove_orphans2 - build_ids = {} - Build.pluck(:id).each do |id| - build_ids[id] = true - end - - repositories = Repository.find_by_sql( - "select * from repositories where current_build_id is not null;" - ) - for_delete = [] - repositories.each do |repo| - for_delete.push(repo.id) if !build_ids[repo.current_build_id] - end - Repository.where(id: for_delete).delete_all - end - - def remove_orphans3 - repositories = Repository.find_by_sql( - "select * from repositories where current_build_id is not null and current_build_id not in (select id from builds);" - ) - for_delete = repositories.map(&:id) - Repository.where(id: for_delete).delete_all - end - - def remove_orphans4 - ActiveRecord::Base.connection.execute("delete from repositories where current_build_id is not null and current_build_id not in (select id from builds);") - end - def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 1cf61a8..86d6339 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -67,52 +67,20 @@ def destination_logs_size describe 'remove_orphans' do let!(:repositories) { - FactoryBot.create_list(:repository, 100_000) + FactoryBot.create_list(:repository, 100) } let!(:repositories_with_builds) { - FactoryBot.create_list(:repository_with_builds, 100_000) + FactoryBot.create_list(:repository_with_builds, 100) } let!(:orphan_repositories) { ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:orphan_repository, 10000) + FactoryBot.create_list(:orphan_repository, 10) ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') } it 'removes orphaned repositories' do expect { - start = Time.now backup.remove_orphans - removing_time = Time.now - start - puts 1 - puts removing_time - - ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:orphan_repository, 10000) - ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') - start = Time.now - backup.remove_orphans2 - removing_time = Time.now - start - puts 2 - puts removing_time - - ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:orphan_repository, 10000) - ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') - start = Time.now - backup.remove_orphans3 - removing_time = Time.now - start - puts 3 - puts removing_time - - ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:orphan_repository, 10000) - ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') - start = Time.now - backup.remove_orphans4 - removing_time = Time.now - start - puts 4 - puts removing_time - - }.to change { Repository.all.size }.by -10000 + }.to change { Repository.all.size }.by -10 end end From 5562d090f123a9237e72aa1ff33e23e331bf7099 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 1 Sep 2021 15:06:47 +0200 Subject: [PATCH 51/84] small refactoring --- lib/travis-backup.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index dec3e54..cf63fa8 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -79,9 +79,15 @@ def move_logs end def remove_orphans - repositories = Repository.find_by_sql( - "select r.* from repositories r left join builds b on r.current_build_id = b.id where (r.current_build_id is not null) and (b.id is null);" - ) + repositories = Repository.find_by_sql(%{ + select r.* + from repositories r + left join builds b + on r.current_build_id = b.id + where + r.current_build_id is not null + and b.id is null; + }) for_delete = repositories.map(&:id) Repository.where(id: for_delete).delete_all end From bd84abd7a0c529bd6cc81f4fe1168bafc2906bd7 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 1 Sep 2021 21:29:12 +0200 Subject: [PATCH 52/84] removing logs with jobs --- lib/models/build.rb | 2 +- lib/models/job.rb | 2 + lib/models/log.rb | 2 + lib/travis-backup.rb | 23 +- spec/backup_spec.rb | 543 ++++++++++++++++++++++++-------------- spec/support/factories.rb | 35 ++- 6 files changed, 401 insertions(+), 206 deletions(-) diff --git a/lib/models/build.rb b/lib/models/build.rb index bfec5f6..dc4f04f 100644 --- a/lib/models/build.rb +++ b/lib/models/build.rb @@ -7,7 +7,7 @@ # Build model class Build < Model belongs_to :repository - has_many :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :delete_all, class_name: 'Job' + has_many :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job' self.table_name = 'builds' end diff --git a/lib/models/job.rb b/lib/models/job.rb index 7b11f88..10dbba3 100644 --- a/lib/models/job.rb +++ b/lib/models/job.rb @@ -2,12 +2,14 @@ require 'models/model' require 'models/repository' +require 'models/log' # Job model class Job < Model self.inheritance_column = :_type_disabled belongs_to :repository + has_many :logs, -> { order('id') }, foreign_key: :job_id, dependent: :destroy, class_name: 'Log' self.table_name = 'jobs' end diff --git a/lib/models/log.rb b/lib/models/log.rb index 165f223..bbb5f4c 100644 --- a/lib/models/log.rb +++ b/lib/models/log.rb @@ -3,5 +3,7 @@ require 'models/model' class Log < Model + belongs_to :job + self.table_name = 'logs' end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 6613fab..928e3e4 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -9,12 +9,13 @@ # main travis-backup class class Backup attr_accessor :config + attr_reader :dry_run_removed def initialize(config_args={}) @config = Config.new(config_args) if @config.dry_run - @dry_run_removed = {builds: [], jobs: []} + @dry_run_removed = {builds: [], jobs: [], logs: []} end connect_db @@ -58,6 +59,7 @@ def run(args={}) puts 'Dry run active. The following data would be removed in normal mode:' puts " - builds: #{@dry_run_removed[:builds].to_json}" puts " - jobs: #{@dry_run_removed[:jobs].to_json}" + puts " - logs: #{@dry_run_removed[:logs].to_json}" end end @@ -107,10 +109,20 @@ def destroy_batch(builds_batch) def destroy_batch_dry(builds_batch) @dry_run_removed[:builds].concat(builds_batch.map(&:id)) + jobs = builds_batch.map do |build| build.jobs.map(&:id) || [] end.flatten + @dry_run_removed[:jobs].concat(jobs) + + logs = builds_batch.map do |build| + build.jobs.map do |job| + job.logs.map(&:id) || [] + end.flatten || [] + end.flatten + + @dry_run_removed[:logs].concat(logs) end def save_file(file_name, content) # rubocop:disable Metrics/MethodLength @@ -149,8 +161,17 @@ def export_builds(builds) def export_jobs(jobs) jobs.map do |job| job_export = job.attributes + job_export[:logs] = export_logs(job.logs) job_export end end + + def export_logs(logs) + logs.map do |log| + log_export = log.attributes + + log_export + end + end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index af86b77..11f9381 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -17,14 +17,27 @@ system("psql '#{config.destination_db_url}' -f db/schema.sql > /dev/null") if config.destination_db_url end + after(:each) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + Log.destroy_all + end + let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location, limit: 5) } describe 'move_logs' do let!(:logs) { - (1..10).to_a.map do - FactoryBot.create(:log) - end + FactoryBot.create_list( + :log, + 10, + job_id: 1, + content: 'some log content', + removed_by: 1, + archiving: false, + archive_verified: true + ) } def connect_db(url) @@ -67,9 +80,7 @@ def destination_logs_size describe 'run' do let!(:unassigned_repositories) { - (1..3).to_a.map do - FactoryBot.create(:repository) - end + FactoryBot.create_list(:repository, 3) } let!(:user1) { FactoryBot.create(:user_with_repos) @@ -144,15 +155,25 @@ def destination_logs_size backup.run end end - end - describe 'process_repo' do - after(:each) do - Repository.destroy_all - Build.destroy_all - Job.destroy_all + context 'when dry run mode is on' do + let!(:backup) { Backup.new(files_location: files_location, limit: 10, dry_run: true, threshold: 0) } + + before do + allow_any_instance_of(IO).to receive(:puts) + end + + it 'prepares proper dry run report' do + backup.run + expect(backup.dry_run_removed[:builds].size).to eql 24 + expect(backup.dry_run_removed[:jobs].size).to eql 48 + expect(backup.dry_run_removed[:logs].size).to eql 96 + end end - + + end + + describe 'process_repo' do let!(:config) { Config.new } let(:datetime) { (config.threshold + 1).months.ago.to_time.utc } let(:org_id) { rand(100000) } @@ -179,179 +200,299 @@ def destination_logs_size let!(:exported_object) { [[ { - "id"=>repository.builds.first.id, - "repository_id"=>repository.id, - "number"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "config"=>nil, - "commit_id"=>nil, - "request_id"=>nil, - "state"=>nil, - "duration"=>nil, - "owner_id"=>nil, - "owner_type"=>nil, - "event_type"=>nil, - "previous_state"=>nil, - "pull_request_title"=>nil, - "pull_request_number"=>nil, - "branch"=>nil, - "canceled_at"=>nil, - "cached_matrix_ids"=>nil, - "received_at"=>nil, - "private"=>nil, - "pull_request_id"=>nil, - "branch_id"=>nil, - "tag_id"=>nil, - "sender_id"=>nil, - "sender_type"=>nil, - :jobs=>[{ - "id"=>repository.builds.first.jobs.first.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>repository.builds.first.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "config"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "stage_number"=>nil, - "stage_id"=>nil + "id": repository.builds.first.id, + "repository_id": repository.id, + "number": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "config": nil, + "commit_id": nil, + "request_id": nil, + "state": nil, + "duration": nil, + "owner_id": nil, + "owner_type": nil, + "event_type": nil, + "previous_state": nil, + "pull_request_title": nil, + "pull_request_number": nil, + "branch": nil, + "canceled_at": nil, + "cached_matrix_ids": nil, + "received_at": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil, + "jobs": [{ + "id": repository.builds.first.jobs.first.id, + "repository_id": repository.id, + "commit_id": nil, + "source_id": repository.builds.first.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": repository.builds.first.jobs.first.logs.first.id, + "job_id": repository.builds.first.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": repository.builds.first.jobs.first.logs.second.id, + "job_id": repository.builds.first.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] }, { - "id"=>repository.builds.first.jobs.second.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>repository.builds.first.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "config"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "stage_number"=>nil, - "stage_id"=>nil + "id": repository.builds.first.jobs.second.id, + "repository_id": repository.id, + "commit_id": nil, + "source_id": repository.builds.first.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": repository.builds.first.jobs.second.logs.first.id, + "job_id": repository.builds.first.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": repository.builds.first.jobs.second.logs.second.id, + "job_id": repository.builds.first.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] }] }, { - "id"=>repository.builds.second.id, - "repository_id"=>repository.id, - "number"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "config"=>nil, - "commit_id"=>nil, - "request_id"=>nil, - "state"=>nil, - "duration"=>nil, - "owner_id"=>nil, - "owner_type"=>nil, - "event_type"=>nil, - "previous_state"=>nil, - "pull_request_title"=>nil, - "pull_request_number"=>nil, - "branch"=>nil, - "canceled_at"=>nil, - "cached_matrix_ids"=>nil, - "received_at"=>nil, - "private"=>nil, - "pull_request_id"=>nil, - "branch_id"=>nil, - "tag_id"=>nil, - "sender_id"=>nil, - "sender_type"=>nil, - :jobs=>[{ - "id"=>repository.builds.second.jobs.first.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>repository.builds.second.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "config"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "stage_number"=>nil, - "stage_id"=>nil + "id": repository.builds.second.id, + "repository_id": repository.id, + "number": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "config": nil, + "commit_id": nil, + "request_id": nil, + "state": nil, + "duration": nil, + "owner_id": nil, + "owner_type": nil, + "event_type": nil, + "previous_state": nil, + "pull_request_title": nil, + "pull_request_number": nil, + "branch": nil, + "canceled_at": nil, + "cached_matrix_ids": nil, + "received_at": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil, + "jobs": [{ + "id": repository.builds.second.jobs.first.id, + "repository_id": repository.id, + "commit_id": nil, + "source_id": repository.builds.second.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": repository.builds.second.jobs.first.logs.first.id, + "job_id": repository.builds.second.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": repository.builds.second.jobs.first.logs.second.id, + "job_id": repository.builds.second.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] }, { - "id"=>repository.builds.second.jobs.second.id, - "repository_id"=>repository.id, - "commit_id"=>nil, - "source_id"=>repository.builds.second.id, - "source_type"=>"Build", - "queue"=>nil, - "type"=>nil, - "state"=>nil, - "number"=>nil, - "config"=>nil, - "worker"=>nil, - "started_at"=>nil, - "finished_at"=>nil, - "created_at"=>datetime, - "updated_at"=>datetime, - "tags"=>nil, - "allow_failure"=>false, - "owner_id"=>nil, - "owner_type"=>nil, - "result"=>nil, - "queued_at"=>nil, - "canceled_at"=>nil, - "received_at"=>nil, - "debug_options"=>nil, - "private"=>nil, - "stage_number"=>nil, - "stage_id"=>nil + "id": repository.builds.second.jobs.second.id, + "repository_id": repository.id, + "commit_id": nil, + "source_id": repository.builds.second.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": datetime, + "updated_at": datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": repository.builds.second.jobs.second.logs.first.id, + "job_id": repository.builds.second.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": repository.builds.second.jobs.second.logs.second.id, + "job_id": repository.builds.second.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": datetime, + "updated_at": datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] }] } ]] @@ -364,23 +505,43 @@ def destination_logs_size end it 'should delete all jobs of removed builds and leave the rest' do - backup.process_repo(repository) + expect { + backup.process_repo(repository) + }.to change { Job.all.size }.by -4 + build_id = Build.first.id expect(Job.all.map(&:source_id)).to eq([build_id, build_id]) end + + it 'should delete all logs of removed jobs and leave the rest' do + expect { + backup.process_repo(repository) + }.to change { Log.all.size }.by -8 + + build_id = Build.first.id + expect(Log.all.map(&:job).map(&:source_id)).to eq(Array.new(4, build_id)) + end end context 'when if_backup config is set to true' do it 'should prepare proper JSON export' do - build_export = backup.process_repo(repository) - build_export.first.first['updated_at'] = datetime - build_export.first.second['updated_at'] = datetime - build_export.first.first[:jobs].first['updated_at'] = datetime - build_export.first.first[:jobs].second['updated_at'] = datetime - build_export.first.second[:jobs].first['updated_at'] = datetime - build_export.first.second[:jobs].second['updated_at'] = datetime - - expect(build_export.to_json).to eq(exported_object.to_json) + result = backup.process_repo(repository) + result.first.first['updated_at'] = datetime + result.first.second['updated_at'] = datetime + result.first.first[:jobs].first['updated_at'] = datetime + result.first.first[:jobs].second['updated_at'] = datetime + result.first.second[:jobs].first['updated_at'] = datetime + result.first.second[:jobs].second['updated_at'] = datetime + result.first.first[:jobs].first[:logs].first['updated_at'] = datetime + result.first.first[:jobs].first[:logs].second['updated_at'] = datetime + result.first.first[:jobs].second[:logs].first['updated_at'] = datetime + result.first.first[:jobs].second[:logs].second['updated_at'] = datetime + result.first.second[:jobs].first[:logs].first['updated_at'] = datetime + result.first.second[:jobs].first[:logs].second['updated_at'] = datetime + result.first.second[:jobs].second[:logs].first['updated_at'] = datetime + result.first.second[:jobs].second[:logs].second['updated_at'] = datetime + + expect(result.to_json).to eq(exported_object.to_json) end it 'should save JSON to file at proper path' do @@ -415,10 +576,6 @@ def destination_logs_size context 'when dry_run config is set to true' do let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } - before do - allow_any_instance_of(IO).to receive(:puts) - end - it 'should not save JSON to file' do expect(File).not_to receive(:open) backup.process_repo(repository) diff --git a/spec/support/factories.rb b/spec/support/factories.rb index a33fcbb..5498086 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -10,7 +10,7 @@ end after(:create) do |organization, evaluator| create_list( - :repository, + :repository_with_builds, evaluator.repos_count, owner_id: organization.id, owner_type: 'Organization' @@ -26,7 +26,7 @@ end after(:create) do |user, evaluator| create_list( - :repository, + :repository_with_builds, evaluator.repos_count, owner_id: user.id, owner_type: 'User' @@ -59,7 +59,7 @@ end after(:create) do |build, evaluator| create_list( - :job, + :job_with_logs, evaluator.jobs_count, repository: build.repository, source_type: 'Build', @@ -71,13 +71,26 @@ end end - factory :job - - factory :log do - job_id { 1 } - content { 'some log content' } - removed_by { 1 } - archiving { false } - archive_verified { true } + factory :job do + factory :job_with_logs do + transient do + logs_count { 2 } + end + after(:create) do |job, evaluator| + create_list( + :log, + evaluator.logs_count, + job_id: job.id, + content: 'some log content', + removed_by: nil, + archiving: false, + archive_verified: true, + created_at: job.created_at, + updated_at: job.updated_at + ) + end + end end + + factory :log end From 0c1683c287358174cb7f53106be131bcb51e5804 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 3 Sep 2021 13:46:16 +0200 Subject: [PATCH 53/84] remove_orphans for all needed tables --- lib/models/{ssl_keys.rb => ssl_key.rb} | 0 lib/travis-backup.rb | 52 +++++- spec/backup_spec.rb | 130 ++++++++++++-- spec/support/factories.rb | 240 ++++++++++++++++++++++++- 4 files changed, 399 insertions(+), 23 deletions(-) rename lib/models/{ssl_keys.rb => ssl_key.rb} (100%) diff --git a/lib/models/ssl_keys.rb b/lib/models/ssl_key.rb similarity index 100% rename from lib/models/ssl_keys.rb rename to lib/models/ssl_key.rb diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index cf63fa8..2e4a778 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -5,6 +5,14 @@ require 'config' require 'models/repository' require 'models/log' +require 'models/branch' +require 'models/tag' +require 'models/commit' +require 'models/cron' +require 'models/pull_request' +require 'models/ssl_key' +require 'models/request' +require 'models/stage' # main travis-backup class class Backup @@ -79,17 +87,45 @@ def move_logs end def remove_orphans - repositories = Repository.find_by_sql(%{ - select r.* - from repositories r - left join builds b - on r.current_build_id = b.id + remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') + remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') + remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id') + remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id') + remove_orphans_for_table(Build, 'builds', 'requests', 'request_id') + remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id') + remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id') + remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id') + remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id') + remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id') + remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id') + remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id') + remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id') + remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id') + remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id') + remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id') + remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id') + remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id') + remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id') + remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id') + remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id') + remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id') + remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id') + remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id') + remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id') + remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id') + end + + def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) + for_delete = model_class.find_by_sql(%{ + select a.* + from #{table_a_name} a + left join #{table_b_name} b + on a.#{fk_name} = b.id where - r.current_build_id is not null + a.#{fk_name} is not null and b.id is null; }) - for_delete = repositories.map(&:id) - Repository.where(id: for_delete).delete_all + model_class.where(id: for_delete.map(&:id)).delete_all end def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 86d6339..c43acd3 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -1,11 +1,11 @@ $: << 'lib' require 'uri' require 'travis-backup' -require 'models/repository' require 'models/build' require 'models/job' require 'models/organization' require 'models/user' + require 'support/factories' require 'pry' @@ -66,21 +66,127 @@ def destination_logs_size end describe 'remove_orphans' do - let!(:repositories) { - FactoryBot.create_list(:repository, 100) - } - let!(:repositories_with_builds) { - FactoryBot.create_list(:repository_with_builds, 100) - } - let!(:orphan_repositories) { + let!(:data) { ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:orphan_repository, 10) + FactoryBot.create_list(:repository, 2) + FactoryBot.create_list(:build, 2) + FactoryBot.create_list(:job, 2) + FactoryBot.create_list(:branch, 2) + FactoryBot.create_list(:tag, 2) + FactoryBot.create_list(:commit, 2) + FactoryBot.create_list(:cron, 2) + FactoryBot.create_list(:pull_request, 2) + FactoryBot.create_list(:request, 2) + FactoryBot.create_list(:stage, 2) + FactoryBot.create_list(:orphan_repository_on_current_build_id, 2) + FactoryBot.create_list(:repository_with_current_build_id, 2) + FactoryBot.create_list(:orphan_repository_on_last_build_id, 2) + FactoryBot.create_list(:repository_with_last_build_id, 2) + FactoryBot.create_list(:orphan_build_on_repository_id, 2) + FactoryBot.create_list(:build_with_repository_id, 2) + FactoryBot.create_list(:orphan_build_on_commit_id, 2) + FactoryBot.create_list(:build_with_commit_id, 2) + FactoryBot.create_list(:orphan_build_on_request_id, 2) + FactoryBot.create_list(:build_with_request_id, 2) + FactoryBot.create_list(:orphan_build_on_pull_request_id, 2) + FactoryBot.create_list(:build_with_pull_request_id, 2) + FactoryBot.create_list(:orphan_build_on_branch_id, 2) + FactoryBot.create_list(:build_with_branch_id, 2) + FactoryBot.create_list(:orphan_build_on_tag_id, 2) + FactoryBot.create_list(:build_with_tag_id, 2) + FactoryBot.create_list(:orphan_job_on_repository_id, 2) + FactoryBot.create_list(:job_with_repository_id, 2) + FactoryBot.create_list(:orphan_job_on_commit_id, 2) + FactoryBot.create_list(:job_with_commit_id, 2) + FactoryBot.create_list(:orphan_job_on_stage_id, 2) + FactoryBot.create_list(:job_with_stage_id, 2) + FactoryBot.create_list(:orphan_branch_on_repository_id, 2) + FactoryBot.create_list(:orphan_branch_on_last_build_id, 2) + FactoryBot.create_list(:branch_with_last_build_id, 2) + FactoryBot.create_list(:orphan_tag_on_repository_id, 2) + FactoryBot.create_list(:tag_with_repository_id, 2) + FactoryBot.create_list(:orphan_tag_on_last_build_id, 2) + FactoryBot.create_list(:tag_with_last_build_id, 2) + FactoryBot.create_list(:orphan_commit_on_repository_id, 2) + FactoryBot.create_list(:commit_with_repository_id, 2) + FactoryBot.create_list(:orphan_commit_on_branch_id, 2) + FactoryBot.create_list(:commit_with_branch_id, 2) + FactoryBot.create_list(:orphan_commit_on_tag_id, 2) + FactoryBot.create_list(:commit_with_tag_id, 2) + FactoryBot.create_list(:orphan_cron_on_branch_id, 2) + FactoryBot.create_list(:cron_with_branch_id, 2) + FactoryBot.create_list(:orphan_pull_request_on_repository_id, 2) + FactoryBot.create_list(:pull_request_with_repository_id, 2) + FactoryBot.create_list(:orphan_request_on_commit_id, 2) + FactoryBot.create_list(:request_with_commit_id, 2) + FactoryBot.create_list(:orphan_request_on_pull_request_id, 2) + FactoryBot.create_list(:request_with_pull_request_id, 2) + FactoryBot.create_list(:orphan_request_on_branch_id, 2) + FactoryBot.create_list(:request_with_branch_id, 2) + FactoryBot.create_list(:orphan_request_on_tag_id, 2) + FactoryBot.create_list(:request_with_tag_id, 2) + FactoryBot.create_list(:orphan_stage_on_build_id, 2) + FactoryBot.create_list(:stage_with_build_id, 2) ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') } it 'removes orphaned repositories' do expect { backup.remove_orphans - }.to change { Repository.all.size }.by -10 + }.to change { Repository.all.size }.by -4 + end + + it 'removes orphaned builds' do + expect { + backup.remove_orphans + }.to change { Build.all.size }.by -12 + end + + it 'removes orphaned jobs' do + expect { + backup.remove_orphans + }.to change { Job.all.size }.by -6 + end + + it 'removes orphaned branches' do + expect { + backup.remove_orphans + }.to change { Branch.all.size }.by -4 + end + + it 'removes orphaned tags' do + expect { + backup.remove_orphans + }.to change { Tag.all.size }.by -4 + end + + it 'removes orphaned commits' do + expect { + backup.remove_orphans + }.to change { Commit.all.size }.by -6 + end + + it 'removes orphaned crons' do + expect { + backup.remove_orphans + }.to change { Cron.all.size }.by -2 + end + + it 'removes orphaned pull requests' do + expect { + backup.remove_orphans + }.to change { PullRequest.all.size }.by -2 + end + + it 'removes orphaned requests' do + expect { + backup.remove_orphans + }.to change { Request.all.size }.by -8 + end + + it 'removes orphaned stages' do + expect { + backup.remove_orphans + }.to change { Stage.all.size }.by -2 end end @@ -180,14 +286,14 @@ def destination_logs_size let(:private_com_id) { rand(100000) } let!(:repository) { FactoryBot.create( - :repository_with_builds, + :repository_with_builds_and_jobs, created_at: datetime, updated_at: datetime ) } let!(:repository2) { FactoryBot.create( - :repository_with_builds, + :repository_with_builds_and_jobs, created_at: datetime, updated_at: datetime, builds_count: 1 diff --git a/spec/support/factories.rb b/spec/support/factories.rb index afe4ee9..4580ae5 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -36,7 +36,7 @@ end factory :repository do - factory :repository_with_builds do + factory :repository_with_builds_and_jobs do transient do builds_count { 2 } end @@ -51,9 +51,36 @@ end end - factory :orphan_repository do + factory :repository_with_builds do + transient do + builds_count { 2 } + end + after(:create) do |repository, evaluator| + create_list( + :build, + evaluator.builds_count, + repository: repository, + created_at: repository.created_at, + updated_at: repository.updated_at + ) + end + end + + factory :orphan_repository_on_current_build_id do current_build_id { 2_000_000_000 } end + + factory :repository_with_current_build_id do + current_build_id { Build.first.id } + end + + factory :orphan_repository_on_last_build_id do + last_build_id { 2_000_000_000 } + end + + factory :repository_with_last_build_id do + last_build_id { Build.first.id } + end end factory :build do @@ -73,9 +100,81 @@ ) end end + + factory :orphan_build_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :build_with_repository_id do + repository_id { Repository.first.id } + end + + factory :orphan_build_on_commit_id do + commit_id { 2_000_000_000 } + end + + factory :build_with_commit_id do + commit_id { Commit.first.id } + end + + factory :orphan_build_on_request_id do + request_id { 2_000_000_000 } + end + + factory :build_with_request_id do + request_id { Request.first.id } + end + + factory :orphan_build_on_pull_request_id do + pull_request_id { 2_000_000_000 } + end + + factory :build_with_pull_request_id do + pull_request_id { PullRequest.first.id } + end + + factory :orphan_build_on_branch_id do + branch_id { 2_000_000_000 } + end + + factory :build_with_branch_id do + branch_id { Branch.first.id } + end + + factory :orphan_build_on_tag_id do + tag_id { 2_000_000_000 } + end + + factory :build_with_tag_id do + tag_id { Tag.first.id } + end end - factory :job + factory :job do + factory :orphan_job_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :job_with_repository_id do + repository_id { Repository.first.id } + end + + factory :orphan_job_on_commit_id do + commit_id { 2_000_000_000 } + end + + factory :job_with_commit_id do + commit_id { Commit.first.id } + end + + factory :orphan_job_on_stage_id do + stage_id { 2_000_000_000 } + end + + factory :job_with_stage_id do + stage_id { Stage.first.id } + end + end factory :log do job_id { 1 } @@ -84,4 +183,139 @@ archiving { false } archive_verified { true } end + + factory :branch do + name { "branch_#{Time.now.to_f}" } + repository_id { 1 } + factory :orphan_branch_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :orphan_branch_on_last_build_id do + last_build_id { 2_000_000_000 } + end + + factory :branch_with_last_build_id do + last_build_id { Build.first.id } + end + end + + factory :tag do + factory :orphan_tag_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :tag_with_repository_id do + repository_id { Repository.first.id } + end + + factory :orphan_tag_on_last_build_id do + last_build_id { 2_000_000_000 } + end + + factory :tag_with_last_build_id do + last_build_id { Tag.first.id } + end + end + + factory :commit do + factory :orphan_commit_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :commit_with_repository_id do + repository_id { Repository.first.id } + end + + factory :orphan_commit_on_branch_id do + branch_id { 2_000_000_000 } + end + + factory :commit_with_branch_id do + branch_id { Branch.first.id } + end + + factory :orphan_commit_on_tag_id do + tag_id { 2_000_000_000 } + end + + factory :commit_with_tag_id do + tag_id { Tag.first.id } + end + end + + factory :cron do + interval { 'test' } + factory :orphan_cron_on_branch_id do + branch_id { 2_000_000_000 } + end + + factory :cron_with_branch_id do + branch_id { Branch.first.id } + end + end + + factory :pull_request do + factory :orphan_pull_request_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :pull_request_with_repository_id do + repository_id { Repository.first.id } + end + end + + factory :ssl_key do + factory :orphan_ssl_key_on_repository_id do + repository_id { 2_000_000_000 } + end + + factory :ssl_key_with_repository_id do + repository_id { Repository.first.id } + end + end + + factory :request do + factory :orphan_request_on_commit_id do + commit_id { 2_000_000_000 } + end + + factory :request_with_commit_id do + commit_id { Commit.first.id } + end + + factory :orphan_request_on_pull_request_id do + pull_request_id { 2_000_000_000 } + end + + factory :request_with_pull_request_id do + pull_request_id { PullRequest.first.id } + end + + factory :orphan_request_on_branch_id do + branch_id { 2_000_000_000 } + end + + factory :request_with_branch_id do + branch_id { Branch.first.id } + end + + factory :orphan_request_on_tag_id do + tag_id { 2_000_000_000 } + end + + factory :request_with_tag_id do + tag_id { Tag.first.id } + end + end + + factory :stage do + factory :orphan_stage_on_build_id do + build_id { 2_000_000_000 } + end + + factory :stage_with_build_id do + build_id { Build.first.id } + end + end end From f596b887017e69d70dd278fb41390245e8bc1db5 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 3 Sep 2021 13:56:09 +0200 Subject: [PATCH 54/84] remove_orphans in config, small renaming --- lib/config.rb | 8 +++++ lib/travis-backup.rb | 1 + spec/backup_spec.rb | 66 +++++++++++++++++++++++---------------- spec/support/factories.rb | 52 +++++++++++++++--------------- 4 files changed, 74 insertions(+), 53 deletions(-) diff --git a/lib/config.rb b/lib/config.rb index 7d1cf8c..42a9f1e 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -12,6 +12,7 @@ class Config :repo_id, :org_id, :move_logs, + :remove_orphans, :destination_db_url def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength @@ -88,6 +89,13 @@ def set_values(args) config.dig('backup', 'move_logs'), false ) + @remove_orphans = first_not_nil( + args[:remove_orphans], + argv_opts[:remove_orphans], + ENV['BACKUP_REMOVE_ORPHANS'], + config.dig('backup', 'remove_orphans'), + false + ) @destination_db_url = first_not_nil( args[:destination_db_url], argv_opts[:destination_db_url], diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 2e4a778..3f9dd1c 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -34,6 +34,7 @@ def connect_db(url=@config.database_url) def run(args={}) return move_logs if @config.move_logs + return remove_orphans if @config.remove_orphans user_id = args[:user_id] || @config.user_id repo_id = args[:repo_id] || @config.repo_id diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index c43acd3..33915e7 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -78,54 +78,54 @@ def destination_logs_size FactoryBot.create_list(:pull_request, 2) FactoryBot.create_list(:request, 2) FactoryBot.create_list(:stage, 2) - FactoryBot.create_list(:orphan_repository_on_current_build_id, 2) + FactoryBot.create_list(:repository_orphaned_on_current_build_id, 2) FactoryBot.create_list(:repository_with_current_build_id, 2) - FactoryBot.create_list(:orphan_repository_on_last_build_id, 2) + FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) FactoryBot.create_list(:repository_with_last_build_id, 2) - FactoryBot.create_list(:orphan_build_on_repository_id, 2) + FactoryBot.create_list(:build_orphaned_on_repository_id, 2) FactoryBot.create_list(:build_with_repository_id, 2) - FactoryBot.create_list(:orphan_build_on_commit_id, 2) + FactoryBot.create_list(:build_orphaned_on_commit_id, 2) FactoryBot.create_list(:build_with_commit_id, 2) - FactoryBot.create_list(:orphan_build_on_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_request_id, 2) FactoryBot.create_list(:build_with_request_id, 2) - FactoryBot.create_list(:orphan_build_on_pull_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_pull_request_id, 2) FactoryBot.create_list(:build_with_pull_request_id, 2) - FactoryBot.create_list(:orphan_build_on_branch_id, 2) + FactoryBot.create_list(:build_orphaned_on_branch_id, 2) FactoryBot.create_list(:build_with_branch_id, 2) - FactoryBot.create_list(:orphan_build_on_tag_id, 2) + FactoryBot.create_list(:build_orphaned_on_tag_id, 2) FactoryBot.create_list(:build_with_tag_id, 2) - FactoryBot.create_list(:orphan_job_on_repository_id, 2) + FactoryBot.create_list(:job_orphaned_on_repository_id, 2) FactoryBot.create_list(:job_with_repository_id, 2) - FactoryBot.create_list(:orphan_job_on_commit_id, 2) + FactoryBot.create_list(:job_orphaned_on_commit_id, 2) FactoryBot.create_list(:job_with_commit_id, 2) - FactoryBot.create_list(:orphan_job_on_stage_id, 2) + FactoryBot.create_list(:job_orphaned_on_stage_id, 2) FactoryBot.create_list(:job_with_stage_id, 2) - FactoryBot.create_list(:orphan_branch_on_repository_id, 2) - FactoryBot.create_list(:orphan_branch_on_last_build_id, 2) + FactoryBot.create_list(:branch_orphaned_on_repository_id, 2) + FactoryBot.create_list(:branch_orphaned_on_last_build_id, 2) FactoryBot.create_list(:branch_with_last_build_id, 2) - FactoryBot.create_list(:orphan_tag_on_repository_id, 2) + FactoryBot.create_list(:tag_orphaned_on_repository_id, 2) FactoryBot.create_list(:tag_with_repository_id, 2) - FactoryBot.create_list(:orphan_tag_on_last_build_id, 2) + FactoryBot.create_list(:tag_orphaned_on_last_build_id, 2) FactoryBot.create_list(:tag_with_last_build_id, 2) - FactoryBot.create_list(:orphan_commit_on_repository_id, 2) + FactoryBot.create_list(:commit_orphaned_on_repository_id, 2) FactoryBot.create_list(:commit_with_repository_id, 2) - FactoryBot.create_list(:orphan_commit_on_branch_id, 2) + FactoryBot.create_list(:commit_orphaned_on_branch_id, 2) FactoryBot.create_list(:commit_with_branch_id, 2) - FactoryBot.create_list(:orphan_commit_on_tag_id, 2) + FactoryBot.create_list(:commit_orphaned_on_tag_id, 2) FactoryBot.create_list(:commit_with_tag_id, 2) - FactoryBot.create_list(:orphan_cron_on_branch_id, 2) + FactoryBot.create_list(:cron_orphaned_on_branch_id, 2) FactoryBot.create_list(:cron_with_branch_id, 2) - FactoryBot.create_list(:orphan_pull_request_on_repository_id, 2) + FactoryBot.create_list(:pull_request_orphaned_on_repository_id, 2) FactoryBot.create_list(:pull_request_with_repository_id, 2) - FactoryBot.create_list(:orphan_request_on_commit_id, 2) + FactoryBot.create_list(:request_orphaned_on_commit_id, 2) FactoryBot.create_list(:request_with_commit_id, 2) - FactoryBot.create_list(:orphan_request_on_pull_request_id, 2) + FactoryBot.create_list(:request_orphaned_on_pull_request_id, 2) FactoryBot.create_list(:request_with_pull_request_id, 2) - FactoryBot.create_list(:orphan_request_on_branch_id, 2) + FactoryBot.create_list(:request_orphaned_on_branch_id, 2) FactoryBot.create_list(:request_with_branch_id, 2) - FactoryBot.create_list(:orphan_request_on_tag_id, 2) + FactoryBot.create_list(:request_orphaned_on_tag_id, 2) FactoryBot.create_list(:request_with_tag_id, 2) - FactoryBot.create_list(:orphan_stage_on_build_id, 2) + FactoryBot.create_list(:stage_orphaned_on_build_id, 2) FactoryBot.create_list(:stage_with_build_id, 2) ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') } @@ -258,17 +258,29 @@ def destination_logs_size let!(:backup) { Backup.new(files_location: files_location, limit: 5, move_logs: true) } it 'does not process repositories' do - repo = Repository.first expect(backup).not_to receive(:process_repo) backup.run end it 'moves logs' do - repo = Repository.first expect(backup).to receive(:move_logs).once backup.run end end + + context 'when remove orphans mode is on' do + let!(:backup) { Backup.new(files_location: files_location, limit: 5, remove_orphans: true) } + + it 'does not process repositories' do + expect(backup).not_to receive(:process_repo) + backup.run + end + + it 'removes orphans' do + expect(backup).to receive(:remove_orphans).once + backup.run + end + end end describe 'process_repo' do diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 4580ae5..a4c546e 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -66,7 +66,7 @@ end end - factory :orphan_repository_on_current_build_id do + factory :repository_orphaned_on_current_build_id do current_build_id { 2_000_000_000 } end @@ -74,7 +74,7 @@ current_build_id { Build.first.id } end - factory :orphan_repository_on_last_build_id do + factory :repository_orphaned_on_last_build_id do last_build_id { 2_000_000_000 } end @@ -101,7 +101,7 @@ end end - factory :orphan_build_on_repository_id do + factory :build_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -109,7 +109,7 @@ repository_id { Repository.first.id } end - factory :orphan_build_on_commit_id do + factory :build_orphaned_on_commit_id do commit_id { 2_000_000_000 } end @@ -117,7 +117,7 @@ commit_id { Commit.first.id } end - factory :orphan_build_on_request_id do + factory :build_orphaned_on_request_id do request_id { 2_000_000_000 } end @@ -125,7 +125,7 @@ request_id { Request.first.id } end - factory :orphan_build_on_pull_request_id do + factory :build_orphaned_on_pull_request_id do pull_request_id { 2_000_000_000 } end @@ -133,7 +133,7 @@ pull_request_id { PullRequest.first.id } end - factory :orphan_build_on_branch_id do + factory :build_orphaned_on_branch_id do branch_id { 2_000_000_000 } end @@ -141,7 +141,7 @@ branch_id { Branch.first.id } end - factory :orphan_build_on_tag_id do + factory :build_orphaned_on_tag_id do tag_id { 2_000_000_000 } end @@ -151,7 +151,7 @@ end factory :job do - factory :orphan_job_on_repository_id do + factory :job_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -159,7 +159,7 @@ repository_id { Repository.first.id } end - factory :orphan_job_on_commit_id do + factory :job_orphaned_on_commit_id do commit_id { 2_000_000_000 } end @@ -167,7 +167,7 @@ commit_id { Commit.first.id } end - factory :orphan_job_on_stage_id do + factory :job_orphaned_on_stage_id do stage_id { 2_000_000_000 } end @@ -187,11 +187,11 @@ factory :branch do name { "branch_#{Time.now.to_f}" } repository_id { 1 } - factory :orphan_branch_on_repository_id do + factory :branch_orphaned_on_repository_id do repository_id { 2_000_000_000 } end - factory :orphan_branch_on_last_build_id do + factory :branch_orphaned_on_last_build_id do last_build_id { 2_000_000_000 } end @@ -201,7 +201,7 @@ end factory :tag do - factory :orphan_tag_on_repository_id do + factory :tag_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -209,7 +209,7 @@ repository_id { Repository.first.id } end - factory :orphan_tag_on_last_build_id do + factory :tag_orphaned_on_last_build_id do last_build_id { 2_000_000_000 } end @@ -219,7 +219,7 @@ end factory :commit do - factory :orphan_commit_on_repository_id do + factory :commit_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -227,7 +227,7 @@ repository_id { Repository.first.id } end - factory :orphan_commit_on_branch_id do + factory :commit_orphaned_on_branch_id do branch_id { 2_000_000_000 } end @@ -235,7 +235,7 @@ branch_id { Branch.first.id } end - factory :orphan_commit_on_tag_id do + factory :commit_orphaned_on_tag_id do tag_id { 2_000_000_000 } end @@ -246,7 +246,7 @@ factory :cron do interval { 'test' } - factory :orphan_cron_on_branch_id do + factory :cron_orphaned_on_branch_id do branch_id { 2_000_000_000 } end @@ -256,7 +256,7 @@ end factory :pull_request do - factory :orphan_pull_request_on_repository_id do + factory :pull_request_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -266,7 +266,7 @@ end factory :ssl_key do - factory :orphan_ssl_key_on_repository_id do + factory :ssl_key_orphaned_on_repository_id do repository_id { 2_000_000_000 } end @@ -276,7 +276,7 @@ end factory :request do - factory :orphan_request_on_commit_id do + factory :request_orphaned_on_commit_id do commit_id { 2_000_000_000 } end @@ -284,7 +284,7 @@ commit_id { Commit.first.id } end - factory :orphan_request_on_pull_request_id do + factory :request_orphaned_on_pull_request_id do pull_request_id { 2_000_000_000 } end @@ -292,7 +292,7 @@ pull_request_id { PullRequest.first.id } end - factory :orphan_request_on_branch_id do + factory :request_orphaned_on_branch_id do branch_id { 2_000_000_000 } end @@ -300,7 +300,7 @@ branch_id { Branch.first.id } end - factory :orphan_request_on_tag_id do + factory :request_orphaned_on_tag_id do tag_id { 2_000_000_000 } end @@ -310,7 +310,7 @@ end factory :stage do - factory :orphan_stage_on_build_id do + factory :stage_orphaned_on_build_id do build_id { 2_000_000_000 } end From f0f066caebe942734e3a0491685f5d7049e5563f Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 3 Sep 2021 14:38:01 +0200 Subject: [PATCH 55/84] README improved --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 026a6e9..c527f55 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ All arguments: -u, --user_id ID # run only for given user -o, --org_id ID # run only for given organization -r, --repo_id ID # run only for given repository + --move_logs # run in move logs mode - move all logs to database at destination_db_url URL + --destination_db_url URL # URL for moving logs to + --remove_orphans # run in remove orphans mode ``` Or inside your app: @@ -54,6 +57,12 @@ backup.run(org_id: 1) backup.run(repo_id: 1) ``` +#### Special modes + +Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). Running gem in this mode no files are created and no other tables are being touched. + +Using `--remove_orphans` flag you can remove all orphaned data from tables. Running gem in this mode no files are created. + ### Configuration Despite of command line arguments, one of the ways you can configure your export is a file `config/settinigs.yml` that you can place in your app's main directory. The gem expects properties in the following format: From 85877971de7985d30613355cbdc7112f49e4e6c8 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 2 Sep 2021 12:29:43 +0200 Subject: [PATCH 56/84] renaming and tests refactoring --- lib/travis-backup.rb | 20 +- spec/backup_spec.rb | 334 ++------------------------------- spec/support/expected_files.rb | 307 ++++++++++++++++++++++++++++++ 3 files changed, 336 insertions(+), 325 deletions(-) create mode 100644 spec/support/expected_files.rb diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 02066d6..22a66a9 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -53,14 +53,14 @@ def run(args={}) if owner_id Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| - process_repo(repository) + process_repo_builds(repository) end elsif repo_id repository = Repository.find(repo_id) - process_repo(repository) + process_repo_builds(repository) else Repository.order(:id).each do |repository| - process_repo(repository) + process_repo_builds(repository) end end @@ -131,34 +131,34 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) model_class.where(id: for_delete.map(&:id)).delete_all end - def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch) + @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch) end.compact end private - def save_and_destroy_batch(builds_batch, repository) + def save_and_destroy_builds_batch(builds_batch, repository) builds_export = export_builds(builds_batch) file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" pretty_json = JSON.pretty_generate(builds_export) if save_file(file_name, pretty_json) - destroy_batch(builds_batch) + destroy_builds_batch(builds_batch) end builds_export end - def destroy_batch(builds_batch) - return destroy_batch_dry(builds_batch) if @config.dry_run + def destroy_builds_batch(builds_batch) + return destroy_builds_batch_dry(builds_batch) if @config.dry_run builds_batch.each(&:destroy) end - def destroy_batch_dry(builds_batch) + def destroy_builds_batch_dry(builds_batch) @dry_run_removed[:builds].concat(builds_batch.map(&:id)) jobs = builds_batch.map do |build| diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 394e109..96996d6 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -7,6 +7,7 @@ require 'models/user' require 'support/factories' +require 'support/expected_files' require 'pry' describe Backup do @@ -223,7 +224,7 @@ def destination_logs_size context 'when no arguments are given' do it 'processes every repository' do Repository.all.each do |repository| - expect(backup).to receive(:process_repo).once.with(repository) + expect(backup).to receive(:process_repo_builds).once.with(repository) end backup.run end @@ -232,7 +233,7 @@ def destination_logs_size context 'when user_id is given' do it 'processes only the repositories of the given user' do processed_repos_ids = [] - allow(backup).to receive(:process_repo) {|repo| processed_repos_ids.push(repo.id)} + allow(backup).to receive(:process_repo_builds) {|repo| processed_repos_ids.push(repo.id)} user_repos_ids = Repository.where( 'owner_id = ? and owner_type = ?', user1.id, @@ -246,7 +247,7 @@ def destination_logs_size context 'when org_id is given' do it 'processes only the repositories of the given organization' do processed_repos_ids = [] - allow(backup).to receive(:process_repo) {|repo| processed_repos_ids.push(repo.id)} + allow(backup).to receive(:process_repo_builds) {|repo| processed_repos_ids.push(repo.id)} organization_repos_ids = Repository.where( 'owner_id = ? and owner_type = ?', organization1.id, @@ -260,7 +261,7 @@ def destination_logs_size context 'when repo_id is given' do it 'processes only the repository with the given id' do repo = Repository.first - expect(backup).to receive(:process_repo).once.with(repo) + expect(backup).to receive(:process_repo_builds).once.with(repo) backup.run(repo_id: repo.id) end end @@ -310,7 +311,7 @@ def destination_logs_size end - describe 'process_repo' do + describe 'process_repo_builds' do after(:each) do Repository.destroy_all Build.destroy_all @@ -341,316 +342,19 @@ def destination_logs_size } - let!(:exported_object) { - [[ - { - "id": repository.builds.first.id, - "repository_id": repository.id, - "number": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "config": nil, - "commit_id": nil, - "request_id": nil, - "state": nil, - "duration": nil, - "owner_id": nil, - "owner_type": nil, - "event_type": nil, - "previous_state": nil, - "pull_request_title": nil, - "pull_request_number": nil, - "branch": nil, - "canceled_at": nil, - "cached_matrix_ids": nil, - "received_at": nil, - "private": nil, - "pull_request_id": nil, - "branch_id": nil, - "tag_id": nil, - "sender_id": nil, - "sender_type": nil, - "jobs": [{ - "id": repository.builds.first.jobs.first.id, - "repository_id": repository.id, - "commit_id": nil, - "source_id": repository.builds.first.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": repository.builds.first.jobs.first.logs.first.id, - "job_id": repository.builds.first.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": repository.builds.first.jobs.first.logs.second.id, - "job_id": repository.builds.first.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }, - { - "id": repository.builds.first.jobs.second.id, - "repository_id": repository.id, - "commit_id": nil, - "source_id": repository.builds.first.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": repository.builds.first.jobs.second.logs.first.id, - "job_id": repository.builds.first.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": repository.builds.first.jobs.second.logs.second.id, - "job_id": repository.builds.first.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }] - }, - { - "id": repository.builds.second.id, - "repository_id": repository.id, - "number": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "config": nil, - "commit_id": nil, - "request_id": nil, - "state": nil, - "duration": nil, - "owner_id": nil, - "owner_type": nil, - "event_type": nil, - "previous_state": nil, - "pull_request_title": nil, - "pull_request_number": nil, - "branch": nil, - "canceled_at": nil, - "cached_matrix_ids": nil, - "received_at": nil, - "private": nil, - "pull_request_id": nil, - "branch_id": nil, - "tag_id": nil, - "sender_id": nil, - "sender_type": nil, - "jobs": [{ - "id": repository.builds.second.jobs.first.id, - "repository_id": repository.id, - "commit_id": nil, - "source_id": repository.builds.second.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": repository.builds.second.jobs.first.logs.first.id, - "job_id": repository.builds.second.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": repository.builds.second.jobs.first.logs.second.id, - "job_id": repository.builds.second.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }, - { - "id": repository.builds.second.jobs.second.id, - "repository_id": repository.id, - "commit_id": nil, - "source_id": repository.builds.second.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": datetime, - "updated_at": datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": repository.builds.second.jobs.second.logs.first.id, - "job_id": repository.builds.second.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": repository.builds.second.jobs.second.logs.second.id, - "job_id": repository.builds.second.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": datetime, - "updated_at": datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }] - } - ]] + let!(:expected_builds_json) { + ExpectedFiles.new(repository, datetime).builds_json } shared_context 'removing builds and jobs' do it 'should delete all builds of the repository' do - backup.process_repo(repository) + backup.process_repo_builds(repository) expect(Build.all.map(&:repository_id)).to eq([repository2.id]) end it 'should delete all jobs of removed builds and leave the rest' do expect { - backup.process_repo(repository) + backup.process_repo_builds(repository) }.to change { Job.all.size }.by -4 build_id = Build.first.id @@ -659,7 +363,7 @@ def destination_logs_size it 'should delete all logs of removed jobs and leave the rest' do expect { - backup.process_repo(repository) + backup.process_repo_builds(repository) }.to change { Log.all.size }.by -8 build_id = Build.first.id @@ -669,7 +373,7 @@ def destination_logs_size context 'when if_backup config is set to true' do it 'should prepare proper JSON export' do - result = backup.process_repo(repository) + result = backup.process_repo_builds(repository) result.first.first['updated_at'] = datetime result.first.second['updated_at'] = datetime result.first.first[:jobs].first['updated_at'] = datetime @@ -685,12 +389,12 @@ def destination_logs_size result.first.second[:jobs].second[:logs].first['updated_at'] = datetime result.first.second[:jobs].second[:logs].second['updated_at'] = datetime - expect(result.to_json).to eq(exported_object.to_json) + expect(result.to_json).to eq(expected_builds_json.to_json) end it 'should save JSON to file at proper path' do expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') - backup.process_repo(repository) + backup.process_repo_builds(repository) end it_behaves_like 'removing builds and jobs' @@ -701,7 +405,7 @@ def destination_logs_size it 'should create needed folders' do expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original - backup.process_repo(repository) + backup.process_repo_builds(repository) end end end @@ -711,7 +415,7 @@ def destination_logs_size it 'should not save JSON to file' do expect(File).not_to receive(:open) - backup.process_repo(repository) + backup.process_repo_builds(repository) end it_behaves_like 'removing builds and jobs' @@ -722,18 +426,18 @@ def destination_logs_size it 'should not save JSON to file' do expect(File).not_to receive(:open) - backup.process_repo(repository) + backup.process_repo_builds(repository) end it 'should not delete builds' do expect { - backup.process_repo(repository) + backup.process_repo_builds(repository) }.not_to change { Build.all.size } end it 'should not delete jobs' do expect { - backup.process_repo(repository) + backup.process_repo_builds(repository) }.not_to change { Job.all.size } end end diff --git a/spec/support/expected_files.rb b/spec/support/expected_files.rb new file mode 100644 index 0000000..1e94cfb --- /dev/null +++ b/spec/support/expected_files.rb @@ -0,0 +1,307 @@ +class ExpectedFiles + def initialize(repository, datetime) + @repository = repository + @datetime = datetime + end + + def builds_json + [[ + { + "id": @repository.builds.first.id, + "repository_id": @repository.id, + "number": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "config": nil, + "commit_id": nil, + "request_id": nil, + "state": nil, + "duration": nil, + "owner_id": nil, + "owner_type": nil, + "event_type": nil, + "previous_state": nil, + "pull_request_title": nil, + "pull_request_number": nil, + "branch": nil, + "canceled_at": nil, + "cached_matrix_ids": nil, + "received_at": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil, + "jobs": [{ + "id": @repository.builds.first.jobs.first.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": @repository.builds.first.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": @repository.builds.first.jobs.first.logs.first.id, + "job_id": @repository.builds.first.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": @repository.builds.first.jobs.first.logs.second.id, + "job_id": @repository.builds.first.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] + }, + { + "id": @repository.builds.first.jobs.second.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": @repository.builds.first.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": @repository.builds.first.jobs.second.logs.first.id, + "job_id": @repository.builds.first.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": @repository.builds.first.jobs.second.logs.second.id, + "job_id": @repository.builds.first.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] + }] + }, + { + "id": @repository.builds.second.id, + "repository_id": @repository.id, + "number": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "config": nil, + "commit_id": nil, + "request_id": nil, + "state": nil, + "duration": nil, + "owner_id": nil, + "owner_type": nil, + "event_type": nil, + "previous_state": nil, + "pull_request_title": nil, + "pull_request_number": nil, + "branch": nil, + "canceled_at": nil, + "cached_matrix_ids": nil, + "received_at": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil, + "jobs": [{ + "id": @repository.builds.second.jobs.first.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": @repository.builds.second.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": @repository.builds.second.jobs.first.logs.first.id, + "job_id": @repository.builds.second.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": @repository.builds.second.jobs.first.logs.second.id, + "job_id": @repository.builds.second.jobs.first.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] + }, + { + "id": @repository.builds.second.jobs.second.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": @repository.builds.second.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + "logs": [ + { + "id": @repository.builds.second.jobs.second.logs.first.id, + "job_id": @repository.builds.second.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": @repository.builds.second.jobs.second.logs.second.id, + "job_id": @repository.builds.second.jobs.second.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] + }] + } + ]] + end +end \ No newline at end of file From 30a89de270be4299e673642779fe70e244a89e8c Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 2 Sep 2021 13:43:31 +0200 Subject: [PATCH 57/84] processing requests --- lib/models/repository.rb | 2 + lib/models/request.rb | 3 + lib/travis-backup.rb | 86 ++++++++++++++++----- spec/backup_spec.rb | 132 ++++++++++++++++++++++++++++++--- spec/support/expected_files.rb | 61 +++++++++++++++ spec/support/factories.rb | 17 +++++ 6 files changed, 273 insertions(+), 28 deletions(-) diff --git a/lib/models/repository.rb b/lib/models/repository.rb index 6022e54..e5cf08e 100644 --- a/lib/models/repository.rb +++ b/lib/models/repository.rb @@ -2,10 +2,12 @@ require 'models/model' require 'models/build' +require 'models/request' # Repository model class Repository < Model has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build' + has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request' self.table_name = 'repositories' end diff --git a/lib/models/request.rb b/lib/models/request.rb index 9ed9d97..4fb1408 100644 --- a/lib/models/request.rb +++ b/lib/models/request.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true require 'models/model' +require 'models/repository' class Request < Model + belongs_to :repository + self.table_name = 'requests' end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 22a66a9..caa4067 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -17,13 +17,13 @@ # main travis-backup class class Backup attr_accessor :config - attr_reader :dry_run_removed + attr_reader :dry_run_report def initialize(config_args={}) @config = Config.new(config_args) if @config.dry_run - @dry_run_removed = {builds: [], jobs: [], logs: []} + @dry_run_report = {builds: [], jobs: [], logs: [], requests: []} end connect_db @@ -47,29 +47,52 @@ def run(args={}) elsif org_id owner_id = org_id owner_type = 'Organization' - elsif repo_id - repo_id = repo_id end if owner_id Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| - process_repo_builds(repository) + process_repo(repository) end elsif repo_id repository = Repository.find(repo_id) - process_repo_builds(repository) + process_repo(repository) else Repository.order(:id).each do |repository| - process_repo_builds(repository) + process_repo(repository) end end - if @config.dry_run - puts 'Dry run active. The following data would be removed in normal mode:' - puts " - builds: #{@dry_run_removed[:builds].to_json}" - puts " - jobs: #{@dry_run_removed[:jobs].to_json}" - puts " - logs: #{@dry_run_removed[:logs].to_json}" - end + print_dry_run_report if @config.dry_run + end + + def print_dry_run_report + puts 'Dry run active. The following data would be removed in normal mode:' + puts " - builds: #{@dry_run_report[:builds].to_json}" + puts " - jobs: #{@dry_run_report[:jobs].to_json}" + puts " - logs: #{@dry_run_report[:logs].to_json}" + puts " - requests: #{@dry_run_report[:requests].to_json}" + end + + def process_repo(repository) + process_repo_builds(repository) + process_repo_requests(repository) + end + + def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + threshold = @config.threshold.to_i.months.ago.to_datetime + current_build_id = repository.current_build_id || -1 + repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) + .in_groups_of(@config.limit.to_i, false).map do |builds_batch| + @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch) + end.compact + end + + def process_repo_requests(repository) + threshold = @config.threshold.to_i.months.ago.to_datetime + repository.requests.where('created_at < ?', threshold) + .in_groups_of(@config.limit.to_i, false).map do |requests_batch| + @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) + end.compact end def move_logs @@ -89,6 +112,7 @@ def move_logs end end +<<<<<<< HEAD def remove_orphans remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') @@ -140,6 +164,8 @@ def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/M end.compact end +======= +>>>>>>> processing requests private def save_and_destroy_builds_batch(builds_batch, repository) @@ -159,13 +185,13 @@ def destroy_builds_batch(builds_batch) end def destroy_builds_batch_dry(builds_batch) - @dry_run_removed[:builds].concat(builds_batch.map(&:id)) + @dry_run_report[:builds].concat(builds_batch.map(&:id)) jobs = builds_batch.map do |build| build.jobs.map(&:id) || [] end.flatten - @dry_run_removed[:jobs].concat(jobs) + @dry_run_report[:jobs].concat(jobs) logs = builds_batch.map do |build| build.jobs.map do |job| @@ -173,7 +199,27 @@ def destroy_builds_batch_dry(builds_batch) end.flatten || [] end.flatten - @dry_run_removed[:logs].concat(logs) + @dry_run_report[:logs].concat(logs) + end + + def save_and_destroy_requests_batch(requests_batch, repository) + requests_export = export_requests(requests_batch) + file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json" + pretty_json = JSON.pretty_generate(requests_export) + if save_file(file_name, pretty_json) + destroy_requests_batch(requests_batch) + end + requests_export + end + + def destroy_requests_batch(requests_batch) + return destroy_requests_batch_dry(requests_batch) if @config.dry_run + + requests_batch.each(&:destroy) + end + + def destroy_requests_batch_dry(requests_batch) + @dry_run_report[:requests].concat(requests_batch.map(&:id)) end def save_file(file_name, content) # rubocop:disable Metrics/MethodLength @@ -220,9 +266,13 @@ def export_jobs(jobs) def export_logs(logs) logs.map do |log| - log_export = log.attributes + log.attributes + end + end - log_export + def export_requests(requests) + requests.map do |request| + request.attributes end end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 96996d6..946285a 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -206,7 +206,7 @@ def destination_logs_size end let!(:unassigned_repositories) { - FactoryBot.create_list(:repository, 3) + FactoryBot.create_list(:repository_with_requests, 3) } let!(:user1) { FactoryBot.create(:user_with_repos) @@ -303,12 +303,28 @@ def destination_logs_size it 'prepares proper dry run report' do backup.run - expect(backup.dry_run_removed[:builds].size).to eql 24 - expect(backup.dry_run_removed[:jobs].size).to eql 48 - expect(backup.dry_run_removed[:logs].size).to eql 96 + expect(backup.dry_run_report[:builds].size).to eql 24 + expect(backup.dry_run_report[:jobs].size).to eql 48 + expect(backup.dry_run_report[:logs].size).to eql 96 + expect(backup.dry_run_report[:requests].size).to eql 6 end end + end + + describe 'process_repo' do + let!(:repository) { + FactoryBot.create(:repository) + } + + it 'processes repository builds' do + expect(backup).to receive(:process_repo_builds).once.with(repository) + backup.process_repo(repository) + end + it 'processes repository requests' do + expect(backup).to receive(:process_repo_requests).once.with(repository) + backup.process_repo(repository) + end end describe 'process_repo_builds' do @@ -319,12 +335,7 @@ def destination_logs_size Log.destroy_all end - let!(:config) { Config.new } - let(:datetime) { (config.threshold + 1).months.ago.to_time.utc } - let(:org_id) { rand(100000) } - let(:com_id) { rand(100000) } - let(:private_org_id) { rand(100000) } - let(:private_com_id) { rand(100000) } + let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } let!(:repository) { FactoryBot.create( :repository_with_builds_jobs_and_logs, @@ -442,4 +453,105 @@ def destination_logs_size end end end + + describe 'process_repo_requests' do + let!(:config) { Config.new } + let(:datetime) { (config.threshold + 1).months.ago.to_time.utc } + let(:org_id) { rand(100000) } + let(:com_id) { rand(100000) } + let(:private_org_id) { rand(100000) } + let(:private_com_id) { rand(100000) } + let!(:repository) { + FactoryBot.create( + :repository_with_requests, + created_at: datetime, + updated_at: datetime + ) + } + let!(:repository2) { + FactoryBot.create( + :repository_with_requests, + created_at: datetime, + updated_at: datetime, + requests_count: 1 + ) + } + + + let!(:expected_requests_json) { + ExpectedFiles.new(repository, datetime).requests_json + } + + shared_context 'removing requests' do + it 'should delete all builds of the repository' do + backup.process_repo_requests(repository) + expect(Request.all.map(&:repository_id)).to eq([repository2.id]) + end + end + + context 'when if_backup config is set to true' do + it 'should prepare proper JSON export' do + result = backup.process_repo_requests(repository) + # result.first.first['updated_at'] = datetime + # result.first.second['updated_at'] = datetime + # result.first.first[:jobs].first['updated_at'] = datetime + # result.first.first[:jobs].second['updated_at'] = datetime + # result.first.second[:jobs].first['updated_at'] = datetime + # result.first.second[:jobs].second['updated_at'] = datetime + # result.first.first[:jobs].first[:logs].first['updated_at'] = datetime + # result.first.first[:jobs].first[:logs].second['updated_at'] = datetime + # result.first.first[:jobs].second[:logs].first['updated_at'] = datetime + # result.first.first[:jobs].second[:logs].second['updated_at'] = datetime + # result.first.second[:jobs].first[:logs].first['updated_at'] = datetime + # result.first.second[:jobs].first[:logs].second['updated_at'] = datetime + # result.first.second[:jobs].second[:logs].first['updated_at'] = datetime + # result.first.second[:jobs].second[:logs].second['updated_at'] = datetime + + expect(result.to_json).to eq(expected_requests_json.to_json) + end + + it 'should save JSON to file at proper path' do + expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') + backup.process_repo_requests(repository) + end + + it_behaves_like 'removing requests' + + context 'when path with nonexistent folders is given' do + let(:random_files_location) { "dump/tests/#{rand(100000)}" } + let!(:backup) { Backup.new(files_location: random_files_location, limit: 2) } + + it 'should create needed folders' do + expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original + backup.process_repo_requests(repository) + end + end + end + + context 'when if_backup config is set to false' do + let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } + + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo_requests(repository) + end + + it_behaves_like 'removing requests' + end + + context 'when dry_run config is set to true' do + let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } + + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo_requests(repository) + end + + it 'should not delete requests' do + expect { + backup.process_repo_requests(repository) + }.not_to change { Request.all.size } + end + end + end end diff --git a/spec/support/expected_files.rb b/spec/support/expected_files.rb index 1e94cfb..b39ce2b 100644 --- a/spec/support/expected_files.rb +++ b/spec/support/expected_files.rb @@ -304,4 +304,65 @@ def builds_json } ]] end + + def requests_json + [[ + { + "id": @repository.requests.first.id, + "repository_id": @repository.id, + "commit_id": nil, + "state": nil, + "source": nil, + "payload": nil, + "token": nil, + "config": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "event_type": nil, + "comments_url": nil, + "base_commit": nil, + "head_commit": nil, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "message": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil + }, + { + "id": @repository.requests.second.id, + "repository_id": @repository.id, + "commit_id": nil, + "state": nil, + "source": nil, + "payload": nil, + "token": nil, + "config": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "event_type": nil, + "comments_url": nil, + "base_commit": nil, + "head_commit": nil, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "message": nil, + "private": nil, + "pull_request_id": nil, + "branch_id": nil, + "tag_id": nil, + "sender_id": nil, + "sender_type": nil + } + ]] + end end \ No newline at end of file diff --git a/spec/support/factories.rb b/spec/support/factories.rb index b7d8181..a37bf1e 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -81,8 +81,25 @@ factory :repository_with_last_build_id do last_build_id { Build.first.id } end + + factory :repository_with_requests do + transient do + requests_count { 2 } + end + after(:create) do |repository, evaluator| + create_list( + :request, + evaluator.requests_count, + repository: repository, + created_at: repository.created_at, + updated_at: repository.updated_at + ) + end + end end + factory :request + factory :build do factory :build_with_jobs_and_logs do transient do From 12f13260b640caa9674495412d3dd1e63503732f Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Thu, 2 Sep 2021 14:49:01 +0200 Subject: [PATCH 58/84] tests improved --- lib/travis-backup.rb | 3 --- spec/backup_spec.rb | 63 +++++++++++++++++--------------------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index caa4067..1775e75 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -112,7 +112,6 @@ def move_logs end end -<<<<<<< HEAD def remove_orphans remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') @@ -164,8 +163,6 @@ def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/M end.compact end -======= ->>>>>>> processing requests private def save_and_destroy_builds_batch(builds_batch, repository) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 946285a..168aa78 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -308,6 +308,11 @@ def destination_logs_size expect(backup.dry_run_report[:logs].size).to eql 96 expect(backup.dry_run_report[:requests].size).to eql 6 end + + it 'prints dry run report' do + expect(backup).to receive(:print_dry_run_report).once + backup.run + end end end @@ -382,6 +387,13 @@ def destination_logs_size end end + shared_context 'not saving JSON to file' do + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo_builds(repository) + end + end + context 'when if_backup config is set to true' do it 'should prepare proper JSON export' do result = backup.process_repo_builds(repository) @@ -424,21 +436,14 @@ def destination_logs_size context 'when if_backup config is set to false' do let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_builds(repository) - end - + it_behaves_like 'not saving JSON to file' it_behaves_like 'removing builds and jobs' end context 'when dry_run config is set to true' do let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_builds(repository) - end + it_behaves_like 'not saving JSON to file' it 'should not delete builds' do expect { @@ -455,12 +460,7 @@ def destination_logs_size end describe 'process_repo_requests' do - let!(:config) { Config.new } - let(:datetime) { (config.threshold + 1).months.ago.to_time.utc } - let(:org_id) { rand(100000) } - let(:com_id) { rand(100000) } - let(:private_org_id) { rand(100000) } - let(:private_com_id) { rand(100000) } + let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } let!(:repository) { FactoryBot.create( :repository_with_requests, @@ -489,24 +489,16 @@ def destination_logs_size end end + shared_context 'not saving JSON to file' do + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + backup.process_repo_requests(repository) + end + end + context 'when if_backup config is set to true' do it 'should prepare proper JSON export' do result = backup.process_repo_requests(repository) - # result.first.first['updated_at'] = datetime - # result.first.second['updated_at'] = datetime - # result.first.first[:jobs].first['updated_at'] = datetime - # result.first.first[:jobs].second['updated_at'] = datetime - # result.first.second[:jobs].first['updated_at'] = datetime - # result.first.second[:jobs].second['updated_at'] = datetime - # result.first.first[:jobs].first[:logs].first['updated_at'] = datetime - # result.first.first[:jobs].first[:logs].second['updated_at'] = datetime - # result.first.first[:jobs].second[:logs].first['updated_at'] = datetime - # result.first.first[:jobs].second[:logs].second['updated_at'] = datetime - # result.first.second[:jobs].first[:logs].first['updated_at'] = datetime - # result.first.second[:jobs].first[:logs].second['updated_at'] = datetime - # result.first.second[:jobs].second[:logs].first['updated_at'] = datetime - # result.first.second[:jobs].second[:logs].second['updated_at'] = datetime - expect(result.to_json).to eq(expected_requests_json.to_json) end @@ -531,21 +523,14 @@ def destination_logs_size context 'when if_backup config is set to false' do let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_requests(repository) - end - + it_behaves_like 'not saving JSON to file' it_behaves_like 'removing requests' end context 'when dry_run config is set to true' do let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_requests(repository) - end + it_behaves_like 'not saving JSON to file' it 'should not delete requests' do expect { From f5707593edbc4007e9eb33a4012a29911e43b432 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 3 Sep 2021 15:15:23 +0200 Subject: [PATCH 59/84] fixes in tests --- spec/backup_spec.rb | 25 +++++++++++++++++++++++-- spec/support/factories.rb | 2 -- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 168aa78..8772bac 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -73,6 +73,19 @@ def destination_logs_size end describe 'remove_orphans' do + after(:all) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + Branch.destroy_all + Tag.destroy_all + Commit.destroy_all + Cron.destroy_all + PullRequest.destroy_all + Request.destroy_all + Stage.destroy_all + end + let!(:data) { ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') FactoryBot.create_list(:repository, 2) @@ -199,10 +212,13 @@ def destination_logs_size describe 'run' do after(:each) do + Organization.destroy_all + User.destroy_all Repository.destroy_all Build.destroy_all Job.destroy_all Log.destroy_all + Request.destroy_all end let!(:unassigned_repositories) { @@ -459,7 +475,12 @@ def destination_logs_size end end - describe 'process_repo_requests' do + describe 'process_repo_requests' do + after(:each) do + Repository.destroy_all + Request.destroy_all + end + let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } let!(:repository) { FactoryBot.create( @@ -483,7 +504,7 @@ def destination_logs_size } shared_context 'removing requests' do - it 'should delete all builds of the repository' do + it 'should delete all requests of the repository' do backup.process_repo_requests(repository) expect(Request.all.map(&:repository_id)).to eq([repository2.id]) end diff --git a/spec/support/factories.rb b/spec/support/factories.rb index a37bf1e..87447f8 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -98,8 +98,6 @@ end end - factory :request - factory :build do factory :build_with_jobs_and_logs do transient do From bbfbab7570abc10449de81d7ed6d0a8801cffe31 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 3 Sep 2021 19:21:27 +0200 Subject: [PATCH 60/84] dry run and README improved, v0.1.0 set --- Gemfile.lock | 2 +- README.md | 17 +++++++++-------- lib/config.rb | 2 +- lib/travis-backup.rb | 42 ++++++++++++++++++++++++++++++++---------- travis-backup.gemspec | 2 +- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index dc75f20..0e6e5a7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.0.3) + travis-backup (0.1.0) activerecord bootsnap pg diff --git a/README.md b/README.md index c527f55..09efe7e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # README -*travis-backup* is an application that removes builds and their corresponding jobs -and exports them (optionally) to json files. +*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. ### Installation and run @@ -59,13 +58,15 @@ backup.run(repo_id: 1) #### Special modes -Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). Running gem in this mode no files are created and no other tables are being touched. +Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). When you run gem in this mode no files are created and no other tables are being touched. -Using `--remove_orphans` flag you can remove all orphaned data from tables. Running gem in this mode no files are created. +Using `--remove_orphans` flag you can remove all orphaned data from tables. When you run gem in this mode no files are created. -### Configuration +Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with `--move_logs` or `--remove_orphans`. -Despite of command line arguments, one of the ways you can configure your export is a file `config/settinigs.yml` that you can place in your app's main directory. The gem expects properties in the following format: +### Configuration options + +Despite of command line arguments, one of the ways you can configure your export is a file `config/settings.yml` that you can place in your app's main directory. The gem expects properties in the following format: ``` backup: @@ -97,9 +98,9 @@ and bundle exec rspec ``` -To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`. +To make tests working properly you should also ensure database connection strings for empty test databases. You can set them as `DATABASE_URL` and `BACKUP_DESTINATION_DB_URL` environment variables or in `config/database.yml`. -**Warning: this database will be cleaned during tests, so ensure that it includes no important data.** +**Warning: these databases will be cleaned during tests, so ensure that they include no important data.** ### Ruby version diff --git a/lib/config.rb b/lib/config.rb index a02d4bb..7409a61 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -105,7 +105,7 @@ def set_values(args) end def check_values - if !@move_logs && !@threshold + if !@move_logs && !@remove_orphans && !@threshold message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " + "Threshold defines number of months from now.") abort message diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 1775e75..8ea6400 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -34,9 +34,6 @@ def connect_db(url=@config.database_url) end def run(args={}) - return move_logs if @config.move_logs - return remove_orphans if @config.remove_orphans - user_id = args[:user_id] || @config.user_id repo_id = args[:repo_id] || @config.repo_id org_id = args[:org_id] || @config.org_id @@ -49,7 +46,11 @@ def run(args={}) owner_type = 'Organization' end - if owner_id + if @config.move_logs + move_logs + elsif @config.remove_orphans + remove_orphans + elsif owner_id Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| process_repo(repository) end @@ -66,11 +67,19 @@ def run(args={}) end def print_dry_run_report - puts 'Dry run active. The following data would be removed in normal mode:' - puts " - builds: #{@dry_run_report[:builds].to_json}" - puts " - jobs: #{@dry_run_report[:jobs].to_json}" - puts " - logs: #{@dry_run_report[:logs].to_json}" - puts " - requests: #{@dry_run_report[:requests].to_json}" + if @dry_run_report.to_a.map(&:second).flatten.empty? + puts 'Dry run active. No data would be removed in normal run.' + else + puts 'Dry run active. The following data would be removed in normal run:' + + @dry_run_report.to_a.map(&:first).each do |symbol| + print_dry_run_report_line(symbol) + end + end + end + + def print_dry_run_report_line(symbol) + puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any? end def process_repo(repository) @@ -96,6 +105,8 @@ def process_repo_requests(repository) end def move_logs + return move_logs_dry if config.dry_run + connect_db(@config.database_url) Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch| log_hashes = logs_batch.as_json @@ -112,6 +123,10 @@ def move_logs end end + def move_logs_dry + dry_run_report[:logs].concat(Log.order(:id).map(&:id)) + end + def remove_orphans remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') @@ -151,7 +166,14 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) a.#{fk_name} is not null and b.id is null; }) - model_class.where(id: for_delete.map(&:id)).delete_all + + if config.dry_run + dry_run_report[table_a_name.to_sym] = [] if dry_run_report[table_a_name.to_sym].nil? + dry_run_report[table_a_name.to_sym].concat(for_delete.map(&:id)) + dry_run_report[table_a_name.to_sym].uniq! + else + model_class.where(id: for_delete.map(&:id)).delete_all + end end def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 516e34c..668265e 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.0.3' + s.version = '0.1.0' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From 0c9c3f77dc9db4f4c79b17be09e95913b3166f32 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 7 Sep 2021 11:40:26 +0200 Subject: [PATCH 61/84] refactoring, README improved, env vars changed, v0.1.1 --- Gemfile.lock | 2 +- README.md | 14 +++++++++----- lib/config.rb | 6 +++--- lib/travis-backup.rb | 43 +++++++++++++++++++++++-------------------- travis-backup.gemspec | 2 +- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0e6e5a7..6908301 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.1.0) + travis-backup (0.1.1) activerecord bootsnap pg diff --git a/README.md b/README.md index 09efe7e..36f5868 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # README -*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. +*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data. ### Installation and run @@ -75,15 +75,19 @@ backup: limit: 1000 # builds limit for one backup file threshold: 6 # number of months from now - data younger than this time won't be backuped files_location: './dump' # path of the folder in which backup files will be placed - user_id # run only for given user - org_id # run only for given organization - repo_id # run only for given repository + user_id: 1 # run only for given user + org_id: 1 # run only for given organization + repo_id: 1 # run only for given repository + move_logs: false # run in move logs mode - move all logs to database at destination_db_url URL + remove_orphans: false # run in remove orphans mode ``` -You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`. +You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`. You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema. +For `move_logs` mode you need also to specify a destination database. You can set it also in `config/database.yml` file, in `destination` subsection, setting the `destination_db_url` hash argument while creating `Backup` instance or using the `BACKUP_DESTINATION_DB_URL` env var. Your destination database should be consistent with the Travis 3.0 database schema. + ### How to run the test suite You can run the test after cloning this repository. Next you should call diff --git a/lib/config.rb b/lib/config.rb index 7409a61..ead7726 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -67,19 +67,19 @@ def set_values(args) @user_id = first_not_nil( args[:user_id], argv_opts[:user_id], - ENV['USER_ID'], + ENV['BACKUP_USER_ID'], config.dig('backup', 'user_id') ) @repo_id = first_not_nil( args[:repo_id], argv_opts[:repo_id], - ENV['REPO_ID'], + ENV['BACKUP_REPO_ID'], config.dig('backup', 'repo_id') ) @org_id = first_not_nil( args[:org_id], argv_opts[:org_id], - ENV['ORG_ID'], + ENV['BACKUP_ORG_ID'], config.dig('backup', 'org_id') ) @move_logs = first_not_nil( diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 8ea6400..eb82ce7 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -51,21 +51,32 @@ def run(args={}) elsif @config.remove_orphans remove_orphans elsif owner_id - Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| - process_repo(repository) - end + process_repos_for_owner(owner_id, owner_type) elsif repo_id - repository = Repository.find(repo_id) - process_repo(repository) + process_repo_with_id(repo_id) else - Repository.order(:id).each do |repository| - process_repo(repository) - end + process_all_repos end print_dry_run_report if @config.dry_run end + def process_repos_for_owner(owner_id, owner_type) + Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| + process_repo(repository) + end + end + + def process_repo_with_id(repo_id) + process_repo(Repository.find(repo_id)) + end + + def process_all_repos + Repository.order(:id).each do |repository| + process_repo(repository) + end + end + def print_dry_run_report if @dry_run_report.to_a.map(&:second).flatten.empty? puts 'Dry run active. No data would be removed in normal run.' @@ -168,23 +179,15 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) }) if config.dry_run - dry_run_report[table_a_name.to_sym] = [] if dry_run_report[table_a_name.to_sym].nil? - dry_run_report[table_a_name.to_sym].concat(for_delete.map(&:id)) - dry_run_report[table_a_name.to_sym].uniq! + key = table_a_name.to_sym + dry_run_report[key] = [] if dry_run_report[key].nil? + dry_run_report[key].concat(for_delete.map(&:id)) + dry_run_report[key].uniq! else model_class.where(id: for_delete.map(&:id)).delete_all end end - def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - threshold = @config.threshold.to_i.months.ago.to_datetime - current_build_id = repository.current_build_id || -1 - repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) - .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch) - end.compact - end - private def save_and_destroy_builds_batch(builds_batch, repository) diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 668265e..3cea92c 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.1.0' + s.version = '0.1.1' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From e075805e041095c773882ece2b825de7b4398d4a Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 13 Sep 2021 11:35:32 +0200 Subject: [PATCH 62/84] bug with fk constraint fixed --- lib/models/build.rb | 1 + lib/travis-backup.rb | 2 +- spec/backup_spec.rb | 6 +++--- spec/support/factories.rb | 10 +++++++++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/models/build.rb b/lib/models/build.rb index dc4f04f..b5aeb35 100644 --- a/lib/models/build.rb +++ b/lib/models/build.rb @@ -8,6 +8,7 @@ class Build < Model belongs_to :repository has_many :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job' + has_one :repo_for_that_this_build_is_current, foreign_key: :current_build_id, dependent: :destroy, class_name: 'Repository' self.table_name = 'builds' end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index eb82ce7..1175741 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -184,7 +184,7 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) dry_run_report[key].concat(for_delete.map(&:id)) dry_run_report[key].uniq! else - model_class.where(id: for_delete.map(&:id)).delete_all + model_class.where(id: for_delete.map(&:id)).destroy_all end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 8772bac..7f48010 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -104,7 +104,7 @@ def destination_logs_size FactoryBot.create_list(:repository_with_last_build_id, 2) FactoryBot.create_list(:build_orphaned_on_repository_id, 2) FactoryBot.create_list(:build_with_repository_id, 2) - FactoryBot.create_list(:build_orphaned_on_commit_id, 2) + FactoryBot.create_list(:build_orphaned_on_commit_id_with_repo, 2) FactoryBot.create_list(:build_with_commit_id, 2) FactoryBot.create_list(:build_orphaned_on_request_id, 2) FactoryBot.create_list(:build_with_request_id, 2) @@ -149,10 +149,10 @@ def destination_logs_size FactoryBot.create_list(:stage_with_build_id, 2) ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') } - it 'removes orphaned repositories' do + it 'removes orphaned repositories (with these dependent on orphaned builds)' do expect { backup.remove_orphans - }.to change { Repository.all.size }.by -4 + }.to change { Repository.all.size }.by -6 end it 'removes orphaned builds' do diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 87447f8..17b2bdb 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -124,8 +124,16 @@ repository_id { Repository.first.id } end - factory :build_orphaned_on_commit_id do + factory :build_orphaned_on_commit_id_with_repo do commit_id { 2_000_000_000 } + after(:create) do |build| + create( + :repository, + current_build_id: build.id, + created_at: build.created_at, + updated_at: build.updated_at + ) + end end factory :build_with_commit_id do From 3041a3ce2ff2c5d434b5014a551c7fa5e51d8dcb Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 13 Sep 2021 13:18:50 +0200 Subject: [PATCH 63/84] v0.1.2 --- Gemfile.lock | 2 +- travis-backup.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6908301..fe0fa82 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.1.1) + travis-backup (0.1.2) activerecord bootsnap pg diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 3cea92c..4c4b582 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.1.1' + s.version = '0.1.2' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From e3100d6e7aab3ba84c971cfd2811c13aae2235ab Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 13 Sep 2021 13:50:30 +0200 Subject: [PATCH 64/84] problem with mutually related repo fixed - v0.1.3 --- Gemfile.lock | 2 +- lib/models/repository.rb | 2 +- spec/backup_spec.rb | 2 +- spec/support/factories.rb | 6 ++++-- travis-backup.gemspec | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index fe0fa82..8b33b7b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.1.2) + travis-backup (0.1.3) activerecord bootsnap pg diff --git a/lib/models/repository.rb b/lib/models/repository.rb index e5cf08e..fc7660f 100644 --- a/lib/models/repository.rb +++ b/lib/models/repository.rb @@ -6,7 +6,7 @@ # Repository model class Repository < Model - has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build' + has_many :builds, -> { order('id') }, foreign_key: :repository_id, class_name: 'Build' has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request' self.table_name = 'repositories' diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 7f48010..3ff2145 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -104,7 +104,7 @@ def destination_logs_size FactoryBot.create_list(:repository_with_last_build_id, 2) FactoryBot.create_list(:build_orphaned_on_repository_id, 2) FactoryBot.create_list(:build_with_repository_id, 2) - FactoryBot.create_list(:build_orphaned_on_commit_id_with_repo, 2) + FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_commit_id, 2) FactoryBot.create_list(:build_orphaned_on_request_id, 2) FactoryBot.create_list(:build_with_request_id, 2) diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 17b2bdb..62c1095 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -124,15 +124,17 @@ repository_id { Repository.first.id } end - factory :build_orphaned_on_commit_id_with_repo do + factory :build_orphaned_on_commit_id_with_mutually_related_repo do commit_id { 2_000_000_000 } after(:create) do |build| - create( + repo = create( :repository, current_build_id: build.id, created_at: build.created_at, updated_at: build.updated_at ) + build.repository_id = repo.id + build.save! end end diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 4c4b582..ebe8cf3 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.1.2' + s.version = '0.1.3' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From ffd7f7f02d194736a4219922e09cbf696cd97138 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 14 Sep 2021 13:16:33 +0200 Subject: [PATCH 65/84] only builds being destroyed, ids of dependent data in dry run --- lib/travis-backup.rb | 42 +++++++++++++++++++++++------- spec/backup_spec.rb | 39 +++++++++++++++++++++++----- spec/support/factories.rb | 54 ++++++++++++++++++++++++--------------- 3 files changed, 100 insertions(+), 35 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 1175741..7404474 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -141,12 +141,24 @@ def move_logs_dry def remove_orphans remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') - remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id') - remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id') - remove_orphans_for_table(Build, 'builds', 'requests', 'request_id') - remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id') - remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id') - remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id') + remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end + remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end + remove_orphans_for_table(Build, 'builds', 'requests', 'request_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end + remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end + remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end + remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id', :destroy_all) do |ids_for_delete| + add_builds_dependencies_to_dry_run_report(ids_for_delete) + end remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id') remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id') remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id') @@ -167,7 +179,14 @@ def remove_orphans remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id') end - def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) + def add_builds_dependencies_to_dry_run_report(ids_for_delete) + repos_for_delete = Repository.where(current_build_id: ids_for_delete) + jobs_for_delete = Job.where(source_id: ids_for_delete) + dry_run_report[:repositories].concat(repos_for_delete.map(&:id)) + dry_run_report[:jobs].concat(jobs_for_delete.map(&:id)) + end + + def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name, method=:delete_all) for_delete = model_class.find_by_sql(%{ select a.* from #{table_a_name} a @@ -178,13 +197,18 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name) and b.id is null; }) + ids_for_delete = for_delete.map(&:id) + if config.dry_run key = table_a_name.to_sym + dry_run_report[key] = [] if dry_run_report[key].nil? - dry_run_report[key].concat(for_delete.map(&:id)) + dry_run_report[key].concat(ids_for_delete) dry_run_report[key].uniq! + + yield(ids_for_delete) if block_given? else - model_class.where(id: for_delete.map(&:id)).destroy_all + model_class.where(id: ids_for_delete).send(method) end end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 3ff2145..94cde56 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -102,17 +102,17 @@ def destination_logs_size FactoryBot.create_list(:repository_with_current_build_id, 2) FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) FactoryBot.create_list(:repository_with_last_build_id, 2) - FactoryBot.create_list(:build_orphaned_on_repository_id, 2) + FactoryBot.create_list(:build_orphaned_on_repository_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_repository_id, 2) FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_commit_id, 2) - FactoryBot.create_list(:build_orphaned_on_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_request_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_pull_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_pull_request_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_pull_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_branch_id, 2) + FactoryBot.create_list(:build_orphaned_on_branch_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_branch_id, 2) - FactoryBot.create_list(:build_orphaned_on_tag_id, 2) + FactoryBot.create_list(:build_orphaned_on_tag_id_with_mutually_related_repo, 2) FactoryBot.create_list(:build_with_tag_id, 2) FactoryBot.create_list(:job_orphaned_on_repository_id, 2) FactoryBot.create_list(:job_with_repository_id, 2) @@ -152,7 +152,7 @@ def destination_logs_size it 'removes orphaned repositories (with these dependent on orphaned builds)' do expect { backup.remove_orphans - }.to change { Repository.all.size }.by -6 + }.to change { Repository.all.size }.by -16 end it 'removes orphaned builds' do @@ -208,6 +208,33 @@ def destination_logs_size backup.remove_orphans }.to change { Stage.all.size }.by -2 end + + context 'when dry run mode is on' do + let!(:backup) { Backup.new(dry_run: true) } + + before do + allow_any_instance_of(IO).to receive(:puts) + end + + it 'prepares proper dry run report' do + backup.remove_orphans + expect(backup.dry_run_report[:repositories].size).to eql 16 + expect(backup.dry_run_report[:builds].size).to eql 12 + expect(backup.dry_run_report[:jobs].size).to eql 6 + expect(backup.dry_run_report[:branches].size).to eql 4 + expect(backup.dry_run_report[:tags].size).to eql 4 + expect(backup.dry_run_report[:commits].size).to eql 6 + expect(backup.dry_run_report[:crons].size).to eql 2 + expect(backup.dry_run_report[:pull_requests].size).to eql 2 + expect(backup.dry_run_report[:requests].size).to eql 8 + expect(backup.dry_run_report[:stages].size).to eql 2 + end + + it 'prints dry run report' do + expect(backup).to receive(:print_dry_run_report).once + backup.run + end + end end describe 'run' do diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 62c1095..8fb9a51 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -116,16 +116,26 @@ end end - factory :build_orphaned_on_repository_id do - repository_id { 2_000_000_000 } + factory :build_with_repo do + after(:create) do |build| + create( + :repository, + current_build_id: build.id, + created_at: build.created_at, + updated_at: build.updated_at + ) + end + + factory :build_orphaned_on_repository_id_with_mutually_related_repo do + repository_id { 2_000_000_000 } + end end factory :build_with_repository_id do repository_id { Repository.first.id } end - factory :build_orphaned_on_commit_id_with_mutually_related_repo do - commit_id { 2_000_000_000 } + factory :build_with_mutually_related_repo do after(:create) do |build| repo = create( :repository, @@ -136,40 +146,44 @@ build.repository_id = repo.id build.save! end + + factory :build_orphaned_on_commit_id_with_mutually_related_repo do + commit_id { 2_000_000_000 } + end + + factory :build_orphaned_on_request_id_with_mutually_related_repo do + request_id { 2_000_000_000 } + end + + factory :build_orphaned_on_pull_request_id_with_mutually_related_repo do + pull_request_id { 2_000_000_000 } + end + + factory :build_orphaned_on_branch_id_with_mutually_related_repo do + branch_id { 2_000_000_000 } + end + + factory :build_orphaned_on_tag_id_with_mutually_related_repo do + tag_id { 2_000_000_000 } + end end factory :build_with_commit_id do commit_id { Commit.first.id } end - factory :build_orphaned_on_request_id do - request_id { 2_000_000_000 } - end - factory :build_with_request_id do request_id { Request.first.id } end - factory :build_orphaned_on_pull_request_id do - pull_request_id { 2_000_000_000 } - end - factory :build_with_pull_request_id do pull_request_id { PullRequest.first.id } end - factory :build_orphaned_on_branch_id do - branch_id { 2_000_000_000 } - end - factory :build_with_branch_id do branch_id { Branch.first.id } end - factory :build_orphaned_on_tag_id do - tag_id { 2_000_000_000 } - end - factory :build_with_tag_id do tag_id { Tag.first.id } end From bd713c01ef19ee99e265fdc320a313508ad15570 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 14 Sep 2021 16:02:38 +0200 Subject: [PATCH 66/84] refactoring --- lib/travis-backup.rb | 160 ++++++++++++++++++++++++++++--------------- 1 file changed, 106 insertions(+), 54 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 7404474..ee3e252 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -38,20 +38,14 @@ def run(args={}) repo_id = args[:repo_id] || @config.repo_id org_id = args[:org_id] || @config.org_id - if user_id - owner_id = user_id - owner_type = 'User' - elsif org_id - owner_id = org_id - owner_type = 'Organization' - end - if @config.move_logs move_logs elsif @config.remove_orphans remove_orphans - elsif owner_id - process_repos_for_owner(owner_id, owner_type) + elsif user_id + process_repos_for_owner(user_id, 'User') + elsif org_id + process_repos_for_owner(org_id, 'Organization') elsif repo_id process_repo_with_id(repo_id) else @@ -139,44 +133,93 @@ def move_logs_dry end def remove_orphans - remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id') - remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id') - remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) - end - remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) - end - remove_orphans_for_table(Build, 'builds', 'requests', 'request_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) - end - remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) - end - remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) - end - remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id', :destroy_all) do |ids_for_delete| - add_builds_dependencies_to_dry_run_report(ids_for_delete) + cases = [ + { + main_model: Repository, + relations: [ + {related_model: Build, fk_name: 'current_build_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Build, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: Request, fk_name: 'request_id'}, + {related_model: PullRequest, fk_name: 'pull_request_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ], + method: :destroy_all, + dry_run_complement: -> (ids) { add_builds_dependencies_to_dry_run_report(ids) } + }, { + main_model: Job, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: Stage, fk_name: 'stage_id'}, + ] + }, { + main_model: Branch, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Tag, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Commit, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ] + }, { + main_model: Cron, + relations: [ + {related_model: Branch, fk_name: 'branch_id'} + ] + }, { + main_model: PullRequest, + relations: [ + {related_model: Repository, fk_name: 'repository_id'} + ] + }, { + main_model: SslKey, + relations: [ + {related_model: Repository, fk_name: 'repository_id'} + ] + }, { + main_model: Request, + relations: [ + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: PullRequest, fk_name: 'pull_request_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ] + }, { + main_model: Stage, + relations: [ + {related_model: Build, fk_name: 'build_id'} + ] + } + ] + + cases.each do |model_block| + model_block[:relations].each do |relation| + remove_orphans_for_table( + main_model: model_block[:main_model], + related_model: relation[:related_model], + fk_name: relation[:fk_name], + method: model_block[:method], + dry_run_complement: model_block[:dry_run_complement] + ) + end end - remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id') - remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id') - remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id') - remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id') - remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id') - remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id') - remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id') - remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id') - remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id') - remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id') - remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id') - remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id') - remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id') - remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id') - remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id') - remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id') - remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id') - remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id') end def add_builds_dependencies_to_dry_run_report(ids_for_delete) @@ -186,11 +229,20 @@ def add_builds_dependencies_to_dry_run_report(ids_for_delete) dry_run_report[:jobs].concat(jobs_for_delete.map(&:id)) end - def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name, method=:delete_all) - for_delete = model_class.find_by_sql(%{ + def remove_orphans_for_table(args) + main_model = args[:main_model] + related_model = args[:related_model] + fk_name = args[:fk_name] + method = args[:method] || :delete_all + dry_run_complement = args[:dry_run_complement] + + main_table = main_model.table_name + related_table = related_model.table_name + + for_delete = main_model.find_by_sql(%{ select a.* - from #{table_a_name} a - left join #{table_b_name} b + from #{main_table} a + left join #{related_table} b on a.#{fk_name} = b.id where a.#{fk_name} is not null @@ -200,15 +252,15 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name, m ids_for_delete = for_delete.map(&:id) if config.dry_run - key = table_a_name.to_sym + key = main_table.to_sym dry_run_report[key] = [] if dry_run_report[key].nil? dry_run_report[key].concat(ids_for_delete) dry_run_report[key].uniq! - yield(ids_for_delete) if block_given? + dry_run_complement.call(ids_for_delete) if dry_run_complement else - model_class.where(id: ids_for_delete).send(method) + main_model.where(id: ids_for_delete).send(method) end end From 2e2b9f9c8d0e582d706c2583ff0fb2c84a954195 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Wed, 15 Sep 2021 12:10:15 +0200 Subject: [PATCH 67/84] problem with updated_at --- spec/backup_spec.rb | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 94cde56..be20dc5 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -385,11 +385,14 @@ def destination_logs_size let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } let!(:repository) { - FactoryBot.create( + ActiveRecord::Base.connection.execute('set session_replication_role = replica;') + repository = FactoryBot.create( :repository_with_builds_jobs_and_logs, created_at: datetime, updated_at: datetime ) + ActiveRecord::Base.connection.execute('set session_replication_role = default;') + repository } let!(:repository2) { FactoryBot.create( @@ -438,24 +441,9 @@ def destination_logs_size end context 'when if_backup config is set to true' do - it 'should prepare proper JSON export' do - result = backup.process_repo_builds(repository) - result.first.first['updated_at'] = datetime - result.first.second['updated_at'] = datetime - result.first.first[:jobs].first['updated_at'] = datetime - result.first.first[:jobs].second['updated_at'] = datetime - result.first.second[:jobs].first['updated_at'] = datetime - result.first.second[:jobs].second['updated_at'] = datetime - result.first.first[:jobs].first[:logs].first['updated_at'] = datetime - result.first.first[:jobs].first[:logs].second['updated_at'] = datetime - result.first.first[:jobs].second[:logs].first['updated_at'] = datetime - result.first.first[:jobs].second[:logs].second['updated_at'] = datetime - result.first.second[:jobs].first[:logs].first['updated_at'] = datetime - result.first.second[:jobs].first[:logs].second['updated_at'] = datetime - result.first.second[:jobs].second[:logs].first['updated_at'] = datetime - result.first.second[:jobs].second[:logs].second['updated_at'] = datetime - - expect(result.to_json).to eq(expected_builds_json.to_json) + it 'should save proper build JSON to file' do + expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_builds_json.first)) + backup.process_repo_builds(repository) end it 'should save JSON to file at proper path' do @@ -545,9 +533,9 @@ def destination_logs_size end context 'when if_backup config is set to true' do - it 'should prepare proper JSON export' do - result = backup.process_repo_requests(repository) - expect(result.to_json).to eq(expected_requests_json.to_json) + it 'should save proper build JSON to file' do + expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_requests_json.first)) + backup.process_repo_requests(repository) end it 'should save JSON to file at proper path' do From 3dcff78df41cb7ff22978c75ea18021e43a50188 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 10:41:52 +0200 Subject: [PATCH 68/84] new file format --- lib/travis-backup.rb | 112 ++++++++--- spec/backup_spec.rb | 127 +++++++++++- spec/support/expected_files.rb | 344 ++++++++++----------------------- 3 files changed, 299 insertions(+), 284 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index ee3e252..abbcda7 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -97,8 +97,13 @@ def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/M current_build_id = repository.current_build_id || -1 repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch) - end.compact + if @config.if_backup + file_prefix = "repository_#{repository.id}" + save_and_destroy_builds_batch(builds_batch, file_prefix) + else + destroy_builds_batch(builds_batch) + end + end.compact.reduce(&:&) end def process_repo_requests(repository) @@ -266,14 +271,57 @@ def remove_orphans_for_table(args) private - def save_and_destroy_builds_batch(builds_batch, repository) - builds_export = export_builds(builds_batch) - file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" - pretty_json = JSON.pretty_generate(builds_export) - if save_file(file_name, pretty_json) - destroy_builds_batch(builds_batch) + def save_and_destroy_builds_batch(builds_batch, file_prefix) + builds_export = builds_batch.map(&:attributes) + + dependencies_saved = builds_batch.map do |build| + save_build_jobs_and_logs(build, file_prefix) + end.reduce(&:&) + + if dependencies_saved + file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" + pretty_json = JSON.pretty_generate(builds_export) + save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false + else + false + end + end + + def save_build_jobs_and_logs(build, file_prefix) + build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| + file_prefix = "#{file_prefix}_build_#{build.id}" + save_jobs_batch(jobs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_jobs_batch(jobs_batch, file_prefix) + jobs_export = jobs_batch.map(&:attributes) + + logs_saved = jobs_batch.map do |job| + save_job_logs(job, file_prefix) + end.reduce(&:&) + + if logs_saved + file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(jobs_export) + save_file(file_name, pretty_json) + else + false end - builds_export + end + + def save_job_logs(job, file_prefix) + job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| + file_prefix = "#{file_prefix}_job_#{job.id}" + save_logs_batch(logs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_logs_batch(logs_batch, file_prefix) + logs_export = logs_batch.map(&:attributes) + file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(logs_export) + save_file(file_name, pretty_json) end def destroy_builds_batch(builds_batch) @@ -344,29 +392,29 @@ def file_path(file_name) "#{@config.files_location}/#{file_name}" end - def export_builds(builds) - builds.map do |build| - build_export = build.attributes - build_export[:jobs] = export_jobs(build.jobs) - - build_export - end - end - - def export_jobs(jobs) - jobs.map do |job| - job_export = job.attributes - job_export[:logs] = export_logs(job.logs) - - job_export - end - end - - def export_logs(logs) - logs.map do |log| - log.attributes - end - end + # def export_builds(builds) + # builds.map do |build| + # build_export = build.attributes + # build_export[:jobs] = export_jobs(build.jobs) + + # build_export + # end + # end + + # def export_jobs(jobs) + # jobs.map do |job| + # job_export = job.attributes + # job_export[:logs] = export_logs(job.logs) + + # job_export + # end + # end + + # def export_logs(logs) + # logs.map do |log| + # log.attributes + # end + # end def export_requests(requests) requests.map do |request| diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index be20dc5..5103833 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -402,10 +402,23 @@ def destination_logs_size builds_count: 1 ) } - - + let(:expected_files_creator) { + ExpectedFiles.new(repository, datetime) + } let!(:expected_builds_json) { - ExpectedFiles.new(repository, datetime).builds_json + expected_files_creator.builds_json + } + let!(:expected_jobs_jsons) { + repository.builds.map do |build| + expected_files_creator.jobs_json(build) + end + } + let!(:expected_logs_jsons) { + repository.builds.map do |build| + build.jobs.map do |job| + expected_files_creator.logs_json(job) + end + end.flatten(1) } shared_context 'removing builds and jobs' do @@ -441,14 +454,78 @@ def destination_logs_size end context 'when if_backup config is set to true' do - it 'should save proper build JSON to file' do - expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_builds_json.first)) - backup.process_repo_builds(repository) + it 'should save proper build JSON file' do + expect_method_calls_on( + File, :write, + [JSON.pretty_generate(expected_builds_json)], + allow_instances: true, + arguments_to_check: :first + ) do + backup.process_repo_builds(repository) + end end - it 'should save JSON to file at proper path' do - expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') - backup.process_repo_builds(repository) + it 'should save proper job JSON files' do + expect_method_calls_on( + File, :write, + [ + JSON.pretty_generate(expected_jobs_jsons.first), + JSON.pretty_generate(expected_jobs_jsons.second) + ], + allow_instances: true, + arguments_to_check: :first + ) do + backup.process_repo_builds(repository) + end + end + + it 'should save proper log JSON files' do + expect_method_calls_on( + File, :write, + [ + JSON.pretty_generate(expected_logs_jsons.first), + JSON.pretty_generate(expected_logs_jsons.second), + JSON.pretty_generate(expected_logs_jsons.third), + JSON.pretty_generate(expected_logs_jsons.fourth), + ], + allow_instances: true, + arguments_to_check: :first + ) do + backup.process_repo_builds(repository) + end + end + + it 'should save JSON files at proper paths' do + expect_method_calls_on( + File, :open, + [ + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_builds_\d+-\d+.json') + ], + match_mode: :match, + arguments_to_check: :first + ) do + backup.process_repo_builds(repository) + end + + + + # saved_files_paths = [] + + # allow(File).to receive(:open).and_wrap_original do |method, *args, &block| + # saved_files_paths.push(args.first) + # method.call(*args, &block) + # end + + # backup.process_repo_builds(repository) + + # expect(saved_files_paths).to match_array([ + # ]) end it_behaves_like 'removing builds and jobs' @@ -534,7 +611,7 @@ def destination_logs_size context 'when if_backup config is set to true' do it 'should save proper build JSON to file' do - expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_requests_json.first)) + expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_requests_json)) backup.process_repo_requests(repository) end @@ -576,3 +653,33 @@ def destination_logs_size end end end + +def expect_method_calls_on(cl, method, call_with, options) + match_mode = options[:mode] || :including + allow_instances = options[:allow_instances] || false + arguments_to_check = options[:arguments_to_check] || :all + + calls_args = [] + + allowed = allow_instances ? allow_any_instance_of(cl) : allow(cl) + + allowed.to receive(method).and_wrap_original do |method, *args, &block| + if arguments_to_check == :all + calls_args.push(args) + else + calls_args.push(args.send(arguments_to_check)) # = args.first, args.second, args.third etc. + end + method.call(*args, &block) + end + + yield + + case match_mode + when :including + call_with.each do |args| + expect(calls_args).to include(args) + end + when :match + expect(call_with).to match_array(calls_args) + end +end diff --git a/spec/support/expected_files.rb b/spec/support/expected_files.rb index b39ce2b..74c82cb 100644 --- a/spec/support/expected_files.rb +++ b/spec/support/expected_files.rb @@ -5,7 +5,7 @@ def initialize(repository, datetime) end def builds_json - [[ + [ { "id": @repository.builds.first.id, "repository_id": @repository.id, @@ -34,125 +34,7 @@ def builds_json "branch_id": nil, "tag_id": nil, "sender_id": nil, - "sender_type": nil, - "jobs": [{ - "id": @repository.builds.first.jobs.first.id, - "repository_id": @repository.id, - "commit_id": nil, - "source_id": @repository.builds.first.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": @datetime, - "updated_at": @datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": @repository.builds.first.jobs.first.logs.first.id, - "job_id": @repository.builds.first.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": @repository.builds.first.jobs.first.logs.second.id, - "job_id": @repository.builds.first.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }, - { - "id": @repository.builds.first.jobs.second.id, - "repository_id": @repository.id, - "commit_id": nil, - "source_id": @repository.builds.first.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": @datetime, - "updated_at": @datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": @repository.builds.first.jobs.second.logs.first.id, - "job_id": @repository.builds.first.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": @repository.builds.first.jobs.second.logs.second.id, - "job_id": @repository.builds.first.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }] + "sender_type": nil }, { "id": @repository.builds.second.id, @@ -183,130 +65,108 @@ def builds_json "tag_id": nil, "sender_id": nil, "sender_type": nil, - "jobs": [{ - "id": @repository.builds.second.jobs.first.id, - "repository_id": @repository.id, - "commit_id": nil, - "source_id": @repository.builds.second.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": @datetime, - "updated_at": @datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": @repository.builds.second.jobs.first.logs.first.id, - "job_id": @repository.builds.second.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": @repository.builds.second.jobs.first.logs.second.id, - "job_id": @repository.builds.second.jobs.first.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }, - { - "id": @repository.builds.second.jobs.second.id, - "repository_id": @repository.id, - "commit_id": nil, - "source_id": @repository.builds.second.id, - "source_type": "Build", - "queue": nil, - "type": nil, - "state": nil, - "number": nil, - "config": nil, - "worker": nil, - "started_at": nil, - "finished_at": nil, - "created_at": @datetime, - "updated_at": @datetime, - "tags": nil, - "allow_failure": false, - "owner_id": nil, - "owner_type": nil, - "result": nil, - "queued_at": nil, - "canceled_at": nil, - "received_at": nil, - "debug_options": nil, - "private": nil, - "stage_number": nil, - "stage_id": nil, - "logs": [ - { - "id": @repository.builds.second.jobs.second.logs.first.id, - "job_id": @repository.builds.second.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - }, - { - "id": @repository.builds.second.jobs.second.logs.second.id, - "job_id": @repository.builds.second.jobs.second.id, - "content": "some log content", - "removed_by": nil, - "created_at": @datetime, - "updated_at": @datetime, - "aggregated_at": nil, - "archived_at": nil, - "purged_at": nil, - "removed_at": nil, - "archiving": false, - "archive_verified": true - } - ] - }] } - ]] + ] + end + + def jobs_json(build) + [ + { + "id": build.jobs.first.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": build.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + }, + { + "id": build.jobs.second.id, + "repository_id": @repository.id, + "commit_id": nil, + "source_id": build.id, + "source_type": "Build", + "queue": nil, + "type": nil, + "state": nil, + "number": nil, + "config": nil, + "worker": nil, + "started_at": nil, + "finished_at": nil, + "created_at": @datetime, + "updated_at": @datetime, + "tags": nil, + "allow_failure": false, + "owner_id": nil, + "owner_type": nil, + "result": nil, + "queued_at": nil, + "canceled_at": nil, + "received_at": nil, + "debug_options": nil, + "private": nil, + "stage_number": nil, + "stage_id": nil, + } + ] + end + + def logs_json(job) + [ + { + "id": job.logs.first.id, + "job_id": job.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + }, + { + "id": job.logs.second.id, + "job_id": job.id, + "content": "some log content", + "removed_by": nil, + "created_at": @datetime, + "updated_at": @datetime, + "aggregated_at": nil, + "archived_at": nil, + "purged_at": nil, + "removed_at": nil, + "archiving": false, + "archive_verified": true + } + ] end def requests_json - [[ + [ { "id": @repository.requests.first.id, "repository_id": @repository.id, @@ -363,6 +223,6 @@ def requests_json "sender_id": nil, "sender_type": nil } - ]] + ] end -end \ No newline at end of file +end From 2ad1423d6beb25b6815c4c263d33ecd2ab85156d Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 10:46:38 +0200 Subject: [PATCH 69/84] v0.2.0 --- Gemfile.lock | 2 +- travis-backup.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8b33b7b..f83f7ad 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - travis-backup (0.1.3) + travis-backup (0.2.0) activerecord bootsnap pg diff --git a/travis-backup.gemspec b/travis-backup.gemspec index ebe8cf3..47c0572 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.1.3' + s.version = '0.2.0' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0") From 94f10b7dc0e5935fa199624754dd9bc4759047ec Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 11:05:23 +0200 Subject: [PATCH 70/84] commented code removed --- lib/travis-backup.rb | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index abbcda7..f3f3718 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -392,30 +392,6 @@ def file_path(file_name) "#{@config.files_location}/#{file_name}" end - # def export_builds(builds) - # builds.map do |build| - # build_export = build.attributes - # build_export[:jobs] = export_jobs(build.jobs) - - # build_export - # end - # end - - # def export_jobs(jobs) - # jobs.map do |job| - # job_export = job.attributes - # job_export[:logs] = export_logs(job.logs) - - # job_export - # end - # end - - # def export_logs(logs) - # logs.map do |log| - # log.attributes - # end - # end - def export_requests(requests) requests.map do |request| request.attributes From 035b1cd731407c3a65bd03451c7fab42b7ae7c10 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 11:41:46 +0200 Subject: [PATCH 71/84] add_to_dry_report method --- lib/travis-backup.rb | 31 ++++++++++++++++++------------- spec/backup_spec.rb | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index f3f3718..74c3bfa 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -23,7 +23,7 @@ def initialize(config_args={}) @config = Config.new(config_args) if @config.dry_run - @dry_run_report = {builds: [], jobs: [], logs: [], requests: []} + @dry_run_report = {} end connect_db @@ -71,6 +71,12 @@ def process_all_repos end end + def add_to_dry_report(key, *values) + dry_run_report[key] = [] if dry_run_report[key].nil? + dry_run_report[key].concat(values) + dry_run_report[key].uniq! + end + def print_dry_run_report if @dry_run_report.to_a.map(&:second).flatten.empty? puts 'Dry run active. No data would be removed in normal run.' @@ -134,7 +140,8 @@ def move_logs end def move_logs_dry - dry_run_report[:logs].concat(Log.order(:id).map(&:id)) + ids = Log.order(:id).map(&:id) + add_to_dry_report(:logs, *ids) end def remove_orphans @@ -230,8 +237,8 @@ def remove_orphans def add_builds_dependencies_to_dry_run_report(ids_for_delete) repos_for_delete = Repository.where(current_build_id: ids_for_delete) jobs_for_delete = Job.where(source_id: ids_for_delete) - dry_run_report[:repositories].concat(repos_for_delete.map(&:id)) - dry_run_report[:jobs].concat(jobs_for_delete.map(&:id)) + add_to_dry_report(:repositories, *repos_for_delete.map(&:id)) + add_to_dry_report(:jobs, *jobs_for_delete.map(&:id)) end def remove_orphans_for_table(args) @@ -259,9 +266,7 @@ def remove_orphans_for_table(args) if config.dry_run key = main_table.to_sym - dry_run_report[key] = [] if dry_run_report[key].nil? - dry_run_report[key].concat(ids_for_delete) - dry_run_report[key].uniq! + add_to_dry_report(key, *ids_for_delete) dry_run_complement.call(ids_for_delete) if dry_run_complement else @@ -331,21 +336,21 @@ def destroy_builds_batch(builds_batch) end def destroy_builds_batch_dry(builds_batch) - @dry_run_report[:builds].concat(builds_batch.map(&:id)) + add_to_dry_report(:builds, *builds_batch.map(&:id)) - jobs = builds_batch.map do |build| + jobs_ids = builds_batch.map do |build| build.jobs.map(&:id) || [] end.flatten - @dry_run_report[:jobs].concat(jobs) + add_to_dry_report(:jobs, *jobs_ids) - logs = builds_batch.map do |build| + logs_ids = builds_batch.map do |build| build.jobs.map do |job| job.logs.map(&:id) || [] end.flatten || [] end.flatten - @dry_run_report[:logs].concat(logs) + add_to_dry_report(:logs, *logs_ids) end def save_and_destroy_requests_batch(requests_batch, repository) @@ -365,7 +370,7 @@ def destroy_requests_batch(requests_batch) end def destroy_requests_batch_dry(requests_batch) - @dry_run_report[:requests].concat(requests_batch.map(&:id)) + add_to_dry_report(:requests, *requests_batch.map(&:id)) end def save_file(file_name, content) # rubocop:disable Metrics/MethodLength diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 5103833..4f9f184 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -213,7 +213,7 @@ def destination_logs_size let!(:backup) { Backup.new(dry_run: true) } before do - allow_any_instance_of(IO).to receive(:puts) + # allow_any_instance_of(IO).to receive(:puts) end it 'prepares proper dry run report' do From 7013e9c87ad14370762fb8ba5372e80d2d64e735 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 16:05:14 +0200 Subject: [PATCH 72/84] refactoring wip --- lib/backup/move_logs.rb | 37 +++++++++++++++++ lib/db_helper.rb | 12 ++++++ lib/dry_run_reporter.rb | 33 ++++++++++++++++ lib/travis-backup.rb | 77 ++++++++---------------------------- spec/backup_spec.rb | 64 +++--------------------------- spec/move_logs_spec.rb | 71 +++++++++++++++++++++++++++++++++ spec/support/before_tests.rb | 11 ++++++ 7 files changed, 186 insertions(+), 119 deletions(-) create mode 100644 lib/backup/move_logs.rb create mode 100644 lib/db_helper.rb create mode 100644 lib/dry_run_reporter.rb create mode 100644 spec/move_logs_spec.rb create mode 100644 spec/support/before_tests.rb diff --git a/lib/backup/move_logs.rb b/lib/backup/move_logs.rb new file mode 100644 index 0000000..c3432a2 --- /dev/null +++ b/lib/backup/move_logs.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class Backup + class MoveLogs + attr_reader :config + + def initialize(config, db_helper, dry_run_reporter=nil) + @config = config + @dry_run_reporter = dry_run_reporter + @db_helper = db_helper + end + + def run + return run_dry if @config.dry_run + + @db_helper.connect_db(@config.database_url) + Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch| + log_hashes = logs_batch.as_json + @db_helper.connect_db(@config.destination_db_url) + + log_hashes.each do |log_hash| + new_log = Log.new(log_hash) + new_log.save! + end + + @db_helper.connect_db(@config.database_url) + + logs_batch.each(&:destroy) + end + end + + def run_dry + ids = Log.order(:id).map(&:id) + @dry_run_reporter.add_to_report(:logs, *ids) + end + end +end diff --git a/lib/db_helper.rb b/lib/db_helper.rb new file mode 100644 index 0000000..319b3be --- /dev/null +++ b/lib/db_helper.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class DbHelper + def initialize(config) + @config = config + connect_db + end + + def connect_db(url=@config.database_url) + ActiveRecord::Base.establish_connection(url) + end +end \ No newline at end of file diff --git a/lib/dry_run_reporter.rb b/lib/dry_run_reporter.rb new file mode 100644 index 0000000..9ae2a86 --- /dev/null +++ b/lib/dry_run_reporter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class DryRunReporter + attr_reader :report + + def initialize + @report = {} + end + + def add_to_report(key, *values) + report[key] = [] if report[key].nil? + report[key].concat(values) + report[key].uniq! + end + + def print_report + if @report.to_a.map(&:second).flatten.empty? + puts 'Dry run active. No data would be removed in normal run.' + else + puts 'Dry run active. The following data would be removed in normal run:' + + @report.to_a.map(&:first).each do |symbol| + print_report_line(symbol) + end + end + end + + private + + def print_report_line(symbol) + puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any? + end +end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 74c3bfa..63464bf 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -3,6 +3,8 @@ require 'active_support/core_ext/array' require 'active_support/time' require 'config' +require 'db_helper' +require 'dry_run_reporter' require 'models/repository' require 'models/log' require 'models/branch' @@ -13,24 +15,25 @@ require 'models/ssl_key' require 'models/request' require 'models/stage' +require 'backup/move_logs' # main travis-backup class class Backup attr_accessor :config - attr_reader :dry_run_report def initialize(config_args={}) @config = Config.new(config_args) + @db_helper = DbHelper.new(@config) if @config.dry_run - @dry_run_report = {} + @dry_run_reporter = DryRunReporter.new end - connect_db + @db_helper.connect_db end - def connect_db(url=@config.database_url) - ActiveRecord::Base.establish_connection(url) + def dry_run_report + @dry_run_reporter.report end def run(args={}) @@ -39,7 +42,7 @@ def run(args={}) org_id = args[:org_id] || @config.org_id if @config.move_logs - move_logs + Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run elsif @config.remove_orphans remove_orphans elsif user_id @@ -52,7 +55,7 @@ def run(args={}) process_all_repos end - print_dry_run_report if @config.dry_run + @dry_run_reporter.print_report if @config.dry_run end def process_repos_for_owner(owner_id, owner_type) @@ -71,28 +74,6 @@ def process_all_repos end end - def add_to_dry_report(key, *values) - dry_run_report[key] = [] if dry_run_report[key].nil? - dry_run_report[key].concat(values) - dry_run_report[key].uniq! - end - - def print_dry_run_report - if @dry_run_report.to_a.map(&:second).flatten.empty? - puts 'Dry run active. No data would be removed in normal run.' - else - puts 'Dry run active. The following data would be removed in normal run:' - - @dry_run_report.to_a.map(&:first).each do |symbol| - print_dry_run_report_line(symbol) - end - end - end - - def print_dry_run_report_line(symbol) - puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any? - end - def process_repo(repository) process_repo_builds(repository) process_repo_requests(repository) @@ -120,30 +101,6 @@ def process_repo_requests(repository) end.compact end - def move_logs - return move_logs_dry if config.dry_run - - connect_db(@config.database_url) - Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch| - log_hashes = logs_batch.as_json - connect_db(@config.destination_db_url) - - log_hashes.each do |log_hash| - new_log = Log.new(log_hash) - new_log.save! - end - - connect_db(@config.database_url) - - logs_batch.each(&:destroy) - end - end - - def move_logs_dry - ids = Log.order(:id).map(&:id) - add_to_dry_report(:logs, *ids) - end - def remove_orphans cases = [ { @@ -237,8 +194,8 @@ def remove_orphans def add_builds_dependencies_to_dry_run_report(ids_for_delete) repos_for_delete = Repository.where(current_build_id: ids_for_delete) jobs_for_delete = Job.where(source_id: ids_for_delete) - add_to_dry_report(:repositories, *repos_for_delete.map(&:id)) - add_to_dry_report(:jobs, *jobs_for_delete.map(&:id)) + @dry_run_reporter.add_to_report(:repositories, *repos_for_delete.map(&:id)) + @dry_run_reporter.add_to_report(:jobs, *jobs_for_delete.map(&:id)) end def remove_orphans_for_table(args) @@ -266,7 +223,7 @@ def remove_orphans_for_table(args) if config.dry_run key = main_table.to_sym - add_to_dry_report(key, *ids_for_delete) + @dry_run_reporter.add_to_report(key, *ids_for_delete) dry_run_complement.call(ids_for_delete) if dry_run_complement else @@ -336,13 +293,13 @@ def destroy_builds_batch(builds_batch) end def destroy_builds_batch_dry(builds_batch) - add_to_dry_report(:builds, *builds_batch.map(&:id)) + @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id)) jobs_ids = builds_batch.map do |build| build.jobs.map(&:id) || [] end.flatten - add_to_dry_report(:jobs, *jobs_ids) + @dry_run_reporter.add_to_report(:jobs, *jobs_ids) logs_ids = builds_batch.map do |build| build.jobs.map do |job| @@ -350,7 +307,7 @@ def destroy_builds_batch_dry(builds_batch) end.flatten || [] end.flatten - add_to_dry_report(:logs, *logs_ids) + @dry_run_reporter.add_to_report(:logs, *logs_ids) end def save_and_destroy_requests_batch(requests_batch, repository) @@ -370,7 +327,7 @@ def destroy_requests_batch(requests_batch) end def destroy_requests_batch_dry(requests_batch) - add_to_dry_report(:requests, *requests_batch.map(&:id)) + @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id)) end def save_file(file_name, content) # rubocop:disable Metrics/MethodLength diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 4f9f184..66c899b 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -5,73 +5,19 @@ require 'models/job' require 'models/organization' require 'models/user' - require 'support/factories' require 'support/expected_files' +require 'support/before_tests' require 'pry' describe Backup do before(:all) do - ARGV = ['-t', '6'] - config = Config.new - system("psql '#{config.database_url}' -f db/schema.sql > /dev/null") - system("psql '#{config.destination_db_url}' -f db/schema.sql > /dev/null") if config.destination_db_url + BeforeTests.new.run end let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location, limit: 5) } - describe 'move_logs' do - let!(:logs) { - FactoryBot.create_list( - :log, - 10, - job_id: 1, - content: 'some log content', - removed_by: 1, - archiving: false, - archive_verified: true - ) - } - - def connect_db(url) - ActiveRecord::Base.establish_connection(url) - end - - def do_in_destination_db(config) - connect_db(config.destination_db_url) - result = yield - connect_db(config.database_url) - result - end - - it 'copies logs to destination database' do - def destination_logs_size - do_in_destination_db(backup.config) do - Log.all.size - end - end - - source_db_logs = Log.all.as_json - - expect { - backup.move_logs - }.to change { destination_logs_size }.by 10 - - destination_db_logs = do_in_destination_db(backup.config) do - Log.all.as_json - end - - expect(destination_db_logs).to eql(source_db_logs) - end - - it 'removes copied logs from source database' do - expect { - backup.move_logs - }.to change { Log.all.size }.by -10 - end - end - describe 'remove_orphans' do after(:all) do Repository.destroy_all @@ -231,7 +177,7 @@ def destination_logs_size end it 'prints dry run report' do - expect(backup).to receive(:print_dry_run_report).once + expect_any_instance_of(DryRunReporter).to receive(:print_report).once backup.run end end @@ -318,7 +264,7 @@ def destination_logs_size end it 'moves logs' do - expect(backup).to receive(:move_logs).once + expect_any_instance_of(Backup::MoveLogs).to receive(:run).once backup.run end end @@ -353,7 +299,7 @@ def destination_logs_size end it 'prints dry run report' do - expect(backup).to receive(:print_dry_run_report).once + expect_any_instance_of(DryRunReporter).to receive(:print_report).once backup.run end end diff --git a/spec/move_logs_spec.rb b/spec/move_logs_spec.rb new file mode 100644 index 0000000..0fe7714 --- /dev/null +++ b/spec/move_logs_spec.rb @@ -0,0 +1,71 @@ +$: << 'lib' +require 'uri' +require 'travis-backup' +require 'models/build' +require 'models/job' +require 'models/organization' +require 'models/user' +require 'support/factories' +require 'support/expected_files' +require 'support/before_tests' +require 'pry' + + +describe Backup::MoveLogs do + before(:all) do + BeforeTests.new.run + end + + let(:files_location) { "dump/tests" } + let!(:config) { Config.new(files_location: files_location, limit: 5) } + let!(:db_helper) { DbHelper.new(config) } + let!(:move_logs) { Backup::MoveLogs.new(config, db_helper, DryRunReporter.new) } + + describe 'run' do + let!(:logs) { + FactoryBot.create_list( + :log, + 10, + job_id: 1, + content: 'some log content', + removed_by: 1, + archiving: false, + archive_verified: true + ) + } + + def do_in_destination_db(config) + db_helper.connect_db(config.destination_db_url) + result = yield + db_helper.connect_db(config.database_url) + result + end + + it 'copies logs to destination database' do + + def destination_logs_size + do_in_destination_db(move_logs.config) do + Log.all.size + end + end + + source_db_logs = Log.all.as_json + + expect { + move_logs.run + }.to change { destination_logs_size }.by 10 + + destination_db_logs = do_in_destination_db(move_logs.config) do + Log.all.as_json + end + + expect(destination_db_logs).to eql(source_db_logs) + end + + it 'removes copied logs from source database' do + expect { + move_logs.run + }.to change { Log.all.size }.by -10 + end + end +end diff --git a/spec/support/before_tests.rb b/spec/support/before_tests.rb new file mode 100644 index 0000000..d3fb128 --- /dev/null +++ b/spec/support/before_tests.rb @@ -0,0 +1,11 @@ +class BeforeTests + def run + config = Config.new + system("psql '#{config.database_url}' -f db/schema.sql > /dev/null 2> /dev/null") + if config.destination_db_url + system("psql '#{config.destination_db_url}' -f db/schema.sql > /dev/null 2> /dev/null") + end + end +end + +ARGV = ['-t', '6'] From 14530428d039fde268c88f2f497220161420855d Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 16:23:24 +0200 Subject: [PATCH 73/84] do_in_other_db - refactoring wip --- lib/db_helper.rb | 12 ++++++++++-- spec/move_logs_spec.rb | 20 ++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/db_helper.rb b/lib/db_helper.rb index 319b3be..f8b06b3 100644 --- a/lib/db_helper.rb +++ b/lib/db_helper.rb @@ -6,7 +6,15 @@ def initialize(config) connect_db end - def connect_db(url=@config.database_url) - ActiveRecord::Base.establish_connection(url) + def connect_db(config_or_url=@config.database_url) + ActiveRecord::Base.establish_connection(config_or_url) + end + + def do_in_other_db(config_or_url) + saved_config = ActiveRecord::Base.connection_db_config + connect_db(config_or_url) + result = yield + connect_db(saved_config) + result end end \ No newline at end of file diff --git a/spec/move_logs_spec.rb b/spec/move_logs_spec.rb index 0fe7714..cf2480e 100644 --- a/spec/move_logs_spec.rb +++ b/spec/move_logs_spec.rb @@ -34,28 +34,24 @@ ) } - def do_in_destination_db(config) - db_helper.connect_db(config.destination_db_url) - result = yield - db_helper.connect_db(config.database_url) - result + def do_in_destination_db(&block) + db_helper.do_in_other_db(config.destination_db_url, &block) end - it 'copies logs to destination database' do - - def destination_logs_size - do_in_destination_db(move_logs.config) do - Log.all.size - end + def destination_logs_size + do_in_destination_db do + Log.all.size end + end + it 'copies logs to destination database' do source_db_logs = Log.all.as_json expect { move_logs.run }.to change { destination_logs_size }.by 10 - destination_db_logs = do_in_destination_db(move_logs.config) do + destination_db_logs = do_in_destination_db do Log.all.as_json end From d3e142aa56c70c9bf14c83a7cec8d3b9787dd597 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 17:06:42 +0200 Subject: [PATCH 74/84] Backup::RemoveOrphans - refactoring wip --- lib/backup/remove_orphans.rb | 146 ++++++++++++++++++++++++ lib/travis-backup.rb | 133 +--------------------- spec/{ => backup}/move_logs_spec.rb | 0 spec/backup/remove_orphans_spec.rb | 165 +++++++++++++++++++++++++++ spec/backup_spec.rb | 167 +--------------------------- 5 files changed, 314 insertions(+), 297 deletions(-) create mode 100644 lib/backup/remove_orphans.rb rename spec/{ => backup}/move_logs_spec.rb (100%) create mode 100644 spec/backup/remove_orphans_spec.rb diff --git a/lib/backup/remove_orphans.rb b/lib/backup/remove_orphans.rb new file mode 100644 index 0000000..c0c5d78 --- /dev/null +++ b/lib/backup/remove_orphans.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +class Backup + class RemoveOrphans + attr_reader :config + + def initialize(config, dry_run_reporter=nil) + @config = config + @dry_run_reporter = dry_run_reporter + end + + def dry_run_report + @dry_run_reporter.report + end + + def run + cases.each do |model_block| + model_block[:relations].each do |relation| + process_table( + main_model: model_block[:main_model], + related_model: relation[:related_model], + fk_name: relation[:fk_name], + method: model_block[:method], + dry_run_complement: model_block[:dry_run_complement] + ) + end + end + end + + def cases + [ + { + main_model: Repository, + relations: [ + {related_model: Build, fk_name: 'current_build_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Build, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: Request, fk_name: 'request_id'}, + {related_model: PullRequest, fk_name: 'pull_request_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ], + method: :destroy_all, + dry_run_complement: -> (ids) { add_builds_dependencies_to_dry_run_report(ids) } + }, { + main_model: Job, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: Stage, fk_name: 'stage_id'}, + ] + }, { + main_model: Branch, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Tag, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Build, fk_name: 'last_build_id'} + ] + }, { + main_model: Commit, + relations: [ + {related_model: Repository, fk_name: 'repository_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ] + }, { + main_model: Cron, + relations: [ + {related_model: Branch, fk_name: 'branch_id'} + ] + }, { + main_model: PullRequest, + relations: [ + {related_model: Repository, fk_name: 'repository_id'} + ] + }, { + main_model: SslKey, + relations: [ + {related_model: Repository, fk_name: 'repository_id'} + ] + }, { + main_model: Request, + relations: [ + {related_model: Commit, fk_name: 'commit_id'}, + {related_model: PullRequest, fk_name: 'pull_request_id'}, + {related_model: Branch, fk_name: 'branch_id'}, + {related_model: Tag, fk_name: 'tag_id'} + ] + }, { + main_model: Stage, + relations: [ + {related_model: Build, fk_name: 'build_id'} + ] + } + ] + end + + def add_builds_dependencies_to_dry_run_report(ids_for_delete) + repos_for_delete = Repository.where(current_build_id: ids_for_delete) + jobs_for_delete = Job.where(source_id: ids_for_delete) + @dry_run_reporter.add_to_report(:repositories, *repos_for_delete.map(&:id)) + @dry_run_reporter.add_to_report(:jobs, *jobs_for_delete.map(&:id)) + end + + def process_table(args) + main_model = args[:main_model] + related_model = args[:related_model] + fk_name = args[:fk_name] + method = args[:method] || :delete_all + dry_run_complement = args[:dry_run_complement] + + main_table = main_model.table_name + related_table = related_model.table_name + + for_delete = main_model.find_by_sql(%{ + select a.* + from #{main_table} a + left join #{related_table} b + on a.#{fk_name} = b.id + where + a.#{fk_name} is not null + and b.id is null; + }) + + ids_for_delete = for_delete.map(&:id) + + if config.dry_run + key = main_table.to_sym + @dry_run_reporter.add_to_report(key, *ids_for_delete) + dry_run_complement.call(ids_for_delete) if dry_run_complement + else + main_model.where(id: ids_for_delete).send(method) + end + end + end +end diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 63464bf..4af2d1f 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -16,6 +16,7 @@ require 'models/request' require 'models/stage' require 'backup/move_logs' +require 'backup/remove_orphans' # main travis-backup class class Backup @@ -44,7 +45,7 @@ def run(args={}) if @config.move_logs Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run elsif @config.remove_orphans - remove_orphans + Backup::RemoveOrphans.new(@config, @dry_run_reporter).run elsif user_id process_repos_for_owner(user_id, 'User') elsif org_id @@ -101,136 +102,6 @@ def process_repo_requests(repository) end.compact end - def remove_orphans - cases = [ - { - main_model: Repository, - relations: [ - {related_model: Build, fk_name: 'current_build_id'}, - {related_model: Build, fk_name: 'last_build_id'} - ] - }, { - main_model: Build, - relations: [ - {related_model: Repository, fk_name: 'repository_id'}, - {related_model: Commit, fk_name: 'commit_id'}, - {related_model: Request, fk_name: 'request_id'}, - {related_model: PullRequest, fk_name: 'pull_request_id'}, - {related_model: Branch, fk_name: 'branch_id'}, - {related_model: Tag, fk_name: 'tag_id'} - ], - method: :destroy_all, - dry_run_complement: -> (ids) { add_builds_dependencies_to_dry_run_report(ids) } - }, { - main_model: Job, - relations: [ - {related_model: Repository, fk_name: 'repository_id'}, - {related_model: Commit, fk_name: 'commit_id'}, - {related_model: Stage, fk_name: 'stage_id'}, - ] - }, { - main_model: Branch, - relations: [ - {related_model: Repository, fk_name: 'repository_id'}, - {related_model: Build, fk_name: 'last_build_id'} - ] - }, { - main_model: Tag, - relations: [ - {related_model: Repository, fk_name: 'repository_id'}, - {related_model: Build, fk_name: 'last_build_id'} - ] - }, { - main_model: Commit, - relations: [ - {related_model: Repository, fk_name: 'repository_id'}, - {related_model: Branch, fk_name: 'branch_id'}, - {related_model: Tag, fk_name: 'tag_id'} - ] - }, { - main_model: Cron, - relations: [ - {related_model: Branch, fk_name: 'branch_id'} - ] - }, { - main_model: PullRequest, - relations: [ - {related_model: Repository, fk_name: 'repository_id'} - ] - }, { - main_model: SslKey, - relations: [ - {related_model: Repository, fk_name: 'repository_id'} - ] - }, { - main_model: Request, - relations: [ - {related_model: Commit, fk_name: 'commit_id'}, - {related_model: PullRequest, fk_name: 'pull_request_id'}, - {related_model: Branch, fk_name: 'branch_id'}, - {related_model: Tag, fk_name: 'tag_id'} - ] - }, { - main_model: Stage, - relations: [ - {related_model: Build, fk_name: 'build_id'} - ] - } - ] - - cases.each do |model_block| - model_block[:relations].each do |relation| - remove_orphans_for_table( - main_model: model_block[:main_model], - related_model: relation[:related_model], - fk_name: relation[:fk_name], - method: model_block[:method], - dry_run_complement: model_block[:dry_run_complement] - ) - end - end - end - - def add_builds_dependencies_to_dry_run_report(ids_for_delete) - repos_for_delete = Repository.where(current_build_id: ids_for_delete) - jobs_for_delete = Job.where(source_id: ids_for_delete) - @dry_run_reporter.add_to_report(:repositories, *repos_for_delete.map(&:id)) - @dry_run_reporter.add_to_report(:jobs, *jobs_for_delete.map(&:id)) - end - - def remove_orphans_for_table(args) - main_model = args[:main_model] - related_model = args[:related_model] - fk_name = args[:fk_name] - method = args[:method] || :delete_all - dry_run_complement = args[:dry_run_complement] - - main_table = main_model.table_name - related_table = related_model.table_name - - for_delete = main_model.find_by_sql(%{ - select a.* - from #{main_table} a - left join #{related_table} b - on a.#{fk_name} = b.id - where - a.#{fk_name} is not null - and b.id is null; - }) - - ids_for_delete = for_delete.map(&:id) - - if config.dry_run - key = main_table.to_sym - - @dry_run_reporter.add_to_report(key, *ids_for_delete) - - dry_run_complement.call(ids_for_delete) if dry_run_complement - else - main_model.where(id: ids_for_delete).send(method) - end - end - private def save_and_destroy_builds_batch(builds_batch, file_prefix) diff --git a/spec/move_logs_spec.rb b/spec/backup/move_logs_spec.rb similarity index 100% rename from spec/move_logs_spec.rb rename to spec/backup/move_logs_spec.rb diff --git a/spec/backup/remove_orphans_spec.rb b/spec/backup/remove_orphans_spec.rb new file mode 100644 index 0000000..fc7f3f0 --- /dev/null +++ b/spec/backup/remove_orphans_spec.rb @@ -0,0 +1,165 @@ +$: << 'lib' +require 'uri' +require 'travis-backup' +require 'models/build' +require 'models/job' +require 'models/organization' +require 'models/user' +require 'support/factories' +require 'support/expected_files' +require 'support/before_tests' +require 'pry' + + +describe Backup::RemoveOrphans do + before(:all) do + BeforeTests.new.run + end + + let(:files_location) { "dump/tests" } + let!(:config) { Config.new(files_location: files_location, limit: 5) } + let!(:connection) { DbHelper.new(config) } + let!(:remove_orphans) { Backup::RemoveOrphans.new(config, DryRunReporter.new) } + + describe 'run' do + after(:all) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + Branch.destroy_all + Tag.destroy_all + Commit.destroy_all + Cron.destroy_all + PullRequest.destroy_all + Request.destroy_all + Stage.destroy_all + end + + let!(:data) { + ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') + FactoryBot.create_list(:repository, 2) + FactoryBot.create_list(:build, 2) + FactoryBot.create_list(:job, 2) + FactoryBot.create_list(:branch, 2) + FactoryBot.create_list(:tag, 2) + FactoryBot.create_list(:commit, 2) + FactoryBot.create_list(:cron, 2) + FactoryBot.create_list(:pull_request, 2) + FactoryBot.create_list(:request, 2) + FactoryBot.create_list(:stage, 2) + FactoryBot.create_list(:repository_orphaned_on_current_build_id, 2) + FactoryBot.create_list(:repository_with_current_build_id, 2) + FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:repository_with_last_build_id, 2) + FactoryBot.create_list(:build_orphaned_on_repository_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_repository_id, 2) + FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_commit_id, 2) + FactoryBot.create_list(:build_orphaned_on_request_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_pull_request_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_pull_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_branch_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_branch_id, 2) + FactoryBot.create_list(:build_orphaned_on_tag_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_tag_id, 2) + FactoryBot.create_list(:job_orphaned_on_repository_id, 2) + FactoryBot.create_list(:job_with_repository_id, 2) + FactoryBot.create_list(:job_orphaned_on_commit_id, 2) + FactoryBot.create_list(:job_with_commit_id, 2) + FactoryBot.create_list(:job_orphaned_on_stage_id, 2) + FactoryBot.create_list(:job_with_stage_id, 2) + FactoryBot.create_list(:branch_orphaned_on_repository_id, 2) + FactoryBot.create_list(:branch_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:branch_with_last_build_id, 2) + FactoryBot.create_list(:tag_orphaned_on_repository_id, 2) + FactoryBot.create_list(:tag_with_repository_id, 2) + FactoryBot.create_list(:tag_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:tag_with_last_build_id, 2) + FactoryBot.create_list(:commit_orphaned_on_repository_id, 2) + FactoryBot.create_list(:commit_with_repository_id, 2) + FactoryBot.create_list(:commit_orphaned_on_branch_id, 2) + FactoryBot.create_list(:commit_with_branch_id, 2) + FactoryBot.create_list(:commit_orphaned_on_tag_id, 2) + FactoryBot.create_list(:commit_with_tag_id, 2) + FactoryBot.create_list(:cron_orphaned_on_branch_id, 2) + FactoryBot.create_list(:cron_with_branch_id, 2) + FactoryBot.create_list(:pull_request_orphaned_on_repository_id, 2) + FactoryBot.create_list(:pull_request_with_repository_id, 2) + FactoryBot.create_list(:request_orphaned_on_commit_id, 2) + FactoryBot.create_list(:request_with_commit_id, 2) + FactoryBot.create_list(:request_orphaned_on_pull_request_id, 2) + FactoryBot.create_list(:request_with_pull_request_id, 2) + FactoryBot.create_list(:request_orphaned_on_branch_id, 2) + FactoryBot.create_list(:request_with_branch_id, 2) + FactoryBot.create_list(:request_orphaned_on_tag_id, 2) + FactoryBot.create_list(:request_with_tag_id, 2) + FactoryBot.create_list(:stage_orphaned_on_build_id, 2) + FactoryBot.create_list(:stage_with_build_id, 2) + ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + } + it 'removes orphaned repositories (with these dependent on orphaned builds)' do + expect { remove_orphans.run }.to change { Repository.all.size }.by -16 + end + + it 'removes orphaned builds' do + expect { remove_orphans.run }.to change { Build.all.size }.by -12 + end + + it 'removes orphaned jobs' do + expect { remove_orphans.run }.to change { Job.all.size }.by -6 + end + + it 'removes orphaned branches' do + expect { remove_orphans.run }.to change { Branch.all.size }.by -4 + end + + it 'removes orphaned tags' do + expect { remove_orphans.run }.to change { Tag.all.size }.by -4 + end + + it 'removes orphaned commits' do + expect { remove_orphans.run }.to change { Commit.all.size }.by -6 + end + + it 'removes orphaned crons' do + expect { remove_orphans.run }.to change { Cron.all.size }.by -2 + end + + it 'removes orphaned pull requests' do + expect { remove_orphans.run }.to change { PullRequest.all.size }.by -2 + end + + it 'removes orphaned requests' do + expect { remove_orphans.run }.to change { Request.all.size }.by -8 + end + + it 'removes orphaned stages' do + expect { remove_orphans.run }.to change { Stage.all.size }.by -2 + end + + context 'when dry run mode is on' do + let!(:config) { Config.new(files_location: files_location, limit: 5, dry_run: true) } + let!(:remove_orphans) { Backup::RemoveOrphans.new(config, DryRunReporter.new) } + + before do + allow_any_instance_of(IO).to receive(:puts) + end + + it 'prepares proper dry run report' do + remove_orphans.run + report = remove_orphans.dry_run_report + expect(report[:repositories].size).to eql 16 + expect(report[:builds].size).to eql 12 + expect(report[:jobs].size).to eql 6 + expect(report[:branches].size).to eql 4 + expect(report[:tags].size).to eql 4 + expect(report[:commits].size).to eql 6 + expect(report[:crons].size).to eql 2 + expect(report[:pull_requests].size).to eql 2 + expect(report[:requests].size).to eql 8 + expect(report[:stages].size).to eql 2 + end + end + end +end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 66c899b..2e1c425 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -18,171 +18,6 @@ let(:files_location) { "dump/tests" } let!(:backup) { Backup.new(files_location: files_location, limit: 5) } - describe 'remove_orphans' do - after(:all) do - Repository.destroy_all - Build.destroy_all - Job.destroy_all - Branch.destroy_all - Tag.destroy_all - Commit.destroy_all - Cron.destroy_all - PullRequest.destroy_all - Request.destroy_all - Stage.destroy_all - end - - let!(:data) { - ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:repository, 2) - FactoryBot.create_list(:build, 2) - FactoryBot.create_list(:job, 2) - FactoryBot.create_list(:branch, 2) - FactoryBot.create_list(:tag, 2) - FactoryBot.create_list(:commit, 2) - FactoryBot.create_list(:cron, 2) - FactoryBot.create_list(:pull_request, 2) - FactoryBot.create_list(:request, 2) - FactoryBot.create_list(:stage, 2) - FactoryBot.create_list(:repository_orphaned_on_current_build_id, 2) - FactoryBot.create_list(:repository_with_current_build_id, 2) - FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:repository_with_last_build_id, 2) - FactoryBot.create_list(:build_orphaned_on_repository_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_repository_id, 2) - FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_commit_id, 2) - FactoryBot.create_list(:build_orphaned_on_request_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_pull_request_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_pull_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_branch_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_branch_id, 2) - FactoryBot.create_list(:build_orphaned_on_tag_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_tag_id, 2) - FactoryBot.create_list(:job_orphaned_on_repository_id, 2) - FactoryBot.create_list(:job_with_repository_id, 2) - FactoryBot.create_list(:job_orphaned_on_commit_id, 2) - FactoryBot.create_list(:job_with_commit_id, 2) - FactoryBot.create_list(:job_orphaned_on_stage_id, 2) - FactoryBot.create_list(:job_with_stage_id, 2) - FactoryBot.create_list(:branch_orphaned_on_repository_id, 2) - FactoryBot.create_list(:branch_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:branch_with_last_build_id, 2) - FactoryBot.create_list(:tag_orphaned_on_repository_id, 2) - FactoryBot.create_list(:tag_with_repository_id, 2) - FactoryBot.create_list(:tag_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:tag_with_last_build_id, 2) - FactoryBot.create_list(:commit_orphaned_on_repository_id, 2) - FactoryBot.create_list(:commit_with_repository_id, 2) - FactoryBot.create_list(:commit_orphaned_on_branch_id, 2) - FactoryBot.create_list(:commit_with_branch_id, 2) - FactoryBot.create_list(:commit_orphaned_on_tag_id, 2) - FactoryBot.create_list(:commit_with_tag_id, 2) - FactoryBot.create_list(:cron_orphaned_on_branch_id, 2) - FactoryBot.create_list(:cron_with_branch_id, 2) - FactoryBot.create_list(:pull_request_orphaned_on_repository_id, 2) - FactoryBot.create_list(:pull_request_with_repository_id, 2) - FactoryBot.create_list(:request_orphaned_on_commit_id, 2) - FactoryBot.create_list(:request_with_commit_id, 2) - FactoryBot.create_list(:request_orphaned_on_pull_request_id, 2) - FactoryBot.create_list(:request_with_pull_request_id, 2) - FactoryBot.create_list(:request_orphaned_on_branch_id, 2) - FactoryBot.create_list(:request_with_branch_id, 2) - FactoryBot.create_list(:request_orphaned_on_tag_id, 2) - FactoryBot.create_list(:request_with_tag_id, 2) - FactoryBot.create_list(:stage_orphaned_on_build_id, 2) - FactoryBot.create_list(:stage_with_build_id, 2) - ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') - } - it 'removes orphaned repositories (with these dependent on orphaned builds)' do - expect { - backup.remove_orphans - }.to change { Repository.all.size }.by -16 - end - - it 'removes orphaned builds' do - expect { - backup.remove_orphans - }.to change { Build.all.size }.by -12 - end - - it 'removes orphaned jobs' do - expect { - backup.remove_orphans - }.to change { Job.all.size }.by -6 - end - - it 'removes orphaned branches' do - expect { - backup.remove_orphans - }.to change { Branch.all.size }.by -4 - end - - it 'removes orphaned tags' do - expect { - backup.remove_orphans - }.to change { Tag.all.size }.by -4 - end - - it 'removes orphaned commits' do - expect { - backup.remove_orphans - }.to change { Commit.all.size }.by -6 - end - - it 'removes orphaned crons' do - expect { - backup.remove_orphans - }.to change { Cron.all.size }.by -2 - end - - it 'removes orphaned pull requests' do - expect { - backup.remove_orphans - }.to change { PullRequest.all.size }.by -2 - end - - it 'removes orphaned requests' do - expect { - backup.remove_orphans - }.to change { Request.all.size }.by -8 - end - - it 'removes orphaned stages' do - expect { - backup.remove_orphans - }.to change { Stage.all.size }.by -2 - end - - context 'when dry run mode is on' do - let!(:backup) { Backup.new(dry_run: true) } - - before do - # allow_any_instance_of(IO).to receive(:puts) - end - - it 'prepares proper dry run report' do - backup.remove_orphans - expect(backup.dry_run_report[:repositories].size).to eql 16 - expect(backup.dry_run_report[:builds].size).to eql 12 - expect(backup.dry_run_report[:jobs].size).to eql 6 - expect(backup.dry_run_report[:branches].size).to eql 4 - expect(backup.dry_run_report[:tags].size).to eql 4 - expect(backup.dry_run_report[:commits].size).to eql 6 - expect(backup.dry_run_report[:crons].size).to eql 2 - expect(backup.dry_run_report[:pull_requests].size).to eql 2 - expect(backup.dry_run_report[:requests].size).to eql 8 - expect(backup.dry_run_report[:stages].size).to eql 2 - end - - it 'prints dry run report' do - expect_any_instance_of(DryRunReporter).to receive(:print_report).once - backup.run - end - end - end - describe 'run' do after(:each) do Organization.destroy_all @@ -278,7 +113,7 @@ end it 'removes orphans' do - expect(backup).to receive(:remove_orphans).once + expect_any_instance_of(Backup::RemoveOrphans).to receive(:run).once backup.run end end From d0dda0c6975748aeb5a660629ca4eea16abaf8f0 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Fri, 17 Sep 2021 17:12:03 +0200 Subject: [PATCH 75/84] do_without_triggers - refactoring wip --- lib/db_helper.rb | 7 ++ spec/backup/remove_orphans_spec.rb | 124 ++++++++++++++--------------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/lib/db_helper.rb b/lib/db_helper.rb index f8b06b3..aa80d0a 100644 --- a/lib/db_helper.rb +++ b/lib/db_helper.rb @@ -17,4 +17,11 @@ def do_in_other_db(config_or_url) connect_db(saved_config) result end + + def do_without_triggers + ActiveRecord::Base.connection.execute('set session_replication_role = replica;') + result = yield + ActiveRecord::Base.connection.execute('set session_replication_role = default;') + result + end end \ No newline at end of file diff --git a/spec/backup/remove_orphans_spec.rb b/spec/backup/remove_orphans_spec.rb index fc7f3f0..a169f3d 100644 --- a/spec/backup/remove_orphans_spec.rb +++ b/spec/backup/remove_orphans_spec.rb @@ -18,7 +18,7 @@ let(:files_location) { "dump/tests" } let!(:config) { Config.new(files_location: files_location, limit: 5) } - let!(:connection) { DbHelper.new(config) } + let!(:db_helper) { DbHelper.new(config) } let!(:remove_orphans) { Backup::RemoveOrphans.new(config, DryRunReporter.new) } describe 'run' do @@ -36,67 +36,67 @@ end let!(:data) { - ActiveRecord::Base.connection.execute('alter table repositories disable trigger all;') - FactoryBot.create_list(:repository, 2) - FactoryBot.create_list(:build, 2) - FactoryBot.create_list(:job, 2) - FactoryBot.create_list(:branch, 2) - FactoryBot.create_list(:tag, 2) - FactoryBot.create_list(:commit, 2) - FactoryBot.create_list(:cron, 2) - FactoryBot.create_list(:pull_request, 2) - FactoryBot.create_list(:request, 2) - FactoryBot.create_list(:stage, 2) - FactoryBot.create_list(:repository_orphaned_on_current_build_id, 2) - FactoryBot.create_list(:repository_with_current_build_id, 2) - FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:repository_with_last_build_id, 2) - FactoryBot.create_list(:build_orphaned_on_repository_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_repository_id, 2) - FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_commit_id, 2) - FactoryBot.create_list(:build_orphaned_on_request_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_pull_request_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_pull_request_id, 2) - FactoryBot.create_list(:build_orphaned_on_branch_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_branch_id, 2) - FactoryBot.create_list(:build_orphaned_on_tag_id_with_mutually_related_repo, 2) - FactoryBot.create_list(:build_with_tag_id, 2) - FactoryBot.create_list(:job_orphaned_on_repository_id, 2) - FactoryBot.create_list(:job_with_repository_id, 2) - FactoryBot.create_list(:job_orphaned_on_commit_id, 2) - FactoryBot.create_list(:job_with_commit_id, 2) - FactoryBot.create_list(:job_orphaned_on_stage_id, 2) - FactoryBot.create_list(:job_with_stage_id, 2) - FactoryBot.create_list(:branch_orphaned_on_repository_id, 2) - FactoryBot.create_list(:branch_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:branch_with_last_build_id, 2) - FactoryBot.create_list(:tag_orphaned_on_repository_id, 2) - FactoryBot.create_list(:tag_with_repository_id, 2) - FactoryBot.create_list(:tag_orphaned_on_last_build_id, 2) - FactoryBot.create_list(:tag_with_last_build_id, 2) - FactoryBot.create_list(:commit_orphaned_on_repository_id, 2) - FactoryBot.create_list(:commit_with_repository_id, 2) - FactoryBot.create_list(:commit_orphaned_on_branch_id, 2) - FactoryBot.create_list(:commit_with_branch_id, 2) - FactoryBot.create_list(:commit_orphaned_on_tag_id, 2) - FactoryBot.create_list(:commit_with_tag_id, 2) - FactoryBot.create_list(:cron_orphaned_on_branch_id, 2) - FactoryBot.create_list(:cron_with_branch_id, 2) - FactoryBot.create_list(:pull_request_orphaned_on_repository_id, 2) - FactoryBot.create_list(:pull_request_with_repository_id, 2) - FactoryBot.create_list(:request_orphaned_on_commit_id, 2) - FactoryBot.create_list(:request_with_commit_id, 2) - FactoryBot.create_list(:request_orphaned_on_pull_request_id, 2) - FactoryBot.create_list(:request_with_pull_request_id, 2) - FactoryBot.create_list(:request_orphaned_on_branch_id, 2) - FactoryBot.create_list(:request_with_branch_id, 2) - FactoryBot.create_list(:request_orphaned_on_tag_id, 2) - FactoryBot.create_list(:request_with_tag_id, 2) - FactoryBot.create_list(:stage_orphaned_on_build_id, 2) - FactoryBot.create_list(:stage_with_build_id, 2) - ActiveRecord::Base.connection.execute('alter table repositories enable trigger all;') + db_helper.do_without_triggers do + FactoryBot.create_list(:repository, 2) + FactoryBot.create_list(:build, 2) + FactoryBot.create_list(:job, 2) + FactoryBot.create_list(:branch, 2) + FactoryBot.create_list(:tag, 2) + FactoryBot.create_list(:commit, 2) + FactoryBot.create_list(:cron, 2) + FactoryBot.create_list(:pull_request, 2) + FactoryBot.create_list(:request, 2) + FactoryBot.create_list(:stage, 2) + FactoryBot.create_list(:repository_orphaned_on_current_build_id, 2) + FactoryBot.create_list(:repository_with_current_build_id, 2) + FactoryBot.create_list(:repository_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:repository_with_last_build_id, 2) + FactoryBot.create_list(:build_orphaned_on_repository_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_repository_id, 2) + FactoryBot.create_list(:build_orphaned_on_commit_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_commit_id, 2) + FactoryBot.create_list(:build_orphaned_on_request_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_pull_request_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_pull_request_id, 2) + FactoryBot.create_list(:build_orphaned_on_branch_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_branch_id, 2) + FactoryBot.create_list(:build_orphaned_on_tag_id_with_mutually_related_repo, 2) + FactoryBot.create_list(:build_with_tag_id, 2) + FactoryBot.create_list(:job_orphaned_on_repository_id, 2) + FactoryBot.create_list(:job_with_repository_id, 2) + FactoryBot.create_list(:job_orphaned_on_commit_id, 2) + FactoryBot.create_list(:job_with_commit_id, 2) + FactoryBot.create_list(:job_orphaned_on_stage_id, 2) + FactoryBot.create_list(:job_with_stage_id, 2) + FactoryBot.create_list(:branch_orphaned_on_repository_id, 2) + FactoryBot.create_list(:branch_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:branch_with_last_build_id, 2) + FactoryBot.create_list(:tag_orphaned_on_repository_id, 2) + FactoryBot.create_list(:tag_with_repository_id, 2) + FactoryBot.create_list(:tag_orphaned_on_last_build_id, 2) + FactoryBot.create_list(:tag_with_last_build_id, 2) + FactoryBot.create_list(:commit_orphaned_on_repository_id, 2) + FactoryBot.create_list(:commit_with_repository_id, 2) + FactoryBot.create_list(:commit_orphaned_on_branch_id, 2) + FactoryBot.create_list(:commit_with_branch_id, 2) + FactoryBot.create_list(:commit_orphaned_on_tag_id, 2) + FactoryBot.create_list(:commit_with_tag_id, 2) + FactoryBot.create_list(:cron_orphaned_on_branch_id, 2) + FactoryBot.create_list(:cron_with_branch_id, 2) + FactoryBot.create_list(:pull_request_orphaned_on_repository_id, 2) + FactoryBot.create_list(:pull_request_with_repository_id, 2) + FactoryBot.create_list(:request_orphaned_on_commit_id, 2) + FactoryBot.create_list(:request_with_commit_id, 2) + FactoryBot.create_list(:request_orphaned_on_pull_request_id, 2) + FactoryBot.create_list(:request_with_pull_request_id, 2) + FactoryBot.create_list(:request_orphaned_on_branch_id, 2) + FactoryBot.create_list(:request_with_branch_id, 2) + FactoryBot.create_list(:request_orphaned_on_tag_id, 2) + FactoryBot.create_list(:request_with_tag_id, 2) + FactoryBot.create_list(:stage_orphaned_on_build_id, 2) + FactoryBot.create_list(:stage_with_build_id, 2) + end } it 'removes orphaned repositories (with these dependent on orphaned builds)' do expect { remove_orphans.run }.to change { Repository.all.size }.by -16 From cebd4be3d6a3f311a94df9a844766a2b5536a693 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 20 Sep 2021 13:48:23 +0200 Subject: [PATCH 76/84] Backup::RemoveOld - refactoring --- lib/backup/remove_old.rb | 205 ++++++++++++++++++ lib/travis-backup.rb | 187 +---------------- spec/backup/remove_old_spec.rb | 312 ++++++++++++++++++++++++++++ spec/backup_spec.rb | 369 +++------------------------------ spec/support/utils.rb | 30 +++ 5 files changed, 574 insertions(+), 529 deletions(-) create mode 100644 lib/backup/remove_old.rb create mode 100644 spec/backup/remove_old_spec.rb create mode 100644 spec/support/utils.rb diff --git a/lib/backup/remove_old.rb b/lib/backup/remove_old.rb new file mode 100644 index 0000000..4c0fae8 --- /dev/null +++ b/lib/backup/remove_old.rb @@ -0,0 +1,205 @@ +# frozen_string_literal: true + +class Backup + class RemoveOld + attr_reader :config + + def initialize(config, dry_run_reporter=nil) + @config = config + @dry_run_reporter = dry_run_reporter + end + + def dry_run_report + @dry_run_reporter.report + end + + def run(args={}) + user_id = args[:user_id] || @config.user_id + repo_id = args[:repo_id] || @config.repo_id + org_id = args[:org_id] || @config.org_id + + if user_id + process_repos_for_owner(user_id, 'User') + elsif org_id + process_repos_for_owner(org_id, 'Organization') + elsif repo_id + process_repo_with_id(repo_id) + else + process_all_repos + end + end + + def process_repos_for_owner(owner_id, owner_type) + Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| + process_repo(repository) + end + end + + def process_repo_with_id(repo_id) + process_repo(Repository.find(repo_id)) + end + + def process_all_repos + Repository.order(:id).each do |repository| + process_repo(repository) + end + end + + def process_repo(repository) + process_repo_builds(repository) + process_repo_requests(repository) + end + + def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + threshold = @config.threshold.to_i.months.ago.to_datetime + current_build_id = repository.current_build_id || -1 + repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) + .in_groups_of(@config.limit.to_i, false).map do |builds_batch| + if @config.if_backup + file_prefix = "repository_#{repository.id}" + save_and_destroy_builds_batch(builds_batch, file_prefix) + else + destroy_builds_batch(builds_batch) + end + end.compact.reduce(&:&) + end + + def process_repo_requests(repository) + threshold = @config.threshold.to_i.months.ago.to_datetime + repository.requests.where('created_at < ?', threshold) + .in_groups_of(@config.limit.to_i, false).map do |requests_batch| + @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) + end.compact + end + + private + + def save_and_destroy_builds_batch(builds_batch, file_prefix) + builds_export = builds_batch.map(&:attributes) + + dependencies_saved = builds_batch.map do |build| + save_build_jobs_and_logs(build, file_prefix) + end.reduce(&:&) + + if dependencies_saved + file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" + pretty_json = JSON.pretty_generate(builds_export) + save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false + else + false + end + end + + def save_build_jobs_and_logs(build, file_prefix) + build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| + file_prefix = "#{file_prefix}_build_#{build.id}" + save_jobs_batch(jobs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_jobs_batch(jobs_batch, file_prefix) + jobs_export = jobs_batch.map(&:attributes) + + logs_saved = jobs_batch.map do |job| + save_job_logs(job, file_prefix) + end.reduce(&:&) + + if logs_saved + file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(jobs_export) + save_file(file_name, pretty_json) + else + false + end + end + + def save_job_logs(job, file_prefix) + job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| + file_prefix = "#{file_prefix}_job_#{job.id}" + save_logs_batch(logs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_logs_batch(logs_batch, file_prefix) + logs_export = logs_batch.map(&:attributes) + file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(logs_export) + save_file(file_name, pretty_json) + end + + def destroy_builds_batch(builds_batch) + return destroy_builds_batch_dry(builds_batch) if @config.dry_run + + builds_batch.each(&:destroy) + end + + def destroy_builds_batch_dry(builds_batch) + @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id)) + + jobs_ids = builds_batch.map do |build| + build.jobs.map(&:id) || [] + end.flatten + + @dry_run_reporter.add_to_report(:jobs, *jobs_ids) + + logs_ids = builds_batch.map do |build| + build.jobs.map do |job| + job.logs.map(&:id) || [] + end.flatten || [] + end.flatten + + @dry_run_reporter.add_to_report(:logs, *logs_ids) + end + + def save_and_destroy_requests_batch(requests_batch, repository) + requests_export = export_requests(requests_batch) + file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json" + pretty_json = JSON.pretty_generate(requests_export) + if save_file(file_name, pretty_json) + destroy_requests_batch(requests_batch) + end + requests_export + end + + def destroy_requests_batch(requests_batch) + return destroy_requests_batch_dry(requests_batch) if @config.dry_run + + requests_batch.each(&:destroy) + end + + def destroy_requests_batch_dry(requests_batch) + @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id)) + end + + def save_file(file_name, content) # rubocop:disable Metrics/MethodLength + return true if @config.dry_run + + saved = false + begin + unless File.directory?(@config.files_location) + FileUtils.mkdir_p(@config.files_location) + end + + File.open(file_path(file_name), 'w') do |file| + file.write(content) + file.close + saved = true + end + rescue => e + print "Failed to save #{file_name}, error: #{e.inspect}\n" + end + saved + end + + def file_path(file_name) + "#{@config.files_location}/#{file_name}" + end + + def export_requests(requests) + requests.map do |request| + request.attributes + end + end + end + end + \ No newline at end of file diff --git a/lib/travis-backup.rb b/lib/travis-backup.rb index 4af2d1f..b0306d6 100644 --- a/lib/travis-backup.rb +++ b/lib/travis-backup.rb @@ -17,6 +17,7 @@ require 'models/stage' require 'backup/move_logs' require 'backup/remove_orphans' +require 'backup/remove_old' # main travis-backup class class Backup @@ -29,8 +30,6 @@ def initialize(config_args={}) if @config.dry_run @dry_run_reporter = DryRunReporter.new end - - @db_helper.connect_db end def dry_run_report @@ -38,196 +37,14 @@ def dry_run_report end def run(args={}) - user_id = args[:user_id] || @config.user_id - repo_id = args[:repo_id] || @config.repo_id - org_id = args[:org_id] || @config.org_id - if @config.move_logs Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run elsif @config.remove_orphans Backup::RemoveOrphans.new(@config, @dry_run_reporter).run - elsif user_id - process_repos_for_owner(user_id, 'User') - elsif org_id - process_repos_for_owner(org_id, 'Organization') - elsif repo_id - process_repo_with_id(repo_id) else - process_all_repos + Backup::RemoveOld.new(@config, @dry_run_reporter).run(args) end @dry_run_reporter.print_report if @config.dry_run end - - def process_repos_for_owner(owner_id, owner_type) - Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| - process_repo(repository) - end - end - - def process_repo_with_id(repo_id) - process_repo(Repository.find(repo_id)) - end - - def process_all_repos - Repository.order(:id).each do |repository| - process_repo(repository) - end - end - - def process_repo(repository) - process_repo_builds(repository) - process_repo_requests(repository) - end - - def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - threshold = @config.threshold.to_i.months.ago.to_datetime - current_build_id = repository.current_build_id || -1 - repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) - .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - if @config.if_backup - file_prefix = "repository_#{repository.id}" - save_and_destroy_builds_batch(builds_batch, file_prefix) - else - destroy_builds_batch(builds_batch) - end - end.compact.reduce(&:&) - end - - def process_repo_requests(repository) - threshold = @config.threshold.to_i.months.ago.to_datetime - repository.requests.where('created_at < ?', threshold) - .in_groups_of(@config.limit.to_i, false).map do |requests_batch| - @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) - end.compact - end - - private - - def save_and_destroy_builds_batch(builds_batch, file_prefix) - builds_export = builds_batch.map(&:attributes) - - dependencies_saved = builds_batch.map do |build| - save_build_jobs_and_logs(build, file_prefix) - end.reduce(&:&) - - if dependencies_saved - file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" - pretty_json = JSON.pretty_generate(builds_export) - save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false - else - false - end - end - - def save_build_jobs_and_logs(build, file_prefix) - build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| - file_prefix = "#{file_prefix}_build_#{build.id}" - save_jobs_batch(jobs_batch, file_prefix) - end.compact.reduce(&:&) - end - - def save_jobs_batch(jobs_batch, file_prefix) - jobs_export = jobs_batch.map(&:attributes) - - logs_saved = jobs_batch.map do |job| - save_job_logs(job, file_prefix) - end.reduce(&:&) - - if logs_saved - file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json" - pretty_json = JSON.pretty_generate(jobs_export) - save_file(file_name, pretty_json) - else - false - end - end - - def save_job_logs(job, file_prefix) - job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| - file_prefix = "#{file_prefix}_job_#{job.id}" - save_logs_batch(logs_batch, file_prefix) - end.compact.reduce(&:&) - end - - def save_logs_batch(logs_batch, file_prefix) - logs_export = logs_batch.map(&:attributes) - file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json" - pretty_json = JSON.pretty_generate(logs_export) - save_file(file_name, pretty_json) - end - - def destroy_builds_batch(builds_batch) - return destroy_builds_batch_dry(builds_batch) if @config.dry_run - - builds_batch.each(&:destroy) - end - - def destroy_builds_batch_dry(builds_batch) - @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id)) - - jobs_ids = builds_batch.map do |build| - build.jobs.map(&:id) || [] - end.flatten - - @dry_run_reporter.add_to_report(:jobs, *jobs_ids) - - logs_ids = builds_batch.map do |build| - build.jobs.map do |job| - job.logs.map(&:id) || [] - end.flatten || [] - end.flatten - - @dry_run_reporter.add_to_report(:logs, *logs_ids) - end - - def save_and_destroy_requests_batch(requests_batch, repository) - requests_export = export_requests(requests_batch) - file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json" - pretty_json = JSON.pretty_generate(requests_export) - if save_file(file_name, pretty_json) - destroy_requests_batch(requests_batch) - end - requests_export - end - - def destroy_requests_batch(requests_batch) - return destroy_requests_batch_dry(requests_batch) if @config.dry_run - - requests_batch.each(&:destroy) - end - - def destroy_requests_batch_dry(requests_batch) - @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id)) - end - - def save_file(file_name, content) # rubocop:disable Metrics/MethodLength - return true if @config.dry_run - - saved = false - begin - unless File.directory?(@config.files_location) - FileUtils.mkdir_p(@config.files_location) - end - - File.open(file_path(file_name), 'w') do |file| - file.write(content) - file.close - saved = true - end - rescue => e - print "Failed to save #{file_name}, error: #{e.inspect}\n" - end - saved - end - - def file_path(file_name) - "#{@config.files_location}/#{file_name}" - end - - def export_requests(requests) - requests.map do |request| - request.attributes - end - end end diff --git a/spec/backup/remove_old_spec.rb b/spec/backup/remove_old_spec.rb new file mode 100644 index 0000000..8f56683 --- /dev/null +++ b/spec/backup/remove_old_spec.rb @@ -0,0 +1,312 @@ +$: << 'lib' +require 'uri' +require 'travis-backup' +require 'models/build' +require 'models/job' +require 'models/organization' +require 'models/user' +require 'support/factories' +require 'support/expected_files' +require 'support/before_tests' +require 'support/utils' +require 'pry' + + +describe Backup::RemoveOld do + before(:all) do + BeforeTests.new.run + end + + let(:files_location) { "dump/tests" } + let!(:config) { Config.new(files_location: files_location, limit: 5) } + let!(:db_helper) { DbHelper.new(config) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + + describe 'process_repo' do + let!(:repository) { + FactoryBot.create(:repository) + } + + it 'processes repository builds' do + expect(remove_old).to receive(:process_repo_builds).once.with(repository) + remove_old.process_repo(repository) + end + + it 'processes repository requests' do + expect(remove_old).to receive(:process_repo_requests).once.with(repository) + remove_old.process_repo(repository) + end + end + + describe 'process_repo_builds' do + after(:each) do + Repository.destroy_all + Build.destroy_all + Job.destroy_all + Log.destroy_all + end + + let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } + let!(:repository) { + ActiveRecord::Base.connection.execute('set session_replication_role = replica;') + repository = FactoryBot.create( + :repository_with_builds_jobs_and_logs, + created_at: datetime, + updated_at: datetime + ) + ActiveRecord::Base.connection.execute('set session_replication_role = default;') + repository + } + let!(:repository2) { + FactoryBot.create( + :repository_with_builds_jobs_and_logs, + created_at: datetime, + updated_at: datetime, + builds_count: 1 + ) + } + let(:expected_files_creator) { + ExpectedFiles.new(repository, datetime) + } + let!(:expected_builds_json) { + expected_files_creator.builds_json + } + let!(:expected_jobs_jsons) { + repository.builds.map do |build| + expected_files_creator.jobs_json(build) + end + } + let!(:expected_logs_jsons) { + repository.builds.map do |build| + build.jobs.map do |job| + expected_files_creator.logs_json(job) + end + end.flatten(1) + } + + shared_context 'removing builds and jobs' do + it 'should delete all builds of the repository' do + remove_old.process_repo_builds(repository) + expect(Build.all.map(&:repository_id)).to eq([repository2.id]) + end + + it 'should delete all jobs of removed builds and leave the rest' do + expect { + remove_old.process_repo_builds(repository) + }.to change { Job.all.size }.by -4 + + build_id = Build.first.id + expect(Job.all.map(&:source_id)).to eq([build_id, build_id]) + end + + it 'should delete all logs of removed jobs and leave the rest' do + expect { + remove_old.process_repo_builds(repository) + }.to change { Log.all.size }.by -8 + + build_id = Build.first.id + expect(Log.all.map(&:job).map(&:source_id)).to eq(Array.new(4, build_id)) + end + end + + shared_context 'not saving JSON to file' do + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + remove_old.process_repo_builds(repository) + end + end + + context 'when if_backup config is set to true' do + it 'should save proper build JSON file' do + expect_method_calls_on( + File, :write, + [JSON.pretty_generate(expected_builds_json)], + allow_instances: true, + arguments_to_check: :first + ) do + remove_old.process_repo_builds(repository) + end + end + + it 'should save proper job JSON files' do + expect_method_calls_on( + File, :write, + [ + JSON.pretty_generate(expected_jobs_jsons.first), + JSON.pretty_generate(expected_jobs_jsons.second) + ], + allow_instances: true, + arguments_to_check: :first + ) do + remove_old.process_repo_builds(repository) + end + end + + it 'should save proper log JSON files' do + expect_method_calls_on( + File, :write, + [ + JSON.pretty_generate(expected_logs_jsons.first), + JSON.pretty_generate(expected_logs_jsons.second), + JSON.pretty_generate(expected_logs_jsons.third), + JSON.pretty_generate(expected_logs_jsons.fourth), + ], + allow_instances: true, + arguments_to_check: :first + ) do + remove_old.process_repo_builds(repository) + end + end + + it 'should save JSON files at proper paths' do + expect_method_calls_on( + File, :open, + [ + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), + Regexp.new('dump/tests/repository_\d+_builds_\d+-\d+.json') + ], + match_mode: :match, + arguments_to_check: :first + ) do + remove_old.process_repo_builds(repository) + end + end + + it_behaves_like 'removing builds and jobs' + + context 'when path with nonexistent folders is given' do + let(:random_files_location) { "dump/tests/#{rand(100000)}" } + let!(:config) { Config.new(files_location: random_files_location, limit: 2) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + + it 'should create needed folders' do + expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original + remove_old.process_repo_builds(repository) + end + end + end + + context 'when if_backup config is set to false' do + let!(:config) { Config.new(files_location: files_location, limit: 2, if_backup: false) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + it_behaves_like 'not saving JSON to file' + it_behaves_like 'removing builds and jobs' + end + + context 'when dry_run config is set to true' do + let!(:config) { Config.new(files_location: files_location, limit: 2, dry_run: true) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + it_behaves_like 'not saving JSON to file' + + it 'should not delete builds' do + expect { + remove_old.process_repo_builds(repository) + }.not_to change { Build.all.size } + end + + it 'should not delete jobs' do + expect { + remove_old.process_repo_builds(repository) + }.not_to change { Job.all.size } + end + end + end + + describe 'process_repo_requests' do + after(:each) do + Repository.destroy_all + Request.destroy_all + end + + let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } + let!(:repository) { + FactoryBot.create( + :repository_with_requests, + created_at: datetime, + updated_at: datetime + ) + } + let!(:repository2) { + FactoryBot.create( + :repository_with_requests, + created_at: datetime, + updated_at: datetime, + requests_count: 1 + ) + } + + + let!(:expected_requests_json) { + ExpectedFiles.new(repository, datetime).requests_json + } + + shared_context 'removing requests' do + it 'should delete all requests of the repository' do + remove_old.process_repo_requests(repository) + expect(Request.all.map(&:repository_id)).to eq([repository2.id]) + end + end + + shared_context 'not saving JSON to file' do + it 'should not save JSON to file' do + expect(File).not_to receive(:open) + remove_old.process_repo_requests(repository) + end + end + + context 'when if_backup config is set to true' do + it 'should save proper build JSON to file' do + expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_requests_json)) + remove_old.process_repo_requests(repository) + end + + it 'should save JSON to file at proper path' do + expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') + remove_old.process_repo_requests(repository) + end + + it_behaves_like 'removing requests' + + context 'when path with nonexistent folders is given' do + let(:random_files_location) { "dump/tests/#{rand(100000)}" } + let!(:config) { Config.new(files_location: random_files_location, limit: 2) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + it 'should create needed folders' do + expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original + remove_old.process_repo_requests(repository) + end + end + end + + context 'when if_backup config is set to false' do + let!(:config) { Config.new(files_location: files_location, limit: 2, if_backup: false) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + it_behaves_like 'not saving JSON to file' + it_behaves_like 'removing requests' + end + + context 'when dry_run config is set to true' do + let!(:config) { Config.new(files_location: files_location, limit: 2, dry_run: true) } + let!(:remove_old) { Backup::RemoveOld.new(config, DryRunReporter.new) } + + it_behaves_like 'not saving JSON to file' + + it 'should not delete requests' do + expect { + remove_old.process_repo_requests(repository) + }.not_to change { Request.all.size } + end + end + end +end diff --git a/spec/backup_spec.rb b/spec/backup_spec.rb index 2e1c425..b7a49d7 100644 --- a/spec/backup_spec.rb +++ b/spec/backup_spec.rb @@ -8,6 +8,7 @@ require 'support/factories' require 'support/expected_files' require 'support/before_tests' +require 'support/utils' require 'pry' describe Backup do @@ -48,7 +49,7 @@ context 'when no arguments are given' do it 'processes every repository' do Repository.all.each do |repository| - expect(backup).to receive(:process_repo_builds).once.with(repository) + expect_any_instance_of(Backup::RemoveOld).to receive(:process_repo_builds).once.with(repository) end backup.run end @@ -56,36 +57,40 @@ context 'when user_id is given' do it 'processes only the repositories of the given user' do - processed_repos_ids = [] - allow(backup).to receive(:process_repo_builds) {|repo| processed_repos_ids.push(repo.id)} - user_repos_ids = Repository.where( - 'owner_id = ? and owner_type = ?', - user1.id, - 'User' - ).map(&:id) - backup.run(user_id: user1.id) - expect(processed_repos_ids).to match_array(user_repos_ids) + user_repos = Repository.where('owner_id = ? and owner_type = ?', user1.id, 'User') + + expect_method_calls_on( + Backup::RemoveOld, + :process_repo_builds, + user_repos, + allow_instances: true, + arguments_to_check: :first + ) do + backup.run(user_id: user1.id) + end end end context 'when org_id is given' do it 'processes only the repositories of the given organization' do - processed_repos_ids = [] - allow(backup).to receive(:process_repo_builds) {|repo| processed_repos_ids.push(repo.id)} - organization_repos_ids = Repository.where( - 'owner_id = ? and owner_type = ?', - organization1.id, - 'Organization' - ).map(&:id) - backup.run(org_id: organization1.id) - expect(processed_repos_ids).to match_array(organization_repos_ids) + org_repos = Repository.where('owner_id = ? and owner_type = ?', organization1.id, 'Organization') + + expect_method_calls_on( + Backup::RemoveOld, + :process_repo_builds, + org_repos, + allow_instances: true, + arguments_to_check: :first + ) do + backup.run(org_id: organization1.id) + end end end context 'when repo_id is given' do it 'processes only the repository with the given id' do repo = Repository.first - expect(backup).to receive(:process_repo_builds).once.with(repo) + expect_any_instance_of(Backup::RemoveOld).to receive(:process_repo_builds).once.with(repo) backup.run(repo_id: repo.id) end end @@ -139,328 +144,4 @@ end end end - - describe 'process_repo' do - let!(:repository) { - FactoryBot.create(:repository) - } - - it 'processes repository builds' do - expect(backup).to receive(:process_repo_builds).once.with(repository) - backup.process_repo(repository) - end - - it 'processes repository requests' do - expect(backup).to receive(:process_repo_requests).once.with(repository) - backup.process_repo(repository) - end - end - - describe 'process_repo_builds' do - after(:each) do - Repository.destroy_all - Build.destroy_all - Job.destroy_all - Log.destroy_all - end - - let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } - let!(:repository) { - ActiveRecord::Base.connection.execute('set session_replication_role = replica;') - repository = FactoryBot.create( - :repository_with_builds_jobs_and_logs, - created_at: datetime, - updated_at: datetime - ) - ActiveRecord::Base.connection.execute('set session_replication_role = default;') - repository - } - let!(:repository2) { - FactoryBot.create( - :repository_with_builds_jobs_and_logs, - created_at: datetime, - updated_at: datetime, - builds_count: 1 - ) - } - let(:expected_files_creator) { - ExpectedFiles.new(repository, datetime) - } - let!(:expected_builds_json) { - expected_files_creator.builds_json - } - let!(:expected_jobs_jsons) { - repository.builds.map do |build| - expected_files_creator.jobs_json(build) - end - } - let!(:expected_logs_jsons) { - repository.builds.map do |build| - build.jobs.map do |job| - expected_files_creator.logs_json(job) - end - end.flatten(1) - } - - shared_context 'removing builds and jobs' do - it 'should delete all builds of the repository' do - backup.process_repo_builds(repository) - expect(Build.all.map(&:repository_id)).to eq([repository2.id]) - end - - it 'should delete all jobs of removed builds and leave the rest' do - expect { - backup.process_repo_builds(repository) - }.to change { Job.all.size }.by -4 - - build_id = Build.first.id - expect(Job.all.map(&:source_id)).to eq([build_id, build_id]) - end - - it 'should delete all logs of removed jobs and leave the rest' do - expect { - backup.process_repo_builds(repository) - }.to change { Log.all.size }.by -8 - - build_id = Build.first.id - expect(Log.all.map(&:job).map(&:source_id)).to eq(Array.new(4, build_id)) - end - end - - shared_context 'not saving JSON to file' do - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_builds(repository) - end - end - - context 'when if_backup config is set to true' do - it 'should save proper build JSON file' do - expect_method_calls_on( - File, :write, - [JSON.pretty_generate(expected_builds_json)], - allow_instances: true, - arguments_to_check: :first - ) do - backup.process_repo_builds(repository) - end - end - - it 'should save proper job JSON files' do - expect_method_calls_on( - File, :write, - [ - JSON.pretty_generate(expected_jobs_jsons.first), - JSON.pretty_generate(expected_jobs_jsons.second) - ], - allow_instances: true, - arguments_to_check: :first - ) do - backup.process_repo_builds(repository) - end - end - - it 'should save proper log JSON files' do - expect_method_calls_on( - File, :write, - [ - JSON.pretty_generate(expected_logs_jsons.first), - JSON.pretty_generate(expected_logs_jsons.second), - JSON.pretty_generate(expected_logs_jsons.third), - JSON.pretty_generate(expected_logs_jsons.fourth), - ], - allow_instances: true, - arguments_to_check: :first - ) do - backup.process_repo_builds(repository) - end - end - - it 'should save JSON files at proper paths' do - expect_method_calls_on( - File, :open, - [ - Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_build_\d+_job_\d+_logs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_build_\d+_jobs_\d+-\d+.json'), - Regexp.new('dump/tests/repository_\d+_builds_\d+-\d+.json') - ], - match_mode: :match, - arguments_to_check: :first - ) do - backup.process_repo_builds(repository) - end - - - - # saved_files_paths = [] - - # allow(File).to receive(:open).and_wrap_original do |method, *args, &block| - # saved_files_paths.push(args.first) - # method.call(*args, &block) - # end - - # backup.process_repo_builds(repository) - - # expect(saved_files_paths).to match_array([ - # ]) - end - - it_behaves_like 'removing builds and jobs' - - context 'when path with nonexistent folders is given' do - let(:random_files_location) { "dump/tests/#{rand(100000)}" } - let!(:backup) { Backup.new(files_location: random_files_location, limit: 2) } - - it 'should create needed folders' do - expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original - backup.process_repo_builds(repository) - end - end - end - - context 'when if_backup config is set to false' do - let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } - - it_behaves_like 'not saving JSON to file' - it_behaves_like 'removing builds and jobs' - end - - context 'when dry_run config is set to true' do - let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } - - it_behaves_like 'not saving JSON to file' - - it 'should not delete builds' do - expect { - backup.process_repo_builds(repository) - }.not_to change { Build.all.size } - end - - it 'should not delete jobs' do - expect { - backup.process_repo_builds(repository) - }.not_to change { Job.all.size } - end - end - end - - describe 'process_repo_requests' do - after(:each) do - Repository.destroy_all - Request.destroy_all - end - - let(:datetime) { (Config.new.threshold + 1).months.ago.to_time.utc } - let!(:repository) { - FactoryBot.create( - :repository_with_requests, - created_at: datetime, - updated_at: datetime - ) - } - let!(:repository2) { - FactoryBot.create( - :repository_with_requests, - created_at: datetime, - updated_at: datetime, - requests_count: 1 - ) - } - - - let!(:expected_requests_json) { - ExpectedFiles.new(repository, datetime).requests_json - } - - shared_context 'removing requests' do - it 'should delete all requests of the repository' do - backup.process_repo_requests(repository) - expect(Request.all.map(&:repository_id)).to eq([repository2.id]) - end - end - - shared_context 'not saving JSON to file' do - it 'should not save JSON to file' do - expect(File).not_to receive(:open) - backup.process_repo_requests(repository) - end - end - - context 'when if_backup config is set to true' do - it 'should save proper build JSON to file' do - expect_any_instance_of(File).to receive(:write).once.with(JSON.pretty_generate(expected_requests_json)) - backup.process_repo_requests(repository) - end - - it 'should save JSON to file at proper path' do - expect(File).to receive(:open).once.with(Regexp.new(files_location), 'w') - backup.process_repo_requests(repository) - end - - it_behaves_like 'removing requests' - - context 'when path with nonexistent folders is given' do - let(:random_files_location) { "dump/tests/#{rand(100000)}" } - let!(:backup) { Backup.new(files_location: random_files_location, limit: 2) } - - it 'should create needed folders' do - expect(FileUtils).to receive(:mkdir_p).once.with(random_files_location).and_call_original - backup.process_repo_requests(repository) - end - end - end - - context 'when if_backup config is set to false' do - let!(:backup) { Backup.new(files_location: files_location, limit: 2, if_backup: false) } - - it_behaves_like 'not saving JSON to file' - it_behaves_like 'removing requests' - end - - context 'when dry_run config is set to true' do - let!(:backup) { Backup.new(files_location: files_location, limit: 2, dry_run: true) } - - it_behaves_like 'not saving JSON to file' - - it 'should not delete requests' do - expect { - backup.process_repo_requests(repository) - }.not_to change { Request.all.size } - end - end - end -end - -def expect_method_calls_on(cl, method, call_with, options) - match_mode = options[:mode] || :including - allow_instances = options[:allow_instances] || false - arguments_to_check = options[:arguments_to_check] || :all - - calls_args = [] - - allowed = allow_instances ? allow_any_instance_of(cl) : allow(cl) - - allowed.to receive(method).and_wrap_original do |method, *args, &block| - if arguments_to_check == :all - calls_args.push(args) - else - calls_args.push(args.send(arguments_to_check)) # = args.first, args.second, args.third etc. - end - method.call(*args, &block) - end - - yield - - case match_mode - when :including - call_with.each do |args| - expect(calls_args).to include(args) - end - when :match - expect(call_with).to match_array(calls_args) - end end diff --git a/spec/support/utils.rb b/spec/support/utils.rb new file mode 100644 index 0000000..6958d34 --- /dev/null +++ b/spec/support/utils.rb @@ -0,0 +1,30 @@ +def expect_method_calls_on(cl, method, call_with, options) + match_mode = options[:mode] || :including + allow_instances = options[:allow_instances] || false + arguments_to_check = options[:arguments_to_check] || :all + + calls_args = [] + + allowed = allow_instances ? allow_any_instance_of(cl) : allow(cl) + + allowed.to receive(method).and_wrap_original do |method, *args, &block| + if arguments_to_check == :all + calls_args.push(args) + else + calls_args.push(args.send(arguments_to_check)) # = args.first, args.second, args.third etc. + end + method.call(*args, &block) + end + + yield + + case match_mode + when :including + call_with.each do |args| + expect(calls_args).to include(args) + end + when :match + expect(call_with).to match_array(calls_args) + end + end + \ No newline at end of file From d4c4ab5d6671af286201e6276c91bd5f0d61c591 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 16:20:35 +0200 Subject: [PATCH 77/84] bug in tag factory fixed --- spec/support/factories.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 8fb9a51..cf345f6 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -272,7 +272,7 @@ end factory :tag_with_last_build_id do - last_build_id { Tag.first.id } + last_build_id { Build.first.id } end end From cfa11a3a4b736585eddae68b2370e742a5fa73cd Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 16:26:01 +0200 Subject: [PATCH 78/84] database cleaner --- spec/backup/remove_orphans_spec.rb | 14 +++----------- travis-backup.gemspec | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/spec/backup/remove_orphans_spec.rb b/spec/backup/remove_orphans_spec.rb index a169f3d..4a94f5f 100644 --- a/spec/backup/remove_orphans_spec.rb +++ b/spec/backup/remove_orphans_spec.rb @@ -9,7 +9,7 @@ require 'support/expected_files' require 'support/before_tests' require 'pry' - +require 'database_cleaner/active_record' describe Backup::RemoveOrphans do before(:all) do @@ -23,16 +23,8 @@ describe 'run' do after(:all) do - Repository.destroy_all - Build.destroy_all - Job.destroy_all - Branch.destroy_all - Tag.destroy_all - Commit.destroy_all - Cron.destroy_all - PullRequest.destroy_all - Request.destroy_all - Stage.destroy_all + DatabaseCleaner.strategy = :truncation + DatabaseCleaner.clean end let!(:data) { diff --git a/travis-backup.gemspec b/travis-backup.gemspec index 47c0572..c714c12 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -26,4 +26,5 @@ Gem::Specification.new do |s| s.add_development_dependency 'listen' s.add_development_dependency 'rubocop', '~> 0.75.1' s.add_development_dependency 'rubocop-rspec' + s.add_development_dependency 'database_cleaner-active_record' end \ No newline at end of file From 2b80693032827d6223d1afc4e46baa78ffd2d409 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 22:11:05 +0200 Subject: [PATCH 79/84] Gemfile.lock removed --- Gemfile.lock | 212 --------------------------------------------------- 1 file changed, 212 deletions(-) delete mode 100644 Gemfile.lock diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index f83f7ad..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,212 +0,0 @@ -PATH - remote: . - specs: - travis-backup (0.2.0) - activerecord - bootsnap - pg - pry - rails - tzinfo-data - -GEM - remote: https://rubygems.org/ - specs: - actioncable (6.1.4.1) - actionpack (= 6.1.4.1) - activesupport (= 6.1.4.1) - nio4r (~> 2.0) - websocket-driver (>= 0.6.1) - actionmailbox (6.1.4.1) - actionpack (= 6.1.4.1) - activejob (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) - mail (>= 2.7.1) - actionmailer (6.1.4.1) - actionpack (= 6.1.4.1) - actionview (= 6.1.4.1) - activejob (= 6.1.4.1) - activesupport (= 6.1.4.1) - mail (~> 2.5, >= 2.5.4) - rails-dom-testing (~> 2.0) - actionpack (6.1.4.1) - actionview (= 6.1.4.1) - activesupport (= 6.1.4.1) - rack (~> 2.0, >= 2.0.9) - rack-test (>= 0.6.3) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.4.1) - actionpack (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) - nokogiri (>= 1.8.5) - actionview (6.1.4.1) - activesupport (= 6.1.4.1) - builder (~> 3.1) - erubi (~> 1.4) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.4.1) - activesupport (= 6.1.4.1) - globalid (>= 0.3.6) - activemodel (6.1.4.1) - activesupport (= 6.1.4.1) - activerecord (6.1.4.1) - activemodel (= 6.1.4.1) - activesupport (= 6.1.4.1) - activestorage (6.1.4.1) - actionpack (= 6.1.4.1) - activejob (= 6.1.4.1) - activerecord (= 6.1.4.1) - activesupport (= 6.1.4.1) - marcel (~> 1.0.0) - mini_mime (>= 1.1.0) - activesupport (6.1.4.1) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 1.6, < 2) - minitest (>= 5.1) - tzinfo (~> 2.0) - zeitwerk (~> 2.3) - ast (2.4.2) - bootsnap (1.7.7) - msgpack (~> 1.0) - brakeman (5.1.1) - builder (3.2.4) - byebug (11.1.3) - coderay (1.1.3) - concurrent-ruby (1.1.9) - crass (1.0.6) - diff-lcs (1.4.4) - erubi (1.10.0) - factory_bot (6.2.0) - activesupport (>= 5.0.0) - ffi (1.15.3) - globalid (0.5.2) - activesupport (>= 5.0) - i18n (1.8.10) - concurrent-ruby (~> 1.0) - jaro_winkler (1.5.4) - listen (3.7.0) - rb-fsevent (~> 0.10, >= 0.10.3) - rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.12.0) - crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) - mini_mime (>= 0.1.1) - marcel (1.0.1) - method_source (1.0.0) - mini_mime (1.1.1) - mini_portile2 (2.6.1) - minitest (5.14.4) - msgpack (1.4.2) - nio4r (2.5.8) - nokogiri (1.12.3) - mini_portile2 (~> 2.6.1) - racc (~> 1.4) - parallel (1.20.1) - parser (3.0.2.0) - ast (~> 2.4.1) - pg (1.2.3) - pry (0.14.1) - coderay (~> 1.1) - method_source (~> 1.0) - racc (1.5.2) - rack (2.2.3) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (6.1.4.1) - actioncable (= 6.1.4.1) - actionmailbox (= 6.1.4.1) - actionmailer (= 6.1.4.1) - actionpack (= 6.1.4.1) - actiontext (= 6.1.4.1) - actionview (= 6.1.4.1) - activejob (= 6.1.4.1) - activemodel (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) - bundler (>= 1.15.0) - railties (= 6.1.4.1) - sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) - nokogiri (>= 1.6) - rails-html-sanitizer (1.4.1) - loofah (~> 2.3) - railties (6.1.4.1) - actionpack (= 6.1.4.1) - activesupport (= 6.1.4.1) - method_source - rake (>= 0.13) - thor (~> 1.0) - rainbow (3.0.0) - rake (13.0.6) - rb-fsevent (0.11.0) - rb-inotify (0.10.1) - ffi (~> 1.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-rails (5.0.2) - actionpack (>= 5.2) - activesupport (>= 5.2) - railties (>= 5.2) - rspec-core (~> 3.10) - rspec-expectations (~> 3.10) - rspec-mocks (~> 3.10) - rspec-support (~> 3.10) - rspec-support (3.10.2) - rubocop (0.75.1) - jaro_winkler (~> 1.5.1) - parallel (~> 1.10) - parser (>= 2.6) - rainbow (>= 2.2.2, < 4.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 1.7) - rubocop-rspec (1.41.0) - rubocop (>= 0.68.1) - ruby-progressbar (1.11.0) - sprockets (4.0.2) - concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.2) - actionpack (>= 4.0) - activesupport (>= 4.0) - sprockets (>= 3.0.0) - thor (1.1.0) - tzinfo (2.0.4) - concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.1) - tzinfo (>= 1.0.0) - unicode-display_width (1.6.1) - websocket-driver (0.7.5) - websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.5) - zeitwerk (2.4.2) - -PLATFORMS - ruby - -DEPENDENCIES - brakeman - byebug - factory_bot - listen - rspec-rails - rubocop (~> 0.75.1) - rubocop-rspec - travis-backup! - -BUNDLED WITH - 2.1.4 From 6f4c02f1d5e141e4098692d1d23aba9be0232489 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 22:22:17 +0200 Subject: [PATCH 80/84] fix --- spec/backup/remove_orphans_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/backup/remove_orphans_spec.rb b/spec/backup/remove_orphans_spec.rb index 4a94f5f..d754685 100644 --- a/spec/backup/remove_orphans_spec.rb +++ b/spec/backup/remove_orphans_spec.rb @@ -22,7 +22,7 @@ let!(:remove_orphans) { Backup::RemoveOrphans.new(config, DryRunReporter.new) } describe 'run' do - after(:all) do + before(:each) do DatabaseCleaner.strategy = :truncation DatabaseCleaner.clean end From 749de232f4566cd68424ff131defb79cce7032a0 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 22:29:51 +0200 Subject: [PATCH 81/84] style in RemoveOld --- lib/backup/remove_old.rb | 371 +++++++++++++++++++-------------------- 1 file changed, 185 insertions(+), 186 deletions(-) diff --git a/lib/backup/remove_old.rb b/lib/backup/remove_old.rb index 4c0fae8..7779915 100644 --- a/lib/backup/remove_old.rb +++ b/lib/backup/remove_old.rb @@ -1,205 +1,204 @@ # frozen_string_literal: true class Backup - class RemoveOld - attr_reader :config - - def initialize(config, dry_run_reporter=nil) - @config = config - @dry_run_reporter = dry_run_reporter - end + class RemoveOld + attr_reader :config - def dry_run_report - @dry_run_reporter.report - end + def initialize(config, dry_run_reporter=nil) + @config = config + @dry_run_reporter = dry_run_reporter + end - def run(args={}) - user_id = args[:user_id] || @config.user_id - repo_id = args[:repo_id] || @config.repo_id - org_id = args[:org_id] || @config.org_id + def dry_run_report + @dry_run_reporter.report + end - if user_id - process_repos_for_owner(user_id, 'User') - elsif org_id - process_repos_for_owner(org_id, 'Organization') - elsif repo_id - process_repo_with_id(repo_id) - else - process_all_repos - end - end + def run(args={}) + user_id = args[:user_id] || @config.user_id + repo_id = args[:repo_id] || @config.repo_id + org_id = args[:org_id] || @config.org_id - def process_repos_for_owner(owner_id, owner_type) - Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| - process_repo(repository) - end - end - - def process_repo_with_id(repo_id) - process_repo(Repository.find(repo_id)) - end - - def process_all_repos - Repository.order(:id).each do |repository| - process_repo(repository) - end - end - - def process_repo(repository) - process_repo_builds(repository) - process_repo_requests(repository) - end - - def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - threshold = @config.threshold.to_i.months.ago.to_datetime - current_build_id = repository.current_build_id || -1 - repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) - .in_groups_of(@config.limit.to_i, false).map do |builds_batch| - if @config.if_backup - file_prefix = "repository_#{repository.id}" - save_and_destroy_builds_batch(builds_batch, file_prefix) - else - destroy_builds_batch(builds_batch) - end - end.compact.reduce(&:&) - end - - def process_repo_requests(repository) - threshold = @config.threshold.to_i.months.ago.to_datetime - repository.requests.where('created_at < ?', threshold) - .in_groups_of(@config.limit.to_i, false).map do |requests_batch| - @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) - end.compact - end - - private - - def save_and_destroy_builds_batch(builds_batch, file_prefix) - builds_export = builds_batch.map(&:attributes) - - dependencies_saved = builds_batch.map do |build| - save_build_jobs_and_logs(build, file_prefix) - end.reduce(&:&) - - if dependencies_saved - file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" - pretty_json = JSON.pretty_generate(builds_export) - save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false - else - false - end + if user_id + process_repos_for_owner(user_id, 'User') + elsif org_id + process_repos_for_owner(org_id, 'Organization') + elsif repo_id + process_repo_with_id(repo_id) + else + process_all_repos + end + end + + def process_repos_for_owner(owner_id, owner_type) + Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository| + process_repo(repository) end - - def save_build_jobs_and_logs(build, file_prefix) - build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| - file_prefix = "#{file_prefix}_build_#{build.id}" - save_jobs_batch(jobs_batch, file_prefix) - end.compact.reduce(&:&) + end + + def process_repo_with_id(repo_id) + process_repo(Repository.find(repo_id)) + end + + def process_all_repos + Repository.order(:id).each do |repository| + process_repo(repository) end - - def save_jobs_batch(jobs_batch, file_prefix) - jobs_export = jobs_batch.map(&:attributes) - - logs_saved = jobs_batch.map do |job| - save_job_logs(job, file_prefix) - end.reduce(&:&) - - if logs_saved - file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json" - pretty_json = JSON.pretty_generate(jobs_export) - save_file(file_name, pretty_json) + end + + def process_repo(repository) + process_repo_builds(repository) + process_repo_requests(repository) + end + + def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + threshold = @config.threshold.to_i.months.ago.to_datetime + current_build_id = repository.current_build_id || -1 + repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) + .in_groups_of(@config.limit.to_i, false).map do |builds_batch| + if @config.if_backup + file_prefix = "repository_#{repository.id}" + save_and_destroy_builds_batch(builds_batch, file_prefix) else - false + destroy_builds_batch(builds_batch) end + end.compact.reduce(&:&) + end + + def process_repo_requests(repository) + threshold = @config.threshold.to_i.months.ago.to_datetime + repository.requests.where('created_at < ?', threshold) + .in_groups_of(@config.limit.to_i, false).map do |requests_batch| + @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) + end.compact + end + + private + + def save_and_destroy_builds_batch(builds_batch, file_prefix) + builds_export = builds_batch.map(&:attributes) + + dependencies_saved = builds_batch.map do |build| + save_build_jobs_and_logs(build, file_prefix) + end.reduce(&:&) + + if dependencies_saved + file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json" + pretty_json = JSON.pretty_generate(builds_export) + save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false + else + false end - - def save_job_logs(job, file_prefix) - job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| - file_prefix = "#{file_prefix}_job_#{job.id}" - save_logs_batch(logs_batch, file_prefix) - end.compact.reduce(&:&) - end - - def save_logs_batch(logs_batch, file_prefix) - logs_export = logs_batch.map(&:attributes) - file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json" - pretty_json = JSON.pretty_generate(logs_export) + end + + def save_build_jobs_and_logs(build, file_prefix) + build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| + file_prefix = "#{file_prefix}_build_#{build.id}" + save_jobs_batch(jobs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_jobs_batch(jobs_batch, file_prefix) + jobs_export = jobs_batch.map(&:attributes) + + logs_saved = jobs_batch.map do |job| + save_job_logs(job, file_prefix) + end.reduce(&:&) + + if logs_saved + file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(jobs_export) save_file(file_name, pretty_json) + else + false end - - def destroy_builds_batch(builds_batch) - return destroy_builds_batch_dry(builds_batch) if @config.dry_run - - builds_batch.each(&:destroy) - end - - def destroy_builds_batch_dry(builds_batch) - @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id)) - - jobs_ids = builds_batch.map do |build| - build.jobs.map(&:id) || [] - end.flatten - - @dry_run_reporter.add_to_report(:jobs, *jobs_ids) - - logs_ids = builds_batch.map do |build| - build.jobs.map do |job| - job.logs.map(&:id) || [] - end.flatten || [] - end.flatten - - @dry_run_reporter.add_to_report(:logs, *logs_ids) - end - - def save_and_destroy_requests_batch(requests_batch, repository) - requests_export = export_requests(requests_batch) - file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json" - pretty_json = JSON.pretty_generate(requests_export) - if save_file(file_name, pretty_json) - destroy_requests_batch(requests_batch) + end + + def save_job_logs(job, file_prefix) + job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| + file_prefix = "#{file_prefix}_job_#{job.id}" + save_logs_batch(logs_batch, file_prefix) + end.compact.reduce(&:&) + end + + def save_logs_batch(logs_batch, file_prefix) + logs_export = logs_batch.map(&:attributes) + file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json" + pretty_json = JSON.pretty_generate(logs_export) + save_file(file_name, pretty_json) + end + + def destroy_builds_batch(builds_batch) + return destroy_builds_batch_dry(builds_batch) if @config.dry_run + + builds_batch.each(&:destroy) + end + + def destroy_builds_batch_dry(builds_batch) + @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id)) + + jobs_ids = builds_batch.map do |build| + build.jobs.map(&:id) || [] + end.flatten + + @dry_run_reporter.add_to_report(:jobs, *jobs_ids) + + logs_ids = builds_batch.map do |build| + build.jobs.map do |job| + job.logs.map(&:id) || [] + end.flatten || [] + end.flatten + + @dry_run_reporter.add_to_report(:logs, *logs_ids) + end + + def save_and_destroy_requests_batch(requests_batch, repository) + requests_export = export_requests(requests_batch) + file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json" + pretty_json = JSON.pretty_generate(requests_export) + if save_file(file_name, pretty_json) + destroy_requests_batch(requests_batch) + end + requests_export + end + + def destroy_requests_batch(requests_batch) + return destroy_requests_batch_dry(requests_batch) if @config.dry_run + + requests_batch.each(&:destroy) + end + + def destroy_requests_batch_dry(requests_batch) + @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id)) + end + + def save_file(file_name, content) # rubocop:disable Metrics/MethodLength + return true if @config.dry_run + + saved = false + begin + unless File.directory?(@config.files_location) + FileUtils.mkdir_p(@config.files_location) end - requests_export - end - - def destroy_requests_batch(requests_batch) - return destroy_requests_batch_dry(requests_batch) if @config.dry_run - - requests_batch.each(&:destroy) - end - - def destroy_requests_batch_dry(requests_batch) - @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id)) - end - - def save_file(file_name, content) # rubocop:disable Metrics/MethodLength - return true if @config.dry_run - - saved = false - begin - unless File.directory?(@config.files_location) - FileUtils.mkdir_p(@config.files_location) - end - - File.open(file_path(file_name), 'w') do |file| - file.write(content) - file.close - saved = true - end - rescue => e - print "Failed to save #{file_name}, error: #{e.inspect}\n" + + File.open(file_path(file_name), 'w') do |file| + file.write(content) + file.close + saved = true end - saved + rescue => e + print "Failed to save #{file_name}, error: #{e.inspect}\n" end - - def file_path(file_name) - "#{@config.files_location}/#{file_name}" - end - - def export_requests(requests) - requests.map do |request| - request.attributes - end - end + saved end + + def file_path(file_name) + "#{@config.files_location}/#{file_name}" + end + + def export_requests(requests) + requests.map do |request| + request.attributes + end + end end - \ No newline at end of file +end From 7e95c317f8f161bc8ff158874d689f576c345915 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Tue, 21 Sep 2021 23:07:55 +0200 Subject: [PATCH 82/84] in_batches instead of in_groups_of - optimalization --- lib/backup/move_logs.rb | 2 +- lib/backup/remove_old.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/backup/move_logs.rb b/lib/backup/move_logs.rb index c3432a2..c29ee6a 100644 --- a/lib/backup/move_logs.rb +++ b/lib/backup/move_logs.rb @@ -14,7 +14,7 @@ def run return run_dry if @config.dry_run @db_helper.connect_db(@config.database_url) - Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch| + Log.order(:id).in_batches(of: @config.limit.to_i).map do |logs_batch| log_hashes = logs_batch.as_json @db_helper.connect_db(@config.destination_db_url) diff --git a/lib/backup/remove_old.rb b/lib/backup/remove_old.rb index 7779915..a4f4fb6 100644 --- a/lib/backup/remove_old.rb +++ b/lib/backup/remove_old.rb @@ -54,7 +54,7 @@ def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/M threshold = @config.threshold.to_i.months.ago.to_datetime current_build_id = repository.current_build_id || -1 repository.builds.where('created_at < ? and id != ?', threshold, current_build_id) - .in_groups_of(@config.limit.to_i, false).map do |builds_batch| + .in_batches(of: @config.limit.to_i).map do |builds_batch| if @config.if_backup file_prefix = "repository_#{repository.id}" save_and_destroy_builds_batch(builds_batch, file_prefix) @@ -67,7 +67,7 @@ def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/M def process_repo_requests(repository) threshold = @config.threshold.to_i.months.ago.to_datetime repository.requests.where('created_at < ?', threshold) - .in_groups_of(@config.limit.to_i, false).map do |requests_batch| + .in_batches(of: @config.limit.to_i).map do |requests_batch| @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch) end.compact end @@ -91,7 +91,7 @@ def save_and_destroy_builds_batch(builds_batch, file_prefix) end def save_build_jobs_and_logs(build, file_prefix) - build.jobs.in_groups_of(@config.limit.to_i, false).map do |jobs_batch| + build.jobs.in_batches(of: @config.limit.to_i).map do |jobs_batch| file_prefix = "#{file_prefix}_build_#{build.id}" save_jobs_batch(jobs_batch, file_prefix) end.compact.reduce(&:&) @@ -114,7 +114,7 @@ def save_jobs_batch(jobs_batch, file_prefix) end def save_job_logs(job, file_prefix) - job.logs.in_groups_of(@config.limit.to_i, false).map do |logs_batch| + job.logs.in_batches(of: @config.limit.to_i).map do |logs_batch| file_prefix = "#{file_prefix}_job_#{job.id}" save_logs_batch(logs_batch, file_prefix) end.compact.reduce(&:&) From 7de4293475743a96ca6b8a1f5dc6c01ae10b87d2 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Sun, 3 Oct 2021 23:50:37 +0200 Subject: [PATCH 83/84] GC.start in MoveLogs --- .travis.yml | 2 +- lib/backup/move_logs.rb | 24 +++++++++++++-------- spec/backup/move_logs_spec.rb | 40 +++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 104c991..f4e6886 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: jobs: include: - stage: "testing time" - script: bundle exec rspec + script: bundle exec rspec --tag ~slow dist: xenial diff --git a/lib/backup/move_logs.rb b/lib/backup/move_logs.rb index c29ee6a..928d84d 100644 --- a/lib/backup/move_logs.rb +++ b/lib/backup/move_logs.rb @@ -15,18 +15,24 @@ def run @db_helper.connect_db(@config.database_url) Log.order(:id).in_batches(of: @config.limit.to_i).map do |logs_batch| - log_hashes = logs_batch.as_json - @db_helper.connect_db(@config.destination_db_url) - - log_hashes.each do |log_hash| - new_log = Log.new(log_hash) - new_log.save! - end + process_logs_batch(logs_batch) + end + end - @db_helper.connect_db(@config.database_url) + def process_logs_batch(logs_batch) + log_hashes = logs_batch.as_json + @db_helper.connect_db(@config.destination_db_url) - logs_batch.each(&:destroy) + log_hashes.each do |log_hash| + new_log = Log.new(log_hash) + new_log.save! end + + @db_helper.connect_db(@config.database_url) + + logs_batch.each(&:destroy) + + GC.start end def run_dry diff --git a/spec/backup/move_logs_spec.rb b/spec/backup/move_logs_spec.rb index cf2480e..3ebf053 100644 --- a/spec/backup/move_logs_spec.rb +++ b/spec/backup/move_logs_spec.rb @@ -63,5 +63,45 @@ def destination_logs_size move_logs.run }.to change { Log.all.size }.by -10 end + + context 'when memory is limited and data amount big', slow: true do + let!(:config) { Config.new(files_location: files_location, limit: 5000) } + let!(:db_helper) { DbHelper.new(config) } + let!(:move_logs) { Backup::MoveLogs.new(config, db_helper, DryRunReporter.new) } + let!(:logs) { + FactoryBot.create( + :log, + job_id: 1, + content: 'some log content', + removed_by: 1, + archiving: false, + archive_verified: true + ) + ActiveRecord::Base.connection.execute(%{ + do $$ + declare + counter integer := 0; + begin + while counter < 49999 loop + insert into logs (content) (select content from logs where content is not null limit 1); + counter := counter + 1; + end loop; + end$$; + }) + } + + def do_with_limited_memory(limit_in_mb) + system("ulimit -Sv #{limit_in_mb * 1000}") + result = yield + system("ulimit -Sv unlimited") + result + end + + it 'runs without memory problems' do + do_with_limited_memory(300) do + move_logs.run + end + end + end end end From faa8f6d610384d4d6da5f3b5843a0c3f66e50294 Mon Sep 17 00:00:00 2001 From: Karol Selak Date: Mon, 4 Oct 2021 15:12:03 +0200 Subject: [PATCH 84/84] v0.2.1 --- travis-backup.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-backup.gemspec b/travis-backup.gemspec index c714c12..77927fd 100644 --- a/travis-backup.gemspec +++ b/travis-backup.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'travis-backup' - s.version = '0.2.0' + s.version = '0.2.1' s.summary = 'Travis CI backup tool' s.authors = ['Karol Selak'] s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")