Speed up CI: faster test job startup and runtime-balanced spec groups

.github/workflows/parallel_ci.yml

@@ -16,10 +16,13 @@ jobs:
       matrix:
         # set N-1 indexes for parallel jobs
         # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc
-        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7]
         # Set N number of parallel jobs you want to run
         # Remember to update ci_node_index above to 0..N-1
-        ci_node_total: [16]
+        # Groups are balanced by recorded runtime (spec/parallel_runtime_rspec.log).
+        # The slowest single spec file sets a floor on group time, so more jobs
+        # than this mostly adds setup cost without finishing sooner.
+        ci_node_total: [8]
 
     services:
       memcached:
@@ -47,14 +50,12 @@ jobs:
           logger.org.opensearch.discovery: "ERROR"
         ports:
           - 9200
-      localstack:
-        image: localstack/localstack:2.3.2
+      floci:
+        # Lightweight LocalStack-compatible AWS emulator for S3 metadata storage.
+        # Signatures aren't validated, so the dummy S3 credentials below work as-is.
+        image: floci/floci:2.3.0
         ports:
-          - "4566:4566" # LocalStack edge port
-        env:
-          SERVICES: s3 # Only start S3
-          AWS_ACCESS_KEY_ID: s3accesskeyid
-          AWS_SECRET_ACCESS_KEY: s3accesskeypass
+          - "4566:4566"
 
     env:
       MYSQL_HOST: "127.0.0.1"
@@ -105,17 +106,30 @@ jobs:
           ELASTIC_PASSWORD: ${{ job.services.elasticsearch.env.OPENSEARCH_INITIAL_ADMIN_PASSWORD }}
           MEMCACHE_SERVERS: localhost:${{ job.services.memcached.ports[11211] }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
-          AWS_ENDPOINT_URL_S3: http://localhost:${{ job.services.localstack.ports[4566] }}
+          AWS_ENDPOINT_URL_S3: http://localhost:${{ job.services.floci.ports[4566] }}
           AWS_ACCESS_KEY_ID_S3: s3accesskeyid
           AWS_SECRET_ACCESS_KEY_S3: s3accesskeypass
+          METADATA_STORAGE_BUCKET_NAME: metadata
           CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
           CI_NODE_INDEX: ${{ matrix.ci_node_index }}
           RAILS_ENV: test
         run: |
+          # Record this group's per-file runtimes so the committed log can be refreshed
+          RSPEC_OPTS="--format progress --format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime_rspec.log"
           if [ "${{ inputs.fail-fast }}" = "true" ]; then
-            bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec -o "--fail-fast"
-          else
-            bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec
+            RSPEC_OPTS="$RSPEC_OPTS --fail-fast"
           fi
+          bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec \
+            --group-by runtime --runtime-log spec/parallel_runtime_rspec.log -o "$RSPEC_OPTS"
           echo $?
+      # To refresh spec/parallel_runtime_rspec.log from a green run:
+      #   gh run download <run-id> -p 'rspec-runtime-*' -D tmp/rspec-runtime
+      #   cat tmp/rspec-runtime/*/parallel_runtime_rspec.log | sort > spec/parallel_runtime_rspec.log
+      - name: Upload test runtimes
+        if: success()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rspec-runtime-${{ matrix.ci_node_index }}
+          path: tmp/parallel_runtime_rspec.log
+          retention-days: 14
 

.github/workflows/pull_request.yml

@@ -10,13 +10,17 @@ on:
         type: boolean
         default: true
         required: false
+# A new push to the PR cancels the run still going on the previous commit
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   lint:
     uses: ./.github/workflows/rubocop.yml
     with:
       run_gem_audit: true
+  # Runs alongside lint rather than after it, so a lint failure doesn't hold up test results
   parallel-test:
-    needs: lint
     uses: ./.github/workflows/parallel_ci.yml
     secrets: inherit
     with:

spec/parallel_runtime_rspec.log

@@ -0,0 +1,163 @@
+spec/concerns/authenticable_spec.rb:7.23616984200001
+spec/concerns/binary_active_flag_spec.rb:0.00849663199997508
+spec/concerns/countable_spec.rb:10.087663592000013
+spec/concerns/crosscitable_spec.rb:0.7613472270000159
+spec/concerns/facetable_spec.rb:0.023880345999998553
+spec/concerns/fieldable_spec.rb:0.008280385999995588
+spec/concerns/helpable_spec.rb:8.357748405999985
+spec/concerns/indexable_spec.rb:34.535168561000006
+spec/concerns/mailable_spec.rb:1.5498162920000027
+spec/concerns/modelable_spec.rb:0.015952293999987432
+spec/concerns/paginatable_spec.rb:0.013771921999989445
+spec/concerns/passwordable_spec.rb:0.012796390000005431
+spec/concerns/rorable_spec.rb:3.502700089999962
+spec/concerns/wikidatable_spec.rb:0.03225019799998563
+spec/graphql/connections/elasticsearch_model_response_connection_spec.rb:0.006885564000015165
+spec/graphql/connections/hash_connection_spec.rb:0.007805345999997826
+spec/graphql/printout_spec.rb:0.00019748900000138292
+spec/graphql/types/actor_item_spec.rb:1.5268952729999796
+spec/graphql/types/address_type_spec.rb:0.00700992600002337
+spec/graphql/types/audiovisual_type_spec.rb:0.0035115580000137925
+spec/graphql/types/book_chapter_type_spec.rb:2.486551798999983
+spec/graphql/types/book_type_spec.rb:2.572460152000019
+spec/graphql/types/collection_type_spec.rb:0.004223553000002767
+spec/graphql/types/contributor_type_spec.rb:0.00902499699998316
+spec/graphql/types/country_type_spec.rb:0.0033182260000046426
+spec/graphql/types/creator_type_spec.rb:0.007820520999999303
+spec/graphql/types/data_catalog_type_spec.rb:0.05557559300001458
+spec/graphql/types/data_management_plan_type_spec.rb:7.104410647999998
+spec/graphql/types/dataset_type_spec.rb:9.946415457
+spec/graphql/types/date_type_spec.rb:0.0025608759999897757
+spec/graphql/types/defined_term_type_spec.rb:0.007254684000002953
+spec/graphql/types/description_type_spec.rb:0.006089913000010938
+spec/graphql/types/dissertation_type_spec.rb:7.030227154999977
+spec/graphql/types/doi_item_spec.rb:0.07502138799998193
+spec/graphql/types/employment_type_spec.rb:0.00658324200000493
+spec/graphql/types/event_type_spec.rb:0.0026494540000072675
+spec/graphql/types/facet_type_spec.rb:0.0037244609999902423
+spec/graphql/types/funder_type_spec.rb:1.0551022740000064
+spec/graphql/types/funding_type_spec.rb:0.00697197499999902
+spec/graphql/types/identifier_type_spec.rb:0.002585369000001947
+spec/graphql/types/image_type_spec.rb:0.0033275030000083916
+spec/graphql/types/instrument_type_spec.rb:1.5004330390000291
+spec/graphql/types/interactive_resource_type_spec.rb:0.0026971580000036965
+spec/graphql/types/issn_type_spec.rb:0.004774549999993383
+spec/graphql/types/journal_article_type_spec.rb:3.20518258300001
+spec/graphql/types/label_type_spec.rb:0.0027216440000046305
+spec/graphql/types/language_type_spec.rb:0.0026398939999978666
+spec/graphql/types/me_type_spec.rb:0.013357471999995596
+spec/graphql/types/member_prefix_type_spec.rb:0.003780732999999259
+spec/graphql/types/member_type_spec.rb:1.8165040540000064
+spec/graphql/types/model_type_spec.rb:0.008434497999999735
+spec/graphql/types/organization_type_spec.rb:7.059419085999991
+spec/graphql/types/peer_review_type_spec.rb:3.111343909999988
+spec/graphql/types/person_type_spec.rb:0.13761978499999827
+spec/graphql/types/physical_object_type_spec.rb:0.004595472999994854
+spec/graphql/types/prefix_type_spec.rb:0.0030883170000208793
+spec/graphql/types/preprint_type_spec.rb:1.6465600800000004
+spec/graphql/types/publication_type_spec.rb:0.003297656999990295
+spec/graphql/types/query_type_spec.rb:0.06846921400000383
+spec/graphql/types/registration_agency_type_spec.rb:0.0042192410000012615
+spec/graphql/types/repository_prefix_type_spec.rb:0.004525596000007681
+spec/graphql/types/repository_type_spec.rb:4.684698427000001
+spec/graphql/types/service_type_spec.rb:2.8001143590000055
+spec/graphql/types/software_type_spec.rb:0.8505349060000071
+spec/graphql/types/sound_type_spec.rb:0.00261251199998469
+spec/graphql/types/title_type_spec.rb:0.005532683999973642
+spec/graphql/types/work_type_spec.rb:81.47088930499999
+spec/graphql/types/workflow_type_spec.rb:0.002773273999991943
+spec/jobs/activity_import_by_id_job_spec.rb:0.24031427299999564
+spec/jobs/datacite_doi_import_in_bulk_job_spec.rb:0.2809385200000065
+spec/jobs/event_import_by_id_job_spec.rb:0.00922902599995723
+spec/jobs/event_registrant_update_by_id_job_spec.rb:0.0198783949999779
+spec/jobs/handle_job_spec.rb:0.24158462200000486
+spec/jobs/import_doi_job_spec.rb:0.22107962399999792
+spec/jobs/index_background_job_spec.rb:0.27061131799996474
+spec/jobs/index_job_spec.rb:0.4184778619999747
+spec/jobs/reindex_by_doi_job_spec.rb:0.5064978469999915
+spec/jobs/target_doi_by_id_job_spec.rb:0.2564392059999818
+spec/jobs/update_doi_job_spec.rb:0.2762944810000363
+spec/jobs/update_state_job_spec.rb:0.2434789920000071
+spec/jobs/url_job_spec.rb:0.2587572329999972
+spec/lib/params_sanitizer_spec.rb:0.016330451000015955
+spec/lib/tasks/contact_rake_spec.rb:0.24447041500002342
+spec/lib/tasks/datacite_doi_rake_spec.rb:0.5857183679999878
+spec/lib/tasks/doi_rake_spec.rb:14.157938868999906
+spec/lib/tasks/other_doi_rake_spec.rb:1.2370564899999863
+spec/models/ability_spec.rb:25.338267057999985
+spec/models/activity_shard_settings_spec.rb:0.00389968700000054
+spec/models/activity_spec.rb:1.0447918539999819
+spec/models/api_key_spec.rb:1.2028797429999827
+spec/models/client_prefix_spec.rb:1.371318030999987
+spec/models/client_spec.rb:18.692525805999992
+spec/models/concerns/indexable_active_index_cache_spec.rb:0.030144491000044127
+spec/models/contact_spec.rb:0.14504918500000485
+spec/models/data_catalog_spec.rb:0.005117537000046468
+spec/models/datacite_doi_shard_settings_spec.rb:0.005353662999993958
+spec/models/datacite_doi_spec.rb:0.7316373580000004
+spec/models/doi/graphql_query_builder_aggregates_spec.rb:0.002607522000005247
+spec/models/doi/graphql_query_builder_filters_spec.rb:0.04410271400001875
+spec/models/doi/graphql_query_builder_spec.rb:0.012373404000015853
+spec/models/doi/related_doi_indexer_spec.rb:0.012823082000011254
+spec/models/doi_related_spec.rb:26.25816549199999
+spec/models/doi_spec.rb:53.63329077899999
+spec/models/doi_verified_related_identifiers_spec.rb:3.3639981399999783
+spec/models/enriched_doi_spec.rb:1.8542901950000044
+spec/models/event_shard_settings_spec.rb:0.004222116999997638
+spec/models/event_spec.rb:0.8721649780000007
+spec/models/funder_spec.rb:0.1953958699999987
+spec/models/media_spec.rb:0.8705206329999555
+spec/models/metadata_spec.rb:7.491840321000012
+spec/models/organization_spec.rb:0.27823526099996343
+spec/models/other_doi_shard_settings_spec.rb:0.00750520700000834
+spec/models/other_doi_spec.rb:0.048826914999992255
+spec/models/person_spec.rb:0.014443773000010651
+spec/models/prefix_spec.rb:2.381553596999993
+spec/models/preloaded_event_relation_spec.rb:0.028499545999977727
+spec/models/provider_prefix_spec.rb:0.19360307899998475
+spec/models/provider_spec.rb:0.6489230849999927
+spec/models/reference_repository_spec.rb:2.244305988000008
+spec/models/resource_type_spec.rb:0.009994913000014094
+spec/models/usage_report_spec.rb:0.021707137000021248
+spec/models/user_spec.rb:0.25523830700001326
+spec/requests/activities_spec.rb:3.3237578410000026
+spec/requests/api_keys_spec.rb:6.368962718999995
+spec/requests/client_prefixes_spec.rb:7.674908447000007
+spec/requests/clients_spec.rb:14.752532505999966
+spec/requests/contacts_spec.rb:2.9021867910000196
+spec/requests/datacite_dois/auth_headers_spec.rb:11.474489887999994
+spec/requests/datacite_dois/content_negotiation_spec.rb:24.872178857999984
+spec/requests/datacite_dois/datacite_dois_gzip_spec.rb:0.370984897999989
+spec/requests/datacite_dois/datacite_dois_spec.rb:23.111433078000005
+spec/requests/datacite_dois/enriched_spec.rb:33.77841522900002
+spec/requests/datacite_dois/get_spec.rb:82.25671590000002
+spec/requests/datacite_dois/patch_geo_spec.rb:1.1677045739999983
+spec/requests/datacite_dois/patch_spec.rb:21.637055855
+spec/requests/datacite_dois/post_geo_spec.rb:1.999131043999995
+spec/requests/datacite_dois/post_spec.rb:36.78908062200003
+spec/requests/datacite_dois/query_spec.rb:35.591979861
+spec/requests/datafile_spec.rb:1.0342397480000045
+spec/requests/events_spec.rb:8.77957791099999
+spec/requests/exports_spec.rb:3.462076139999965
+spec/requests/heartbeat_spec.rb:0.00028958999999417756
+spec/requests/index_spec.rb:9.08730690599998
+spec/requests/media_spec.rb:7.762619322999996
+spec/requests/metadata_spec.rb:5.385189698000033
+spec/requests/old_events_spec.rb:6.879919326999982
+spec/requests/prefixes_spec.rb:3.741922211000002
+spec/requests/provider_prefixes_spec.rb:14.932324795
+spec/requests/providers_spec.rb:8.624034934999997
+spec/requests/reference_repositories_spec.rb:4.659708487999978
+spec/requests/repositories_spec.rb:10.090219087000008
+spec/requests/repository_prefixes_spec.rb:5.55214925300001
+spec/requests/sessions_spec.rb:0.8150180790000263
+spec/routing/clients_routing_spec.rb:0.010428230000002259
+spec/routing/dois_routing_spec.rb:0.010141721000024972
+spec/routing/media_routing_spec.rb:0.011999884999994492
+spec/routing/metadata_routing_spec.rb:0.009665776000019832
+spec/routing/prefixes_routing_spec.rb:0.01218184999999039
+spec/routing/providers_routing_spec.rb:0.09732675600000107
+spec/routing/repositories_routing_spec.rb:0.021495101999960298
+spec/services/events_preloader_spec.rb:3.873309336999995
+spec/services/ror_reference_store_spec.rb:0.03799921299997777
+spec/services/shared_container_settings_spec.rb:0.034730493000012075

spec/services/events_preloader_spec.rb

@@ -88,23 +88,26 @@
       end
     end
 
-    context "with large batch" do
-      it "chunks large DOI lists" do
-        # Create more than CHUNK_SIZE DOIs
-        large_batch = create_list(:doi, EventsPreloader::CHUNK_SIZE + 100, client: client, aasm_state: "findable")
+    context "with more DOIs than CHUNK_SIZE" do
+      before { stub_const("EventsPreloader::CHUNK_SIZE", 2) }
 
-        # Create events for some of them
+      # doi1 and doi3 fall in different chunks
+      let!(:event) do
         create(:event_for_crossref, {
-          subj_id: "https://doi.org/#{large_batch.first.doi}",
-          obj_id: "https://doi.org/#{large_batch.last.doi}",
+          subj_id: "https://doi.org/#{doi1.doi}",
+          obj_id: "https://doi.org/#{doi3.doi}",
           relation_type_id: "references",
         })
+      end
 
-        expect {
-          EventsPreloader.new(large_batch).preload!
-        }.not_to raise_error
+      it "queries once per chunk and attaches events across chunks" do
+        preloader = EventsPreloader.new([doi1, doi2, doi3])
+
+        expect { preloader.preload! }.not_to exceed_query_limit(2)
 
-        expect(large_batch.first.preloaded_events).not_to be_nil
+        expect(doi1.preloaded_events).to eq([event])
+        expect(doi3.preloaded_events).to eq([event])
+        expect(doi2.preloaded_events).to eq([])
       end
     end