diff --git a/.github/workflows/parallel_ci.yml b/.github/workflows/parallel_ci.yml index ff71f0724..c4146a76b 100644 --- a/.github/workflows/parallel_ci.yml +++ b/.github/workflows/parallel_ci.yml @@ -16,10 +16,13 @@ jobs: matrix: # set N-1 indexes for parallel jobs # When you run 2 parallel jobs then first job will have index 0, the second job will have index 1 etc - ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7] # Set N number of parallel jobs you want to run # Remember to update ci_node_index above to 0..N-1 - ci_node_total: [16] + # Groups are balanced by recorded runtime (spec/parallel_runtime_rspec.log). + # The slowest single spec file sets a floor on group time, so more jobs + # than this mostly adds setup cost without finishing sooner. + ci_node_total: [8] services: memcached: @@ -47,14 +50,12 @@ jobs: logger.org.opensearch.discovery: "ERROR" ports: - 9200 - localstack: - image: localstack/localstack:2.3.2 + floci: + # Lightweight LocalStack-compatible AWS emulator for S3 metadata storage. + # Signatures aren't validated, so the dummy S3 credentials below work as-is. + image: floci/floci:2.3.0 ports: - - "4566:4566" # LocalStack edge port - env: - SERVICES: s3 # Only start S3 - AWS_ACCESS_KEY_ID: s3accesskeyid - AWS_SECRET_ACCESS_KEY: s3accesskeypass + - "4566:4566" env: MYSQL_HOST: "127.0.0.1" @@ -105,17 +106,30 @@ jobs: ELASTIC_PASSWORD: ${{ job.services.elasticsearch.env.OPENSEARCH_INITIAL_ADMIN_PASSWORD }} MEMCACHE_SERVERS: localhost:${{ job.services.memcached.ports[11211] }} AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ENDPOINT_URL_S3: http://localhost:${{ job.services.localstack.ports[4566] }} + AWS_ENDPOINT_URL_S3: http://localhost:${{ job.services.floci.ports[4566] }} AWS_ACCESS_KEY_ID_S3: s3accesskeyid AWS_SECRET_ACCESS_KEY_S3: s3accesskeypass + METADATA_STORAGE_BUCKET_NAME: metadata CI_NODE_TOTAL: ${{ matrix.ci_node_total }} CI_NODE_INDEX: ${{ matrix.ci_node_index }} RAILS_ENV: test run: | + # Record this group's per-file runtimes so the committed log can be refreshed + RSPEC_OPTS="--format progress --format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime_rspec.log" if [ "${{ inputs.fail-fast }}" = "true" ]; then - bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec -o "--fail-fast" - else - bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec + RSPEC_OPTS="$RSPEC_OPTS --fail-fast" fi + bundle exec parallel_test spec/ -n $CI_NODE_TOTAL --only-group $CI_NODE_INDEX --type rspec \ + --group-by runtime --runtime-log spec/parallel_runtime_rspec.log -o "$RSPEC_OPTS" echo $? + # To refresh spec/parallel_runtime_rspec.log from a green run: + # gh run download -p 'rspec-runtime-*' -D tmp/rspec-runtime + # cat tmp/rspec-runtime/*/parallel_runtime_rspec.log | sort > spec/parallel_runtime_rspec.log + - name: Upload test runtimes + if: success() + uses: actions/upload-artifact@v7 + with: + name: rspec-runtime-${{ matrix.ci_node_index }} + path: tmp/parallel_runtime_rspec.log + retention-days: 14 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8bb3ecfd8..ed44430c8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -10,13 +10,17 @@ on: type: boolean default: true required: false +# A new push to the PR cancels the run still going on the previous commit +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: lint: uses: ./.github/workflows/rubocop.yml with: run_gem_audit: true + # Runs alongside lint rather than after it, so a lint failure doesn't hold up test results parallel-test: - needs: lint uses: ./.github/workflows/parallel_ci.yml secrets: inherit with: diff --git a/spec/parallel_runtime_rspec.log b/spec/parallel_runtime_rspec.log new file mode 100644 index 000000000..e25543992 --- /dev/null +++ b/spec/parallel_runtime_rspec.log @@ -0,0 +1,163 @@ +spec/concerns/authenticable_spec.rb:7.23616984200001 +spec/concerns/binary_active_flag_spec.rb:0.00849663199997508 +spec/concerns/countable_spec.rb:10.087663592000013 +spec/concerns/crosscitable_spec.rb:0.7613472270000159 +spec/concerns/facetable_spec.rb:0.023880345999998553 +spec/concerns/fieldable_spec.rb:0.008280385999995588 +spec/concerns/helpable_spec.rb:8.357748405999985 +spec/concerns/indexable_spec.rb:34.535168561000006 +spec/concerns/mailable_spec.rb:1.5498162920000027 +spec/concerns/modelable_spec.rb:0.015952293999987432 +spec/concerns/paginatable_spec.rb:0.013771921999989445 +spec/concerns/passwordable_spec.rb:0.012796390000005431 +spec/concerns/rorable_spec.rb:3.502700089999962 +spec/concerns/wikidatable_spec.rb:0.03225019799998563 +spec/graphql/connections/elasticsearch_model_response_connection_spec.rb:0.006885564000015165 +spec/graphql/connections/hash_connection_spec.rb:0.007805345999997826 +spec/graphql/printout_spec.rb:0.00019748900000138292 +spec/graphql/types/actor_item_spec.rb:1.5268952729999796 +spec/graphql/types/address_type_spec.rb:0.00700992600002337 +spec/graphql/types/audiovisual_type_spec.rb:0.0035115580000137925 +spec/graphql/types/book_chapter_type_spec.rb:2.486551798999983 +spec/graphql/types/book_type_spec.rb:2.572460152000019 +spec/graphql/types/collection_type_spec.rb:0.004223553000002767 +spec/graphql/types/contributor_type_spec.rb:0.00902499699998316 +spec/graphql/types/country_type_spec.rb:0.0033182260000046426 +spec/graphql/types/creator_type_spec.rb:0.007820520999999303 +spec/graphql/types/data_catalog_type_spec.rb:0.05557559300001458 +spec/graphql/types/data_management_plan_type_spec.rb:7.104410647999998 +spec/graphql/types/dataset_type_spec.rb:9.946415457 +spec/graphql/types/date_type_spec.rb:0.0025608759999897757 +spec/graphql/types/defined_term_type_spec.rb:0.007254684000002953 +spec/graphql/types/description_type_spec.rb:0.006089913000010938 +spec/graphql/types/dissertation_type_spec.rb:7.030227154999977 +spec/graphql/types/doi_item_spec.rb:0.07502138799998193 +spec/graphql/types/employment_type_spec.rb:0.00658324200000493 +spec/graphql/types/event_type_spec.rb:0.0026494540000072675 +spec/graphql/types/facet_type_spec.rb:0.0037244609999902423 +spec/graphql/types/funder_type_spec.rb:1.0551022740000064 +spec/graphql/types/funding_type_spec.rb:0.00697197499999902 +spec/graphql/types/identifier_type_spec.rb:0.002585369000001947 +spec/graphql/types/image_type_spec.rb:0.0033275030000083916 +spec/graphql/types/instrument_type_spec.rb:1.5004330390000291 +spec/graphql/types/interactive_resource_type_spec.rb:0.0026971580000036965 +spec/graphql/types/issn_type_spec.rb:0.004774549999993383 +spec/graphql/types/journal_article_type_spec.rb:3.20518258300001 +spec/graphql/types/label_type_spec.rb:0.0027216440000046305 +spec/graphql/types/language_type_spec.rb:0.0026398939999978666 +spec/graphql/types/me_type_spec.rb:0.013357471999995596 +spec/graphql/types/member_prefix_type_spec.rb:0.003780732999999259 +spec/graphql/types/member_type_spec.rb:1.8165040540000064 +spec/graphql/types/model_type_spec.rb:0.008434497999999735 +spec/graphql/types/organization_type_spec.rb:7.059419085999991 +spec/graphql/types/peer_review_type_spec.rb:3.111343909999988 +spec/graphql/types/person_type_spec.rb:0.13761978499999827 +spec/graphql/types/physical_object_type_spec.rb:0.004595472999994854 +spec/graphql/types/prefix_type_spec.rb:0.0030883170000208793 +spec/graphql/types/preprint_type_spec.rb:1.6465600800000004 +spec/graphql/types/publication_type_spec.rb:0.003297656999990295 +spec/graphql/types/query_type_spec.rb:0.06846921400000383 +spec/graphql/types/registration_agency_type_spec.rb:0.0042192410000012615 +spec/graphql/types/repository_prefix_type_spec.rb:0.004525596000007681 +spec/graphql/types/repository_type_spec.rb:4.684698427000001 +spec/graphql/types/service_type_spec.rb:2.8001143590000055 +spec/graphql/types/software_type_spec.rb:0.8505349060000071 +spec/graphql/types/sound_type_spec.rb:0.00261251199998469 +spec/graphql/types/title_type_spec.rb:0.005532683999973642 +spec/graphql/types/work_type_spec.rb:81.47088930499999 +spec/graphql/types/workflow_type_spec.rb:0.002773273999991943 +spec/jobs/activity_import_by_id_job_spec.rb:0.24031427299999564 +spec/jobs/datacite_doi_import_in_bulk_job_spec.rb:0.2809385200000065 +spec/jobs/event_import_by_id_job_spec.rb:0.00922902599995723 +spec/jobs/event_registrant_update_by_id_job_spec.rb:0.0198783949999779 +spec/jobs/handle_job_spec.rb:0.24158462200000486 +spec/jobs/import_doi_job_spec.rb:0.22107962399999792 +spec/jobs/index_background_job_spec.rb:0.27061131799996474 +spec/jobs/index_job_spec.rb:0.4184778619999747 +spec/jobs/reindex_by_doi_job_spec.rb:0.5064978469999915 +spec/jobs/target_doi_by_id_job_spec.rb:0.2564392059999818 +spec/jobs/update_doi_job_spec.rb:0.2762944810000363 +spec/jobs/update_state_job_spec.rb:0.2434789920000071 +spec/jobs/url_job_spec.rb:0.2587572329999972 +spec/lib/params_sanitizer_spec.rb:0.016330451000015955 +spec/lib/tasks/contact_rake_spec.rb:0.24447041500002342 +spec/lib/tasks/datacite_doi_rake_spec.rb:0.5857183679999878 +spec/lib/tasks/doi_rake_spec.rb:14.157938868999906 +spec/lib/tasks/other_doi_rake_spec.rb:1.2370564899999863 +spec/models/ability_spec.rb:25.338267057999985 +spec/models/activity_shard_settings_spec.rb:0.00389968700000054 +spec/models/activity_spec.rb:1.0447918539999819 +spec/models/api_key_spec.rb:1.2028797429999827 +spec/models/client_prefix_spec.rb:1.371318030999987 +spec/models/client_spec.rb:18.692525805999992 +spec/models/concerns/indexable_active_index_cache_spec.rb:0.030144491000044127 +spec/models/contact_spec.rb:0.14504918500000485 +spec/models/data_catalog_spec.rb:0.005117537000046468 +spec/models/datacite_doi_shard_settings_spec.rb:0.005353662999993958 +spec/models/datacite_doi_spec.rb:0.7316373580000004 +spec/models/doi/graphql_query_builder_aggregates_spec.rb:0.002607522000005247 +spec/models/doi/graphql_query_builder_filters_spec.rb:0.04410271400001875 +spec/models/doi/graphql_query_builder_spec.rb:0.012373404000015853 +spec/models/doi/related_doi_indexer_spec.rb:0.012823082000011254 +spec/models/doi_related_spec.rb:26.25816549199999 +spec/models/doi_spec.rb:53.63329077899999 +spec/models/doi_verified_related_identifiers_spec.rb:3.3639981399999783 +spec/models/enriched_doi_spec.rb:1.8542901950000044 +spec/models/event_shard_settings_spec.rb:0.004222116999997638 +spec/models/event_spec.rb:0.8721649780000007 +spec/models/funder_spec.rb:0.1953958699999987 +spec/models/media_spec.rb:0.8705206329999555 +spec/models/metadata_spec.rb:7.491840321000012 +spec/models/organization_spec.rb:0.27823526099996343 +spec/models/other_doi_shard_settings_spec.rb:0.00750520700000834 +spec/models/other_doi_spec.rb:0.048826914999992255 +spec/models/person_spec.rb:0.014443773000010651 +spec/models/prefix_spec.rb:2.381553596999993 +spec/models/preloaded_event_relation_spec.rb:0.028499545999977727 +spec/models/provider_prefix_spec.rb:0.19360307899998475 +spec/models/provider_spec.rb:0.6489230849999927 +spec/models/reference_repository_spec.rb:2.244305988000008 +spec/models/resource_type_spec.rb:0.009994913000014094 +spec/models/usage_report_spec.rb:0.021707137000021248 +spec/models/user_spec.rb:0.25523830700001326 +spec/requests/activities_spec.rb:3.3237578410000026 +spec/requests/api_keys_spec.rb:6.368962718999995 +spec/requests/client_prefixes_spec.rb:7.674908447000007 +spec/requests/clients_spec.rb:14.752532505999966 +spec/requests/contacts_spec.rb:2.9021867910000196 +spec/requests/datacite_dois/auth_headers_spec.rb:11.474489887999994 +spec/requests/datacite_dois/content_negotiation_spec.rb:24.872178857999984 +spec/requests/datacite_dois/datacite_dois_gzip_spec.rb:0.370984897999989 +spec/requests/datacite_dois/datacite_dois_spec.rb:23.111433078000005 +spec/requests/datacite_dois/enriched_spec.rb:33.77841522900002 +spec/requests/datacite_dois/get_spec.rb:82.25671590000002 +spec/requests/datacite_dois/patch_geo_spec.rb:1.1677045739999983 +spec/requests/datacite_dois/patch_spec.rb:21.637055855 +spec/requests/datacite_dois/post_geo_spec.rb:1.999131043999995 +spec/requests/datacite_dois/post_spec.rb:36.78908062200003 +spec/requests/datacite_dois/query_spec.rb:35.591979861 +spec/requests/datafile_spec.rb:1.0342397480000045 +spec/requests/events_spec.rb:8.77957791099999 +spec/requests/exports_spec.rb:3.462076139999965 +spec/requests/heartbeat_spec.rb:0.00028958999999417756 +spec/requests/index_spec.rb:9.08730690599998 +spec/requests/media_spec.rb:7.762619322999996 +spec/requests/metadata_spec.rb:5.385189698000033 +spec/requests/old_events_spec.rb:6.879919326999982 +spec/requests/prefixes_spec.rb:3.741922211000002 +spec/requests/provider_prefixes_spec.rb:14.932324795 +spec/requests/providers_spec.rb:8.624034934999997 +spec/requests/reference_repositories_spec.rb:4.659708487999978 +spec/requests/repositories_spec.rb:10.090219087000008 +spec/requests/repository_prefixes_spec.rb:5.55214925300001 +spec/requests/sessions_spec.rb:0.8150180790000263 +spec/routing/clients_routing_spec.rb:0.010428230000002259 +spec/routing/dois_routing_spec.rb:0.010141721000024972 +spec/routing/media_routing_spec.rb:0.011999884999994492 +spec/routing/metadata_routing_spec.rb:0.009665776000019832 +spec/routing/prefixes_routing_spec.rb:0.01218184999999039 +spec/routing/providers_routing_spec.rb:0.09732675600000107 +spec/routing/repositories_routing_spec.rb:0.021495101999960298 +spec/services/events_preloader_spec.rb:3.873309336999995 +spec/services/ror_reference_store_spec.rb:0.03799921299997777 +spec/services/shared_container_settings_spec.rb:0.034730493000012075 diff --git a/spec/services/events_preloader_spec.rb b/spec/services/events_preloader_spec.rb index ee43050a9..d4bb68389 100644 --- a/spec/services/events_preloader_spec.rb +++ b/spec/services/events_preloader_spec.rb @@ -88,23 +88,26 @@ end end - context "with large batch" do - it "chunks large DOI lists" do - # Create more than CHUNK_SIZE DOIs - large_batch = create_list(:doi, EventsPreloader::CHUNK_SIZE + 100, client: client, aasm_state: "findable") + context "with more DOIs than CHUNK_SIZE" do + before { stub_const("EventsPreloader::CHUNK_SIZE", 2) } - # Create events for some of them + # doi1 and doi3 fall in different chunks + let!(:event) do create(:event_for_crossref, { - subj_id: "https://doi.org/#{large_batch.first.doi}", - obj_id: "https://doi.org/#{large_batch.last.doi}", + subj_id: "https://doi.org/#{doi1.doi}", + obj_id: "https://doi.org/#{doi3.doi}", relation_type_id: "references", }) + end - expect { - EventsPreloader.new(large_batch).preload! - }.not_to raise_error + it "queries once per chunk and attaches events across chunks" do + preloader = EventsPreloader.new([doi1, doi2, doi3]) + + expect { preloader.preload! }.not_to exceed_query_limit(2) - expect(large_batch.first.preloaded_events).not_to be_nil + expect(doi1.preloaded_events).to eq([event]) + expect(doi3.preloaded_events).to eq([event]) + expect(doi2.preloaded_events).to eq([]) end end