diff --git a/README.md b/README.md index b25618519..ebd214940 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,10 @@ The scenarios that are available to use with Crank and some sample command lines # Contributing Microsoft employees can approve and merge contributions by joining [this team](https://repos.opensource.microsoft.com/orgs/aspnet/teams/aspnet-push) + +# Continuous benchmarking pipelines + +The Azure DevOps pipelines that drive the daily benchmark sweep are generated +from JSON configs by the [pod-scheduler](scripts/pod-scheduler/README.md). +See [`build/README.md`](build/README.md) for the mapping between configs and +generated YAML files, and how to regenerate them after editing. diff --git a/build/README.md b/build/README.md new file mode 100644 index 000000000..205883ac0 --- /dev/null +++ b/build/README.md @@ -0,0 +1,71 @@ +# `build/` — Pipeline definitions + +This directory contains Azure DevOps pipeline YAML files and the pod-scheduler +configs that generate them. + +## Generated files (do not edit by hand) + +The CI pipelines that run the daily benchmark sweep are generated by the +[pod-scheduler](../scripts/pod-scheduler/README.md). Each pipeline maps to a +JSON config that lists the available pods (machine groups) and scenarios. + +| Generated YAML | Source config | Notes | +|---|---|---| +| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.json` | On-prem fleet (citrine + mono); split across two YAMLs to spread load | +| `benchmarks-ci-azure.yml` | `benchmarks_ci_azure_pods.json` | Azure cloud pods (includes the merged eastus2 cobalt cloud pods) | +| `benchmarks-ci-cobalt.yml` | `benchmarks_ci_cobalt_pods.json` | Cobalt-hosted pods | + +To regenerate, from the repo root: + +```bash +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_pods.json \ + --yaml-output ./build + +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_azure_pods.json \ + --base-name benchmarks-ci-azure \ + --yaml-output ./build + +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_cobalt_pods.json \ + --base-name benchmarks-ci-cobalt \ + --yaml-output ./build +``` + +The header of each generated YAML embeds the exact regen command for that +file. + +## Hand-maintained scenario templates + +Everything else in this directory is hand-maintained. The most important +groups: + +- `*-scenarios.yml` — AzDO job templates that the generated pipelines invoke + (e.g. `proxies-scenarios.yml`, `grpc-scenarios.yml`). One per benchmark + family. The pod-scheduler picks which template to run via the + `template` field on each scenario in the JSON config. +- `prbenchmarks.*.config.yml` — PR-trigger pipelines for individual PR runs. +- `job-variables.yml` — shared variable definitions consumed by the scenario + templates. + +When adding a new benchmark family, drop a `*-scenarios.yml` template in this +directory and reference it from the relevant `benchmarks_ci_*_pods.json`. + +## Editing the schedule + +Common edits, all done in JSON: + +- **Add a scenario**: append to `scenarios[]` in the appropriate + `benchmarks_ci_*_pods.json`. +- **Add a pod / machine class**: append to `pods[]` and reference its name + from any scenario you want to target it. +- **Change cadence or splits**: edit `metadata.schedule` (cron), + `metadata.yaml_generation.target_yaml_count`, or + `metadata.yaml_generation.schedule_offset_hours`. +- **Override a job's timeout**: add `"timeout": ` to the scenario. + +After any edit, regenerate the YAML and commit both the JSON and the YAML +together. The snapshot tests in +`scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the +generated YAML drifts from the configs. diff --git a/build/benchmarks-ci-01.yml b/build/benchmarks-ci-01.yml index 975cd715d..30e73842e 100644 --- a/build/benchmarks-ci-01.yml +++ b/build/benchmarks-ci-01.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none schedules: - cron: "0 3/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Proxies_Gold_Lin - displayName: 1- Proxies Gold Lin +- job: Proxies_gold_lin + displayName: 1- Proxies gold-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 240 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Proxies_Intel_Lin - displayName: 1- Proxies Intel Lin + +- job: Proxies_intel_lin + displayName: 1- Proxies intel-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 240 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,11 +57,11 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Grpc_Gold_Win - displayName: 1- Grpc Gold Win + +- job: Grpc_gold_win + displayName: 1- Grpc gold-win pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 140 dependsOn: [] condition: succeededOrFailed() steps: @@ -73,11 +71,11 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: Grpc_Intel_Win - displayName: 1- Grpc Intel Win + +- job: Grpc_intel_win + displayName: 1- Grpc intel-win pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 140 dependsOn: [] condition: succeededOrFailed() steps: @@ -87,14 +85,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + # GROUP 2 -- job: PGO_Gold_Win - displayName: 2- PGO Gold Win +- job: PGO_gold_win + displayName: 2- PGO gold-win pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 180 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -103,12 +101,12 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: PGO_Intel_Win - displayName: 2- PGO Intel Win + +- job: PGO_intel_win + displayName: 2- PGO intel-win pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 180 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -117,12 +115,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Grpc_Gold_Lin - displayName: 2- Grpc Gold Lin + +- job: Grpc_gold_lin + displayName: 2- Grpc gold-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 140 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: grpc-scenarios.yml @@ -131,12 +129,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Grpc_Intel_Lin - displayName: 2- Grpc Intel Lin + +- job: Grpc_intel_lin + displayName: 2- Grpc intel-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 140 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: grpc-scenarios.yml @@ -145,14 +143,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - + # GROUP 3 -- job: Baselines_Database_Amd_Lin2 - displayName: 3- Baselines Database Amd Lin2 +- job: Baselines_Database_gold_win + displayName: 3- Baselines Database gold-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -160,27 +158,27 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: HttpClient_Gold_Lin - displayName: 3- HttpClient Gold Lin + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: Blazor_gold_lin + displayName: 3- Blazor gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - - template: httpclient-scenarios.yml + - template: blazor-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load2-load " - -- job: HttpClient_Intel_Lin - displayName: 3- HttpClient Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +- job: HttpClient_intel_lin + displayName: 3- HttpClient intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: httpclient-scenarios.yml @@ -189,12 +187,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Intel_Win - displayName: 3- SignalR Intel Win + +- job: SignalR_intel_win + displayName: 3- SignalR intel-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: signalr-scenarios.yml @@ -203,28 +201,28 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + # GROUP 4 -- job: Baselines_Gold_Lin - displayName: 4- Baselines Gold Lin +- job: SslStream_gold_lin + displayName: 4- SslStream gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - - template: baselines-scenarios.yml + - template: sslstream-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Baselines_Intel_Lin - displayName: 4- Baselines Intel Lin + +- job: Baselines_intel_lin + displayName: 4- Baselines intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -233,26 +231,26 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Frameworks_Database_Amd_Lin2 - displayName: 4- Frameworks Database Amd Lin2 + +- job: NativeAOT_gold_win + displayName: 4- NativeAOT gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: nativeaot-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load --profile gold-db-db " - -- job: NativeAOT_Intel_Win - displayName: 4- NativeAOT Intel Win + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: NativeAOT_intel_win + displayName: 4- NativeAOT intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -261,86 +259,86 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + # GROUP 5 -- job: SignalR_Gold_Lin - displayName: 5- SignalR Gold Lin +- job: Baselines_gold_lin + displayName: 5- Baselines gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: baselines-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: SignalR_Intel_Lin - displayName: 5- SignalR Intel Lin + +- job: Frameworks_Database_intel_lin + displayName: 5- Frameworks Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: MVC_Gold_Win - displayName: 5- MVC Gold Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " + +- job: EF_Core_intel_win + displayName: 5- EF Core intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: mvc-scenarios.yml + - template: efcore-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Trends_Intel_Win - displayName: 5- Trends Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: GC_gold_win + displayName: 5- GC gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app " + # GROUP 6 -- job: NativeAOT_Gold_Lin - displayName: 6- NativeAOT Gold Lin +- job: Frameworks_Database_amd_lin2 + displayName: 6- Frameworks Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: nativeaot-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: NativeAOT_Intel_Lin - displayName: 6- NativeAOT Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: NativeAOT_intel_lin + displayName: 6- NativeAOT intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -349,56 +347,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: EF_Core_Gold_Win - displayName: 6- EF Core Gold Win + +- job: GC_gold_lin + displayName: 6- GC gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: efcore-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: GC_Intel_Win - displayName: 6- GC Intel Win + arguments: "$(ciProfile) --profile gold-lin-app " + +- job: Single_File_gold_win + displayName: 6- Single File gold-win pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 7 -- job: Frameworks_Gold_Lin - displayName: 7- Frameworks Gold Lin +- job: MVC_gold_lin + displayName: 7- MVC gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - - template: frameworks-scenarios.yml + - template: mvc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Frameworks_Intel_Lin - displayName: 7- Frameworks Intel Lin + +- job: Frameworks_intel_lin + displayName: 7- Frameworks intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -407,12 +405,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Single_File_Gold_Win - displayName: 7- Single File Gold Win + +- job: Single_File_intel_win + displayName: 7- Single File intel-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - template: singlefile-scenarios.yml @@ -420,43 +418,43 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: Single_File_Intel_Win - displayName: 7- Single File Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: Crossgen_gold_win + displayName: 7- Crossgen gold-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - - template: singlefile-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 8 -- job: Trends_Database_Gold_Lin - displayName: 8- Trends Database Gold Lin +- job: Trends_amd_lin2 + displayName: 8- Trends amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Trends_Database_Intel_Lin - displayName: 8- Trends Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +- job: Trends_Database_intel_lin + displayName: 8- Trends Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -465,56 +463,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: WebSockets_Intel_Win - displayName: 8- WebSockets Intel Win + +- job: Crossgen_intel_win + displayName: 8- Crossgen intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Crossgen_Gold_Win - displayName: 8- Crossgen Gold Win + +- job: Build_gold_lin + displayName: 8- Build gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-lin-app " + # GROUP 9 -- job: Trends_Database_Amd_Lin2 - displayName: 9- Trends Database Amd Lin2 +- job: Frameworks_amd_lin2 + displayName: 9- Frameworks amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Trends_amd_lin2, Trends_Database_intel_lin, Crossgen_intel_win, Build_gold_lin] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: frameworks-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: WebSockets_Gold_Lin - displayName: 9- WebSockets Gold Lin + +- job: WebSockets_intel_lin + displayName: 9- WebSockets intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Trends_amd_lin2, Trends_Database_intel_lin, Crossgen_intel_win, Build_gold_lin] condition: succeededOrFailed() steps: - template: websocket-scenarios.yml @@ -522,63 +520,97 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load2-load " - -- job: WebSockets_Intel_Lin - displayName: 9- WebSockets Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +# GROUP 10 + +- job: Frameworks_gold_lin + displayName: 10- Frameworks gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Frameworks_amd_lin2, WebSockets_intel_lin] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: frameworks-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 + serviceBusQueueName: citrine1 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Crossgen_intel_lin + displayName: 10- Crossgen intel-lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Frameworks_amd_lin2, WebSockets_intel_lin] + condition: succeededOrFailed() + steps: + - template: crossgen2-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Build_Gold_Win - displayName: 9- Build Gold Win + +# GROUP 11 + +- job: Trends_Database_gold_lin + displayName: 11- Trends Database gold-lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Frameworks_gold_lin, Crossgen_intel_lin] + condition: succeededOrFailed() + steps: + - template: trend-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: citrine1 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Build_intel_lin + displayName: 11- Build intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Frameworks_gold_lin, Crossgen_intel_lin] condition: succeededOrFailed() steps: - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app " - -# GROUP 10 + arguments: "$(ciProfile) --profile intel-lin-app " + +# GROUP 12 -- job: Custom_Proxies_Gold_Lin - displayName: 10- Custom Proxies Gold Lin +- job: Crossgen_amd_lin2 + displayName: 12- Crossgen amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Lin2, WebSockets_Gold_Lin, WebSockets_Intel_Lin, Build_Gold_Win] + dependsOn: [Trends_Database_gold_lin, Build_intel_lin] condition: succeededOrFailed() steps: - - template: proxies-custom-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Custom_Proxies_Intel_Lin - displayName: 10- Custom Proxies Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +# GROUP 13 + +- job: Custom_Proxies_gold_lin + displayName: 13- Custom Proxies gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Lin2, WebSockets_Gold_Lin, WebSockets_Intel_Lin, Build_Gold_Win] + dependsOn: [Crossgen_amd_lin2] condition: succeededOrFailed() steps: - template: proxies-custom-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + diff --git a/build/benchmarks-ci-02.yml b/build/benchmarks-ci-02.yml index a833e5cbe..f9ee4c37b 100644 --- a/build/benchmarks-ci-02.yml +++ b/build/benchmarks-ci-02.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none schedules: - cron: "0 9/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Gold_Lin - displayName: 1- Containers Gold Lin +- job: Containers_gold_lin + displayName: 1- Containers gold-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Containers_Intel_Lin - displayName: 1- Containers Intel Lin + +- job: Containers_intel_lin + displayName: 1- Containers intel-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,9 +57,9 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Blazor_Intel_Perflin - displayName: 1- Blazor Intel Perflin + +- job: Blazor_intel_perflin + displayName: 1- Blazor intel-perflin pool: server timeoutInMinutes: 120 dependsOn: [] @@ -73,9 +71,9 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-perflin-app --profile intel-perfload-load " - -- job: HttpClient_Gold_Win - displayName: 1- HttpClient Gold Win + +- job: HttpClient_gold_win + displayName: 1- HttpClient gold-win pool: server timeoutInMinutes: 120 dependsOn: [] @@ -87,14 +85,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + # GROUP 2 -- job: PGO_Gold_Lin - displayName: 2- PGO Gold Lin +- job: PGO_gold_lin + displayName: 2- PGO gold-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + timeoutInMinutes: 180 + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -103,12 +101,12 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: PGO_Intel_Lin - displayName: 2- PGO Intel Lin + +- job: PGO_intel_lin + displayName: 2- PGO intel-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + timeoutInMinutes: 180 + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -117,12 +115,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: HttpClient_Intel_Win - displayName: 2- HttpClient Intel Win + +- job: HttpClient_intel_win + displayName: 2- HttpClient intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: httpclient-scenarios.yml @@ -131,12 +129,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Gold_Win - displayName: 2- Baselines Gold Win + +- job: Baselines_gold_win + displayName: 2- Baselines gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -145,14 +143,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + # GROUP 3 -- job: Baselines_Database_Gold_Lin - displayName: 3- Baselines Database Gold Lin +- job: Baselines_Database_amd_lin2 + displayName: 3- Baselines Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -160,13 +158,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Baselines_Database_Intel_Lin - displayName: 3- Baselines Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: Baselines_Database_intel_lin + displayName: 3- Baselines Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -175,12 +173,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Win - displayName: 3- Baselines Intel Win + +- job: Baselines_intel_win + displayName: 3- Baselines intel-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -189,28 +187,28 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Amd_Lin2 - displayName: 3- Baselines Amd Lin2 + +- job: SignalR_gold_win + displayName: 3- SignalR gold-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - - template: baselines-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 4 -- job: Baselines_Database_Gold_Win - displayName: 4- Baselines Database Gold Win +- job: Baselines_Database_gold_lin + displayName: 4- Baselines Database gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -218,13 +216,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Baselines_Database_Intel_Win - displayName: 4- Baselines Database Intel Win + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Baselines_Database_intel_win + displayName: 4- Baselines Database intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -233,12 +231,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Blazor_Gold_Lin - displayName: 4- Blazor Gold Lin + +- job: Blazor_intel_lin + displayName: 4- Blazor intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: blazor-scenarios.yml @@ -246,43 +244,43 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Blazor_Intel_Lin - displayName: 4- Blazor Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_gold_win + displayName: 4- Trends gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - - template: blazor-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 5 -- job: SslStream_Gold_Lin - displayName: 5- SslStream Gold Lin +- job: HttpClient_gold_lin + displayName: 5- HttpClient gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - - template: sslstream-scenarios.yml + - template: httpclient-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: SslStream_Intel_Lin - displayName: 5- SslStream Intel Lin + +- job: SslStream_intel_lin + displayName: 5- SslStream intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - template: sslstream-scenarios.yml @@ -291,26 +289,26 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Gold_Win - displayName: 5- SignalR Gold Win + +- job: MVC_gold_win + displayName: 5- MVC gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: mvc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: MVC_Intel_Win - displayName: 5- MVC Intel Win + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: MVC_intel_win + displayName: 5- MVC intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -319,42 +317,42 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + # GROUP 6 -- job: Frameworks_Database_Gold_Lin - displayName: 6- Frameworks Database Gold Lin +- job: Baselines_amd_lin2 + displayName: 6- Baselines amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: baselines-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Frameworks_Database_Intel_Lin - displayName: 6- Frameworks Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +- job: SignalR_intel_lin + displayName: 6- SignalR intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Gold_Win - displayName: 6- Trends Gold Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_intel_win + displayName: 6- Trends intel-win pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -362,13 +360,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: EF_Core_Intel_Win - displayName: 6- EF Core Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: EF_Core_gold_win + displayName: 6- EF Core gold-win pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - template: efcore-scenarios.yml @@ -376,29 +374,29 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 7 -- job: MVC_Gold_Lin - displayName: 7- MVC Gold Lin +- job: SignalR_gold_lin + displayName: 7- SignalR gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: mvc-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: MVC_Intel_Lin - displayName: 7- MVC Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +- job: MVC_intel_lin + displayName: 7- MVC intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -407,56 +405,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Amd_Lin2 - displayName: 7- Trends Amd Lin2 + +- job: GC_intel_win + displayName: 7- GC intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - -- job: GC_Gold_Win - displayName: 7- GC Gold Win + arguments: "$(ciProfile) --profile intel-win-app " + +- job: Trends_Database_gold_win + displayName: 7- Trends Database gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + # GROUP 8 -- job: NativeAOT_Gold_Win - displayName: 8- NativeAOT Gold Win +- job: Frameworks_Database_gold_lin + displayName: 8- Frameworks Database gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: nativeaot-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Trends_Gold_Lin - displayName: 8- Trends Gold Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Trends_intel_lin + displayName: 8- Trends intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -464,57 +462,57 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Trends_Intel_Lin - displayName: 8- Trends Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_Database_intel_win + displayName: 8- Trends Database intel-win pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Trends_Database_Intel_Win - displayName: 8- Trends Database Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " + +- job: WebSockets_gold_win + displayName: 8- WebSockets gold-win pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 9 -- job: Frameworks_Amd_Lin2 - displayName: 9- Frameworks Amd Lin2 +- job: NativeAOT_gold_lin + displayName: 9- NativeAOT gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: frameworks-scenarios.yml + - template: nativeaot-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: GC_Gold_Lin - displayName: 9- GC Gold Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: GC_intel_lin + displayName: 9- GC intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -522,179 +520,155 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app " - -- job: GC_Intel_Lin - displayName: 9- GC Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app " + +- job: WebSockets_intel_win + displayName: 9- WebSockets intel-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app " - -- job: WebSockets_Gold_Win - displayName: 9- WebSockets Gold Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: Build_gold_win + displayName: 9- Build gold-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app " + # GROUP 10 -- job: Trends_Database_Gold_Win - displayName: 10- Trends Database Gold Win +- job: Trends_gold_lin + displayName: 10- Trends gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Single_File_Gold_Lin - displayName: 10- Single File Gold Lin - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 - serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Single_File_Intel_Lin - displayName: 10- Single File Intel Lin + +- job: Single_File_intel_lin + displayName: 10- Single File intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Crossgen_Intel_Win - displayName: 10- Crossgen Intel Win + +- job: Build_intel_win + displayName: 10- Build intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile intel-win-app " + # GROUP 11 -- job: Crossgen_Gold_Lin - displayName: 11- Crossgen Gold Lin +- job: Trends_Database_amd_lin2 + displayName: 11- Trends Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_gold_lin, Single_File_intel_lin, Build_intel_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Crossgen_Intel_Lin - displayName: 11- Crossgen Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: Custom_Proxies_intel_lin + displayName: 11- Custom Proxies intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_gold_lin, Single_File_intel_lin, Build_intel_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: proxies-custom-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Crossgen_Amd_Lin2 - displayName: 11- Crossgen Amd Lin2 - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - -- job: Build_Intel_Win - displayName: 11- Build Intel Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " + +# GROUP 12 + +- job: Single_File_gold_lin + displayName: 12- Single File gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_Database_amd_lin2, Custom_Proxies_intel_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app " - -# GROUP 12 + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +# GROUP 13 -- job: Build_Gold_Lin - displayName: 12- Build Gold Lin +- job: WebSockets_gold_lin + displayName: 13- WebSockets gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Gold_Lin, Crossgen_Intel_Lin, Crossgen_Amd_Lin2, Build_Intel_Win] + dependsOn: [Single_File_gold_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app " - -- job: Build_Intel_Lin - displayName: 12- Build Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +# GROUP 14 + +- job: Crossgen_gold_lin + displayName: 14- Crossgen gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Gold_Lin, Crossgen_Intel_Lin, Crossgen_Amd_Lin2, Build_Intel_Win] + dependsOn: [WebSockets_gold_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app " - + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 868e76e43..2406d43af 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_azure_pods.json --base-name benchmarks-ci-azure --yaml-output ./build trigger: none pr: none schedules: - cron: "0 2/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Azure_Server_Arm64 - displayName: 1- Containers Azure Server Arm64 +- job: Containers_azure_arm64 + displayName: 1- Containers azure-arm64 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - -- job: Containers_Azure2_Server_Amd64 - displayName: 1- Containers Azure2 Server Amd64 + +- job: Containers_azure2_amd64 + displayName: 1- Containers azure2-amd64 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,14 +57,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 2 -- job: Containers_Cobalt_Cloud_Lin_Server - displayName: 2- Containers Cobalt Cloud Lin Server +- job: Containers_cobalt_cloud_lin + displayName: 2- Containers cobalt-cloud-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + timeoutInMinutes: 180 + dependsOn: [Containers_azure_arm64, Containers_azure2_amd64] condition: succeededOrFailed() steps: - template: containers-scenarios.yml @@ -75,12 +73,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Database_Azure_Server_Arm64 - displayName: 2- Baselines Database Azure Server Arm64 + +- job: Baselines_Database_azure_arm64 + displayName: 2- Baselines Database azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + dependsOn: [Containers_azure_arm64, Containers_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -89,14 +87,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - + # GROUP 3 -- job: Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 3- Containers Cobalt Cloud Lin Server Azure Linux3 +- job: Containers_cobalt_cloud_lin_azl3 + displayName: 3- Containers cobalt-cloud-lin-azl3 pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_cloud_lin, Baselines_Database_azure_arm64] condition: succeededOrFailed() steps: - template: containers-scenarios.yml @@ -105,12 +103,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Database_Azure2_Server_Amd64 - displayName: 3- Baselines Database Azure2 Server Amd64 + +- job: Baselines_Database_azure2_amd64 + displayName: 3- Baselines Database azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] + dependsOn: [Containers_cobalt_cloud_lin, Baselines_Database_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -119,14 +117,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 4 -- job: Baselines_Database_Cobalt_Cloud_Lin_Server - displayName: 4- Baselines Database Cobalt Cloud Lin Server +- job: Baselines_Database_cobalt_cloud_lin + displayName: 4- Baselines Database cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Containers_cobalt_cloud_lin_azl3, Baselines_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -135,12 +133,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Azure_Server_Arm64 - displayName: 4- Baselines Azure Server Arm64 + +- job: Baselines_azure_arm64 + displayName: 4- Baselines azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Containers_cobalt_cloud_lin_azl3, Baselines_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -149,14 +147,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " - + # GROUP 5 -- job: Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 5- Baselines Database Cobalt Cloud Lin Server Azure Linux3 +- job: Baselines_Database_cobalt_cloud_lin_azl3 + displayName: 5- Baselines Database cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] + dependsOn: [Baselines_Database_cobalt_cloud_lin, Baselines_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -165,12 +163,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Azure2_Server_Amd64 - displayName: 5- Baselines Azure2 Server Amd64 + +- job: Baselines_azure2_amd64 + displayName: 5- Baselines azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] + dependsOn: [Baselines_Database_cobalt_cloud_lin, Baselines_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -179,14 +177,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " - + # GROUP 6 -- job: Baselines_Cobalt_Cloud_Lin_Server - displayName: 6- Baselines Cobalt Cloud Lin Server +- job: Baselines_cobalt_cloud_lin + displayName: 6- Baselines cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + dependsOn: [Baselines_Database_cobalt_cloud_lin_azl3, Baselines_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -195,12 +193,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -- job: Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 6- Baselines Cobalt Cloud Lin Server Azure Linux3 + +- job: Baselines_cobalt_cloud_lin_azl3_dual + displayName: 6- Baselines cobalt-cloud-lin-azl3-dual pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + dependsOn: [Baselines_Database_cobalt_cloud_lin_azl3, Baselines_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -209,14 +207,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " - + # GROUP 7 -- job: MVC_Cobalt_Cloud_Lin_Server - displayName: 7- MVC Cobalt Cloud Lin Server +- job: MVC_cobalt_cloud_lin + displayName: 7- MVC cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_cloud_lin, Baselines_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -225,12 +223,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Trends_Azure_Server_Arm64 - displayName: 7- Trends Azure Server Arm64 + +- job: Trends_azure_arm64 + displayName: 7- Trends azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_cloud_lin, Baselines_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -239,14 +237,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " - + # GROUP 8 -- job: MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 8- MVC Cobalt Cloud Lin Server Azure Linux3 +- job: MVC_cobalt_cloud_lin_azl3 + displayName: 8- MVC cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] + dependsOn: [MVC_cobalt_cloud_lin, Trends_azure_arm64] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -255,12 +253,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Trends_Azure2_Server_Amd64 - displayName: 8- Trends Azure2 Server Amd64 + +- job: Trends_azure2_amd64 + displayName: 8- Trends azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] + dependsOn: [MVC_cobalt_cloud_lin, Trends_azure_arm64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -269,14 +267,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " - + # GROUP 9 -- job: Trends_Cobalt_Cloud_Lin_Server - displayName: 9- Trends Cobalt Cloud Lin Server +- job: Trends_cobalt_cloud_lin + displayName: 9- Trends cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_cobalt_cloud_lin_azl3, Trends_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -285,12 +283,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -- job: Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 9- Trends Cobalt Cloud Lin Server Azure Linux3 + +- job: Trends_cobalt_cloud_lin_azl3_dual + displayName: 9- Trends cobalt-cloud-lin-azl3-dual pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_cobalt_cloud_lin_azl3, Trends_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -299,14 +297,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " - + # GROUP 10 -- job: Trends_Idna_Amd_Lin - displayName: 10- Trends Idna Amd Lin +- job: Trends_idna_amd_lin + displayName: 10- Trends idna-amd-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_cloud_lin, Trends_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -314,13 +312,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-amd-win-load " - -- job: Trends_Idna_Intel_Lin - displayName: 10- Trends Idna Intel Lin + arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-intel-lin-load " + +- job: Trends_idna_amd_win + displayName: 10- Trends idna-amd-win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_cloud_lin, Trends_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -328,15 +326,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-intel-win-load " - + arguments: "$(ciProfile) --profile idna-amd-win-app --profile azure2-db-load " + # GROUP 11 -- job: Trends_Idna_Amd_Win - displayName: 11- Trends Idna Amd Win +- job: Trends_idna_intel_lin + displayName: 11- Trends idna-intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] + dependsOn: [Trends_idna_amd_lin, Trends_idna_amd_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -344,13 +342,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-win-app --profile idna-intel-lin-load " - -- job: Trends_Idna_Intel_Win - displayName: 11- Trends Idna Intel Win + arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-amd-lin-load " + +- job: Trends_idna_intel_win + displayName: 11- Trends idna-intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] + dependsOn: [Trends_idna_amd_lin, Trends_idna_amd_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -358,15 +356,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-win-app --profile idna-amd-lin-load " - + arguments: "$(ciProfile) --profile idna-intel-win-app --profile azure2-db-load " + # GROUP 12 -- job: Trends_Database_Azure_Server_Arm64 - displayName: 12- Trends Database Azure Server Arm64 +- job: Trends_Database_azure_arm64 + displayName: 12- Trends Database azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -375,12 +373,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - -- job: Trends_Database_Azure2_Server_Amd64 - displayName: 12- Trends Database Azure2 Server Amd64 + +- job: Trends_Database_azure2_amd64 + displayName: 12- Trends Database azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -389,14 +387,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 13 -- job: Trends_Database_Cobalt_Cloud_Lin_Server - displayName: 13- Trends Database Cobalt Cloud Lin Server +- job: Trends_Database_cobalt_cloud_lin + displayName: 13- Trends Database cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Azure_Server_Arm64, Trends_Database_Azure2_Server_Amd64] + dependsOn: [Trends_Database_azure_arm64, Trends_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -405,14 +403,14 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - + # GROUP 14 -- job: Trends_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 14- Trends Database Cobalt Cloud Lin Server Azure Linux3 +- job: Trends_Database_cobalt_cloud_lin_azl3 + displayName: 14- Trends Database cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Cloud_Lin_Server] + dependsOn: [Trends_Database_cobalt_cloud_lin] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -421,4 +419,4 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - + diff --git a/build/benchmarks-ci-cobalt.yml b/build/benchmarks-ci-cobalt.yml index 571c66798..1d0c2d0b5 100644 --- a/build/benchmarks-ci-cobalt.yml +++ b/build/benchmarks-ci-cobalt.yml @@ -1,15 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_cobalt_pods.json --base-name benchmarks-ci-cobalt --yaml-output ./build trigger: none pr: none schedules: - cron: "0 3/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -31,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Cobalt_Hosted_Lin_Server - displayName: 1- Containers Cobalt Hosted Lin Server +- job: Containers_cobalt_hosted_lin + displayName: 1- Containers cobalt-hosted-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -44,11 +43,11 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 1- Containers Cobalt Hosted Lin Server Azure Linux3 + +- job: Containers_cobalt_hosted_lin_azl3 + displayName: 1- Containers cobalt-hosted-lin-azl3 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -58,14 +57,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 2 -- job: PGO_Cobalt_Hosted_Lin_Server - displayName: 2- PGO Cobalt Hosted Lin Server +- job: PGO_cobalt_hosted_lin + displayName: 2- PGO cobalt-hosted-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Hosted_Lin_Server, Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_hosted_lin, Containers_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -74,12 +73,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 2- PGO Cobalt Hosted Lin Server Azure Linux3 + +- job: PGO_cobalt_hosted_lin_azl3 + displayName: 2- PGO cobalt-hosted-lin-azl3 pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Hosted_Lin_Server, Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_hosted_lin, Containers_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -88,14 +87,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 3 -- job: PGO_28_Core_Cobalt_Hosted_Lin_Server - displayName: 3- PGO 28 Core Cobalt Hosted Lin Server +- job: PGO_cobalt_hosted_lin_28 + displayName: 3- PGO cobalt-hosted-lin-28 pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Cobalt_Hosted_Lin_Server, PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [PGO_cobalt_hosted_lin, PGO_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -104,12 +103,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 3- PGO 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: PGO_cobalt_hosted_lin_azl3_28 + displayName: 3- PGO cobalt-hosted-lin-azl3-28 pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Cobalt_Hosted_Lin_Server, PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [PGO_cobalt_hosted_lin, PGO_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -118,14 +117,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 4 -- job: Baselines_Database_Cobalt_Hosted_Lin_Server - displayName: 4- Baselines Database Cobalt Hosted Lin Server +- job: Baselines_Database_cobalt_hosted_lin + displayName: 4- Baselines Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_28_Core_Cobalt_Hosted_Lin_Server, PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [PGO_cobalt_hosted_lin_28, PGO_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -134,12 +133,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 4- Baselines Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_Database_cobalt_hosted_lin_azl3 + displayName: 4- Baselines Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [PGO_28_Core_Cobalt_Hosted_Lin_Server, PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [PGO_cobalt_hosted_lin_28, PGO_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -148,14 +147,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 5 -- job: Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 5- Baselines Database 28 Core Cobalt Hosted Lin Server +- job: Baselines_Database_cobalt_hosted_lin_28 + displayName: 5- Baselines Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Hosted_Lin_Server, Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin, Baselines_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -164,12 +163,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 5- Baselines Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_Database_cobalt_hosted_lin_azl3_28 + displayName: 5- Baselines Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Hosted_Lin_Server, Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin, Baselines_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -178,14 +177,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 6 -- job: Baselines_Cobalt_Hosted_Lin_Server - displayName: 6- Baselines Cobalt Hosted Lin Server +- job: Baselines_cobalt_hosted_lin + displayName: 6- Baselines cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server, Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin_28, Baselines_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -194,12 +193,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 6- Baselines Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_cobalt_hosted_lin_azl3 + displayName: 6- Baselines cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server, Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin_28, Baselines_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -208,14 +207,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 7 -- job: Baselines_28_Core_Cobalt_Hosted_Lin_Server - displayName: 7- Baselines 28 Core Cobalt Hosted Lin Server +- job: Baselines_cobalt_hosted_lin_28 + displayName: 7- Baselines cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Hosted_Lin_Server, Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin, Baselines_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -224,12 +223,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 7- Baselines 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_cobalt_hosted_lin_azl3_28 + displayName: 7- Baselines cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Hosted_Lin_Server, Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin, Baselines_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -238,14 +237,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 8 -- job: Frameworks_Database_Cobalt_Hosted_Lin_Server - displayName: 8- Frameworks Database Cobalt Hosted Lin Server +- job: Frameworks_Database_cobalt_hosted_lin + displayName: 8- Frameworks Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_28_Core_Cobalt_Hosted_Lin_Server, Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin_28, Baselines_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -254,12 +253,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 8- Frameworks Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_Database_cobalt_hosted_lin_azl3 + displayName: 8- Frameworks Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_28_Core_Cobalt_Hosted_Lin_Server, Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin_28, Baselines_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -268,14 +267,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 9 -- job: Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 9- Frameworks Database 28 Core Cobalt Hosted Lin Server +- job: Frameworks_Database_cobalt_hosted_lin_28 + displayName: 9- Frameworks Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Cobalt_Hosted_Lin_Server, Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin, Frameworks_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -284,12 +283,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 9- Frameworks Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_Database_cobalt_hosted_lin_azl3_28 + displayName: 9- Frameworks Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Cobalt_Hosted_Lin_Server, Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin, Frameworks_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -298,14 +297,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 10 -- job: MVC_Cobalt_Hosted_Lin_Server - displayName: 10- MVC Cobalt Hosted Lin Server +- job: MVC_cobalt_hosted_lin + displayName: 10- MVC cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin_28, Frameworks_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -314,12 +313,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 10- MVC Cobalt Hosted Lin Server Azure Linux3 + +- job: MVC_cobalt_hosted_lin_azl3 + displayName: 10- MVC cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin_28, Frameworks_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -328,14 +327,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 11 -- job: NativeAOT_Cobalt_Hosted_Lin_Server - displayName: 11- NativeAOT Cobalt Hosted Lin Server +- job: NativeAOT_cobalt_hosted_lin + displayName: 11- NativeAOT cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Hosted_Lin_Server, MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [MVC_cobalt_hosted_lin, MVC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -344,12 +343,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 11- NativeAOT Cobalt Hosted Lin Server Azure Linux3 + +- job: NativeAOT_cobalt_hosted_lin_azl3 + displayName: 11- NativeAOT cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Hosted_Lin_Server, MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [MVC_cobalt_hosted_lin, MVC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -358,14 +357,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 12 -- job: NativeAOT_28_Core_Cobalt_Hosted_Lin_Server - displayName: 12- NativeAOT 28 Core Cobalt Hosted Lin Server +- job: NativeAOT_cobalt_hosted_lin_28 + displayName: 12- NativeAOT cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Cobalt_Hosted_Lin_Server, NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin, NativeAOT_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -374,12 +373,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 12- NativeAOT 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: NativeAOT_cobalt_hosted_lin_azl3_28 + displayName: 12- NativeAOT cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Cobalt_Hosted_Lin_Server, NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin, NativeAOT_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -388,14 +387,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 13 -- job: Trends_Cobalt_Hosted_Lin_Server - displayName: 13- Trends Cobalt Hosted Lin Server +- job: Trends_cobalt_hosted_lin + displayName: 13- Trends cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_28_Core_Cobalt_Hosted_Lin_Server, NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin_28, NativeAOT_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -404,12 +403,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 13- Trends Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_cobalt_hosted_lin_azl3 + displayName: 13- Trends cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_28_Core_Cobalt_Hosted_Lin_Server, NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin_28, NativeAOT_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -418,14 +417,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 14 -- job: Trends_28_Core_Cobalt_Hosted_Lin_Server - displayName: 14- Trends 28 Core Cobalt Hosted Lin Server +- job: Trends_cobalt_hosted_lin_28 + displayName: 14- Trends cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Hosted_Lin_Server, Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin, Trends_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -434,12 +433,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 14- Trends 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_cobalt_hosted_lin_azl3_28 + displayName: 14- Trends cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Hosted_Lin_Server, Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin, Trends_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -448,14 +447,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 15 -- job: Frameworks_Cobalt_Hosted_Lin_Server - displayName: 15- Frameworks Cobalt Hosted Lin Server +- job: Frameworks_cobalt_hosted_lin + displayName: 15- Frameworks cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_28_Core_Cobalt_Hosted_Lin_Server, Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin_28, Trends_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -464,12 +463,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 15- Frameworks Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_cobalt_hosted_lin_azl3 + displayName: 15- Frameworks cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_28_Core_Cobalt_Hosted_Lin_Server, Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin_28, Trends_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -478,14 +477,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 16 -- job: Frameworks_28_Core_Cobalt_Hosted_Lin_Server - displayName: 16- Frameworks 28 Core Cobalt Hosted Lin Server +- job: Frameworks_cobalt_hosted_lin_28 + displayName: 16- Frameworks cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Cobalt_Hosted_Lin_Server, Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin, Frameworks_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -494,12 +493,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 16- Frameworks 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_cobalt_hosted_lin_azl3_28 + displayName: 16- Frameworks cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Cobalt_Hosted_Lin_Server, Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin, Frameworks_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -508,14 +507,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 17 -- job: GC_Cobalt_Hosted_Lin_Server - displayName: 17- GC Cobalt Hosted Lin Server +- job: GC_cobalt_hosted_lin + displayName: 17- GC cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin_28, Frameworks_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -524,12 +523,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app " - -- job: GC_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 17- GC Cobalt Hosted Lin Server Azure Linux3 + +- job: GC_cobalt_hosted_lin_azl3 + displayName: 17- GC cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin_28, Frameworks_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -538,14 +537,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app " - + # GROUP 18 -- job: GC_28_Core_Cobalt_Hosted_Lin_Server - displayName: 18- GC 28 Core Cobalt Hosted Lin Server +- job: GC_cobalt_hosted_lin_28 + displayName: 18- GC cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [GC_Cobalt_Hosted_Lin_Server, GC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin, GC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -554,12 +553,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app " - -- job: GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 18- GC 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: GC_cobalt_hosted_lin_azl3_28 + displayName: 18- GC cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [GC_Cobalt_Hosted_Lin_Server, GC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin, GC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -568,14 +567,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app " - + # GROUP 19 -- job: Trends_Database_Cobalt_Hosted_Lin_Server - displayName: 19- Trends Database Cobalt Hosted Lin Server +- job: Trends_Database_cobalt_hosted_lin + displayName: 19- Trends Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [GC_28_Core_Cobalt_Hosted_Lin_Server, GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin_28, GC_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -584,12 +583,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 19- Trends Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_Database_cobalt_hosted_lin_azl3 + displayName: 19- Trends Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [GC_28_Core_Cobalt_Hosted_Lin_Server, GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin_28, GC_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -598,14 +597,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 20 -- job: Trends_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 20- Trends Database 28 Core Cobalt Hosted Lin Server +- job: Trends_Database_cobalt_hosted_lin_28 + displayName: 20- Trends Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Hosted_Lin_Server, Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin, Trends_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -614,12 +613,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 20- Trends Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_Database_cobalt_hosted_lin_azl3_28 + displayName: 20- Trends Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Hosted_Lin_Server, Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin, Trends_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -628,14 +627,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 21 -- job: Crossgen_Cobalt_Hosted_Lin_Server - displayName: 21- Crossgen Cobalt Hosted Lin Server +- job: Crossgen_cobalt_hosted_lin + displayName: 21- Crossgen cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_28_Core_Cobalt_Hosted_Lin_Server, Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin_28, Trends_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -644,12 +643,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 21- Crossgen Cobalt Hosted Lin Server Azure Linux3 + +- job: Crossgen_cobalt_hosted_lin_azl3 + displayName: 21- Crossgen cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_28_Core_Cobalt_Hosted_Lin_Server, Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin_28, Trends_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -658,14 +657,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 22 -- job: Crossgen_28_Core_Cobalt_Hosted_Lin_Server - displayName: 22- Crossgen 28 Core Cobalt Hosted Lin Server +- job: Crossgen_cobalt_hosted_lin_28 + displayName: 22- Crossgen cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Cobalt_Hosted_Lin_Server, Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Crossgen_cobalt_hosted_lin, Crossgen_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -674,12 +673,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Crossgen_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 22- Crossgen 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Crossgen_cobalt_hosted_lin_azl3_28 + displayName: 22- Crossgen cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Cobalt_Hosted_Lin_Server, Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Crossgen_cobalt_hosted_lin, Crossgen_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -688,4 +687,4 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + diff --git a/build/benchmarks.matrix.01.yml b/build/benchmarks.matrix.01.yml deleted file mode 100644 index 1540e7b61..000000000 --- a/build/benchmarks.matrix.01.yml +++ /dev/null @@ -1,299 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - citrine1 - - citrine2 - - citrine3 - - mono - -schedule: "0 9/12 * * *" - -groups: - - - jobs: - - - name: Trends Database Intel Linux - template: trend-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Trends Intel Windows - template: trend-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Trends Database Arm 28 Linux - template: trend-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Trends Database Gold Linux - template: trend-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Trends Database Intel Windows - template: trend-database-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - intel-db-db - - - name: Trends Intel Linux - template: trend-scenarios.yml - profiles: - - intel-lin-app - - intel-load2-load - - - name: Native Aot Arm 28 Linux - template: nativeaot-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Trends Gold Windows - template: trend-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Trends Database Amd Linux - template: trend-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Trends Arm 28 Linux - template: trend-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - - name: GC Intel Windows - template: gc-scenarios.yml - profiles: - - intel-win-app - - - name: Trends Gold Linux - template: trend-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Trends Amd Linux - template: trend-scenarios.yml - profiles: - - amd-lin-app - - intel-load2-load - - - name: Containers Intel Linux - template: containers-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: EF Core Intel Windows - template: efcore-scenarios.yml - profiles: - - intel-win-app - - - name: Trends Database Gold Windows - template: trend-database-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Baselines Database Intel Linux - template: baselines-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Intel Windows - template: baselines-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Baselines Database Arm 28 Linux - template: baselines-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Baselines Database Gold Linux - template: baselines-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Baselines Database Intel Windows - template: baselines-database-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - intel-db-db - - - name: Baselines Intel Linux - template: baselines-scenarios.yml - profiles: - - intel-lin-app - - intel-load2-load - - - name: PGO Arm 28 Linux - template: pgo-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Baselines Gold Windows - template: baselines-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Baselines Database Amd Linux - template: baselines-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Arm 28 Linux - template: baselines-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - - name: Native Aot Intel Windows - template: nativeaot-scenarios.yml - profiles: - - intel-win-app - - intel-lin-load - - amd-lin2-db - - - name: Baselines Gold Linux - template: baselines-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Baselines Amd Linux - template: baselines-scenarios.yml - profiles: - - amd-lin-app - - intel-lin-load - - - name: Frameworks Intel Linux - template: frameworks-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: Grpc Intel Windows - template: grpc-scenarios.yml - profiles: - - intel-win-app - - intel-db-load - - - name: Baselines Database Gold Windows - template: baselines-database-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Frameworks Database Intel Linux - template: frameworks-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: WebSockets Intel Windows - template: websocket-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Frameworks Database Arm 28 Linux - template: frameworks-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Frameworks Database Intel Gold Linux - template: frameworks-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Frameworks Database Amd Linux - template: frameworks-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Frameworks Arm 80 Linux - template: frameworks-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: Trends Siryn Linux - template: trend-scenarios.yml - profiles: - - siryn-lin-app - - intel-load2-load - - - name: Frameworks Intel Gold Linux - template: frameworks-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Frameworks Amd Linux - template: frameworks-scenarios.yml - profiles: - - amd-lin-app - - amd-lin2-load - - intel-load2-db # required by nodejs and tfb-database argument - - - name: Native Aot Intel Linux - template: nativeaot-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Siryn Linux - template: baselines-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load diff --git a/build/benchmarks.matrix.02.yml b/build/benchmarks.matrix.02.yml deleted file mode 100644 index 725ba9640..000000000 --- a/build/benchmarks.matrix.02.yml +++ /dev/null @@ -1,231 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - citrine1 - - citrine2 - - citrine3 - - mono - -schedule: "0 3/12 * * *" - -groups: - - - jobs: - - - name: Frameworks Arm 28 Linux - template: frameworks-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: GC Intel Linux - template: gc-scenarios.yml - profiles: - - intel-lin-app - - - name: SslStream Intel Linux - template: sslstream-scenarios.yml - profiles: - - intel-load-app - - amd-lin-load - - - jobs: - - - name: PGO Intel Windows - template: pgo-scenarios.yml - profiles: - - intel-win-app - - amd-lin2-load - - intel-db-db - - - name: PGO Intel Linux - template: pgo-scenarios.yml - profiles: - - intel-lin-app - - amd-lin-load - - intel-load-db - - - name: Mvc Siryn Linux - template: mvc-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load - - intel-load2-db - - - jobs: - - - name: SignalR Intel Linux - template: signalr-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: SignalR Intel Windows - template: signalr-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Blazor Intel Linux - template: blazor-scenarios.yml - profiles: - - intel-perflin-app - - intel-perfload-load - - - jobs: - - - name: WebSockets Intel Linux - template: websocket-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Build Intel Linux - template: build-perf-scenarios.yml - profiles: - - intel-load2-app - - intel-db-load - - - name: Trends Database Arm 80 Linux - template: trend-database-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db - - - jobs: - - - name: Single File Intel Linux - template: singlefile-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Mvc Intel Windows - template: mvc-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - intel-db-db - - - name: Frameworks Database Arm 80 Linux - template: frameworks-database-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db - - - jobs: - - - name: Mvc Intel Linux - template: mvc-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Single File Intel Windows - template: singlefile-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: GC Arm 28 Linux - template: gc-scenarios.yml - profiles: - - arm-lin-28-app - - - jobs: - - - name: Custom Proxies Intel Linux - template: proxies-custom-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Build Intel Windows - template: build-perf-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Trends Arm 80 Linux - template: trend-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - - jobs: - - - name: Proxies Intel Linux - template: proxies-scenarios.yml - timeout: 180 - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Grpc Intel Linux - template: grpc-scenarios.yml - profiles: - - intel-load2-app - - amd-lin-load - - - name: Baselines Database Siryn Linux - template: baselines-database-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load - - amd-lin2-db - - - jobs: - - - name: Crossgen Arm 28 Linux - template: crossgen2-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - - name: Crossgen Intel Linux - template: crossgen2-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Trends Database Siryn Linux - template: trend-database-scenarios.yml - profiles: - - siryn-lin-app - - intel-load2-load - - intel-db-db - - - jobs: - - - name: Crossgen Intel Windows - template: crossgen2-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - - name: Crossgen Amd Linux - template: crossgen2-scenarios.yml - profiles: - - amd-lin-app - - intel-load2-load - - - name: HttpClient Intel Linux - template: httpclient-scenarios.yml - profiles: - - intel-lin-app - - amd-lin2-load - - - jobs: - - - name: HttpClient Intel Windows - template: httpclient-scenarios.yml - profiles: - - intel-win-app - - amd-lin2-load diff --git a/build/benchmarks.template.liquid b/build/benchmarks.template.liquid deleted file mode 100644 index 2aa230237..000000000 --- a/build/benchmarks.template.liquid +++ /dev/null @@ -1,56 +0,0 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build - -{%- assign defaultTimeout = 120 %} - -trigger: none -pr: none - -schedules: -- cron: "{{ schedule }}" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. - branches: - include: - - main - -variables: - - template: job-variables.yml - - name: session - value: $(Build.BuildNumber) - - name: buildId - value: $(Build.BuildId) - - name: buildNumber - value: $(Build.BuildNumber) - - name: am - value: $[lt(format('{0:HH}', pipeline.startTime), 12)] - - name: pm - value: $[ge(format('{0:HH}', pipeline.startTime), 12)] - -jobs: - -{% for group in groups -%} -{% assign n = forloop.index -%} -# GROUP {{ n }} - -{% for job in group.jobs -%} -- job: {{job.name | replace: ' ', '_'}} - displayName: {{ n }}- {{job.name}} - pool: server - timeoutInMinutes: {{ job.timeout | default: defaultTimeout }} - dependsOn: [{%for previousJob in previousGroup.jobs%}{{previousJob.name | replace: ' ', '_'}}{%if forloop.last == false%}, {%endif%}{%endfor%}] - condition: succeededOrFailed() - steps: - - template: {{job.template}} - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: {{queues[forloop.index0]}} - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) {%for p in job.profiles%}--profile {{p}} {% endfor %}" - -{% endfor %} -{%-assign previousGroup = group -%} -{% endfor %} \ No newline at end of file diff --git a/build/benchmarks.yml b/build/benchmarks.yml deleted file mode 100644 index 39149baff..000000000 --- a/build/benchmarks.yml +++ /dev/null @@ -1,847 +0,0 @@ - -# Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation - -variables: - - template: job-variables.yml - - name: session - value: $(Build.BuildNumber) - - name: buildId - value: $(Build.BuildId) - - name: buildNumber - value: $(Build.BuildNumber) - - name: am - value: $[lt(format('{0:HH}', pipeline.startTime), 12)] - - name: pm - value: $[ge(format('{0:HH}', pipeline.startTime), 12)] - -jobs: - -# GROUP 1 - -- job: Trends_Database_Intel_Linux - displayName: 1- Trends Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Intel_Windows - displayName: 1- Trends Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Database_Arm_28_Linux - displayName: 1- Trends Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -- job: Trends_Database_Azure_Linux - displayName: 1- Trends Database Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 2 - -- job: Trends_Database_Intel_Windows - displayName: 2- Trends Database Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Intel_Linux - displayName: 2- Trends Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load2-load " - -- job: Trends_Database_Arm_28_Windows - displayName: 2- Trends Database Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin-load --profile amd-lin2-db " - -- job: Trends_Azure_Linux - displayName: 2- Trends Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 3 - -- job: Trends_Database_Amd_Linux - displayName: 3- Trends Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Arm_28_Linux - displayName: 3- Trends Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Trends_Arm_28_Windows - displayName: 3- Trends Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile intel-lin-load " - -- job: Baselines_Database_Azure_Linux - displayName: 3- Baselines Database Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 4 - -- job: Trends_Amd_Linux - displayName: 4- Trends Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - -- job: Containers_Intel_Linux - displayName: 4- Containers Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: containers-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: EF_Core_Intel_Windows - displayName: 4- EF Core Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: efcore-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-win-app " - -- job: Baselines_Azure_Linux - displayName: 4- Baselines Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 5 - -- job: Baselines_Database_Intel_Linux - displayName: 5- Baselines Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Windows - displayName: 5- Baselines Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Database_Arm_28_Linux - displayName: 5- Baselines Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 6 - -- job: Baselines_Database_Intel_Windows - displayName: 6- Baselines Database Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Linux - displayName: 6- Baselines Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load2-load " - -- job: Baselines_Database_Arm_28_Windows - displayName: 6- Baselines Database Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 7 - -- job: Baselines_Database_Amd_Linux - displayName: 7- Baselines Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Arm_28_Linux - displayName: 7- Baselines Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Baselines_Arm_28_Windows - displayName: 7- Baselines Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile intel-lin-load " - -# GROUP 8 - -- job: Baselines_Amd_Linux - displayName: 8- Baselines Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - -- job: Frameworks_Intel_Linux - displayName: 8- Frameworks Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile amd-lin2-db " - -- job: Grpc_Intel_Windows - displayName: 8- Grpc Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: grpc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-db-load " - -# GROUP 9 - -- job: Frameworks_Database_Intel_Linux - displayName: 9- Frameworks Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: WebSockets_Intel_Windows - displayName: 9- WebSockets Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: websocket-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Frameworks_Database_Arm_28_Linux - displayName: 9- Frameworks Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 10 - -- job: Frameworks_Database_Amd_Linux - displayName: 10- Frameworks Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: PGO_Intel_Windows - displayName: 10- PGO Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-lin-load --profile intel-load2-db " - -- job: Frameworks_Arm_80_Linux - displayName: 10- Frameworks Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 11 - -- job: Frameworks_Amd_Linux - displayName: 11- Frameworks Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Amd_Linux, PGO_Intel_Windows, Frameworks_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load --profile amd-lin2-db " - -- job: PGO_Intel_Linux - displayName: 11- PGO Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Amd_Linux, PGO_Intel_Windows, Frameworks_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -# GROUP 12 - -- job: Mono_Database_Intel_Linux - displayName: 12- Mono Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Linux, PGO_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: mono-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile amd-lin-load --profile intel-db-db " - -- job: Mono_Database_Arm_28_Linux - displayName: 12- Mono Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Linux, PGO_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: mono-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin2-load --profile intel-load2-db " - -# GROUP 13 - -- job: Mono_Arm_28_Linux - displayName: 13- Mono Arm 28 Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Mono_Database_Intel_Linux, Mono_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: mono-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Mono_Intel_Linux - displayName: 13- Mono Intel Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Mono_Database_Intel_Linux, Mono_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: mono-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -# GROUP 14 - -- job: PGO_Arm_28_Linux - displayName: 14- PGO Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load --profile intel-db-db " - -- job: Grpc_Intel_Linux - displayName: 14- Grpc Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: grpc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Trends_Arm_80_Windows - displayName: 14- Trends Arm 80 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-app --profile amd-lin2-load " - -# GROUP 15 - -- job: SignalR_Intel_Linux - displayName: 15- SignalR Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: signalr-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Intel_Windows - displayName: 15- SignalR Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: signalr-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load " - -- job: Blazor_Intel_Linux - displayName: 15- Blazor Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: blazor-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-perflin-app " - -# GROUP 16 - -- job: WebSockets_Intel_Linux - displayName: 16- WebSockets Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: websocket-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Build_Intel_Linux - displayName: 16- Build Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: build-perf-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-load2-app --profile intel-db-load " - -- job: Trends_Database_Arm_80_Linux - displayName: 16- Trends Database Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 17 - -- job: Single_File_Intel_Linux - displayName: 17- Single File Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Mvc_Intel_Windows - displayName: 17- Mvc Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Frameworks_Database_Arm_80_Linux - displayName: 17- Frameworks Database Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 18 - -- job: Mvc_Intel_Linux - displayName: 18- Mvc Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Single_File_Intel_Windows - displayName: 18- Single File Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Database_Arm_80_Windows - displayName: 18- Trends Database Arm 80 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 19 - -- job: Custom_Proxies_Intel_Linux - displayName: 19- Custom Proxies Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: proxies-custom-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Build_Intel_Windows - displayName: 19- Build Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: build-perf-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Arm_80_Linux - displayName: 19- Trends Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load " - -# GROUP 20 - -- job: Proxies_Intel_Linux - displayName: 20- Proxies Intel Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Custom_Proxies_Intel_Linux, Build_Intel_Windows, Trends_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: proxies-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Frameworks_Arm_28_Linux - displayName: 20- Frameworks Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Custom_Proxies_Intel_Linux, Build_Intel_Windows, Trends_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load --profile amd-lin2-db " - -# GROUP 21 - -- job: Crossgen_Arm_28_Linux - displayName: 21- Crossgen Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile intel-db-db " - -- job: Crossgen_Arm_28_Windows - displayName: 21- Crossgen Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin2-load " - -- job: Crossgen_Intel_Linux - displayName: 21- Crossgen Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -# GROUP 22 - -- job: Crossgen_Intel_Windows - displayName: 22- Crossgen Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Crossgen_Arm_28_Linux, Crossgen_Arm_28_Windows, Crossgen_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load " - -- job: Crossgen_Amd_Linux - displayName: 22- Crossgen Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Crossgen_Arm_28_Linux, Crossgen_Arm_28_Windows, Crossgen_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - - diff --git a/build/benchmarks_ci.json b/build/benchmarks_ci.json deleted file mode 100644 index 2c1fdec3b..000000000 --- a/build/benchmarks_ci.json +++ /dev/null @@ -1,522 +0,0 @@ -{ - "metadata": { - "name": "CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks", - "version": "1.0", - "schedule": "0 3/12 * * *", - "queues": [ - "citrine1", - "citrine2", - "citrine3", - "mono" - ], - "yaml_generation": { - "target_yaml_count": 2, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "intel-lin", - "machine_group": "intel", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "intel-lin-load" - ] - } - }, - "preferred_partners": [ - "intel-load", - "intel-load2", - "intel-db" - ] - }, - { - "name": "intel-win", - "machine_group": "intel", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "intel-win-load" - ] - } - }, - "preferred_partners": [ - "intel-load2", - "intel-load", - "intel-db" - ] - }, - { - "name": "intel-load", - "machine_group": "intel", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-load-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-load-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "intel-load-db" - ] - } - } - }, - { - "name": "intel-load2", - "machine_group": "intel", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-load2-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-load2-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "intel-load2-db" - ] - } - } - }, - { - "name": "intel-db", - "machine_group": "intel", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "intel-db-db" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-db-app" - ] - }, - "load": { - "priority": 3, - "profiles": [ - "intel-db-load" - ] - } - } - }, - { - "name": "intel-perflin", - "machine_group": "intel-perf", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-perflin-app" - ] - } - } - }, - { - "name": "intel-perfload", - "machine_group": "intel-perf", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-perfload-load" - ] - } - } - }, - { - "name": "amd-lin2", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "amd-lin2-app" - ] - } - } - }, - { - "name": "gold-lin", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "gold-lin-app", - "gold-lin-28-app" - ] - } - }, - "preferred_partners": [ - "gold-load", - "gold-load2", - "gold-db" - ] - }, - { - "name": "gold-win", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "gold-win-app", - "gold-win-28-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "gold-win-load" - ] - } - }, - "preferred_partners": [ - "gold-load2", - "gold-load", - "gold-db" - ] - }, - { - "name": "gold-load", - "machine_group": "gold", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "gold-load-load" - ] - } - } - }, - { - "name": "gold-load2", - "machine_group": "gold", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "gold-load2-load" - ] - } - } - }, - { - "name": "gold-db", - "machine_group": "gold", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "gold-db-db" - ] - } - } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Blazor", - "template": "blazor-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "intel-lin", - "intel-perflin" - ], - "estimated_runtime": null - }, - { - "name": "Build", - "template": "build-perf-scenarios.yml", - "type": 1, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 1.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 5.0 - }, - { - "name": "Custom Proxies", - "template": "proxies-custom-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 5.0 - }, - { - "name": "EF Core", - "template": "efcore-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-win", - "intel-win" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin", - "amd-lin2" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin", - "amd-lin2" - ], - "estimated_runtime": 20.0 - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Grpc", - "template": "grpc-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 70.0 - }, - { - "name": "HttpClient", - "template": "httpclient-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": null - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Proxies", - "template": "proxies-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 150.0 - }, - { - "name": "SignalR", - "template": "signalr-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Single File", - "template": "singlefile-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 10.0 - }, - { - "name": "SslStream", - "template": "sslstream-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": null - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 15.0 - }, - { - "name": "WebSockets", - "template": "websocket-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 6.0 - } - ] -} \ No newline at end of file diff --git a/build/benchmarks_ci_azure.json b/build/benchmarks_ci_azure.json deleted file mode 100644 index 20160bf17..000000000 --- a/build/benchmarks_ci_azure.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "metadata": { - "name": "Azure CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks on Azure", - "version": "1.0", - "schedule": "0 2/12 * * *", - "queues": [ - "azure", - "azurearm64" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "azure-db", - "machine_group": "azure", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "azure-db-db" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure-db-load" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "azure-db-app" - ] - } - } - }, - { - "name": "azure-client", - "machine_group": "azure", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "azure-client-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure-client-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure-client-db" - ] - } - } - }, - { - "name": "azure-server-arm64", - "machine_group": "azure", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "azure-server-arm64-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure-server-arm64-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure-server-arm64-db" - ] - } - }, - "preferred_partners": [ - "azure-client", - "azure-db" - ] - }, - { - "name": "azure2-client", - "machine_group": "azure2", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "azure2-client-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure2-client-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure2-client-db" - ] - } - } - }, - { - "name": "azure2-db", - "machine_group": "azure2", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "azure2-db-db" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure2-db-app" - ] - }, - "load": { - "priority": 3, - "profiles": [ - "azure2-db-load" - ] - } - } - }, - { - "name": "azure2-server-amd64", - "machine_group": "azure2", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "azure2-server-amd64-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure2-server-amd64-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure2-server-amd64-db" - ] - } - }, - "preferred_partners": [ - "azure2-client", - "azure2-db" - ] - }, - { - "name": "idna-amd-lin", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-amd-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-amd-lin-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-amd-lin-db" - ] - } - }, - "preferred_partners": [ - "idna-intel-lin", - "azure2-db" - ] - }, - { - "name": "idna-amd-win", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-amd-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-amd-win-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-amd-win-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "idna-intel-lin", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-intel-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-intel-lin-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-intel-lin-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "idna-intel-win", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-intel-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-intel-win-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-intel-win-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "cobalt-cloud-lin-server", - "machine_group": "cobalt-cloud", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - }, - { - "name": "cobalt-cloud-lin-client", - "machine_group": "cobalt-cloud", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-client-load" - ] - }, - "db": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-client-db" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-client-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-db", - "machine_group": "cobalt-cloud", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-db-db" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-db-load" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-db-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-server-azure-linux3", - "machine_group": "cobalt-cloud", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3", - "idna-amd-lin", - "idna-amd-win", - "idna-intel-lin", - "idna-intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - } - ] -} diff --git a/build/benchmarks_ci_azure_pods.json b/build/benchmarks_ci_azure_pods.json new file mode 100644 index 000000000..fead30052 --- /dev/null +++ b/build/benchmarks_ci_azure_pods.json @@ -0,0 +1,107 @@ +{ + "metadata": { + "name": "Azure CI Benchmarks Configuration", + "description": "Pod-based scheduling for Azure CI benchmarks (includes cobalt cloud from eastus2)", + "version": "2.1", + "schedule": "0 2/12 * * *", + "queues": [ + "azure", + "azurearm64" + ], + "yaml_generation": { + "target_yaml_count": 1, + "schedule_offset_hours": 0 + } + }, + "pods": [ + { + "name": "azure-arm64", + "machines": { "sut": "azure-server-arm64", "load": "azure-client", "db": "azure-db" }, + "profiles": { "sut": "azure-server-arm64-app", "load": "azure-client-load", "db": "azure-db-db" } + }, + { + "name": "azure2-amd64", + "machines": { "sut": "azure2-server-amd64", "load": "azure2-client", "db": "azure2-db" }, + "profiles": { "sut": "azure2-server-amd64-app", "load": "azure2-client-load", "db": "azure2-db-db" } + }, + { + "name": "cobalt-cloud-lin", + "machines": { "sut": "cobalt-cloud-lin-server", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } + }, + { + "name": "cobalt-cloud-lin-azl3", + "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } + }, + { + "name": "cobalt-cloud-lin-azl3-dual", + "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-db-load" } + }, + { + "name": "idna-amd-lin", + "machines": { "sut": "idna-amd-lin", "load": "idna-intel-lin" }, + "profiles": { "sut": "idna-amd-lin-app", "load": "idna-intel-lin-load" } + }, + { + "name": "idna-amd-win", + "machines": { "sut": "idna-amd-win", "load": "azure2-db" }, + "profiles": { "sut": "idna-amd-win-app", "load": "azure2-db-load" } + }, + { + "name": "idna-intel-lin", + "machines": { "sut": "idna-intel-lin", "load": "idna-amd-lin" }, + "profiles": { "sut": "idna-intel-lin-app", "load": "idna-amd-lin-load" } + }, + { + "name": "idna-intel-win", + "machines": { "sut": "idna-intel-win", "load": "azure2-db" }, + "profiles": { "sut": "idna-intel-win-app", "load": "azure2-db-load" } + } + ], + "scenarios": [ + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 90.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 45.0 + }, + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual"], + "estimated_runtime": 30.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual", "idna-amd-lin", "idna-amd-win", "idna-intel-lin", "idna-intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 15.0 + } + ] +} diff --git a/build/benchmarks_ci_cobalt.json b/build/benchmarks_ci_cobalt.json deleted file mode 100644 index acf0b9677..000000000 --- a/build/benchmarks_ci_cobalt.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "metadata": { - "name": "CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks", - "version": "1.0", - "schedule": "0 3/12 * * *", - "queues": [ - "cobalthosted", - "cobalthosted_azurelinux3" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "cobalt-hosted-lin-server", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-server-app", - "cobalt-hosted-lin-server-28-app" - ] - } - }, - "preferred_partners": [ - "cobalt-hosted-lin-client", - "cobalt-hosted-lin-db" - ] - }, - { - "name": "cobalt-hosted-lin-client", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-client-load" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-db", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-db-db" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-server-azure-linux3", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-server-azure-linux3-app", - "cobalt-hosted-lin-server-azure-linux3-28-app" - ] - } - }, - "preferred_partners": [ - "cobalt-hosted-lin-client-azure-linux3", - "cobalt-hosted-lin-db-azure-linux3" - ] - }, - { - "name": "cobalt-hosted-lin-client-azure-linux3", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-client-azure-linux3-load" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-db-azure-linux3", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-db-azure-linux3-db" - ] - } - } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines 28 Core", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Baselines Database 28 Core", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 5.0 - }, - { - "name": "Crossgen 28 Core", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 5.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks 28 Core", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Frameworks Database 28 Core", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "GC 28 Core", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT 28 Core", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "PGO 28 Core", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends 28 Core", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Trends Database 28 Core", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - } - ] -} \ No newline at end of file diff --git a/build/benchmarks_ci_cobalt_pods.json b/build/benchmarks_ci_cobalt_pods.json new file mode 100644 index 000000000..aed674790 --- /dev/null +++ b/build/benchmarks_ci_cobalt_pods.json @@ -0,0 +1,124 @@ +{ + "metadata": { + "name": "Cobalt Hosted CI Benchmarks Configuration", + "description": "Pod-based scheduling for cobalt hosted CI benchmarks", + "version": "2.0", + "schedule": "0 3/12 * * *", + "queues": [ + "cobalthosted", + "cobalthosted_azurelinux3" + ], + "yaml_generation": { + "target_yaml_count": 1, + "schedule_offset_hours": 0 + } + }, + "pods": [ + { + "name": "cobalt-hosted-lin", + "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, + "profiles": { "sut": "cobalt-hosted-lin-server-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } + }, + { + "name": "cobalt-hosted-lin-azl3", + "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, + "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } + }, + { + "name": "cobalt-hosted-lin-28", + "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, + "profiles": { "sut": "cobalt-hosted-lin-server-28-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } + }, + { + "name": "cobalt-hosted-lin-azl3-28", + "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, + "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-28-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } + } + ], + "scenarios": [ + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], + "estimated_runtime": 90.0 + }, + { + "name": "PGO", + "template": "pgo-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 90.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 45.0 + }, + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 30.0 + }, + { + "name": "Frameworks Database", + "template": "frameworks-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], + "estimated_runtime": 20.0 + }, + { + "name": "NativeAOT", + "template": "nativeaot-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + }, + { + "name": "Crossgen", + "template": "crossgen2-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 5.0 + }, + { + "name": "Frameworks", + "template": "frameworks-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + }, + { + "name": "GC", + "template": "gc-scenarios.yml", + "type": 1, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + } + ] +} diff --git a/build/benchmarks_ci_pods.json b/build/benchmarks_ci_pods.json new file mode 100644 index 000000000..ad4e75104 --- /dev/null +++ b/build/benchmarks_ci_pods.json @@ -0,0 +1,213 @@ +{ + "metadata": { + "name": "CI Benchmarks Configuration", + "description": "Pod-based scheduling for continuous integration benchmarks", + "version": "2.0", + "schedule": "0 3/12 * * *", + "queues": [ + "citrine1", + "citrine2", + "citrine3", + "mono" + ], + "yaml_generation": { + "target_yaml_count": 2, + "schedule_offset_hours": 6 + } + }, + "pods": [ + { + "name": "intel-lin", + "machines": { "sut": "intel-lin", "load": "intel-load", "db": "intel-db" }, + "profiles": { "sut": "intel-lin-app", "load": "intel-load-load", "db": "intel-db-db" } + }, + { + "name": "intel-win", + "machines": { "sut": "intel-win", "load": "intel-load2", "db": "intel-db" }, + "profiles": { "sut": "intel-win-app", "load": "intel-load2-load", "db": "intel-db-db" } + }, + { + "name": "gold-lin", + "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } + }, + { + "name": "gold-win", + "machines": { "sut": "gold-win", "load": "gold-load2", "db": "gold-db" }, + "profiles": { "sut": "gold-win-app", "load": "gold-load2-load", "db": "gold-db-db" } + }, + { + "name": "amd-lin2", + "machines": { "sut": "amd-lin2", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "amd-lin2-app", "load": "gold-load-load", "db": "gold-db-db" } + }, + { + "name": "intel-perflin", + "machines": { "sut": "intel-perflin", "load": "intel-perfload" }, + "profiles": { "sut": "intel-perflin-app", "load": "intel-perfload-load" } + } + ], + "scenarios": [ + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 30.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 45.0 + }, + { + "name": "Blazor", + "template": "blazor-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "intel-lin", "intel-perflin"], + "estimated_runtime": 45.0 + }, + { + "name": "Build", + "template": "build-perf-scenarios.yml", + "type": 1, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 1.0 + }, + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 90.0 + }, + { + "name": "Crossgen", + "template": "crossgen2-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 5.0 + }, + { + "name": "Custom Proxies", + "template": "proxies-custom-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 5.0 + }, + { + "name": "EF Core", + "template": "efcore-scenarios.yml", + "type": 2, + "pods": ["gold-win", "intel-win"], + "estimated_runtime": 15.0 + }, + { + "name": "Frameworks", + "template": "frameworks-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin", "amd-lin2"], + "estimated_runtime": 15.0 + }, + { + "name": "Frameworks Database", + "template": "frameworks-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin", "amd-lin2"], + "estimated_runtime": 20.0 + }, + { + "name": "GC", + "template": "gc-scenarios.yml", + "type": 1, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 15.0 + }, + { + "name": "Grpc", + "template": "grpc-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 70.0 + }, + { + "name": "HttpClient", + "template": "httpclient-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 45.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "NativeAOT", + "template": "nativeaot-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "PGO", + "template": "pgo-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 90.0 + }, + { + "name": "Proxies", + "template": "proxies-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 150.0 + }, + { + "name": "SignalR", + "template": "signalr-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 30.0 + }, + { + "name": "Single File", + "template": "singlefile-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 10.0 + }, + { + "name": "SslStream", + "template": "sslstream-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 45.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 15.0 + }, + { + "name": "WebSockets", + "template": "websocket-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 6.0 + } + ] +} diff --git a/scripts/pod-scheduler/README.md b/scripts/pod-scheduler/README.md new file mode 100644 index 000000000..4e46f0b62 --- /dev/null +++ b/scripts/pod-scheduler/README.md @@ -0,0 +1,197 @@ +# Pod-Based Crank Scheduler + +A simplified scheduler for assigning benchmark scenarios to machine "pods" and +generating Azure DevOps pipeline YAML files. + +## Concept + +A **pod** is a fixed group of machines that always run together: +- **SUT** (System Under Test) — required +- **Load** generator — optional, for dual/triple scenarios +- **DB** (Database) — optional, for triple scenarios + +Pods that share physical machines (e.g., two pods using the same DB machine) +cannot run in the same stage. The scheduler handles this automatically. + +## Quick Start + +Run from the repository root: + +```bash +# Show schedule summary +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json + +# Generate pipeline YAML files +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_pods.json \ + --yaml-output build + +# Regenerate the Azure or Cobalt pipelines +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_azure_pods.json \ + --base-name benchmarks-ci-azure --yaml-output build + +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_cobalt_pods.json \ + --base-name benchmarks-ci-cobalt --yaml-output build + +# Show which pods share machines +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --show-conflicts + +# List all runs without scheduling +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --list-runs +``` + +The header of every generated YAML embeds the exact regen command for that +file, so you can also copy the command from there. + +The scheduler runs in **strict mode** by default: any unknown pod or invalid +pod-for-scenario reference fails with a non-zero exit code so config typos +cannot silently drop scenarios from the pipeline. Pass `--lenient` to fall +back to the previous warn-and-skip behavior. + +Output is **deterministic**: identical input JSON always produces identical +YAML, so regenerations diff cleanly. To verify, run the snapshot tests: + +```bash +cd scripts/pod-scheduler +python -m unittest discover tests +``` + +## Configuration Format + +```json +{ + "metadata": { + "name": "Config Name", + "schedule": "0 3/12 * * *", + "queues": ["citrine1", "citrine2", "citrine3", "mono"], + "yaml_generation": { + "target_yaml_count": 2, + "schedule_offset_hours": 6 + }, + "pipeline": { + "pool": "server", + "service_bus_connection": "ASPNET Benchmarks Service Bus", + "service_bus_namespace": "aspnetbenchmarks" + } + }, + "pods": [ + { + "name": "gold-lin", + "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } + } + ], + "scenarios": [ + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win"], + "estimated_runtime": 30.0, + "timeout": 120 + } + ] +} +``` + +The `pipeline` block is optional; defaults match the legacy hardcoded values. + +The `schedule` field's **hour** must be a `H` or `H/N` cron expression +(e.g. `3` or `3/12`). Lists, ranges, and `*` are rejected at load time so the +hour-offset used for split YAMLs cannot silently no-op. + +### Pod Definition + +| Field | Description | +|-------|-------------| +| `name` | Unique identifier for the pod | +| `machines.sut` | Physical machine name for SUT role | +| `machines.load` | Physical machine name for Load role (optional) | +| `machines.db` | Physical machine name for DB role (optional) | +| `profiles.sut` | Crank profile name for SUT | +| `profiles.load` | Crank profile name for Load (optional) | +| `profiles.db` | Crank profile name for DB (optional) | + +### Scenario Definition + +| Field | Description | +|-------|-------------| +| `name` | Display name (also used as part of the AzDO job id) | +| `template` | YAML scenario template to invoke | +| `type` | 1=SINGLE, 2=DUAL, 3=TRIPLE (see below) | +| `pods` | List of pod names this scenario targets (no duplicates) | +| `estimated_runtime` | Runtime estimate in minutes; defaults per type if omitted | +| `timeout` | Optional explicit AzDO `timeoutInMinutes` override. When unset, the generator picks `max(120, min(240, ceil(2 * estimated_runtime)))` | + +### Scenario Types + +| Type | Machines Used | Example | +|------|--------------|---------| +| 1 (SINGLE) | SUT only | Build, GC | +| 2 (DUAL) | SUT + Load | Baselines, Grpc, SignalR | +| 3 (TRIPLE) | SUT + Load + DB | Baselines Database, PGO, Proxies | + +### Queue Assignment + +The N-th run within a stage is assigned to `queues[N % len(queues)]`. Queues +are treated as interchangeable workers; if a queue is pinned to specific +hardware in your service-bus topology, set `metadata.queues` accordingly so +the order matches your hardware layout. + +### Handling Shared Machines + +Two pods can share load/DB machines. For example: +- `gold-lin` pod: SUT=gold-lin, Load=gold-load, DB=gold-db +- `gold-win` pod: SUT=gold-win, Load=gold-load2, DB=gold-db + +These pods share `gold-db`. When both run type-3 scenarios, they cannot be in +the same stage. When `gold-win` runs a type-2 scenario (no DB), there's no +conflict. + +### Future: Multiple SUTs per Class + +If you get 2 SUT machines of the same class (e.g., gold-lin-1 and gold-lin-2), +create separate pods for each. They can share load/DB: + +```json +{"name": "gold-lin-1", "machines": {"sut": "gold-lin-1", "load": "gold-load", "db": "gold-db"}, ...}, +{"name": "gold-lin-2", "machines": {"sut": "gold-lin-2", "load": "gold-load", "db": "gold-db"}, ...} +``` + +The scheduler automatically prevents them from running simultaneously when they +share load/DB machines. + +## Algorithm + +1. **Expand** each scenario × pod into individual "runs" +2. **Sort** runs by runtime descending (longest-job-first), with the run name + as a stable tie-breaker so output is deterministic +3. **Pack** into stages greedily — each run goes into the first stage where no + physical machines conflict and the queue limit isn't exceeded +4. **Split** stages across multiple YAML files using bin-packing for balanced + runtime, restoring the original stage order within each bin + +## Files + +| File | Purpose | +|------|---------| +| `main.py` | CLI entry point, summary display | +| `models.py` | Data classes (Pod, Scenario, Run, Stage, Schedule, PipelineSettings) | +| `scheduler.py` | Scheduling algorithm | +| `config_loader.py` | JSON config parser + validation | +| `generator.py` | YAML generation | +| `tests/` | Unit + snapshot tests (`python -m unittest`) | + +This is intentionally script-style: the modules use absolute imports +(`from models import …`) and are run as `python main.py …`. To run the +tests, `cd scripts/pod-scheduler && python -m unittest discover tests`. + +## Tradeoffs vs. the full crank-scheduler + +By collapsing capabilities, priorities, preferred-partners, and machine groups +into fixed pod definitions, this scheduler is much smaller — but loses some +expressivity. If the hardware layout grows beyond "fixed SUT + load + DB +triples", the constraint solver from the full crank-scheduler may be a better +fit. For today's hardware, the simplification is intentional. diff --git a/scripts/pod-scheduler/config_loader.py b/scripts/pod-scheduler/config_loader.py new file mode 100644 index 000000000..f5b1d6377 --- /dev/null +++ b/scripts/pod-scheduler/config_loader.py @@ -0,0 +1,134 @@ +""" +JSON configuration loader for pod-based scheduling. +""" + +import json +import re +from typing import Any, Dict + +from models import ( + PipelineSettings, + Pod, + Scenario, + ScenarioType, + ScheduleConfig, +) + + +class ConfigError(ValueError): + """Raised when a JSON config is malformed or self-inconsistent.""" + + +_CRON_HOUR_RE = re.compile(r"^\d+(/\d+)?$") + + +def _require(node: Dict[str, Any], key: str, context: str) -> Any: + if key not in node: + raise ConfigError(f"Missing required field '{key}' in {context}") + return node[key] + + +def _validate_cron(schedule: str) -> None: + """Confirm we can later offset the cron's hour field deterministically.""" + parts = schedule.split() + if len(parts) != 5: + raise ConfigError( + f"Schedule {schedule!r} is not a 5-field cron expression" + ) + if not _CRON_HOUR_RE.match(parts[1]): + raise ConfigError( + f"Schedule {schedule!r} uses an unsupported hour field " + f"{parts[1]!r}. Pod-scheduler only supports 'H' or 'H/N' here so " + f"it can offset the hour for split YAMLs without ambiguity." + ) + + +def load_config(path: str) -> ScheduleConfig: + """Load and validate a pod-scheduler JSON configuration file.""" + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + metadata = _require(data, "metadata", "config root") + schedule = _require(metadata, "schedule", "metadata") + _validate_cron(schedule) + + queues = _require(metadata, "queues", "metadata") + if not isinstance(queues, list) or not queues: + raise ConfigError("metadata.queues must be a non-empty list") + if len(queues) != len(set(queues)): + raise ConfigError(f"metadata.queues contains duplicates: {queues}") + + yaml_gen = metadata.get("yaml_generation", {}) + + pipeline_meta = metadata.get("pipeline", {}) + pipeline = PipelineSettings( + pool=pipeline_meta.get("pool", PipelineSettings.pool), + service_bus_connection=pipeline_meta.get( + "service_bus_connection", + PipelineSettings.service_bus_connection, + ), + service_bus_namespace=pipeline_meta.get( + "service_bus_namespace", + PipelineSettings.service_bus_namespace, + ), + ) + + pods: Dict[str, Pod] = {} + raw_pods = _require(data, "pods", "config root") + for pod_data in raw_pods: + pod_name = _require(pod_data, "name", "pod entry") + if pod_name in pods: + raise ConfigError(f"Duplicate pod name: {pod_name!r}") + machines = _require(pod_data, "machines", f"pod '{pod_name}'") + profiles = _require(pod_data, "profiles", f"pod '{pod_name}'") + pods[pod_name] = Pod( + name=pod_name, + sut=_require(machines, "sut", f"pod '{pod_name}'.machines"), + load=machines.get("load"), + db=machines.get("db"), + sut_profile=_require(profiles, "sut", f"pod '{pod_name}'.profiles"), + load_profile=profiles.get("load"), + db_profile=profiles.get("db"), + ) + + scenarios = [] + raw_scenarios = _require(data, "scenarios", "config root") + for sc_data in raw_scenarios: + name = _require(sc_data, "name", "scenario entry") + scenario_pods = _require(sc_data, "pods", f"scenario '{name}'") + if not scenario_pods: + raise ConfigError(f"scenario '{name}' has empty pods list") + if len(scenario_pods) != len(set(scenario_pods)): + dupes = sorted({ + p for p in scenario_pods if scenario_pods.count(p) > 1 + }) + raise ConfigError( + f"scenario '{name}' lists duplicate pods: {dupes}" + ) + runtime_raw = sc_data.get("estimated_runtime") or 0 + timeout = sc_data.get("timeout") + if timeout is not None: + timeout = int(timeout) + if timeout <= 0: + raise ConfigError( + f"scenario '{name}' has non-positive timeout {timeout}" + ) + scenarios.append(Scenario( + name=name, + template=_require(sc_data, "template", f"scenario '{name}'"), + type=ScenarioType(_require(sc_data, "type", f"scenario '{name}'")), + pods=list(scenario_pods), + estimated_runtime=float(runtime_raw) if runtime_raw else 0.0, + timeout=timeout, + )) + + return ScheduleConfig( + name=metadata.get("name", ""), + schedule=schedule, + queues=list(queues), + target_yaml_count=yaml_gen.get("target_yaml_count", 1), + schedule_offset_hours=yaml_gen.get("schedule_offset_hours", 6), + pods=pods, + scenarios=scenarios, + pipeline=pipeline, + ) diff --git a/scripts/pod-scheduler/generator.py b/scripts/pod-scheduler/generator.py new file mode 100644 index 000000000..efde9d19a --- /dev/null +++ b/scripts/pod-scheduler/generator.py @@ -0,0 +1,246 @@ +""" +YAML generator for pod-based scheduling. + +Converts schedule data into Azure DevOps pipeline YAML files. The output +matches the structure previously produced by the Liquid-template based +crank-scheduler so existing pipeline consumers continue to work. +""" + +import os +import re +from typing import Any, Dict, List, Optional + +from models import ( + PipelineSettings, + Run, + Schedule, + ScheduleConfig, + sanitize_job_id, +) + + +_CRON_HOUR_RE = re.compile(r"^(\d+)(/\d+)?$") + + +class GeneratorError(ValueError): + """Raised when YAML generation cannot proceed safely.""" + + +def _offset_cron(cron: str, offset_hours: int) -> str: + """Offset a cron schedule's hour field by the given hours. + + Only ``H`` and ``H/N`` hour fields are supported; anything else (lists, + ranges, ``*`` etc.) raises :class:`GeneratorError` rather than silently + no-op'ing, which would otherwise cause every split YAML to fire at the + same time. + """ + parts = cron.split() + if len(parts) != 5: + raise GeneratorError( + f"Cron {cron!r} is not a 5-field expression; cannot offset" + ) + + hour_part = parts[1] + match = _CRON_HOUR_RE.match(hour_part) + if not match: + raise GeneratorError( + f"Cron {cron!r} uses an unsupported hour field {hour_part!r}; " + f"only 'H' or 'H/N' may be safely offset for split YAMLs" + ) + + base = int(match.group(1)) + suffix = match.group(2) or "" + new_base = (base + offset_hours) % 24 + parts[1] = f"{new_base}{suffix}" + return " ".join(parts) + + +def _job_timeout(run: Run) -> int: + """Pick a YAML timeout for the run. + + If the scenario specifies a timeout explicitly, use it. Otherwise derive + one from estimated_runtime (capped at [120, 240] minutes) so jobs that + historically took longer than the old flat 120-minute default still get + enough headroom. + """ + if run.scenario.timeout is not None: + return run.scenario.timeout + return max(120, min(240, int(run.estimated_runtime * 2))) + + +def schedule_to_template_data( + schedule: Schedule, + config: ScheduleConfig, + cron_override: Optional[str] = None, +) -> Dict[str, Any]: + """Project a Schedule into a JSON-serialisable shape useful for tests.""" + groups = [] + for stage in schedule.stages: + jobs = [] + for run in stage.runs: + jobs.append({ + "name": run.name, + "job_id": run.job_name, + "template": run.scenario.template, + "profiles": run.profiles, + "timeout": _job_timeout(run), + }) + groups.append({"jobs": jobs}) + + return { + "schedule": cron_override or config.schedule, + "queues": config.queues, + "groups": groups, + } + + +def _render_yaml( + data: Dict[str, Any], + pipeline: PipelineSettings, + source_config: Optional[str] = None, + base_name: str = "benchmarks-ci", +) -> str: + """Render the template data into Azure DevOps pipeline YAML.""" + lines: List[str] = [] + lines.append("# Do not change this file, it is generated by the pod-scheduler.") + lines.append("# Source of truth: see ../scripts/pod-scheduler/README.md") + lines.append("# To regenerate, run from the repo root:") + if source_config: + lines.append( + f"# python ./scripts/pod-scheduler/main.py " + f"--config {source_config} --base-name {base_name} " + f"--yaml-output ./build" + ) + else: + lines.append( + "# python ./scripts/pod-scheduler/main.py " + "--config ./build/.json " + "--yaml-output ./build" + ) + lines.append("") + lines.append("trigger: none") + lines.append("pr: none") + lines.append("") + lines.append("schedules:") + lines.append(f'- cron: "{data["schedule"]}"') + lines.append(" always: true") + lines.append(" branches:") + lines.append(" include:") + lines.append(" - main") + lines.append("") + lines.append("variables:") + lines.append(" - template: job-variables.yml") + lines.append(" - name: session") + lines.append(" value: $(Build.BuildNumber)") + lines.append(" - name: buildId") + lines.append(" value: $(Build.BuildId)") + lines.append(" - name: buildNumber") + lines.append(" value: $(Build.BuildNumber)") + lines.append(" - name: am") + lines.append(" value: $[lt(format('{0:HH}', pipeline.startTime), 12)]") + lines.append(" - name: pm") + lines.append(" value: $[ge(format('{0:HH}', pipeline.startTime), 12)]") + lines.append("") + lines.append("jobs:") + lines.append("") + + queues = data["queues"] + prev_group_jobs: List[str] = [] + seen_job_ids = set() + + for group_idx, group in enumerate(data["groups"]): + group_num = group_idx + 1 + lines.append(f"# GROUP {group_num}") + lines.append("") + + current_jobs: List[str] = [] + for job_idx, job in enumerate(group["jobs"]): + job_id = job["job_id"] + if job_id in seen_job_ids: + raise GeneratorError( + f"Duplicate job id {job_id!r} produced for jobs named " + f"{job['name']!r}. Rename the offending pod or scenario." + ) + seen_job_ids.add(job_id) + current_jobs.append(job_id) + + queue = queues[job_idx % len(queues)] + depends = ", ".join(prev_group_jobs) if prev_group_jobs else "" + profiles_args = " ".join( + f"--profile {p}" for p in job["profiles"] + ) + + lines.append(f"- job: {job_id}") + lines.append(f" displayName: {group_num}- {job['name']}") + lines.append(f" pool: {pipeline.pool}") + lines.append(f" timeoutInMinutes: {job['timeout']}") + lines.append(f" dependsOn: [{depends}]") + lines.append(" condition: succeededOrFailed()") + lines.append(" steps:") + lines.append(f" - template: {job['template']}") + lines.append(" parameters:") + lines.append( + f" connection: {pipeline.service_bus_connection}" + ) + lines.append(f" serviceBusQueueName: {queue}") + lines.append( + f" serviceBusNamespace: {pipeline.service_bus_namespace}" + ) + lines.append( + f' arguments: "$(ciProfile) {profiles_args} "' + ) + lines.append("") + + prev_group_jobs = current_jobs + + return "\n".join(lines) + "\n" + + +def generate_yamls( + schedules: List[Schedule], + config: ScheduleConfig, + output_dir: str, + base_name: str = "benchmarks-ci", + source_config: Optional[str] = None, +) -> List[str]: + """Generate YAML pipeline files for each sub-schedule. + + All job ids are sanitized via :func:`sanitize_job_id` and verified to be + unique within the produced YAML; collisions raise GeneratorError instead + of producing AzDO-invalid output. + + ``source_config`` is the path to the JSON config that produced the + schedule. When provided, it's embedded in the generated YAML header so + each file documents the exact command needed to regenerate it. + """ + os.makedirs(output_dir, exist_ok=True) + output_files = [] + + for i, sched in enumerate(schedules): + # Pre-flight: ensure every run already has a sanitized id. + for stage in sched.stages: + for run in stage.runs: + _ = sanitize_job_id(run.name) + + suffix = f"-{i + 1:02d}" if len(schedules) > 1 else "" + filename = f"{base_name}{suffix}.yml" + filepath = os.path.join(output_dir, filename) + + cron = _offset_cron( + config.schedule, config.schedule_offset_hours * i + ) + data = schedule_to_template_data(sched, config, cron_override=cron) + yaml_content = _render_yaml( + data, + config.pipeline, + source_config=source_config, + base_name=base_name, + ) + + with open(filepath, "w", newline="\n", encoding="utf-8") as f: + f.write(yaml_content) + + output_files.append(filepath) + print(f" Generated: {filepath}") + + return output_files diff --git a/scripts/pod-scheduler/main.py b/scripts/pod-scheduler/main.py new file mode 100644 index 000000000..00cdc0236 --- /dev/null +++ b/scripts/pod-scheduler/main.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Pod-based crank scheduler. + +Assigns benchmark scenarios to machine pods and generates Azure DevOps +pipeline YAML files. Pods define fixed machine groupings (SUT + load + DB), +and the scheduler ensures no physical machine is double-booked per stage. + +Usage: + python main.py --config ./build/benchmarks_ci_pods.json + python main.py --config ./build/benchmarks_ci_pods.json \\ + --yaml-output ./build +""" + +import argparse +import json +import os +import sys +from typing import List + +from config_loader import ConfigError, load_config +from generator import GeneratorError, generate_yamls, schedule_to_template_data +from models import Schedule, ScheduleConfig +from scheduler import ( + SchedulerError, + create_schedule, + expand_runs, + split_schedule, +) + + +def print_summary(config: ScheduleConfig, schedule: Schedule) -> None: + """Print a human-readable schedule summary.""" + print(f"\n{'=' * 70}") + print(f"SCHEDULE SUMMARY: {config.name}") + print(f"{'=' * 70}") + print(f" Pods: {len(config.pods)}") + print(f" Scenarios: {len(config.scenarios)}") + print(f" Total runs: {schedule.total_runs}") + print(f" Stages: {len(schedule.stages)}") + print(f" Queues: {len(config.queues)} ({', '.join(config.queues)})") + print(f" Est. total time: {schedule.total_duration:.0f} min " + f"({schedule.total_duration / 60:.1f} hrs)") + print() + + machine_time = {} + machine_total = {} + for stage in schedule.stages: + stage_machines = set() + for run in stage.runs: + for m in run.machines_used: + stage_machines.add(m) + machine_time.setdefault(m, 0) + machine_time[m] += run.estimated_runtime + for m in stage_machines: + machine_total.setdefault(m, 0) + machine_total[m] += stage.duration + + if machine_total: + print("MACHINE UTILIZATION:") + for m in sorted(machine_total.keys()): + busy = machine_time.get(m, 0) + total = machine_total.get(m, 1) + pct = (busy / total * 100) if total > 0 else 0 + filled = int(pct / 5) + bar = "#" * filled + "." * (20 - filled) + print(f" {m:<25} {bar} {pct:5.1f}% " + f"({busy:.0f}/{total:.0f} min)") + print() + + print("STAGE BREAKDOWN:") + for i, stage in enumerate(schedule.stages): + print(f"\n Stage {i} (Duration: {stage.duration:.0f} min)") + print(f" {'Queue':<8} {'Scenario':<40} {'Runtime':>8} " + f"Machines") + print(f" {'-' * 80}") + for j, run in enumerate(stage.runs): + queue = config.queues[j % len(config.queues)] + machines = ", ".join(sorted(run.machines_used)) + print(f" {queue:<8} {run.name:<40} {run.estimated_runtime:>6.0f}m " + f"{machines}") + print() + + +def print_split_summary(schedules: List[Schedule], config: ScheduleConfig) -> None: + """Print summary of multi-YAML split.""" + if len(schedules) <= 1: + return + print(f"YAML SPLIT ({len(schedules)} files):") + for i, sched in enumerate(schedules): + print(f" YAML {i + 1}: {len(sched.stages)} stages, " + f"{sched.total_runs} runs, " + f"{sched.total_duration:.0f} min") + print() + + +def print_pod_conflicts(config: ScheduleConfig) -> None: + """Show which pods share physical machines (potential conflicts).""" + machine_pods = {} + for pod in config.pods.values(): + for role in ["sut", "load", "db"]: + machine = getattr(pod, role) + if machine: + machine_pods.setdefault(machine, []).append( + (pod.name, role) + ) + + shared = {m: pods for m, pods in machine_pods.items() if len(pods) > 1} + if shared: + print("SHARED MACHINES (pods that cannot run simultaneously):") + for machine, pods in sorted(shared.items()): + pod_strs = [f"{name}({role})" for name, role in pods] + print(f" {machine:<20} used by: {', '.join(pod_strs)}") + print() + + +def _format_source_path(path: str) -> str: + """Render a config path for embedding in generated YAML headers. + + Tries to express the path relative to the repo root using forward slashes + so the regen command is identical on Windows and POSIX shells. Falls back + to ``./`` if the path is outside the repo. + """ + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") + ) + abs_path = os.path.abspath(path) + try: + rel = os.path.relpath(abs_path, repo_root) + except ValueError: + rel = os.path.basename(abs_path) + rel = rel.replace(os.sep, "/") + if rel.startswith("../"): + rel = os.path.basename(abs_path) + return f"./{rel}" + + +def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Pod-based crank scheduler" + ) + parser.add_argument( + "--config", required=True, + help="Path to JSON configuration file" + ) + parser.add_argument( + "--yaml-output", + help="Directory to write generated YAML files" + ) + parser.add_argument( + "--base-name", default="benchmarks-ci", + help="Base filename for generated YAMLs (default: benchmarks-ci)" + ) + parser.add_argument( + "--target-yamls", type=int, + help="Override number of YAML files to generate" + ) + parser.add_argument( + "--template-data", action="store_true", + help="Print template data as JSON (for debugging)" + ) + parser.add_argument( + "--list-runs", action="store_true", + help="List all expanded runs without scheduling" + ) + parser.add_argument( + "--show-conflicts", action="store_true", + help="Show pods that share physical machines" + ) + parser.add_argument( + "--lenient", action="store_true", + help="Warn instead of fail on unknown or invalid pod references. " + "Off by default so config typos do not silently drop scenarios." + ) + return parser + + +def main(argv: List[str] = None) -> int: + parser = _build_arg_parser() + args = parser.parse_args(argv) + + try: + print(f"Loading config: {args.config}") + config = load_config(args.config) + print(f" Loaded {len(config.pods)} pods, " + f"{len(config.scenarios)} scenarios") + + strict = not args.lenient + + if args.show_conflicts: + print_pod_conflicts(config) + + if args.list_runs: + runs = expand_runs(config, strict=strict) + print(f"\nAll runs ({len(runs)} total):") + for r in runs: + machines = ", ".join(sorted(r.machines_used)) + print(f" {r.name:<45} type={r.scenario.type.value} " + f"runtime={r.estimated_runtime:.0f}m " + f"machines=[{machines}]") + return 0 + + schedule = create_schedule(config, strict=strict) + print_summary(config, schedule) + print_pod_conflicts(config) + + yaml_count = args.target_yamls or config.target_yaml_count + schedules = split_schedule(schedule, yaml_count) + print_split_summary(schedules, config) + + if args.template_data: + for i, sched in enumerate(schedules): + data = schedule_to_template_data(sched, config) + print(f"\n--- Template data for YAML {i + 1} ---") + print(json.dumps(data, indent=2)) + + if args.yaml_output: + print(f"Generating {len(schedules)} YAML file(s)...") + generate_yamls( + schedules, config, args.yaml_output, + base_name=args.base_name, + source_config=_format_source_path(args.config), + ) + print("Done!") + return 0 + except (ConfigError, SchedulerError, GeneratorError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pod-scheduler/models.py b/scripts/pod-scheduler/models.py new file mode 100644 index 000000000..22ec8503f --- /dev/null +++ b/scripts/pod-scheduler/models.py @@ -0,0 +1,194 @@ +""" +Data models for pod-based crank scheduling. + +A "pod" is a fixed group of machines (SUT + optional load + optional DB) that +always run together. Pods sharing physical machines cannot run simultaneously, +which the scheduler enforces automatically. +""" + +import re +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Dict, List, Optional, Set + + +# Default per-type runtime estimates (minutes) used when a scenario provides +# none of its own. Lives here so models, scheduler, and tests share one source. +DEFAULT_RUNTIMES: Dict["ScenarioType", float] = {} # populated below + +# Pipeline plumbing defaults. Override in JSON metadata.pipeline.* if needed. +DEFAULT_PIPELINE_POOL = "server" +DEFAULT_PIPELINE_CONNECTION = "ASPNET Benchmarks Service Bus" +DEFAULT_PIPELINE_NAMESPACE = "aspnetbenchmarks" + +# AzDO job identifier rule. Letters/digits/underscore, must not start with a +# digit, no longer than 100 characters. +JOB_ID_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,99}$") + + +class ScenarioType(IntEnum): + """Number of machine roles required for a scenario.""" + SINGLE = 1 # SUT only + DUAL = 2 # SUT + Load + TRIPLE = 3 # SUT + Load + DB + + +DEFAULT_RUNTIMES.update({ + ScenarioType.SINGLE: 30.0, + ScenarioType.DUAL: 45.0, + ScenarioType.TRIPLE: 60.0, +}) + + +def sanitize_job_id(raw: str) -> str: + """Sanitize an arbitrary string into a valid AzDO job identifier. + + Replaces any character that isn't [A-Za-z0-9_] with '_', collapses runs of + underscores, prefixes a leading digit with '_', and truncates at 100 + characters. The result is guaranteed to match ``JOB_ID_RE``. + """ + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", raw) + cleaned = re.sub(r"_+", "_", cleaned).strip("_") or "_" + if cleaned[0].isdigit(): + cleaned = "_" + cleaned + cleaned = cleaned[:100] + if not JOB_ID_RE.match(cleaned): + # Should be unreachable, but assert keeps callers honest. + raise ValueError(f"Could not sanitize {raw!r} into a valid job id") + return cleaned + + +@dataclass +class Pod: + """A fixed group of machines that run scenarios together.""" + name: str + # Physical machine names for each role + sut: str + load: Optional[str] = None + db: Optional[str] = None + # Crank profile names for each role + sut_profile: str = "" + load_profile: Optional[str] = None + db_profile: Optional[str] = None + + def machines_for_type(self, scenario_type: ScenarioType) -> Set[str]: + """Return the set of physical machines used for a given scenario type.""" + machines = {self.sut} + if scenario_type >= ScenarioType.DUAL and self.load: + machines.add(self.load) + if scenario_type >= ScenarioType.TRIPLE and self.db: + machines.add(self.db) + return machines + + def profiles_for_type(self, scenario_type: ScenarioType) -> List[str]: + """Return the ordered list of profiles for a given scenario type.""" + profiles = [self.sut_profile] + if scenario_type >= ScenarioType.DUAL and self.load_profile: + profiles.append(self.load_profile) + if scenario_type >= ScenarioType.TRIPLE and self.db_profile: + profiles.append(self.db_profile) + return profiles + + def validate(self, scenario_type: ScenarioType) -> Optional[str]: + """Check if this pod can run the given scenario type. Returns error or None.""" + if scenario_type >= ScenarioType.DUAL and not self.load: + return f"Pod '{self.name}' has no load machine for DUAL/TRIPLE scenario" + if scenario_type >= ScenarioType.TRIPLE and not self.db: + return f"Pod '{self.name}' has no db machine for TRIPLE scenario" + return None + + +@dataclass +class Scenario: + """A benchmark scenario that runs on one or more pods.""" + name: str + template: str + type: ScenarioType + pods: List[str] + estimated_runtime: float = 45.0 + # Optional explicit timeout (minutes) for the generated AzDO job. When + # None, the generator derives one from estimated_runtime. + timeout: Optional[int] = None + + +@dataclass +class Run: + """A single execution: one scenario on one pod.""" + scenario: Scenario + pod: Pod + estimated_runtime: float + + @property + def name(self) -> str: + return f"{self.scenario.name} {self.pod.name}" + + @property + def job_name(self) -> str: + """Sanitized identifier suitable for AzDO ``- job:`` use.""" + return sanitize_job_id(self.name) + + @property + def machines_used(self) -> Set[str]: + return self.pod.machines_for_type(self.scenario.type) + + @property + def profiles(self) -> List[str]: + return self.pod.profiles_for_type(self.scenario.type) + + +@dataclass +class Stage: + """A group of runs that execute in parallel (no machine conflicts).""" + runs: List[Run] = field(default_factory=list) + + @property + def machines_in_use(self) -> Set[str]: + result: Set[str] = set() + for run in self.runs: + result |= run.machines_used + return result + + @property + def duration(self) -> float: + return max((r.estimated_runtime for r in self.runs), default=0) + + def can_add(self, run: Run, queue_count: int) -> bool: + """True if the run fits without machine conflicts or queue overflow.""" + if len(self.runs) >= queue_count: + return False + return run.machines_used.isdisjoint(self.machines_in_use) + + +@dataclass +class Schedule: + """Complete schedule: ordered list of stages.""" + stages: List[Stage] = field(default_factory=list) + + @property + def total_duration(self) -> float: + return sum(s.duration for s in self.stages) + + @property + def total_runs(self) -> int: + return sum(len(s.runs) for s in self.stages) + + +@dataclass +class PipelineSettings: + """Pipeline-level plumbing values rendered into the YAML.""" + pool: str = DEFAULT_PIPELINE_POOL + service_bus_connection: str = DEFAULT_PIPELINE_CONNECTION + service_bus_namespace: str = DEFAULT_PIPELINE_NAMESPACE + + +@dataclass +class ScheduleConfig: + """Top-level configuration loaded from JSON.""" + name: str + schedule: str + queues: List[str] + target_yaml_count: int + schedule_offset_hours: int + pods: Dict[str, Pod] + scenarios: List[Scenario] + pipeline: PipelineSettings = field(default_factory=PipelineSettings) diff --git a/scripts/pod-scheduler/scheduler.py b/scripts/pod-scheduler/scheduler.py new file mode 100644 index 000000000..c86284bea --- /dev/null +++ b/scripts/pod-scheduler/scheduler.py @@ -0,0 +1,121 @@ +""" +Core scheduling algorithm for pod-based crank scheduling. + +Uses a greedy longest-job-first approach with machine-collision avoidance. +Since pods define fixed machine groupings, the scheduler only needs to +ensure no physical machine is used twice in the same stage. + +Output is deterministic: identical input JSON always produces identical +schedules, so generated YAML files diff cleanly across regenerations. +""" + +from typing import List, Tuple + +from models import ( + DEFAULT_RUNTIMES, + Run, + Schedule, + ScheduleConfig, + Stage, +) + + +class SchedulerError(ValueError): + """Raised when the scheduler refuses to build a schedule.""" + + +def expand_runs(config: ScheduleConfig, strict: bool = True) -> List[Run]: + """Expand scenarios x pods into individual runs. + + With ``strict=True`` (the default) an unknown pod or a pod that cannot + satisfy a scenario's type raises :class:`SchedulerError`. With + ``strict=False`` the offending entry is skipped and a warning is printed. + """ + runs: List[Run] = [] + for scenario in config.scenarios: + for pod_name in scenario.pods: + pod = config.pods.get(pod_name) + if pod is None: + msg = ( + f"Scenario '{scenario.name}' references unknown pod " + f"'{pod_name}'" + ) + if strict: + raise SchedulerError(msg) + print(f" WARNING: {msg}, skipping") + continue + error = pod.validate(scenario.type) + if error: + msg = f"{error} for scenario '{scenario.name}'" + if strict: + raise SchedulerError(msg) + print(f" WARNING: {msg}, skipping") + continue + runtime = scenario.estimated_runtime + if runtime <= 0: + runtime = DEFAULT_RUNTIMES.get(scenario.type, 45.0) + runs.append(Run( + scenario=scenario, + pod=pod, + estimated_runtime=runtime, + )) + return runs + + +def create_schedule(config: ScheduleConfig, strict: bool = True) -> Schedule: + """Create a schedule by greedy longest-job-first packing. + + 1. Expand all scenario x pod combinations into runs. + 2. Sort by runtime descending (longest-job-first heuristic). + 3. Greedily pack runs into stages, checking machine collisions. + + Sort key includes the run name as a tie-breaker so the result is stable. + """ + runs = expand_runs(config, strict=strict) + queue_count = len(config.queues) + if queue_count == 0: + raise SchedulerError( + "Cannot schedule with zero queues. Configure metadata.queues." + ) + + runs.sort(key=lambda r: (-r.estimated_runtime, r.name)) + + schedule = Schedule() + for run in runs: + for stage in schedule.stages: + if stage.can_add(run, queue_count): + stage.runs.append(run) + break + else: + schedule.stages.append(Stage(runs=[run])) + + return schedule + + +def split_schedule(schedule: Schedule, target_count: int) -> List[Schedule]: + """Split a schedule into multiple sub-schedules using bin-packing. + + Stages are packed longest-first into the lightest bin to balance total + runtime, then each bin's stages are restored to their original ordering + so the generated YAML files preserve scenario sequence. + """ + if target_count <= 1: + return [schedule] + + indexed: List[Tuple[int, Stage]] = list(enumerate(schedule.stages)) + indexed.sort(key=lambda pair: -pair[1].duration) + + bins: List[List[Tuple[int, Stage]]] = [[] for _ in range(target_count)] + bin_durations = [0.0] * target_count + for original_index, stage in indexed: + target = min(range(target_count), key=lambda i: bin_durations[i]) + bins[target].append((original_index, stage)) + bin_durations[target] += stage.duration + + result: List[Schedule] = [] + for entries in bins: + if not entries: + continue + entries.sort(key=lambda pair: pair[0]) + result.append(Schedule(stages=[stage for _, stage in entries])) + return result diff --git a/scripts/pod-scheduler/tests/__init__.py b/scripts/pod-scheduler/tests/__init__.py new file mode 100644 index 000000000..54b55fbcc --- /dev/null +++ b/scripts/pod-scheduler/tests/__init__.py @@ -0,0 +1,14 @@ +"""Pod-scheduler tests. + +Each test module adds the parent ``scripts/pod-scheduler`` directory to +``sys.path`` so it can ``import models`` etc. without requiring the package +to be installed. +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_PARENT = os.path.dirname(_HERE) +if _PARENT not in sys.path: + sys.path.insert(0, _PARENT) \ No newline at end of file diff --git a/scripts/pod-scheduler/tests/test_config_loader.py b/scripts/pod-scheduler/tests/test_config_loader.py new file mode 100644 index 000000000..9dd698939 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_config_loader.py @@ -0,0 +1,117 @@ +import json +import os +import tempfile +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from config_loader import ConfigError, load_config + + +_BASE = { + "metadata": { + "name": "t", + "schedule": "0 3/12 * * *", + "queues": ["a", "b"], + }, + "pods": [ + { + "name": "p1", + "machines": {"sut": "m1"}, + "profiles": {"sut": "m1-app"}, + } + ], + "scenarios": [ + { + "name": "S", + "template": "s.yml", + "type": 1, + "pods": ["p1"], + } + ], +} + + +def _write(tmp, payload): + path = os.path.join(tmp, "cfg.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f) + return path + + +class TestLoadConfig(unittest.TestCase): + def test_happy_path(self): + with tempfile.TemporaryDirectory() as tmp: + path = _write(tmp, _BASE) + cfg = load_config(path) + self.assertIn("p1", cfg.pods) + self.assertEqual(len(cfg.scenarios), 1) + self.assertEqual(cfg.pipeline.pool, "server") + + def test_unsupported_cron_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["schedule"] = "0 * * * *" + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_empty_queues_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["queues"] = [] + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_duplicate_pods_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["pods"].append(dict(payload["pods"][0])) + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_duplicate_pods_in_scenario_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["pods"] = ["p1", "p1"] + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_pipeline_settings_loaded(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["pipeline"] = { + "pool": "custompool", + "service_bus_namespace": "myns", + } + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.pipeline.pool, "custompool") + self.assertEqual(cfg.pipeline.service_bus_namespace, "myns") + self.assertEqual( + cfg.pipeline.service_bus_connection, + "ASPNET Benchmarks Service Bus", + ) + + def test_optional_timeout_loaded(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["timeout"] = 199 + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.scenarios[0].timeout, 199) + + def test_negative_timeout_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["timeout"] = 0 + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_generator.py b/scripts/pod-scheduler/tests/test_generator.py new file mode 100644 index 000000000..91a59f617 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_generator.py @@ -0,0 +1,81 @@ +import os +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from generator import GeneratorError, _job_timeout, _offset_cron +from main import _format_source_path +from models import Pod, Run, Scenario, ScenarioType + + +class TestOffsetCron(unittest.TestCase): + def test_offset_h_field(self): + self.assertEqual(_offset_cron("0 3 * * *", 6), "0 9 * * *") + + def test_offset_h_slash_n(self): + self.assertEqual(_offset_cron("0 3/12 * * *", 6), "0 9/12 * * *") + + def test_offset_wraps_at_24(self): + self.assertEqual(_offset_cron("0 22 * * *", 6), "0 4 * * *") + + def test_offset_zero_is_identity(self): + self.assertEqual(_offset_cron("0 3/12 * * *", 0), "0 3/12 * * *") + + def test_unsupported_hour_field_raises(self): + for cron in [ + "0 * * * *", + "0 0,12 * * *", + "0 1-5 * * *", + "0 */6 * * *", + ]: + with self.assertRaises(GeneratorError, msg=cron): + _offset_cron(cron, 6) + + def test_wrong_field_count_raises(self): + with self.assertRaises(GeneratorError): + _offset_cron("0 3 * *", 6) + + +class TestJobTimeout(unittest.TestCase): + def _run(self, runtime, timeout=None): + scenario = Scenario( + name="s", template="s.yml", type=ScenarioType.SINGLE, + pods=["p"], estimated_runtime=runtime, timeout=timeout, + ) + pod = Pod(name="p", sut="sut", sut_profile="sut") + return Run(scenario=scenario, pod=pod, estimated_runtime=runtime) + + def test_uses_explicit_timeout(self): + self.assertEqual(_job_timeout(self._run(30, timeout=42)), 42) + + def test_short_runtime_floors_at_120(self): + self.assertEqual(_job_timeout(self._run(5)), 120) + + def test_long_runtime_caps_at_240(self): + self.assertEqual(_job_timeout(self._run(200)), 240) + + def test_mid_runtime_doubles(self): + self.assertEqual(_job_timeout(self._run(90)), 180) + + +class TestFormatSourcePath(unittest.TestCase): + def test_paths_in_repo_become_repo_relative(self): + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..") + ) + candidate = os.path.join(repo_root, "build", "benchmarks_ci_pods.json") + self.assertEqual( + _format_source_path(candidate), + "./build/benchmarks_ci_pods.json", + ) + + def test_outside_repo_falls_back_to_basename(self): + # Use a path that's definitely outside the repo by jumping above root. + far = os.path.abspath(os.sep + "definitely-not-in-repo.json") + self.assertEqual( + _format_source_path(far), "./definitely-not-in-repo.json" + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/scripts/pod-scheduler/tests/test_models.py b/scripts/pod-scheduler/tests/test_models.py new file mode 100644 index 000000000..a0884f957 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_models.py @@ -0,0 +1,106 @@ +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from models import ( + JOB_ID_RE, + Pod, + Run, + Scenario, + ScenarioType, + Stage, + sanitize_job_id, +) + + +class TestSanitizeJobId(unittest.TestCase): + def test_replaces_spaces_and_hyphens(self): + self.assertEqual(sanitize_job_id("Proxies gold-lin"), "Proxies_gold_lin") + + def test_collapses_multiple_separators(self): + self.assertEqual( + sanitize_job_id("Foo --bar..baz"), "Foo_bar_baz" + ) + + def test_prefixes_leading_digit(self): + self.assertEqual(sanitize_job_id("01-stage"), "_01_stage") + + def test_handles_unicode_and_punctuation(self): + result = sanitize_job_id("Frënch (test)/v2") + self.assertRegex(result, JOB_ID_RE) + self.assertNotIn(" ", result) + self.assertNotIn("/", result) + self.assertNotIn("(", result) + + def test_truncates_to_100(self): + long_name = "a" * 250 + result = sanitize_job_id(long_name) + self.assertEqual(len(result), 100) + + def test_result_always_matches_pattern(self): + for sample in ["x.y", "1abc", " hello ", "---", "a/b/c"]: + self.assertRegex(sanitize_job_id(sample), JOB_ID_RE, sample) + + +class TestPodValidation(unittest.TestCase): + def _pod(self, **kwargs): + defaults = dict( + name="p", sut="sut", + sut_profile="sut-app", + ) + defaults.update(kwargs) + return Pod(**defaults) + + def test_single_only_pod_rejects_dual(self): + pod = self._pod() + self.assertIsNone(pod.validate(ScenarioType.SINGLE)) + self.assertIsNotNone(pod.validate(ScenarioType.DUAL)) + self.assertIsNotNone(pod.validate(ScenarioType.TRIPLE)) + + def test_dual_pod_rejects_triple(self): + pod = self._pod(load="load", load_profile="load-load") + self.assertIsNone(pod.validate(ScenarioType.DUAL)) + self.assertIsNotNone(pod.validate(ScenarioType.TRIPLE)) + + def test_triple_pod_accepts_all(self): + pod = self._pod( + load="load", load_profile="load-load", + db="db", db_profile="db-db", + ) + for t in ScenarioType: + self.assertIsNone(pod.validate(t), t) + + +class TestStageCanAdd(unittest.TestCase): + def _run(self, name, sut, load=None, db=None, runtime=10): + scenario = Scenario( + name=name, + template=f"{name}.yml", + type=ScenarioType.TRIPLE if db else ( + ScenarioType.DUAL if load else ScenarioType.SINGLE + ), + pods=["p"], + estimated_runtime=runtime, + ) + pod = Pod( + name="p", sut=sut, load=load, db=db, + sut_profile="sut", load_profile="load", db_profile="db", + ) + return Run(scenario=scenario, pod=pod, estimated_runtime=runtime) + + def test_collision_blocks(self): + stage = Stage(runs=[self._run("a", sut="m1")]) + self.assertFalse(stage.can_add(self._run("b", sut="m1"), 10)) + + def test_no_collision_allows(self): + stage = Stage(runs=[self._run("a", sut="m1")]) + self.assertTrue(stage.can_add(self._run("b", sut="m2"), 10)) + + def test_queue_limit_enforced(self): + stage = Stage(runs=[self._run(f"a{i}", sut=f"m{i}") for i in range(3)]) + self.assertFalse(stage.can_add(self._run("a3", sut="m3"), queue_count=3)) + self.assertTrue(stage.can_add(self._run("a3", sut="m3"), queue_count=4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_scheduler.py b/scripts/pod-scheduler/tests/test_scheduler.py new file mode 100644 index 000000000..0c52c44c6 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_scheduler.py @@ -0,0 +1,160 @@ +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from models import ( + PipelineSettings, + Pod, + Scenario, + ScenarioType, + Schedule, + ScheduleConfig, + Stage, +) +from scheduler import ( + SchedulerError, + create_schedule, + expand_runs, + split_schedule, +) + + +def _config(pods, scenarios, queues=("q1", "q2")): + return ScheduleConfig( + name="t", + schedule="0 3/12 * * *", + queues=list(queues), + target_yaml_count=1, + schedule_offset_hours=6, + pods={p.name: p for p in pods}, + scenarios=list(scenarios), + pipeline=PipelineSettings(), + ) + + +def _pod(name, sut, load=None, db=None): + return Pod( + name=name, sut=sut, load=load, db=db, + sut_profile=f"{sut}-app", + load_profile=f"{load}-load" if load else None, + db_profile=f"{db}-db" if db else None, + ) + + +def _scn(name, type_, pods, runtime=10): + return Scenario( + name=name, template=f"{name}.yml", + type=type_, pods=list(pods), estimated_runtime=runtime, + ) + + +class TestExpandRuns(unittest.TestCase): + def test_strict_raises_on_unknown_pod(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["missing"])], + ) + with self.assertRaises(SchedulerError): + expand_runs(cfg, strict=True) + + def test_lenient_skips_unknown_pod(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["missing", "p1"])], + ) + runs = expand_runs(cfg, strict=False) + self.assertEqual([r.pod.name for r in runs], ["p1"]) + + def test_strict_raises_on_invalid_type(self): + cfg = _config( + pods=[_pod("p1", "m1")], # no load + scenarios=[_scn("S", ScenarioType.DUAL, ["p1"])], + ) + with self.assertRaises(SchedulerError): + expand_runs(cfg, strict=True) + + +class TestCreateSchedule(unittest.TestCase): + def test_collisions_split_into_stages(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=5), + _scn("B", ScenarioType.SINGLE, ["p1"], runtime=10), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 2) + + def test_independent_pods_share_a_stage(self): + cfg = _config( + pods=[_pod("p1", "m1"), _pod("p2", "m2")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"]), + _scn("B", ScenarioType.SINGLE, ["p2"]), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 1) + self.assertEqual(len(sched.stages[0].runs), 2) + + def test_queue_count_limits_stage(self): + cfg = _config( + pods=[_pod(f"p{i}", f"m{i}") for i in range(3)], + scenarios=[ + _scn(f"S{i}", ScenarioType.SINGLE, [f"p{i}"]) for i in range(3) + ], + queues=("q1", "q2"), + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 2) + + def test_zero_queues_raises(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["p1"])], + queues=(), + ) + with self.assertRaises(SchedulerError): + create_schedule(cfg) + + def test_deterministic_output(self): + cfg = _config( + pods=[_pod("p1", "m1"), _pod("p2", "m2")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=10), + _scn("B", ScenarioType.SINGLE, ["p2"], runtime=10), + ], + ) + s1 = create_schedule(cfg) + s2 = create_schedule(cfg) + names_of = lambda s: [ + [r.name for r in stage.runs] for stage in s.stages + ] + self.assertEqual(names_of(s1), names_of(s2)) + + +class TestSplitSchedule(unittest.TestCase): + def test_single_target_returns_input(self): + sched = Schedule(stages=[Stage(runs=[])]) + self.assertEqual(split_schedule(sched, 1), [sched]) + + def test_balances_durations(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=30), + _scn("B", ScenarioType.SINGLE, ["p1"], runtime=20), + _scn("C", ScenarioType.SINGLE, ["p1"], runtime=10), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 3) + bins = split_schedule(sched, 2) + self.assertEqual(len(bins), 2) + durations = sorted(b.total_duration for b in bins) + self.assertEqual(durations, [30, 30]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_snapshots.py b/scripts/pod-scheduler/tests/test_snapshots.py new file mode 100644 index 000000000..849430daf --- /dev/null +++ b/scripts/pod-scheduler/tests/test_snapshots.py @@ -0,0 +1,79 @@ +"""Snapshot tests: regenerate the committed YAML files and compare. + +Run from the ``scripts/pod-scheduler`` directory: + + python -m unittest discover tests +""" + +import os +import tempfile +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from config_loader import load_config +from generator import generate_yamls +from scheduler import create_schedule, split_schedule + + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) +_BUILD = os.path.join(_REPO, "build") + + +CASES = [ + ("benchmarks_ci_pods.json", "benchmarks-ci", + ["benchmarks-ci-01.yml", "benchmarks-ci-02.yml"]), + ("benchmarks_ci_azure_pods.json", "benchmarks-ci-azure", + ["benchmarks-ci-azure.yml"]), + ("benchmarks_ci_cobalt_pods.json", "benchmarks-ci-cobalt", + ["benchmarks-ci-cobalt.yml"]), +] + + +class TestSnapshots(unittest.TestCase): + def test_each_config_produces_committed_yaml(self): + for config_name, base_name, expected_files in CASES: + with self.subTest(config=config_name): + config_path = os.path.join(_BUILD, config_name) + self.assertTrue( + os.path.exists(config_path), f"missing {config_path}" + ) + config = load_config(config_path) + schedule = create_schedule(config) + schedules = split_schedule(schedule, config.target_yaml_count) + + with tempfile.TemporaryDirectory() as tmp: + generate_yamls( + schedules, config, tmp, base_name=base_name, + source_config=f"./build/{config_name}", + ) + for name in expected_files: + generated = os.path.join(tmp, name) + committed = os.path.join(_BUILD, name) + self.assertTrue( + os.path.exists(generated), + f"generator did not produce {generated}", + ) + self.assertTrue( + os.path.exists(committed), + f"committed YAML missing: {committed}", + ) + with open(generated, encoding="utf-8") as f: + gen_text = f.read() + with open(committed, encoding="utf-8") as f: + comm_text = f.read() + self.assertEqual( + gen_text, + comm_text, + f"{name} differs from committed snapshot. " + f"Run from repo root: python " + f"scripts/pod-scheduler/main.py " + f"--config build/{config_name} " + f"--yaml-output build " + f"--base-name {base_name}", + ) + + +if __name__ == "__main__": + unittest.main()