diff --git a/cpp/arcticdb/storage/s3/s3_storage.cpp b/cpp/arcticdb/storage/s3/s3_storage.cpp index ac407653544..adaed5ec70c 100644 --- a/cpp/arcticdb/storage/s3/s3_storage.cpp +++ b/cpp/arcticdb/storage/s3/s3_storage.cpp @@ -215,10 +215,14 @@ void S3Storage::create_s3_client(const S3Settings& conf, const Aws::Auth::AWSCre } else if (creds.GetAWSAccessKeyId() == USE_AWS_CRED_PROVIDERS_TOKEN && creds.GetAWSSecretKey() == USE_AWS_CRED_PROVIDERS_TOKEN) { ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using AWS auth mechanisms"); + auto client_config = get_s3_config_and_set_env_var(conf); + if (!conf.aws_profile().empty()) { + ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using AWS profile {}", conf.aws_profile()); + Aws::Config::ReloadCachedConfigFile(); // config files loaded once in Aws::InitAPI; reload to get latest + client_config.profileName = conf.aws_profile(); + } s3_client_ = std::make_unique( - get_s3_config_and_set_env_var(conf), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - conf.use_virtual_addressing() + client_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, conf.use_virtual_addressing() ); } else { ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using provided auth credentials"); diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/configure-binary-dir.patch b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/configure-binary-dir.patch new file mode 120000 index 00000000000..ac1c8348678 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/configure-binary-dir.patch @@ -0,0 +1 @@ +../../../vcpkg/ports/aws-sdk-cpp/configure-binary-dir.patch \ No newline at end of file diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-refresh.patch b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-refresh.patch deleted file mode 100644 index 7e6f1cf298f..00000000000 --- a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-refresh.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/aws-cpp-sdk-identity-management/source/auth/STSProfileCredentialsProvider.cpp b/src/aws-cpp-sdk-identity-management/source/auth/STSProfileCredentialsProvider.cpp -index fd82b678fba..9fd0537e217 100644 ---- a/src/aws-cpp-sdk-identity-management/source/auth/STSProfileCredentialsProvider.cpp -+++ b/src/aws-cpp-sdk-identity-management/source/auth/STSProfileCredentialsProvider.cpp -@@ -45,13 +45,13 @@ AWSCredentials STSProfileCredentialsProvider::GetAWSCredentials() - void STSProfileCredentialsProvider::RefreshIfExpired() - { - Utils::Threading::ReaderLockGuard guard(m_reloadLock); -- if (!IsTimeToRefresh(static_cast(m_reloadFrequency.count())) || !m_credentials.IsExpiredOrEmpty()) -+ if (!IsTimeToRefresh(static_cast(m_reloadFrequency.count())) && !m_credentials.IsExpiredOrEmpty()) - { - return; - } - - guard.UpgradeToWriterLock(); -- if (!IsTimeToRefresh(static_cast(m_reloadFrequency.count())) || !m_credentials.IsExpiredOrEmpty()) // double-checked lock to avoid refreshing twice -+ if (!IsTimeToRefresh(static_cast(m_reloadFrequency.count())) && !m_credentials.IsExpiredOrEmpty()) // double-checked lock to avoid refreshing twice - { - return; - } diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-win-https-conn.patch b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-win-https-conn.patch deleted file mode 100644 index b282d28abab..00000000000 --- a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/fix-win-https-conn.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -index 9c82caa6811..2d91cc0da32 100644 ---- a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -+++ b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -@@ -533,7 +533,7 @@ void* WinHttpSyncHttpClient::OpenRequest(const std::shared_ptr& req - { - LPCWSTR accept[2] = { nullptr, nullptr }; - -- DWORD requestFlags = request->GetUri().GetScheme() == Scheme::HTTPS && m_verifySSL ? WINHTTP_FLAG_SECURE : 0; -+ DWORD requestFlags = request->GetUri().GetScheme() == Scheme::HTTPS ? WINHTTP_FLAG_SECURE : 0; - if (m_usingProxy) { - // Avoid force adding "Cache-Control: no-cache" header. - requestFlags |= WINHTTP_FLAG_REFRESH; diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/portfile.cmake b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/portfile.cmake index 1f6543d389d..fd5f624935f 100644 --- a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/portfile.cmake +++ b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/portfile.cmake @@ -4,14 +4,13 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO aws/aws-sdk-cpp REF "${VERSION}" - SHA512 f81b0afd9c3bb6e8181c6edc04de9b83af8a17e5bdf993e68fc00abc07980fadbe9e6b98635632cb96ed62c9b2753771f9cf6d91088d6ab42f409eec5d136faa + SHA512 976cf53e9b15d555656474de8d58429b17bd8083ace1003f2f056fdc21732f6fd0a309ff6b3498b5f3675d011e447575387f1e00a05a0ae3487d8f7ad26e5bba PATCHES fix-aws-root.patch lock-curl-http-and-tls-settings.patch fix_find_curl.patch find-dependency.patch - fix-refresh.patch - fix-win-https-conn.patch + configure-binary-dir.patch # https://github.com/aws/aws-sdk-cpp/pull/3459 ) string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.in.json b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.in.json index e62791117ad..dd584a01ccd 100644 --- a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.in.json +++ b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.in.json @@ -1,6 +1,6 @@ { "name": "aws-sdk-cpp", - "version": "1.11.474", + "version": "1.11.820", "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", "license": "Apache-2.0", diff --git a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.json b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.json index 2ca1eff1d6e..7eebf88bc06 100644 --- a/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.json +++ b/cpp/third_party/vcpkg_overlays/aws-sdk-cpp/vcpkg.json @@ -1,7 +1,7 @@ { "$note": "Automatically generated by generateFeatures.ps1 from vcpkg.in.json, do not edit manually", "name": "aws-sdk-cpp", - "version": "1.11.474", + "version": "1.11.820", "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", "license": "Apache-2.0", @@ -60,6 +60,9 @@ "acm-pca": { "description": "C++ SDK for the AWS acm-pca service" }, + "aiops": { + "description": "C++ SDK for the AWS aiops service" + }, "amp": { "description": "C++ SDK for the AWS amp service" }, @@ -120,8 +123,8 @@ "appsync": { "description": "C++ SDK for the AWS appsync service" }, - "apptest": { - "description": "C++ SDK for the AWS apptest service" + "arc-region-switch": { + "description": "C++ SDK for the AWS arc-region-switch service" }, "arc-zonal-shift": { "description": "C++ SDK for the AWS arc-zonal-shift service" @@ -162,12 +165,18 @@ "batch": { "description": "C++ SDK for the AWS batch service" }, + "bcm-dashboards": { + "description": "C++ SDK for the AWS bcm-dashboards service" + }, "bcm-data-exports": { "description": "C++ SDK for the AWS bcm-data-exports service" }, "bcm-pricing-calculator": { "description": "C++ SDK for the AWS bcm-pricing-calculator service" }, + "bcm-recommended-actions": { + "description": "C++ SDK for the AWS bcm-recommended-actions service" + }, "bedrock": { "description": "C++ SDK for the AWS bedrock service" }, @@ -177,6 +186,12 @@ "bedrock-agent-runtime": { "description": "C++ SDK for the AWS bedrock-agent-runtime service" }, + "bedrock-agentcore": { + "description": "C++ SDK for the AWS bedrock-agentcore service" + }, + "bedrock-agentcore-control": { + "description": "C++ SDK for the AWS bedrock-agentcore-control service" + }, "bedrock-data-automation": { "description": "C++ SDK for the AWS bedrock-data-automation service" }, @@ -318,6 +333,9 @@ "compute-optimizer": { "description": "C++ SDK for the AWS compute-optimizer service" }, + "compute-optimizer-automation": { + "description": "C++ SDK for the AWS compute-optimizer-automation service" + }, "config": { "description": "C++ SDK for the AWS config service" }, @@ -336,6 +354,9 @@ "connectcases": { "description": "C++ SDK for the AWS connectcases service" }, + "connecthealth": { + "description": "C++ SDK for the AWS connecthealth service" + }, "connectparticipant": { "description": "C++ SDK for the AWS connectparticipant service" }, @@ -381,6 +402,9 @@ "devicefarm": { "description": "C++ SDK for the AWS devicefarm service" }, + "devops-agent": { + "description": "C++ SDK for the AWS devops-agent service" + }, "devops-guru": { "description": "C++ SDK for the AWS devops-guru service" }, @@ -444,9 +468,6 @@ "eks-auth": { "description": "C++ SDK for the AWS eks-auth service" }, - "elastic-inference": { - "description": "C++ SDK for the AWS elastic-inference service" - }, "elasticache": { "description": "C++ SDK for the AWS elasticache service" }, @@ -465,8 +486,8 @@ "elasticmapreduce": { "description": "C++ SDK for the AWS elasticmapreduce service" }, - "elastictranscoder": { - "description": "C++ SDK for the AWS elastictranscoder service" + "elementalinference": { + "description": "C++ SDK for the AWS elementalinference service" }, "email": { "description": "C++ SDK for the AWS email service" @@ -489,8 +510,8 @@ "events": { "description": "C++ SDK for the AWS events service" }, - "evidently": { - "description": "C++ SDK for the AWS evidently service" + "evs": { + "description": "C++ SDK for the AWS evs service" }, "finspace": { "description": "C++ SDK for the AWS finspace service" @@ -525,6 +546,9 @@ "gamelift": { "description": "C++ SDK for the AWS gamelift service" }, + "gameliftstreams": { + "description": "C++ SDK for the AWS gameliftstreams service" + }, "geo-maps": { "description": "C++ SDK for the AWS geo-maps service" }, @@ -598,6 +622,9 @@ "inspector2": { "description": "C++ SDK for the AWS inspector2 service" }, + "interconnect": { + "description": "C++ SDK for the AWS interconnect service" + }, "internetmonitor": { "description": "C++ SDK for the AWS internetmonitor service" }, @@ -613,14 +640,8 @@ "iot-jobs-data": { "description": "C++ SDK for the AWS iot-jobs-data service" }, - "iot1click-devices": { - "description": "C++ SDK for the AWS iot1click-devices service" - }, - "iot1click-projects": { - "description": "C++ SDK for the AWS iot1click-projects service" - }, - "iotanalytics": { - "description": "C++ SDK for the AWS iotanalytics service" + "iot-managed-integrations": { + "description": "C++ SDK for the AWS iot-managed-integrations service" }, "iotdeviceadvisor": { "description": "C++ SDK for the AWS iotdeviceadvisor service" @@ -631,9 +652,6 @@ "iotevents-data": { "description": "C++ SDK for the AWS iotevents-data service" }, - "iotfleethub": { - "description": "C++ SDK for the AWS iotfleethub service" - }, "iotfleetwise": { "description": "C++ SDK for the AWS iotfleetwise service" }, @@ -676,6 +694,9 @@ "keyspaces": { "description": "C++ SDK for the AWS keyspaces service" }, + "keyspacesstreams": { + "description": "C++ SDK for the AWS keyspacesstreams service" + }, "kinesis": { "description": "C++ SDK for the AWS kinesis service" }, @@ -745,12 +766,6 @@ "lookoutequipment": { "description": "C++ SDK for the AWS lookoutequipment service" }, - "lookoutmetrics": { - "description": "C++ SDK for the AWS lookoutmetrics service" - }, - "lookoutvision": { - "description": "C++ SDK for the AWS lookoutvision service" - }, "m2": { "description": "C++ SDK for the AWS m2 service" }, @@ -778,6 +793,9 @@ "marketplace-deployment": { "description": "C++ SDK for the AWS marketplace-deployment service" }, + "marketplace-discovery": { + "description": "C++ SDK for the AWS marketplace-discovery service" + }, "marketplace-entitlement": { "description": "C++ SDK for the AWS marketplace-entitlement service" }, @@ -841,6 +859,9 @@ "monitoring": { "description": "C++ SDK for the AWS monitoring service" }, + "mpa": { + "description": "C++ SDK for the AWS mpa service" + }, "mq": { "description": "C++ SDK for the AWS mq service" }, @@ -850,6 +871,9 @@ "mwaa": { "description": "C++ SDK for the AWS mwaa service" }, + "mwaa-serverless": { + "description": "C++ SDK for the AWS mwaa-serverless service" + }, "neptune": { "description": "C++ SDK for the AWS neptune service" }, @@ -877,12 +901,18 @@ "notificationscontacts": { "description": "C++ SDK for the AWS notificationscontacts service" }, + "nova-act": { + "description": "C++ SDK for the AWS nova-act service" + }, "oam": { "description": "C++ SDK for the AWS oam service" }, "observabilityadmin": { "description": "C++ SDK for the AWS observabilityadmin service" }, + "odb": { + "description": "C++ SDK for the AWS odb service" + }, "omics": { "description": "C++ SDK for the AWS omics service" }, @@ -892,12 +922,6 @@ "opensearchserverless": { "description": "C++ SDK for the AWS opensearchserverless service" }, - "opsworks": { - "description": "C++ SDK for the AWS opsworks service" - }, - "opsworkscm": { - "description": "C++ SDK for the AWS opsworkscm service" - }, "organizations": { "description": "C++ SDK for the AWS organizations service" }, @@ -910,6 +934,15 @@ "panorama": { "description": "C++ SDK for the AWS panorama service" }, + "partnercentral-account": { + "description": "C++ SDK for the AWS partnercentral-account service" + }, + "partnercentral-benefits": { + "description": "C++ SDK for the AWS partnercentral-benefits service" + }, + "partnercentral-channel": { + "description": "C++ SDK for the AWS partnercentral-channel service" + }, "partnercentral-selling": { "description": "C++ SDK for the AWS partnercentral-selling service" }, @@ -958,9 +991,6 @@ "pricing": { "description": "C++ SDK for the AWS pricing service" }, - "privatenetworks": { - "description": "C++ SDK for the AWS privatenetworks service" - }, "proton": { "description": "C++ SDK for the AWS proton service" }, @@ -973,12 +1003,6 @@ "qconnect": { "description": "C++ SDK for the AWS qconnect service" }, - "qldb": { - "description": "C++ SDK for the AWS qldb service" - }, - "qldb-session": { - "description": "C++ SDK for the AWS qldb-session service" - }, "queues": { "description": "C++ SDK for the AWS queues service", "dependencies": [ @@ -1024,6 +1048,9 @@ "resiliencehub": { "description": "C++ SDK for the AWS resiliencehub service" }, + "resiliencehubv2": { + "description": "C++ SDK for the AWS resiliencehubv2 service" + }, "resource-explorer-2": { "description": "C++ SDK for the AWS resource-explorer-2 service" }, @@ -1033,9 +1060,6 @@ "resourcegroupstaggingapi": { "description": "C++ SDK for the AWS resourcegroupstaggingapi service" }, - "robomaker": { - "description": "C++ SDK for the AWS robomaker service" - }, "rolesanywhere": { "description": "C++ SDK for the AWS rolesanywhere service" }, @@ -1054,12 +1078,18 @@ "route53domains": { "description": "C++ SDK for the AWS route53domains service" }, + "route53globalresolver": { + "description": "C++ SDK for the AWS route53globalresolver service" + }, "route53profiles": { "description": "C++ SDK for the AWS route53profiles service" }, "route53resolver": { "description": "C++ SDK for the AWS route53resolver service" }, + "rtbfabric": { + "description": "C++ SDK for the AWS rtbfabric service" + }, "rum": { "description": "C++ SDK for the AWS rum service" }, @@ -1085,12 +1115,18 @@ "s3control": { "description": "C++ SDK for the AWS s3control service" }, + "s3files": { + "description": "C++ SDK for the AWS s3files service" + }, "s3outposts": { "description": "C++ SDK for the AWS s3outposts service" }, "s3tables": { "description": "C++ SDK for the AWS s3tables service" }, + "s3vectors": { + "description": "C++ SDK for the AWS s3vectors service" + }, "sagemaker": { "description": "C++ SDK for the AWS sagemaker service" }, @@ -1112,6 +1148,12 @@ "sagemaker-runtime": { "description": "C++ SDK for the AWS sagemaker-runtime service" }, + "sagemaker-runtime-http2": { + "description": "C++ SDK for the AWS sagemaker-runtime-http2 service" + }, + "sagemakerjobruntime": { + "description": "C++ SDK for the AWS sagemakerjobruntime service" + }, "savingsplans": { "description": "C++ SDK for the AWS savingsplans service" }, @@ -1130,6 +1172,9 @@ "security-ir": { "description": "C++ SDK for the AWS security-ir service" }, + "securityagent": { + "description": "C++ SDK for the AWS securityagent service" + }, "securityhub": { "description": "C++ SDK for the AWS securityhub service" }, @@ -1160,12 +1205,18 @@ "signer": { "description": "C++ SDK for the AWS signer service" }, + "signer-data": { + "description": "C++ SDK for the AWS signer-data service" + }, + "signin": { + "description": "C++ SDK for the AWS signin service" + }, + "simpledbv2": { + "description": "C++ SDK for the AWS simpledbv2 service" + }, "simspaceweaver": { "description": "C++ SDK for the AWS simspaceweaver service" }, - "sms": { - "description": "C++ SDK for the AWS sms service" - }, "sms-voice": { "description": "C++ SDK for the AWS sms-voice service" }, @@ -1190,6 +1241,9 @@ "ssm-contacts": { "description": "C++ SDK for the AWS ssm-contacts service" }, + "ssm-guiconnect": { + "description": "C++ SDK for the AWS ssm-guiconnect service" + }, "ssm-incidents": { "description": "C++ SDK for the AWS ssm-incidents service" }, @@ -1226,6 +1280,9 @@ "support-app": { "description": "C++ SDK for the AWS support-app service" }, + "sustainability": { + "description": "C++ SDK for the AWS sustainability service" + }, "swf": { "description": "C++ SDK for the AWS swf service" }, @@ -1286,6 +1343,9 @@ "trustedadvisor": { "description": "C++ SDK for the AWS trustedadvisor service" }, + "uxc": { + "description": "C++ SDK for the AWS uxc service" + }, "verifiedpermissions": { "description": "C++ SDK for the AWS verifiedpermissions service" }, @@ -1307,15 +1367,15 @@ "wellarchitected": { "description": "C++ SDK for the AWS wellarchitected service" }, + "wickr": { + "description": "C++ SDK for the AWS wickr service" + }, "wisdom": { "description": "C++ SDK for the AWS wisdom service" }, "workdocs": { "description": "C++ SDK for the AWS workdocs service" }, - "worklink": { - "description": "C++ SDK for the AWS worklink service" - }, "workmail": { "description": "C++ SDK for the AWS workmail service" }, @@ -1325,6 +1385,9 @@ "workspaces": { "description": "C++ SDK for the AWS workspaces service" }, + "workspaces-instances": { + "description": "C++ SDK for the AWS workspaces-instances service" + }, "workspaces-thin-client": { "description": "C++ SDK for the AWS workspaces-thin-client service" }, diff --git a/cpp/vcpkg b/cpp/vcpkg index bb14c5ab24c..cea592f4772 160000 --- a/cpp/vcpkg +++ b/cpp/vcpkg @@ -1 +1 @@ -Subproject commit bb14c5ab24c8d4de66092ad655d45f17a38aa1d5 +Subproject commit cea592f4772491abdb7c483387a59ea89889f4be diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index f3b3bf5f2f7..0308cc7d03e 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -86,18 +86,18 @@ { "name": "openssl", "version-string": "3.3.0" }, { "name": "arcticdb-sparrow", "version": "2.4.0" }, { "name": "arrow", "version": "21.0.0#2" }, - { "name": "aws-sdk-cpp", "version": "1.11.474", "$note": "Update overlay json to upgrade; Upgrade to >=1.11.486 blocked by default integrity change" }, - { "name": "aws-crt-cpp", "version": "0.29.7" }, - { "name": "aws-c-mqtt", "version": "0.11.0" }, - { "name": "aws-c-s3", "version": "0.6.6" }, - { "name": "aws-c-io", "version": "0.14.18" }, - { "name": "aws-c-common", "version": "0.9.28" }, - { "name": "aws-c-auth", "version": "0.7.31" }, - { "name": "aws-c-cal", "version": "0.7.4" }, - { "name": "aws-c-http", "version": "0.8.10" }, - { "name": "aws-c-sdkutils", "version": "0.1.19" }, - { "name": "aws-c-event-stream", "version": "0.4.3" }, - { "name": "aws-checksums", "version": "0.1.20" }, + { "name": "aws-sdk-cpp", "version": "1.11.820", "$note": "Update overlay json to upgrade." }, + { "name": "aws-crt-cpp", "version": "0.39.1" }, + { "name": "aws-c-mqtt", "version": "0.15.2" }, + { "name": "aws-c-s3", "version": "0.12.4" }, + { "name": "aws-c-io", "version": "0.26.3" }, + { "name": "aws-c-common", "version": "0.13.1" }, + { "name": "aws-c-auth", "version": "0.10.3" }, + { "name": "aws-c-cal", "version": "0.9.14" }, + { "name": "aws-c-http", "version": "0.11.0" }, + { "name": "aws-c-sdkutils", "version": "0.2.4" }, + { "name": "aws-c-event-stream", "version": "0.7.1" }, + { "name": "aws-checksums", "version": "0.2.10" }, { "name": "azure-core-cpp", "version": "1.12.0" }, { "name": "azure-identity-cpp", "version": "1.6.0" }, { "name": "benchmark", "version": "1.9.0" }, @@ -116,8 +116,9 @@ { "name": "prometheus-cpp", "version": "1.1.0" }, { "name": "protobuf", "version": "3.21.8" }, { "name": "rapidcheck", "version": "2023-12-14" }, - { "name": "s2n", "version": "1.3.5" }, + { "name": "s2n", "version": "1.7.3" }, { "name": "spdlog", "version": "1.13.0" }, + { "name": "thrift", "version": "0.22.0#2" }, { "name": "xxhash", "version": "0.8.2" }, { "name": "zlib", "version": "1.3.1" }, { "name": "zstd", "version": "1.5.2" } @@ -126,5 +127,5 @@ "overlay-ports": ["third_party/vcpkg_overlays"] }, "$note on builtin-baseline": "Remember to regenerate third_party/vcpkg_overlays", - "builtin-baseline": "bb14c5ab24c8d4de66092ad655d45f17a38aa1d5" + "builtin-baseline": "cea592f4772491abdb7c483387a59ea89889f4be" } diff --git a/docs/mkdocs/docs/api/arctic_uri.md b/docs/mkdocs/docs/api/arctic_uri.md index f143dcddf21..335b76ebd65 100644 --- a/docs/mkdocs/docs/api/arctic_uri.md +++ b/docs/mkdocs/docs/api/arctic_uri.md @@ -23,7 +23,7 @@ Available options for S3: | secret | S3 secret access key | | path_prefix | Path within S3 bucket to use for data storage | | aws_auth | AWS authentication method. If setting is `default` (or `true` for backward compatibility), authentication to endpoint will be computed via [AWS default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credproviders.html). If setting is `sts`, AWS Security Token Service (STS) will be the authentication method used. If no options are provided AWS authentication will not be used and you should specify access and secret in the URI. More info about `sts` is provided [here](https://docs.arcticdb.io/latest/aws/#aws-security-token-service-sts-setup) | -| aws_profile | Only when `aws_auth` is set to be `sts`. AWS profile to be used with AWS Security Token Service (STS). More info about `sts` is provided [here](https://docs.arcticdb.io/latest/aws/#aws-security-token-service-sts-setup) | +| aws_profile | Named AWS profile (from the shared config/credentials files) to authenticate with. Requires `aws_auth` to be set. With `aws_auth=default` the profile is resolved by the default credentials provider chain; with `aws_auth=sts` it is used by AWS Security Token Service (STS). More info about `sts` is provided [here](https://docs.arcticdb.io/latest/aws/#aws-security-token-service-sts-setup) | Note: When connecting to AWS, `region` can be automatically deduced from the endpoint if the given endpoint specifies the region and `region` is not set. diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index caa2c0ed355..b84afee3bea 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -106,6 +106,8 @@ def __init__( self.arctic_uri += f"&aws_profile={factory.aws_profile}" else: self.arctic_uri += "aws_auth=default" + if factory.aws_profile: + self.arctic_uri += f"&aws_profile={factory.aws_profile}" if port: self.arctic_uri += f"&port={port}" if factory.default_prefix: @@ -592,6 +594,39 @@ def real_s3_sts_write_local_credentials(factory: BaseS3StorageFixtureFactory, co config_file.write(aws_credentials) +def real_s3_default_profile_from_environment_variables( + profile_name: str, config_file_path: str +) -> BaseS3StorageFixtureFactory: + """Factory for testing aws_profile with the default credentials provider chain (non-STS): writes a shared config + file with the real S3 static credentials under the named profile, and configures the fixture to authenticate via + aws_auth=default + that profile.""" + additional_suffix = f"{random.randint(0, 999)}_{datetime.utcnow().strftime('%Y-%m-%dT%H_%M_%S_%f')}" + out = real_s3_from_environment_variables(False, NativeVariantStorage(), additional_suffix) + aws_config = f""" +[profile {profile_name}] +aws_access_key_id = {out.default_key.id} +aws_secret_access_key = {out.default_key.secret} +region = {out.region} +""" + config_dir = os.path.dirname(config_file_path) + os.makedirs(config_dir, exist_ok=True) + with open(config_file_path, "w") as config_file: + config_file.write(aws_config) + + out.native_config = NativeVariantStorage( + NativeS3Settings( + aws_auth=AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN, + aws_profile=profile_name, + use_internal_client_wrapper_for_testing=False, + ) + ) + out.aws_auth = AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN + out.aws_profile = profile_name + # Reset to ensure the client can't fall back to default key+secret auth and must resolve the profile + out.default_key = Key(id="", secret="", user_name="unknown user") + return out + + def real_s3_sts_resources_ready(factory: BaseS3StorageFixtureFactory): sts_client = boto3.client( "sts", aws_access_key_id=factory.sts_test_key.id, aws_secret_access_key=factory.sts_test_key.secret @@ -675,6 +710,15 @@ def mock_s3_with_error_simulation(): class HostDispatcherApplication(DomainDispatcherApplication): _reqs_till_rate_limit = -1 + # When set to an access key id, any S3 request not signed with that key is rejected with 403. None disables. + _enforced_access_key = None + + @staticmethod + def _request_access_key(environ): + # SigV4 Authorization header: "AWS4-HMAC-SHA256 Credential=///s3/aws4_request, ..." + auth = environ.get("HTTP_AUTHORIZATION", "") + match = re.search(r"Credential=([^/,\s]+)/", auth) + return match.group(1) if match else None def get_backend_for_host(self, host): """The stand-alone server needs a way to distinguish between S3 and IAM. We use the host for that""" @@ -720,6 +764,14 @@ def __call__(self, environ, start_response): start_response("200 OK", [("Content-Type", "text/plain")]) return [b"Limit accepted"] + # Allow enforcing that requests are signed with a specific access key id (empty body disables) + if path_info in ("/enforce_access_key", b"/enforce_access_key"): + length = int(environ["CONTENT_LENGTH"]) + body = environ["wsgi.input"].read(length).decode("ascii") + self._enforced_access_key = body or None + start_response("200 OK", [("Content-Type", "text/plain")]) + return [b"Access key enforcement set"] + if self._reqs_till_rate_limit == 0: response_body = ( b'SlowDownPlease reduce your request rate.' @@ -737,6 +789,16 @@ def __call__(self, environ, start_response): start_response("200 OK", [("Content-Type", "text/plain")]) return [b"Moto AWS S3"] + if self._enforced_access_key is not None and self._request_access_key(environ) != self._enforced_access_key: + response_body = ( + b'AccessDenied' + b"Access Denied" + ) + start_response( + "403 Forbidden", [("Content-Type", "text/xml"), ("Content-Length", str(len(response_body)))] + ) + return [response_body] + return super().__call__(environ, start_response) diff --git a/python/arcticdb/version_store/helper.py b/python/arcticdb/version_store/helper.py index 510f71598c4..bdcfc405535 100644 --- a/python/arcticdb/version_store/helper.py +++ b/python/arcticdb/version_store/helper.py @@ -288,8 +288,10 @@ def get_s3_proto( if aws_auth == AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER or aws_profile: if is_nfs_layout: raise UserInputException("aws_auth and aws_profile can only be set for S3") - if aws_auth != AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER or not aws_profile: - raise UserInputException("STS credential provider and aws_profile must be set together") + if aws_auth == AWSAuthMethod.DISABLED: + raise UserInputException("aws_profile cannot be set unless aws_auth is used") + if aws_auth == AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER and not aws_profile: + raise UserInputException("STS credential provider requires aws_profile to be set") if use_internal_client_wrapper_for_testing: assert ( diff --git a/python/tests/conftest.py b/python/tests/conftest.py index d336e0a1649..77db85df28e 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -49,6 +49,7 @@ real_s3_sts_from_environment_variables, real_s3_sts_resources_ready, real_s3_sts_clean_up, + real_s3_default_profile_from_environment_variables, ) from arcticdb.storage_fixtures.azure import real_azure_from_environment_variables from arcticdb.storage_fixtures.mongo import ManagedMongoDBServer, auto_detect_server @@ -687,6 +688,30 @@ def real_s3_sts_storage(real_s3_sts_storage_factory) -> Generator[S3Bucket, None yield f +@pytest.fixture(scope="session") +def real_s3_default_profile_storage_factory(monkeypatch_session) -> Generator[BaseS3StorageFixtureFactory, None, None]: + profile_name = "default_cred_test_profile" + working_dir = mkdtemp(suffix="S3DefaultProfileStorageFixtureFactory") + config_file_path = os.path.join(working_dir, "config") + try: + f = real_s3_default_profile_from_environment_variables( + profile_name=profile_name, config_file_path=config_file_path + ) + monkeypatch_session.setenv("AWS_CONFIG_FILE", config_file_path) + # Stop the default credentials provider chain finding credentials anywhere except the named profile, so a + # wrong profile deterministically fails to authenticate instead of falling back to env/IMDS credentials + monkeypatch_session.setenv("AWS_EC2_METADATA_DISABLED", "true") + yield f + finally: + safer_rmtree(None, working_dir) + + +@pytest.fixture +def real_s3_default_profile_storage(real_s3_default_profile_storage_factory) -> Generator[S3Bucket, None, None]: + with real_s3_default_profile_storage_factory.create_fixture() as f: + yield f + + # ssl cannot be ON by default due to azurite performance constraints https://github.com/man-group/ArcticDB/issues/1539 @pytest.fixture(scope="session") def azurite_storage_factory() -> Generator[AzuriteStorageFixtureFactory, None, None]: diff --git a/python/tests/integration/arcticdb/test_arctic.py b/python/tests/integration/arcticdb/test_arctic.py index a227b2ba951..9e2127d0b53 100644 --- a/python/tests/integration/arcticdb/test_arctic.py +++ b/python/tests/integration/arcticdb/test_arctic.py @@ -13,6 +13,7 @@ import pytz import math import pytest +import requests import pandas as pd import numpy as np from datetime import datetime, timedelta @@ -21,7 +22,7 @@ import multiprocessing from arcticdb_ext import get_config_int, set_config_int -from arcticdb_ext.exceptions import InternalException, UnsortedDataException, UserInputException +from arcticdb_ext.exceptions import InternalException, PermissionException, UnsortedDataException, UserInputException from arcticdb_ext.storage import NoDataFoundException, KeyType, AWSAuthMethod from arcticdb.exceptions import ArcticDbNotYetImplemented, NoSuchVersionException from arcticdb.adapters.mongo_library_adapter import MongoLibraryAdapter @@ -246,6 +247,83 @@ def test_s3_sts_auth_store(real_s3_sts_version_store): assert_frame_equal(lib.read("sym").data, df) +def _uri_with_profile(s3_storage, profile): + # Reuse the fixture's URI (host, port, bucket, prefix, ...) and swap key auth for default-chain + profile auth + # s3s://HOST:PORT?access=*&secret=*&port=12126&ssl=True&CA_cert_path=* + key_auth = f"access={s3_storage.key.id}&secret={s3_storage.key.secret}" + assert key_auth in s3_storage.arctic_uri, s3_storage.arctic_uri + return s3_storage.arctic_uri.replace(key_auth, f"aws_auth=default&aws_profile={profile}") + + +@pytest.mark.storage +@pytest.mark.authentication +def test_s3_default_auth_with_profile(s3_storage, lib_name, tmp_path, monkeypatch): + """aws_profile with the default credentials provider chain: the client must sign with the profile's + credentials. We make moto enforce that requests carry profile A's access key, so the wrong profile is rejected.""" + profile_a, key_a = "arctic_profile_a", "arctic_profile_a_key" + profile_b, key_b = "arctic_profile_b", "arctic_profile_b_key" + config_file = tmp_path / "aws_config" + config_file.write_text( + f"[profile {profile_a}]\n" + f"aws_access_key_id = {key_a}\n" + f"aws_secret_access_key = arctic_profile_a_secret\n\n" + f"[profile {profile_b}]\n" + f"aws_access_key_id = {key_b}\n" + f"aws_secret_access_key = arctic_profile_b_secret\n" + ) + monkeypatch.setenv("AWS_CONFIG_FILE", str(config_file)) + + endpoint = s3_storage.factory.endpoint + requests.post(endpoint + "/enforce_access_key", key_a.encode("ascii"), verify=False).raise_for_status() + try: + # Profile A is signed with key_a, which the server accepts + ac = Arctic(_uri_with_profile(s3_storage, profile_a)) + try: + lib = ac.create_library(lib_name) + df = pd.DataFrame({"a": [1, 2, 3]}) + lib.write("sym", df) + assert_frame_equal(lib.read("sym").data, df) + finally: + ac.delete_library(lib_name) + + # Profile B is signed with key_b, which the server rejects with 403 AccessDenied + with pytest.raises(PermissionException): + ac_wrong = Arctic(_uri_with_profile(s3_storage, profile_b)) + ac_wrong.create_library(lib_name + "_wrong") + finally: + requests.post(endpoint + "/enforce_access_key", b"", verify=False).raise_for_status() + + +@REAL_S3_TESTS_MARK +@pytest.mark.storage +@pytest.mark.authentication +def test_s3_default_auth_profile_real(lib_name, real_s3_default_profile_storage): + ac = Arctic(real_s3_default_profile_storage.arctic_uri) + try: + ac.delete_library(lib_name) # make sure we delete any previously existing library + lib = ac.create_library(lib_name) + df = pd.DataFrame({"a": [1, 2, 3]}) + lib.write("sym", df) + assert_frame_equal(lib.read("sym").data, df) + finally: + ac.delete_library(lib_name) + + +@REAL_S3_TESTS_MARK +@pytest.mark.storage +@pytest.mark.authentication +def test_s3_default_auth_wrong_profile_real(lib_name, real_s3_default_profile_storage): + # A non-existent profile resolves no credentials (env/IMDS fallback is disabled in the fixture), so the request + # is unauthenticated and S3 rejects it + storage = real_s3_default_profile_storage + bad_uri = storage.arctic_uri.replace( + f"aws_profile={storage.factory.aws_profile}", "aws_profile=nonexistent_profile_xyz" + ) + with pytest.raises(PermissionException): + ac = Arctic(bad_uri) + ac.create_library(lib_name + "_wrong") + + @pytest.mark.parametrize("prefix", ["s3", "gcpxml"]) def test_colon_in_query_parameters(prefix): ac = Arctic(f"{prefix}://10.0.0.3:tabulardata?access=with_a:colon&secret=with_another:colon&port=8010") diff --git a/python/tests/unit/arcticdb/test_library_adapters.py b/python/tests/unit/arcticdb/test_library_adapters.py index b629223eb2a..14f69e366aa 100644 --- a/python/tests/unit/arcticdb/test_library_adapters.py +++ b/python/tests/unit/arcticdb/test_library_adapters.py @@ -1,9 +1,48 @@ import pytest from arcticdb.adapters import S3LibraryAdapter, GCPXMLLibraryAdapter +from arcticdb.adapters.s3_library_adapter import USE_AWS_CRED_PROVIDERS_TOKEN from arcticdb.encoding_version import EncodingVersion +from arcticdb.exceptions import UserInputException +from arcticdb.version_store.helper import add_s3_library_to_env +from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap from arcticdb_ext.storage import AWSAuthMethod +def _add_s3_library(aws_auth, aws_profile, is_nfs_layout=False): + add_s3_library_to_env( + EnvironmentConfigsMap(), + lib_name="lib", + env_name="env", + credential_name=USE_AWS_CRED_PROVIDERS_TOKEN, + credential_key=USE_AWS_CRED_PROVIDERS_TOKEN, + bucket_name="bucket", + endpoint="endpoint", + aws_auth=aws_auth, + aws_profile=aws_profile, + is_nfs_layout=is_nfs_layout, + ) + + +def test_s3_default_auth_with_profile_allowed(): + # aws_profile is now allowed with the default credentials provider chain, not just STS + _add_s3_library(AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN, "my_profile") + + +def test_s3_profile_requires_aws_auth(): + with pytest.raises(UserInputException, match="aws_profile cannot be set"): + _add_s3_library(AWSAuthMethod.DISABLED, "my_profile") + + +def test_s3_sts_auth_requires_profile(): + with pytest.raises(UserInputException, match="STS credential provider requires aws_profile"): + _add_s3_library(AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER, "") + + +def test_s3_profile_not_allowed_for_nfs(): + with pytest.raises(UserInputException, match="can only be set for S3"): + _add_s3_library(AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN, "my_profile", is_nfs_layout=True) + + def test_s3_native_cfg_sdk_default(): adapter = S3LibraryAdapter( "s3://my_endpoint:my_bucket?aws_auth=true&aws_profile=my_profile", encoding_version=EncodingVersion.V1