diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs new file mode 100644 index 00000000..f88f080e --- /dev/null +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -0,0 +1,102 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.Extensions.Logging; +using Moq; + +namespace Cosmos.DataTransfer.CosmosExtension.UnitTests +{ + [TestClass] + public class CertificateConfigurationTests + { + [TestMethod] + public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessfully() + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = true + }; + + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); + } + + [TestMethod] + public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultValidation() + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = false + }; + + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); + } + + [TestMethod] + public void CreateClient_WithDisableSslValidation_LogsWarningAndSetsCallback() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = true + }; + + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + // Verify warning was logged when DisableSslValidation is true + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")), + It.IsAny(), + It.IsAny>()), + Times.Once); + + // Verify client was created successfully + Assert.IsNotNull(client, "CosmosClient should be created"); + } + + [TestMethod] + public void CreateClient_WithoutDisableSslValidation_DoesNotLogWarning() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = false + }; + + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + // Verify warning was NOT logged when DisableSslValidation is false + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")), + It.IsAny(), + It.IsAny>()), + Times.Never); + + // Verify client was created successfully + Assert.IsNotNull(client, "CosmosClient should be created"); + } + + private class TestableCosmosSettings : CosmosSettingsBase + { + } + } +} \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs new file mode 100644 index 00000000..6ca5f344 --- /dev/null +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs @@ -0,0 +1,25 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Cosmos.DataTransfer.CosmosExtension; + +namespace Cosmos.DataTransfer.CosmosExtension.UnitTests +{ + [TestClass] + public class CosmosSettingsBaseTests + { + private class TestableCosmosSettings : CosmosSettingsBase { } + + [TestMethod] + public void AllowBulkExecution_Property_ShouldSetAndGet() + { + var settings = new TestableCosmosSettings + { + AllowBulkExecution = true + }; + + Assert.IsTrue(settings.AllowBulkExecution, "AllowBulkExecution should be true when set to true"); + + settings.AllowBulkExecution = false; + Assert.IsFalse(settings.AllowBulkExecution, "AllowBulkExecution should be false when set to false"); + } + } +} diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs index 33053919..4dae712f 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs @@ -139,7 +139,7 @@ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfigurati var settings = config.Get(); settings.Validate(); - var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, dataSource.DisplayName); + var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, logger, dataSource.DisplayName); Container container; if (settings!.UseRbacAuth) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs index 5a13ff87..03eae830 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs @@ -18,7 +18,7 @@ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogge var settings = config.Get(); settings.Validate(); - var client = CosmosExtensionServices.CreateClient(settings!, DisplayName); + var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, logger); Container container; diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index ba585cae..c158a084 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -1,15 +1,14 @@ -using Azure.Identity; -using Cosmos.DataTransfer.Interfaces; +using Azure.Core; +using Azure.Identity; +using Azure.Security.KeyVault.Keys.Cryptography; using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Encryption; using Microsoft.Extensions.Logging; using System.Globalization; -using System.Reflection; -using Azure.Core; -using System.Text.RegularExpressions; -using Microsoft.Azure.Cosmos.Encryption; -using Azure.Security.KeyVault.Keys.Cryptography; using System.Net; using System.Net.Http; +using System.Reflection; +using System.Text.RegularExpressions; namespace Cosmos.DataTransfer.CosmosExtension { @@ -37,7 +36,7 @@ public static class CosmosExtensionServices return new HttpClient(handler); }); - public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, string? sourceDisplayName = null) + public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, ILogger logger, string? sourceDisplayName = null) { string userAgentString = CreateUserAgentString(displayName, sourceDisplayName); @@ -76,6 +75,13 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp ? () => _httpClientWithDefaultCredentialsAndPreAuth.Value : () => _httpClientWithDefaultCredentials.Value; } + + // Disable SSL certificate validation for development scenarios + if (settings.DisableSslValidation) + { + logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development scenarios. Never use in production."); + clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => true; + } CosmosClient? cosmosClient; if (settings.UseRbacAuth) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index c3814796..d147ec0d 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -27,6 +27,26 @@ public abstract class CosmosSettingsBase : IValidatableObject /// public bool LimitToEndpoint { get; set; } = false; + /// + /// Disables SSL certificate validation for the Cosmos DB connection. + /// This is intended for use with local development environments. + /// WARNING: Never use this option in production as it disables critical security checks. + /// + public bool DisableSslValidation { get; set; } = false; + + /// + /// Enables bulk execution for Cosmos DB operations. + /// When set to true, operations such as bulk inserts and updates are optimized for performance. + /// + /// Default: false + /// + /// + /// Warning: Use with caution, as enabling bulk execution may affect consistency and error handling. + /// Review Cosmos DB documentation for bulk operation caveats before enabling in production environments. + /// + /// + public bool AllowBulkExecution { get; set; } = false; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index ba3d5194..0b62ca6f 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -6,11 +6,44 @@ The Cosmos data transfer extension provides source and sink capabilities for rea ## Settings + +### Main Settings + +| Setting | Description | Default | +|------------------------|---------------------------------------------------------------------------------------------------|-----------| +| ConnectionString | Cosmos DB connection string (AccountEndpoint + AccountKey) | | +| UseRbacAuth | Use Role Based Access Control for authentication | false | +| AccountEndpoint | Cosmos DB account endpoint (required for RBAC) | | +| EnableInteractiveCredentials | Prompt for Azure login if default credentials are unavailable | false | +| Database | Cosmos DB database name | | +| Container | Cosmos DB container name | | +| WebProxy | Proxy server URL for Cosmos DB connections | | +| InitClientEncryption | Enable Always Encrypted feature | false | +| LimitToEndpoint | Restrict client to endpoint (see CosmosClientOptions.LimitToEndpoint) | false | +| DisableSslValidation | Disable SSL certificate validation (for local dev only; not for production) | false | +| AllowBulkExecution | Enable bulk execution for optimized performance.
**Warning:** May affect consistency and error handling. | false | + Source and sink require settings used to locate and access the Cosmos DB account. This can be done in one of two ways: - Using a `ConnectionString` that includes an AccountEndpoint and AccountKey - Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. See ([migrate-passwordless](https://learn.microsoft.com/azure/cosmos-db/nosql/migrate-passwordless?tabs=sign-in-azure-cli%2Cdotnet%2Cazure-portal-create%2Cazure-portal-associate%2Capp-service-identity) for how to configure Cosmos DB for passwordless access. + +### Bulk Execution + +The extension supports bulk execution for Cosmos DB operations. When the `AllowBulkExecution` setting is set to `true`, operations such as bulk inserts and updates are optimized for performance. Use with caution, as bulk execution may affect consistency and error handling. Default is `false`. + +Example: + +```json +{ + "ConnectionString": "AccountEndpoint=https://...", + "Database": "myDb", + "Container": "myContainer", + "AllowBulkExecution": true +} +``` + Source and sink settings also both require parameters to specify the data location within a Cosmos DB account: - `Database` @@ -27,7 +60,7 @@ Source supports the following optional parameters: ### Always Encrypted -Source and Sink support Always Encrypted as an optional parameter. When `InitClientEncryption` is set to `true`, the extension will initialize the Cosmos client with the Always Encrypted feature enabled. This allows for the use of encrypted fields in the Cosmos DB container. The extension will automatically decrypt the fields when reading from the source and encrypt the fields when writing to the sink. +Source and Sink support Always Encrypted as an optional parameter. When `InitClientEncryption` is set to `true`, the extension will initialize the Cosmos client with the Always Encrypted feature enabled. This allows for the use of encrypted fields in the Cosmos DB container. The extension will automatically decrypt the fields when reading from the source and encrypt the fields when writing to the sink.
The extension will also automatically handle the encryption keys and encryption policy for the client, but it requires `UseRbacAuth` to be set to `true` and the user to have the necessary permissions to access the key vault.
@@ -70,22 +103,39 @@ Or with RBAC: } ``` +#### Disable SSL Validation Configuration Example + +For development purposes with SSL validation disabled: + +```json +{ + "ConnectionString": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + "Database":"myDb", + "Container":"myContainer", + "DisableSslValidation": true +} +``` + ### Sink Settings #### **Partition Key Settings** + - **`PartitionKeyPath`**: Specifies the partition key path when creating the container (e.g., `/id`) if it does not exist. - **`PartitionKeyPaths`**: Use this to supply an array of up to 3 paths for hierarchical partition keys. #### **Database Management** + - **`UseAutoscaleForDatabase`**: Specifies if the database will be created with autoscale enabled or manual. Defaults to `false`. manual. #### **Container Management** + - **`RecreateContainer`**: Optional, defaults to `false`. Deletes and recreates the container to ensure only newly imported data is present. - **`CreatedContainerMaxThroughput`**: Specifies the initial throughput (in RUs) for a newly created container. - **`UseAutoscaleForCreatedContainer`**: Enables autoscale for the newly created container. - **`UseSharedThroughput`**: Set to `true` to use shared throughput provisioned at the database level. #### **Batching and Write Behavior** + - **`BatchSize`**: Optional, defaults to `100`. Sets the number of items to accumulate before inserting. - **`WriteMode`**: Specifies the type of data write to use. Options: - `InsertStream` @@ -94,6 +144,7 @@ Or with RBAC: - `Upsert` #### **Connection Settings** + - **`ConnectionMode`**: Controls how the client connects to the Cosmos DB service. Options: - `Gateway` (default) - `Direct` @@ -104,16 +155,23 @@ Or with RBAC: - **`PreAuthenticate`**: Optional, defaults to `false`. When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. - **`LimitToEndpoint`**: Optional, defaults to `false`. When the value of this property is false, the Cosmos DB SDK will automatically discover - write and read regions, and use them when the configured application region is not available. + write and read regions, and use them when the configured application region is not available. When set to `true`, availability is limited to the endpoint specified. - **Note**: [CosmosClientOptions.LimitToEndpoint Property](https://learn.microsoft.com/dotnet/api/microsoft.azure.cosmos.cosmosclientoptions.limittoendpoint?view=azure-dotnet). When using the Cosmos DB Emulator Container for Linux it's been observed - setting the value to `true` enables import and export of data. + setting the value to `true` enables import and export of data. + +#### **SSL/Certificate Settings** + +- **`DisableSslValidation`**: Optional, defaults to `false`. Disables SSL certificate validation for development/emulator scenarios. + - **⚠️ WARNING**: Only use this for development purposes. Never use in production environments as it disables critical security checks and makes connections vulnerable to man-in-the-middle attacks. #### **Serverless Account** + - **`IsServerlessAccount`**: Specifies whether the target account uses Serverless instead of Provisioned throughput, which affects the way containers are created. - **Note**: Serverless accounts cannot have shared throughput. See [Azure Cosmos DB serverless account type](https://learn.microsoft.com/azure/cosmos-db/serverless#use-serverless-resources). #### **Client Behavior** + - **`PreserveMixedCaseIds`**: Optional, defaults to `false`. Writes `id` fields with their original casing while generating a separate lowercased `id` field as required by Cosmos. - **`IgnoreNullValues`**: Optional. Excludes fields with null values when writing to Cosmos DB. - **`InitClientEncryption`**: Optional, defaults to `false`. Uses client-side encryption with the container. Can only be used with `UseRbacAuth` set to `true`