From af27c177bc6feddacabb40d665c611ba08c3b22e Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 2 Nov 2025 05:36:14 -0800 Subject: [PATCH 01/31] Disabling SSL option for development purposes to work with vnext emulator for Partial certchain --- Extensions/Cosmos/CERTIFICATE_EXAMPLES.md | 125 ++++++++++++++++++ .../CertificateConfigurationTests.cs | 87 ++++++++++++ .../CosmosExtensionServices.cs | 50 +++++++ .../CosmosSettingsBase.cs | 16 +++ Extensions/Cosmos/README.md | 45 ++++++- 5 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 Extensions/Cosmos/CERTIFICATE_EXAMPLES.md create mode 100644 Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs diff --git a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md new file mode 100644 index 00000000..74a5d9e8 --- /dev/null +++ b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md @@ -0,0 +1,125 @@ +# Custom Certificate Configuration Examples + +This file demonstrates how to use the new custom certificate configuration options in the Cosmos DB data migration tool. + +## Example 1: Disable SSL Validation (Development/Emulator Only) + +For development with the Cosmos DB emulator when certificates are problematic: + +```json +{ + "Source": "json", + "Sink": "cosmos-nosql", + "SourceSettings": { + "FilePath": "C:\\data\\sample-data.json" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + "Database": "SampleDB", + "Container": "SampleContainer", + "DisableSslValidation": true + } +} +``` + +## Example 2: Custom Certificate Path + +For custom Cosmos DB instances with self-signed or corporate certificates: + +```json +{ + "Source": "json", + "Sink": "cosmos-nosql", + "SourceSettings": { + "FilePath": "C:\\data\\sample-data.json" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.company.com:8081/;AccountKey=...", + "Database": "ProductionDB", + "Container": "DataContainer", + "CustomCertificatePath": "C:\\certificates\\cosmos-custom.cer", + "PartitionKeyPath": "/id" + } +} +``` + +## Example 3: RBAC with Custom Certificate + +Using RBAC authentication with a custom certificate: + +```json +{ + "Source": "json", + "Sink": "cosmos-nosql", + "SourceSettings": { + "FilePath": "C:\\data\\sample-data.json" + }, + "SinkSettings": { + "UseRbacAuth": true, + "AccountEndpoint": "https://my-cosmos-account.documents.azure.com:443/", + "EnableInteractiveCredentials": true, + "Database": "MyDatabase", + "Container": "MyContainer", + "CustomCertificatePath": "C:\\certificates\\my-cosmos-cert.pem", + "PartitionKeyPath": "/partitionKey" + } +} +``` + +## Example 4: Cosmos-to-Cosmos Migration with Custom Certificate + +Migrating from one Cosmos DB instance to another with custom certificate validation: + +```json +{ + "Source": "cosmos-nosql", + "Sink": "cosmos-nosql", + "SourceSettings": { + "ConnectionString": "AccountEndpoint=https://source-cosmos.domain.com:8081/;AccountKey=...", + "Database": "SourceDB", + "Container": "SourceContainer", + "CustomCertificatePath": "C:\\certificates\\source-cosmos.cer" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://dest-cosmos.domain.com:8081/;AccountKey=...", + "Database": "DestDB", + "Container": "DestContainer", + "CustomCertificatePath": "C:\\certificates\\dest-cosmos.cer", + "PartitionKeyPath": "/id", + "WriteMode": "Upsert" + } +} +``` + +## Security Notes + +⚠️ **Important Security Considerations:** + +1. **Never use `DisableSslValidation: true` in production environments** + - This disables all SSL certificate validation + - Only use for development with local emulators + +2. **Store certificate files securely** + - Use proper file permissions + - Store in protected directories + - Consider using Azure Key Vault for production scenarios + +3. **Certificate file formats supported** + - `.cer` (binary or base64 encoded) + - `.crt` (binary or base64 encoded) + - `.pem` (base64 encoded) + +4. **Validate certificate authenticity** + - Ensure certificate files come from trusted sources + - Verify certificate thumbprints when possible + - Use proper certificate authorities for production + +## Troubleshooting + +If you encounter certificate validation issues: + +1. **Verify certificate file exists and is readable** +2. **Check certificate format and encoding** +3. **Ensure certificate matches the server's certificate** +4. **For emulator issues, try `DisableSslValidation: true` (development only)** +5. **For production, contact your network administrator for proper certificates** diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs new file mode 100644 index 00000000..9f33a914 --- /dev/null +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -0,0 +1,87 @@ +using Cosmos.DataTransfer.CosmosExtension; +using System.ComponentModel.DataAnnotations; + +namespace Cosmos.DataTransfer.CosmosExtension.UnitTests +{ + [TestClass] + public class CertificateConfigurationTests + { + [TestMethod] + public void CosmosSettingsBase_WithValidCertificatePath_ShouldValidateSuccessfully() + { + // Arrange - Create a temp certificate file for testing + var tempCertPath = Path.GetTempFileName(); + File.WriteAllText(tempCertPath, "dummy cert content"); // Not a real cert, but file exists + + try + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + CustomCertificatePath = tempCertPath, + DisableSslValidation = false + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); + } + finally + { + // Cleanup + if (File.Exists(tempCertPath)) + File.Delete(tempCertPath); + } + } + + [TestMethod] + public void CosmosSettingsBase_WithInvalidCertificatePath_ShouldReturnValidationError() + { + // Arrange + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + CustomCertificatePath = "C:\\nonexistent\\path\\cert.cer", + DisableSslValidation = false + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(1, validationResults.Count); + Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CustomCertificatePath file does not exist")); + } + + [TestMethod] + public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessfully() + { + // Arrange + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + CustomCertificatePath = null, + DisableSslValidation = true + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); + } + + // Test implementation of CosmosSettingsBase for testing + private class TestableCosmosSettings : CosmosSettingsBase + { + } + } +} \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index dd9729f4..e9bea532 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -9,6 +9,8 @@ using Microsoft.Azure.Cosmos.Encryption; using Azure.Security.KeyVault.Keys.Cryptography; using System.Net; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; namespace Cosmos.DataTransfer.CosmosExtension { @@ -39,6 +41,12 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp if (!string.IsNullOrEmpty(settings.WebProxy)){ clientOptions.WebProxy = new WebProxy(settings.WebProxy); } + + // Configure custom certificate validation + if (settings.DisableSslValidation || !string.IsNullOrEmpty(settings.CustomCertificatePath)) + { + clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(settings); + } CosmosClient? cosmosClient; if (settings.UseRbacAuth) @@ -111,5 +119,47 @@ public static async Task VerifyContainerAccess(Container? container, string? nam throw new InvalidOperationException("Failed to create CosmosClient"); } } + + private static Func CreateCertificateValidationCallback(CosmosSettingsBase settings) + { + return (cert, chain, errors) => + { + // If SSL validation is disabled (development/emulator only) + if (settings.DisableSslValidation) + { + return true; + } + + // If a custom certificate path is specified + if (!string.IsNullOrEmpty(settings.CustomCertificatePath)) + { + try + { + var trustedCert = new X509Certificate2(settings.CustomCertificatePath); + + // Compare certificate thumbprints + bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); + + if (!thumbprintMatch) + { + // Also try comparing by subject and issuer as fallback + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } + + return thumbprintMatch; + } + catch (Exception) + { + // If we can't load the custom certificate, fail validation + return false; + } + } + + // Default validation - accept only if no SSL policy errors + return errors == SslPolicyErrors.None; + }; + } } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index a81fac44..e3348b3e 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -24,6 +24,18 @@ public abstract class CosmosSettingsBase : IValidatableObject /// public bool LimitToEndpoint { get; set; } = false; + /// + /// Path to a custom certificate file (.cer, .crt, .pem) for SSL validation. + /// When specified, only connections using this certificate will be accepted. + /// + public string? CustomCertificatePath { get; set; } + + /// + /// Disable SSL certificate validation. + /// WARNING: Only use this for development with the emulator. Never use in production. + /// + public bool DisableSslValidation { get; set; } = false; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) @@ -38,6 +50,10 @@ public virtual IEnumerable Validate(ValidationContext validati { yield return new ValidationResult("InitClientEncryption can only be used when UseRbacAuth is true", new[] { nameof(InitClientEncryption) }); } + if (!string.IsNullOrEmpty(CustomCertificatePath) && !File.Exists(CustomCertificatePath)) + { + yield return new ValidationResult($"CustomCertificatePath file does not exist: {CustomCertificatePath}", new[] { nameof(CustomCertificatePath) }); + } } } } \ No newline at end of file diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 82ea46fb..23d8db94 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -20,7 +20,7 @@ Source supports an optional `IncludeMetadataFields` parameter (`false` by defaul ### Always Encrypted -Source and Sink support Always Encrypted as an optional parameter. When `InitClientEncryption` is set to `true`, the extension will initialize the Cosmos client with the Always Encrypted feature enabled. This allows for the use of encrypted fields in the Cosmos DB container. The extension will automatically decrypt the fields when reading from the source and encrypt the fields when writing to the sink. +Source and Sink support Always Encrypted as an optional parameter. When `InitClientEncryption` is set to `true`, the extension will initialize the Cosmos client with the Always Encrypted feature enabled. This allows for the use of encrypted fields in the Cosmos DB container. The extension will automatically decrypt the fields when reading from the source and encrypt the fields when writing to the sink.
The extension will also automatically handle the encryption keys and encryption policy for the client, but it requires `UseRbacAuth` to be set to `true` and the user to have the necessary permissions to access the key vault.
@@ -52,27 +52,55 @@ Or with RBAC: "IncludeMetadataFields": false, "PartitionKeyValue":"123", "Query":"SELECT * FROM c WHERE c.category='event'", - "InitClientEncryption": false + "InitClientEncryption": false, "WebProxy":"http://yourproxy.server.com/" } ``` +#### Certificate Configuration Examples + +For emulator development with SSL validation disabled: + +```json +{ + "ConnectionString": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + "Database":"myDb", + "Container":"myContainer", + "DisableSslValidation": true +} +``` + +For custom certificate validation: + +```json +{ + "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.domain.com:8081/;AccountKey=...", + "Database":"myDb", + "Container":"myContainer", + "CustomCertificatePath": "C:\\certs\\cosmos-custom.cer" +} +``` + ### Sink Settings #### **Partition Key Settings** + - **`PartitionKeyPath`**: Specifies the partition key path when creating the container (e.g., `/id`) if it does not exist. - **`PartitionKeyPaths`**: Use this to supply an array of up to 3 paths for hierarchical partition keys. #### **Database Management** + - **`UseAutoscaleForDatabase`**: Specifies if the database will be created with autoscale enabled or manual. Defaults to `false`. manual. #### **Container Management** + - **`RecreateContainer`**: Optional, defaults to `false`. Deletes and recreates the container to ensure only newly imported data is present. - **`CreatedContainerMaxThroughput`**: Specifies the initial throughput (in RUs) for a newly created container. - **`UseAutoscaleForCreatedContainer`**: Enables autoscale for the newly created container. - **`UseSharedThroughput`**: Set to `true` to use shared throughput provisioned at the database level. #### **Batching and Write Behavior** + - **`BatchSize`**: Optional, defaults to `100`. Sets the number of items to accumulate before inserting. - **`WriteMode`**: Specifies the type of data write to use. Options: - `InsertStream` @@ -81,21 +109,30 @@ Or with RBAC: - `Upsert` #### **Connection Settings** + - **`ConnectionMode`**: Controls how the client connects to the Cosmos DB service. Options: - `Gateway` (default) - `Direct` - **`LimitToEndpoint`**: Optional, defaults to `false`. When the value of this property is false, the Cosmos DB SDK will automatically discover - write and read regions, and use them when the configured application region is not available. + write and read regions, and use them when the configured application region is not available. When set to `true`, availability is limited to the endpoint specified. - **Note**: [CosmosClientOptions.LimitToEndpoint Property](https://learn.microsoft.com/dotnet/api/microsoft.azure.cosmos.cosmosclientoptions.limittoendpoint?view=azure-dotnet). When using the Cosmos DB Emulator Container for Linux it's been observed - setting the value to `true` enables import and export of data. + setting the value to `true` enables import and export of data. + +#### **SSL/Certificate Settings** + +- **`CustomCertificatePath`**: Optional. Path to a custom certificate file (.cer, .crt, .pem) for SSL validation. When specified, only connections using this certificate will be accepted. Useful for private/self-signed certificates or custom certificate authorities. +- **`DisableSslValidation`**: Optional, defaults to `false`. Disables SSL certificate validation entirely. + - **⚠️ WARNING**: Only use this for development with the emulator. Never use in production environments as it makes connections vulnerable to man-in-the-middle attacks. #### **Serverless Account** + - **`IsServerlessAccount`**: Specifies whether the target account uses Serverless instead of Provisioned throughput, which affects the way containers are created. - **Note**: Serverless accounts cannot have shared throughput. See [Azure Cosmos DB serverless account type](https://learn.microsoft.com/azure/cosmos-db/serverless#use-serverless-resources). #### **Client Behavior** + - **`PreserveMixedCaseIds`**: Optional, defaults to `false`. Writes `id` fields with their original casing while generating a separate lowercased `id` field as required by Cosmos. - **`IgnoreNullValues`**: Optional. Excludes fields with null values when writing to Cosmos DB. - **`InitClientEncryption`**: Optional, defaults to `false`. Uses client-side encryption with the container. Can only be used with `UseRbacAuth` set to `true` From 24546e51f4d8276043fdacfe37e0fc297b855488 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 2 Nov 2025 07:47:08 -0800 Subject: [PATCH 02/31] Add ability to provide password with PFX --- Extensions/Cosmos/CERTIFICATE_EXAMPLES.md | 99 ++++++++++++- .../CertificateConfigurationTests.cs | 132 +++++++++++++++++- .../CosmosExtensionServices.cs | 68 +++++++-- .../CosmosSettingsBase.cs | 20 ++- Extensions/Cosmos/README.md | 32 ++++- 5 files changed, 323 insertions(+), 28 deletions(-) diff --git a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md index 74a5d9e8..4f595c93 100644 --- a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md +++ b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md @@ -37,7 +37,7 @@ For custom Cosmos DB instances with self-signed or corporate certificates: "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.company.com:8081/;AccountKey=...", "Database": "ProductionDB", "Container": "DataContainer", - "CustomCertificatePath": "C:\\certificates\\cosmos-custom.cer", + "CertificatePath": "C:\\certificates\\cosmos-custom.cer", "PartitionKeyPath": "/id" } } @@ -60,7 +60,7 @@ Using RBAC authentication with a custom certificate: "EnableInteractiveCredentials": true, "Database": "MyDatabase", "Container": "MyContainer", - "CustomCertificatePath": "C:\\certificates\\my-cosmos-cert.pem", + "CertificatePath": "C:\\certificates\\my-cosmos-cert.pem", "PartitionKeyPath": "/partitionKey" } } @@ -78,13 +78,85 @@ Migrating from one Cosmos DB instance to another with custom certificate validat "ConnectionString": "AccountEndpoint=https://source-cosmos.domain.com:8081/;AccountKey=...", "Database": "SourceDB", "Container": "SourceContainer", - "CustomCertificatePath": "C:\\certificates\\source-cosmos.cer" + "CertificatePath": "C:\\certificates\\source-cosmos.cer" }, "SinkSettings": { "ConnectionString": "AccountEndpoint=https://dest-cosmos.domain.com:8081/;AccountKey=...", "Database": "DestDB", "Container": "DestContainer", - "CustomCertificatePath": "C:\\certificates\\dest-cosmos.cer", + "CertificatePath": "C:\\certificates\\dest-cosmos.cer", + "PartitionKeyPath": "/id", + "WriteMode": "Upsert" + } +} +``` + +## Example 5: Enterprise PFX Certificate with Password + +For enterprise environments using PFX/P12 certificates with mutual TLS authentication: + +```json +{ + "Source": "json", + "Sink": "cosmos-nosql", + "SourceSettings": { + "FilePath": "C:\\data\\enterprise-data.json" + }, + "SinkSettings": { + "UseRbacAuth": true, + "AccountEndpoint": "https://enterprise-cosmos.company.com:443/", + "Database": "EnterpriseDB", + "Container": "SecureContainer", + "CertificatePath": "C:\\enterprise-certs\\cosmos-client.pfx", + "CertificatePassword": "SecureP@ssw0rd!", + "PartitionKeyPath": "/tenantId" + } +} +``` + +## Example 6: Enterprise PFX Certificate without Password + +For enterprise environments with unprotected PFX certificates: + +```json +{ + "Source": "cosmos-nosql", + "Sink": "json", + "SourceSettings": { + "ConnectionString": "AccountEndpoint=https://enterprise-cosmos.company.com:8081/;AccountKey=...", + "Database": "EnterpriseDB", + "Container": "AuditLogs", + "CertificatePath": "C:\\enterprise-certs\\cosmos-client.p12" + }, + "SinkSettings": { + "FilePath": "C:\\exports\\audit-logs.json" + } +} +``` + +## Example 7: Enterprise Multi-Environment Migration + +Migrating from staging to production with different PFX certificates: + +```json +{ + "Source": "cosmos-nosql", + "Sink": "cosmos-nosql", + "SourceSettings": { + "UseRbacAuth": true, + "AccountEndpoint": "https://staging-cosmos.company.com:443/", + "Database": "StagingDB", + "Container": "DataContainer", + "CertificatePath": "C:\\certs\\staging-client.pfx", + "CertificatePassword": "StagingPass123!" + }, + "SinkSettings": { + "UseRbacAuth": true, + "AccountEndpoint": "https://prod-cosmos.company.com:443/", + "Database": "ProductionDB", + "Container": "DataContainer", + "CertificatePath": "C:\\certs\\prod-client.pfx", + "CertificatePassword": "ProductionSecurePass456!", "PartitionKeyPath": "/id", "WriteMode": "Upsert" } @@ -108,11 +180,21 @@ Migrating from one Cosmos DB instance to another with custom certificate validat - `.cer` (binary or base64 encoded) - `.crt` (binary or base64 encoded) - `.pem` (base64 encoded) + - `.pfx` (PKCS#12, with or without password) + - `.p12` (PKCS#12, with or without password) + +4. **PFX/P12 Certificate Security** + - Never hardcode `CertificatePassword` in configuration files + - Use environment variables or secure configuration providers for passwords + - Store PFX files with restricted access permissions (600 on Unix, appropriate ACLs on Windows) + - Consider using Azure Key Vault or other secure key management systems + - Ensure PFX files contain both certificate and private key -4. **Validate certificate authenticity** +5. **Validate certificate authenticity** - Ensure certificate files come from trusted sources - Verify certificate thumbprints when possible - Use proper certificate authorities for production + - For PFX certificates, verify the certificate chain and expiration dates ## Troubleshooting @@ -121,5 +203,8 @@ If you encounter certificate validation issues: 1. **Verify certificate file exists and is readable** 2. **Check certificate format and encoding** 3. **Ensure certificate matches the server's certificate** -4. **For emulator issues, try `DisableSslValidation: true` (development only)** -5. **For production, contact your network administrator for proper certificates** +4. **For PFX certificates, verify the password is correct** +5. **For emulator issues, try `DisableSslValidation: true` (development only)** +6. **For production, contact your network administrator for proper certificates** +7. **Check that PFX files contain both certificate and private key** +8. **Verify certificate expiration dates and certificate chain validity** diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index 9f33a914..0efc94bf 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -20,7 +20,7 @@ public void CosmosSettingsBase_WithValidCertificatePath_ShouldValidateSuccessful ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CustomCertificatePath = tempCertPath, + CertificatePath = tempCertPath, DisableSslValidation = false }; @@ -47,7 +47,7 @@ public void CosmosSettingsBase_WithInvalidCertificatePath_ShouldReturnValidation ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CustomCertificatePath = "C:\\nonexistent\\path\\cert.cer", + CertificatePath = "C:\\nonexistent\\path\\cert.cer", DisableSslValidation = false }; @@ -56,7 +56,7 @@ public void CosmosSettingsBase_WithInvalidCertificatePath_ShouldReturnValidation // Assert Assert.AreEqual(1, validationResults.Count); - Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CustomCertificatePath file does not exist")); + Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CertificatePath file does not exist")); } [TestMethod] @@ -68,7 +68,7 @@ public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessful ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CustomCertificatePath = null, + CertificatePath = null, DisableSslValidation = true }; @@ -79,6 +79,130 @@ public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessful Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); } + [TestMethod] + public void CosmosSettingsBase_WithValidPfxCertificatePath_ShouldValidateSuccessfully() + { + // Arrange - Create a temp PFX file for testing + var tempPfxPath = Path.GetTempFileName(); + File.WriteAllText(tempPfxPath, "dummy pfx content"); // Not a real PFX, but file exists + + try + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", + Database = "testDb", + Container = "testContainer", + CertificatePath = tempPfxPath, + CertificatePassword = "testPassword" + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); + } + finally + { + // Cleanup + if (File.Exists(tempPfxPath)) + File.Delete(tempPfxPath); + } + } + + [TestMethod] + public void CosmosSettingsBase_WithInvalidPfxCertificatePath_ShouldReturnValidationError() + { + // Arrange + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", + Database = "testDb", + Container = "testContainer", + CertificatePath = "C:\\nonexistent\\path\\cert.pfx", + CertificatePassword = "testPassword" + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(1, validationResults.Count); + Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CertificatePath file does not exist")); + } + + [TestMethod] + public void CosmosSettingsBase_WithPfxCertificateWithoutPassword_ShouldValidateSuccessfully() + { + // Arrange - Create a temp PFX file for testing + var tempPfxPath = Path.GetTempFileName(); + File.WriteAllText(tempPfxPath, "dummy pfx content"); // Not a real PFX, but file exists + + try + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", + Database = "testDb", + Container = "testContainer", + CertificatePath = tempPfxPath, + CertificatePassword = null // No password specified + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors for PFX without password"); + } + finally + { + // Cleanup + if (File.Exists(tempPfxPath)) + File.Delete(tempPfxPath); + } + } + + [TestMethod] + public void CosmosSettingsBase_WithCertificateTypeDetection_ShouldSupportMultipleFormats() + { + // Arrange - Test that the unified CertificatePath supports different file extensions + var extensions = new[] { ".cer", ".crt", ".pem", ".pfx", ".p12" }; + + foreach (var extension in extensions) + { + var tempCertPath = Path.GetTempFileName(); + var certPathWithExtension = Path.ChangeExtension(tempCertPath, extension); + File.Move(tempCertPath, certPathWithExtension); + File.WriteAllText(certPathWithExtension, "dummy cert content"); + + try + { + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + CertificatePath = certPathWithExtension, + CertificatePassword = (extension == ".pfx" || extension == ".p12") ? "password" : null + }; + + // Act + var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); + + // Assert + Assert.AreEqual(0, validationResults.Count, $"Should have no validation errors for {extension} files"); + } + finally + { + // Cleanup + if (File.Exists(certPathWithExtension)) + File.Delete(certPathWithExtension); + } + } + } + // Test implementation of CosmosSettingsBase for testing private class TestableCosmosSettings : CosmosSettingsBase { diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index e9bea532..c22a805b 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -43,7 +43,7 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp } // Configure custom certificate validation - if (settings.DisableSslValidation || !string.IsNullOrEmpty(settings.CustomCertificatePath)) + if (settings.DisableSslValidation || !string.IsNullOrEmpty(settings.CertificatePath)) { clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(settings); } @@ -130,29 +130,71 @@ private static Func CreateCe return true; } - // If a custom certificate path is specified - if (!string.IsNullOrEmpty(settings.CustomCertificatePath)) + // If a certificate path is specified + if (!string.IsNullOrEmpty(settings.CertificatePath)) { try { - var trustedCert = new X509Certificate2(settings.CustomCertificatePath); + X509Certificate2 trustedCert; + bool isPfxCertificate = IsPfxCertificate(settings.CertificatePath); - // Compare certificate thumbprints + // Load certificate based on type and password availability + if (isPfxCertificate) + { + // Load PFX certificate with or without password + trustedCert = string.IsNullOrEmpty(settings.CertificatePassword) + ? new X509Certificate2(settings.CertificatePath) + : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword); + } + else + { + // Load standard certificate file (.cer, .crt, .pem) + trustedCert = new X509Certificate2(settings.CertificatePath); + } + + // Compare certificate thumbprints (most reliable method) bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); if (!thumbprintMatch) { - // Also try comparing by subject and issuer as fallback - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; + // For PFX certificates, also check if the server cert was issued by our trusted cert + // This handles cases where the PFX is a CA certificate + if (isPfxCertificate) + { + try + { + var certChain = new X509Chain(); + certChain.ChainPolicy.ExtraStore.Add(trustedCert); + certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; + + bool chainIsValid = certChain.Build(cert); + return chainIsValid && certChain.ChainElements + .Cast() + .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + } + catch + { + // Fallback to subject and issuer comparison + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } + } + else + { + // For standard certificates, try comparing by subject and issuer as fallback + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } } return thumbprintMatch; } catch (Exception) { - // If we can't load the custom certificate, fail validation + // If we can't load the certificate, fail validation return false; } } @@ -161,5 +203,11 @@ private static Func CreateCe return errors == SslPolicyErrors.None; }; } + + private static bool IsPfxCertificate(string certificatePath) + { + var extension = Path.GetExtension(certificatePath).ToLowerInvariant(); + return extension == ".pfx" || extension == ".p12"; + } } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index e3348b3e..db2031d2 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -25,10 +25,20 @@ public abstract class CosmosSettingsBase : IValidatableObject public bool LimitToEndpoint { get; set; } = false; /// - /// Path to a custom certificate file (.cer, .crt, .pem) for SSL validation. - /// When specified, only connections using this certificate will be accepted. + /// Path to a certificate file for SSL validation and client authentication. + /// Supports multiple formats: + /// - .cer, .crt, .pem files for basic SSL validation + /// - .pfx, .p12 files for client authentication (enterprise scenarios) + /// For PFX/P12 files, use CertificatePassword if the file is password-protected. /// - public string? CustomCertificatePath { get; set; } + public string? CertificatePath { get; set; } + + /// + /// Password for PFX/P12 certificate files when they are password-protected. + /// Only used when CertificatePath points to a .pfx or .p12 file. + /// WARNING: Store this securely and avoid hardcoding in configuration files. + /// + public string? CertificatePassword { get; set; } /// /// Disable SSL certificate validation. @@ -50,9 +60,9 @@ public virtual IEnumerable Validate(ValidationContext validati { yield return new ValidationResult("InitClientEncryption can only be used when UseRbacAuth is true", new[] { nameof(InitClientEncryption) }); } - if (!string.IsNullOrEmpty(CustomCertificatePath) && !File.Exists(CustomCertificatePath)) + if (!string.IsNullOrEmpty(CertificatePath) && !File.Exists(CertificatePath)) { - yield return new ValidationResult($"CustomCertificatePath file does not exist: {CustomCertificatePath}", new[] { nameof(CustomCertificatePath) }); + yield return new ValidationResult($"CertificatePath file does not exist: {CertificatePath}", new[] { nameof(CertificatePath) }); } } } diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 23d8db94..ab93896f 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -77,7 +77,31 @@ For custom certificate validation: "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.domain.com:8081/;AccountKey=...", "Database":"myDb", "Container":"myContainer", - "CustomCertificatePath": "C:\\certs\\cosmos-custom.cer" + "CertificatePath": "C:\\certs\\cosmos-custom.cer" +} +``` + +For enterprise PFX certificate with password: + +```json +{ + "ConnectionString": "AccountEndpoint=https://enterprise-cosmos.company.com:8081/;AccountKey=...", + "Database":"EnterpriseDB", + "Container":"SecureContainer", + "CertificatePath": "C:\\enterprise-certs\\cosmos-client.pfx", + "CertificatePassword": "SecureP@ssw0rd!" +} +``` + +For enterprise PFX certificate without password: + +```json +{ + "UseRbacAuth": true, + "AccountEndpoint": "https://enterprise-cosmos.company.com:443/", + "Database":"EnterpriseDB", + "Container":"SecureContainer", + "CertificatePath": "C:\\enterprise-certs\\cosmos-client.p12" } ``` @@ -122,7 +146,11 @@ For custom certificate validation: #### **SSL/Certificate Settings** -- **`CustomCertificatePath`**: Optional. Path to a custom certificate file (.cer, .crt, .pem) for SSL validation. When specified, only connections using this certificate will be accepted. Useful for private/self-signed certificates or custom certificate authorities. +- **`CertificatePath`**: Optional. Path to a certificate file for SSL validation and client authentication. Supports multiple formats: + - `.cer`, `.crt`, `.pem` files for basic SSL validation + - `.pfx`, `.p12` files for client authentication (enterprise scenarios) + For PFX/P12 files, use `CertificatePassword` if the file is password-protected. +- **`CertificatePassword`**: Optional. Password for PFX/P12 certificate files when they are password-protected. Only used when `CertificatePath` points to a `.pfx` or `.p12` file. ⚠️ Store securely and avoid hardcoding in configuration files. - **`DisableSslValidation`**: Optional, defaults to `false`. Disables SSL certificate validation entirely. - **⚠️ WARNING**: Only use this for development with the emulator. Never use in production environments as it makes connections vulnerable to man-in-the-middle attacks. From d3e59960c8980842aaa23dd35554db349fe02389 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 9 Nov 2025 12:51:58 -0800 Subject: [PATCH 03/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index c22a805b..b044ba4d 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -192,8 +192,10 @@ private static Func CreateCe return thumbprintMatch; } - catch (Exception) + catch (Exception ex) { + // Log the exception details to help diagnose certificate loading issues + Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); // If we can't load the certificate, fail validation return false; } From cbd828a2e911fc6beaf5cbd438ac1306a24f0a2e Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 9 Nov 2025 12:52:35 -0800 Subject: [PATCH 04/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 75 +++++++++---------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index b044ba4d..6efb7064 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -135,62 +135,55 @@ private static Func CreateCe { try { - X509Certificate2 trustedCert; bool isPfxCertificate = IsPfxCertificate(settings.CertificatePath); // Load certificate based on type and password availability - if (isPfxCertificate) - { - // Load PFX certificate with or without password - trustedCert = string.IsNullOrEmpty(settings.CertificatePassword) + using (X509Certificate2 trustedCert = isPfxCertificate + ? (string.IsNullOrEmpty(settings.CertificatePassword) ? new X509Certificate2(settings.CertificatePath) - : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword); - } - else + : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword)) + : new X509Certificate2(settings.CertificatePath)) { - // Load standard certificate file (.cer, .crt, .pem) - trustedCert = new X509Certificate2(settings.CertificatePath); - } - - // Compare certificate thumbprints (most reliable method) - bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); - - if (!thumbprintMatch) - { - // For PFX certificates, also check if the server cert was issued by our trusted cert - // This handles cases where the PFX is a CA certificate - if (isPfxCertificate) + // Compare certificate thumbprints (most reliable method) + bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); + + if (!thumbprintMatch) { - try + // For PFX certificates, also check if the server cert was issued by our trusted cert + // This handles cases where the PFX is a CA certificate + if (isPfxCertificate) { - var certChain = new X509Chain(); - certChain.ChainPolicy.ExtraStore.Add(trustedCert); - certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; - - bool chainIsValid = certChain.Build(cert); - return chainIsValid && certChain.ChainElements - .Cast() - .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + try + { + var certChain = new X509Chain(); + certChain.ChainPolicy.ExtraStore.Add(trustedCert); + certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; + + bool chainIsValid = certChain.Build(cert); + return chainIsValid && certChain.ChainElements + .Cast() + .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + } + catch + { + // Fallback to subject and issuer comparison + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } } - catch + else { - // Fallback to subject and issuer comparison + // For standard certificates, try comparing by subject and issuer as fallback bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); return subjectMatch && issuerMatch; } } - else - { - // For standard certificates, try comparing by subject and issuer as fallback - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; - } + + return thumbprintMatch; } - - return thumbprintMatch; } catch (Exception ex) { From 1752de40abf0ec79d92953fe3e04c3f2b2ea4297 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 10 Nov 2025 13:10:45 -0800 Subject: [PATCH 05/31] Dispose properly of the X509Chain --- .../CosmosExtensionServices.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 6efb7064..5500441b 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -155,15 +155,17 @@ private static Func CreateCe { try { - var certChain = new X509Chain(); - certChain.ChainPolicy.ExtraStore.Add(trustedCert); - certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; - - bool chainIsValid = certChain.Build(cert); - return chainIsValid && certChain.ChainElements - .Cast() - .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + using (var certChain = new X509Chain()) + { + certChain.ChainPolicy.ExtraStore.Add(trustedCert); + certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; + + bool chainIsValid = certChain.Build(cert); + return chainIsValid && certChain.ChainElements + .Cast() + .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + } } catch { From 74671a2cf33f1e3faddb6b41785205ccf3688f31 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 10 Nov 2025 13:16:57 -0800 Subject: [PATCH 06/31] Updates to log on failure --- .../CosmosExtensionServices.cs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 5500441b..b7b67149 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -10,6 +10,7 @@ using Azure.Security.KeyVault.Keys.Cryptography; using System.Net; using System.Net.Security; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; namespace Cosmos.DataTransfer.CosmosExtension @@ -167,13 +168,44 @@ private static Func CreateCe .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); } } - catch + catch (ArgumentException ex) { + // Certificate is invalid or null - log and fallback + Console.Error.WriteLine($"Certificate chain validation failed - Invalid certificate: {ex.Message}"); + // Fallback to subject and issuer comparison bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); return subjectMatch && issuerMatch; } + catch (CryptographicException ex) + { + // Certificate is unreadable - log and fallback + Console.Error.WriteLine($"Certificate chain validation failed - Certificate unreadable: {ex.Message}"); + + // Fallback to subject and issuer comparison + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } + catch (InvalidOperationException ex) + { + // Chain elements collection is in invalid state - log and fallback + Console.Error.WriteLine($"Certificate chain validation failed - Invalid chain state: {ex.Message}"); + + // Fallback to subject and issuer comparison + bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); + bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + return subjectMatch && issuerMatch; + } + catch (Exception ex) + { + // Unexpected exception - log with full details and fail validation for security + Console.Error.WriteLine($"Certificate chain validation failed - Unexpected error: {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}"); + + // For unknown exceptions, fail validation for security rather than fallback + return false; + } } else { From d7a2c0c35f01640162478503feee11682b10df85 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 23 Nov 2025 09:16:09 -0800 Subject: [PATCH 07/31] Add logger to certificates as supposed to writeline --- .../CosmosDataSinkExtension.cs | 2 +- .../CosmosExtensionServices.cs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs index 4fe91639..5ca55b90 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs @@ -139,7 +139,7 @@ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfigurati var settings = config.Get(); settings.Validate(); - var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, dataSource.DisplayName); + var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, logger, dataSource.DisplayName); Container container; if (settings!.UseRbacAuth) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index b7b67149..94cec078 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -17,7 +17,7 @@ namespace Cosmos.DataTransfer.CosmosExtension { public static class CosmosExtensionServices { - public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, string? sourceDisplayName = null) + public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, ILogger logger, string? sourceDisplayName = null) { string userAgentString = CreateUserAgentString(displayName, sourceDisplayName); @@ -46,7 +46,7 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp // Configure custom certificate validation if (settings.DisableSslValidation || !string.IsNullOrEmpty(settings.CertificatePath)) { - clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(settings); + clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(settings, logger); } CosmosClient? cosmosClient; @@ -121,7 +121,7 @@ public static async Task VerifyContainerAccess(Container? container, string? nam } } - private static Func CreateCertificateValidationCallback(CosmosSettingsBase settings) + private static Func CreateCertificateValidationCallback(CosmosSettingsBase settings, ILogger logger) { return (cert, chain, errors) => { @@ -171,7 +171,7 @@ private static Func CreateCe catch (ArgumentException ex) { // Certificate is invalid or null - log and fallback - Console.Error.WriteLine($"Certificate chain validation failed - Invalid certificate: {ex.Message}"); + logger.LogError($"Certificate chain validation failed - Invalid certificate: {ex.Message}"); // Fallback to subject and issuer comparison bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); @@ -181,7 +181,7 @@ private static Func CreateCe catch (CryptographicException ex) { // Certificate is unreadable - log and fallback - Console.Error.WriteLine($"Certificate chain validation failed - Certificate unreadable: {ex.Message}"); + logger.LogError($"Certificate chain validation failed - Certificate unreadable: {ex.Message}"); // Fallback to subject and issuer comparison bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); @@ -191,7 +191,7 @@ private static Func CreateCe catch (InvalidOperationException ex) { // Chain elements collection is in invalid state - log and fallback - Console.Error.WriteLine($"Certificate chain validation failed - Invalid chain state: {ex.Message}"); + logger.LogError($"Certificate chain validation failed - Invalid chain state: {ex.Message}"); // Fallback to subject and issuer comparison bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); @@ -201,7 +201,7 @@ private static Func CreateCe catch (Exception ex) { // Unexpected exception - log with full details and fail validation for security - Console.Error.WriteLine($"Certificate chain validation failed - Unexpected error: {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}"); + logger.LogError($"Certificate chain validation failed - Unexpected error: {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}"); // For unknown exceptions, fail validation for security rather than fallback return false; @@ -222,7 +222,7 @@ private static Func CreateCe catch (Exception ex) { // Log the exception details to help diagnose certificate loading issues - Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + logger.LogError($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); // If we can't load the certificate, fail validation return false; } From 51c5ac27fb2b50a12bd9085d84b4cf11b21c7dfc Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 23 Nov 2025 09:20:03 -0800 Subject: [PATCH 08/31] remove duplicate cert check --- .../CosmosExtensionServices.cs | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 94cec078..9afbc924 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -171,32 +171,26 @@ private static Func CreateCe catch (ArgumentException ex) { // Certificate is invalid or null - log and fallback - logger.LogError($"Certificate chain validation failed - Invalid certificate: {ex.Message}"); + logger.LogError("Certificate chain validation failed - Invalid certificate: {message}", ex.Message); // Fallback to subject and issuer comparison - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; + return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); } catch (CryptographicException ex) { // Certificate is unreadable - log and fallback - logger.LogError($"Certificate chain validation failed - Certificate unreadable: {ex.Message}"); + logger.LogError("Certificate chain validation failed - Certificate unreadable: {message}", ex.Message); // Fallback to subject and issuer comparison - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; + return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); } catch (InvalidOperationException ex) { // Chain elements collection is in invalid state - log and fallback - logger.LogError($"Certificate chain validation failed - Invalid chain state: {ex.Message}"); + logger.LogError("Certificate chain validation failed - Invalid chain state: {message}", ex.Message); // Fallback to subject and issuer comparison - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; + return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); } catch (Exception ex) { @@ -210,9 +204,7 @@ private static Func CreateCe else { // For standard certificates, try comparing by subject and issuer as fallback - bool subjectMatch = cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase); - bool issuerMatch = cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - return subjectMatch && issuerMatch; + return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); } } @@ -238,5 +230,11 @@ private static bool IsPfxCertificate(string certificatePath) var extension = Path.GetExtension(certificatePath).ToLowerInvariant(); return extension == ".pfx" || extension == ".p12"; } + + private static bool ValidateCertificateBySubjectAndIssuer(X509Certificate2 cert, X509Certificate2 trustedCert) + { + return cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase) + && cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); + } } } \ No newline at end of file From c4c6a20ec9cfe00accf8368a1908df49cd147ae2 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 23 Nov 2025 22:42:54 -0800 Subject: [PATCH 09/31] PR feedback --- .../CosmosExtensionServices.cs | 52 ++++++++----------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 9afbc924..02396b9e 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -128,6 +128,7 @@ private static Func CreateCe // If SSL validation is disabled (development/emulator only) if (settings.DisableSslValidation) { + logger.LogWarning("SSL validation is disabled. This should only be used for development/emulator scenarios."); return true; } @@ -163,48 +164,45 @@ private static Func CreateCe certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; bool chainIsValid = certChain.Build(cert); - return chainIsValid && certChain.ChainElements + bool issuedByTrustedCert = chainIsValid && certChain.ChainElements .Cast() .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); + + if (!issuedByTrustedCert) + { + logger.LogWarning("Certificate chain validation failed. Server certificate was not issued by the trusted CA certificate."); + } + + return issuedByTrustedCert; } } catch (ArgumentException ex) { - // Certificate is invalid or null - log and fallback - logger.LogError("Certificate chain validation failed - Invalid certificate: {message}", ex.Message); - - // Fallback to subject and issuer comparison - return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); + logger.LogError(ex, "Certificate chain validation failed - Invalid certificate"); + return false; } catch (CryptographicException ex) { - // Certificate is unreadable - log and fallback - logger.LogError("Certificate chain validation failed - Certificate unreadable: {message}", ex.Message); - - // Fallback to subject and issuer comparison - return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); + logger.LogError(ex, "Certificate chain validation failed - Certificate unreadable"); + return false; } catch (InvalidOperationException ex) { - // Chain elements collection is in invalid state - log and fallback - logger.LogError("Certificate chain validation failed - Invalid chain state: {message}", ex.Message); - - // Fallback to subject and issuer comparison - return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); + logger.LogError(ex, "Certificate chain validation failed - Invalid chain state"); + return false; } catch (Exception ex) { - // Unexpected exception - log with full details and fail validation for security - logger.LogError($"Certificate chain validation failed - Unexpected error: {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}"); - - // For unknown exceptions, fail validation for security rather than fallback + logger.LogError(ex, "Certificate chain validation failed - Unexpected error"); return false; } } else { - // For standard certificates, try comparing by subject and issuer as fallback - return ValidateCertificateBySubjectAndIssuer(cert, trustedCert); + // For standard certificates, thumbprint must match exactly + logger.LogWarning("Certificate thumbprint mismatch. Expected: {ExpectedThumbprint}, Got: {ActualThumbprint}", + trustedCert.Thumbprint, cert.Thumbprint); + return false; } } @@ -213,9 +211,7 @@ private static Func CreateCe } catch (Exception ex) { - // Log the exception details to help diagnose certificate loading issues - logger.LogError($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); - // If we can't load the certificate, fail validation + logger.LogError(ex, "Certificate loading failed from path: {CertificatePath}", settings.CertificatePath); return false; } } @@ -230,11 +226,5 @@ private static bool IsPfxCertificate(string certificatePath) var extension = Path.GetExtension(certificatePath).ToLowerInvariant(); return extension == ".pfx" || extension == ".p12"; } - - private static bool ValidateCertificateBySubjectAndIssuer(X509Certificate2 cert, X509Certificate2 trustedCert) - { - return cert.Subject.Equals(trustedCert.Subject, StringComparison.OrdinalIgnoreCase) - && cert.Issuer.Equals(trustedCert.Issuer, StringComparison.OrdinalIgnoreCase); - } } } \ No newline at end of file From c2b8dc89bb74ae7c1ca2901f9f8b9fd5cdac768b Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 06:35:43 -0800 Subject: [PATCH 10/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 02396b9e..a6b09566 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -207,11 +207,25 @@ private static Func CreateCe } return thumbprintMatch; - } + catch (CryptographicException ex) + { + // Log the exception details to help diagnose certificate loading issues + Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + // If we can't load the certificate, fail validation + return false; + } + catch (IOException ex) + { + // Log the exception details to help diagnose certificate loading issues + Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + // If we can't load the certificate, fail validation + return false; } - catch (Exception ex) + catch (UnauthorizedAccessException ex) { - logger.LogError(ex, "Certificate loading failed from path: {CertificatePath}", settings.CertificatePath); + // Log the exception details to help diagnose certificate loading issues + Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + // If we can't load the certificate, fail validation return false; } } From 24205f399341ed2ce58b42f5b14311cd94b209de Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 06:37:52 -0800 Subject: [PATCH 11/31] add null check --- .../CosmosExtensionServices.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index a6b09566..dc6c02e5 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -132,6 +132,12 @@ private static Func CreateCe return true; } + if (cert == null) + { + logger.LogError("Server certificate is null during SSL validation."); + return false; + } + // If a certificate path is specified if (!string.IsNullOrEmpty(settings.CertificatePath)) { From fbae2e88f5b06548459d049bfcd9b9798ba3cadd Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 06:39:50 -0800 Subject: [PATCH 12/31] add date check to cert --- .../CosmosExtensionServices.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index dc6c02e5..a8bd9fd6 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -138,6 +138,12 @@ private static Func CreateCe return false; } + if (cert.NotAfter < DateTime.UtcNow || cert.NotBefore > DateTime.UtcNow) + { + logger.LogError("Certificate expired or not yet valid. Valid from {NotBefore} to {NotAfter}", cert.NotBefore, cert.NotAfter); + return false; + } + // If a certificate path is specified if (!string.IsNullOrEmpty(settings.CertificatePath)) { From d11b2e335654ac79ceaf389ed43e7accc50b131b Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 07:33:10 -0800 Subject: [PATCH 13/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index a8bd9fd6..bf9b96e8 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -222,21 +222,21 @@ private static Func CreateCe catch (CryptographicException ex) { // Log the exception details to help diagnose certificate loading issues - Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + logger.LogError(ex, "Certificate loading failed"); // If we can't load the certificate, fail validation return false; } catch (IOException ex) { // Log the exception details to help diagnose certificate loading issues - Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + logger.LogError(ex, "Certificate loading failed"); // If we can't load the certificate, fail validation return false; } catch (UnauthorizedAccessException ex) { // Log the exception details to help diagnose certificate loading issues - Console.Error.WriteLine($"Certificate loading failed: {ex.Message}\n{ex.StackTrace}"); + logger.LogError(ex, "Certificate loading failed"); // If we can't load the certificate, fail validation return false; } From 1c98b72f1aa8a874ada47dfb9b534e994f6e40dc Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 07:33:27 -0800 Subject: [PATCH 14/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index bf9b96e8..70e74589 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -219,6 +219,7 @@ private static Func CreateCe } return thumbprintMatch; + } catch (CryptographicException ex) { // Log the exception details to help diagnose certificate loading issues From 424c5f6c068579289c50ffbc6e4b600216ca25be Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 10:03:37 -0800 Subject: [PATCH 15/31] fix build issues --- .../CosmosDataSourceExtension.cs | 2 +- .../CosmosExtensionServices.cs | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs index 5a13ff87..03eae830 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs @@ -18,7 +18,7 @@ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogge var settings = config.Get(); settings.Validate(); - var client = CosmosExtensionServices.CreateClient(settings!, DisplayName); + var client = CosmosExtensionServices.CreateClient(settings!, DisplayName, logger); Container container; diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index a8bd9fd6..ae8f71fe 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -150,7 +150,7 @@ private static Func CreateCe try { bool isPfxCertificate = IsPfxCertificate(settings.CertificatePath); - + // Load certificate based on type and password availability using (X509Certificate2 trustedCert = isPfxCertificate ? (string.IsNullOrEmpty(settings.CertificatePassword) @@ -160,7 +160,7 @@ private static Func CreateCe { // Compare certificate thumbprints (most reliable method) bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); - + if (!thumbprintMatch) { // For PFX certificates, also check if the server cert was issued by our trusted cert @@ -174,17 +174,17 @@ private static Func CreateCe certChain.ChainPolicy.ExtraStore.Add(trustedCert); certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; - + bool chainIsValid = certChain.Build(cert); bool issuedByTrustedCert = chainIsValid && certChain.ChainElements .Cast() .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); - + if (!issuedByTrustedCert) { logger.LogWarning("Certificate chain validation failed. Server certificate was not issued by the trusted CA certificate."); } - + return issuedByTrustedCert; } } @@ -212,13 +212,15 @@ private static Func CreateCe else { // For standard certificates, thumbprint must match exactly - logger.LogWarning("Certificate thumbprint mismatch. Expected: {ExpectedThumbprint}, Got: {ActualThumbprint}", + logger.LogWarning("Certificate thumbprint mismatch. Expected: {ExpectedThumbprint}, Got: {ActualThumbprint}", trustedCert.Thumbprint, cert.Thumbprint); return false; } } - + return thumbprintMatch; + } + } catch (CryptographicException ex) { // Log the exception details to help diagnose certificate loading issues From 3c2af0032f8195cbd1f0fc26525dcded3d703a3f Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 10:17:33 -0800 Subject: [PATCH 16/31] Update Extensions/Cosmos/CERTIFICATE_EXAMPLES.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Extensions/Cosmos/CERTIFICATE_EXAMPLES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md index 4f595c93..1a13ffd2 100644 --- a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md +++ b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md @@ -178,7 +178,7 @@ Migrating from staging to production with different PFX certificates: 3. **Certificate file formats supported** - `.cer` (binary or base64 encoded) - - `.crt` (binary or base64 encoded) + - `.crt` (binary or base64 encoded) - `.pem` (base64 encoded) - `.pfx` (PKCS#12, with or without password) - `.p12` (PKCS#12, with or without password) From e5965c3ead003adc34b1a1c63049b6f9c743fb8d Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 24 Nov 2025 10:18:37 -0800 Subject: [PATCH 17/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index db2031d2..c2799ecb 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -41,7 +41,7 @@ public abstract class CosmosSettingsBase : IValidatableObject public string? CertificatePassword { get; set; } /// - /// Disable SSL certificate validation. + /// Disable SSL certificate validation. /// WARNING: Only use this for development with the emulator. Never use in production. /// public bool DisableSslValidation { get; set; } = false; From c8170dfb9dbba18bc20486703cc9b4923c34981f Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 05:48:40 -0800 Subject: [PATCH 18/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 494053bb..c1d74eeb 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -172,9 +172,15 @@ private static Func CreateCe using (var certChain = new X509Chain()) { certChain.ChainPolicy.ExtraStore.Add(trustedCert); - certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + certChain.ChainPolicy.RevocationMode = disableRevocationCheck + ? X509RevocationMode.NoCheck + : X509RevocationMode.Online; certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; + if (disableRevocationCheck) + { + logger.LogWarning("Certificate revocation checking is disabled. This may allow revoked certificates to be accepted."); + } bool chainIsValid = certChain.Build(cert); bool issuedByTrustedCert = chainIsValid && certChain.ChainElements .Cast() From a95d075dd63628145e0dd08abe8ddf4fa0045104 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 05:51:24 -0800 Subject: [PATCH 19/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index c2799ecb..e7c7ad9f 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -64,6 +64,10 @@ public virtual IEnumerable Validate(ValidationContext validati { yield return new ValidationResult($"CertificatePath file does not exist: {CertificatePath}", new[] { nameof(CertificatePath) }); } + if (!string.IsNullOrEmpty(CertificatePassword) && string.IsNullOrEmpty(CertificatePath)) + { + yield return new ValidationResult("CertificatePassword is specified but CertificatePath is missing", new[] { nameof(CertificatePassword) }); + } } } } \ No newline at end of file From d2cfeccf2ac1a05bc286a260ddb9b219c061af6b Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 08:35:04 -0800 Subject: [PATCH 20/31] Remove all the complexity and simplify to just the disabledsetting --- .../CertificateConfigurationTests.cs | 193 ++++-------------- .../CosmosExtensionServices.cs | 153 ++------------ .../CosmosSettingsBase.cs | 25 +-- Extensions/Cosmos/README.md | 45 +--- 4 files changed, 66 insertions(+), 350 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index 0efc94bf..b6612f3c 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -1,5 +1,10 @@ using Cosmos.DataTransfer.CosmosExtension; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; using System.ComponentModel.DataAnnotations; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; namespace Cosmos.DataTransfer.CosmosExtension.UnitTests { @@ -7,203 +12,83 @@ namespace Cosmos.DataTransfer.CosmosExtension.UnitTests public class CertificateConfigurationTests { [TestMethod] - public void CosmosSettingsBase_WithValidCertificatePath_ShouldValidateSuccessfully() - { - // Arrange - Create a temp certificate file for testing - var tempCertPath = Path.GetTempFileName(); - File.WriteAllText(tempCertPath, "dummy cert content"); // Not a real cert, but file exists - - try - { - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", - Database = "testDb", - Container = "testContainer", - CertificatePath = tempCertPath, - DisableSslValidation = false - }; - - // Act - var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - - // Assert - Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); - } - finally - { - // Cleanup - if (File.Exists(tempCertPath)) - File.Delete(tempCertPath); - } - } - - [TestMethod] - public void CosmosSettingsBase_WithInvalidCertificatePath_ShouldReturnValidationError() + public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessfully() { - // Arrange var settings = new TestableCosmosSettings { ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CertificatePath = "C:\\nonexistent\\path\\cert.cer", - DisableSslValidation = false + DisableSslValidation = true }; - // Act var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - // Assert - Assert.AreEqual(1, validationResults.Count); - Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CertificatePath file does not exist")); + Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); } [TestMethod] - public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessfully() + public void CertificateValidation_WithDisableSslValidation_ShouldBypassAllChecks() { - // Arrange var settings = new TestableCosmosSettings { ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CertificatePath = null, DisableSslValidation = true }; - // Act - var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - - // Assert - Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); - } - - [TestMethod] - public void CosmosSettingsBase_WithValidPfxCertificatePath_ShouldValidateSuccessfully() - { - // Arrange - Create a temp PFX file for testing - var tempPfxPath = Path.GetTempFileName(); - File.WriteAllText(tempPfxPath, "dummy pfx content"); // Not a real PFX, but file exists - - try - { - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", - Database = "testDb", - Container = "testContainer", - CertificatePath = tempPfxPath, - CertificatePassword = "testPassword" - }; - - // Act - var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - - // Assert - Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); - } - finally - { - // Cleanup - if (File.Exists(tempPfxPath)) - File.Delete(tempPfxPath); - } + var mockLogger = new Mock(); + var callback = CreateCertificateValidationCallbackForTesting(mockLogger.Object); + + using var cert = CreateDummyCertificate(); + var isValid = callback(cert, new X509Chain(), System.Net.Security.SslPolicyErrors.RemoteCertificateChainErrors); + + Assert.IsTrue(isValid, "Callback should always return true, bypassing all SSL errors"); + mockLogger.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((o, t) => o.ToString()!.Contains("SSL certificate validation is DISABLED")), + It.IsAny(), + It.IsAny>()), + Times.Once); } [TestMethod] - public void CosmosSettingsBase_WithInvalidPfxCertificatePath_ShouldReturnValidationError() + public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultValidation() { - // Arrange var settings = new TestableCosmosSettings { - ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", Database = "testDb", Container = "testContainer", - CertificatePath = "C:\\nonexistent\\path\\cert.pfx", - CertificatePassword = "testPassword" + DisableSslValidation = false }; - // Act var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - // Assert - Assert.AreEqual(1, validationResults.Count); - Assert.IsTrue(validationResults[0].ErrorMessage!.Contains("CertificatePath file does not exist")); + Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); } - [TestMethod] - public void CosmosSettingsBase_WithPfxCertificateWithoutPassword_ShouldValidateSuccessfully() + private static Func CreateCertificateValidationCallbackForTesting( + ILogger logger) { - // Arrange - Create a temp PFX file for testing - var tempPfxPath = Path.GetTempFileName(); - File.WriteAllText(tempPfxPath, "dummy pfx content"); // Not a real PFX, but file exists - - try - { - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://enterprise.cosmos.com:8081/;AccountKey=...", - Database = "testDb", - Container = "testContainer", - CertificatePath = tempPfxPath, - CertificatePassword = null // No password specified - }; + var method = typeof(CosmosExtensionServices).GetMethod( + "CreateCertificateValidationCallback", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); - // Act - var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - - // Assert - Assert.AreEqual(0, validationResults.Count, "Should have no validation errors for PFX without password"); - } - finally - { - // Cleanup - if (File.Exists(tempPfxPath)) - File.Delete(tempPfxPath); - } + var callback = method!.Invoke(null, new object[] { logger }) as Func; + return callback!; } - [TestMethod] - public void CosmosSettingsBase_WithCertificateTypeDetection_ShouldSupportMultipleFormats() + private static X509Certificate2 CreateDummyCertificate() { - // Arrange - Test that the unified CertificatePath supports different file extensions - var extensions = new[] { ".cer", ".crt", ".pem", ".pfx", ".p12" }; - - foreach (var extension in extensions) - { - var tempCertPath = Path.GetTempFileName(); - var certPathWithExtension = Path.ChangeExtension(tempCertPath, extension); - File.Move(tempCertPath, certPathWithExtension); - File.WriteAllText(certPathWithExtension, "dummy cert content"); - - try - { - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", - Database = "testDb", - Container = "testContainer", - CertificatePath = certPathWithExtension, - CertificatePassword = (extension == ".pfx" || extension == ".p12") ? "password" : null - }; - - // Act - var validationResults = settings.Validate(new ValidationContext(settings)).ToList(); - - // Assert - Assert.AreEqual(0, validationResults.Count, $"Should have no validation errors for {extension} files"); - } - finally - { - // Cleanup - if (File.Exists(certPathWithExtension)) - File.Delete(certPathWithExtension); - } - } + using var rsa = RSA.Create(2048); + var certRequest = new CertificateRequest("CN=TestCert", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + return certRequest.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(365)); } - // Test implementation of CosmosSettingsBase for testing private class TestableCosmosSettings : CosmosSettingsBase { } diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 494053bb..edd25714 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -43,10 +43,10 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp clientOptions.WebProxy = new WebProxy(settings.WebProxy); } - // Configure custom certificate validation - if (settings.DisableSslValidation || !string.IsNullOrEmpty(settings.CertificatePath)) + // Configure custom certificate validation for emulator scenarios + if (settings.DisableSslValidation) { - clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(settings, logger); + clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(logger); } CosmosClient? cosmosClient; @@ -121,138 +121,29 @@ public static async Task VerifyContainerAccess(Container? container, string? nam } } - private static Func CreateCertificateValidationCallback(CosmosSettingsBase settings, ILogger logger) + /// + /// Creates a custom SSL certificate validation callback that bypasses all certificate checks. + /// This is intended solely for use with the Cosmos DB vNext emulator in development environments. + /// + /// Logger for diagnostic information + /// A callback function that always accepts server certificates + /// + /// WARNING: This callback disables all SSL/TLS security checks including: + /// - Certificate chain validation + /// - Certificate revocation checking + /// - Trusted CA verification + /// - Hostname verification + /// - Certificate expiration checking + /// + /// Never use this in production environments as it makes connections vulnerable to man-in-the-middle attacks. + /// + private static Func CreateCertificateValidationCallback(ILogger logger) { return (cert, chain, errors) => { - // If SSL validation is disabled (development/emulator only) - if (settings.DisableSslValidation) - { - logger.LogWarning("SSL validation is disabled. This should only be used for development/emulator scenarios."); - return true; - } - - if (cert == null) - { - logger.LogError("Server certificate is null during SSL validation."); - return false; - } - - if (cert.NotAfter < DateTime.UtcNow || cert.NotBefore > DateTime.UtcNow) - { - logger.LogError("Certificate expired or not yet valid. Valid from {NotBefore} to {NotAfter}", cert.NotBefore, cert.NotAfter); - return false; - } - - // If a certificate path is specified - if (!string.IsNullOrEmpty(settings.CertificatePath)) - { - try - { - bool isPfxCertificate = IsPfxCertificate(settings.CertificatePath); - - // Load certificate based on type and password availability - using (X509Certificate2 trustedCert = isPfxCertificate - ? (string.IsNullOrEmpty(settings.CertificatePassword) - ? new X509Certificate2(settings.CertificatePath) - : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword)) - : new X509Certificate2(settings.CertificatePath)) - { - // Compare certificate thumbprints (most reliable method) - bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); - - if (!thumbprintMatch) - { - // For PFX certificates, also check if the server cert was issued by our trusted cert - // This handles cases where the PFX is a CA certificate - if (isPfxCertificate) - { - try - { - using (var certChain = new X509Chain()) - { - certChain.ChainPolicy.ExtraStore.Add(trustedCert); - certChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; - - bool chainIsValid = certChain.Build(cert); - bool issuedByTrustedCert = chainIsValid && certChain.ChainElements - .Cast() - .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); - - if (!issuedByTrustedCert) - { - logger.LogWarning("Certificate chain validation failed. Server certificate was not issued by the trusted CA certificate."); - } - - return issuedByTrustedCert; - } - } - catch (ArgumentException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Invalid certificate"); - return false; - } - catch (CryptographicException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Certificate unreadable"); - return false; - } - catch (InvalidOperationException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Invalid chain state"); - return false; - } - catch (Exception ex) - { - logger.LogError(ex, "Certificate chain validation failed - Unexpected error"); - return false; - } - } - else - { - // For standard certificates, thumbprint must match exactly - logger.LogWarning("Certificate thumbprint mismatch. Expected: {ExpectedThumbprint}, Got: {ActualThumbprint}", - trustedCert.Thumbprint, cert.Thumbprint); - return false; - } - } - - return thumbprintMatch; - } - } - catch (CryptographicException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - catch (IOException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - catch (UnauthorizedAccessException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - } - - // Default validation - accept only if no SSL policy errors - return errors == SslPolicyErrors.None; + logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); + return true; }; } - - private static bool IsPfxCertificate(string certificatePath) - { - var extension = Path.GetExtension(certificatePath).ToLowerInvariant(); - return extension == ".pfx" || extension == ".p12"; - } } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index c2799ecb..5a4c2bc0 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -25,24 +25,9 @@ public abstract class CosmosSettingsBase : IValidatableObject public bool LimitToEndpoint { get; set; } = false; /// - /// Path to a certificate file for SSL validation and client authentication. - /// Supports multiple formats: - /// - .cer, .crt, .pem files for basic SSL validation - /// - .pfx, .p12 files for client authentication (enterprise scenarios) - /// For PFX/P12 files, use CertificatePassword if the file is password-protected. - /// - public string? CertificatePath { get; set; } - - /// - /// Password for PFX/P12 certificate files when they are password-protected. - /// Only used when CertificatePath points to a .pfx or .p12 file. - /// WARNING: Store this securely and avoid hardcoding in configuration files. - /// - public string? CertificatePassword { get; set; } - - /// - /// Disable SSL certificate validation. - /// WARNING: Only use this for development with the emulator. Never use in production. + /// Disables SSL certificate validation for the Cosmos DB connection. + /// This is intended for use with the Cosmos DB vNext emulator or local development environments. + /// WARNING: Never use this option in production as it disables critical security checks. /// public bool DisableSslValidation { get; set; } = false; @@ -60,10 +45,6 @@ public virtual IEnumerable Validate(ValidationContext validati { yield return new ValidationResult("InitClientEncryption can only be used when UseRbacAuth is true", new[] { nameof(InitClientEncryption) }); } - if (!string.IsNullOrEmpty(CertificatePath) && !File.Exists(CertificatePath)) - { - yield return new ValidationResult($"CertificatePath file does not exist: {CertificatePath}", new[] { nameof(CertificatePath) }); - } } } } \ No newline at end of file diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index ab93896f..951761cb 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -70,41 +70,6 @@ For emulator development with SSL validation disabled: } ``` -For custom certificate validation: - -```json -{ - "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.domain.com:8081/;AccountKey=...", - "Database":"myDb", - "Container":"myContainer", - "CertificatePath": "C:\\certs\\cosmos-custom.cer" -} -``` - -For enterprise PFX certificate with password: - -```json -{ - "ConnectionString": "AccountEndpoint=https://enterprise-cosmos.company.com:8081/;AccountKey=...", - "Database":"EnterpriseDB", - "Container":"SecureContainer", - "CertificatePath": "C:\\enterprise-certs\\cosmos-client.pfx", - "CertificatePassword": "SecureP@ssw0rd!" -} -``` - -For enterprise PFX certificate without password: - -```json -{ - "UseRbacAuth": true, - "AccountEndpoint": "https://enterprise-cosmos.company.com:443/", - "Database":"EnterpriseDB", - "Container":"SecureContainer", - "CertificatePath": "C:\\enterprise-certs\\cosmos-client.p12" -} -``` - ### Sink Settings #### **Partition Key Settings** @@ -146,13 +111,8 @@ For enterprise PFX certificate without password: #### **SSL/Certificate Settings** -- **`CertificatePath`**: Optional. Path to a certificate file for SSL validation and client authentication. Supports multiple formats: - - `.cer`, `.crt`, `.pem` files for basic SSL validation - - `.pfx`, `.p12` files for client authentication (enterprise scenarios) - For PFX/P12 files, use `CertificatePassword` if the file is password-protected. -- **`CertificatePassword`**: Optional. Password for PFX/P12 certificate files when they are password-protected. Only used when `CertificatePath` points to a `.pfx` or `.p12` file. ⚠️ Store securely and avoid hardcoding in configuration files. -- **`DisableSslValidation`**: Optional, defaults to `false`. Disables SSL certificate validation entirely. - - **⚠️ WARNING**: Only use this for development with the emulator. Never use in production environments as it makes connections vulnerable to man-in-the-middle attacks. +- **`DisableSslValidation`**: Optional, defaults to `false`. Disables SSL certificate validation for development/emulator scenarios. + - **⚠️ WARNING**: Only use this for development purposes. Never use in production environments as it disables critical security checks and makes connections vulnerable to man-in-the-middle attacks. #### **Serverless Account** @@ -189,4 +149,3 @@ For enterprise PFX certificate without password: "InitClientEncryption": false, "LimitToEndpoint": false } -``` From 7387b3a2a9293c93d46fe2e7e4865caa6702cb22 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 08:39:54 -0800 Subject: [PATCH 21/31] missed merge --- .../CosmosExtensionServices.cs | 130 ------------------ 1 file changed, 130 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 1e82eda3..edd25714 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -141,138 +141,8 @@ private static Func CreateCe { return (cert, chain, errors) => { -<<<<<<< HEAD logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); return true; -======= - // If SSL validation is disabled (development/emulator only) - if (settings.DisableSslValidation) - { - logger.LogWarning("SSL validation is disabled. This should only be used for development/emulator scenarios."); - return true; - } - - if (cert == null) - { - logger.LogError("Server certificate is null during SSL validation."); - return false; - } - - if (cert.NotAfter < DateTime.UtcNow || cert.NotBefore > DateTime.UtcNow) - { - logger.LogError("Certificate expired or not yet valid. Valid from {NotBefore} to {NotAfter}", cert.NotBefore, cert.NotAfter); - return false; - } - - // If a certificate path is specified - if (!string.IsNullOrEmpty(settings.CertificatePath)) - { - try - { - bool isPfxCertificate = IsPfxCertificate(settings.CertificatePath); - - // Load certificate based on type and password availability - using (X509Certificate2 trustedCert = isPfxCertificate - ? (string.IsNullOrEmpty(settings.CertificatePassword) - ? new X509Certificate2(settings.CertificatePath) - : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword)) - : new X509Certificate2(settings.CertificatePath)) - { - // Compare certificate thumbprints (most reliable method) - bool thumbprintMatch = cert.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase); - - if (!thumbprintMatch) - { - // For PFX certificates, also check if the server cert was issued by our trusted cert - // This handles cases where the PFX is a CA certificate - if (isPfxCertificate) - { - try - { - using (var certChain = new X509Chain()) - { - certChain.ChainPolicy.ExtraStore.Add(trustedCert); - certChain.ChainPolicy.RevocationMode = disableRevocationCheck - ? X509RevocationMode.NoCheck - : X509RevocationMode.Online; - certChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority; - - if (disableRevocationCheck) - { - logger.LogWarning("Certificate revocation checking is disabled. This may allow revoked certificates to be accepted."); - } - bool chainIsValid = certChain.Build(cert); - bool issuedByTrustedCert = chainIsValid && certChain.ChainElements - .Cast() - .Any(element => element.Certificate.Thumbprint.Equals(trustedCert.Thumbprint, StringComparison.OrdinalIgnoreCase)); - - if (!issuedByTrustedCert) - { - logger.LogWarning("Certificate chain validation failed. Server certificate was not issued by the trusted CA certificate."); - } - - return issuedByTrustedCert; - } - } - catch (ArgumentException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Invalid certificate"); - return false; - } - catch (CryptographicException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Certificate unreadable"); - return false; - } - catch (InvalidOperationException ex) - { - logger.LogError(ex, "Certificate chain validation failed - Invalid chain state"); - return false; - } - catch (Exception ex) - { - logger.LogError(ex, "Certificate chain validation failed - Unexpected error"); - return false; - } - } - else - { - // For standard certificates, thumbprint must match exactly - logger.LogWarning("Certificate thumbprint mismatch. Expected: {ExpectedThumbprint}, Got: {ActualThumbprint}", - trustedCert.Thumbprint, cert.Thumbprint); - return false; - } - } - - return thumbprintMatch; - } - } - catch (CryptographicException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - catch (IOException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - catch (UnauthorizedAccessException ex) - { - // Log the exception details to help diagnose certificate loading issues - logger.LogError(ex, "Certificate loading failed"); - // If we can't load the certificate, fail validation - return false; - } - } - - // Default validation - accept only if no SSL policy errors - return errors == SslPolicyErrors.None; ->>>>>>> a95d075dd63628145e0dd08abe8ddf4fa0045104 }; } } From 84694862cf9cd51b8ce487b8a13fabf555ddfb18 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 08:43:36 -0800 Subject: [PATCH 22/31] remove examples since we don't support any of them --- Extensions/Cosmos/CERTIFICATE_EXAMPLES.md | 210 ---------------------- 1 file changed, 210 deletions(-) delete mode 100644 Extensions/Cosmos/CERTIFICATE_EXAMPLES.md diff --git a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md b/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md deleted file mode 100644 index 1a13ffd2..00000000 --- a/Extensions/Cosmos/CERTIFICATE_EXAMPLES.md +++ /dev/null @@ -1,210 +0,0 @@ -# Custom Certificate Configuration Examples - -This file demonstrates how to use the new custom certificate configuration options in the Cosmos DB data migration tool. - -## Example 1: Disable SSL Validation (Development/Emulator Only) - -For development with the Cosmos DB emulator when certificates are problematic: - -```json -{ - "Source": "json", - "Sink": "cosmos-nosql", - "SourceSettings": { - "FilePath": "C:\\data\\sample-data.json" - }, - "SinkSettings": { - "ConnectionString": "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", - "Database": "SampleDB", - "Container": "SampleContainer", - "DisableSslValidation": true - } -} -``` - -## Example 2: Custom Certificate Path - -For custom Cosmos DB instances with self-signed or corporate certificates: - -```json -{ - "Source": "json", - "Sink": "cosmos-nosql", - "SourceSettings": { - "FilePath": "C:\\data\\sample-data.json" - }, - "SinkSettings": { - "ConnectionString": "AccountEndpoint=https://my-custom-cosmos.company.com:8081/;AccountKey=...", - "Database": "ProductionDB", - "Container": "DataContainer", - "CertificatePath": "C:\\certificates\\cosmos-custom.cer", - "PartitionKeyPath": "/id" - } -} -``` - -## Example 3: RBAC with Custom Certificate - -Using RBAC authentication with a custom certificate: - -```json -{ - "Source": "json", - "Sink": "cosmos-nosql", - "SourceSettings": { - "FilePath": "C:\\data\\sample-data.json" - }, - "SinkSettings": { - "UseRbacAuth": true, - "AccountEndpoint": "https://my-cosmos-account.documents.azure.com:443/", - "EnableInteractiveCredentials": true, - "Database": "MyDatabase", - "Container": "MyContainer", - "CertificatePath": "C:\\certificates\\my-cosmos-cert.pem", - "PartitionKeyPath": "/partitionKey" - } -} -``` - -## Example 4: Cosmos-to-Cosmos Migration with Custom Certificate - -Migrating from one Cosmos DB instance to another with custom certificate validation: - -```json -{ - "Source": "cosmos-nosql", - "Sink": "cosmos-nosql", - "SourceSettings": { - "ConnectionString": "AccountEndpoint=https://source-cosmos.domain.com:8081/;AccountKey=...", - "Database": "SourceDB", - "Container": "SourceContainer", - "CertificatePath": "C:\\certificates\\source-cosmos.cer" - }, - "SinkSettings": { - "ConnectionString": "AccountEndpoint=https://dest-cosmos.domain.com:8081/;AccountKey=...", - "Database": "DestDB", - "Container": "DestContainer", - "CertificatePath": "C:\\certificates\\dest-cosmos.cer", - "PartitionKeyPath": "/id", - "WriteMode": "Upsert" - } -} -``` - -## Example 5: Enterprise PFX Certificate with Password - -For enterprise environments using PFX/P12 certificates with mutual TLS authentication: - -```json -{ - "Source": "json", - "Sink": "cosmos-nosql", - "SourceSettings": { - "FilePath": "C:\\data\\enterprise-data.json" - }, - "SinkSettings": { - "UseRbacAuth": true, - "AccountEndpoint": "https://enterprise-cosmos.company.com:443/", - "Database": "EnterpriseDB", - "Container": "SecureContainer", - "CertificatePath": "C:\\enterprise-certs\\cosmos-client.pfx", - "CertificatePassword": "SecureP@ssw0rd!", - "PartitionKeyPath": "/tenantId" - } -} -``` - -## Example 6: Enterprise PFX Certificate without Password - -For enterprise environments with unprotected PFX certificates: - -```json -{ - "Source": "cosmos-nosql", - "Sink": "json", - "SourceSettings": { - "ConnectionString": "AccountEndpoint=https://enterprise-cosmos.company.com:8081/;AccountKey=...", - "Database": "EnterpriseDB", - "Container": "AuditLogs", - "CertificatePath": "C:\\enterprise-certs\\cosmos-client.p12" - }, - "SinkSettings": { - "FilePath": "C:\\exports\\audit-logs.json" - } -} -``` - -## Example 7: Enterprise Multi-Environment Migration - -Migrating from staging to production with different PFX certificates: - -```json -{ - "Source": "cosmos-nosql", - "Sink": "cosmos-nosql", - "SourceSettings": { - "UseRbacAuth": true, - "AccountEndpoint": "https://staging-cosmos.company.com:443/", - "Database": "StagingDB", - "Container": "DataContainer", - "CertificatePath": "C:\\certs\\staging-client.pfx", - "CertificatePassword": "StagingPass123!" - }, - "SinkSettings": { - "UseRbacAuth": true, - "AccountEndpoint": "https://prod-cosmos.company.com:443/", - "Database": "ProductionDB", - "Container": "DataContainer", - "CertificatePath": "C:\\certs\\prod-client.pfx", - "CertificatePassword": "ProductionSecurePass456!", - "PartitionKeyPath": "/id", - "WriteMode": "Upsert" - } -} -``` - -## Security Notes - -⚠️ **Important Security Considerations:** - -1. **Never use `DisableSslValidation: true` in production environments** - - This disables all SSL certificate validation - - Only use for development with local emulators - -2. **Store certificate files securely** - - Use proper file permissions - - Store in protected directories - - Consider using Azure Key Vault for production scenarios - -3. **Certificate file formats supported** - - `.cer` (binary or base64 encoded) - - `.crt` (binary or base64 encoded) - - `.pem` (base64 encoded) - - `.pfx` (PKCS#12, with or without password) - - `.p12` (PKCS#12, with or without password) - -4. **PFX/P12 Certificate Security** - - Never hardcode `CertificatePassword` in configuration files - - Use environment variables or secure configuration providers for passwords - - Store PFX files with restricted access permissions (600 on Unix, appropriate ACLs on Windows) - - Consider using Azure Key Vault or other secure key management systems - - Ensure PFX files contain both certificate and private key - -5. **Validate certificate authenticity** - - Ensure certificate files come from trusted sources - - Verify certificate thumbprints when possible - - Use proper certificate authorities for production - - For PFX certificates, verify the certificate chain and expiration dates - -## Troubleshooting - -If you encounter certificate validation issues: - -1. **Verify certificate file exists and is readable** -2. **Check certificate format and encoding** -3. **Ensure certificate matches the server's certificate** -4. **For PFX certificates, verify the password is correct** -5. **For emulator issues, try `DisableSslValidation: true` (development only)** -6. **For production, contact your network administrator for proper certificates** -7. **Check that PFX files contain both certificate and private key** -8. **Verify certificate expiration dates and certificate chain validity** From a1cc6d4637233a5047ee4abc6b351f8e3b53e23b Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 25 Nov 2025 08:46:25 -0800 Subject: [PATCH 23/31] remove vnext mention --- .../CosmosExtensionServices.cs | 2 +- .../Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index edd25714..4e55e95f 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -123,7 +123,7 @@ public static async Task VerifyContainerAccess(Container? container, string? nam /// /// Creates a custom SSL certificate validation callback that bypasses all certificate checks. - /// This is intended solely for use with the Cosmos DB vNext emulator in development environments. + /// This is intended solely for use with development environments. /// /// Logger for diagnostic information /// A callback function that always accepts server certificates diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index 5a4c2bc0..f71fb7c5 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -26,7 +26,7 @@ public abstract class CosmosSettingsBase : IValidatableObject /// /// Disables SSL certificate validation for the Cosmos DB connection. - /// This is intended for use with the Cosmos DB vNext emulator or local development environments. + /// This is intended for use with local development environments. /// WARNING: Never use this option in production as it disables critical security checks. /// public bool DisableSslValidation { get; set; } = false; From f0a53e7f8db8c6a1ad5f686fce6457f86a400a0e Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 26 Nov 2025 05:53:16 -0800 Subject: [PATCH 24/31] simplify --- .../CertificateConfigurationTests.cs | 52 ------------------- .../CosmosExtensionServices.cs | 47 ++++------------- 2 files changed, 11 insertions(+), 88 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index b6612f3c..57fa6fde 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -1,10 +1,4 @@ -using Cosmos.DataTransfer.CosmosExtension; -using Microsoft.Extensions.Logging; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; using System.ComponentModel.DataAnnotations; -using System.Security.Cryptography; -using System.Security.Cryptography.X509Certificates; namespace Cosmos.DataTransfer.CosmosExtension.UnitTests { @@ -27,34 +21,6 @@ public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessful Assert.AreEqual(0, validationResults.Count, "Should have no validation errors"); } - [TestMethod] - public void CertificateValidation_WithDisableSslValidation_ShouldBypassAllChecks() - { - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", - Database = "testDb", - Container = "testContainer", - DisableSslValidation = true - }; - - var mockLogger = new Mock(); - var callback = CreateCertificateValidationCallbackForTesting(mockLogger.Object); - - using var cert = CreateDummyCertificate(); - var isValid = callback(cert, new X509Chain(), System.Net.Security.SslPolicyErrors.RemoteCertificateChainErrors); - - Assert.IsTrue(isValid, "Callback should always return true, bypassing all SSL errors"); - mockLogger.Verify( - x => x.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((o, t) => o.ToString()!.Contains("SSL certificate validation is DISABLED")), - It.IsAny(), - It.IsAny>()), - Times.Once); - } - [TestMethod] public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultValidation() { @@ -71,24 +37,6 @@ public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultVal Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); } - private static Func CreateCertificateValidationCallbackForTesting( - ILogger logger) - { - var method = typeof(CosmosExtensionServices).GetMethod( - "CreateCertificateValidationCallback", - System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); - - var callback = method!.Invoke(null, new object[] { logger }) as Func; - return callback!; - } - - private static X509Certificate2 CreateDummyCertificate() - { - using var rsa = RSA.Create(2048); - var certRequest = new CertificateRequest("CN=TestCert", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); - return certRequest.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(365)); - } - private class TestableCosmosSettings : CosmosSettingsBase { } diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 4e55e95f..d2f1a058 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -1,17 +1,13 @@ -using Azure.Identity; -using Cosmos.DataTransfer.Interfaces; +using Azure.Core; +using Azure.Identity; +using Azure.Security.KeyVault.Keys.Cryptography; using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Encryption; using Microsoft.Extensions.Logging; using System.Globalization; +using System.Net; using System.Reflection; -using Azure.Core; using System.Text.RegularExpressions; -using Microsoft.Azure.Cosmos.Encryption; -using Azure.Security.KeyVault.Keys.Cryptography; -using System.Net; -using System.Net.Security; -using System.Security.Cryptography; -using System.Security.Cryptography.X509Certificates; namespace Cosmos.DataTransfer.CosmosExtension { @@ -43,10 +39,14 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp clientOptions.WebProxy = new WebProxy(settings.WebProxy); } - // Configure custom certificate validation for emulator scenarios + // Disable SSL certificate validation for emulator scenarios if (settings.DisableSslValidation) { - clientOptions.ServerCertificateCustomValidationCallback = CreateCertificateValidationCallback(logger); + clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => + { + logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); + return true; + }; } CosmosClient? cosmosClient; @@ -120,30 +120,5 @@ public static async Task VerifyContainerAccess(Container? container, string? nam throw new InvalidOperationException("Failed to create CosmosClient"); } } - - /// - /// Creates a custom SSL certificate validation callback that bypasses all certificate checks. - /// This is intended solely for use with development environments. - /// - /// Logger for diagnostic information - /// A callback function that always accepts server certificates - /// - /// WARNING: This callback disables all SSL/TLS security checks including: - /// - Certificate chain validation - /// - Certificate revocation checking - /// - Trusted CA verification - /// - Hostname verification - /// - Certificate expiration checking - /// - /// Never use this in production environments as it makes connections vulnerable to man-in-the-middle attacks. - /// - private static Func CreateCertificateValidationCallback(ILogger logger) - { - return (cert, chain, errors) => - { - logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); - return true; - }; - } } } \ No newline at end of file From ccc479f1626c976c0c0507f966533f2d2f450611 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 26 Nov 2025 05:56:11 -0800 Subject: [PATCH 25/31] slight fix on readme --- Extensions/Cosmos/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 951761cb..3c6c66fd 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -59,7 +59,7 @@ Or with RBAC: #### Certificate Configuration Examples -For emulator development with SSL validation disabled: +For development purposes with SSL validation disabled: ```json { From e9b085a4db21862c730d9300013a7f71059982ed Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 26 Nov 2025 05:57:16 -0800 Subject: [PATCH 26/31] fix last thing copilot was complaining about --- Extensions/Cosmos/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 3c6c66fd..d07113e8 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -149,3 +149,4 @@ For development purposes with SSL validation disabled: "InitClientEncryption": false, "LimitToEndpoint": false } +``` From 8709b82eabc6d1cd52229c714d153c21a03fdd2d Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 26 Nov 2025 22:27:48 -0800 Subject: [PATCH 27/31] Update Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../CosmosExtensionServices.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index d2f1a058..73705b56 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -42,11 +42,8 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp // Disable SSL certificate validation for emulator scenarios if (settings.DisableSslValidation) { - clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => - { - logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); - return true; - }; + logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); + clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => true; } CosmosClient? cosmosClient; From 47a50c9d458cc6c629ccb812a7e577a990f33101 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sat, 29 Nov 2025 06:16:44 -0800 Subject: [PATCH 28/31] Add AllowBulkExecution --- .../Properties/launchSettings.json | 5 ++- .../CertificateConfigurationTests.cs | 1 + .../CosmosSettingsBaseTests.cs | 25 ++++++++++++++ .../CosmosSettingsBase.cs | 13 ++++++++ Extensions/Cosmos/README.md | 33 +++++++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs diff --git a/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json b/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json index 05814317..bd73cce9 100644 --- a/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json +++ b/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json @@ -1,8 +1,7 @@ { "profiles": { "Cosmos.DataTransfer.Core": { - "commandName": "Project", - "commandLineArgs": "" + "commandName": "Project" }, "JSON->Cosmos": { "commandName": "Project", @@ -34,7 +33,7 @@ }, "SettingsCommand": { "commandName": "Project", - "commandLineArgs": "settings --sink" + "commandLineArgs": "--source json --sink cosmos-nosql --settings=\"C:\\temp\\datamigration.json\"" } } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index 57fa6fde..7f91f177 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -37,6 +37,7 @@ public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultVal Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); } + private class TestableCosmosSettings : CosmosSettingsBase { } diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs new file mode 100644 index 00000000..6ca5f344 --- /dev/null +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSettingsBaseTests.cs @@ -0,0 +1,25 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Cosmos.DataTransfer.CosmosExtension; + +namespace Cosmos.DataTransfer.CosmosExtension.UnitTests +{ + [TestClass] + public class CosmosSettingsBaseTests + { + private class TestableCosmosSettings : CosmosSettingsBase { } + + [TestMethod] + public void AllowBulkExecution_Property_ShouldSetAndGet() + { + var settings = new TestableCosmosSettings + { + AllowBulkExecution = true + }; + + Assert.IsTrue(settings.AllowBulkExecution, "AllowBulkExecution should be true when set to true"); + + settings.AllowBulkExecution = false; + Assert.IsFalse(settings.AllowBulkExecution, "AllowBulkExecution should be false when set to false"); + } + } +} diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index f71fb7c5..9a2d9038 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -31,6 +31,19 @@ public abstract class CosmosSettingsBase : IValidatableObject /// public bool DisableSslValidation { get; set; } = false; + /// + /// Enables bulk execution for Cosmos DB operations. + /// When set to true, operations such as bulk inserts and updates are optimized for performance. + /// + /// Default: false + /// + /// + /// Warning: Use with caution, as enabling bulk execution may affect consistency and error handling. + /// Review Cosmos DB documentation for bulk operation caveats before enabling in production environments. + /// + /// + public bool AllowBulkExecution { get; set; } = false; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index d07113e8..ef216124 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -6,11 +6,44 @@ The Cosmos data transfer extension provides source and sink capabilities for rea ## Settings + +### Main Settings + +| Setting | Description | Default | +|------------------------|---------------------------------------------------------------------------------------------------|-----------| +| ConnectionString | Cosmos DB connection string (AccountEndpoint + AccountKey) | | +| UseRbacAuth | Use Role Based Access Control for authentication | false | +| AccountEndpoint | Cosmos DB account endpoint (required for RBAC) | | +| EnableInteractiveCredentials | Prompt for Azure login if default credentials are unavailable | false | +| Database | Cosmos DB database name | | +| Container | Cosmos DB container name | | +| WebProxy | Proxy server URL for Cosmos DB connections | | +| InitClientEncryption | Enable Always Encrypted feature | false | +| LimitToEndpoint | Restrict client to endpoint (see CosmosClientOptions.LimitToEndpoint) | false | +| DisableSslValidation | Disable SSL certificate validation (for local dev only; not for production) | false | +| AllowBulkExecution | Enable bulk execution for optimized performance.
**Warning:** May affect consistency and error handling. | false | + Source and sink require settings used to locate and access the Cosmos DB account. This can be done in one of two ways: - Using a `ConnectionString` that includes an AccountEndpoint and AccountKey - Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. See ([migrate-passwordless](https://learn.microsoft.com/azure/cosmos-db/nosql/migrate-passwordless?tabs=sign-in-azure-cli%2Cdotnet%2Cazure-portal-create%2Cazure-portal-associate%2Capp-service-identity) for how to configure Cosmos DB for passwordless access. + +### Bulk Execution + +The extension supports bulk execution for Cosmos DB operations. When the `AllowBulkExecution` setting is set to `true`, operations such as bulk inserts and updates are optimized for performance. Use with caution, as bulk execution may affect consistency and error handling. Default is `false`. + +Example: + +```json +{ + "ConnectionString": "AccountEndpoint=https://...", + "Database": "myDb", + "Container": "myContainer", + "AllowBulkExecution": true +} +``` + Source and sink settings also both require parameters to specify the data location within a Cosmos DB account: - `Database` From ca8f2f0ab4995afd651a6cdcd2dfe91ff52fd232 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sat, 29 Nov 2025 06:24:12 -0800 Subject: [PATCH 29/31] Minor wording tweaks --- Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json | 5 +++-- .../CosmosExtensionServices.cs | 4 ++-- Extensions/Cosmos/README.md | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json b/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json index bd73cce9..05814317 100644 --- a/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json +++ b/Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json @@ -1,7 +1,8 @@ { "profiles": { "Cosmos.DataTransfer.Core": { - "commandName": "Project" + "commandName": "Project", + "commandLineArgs": "" }, "JSON->Cosmos": { "commandName": "Project", @@ -33,7 +34,7 @@ }, "SettingsCommand": { "commandName": "Project", - "commandLineArgs": "--source json --sink cosmos-nosql --settings=\"C:\\temp\\datamigration.json\"" + "commandLineArgs": "settings --sink" } } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 73705b56..fc03fcb7 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -39,10 +39,10 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp clientOptions.WebProxy = new WebProxy(settings.WebProxy); } - // Disable SSL certificate validation for emulator scenarios + // Disable SSL certificate validation for development scenarios if (settings.DisableSslValidation) { - logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development or Cosmos DB emulator scenarios. Never use in production."); + logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development scenarios. Never use in production."); clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => true; } diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index ef216124..8724d8c7 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -90,7 +90,7 @@ Or with RBAC: } ``` -#### Certificate Configuration Examples +#### Disable SSL Validation Configuration Example For development purposes with SSL validation disabled: From 51693819588f7aba4a3ae1bac82c4d6f1f64a781 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sat, 29 Nov 2025 06:48:15 -0800 Subject: [PATCH 30/31] Add a test for the SSL disable --- .../CertificateConfigurationTests.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index 7f91f177..92dc142c 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -1,5 +1,7 @@ using System.ComponentModel.DataAnnotations; +using Moq; + namespace Cosmos.DataTransfer.CosmosExtension.UnitTests { [TestClass] @@ -37,6 +39,36 @@ public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultVal Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); } + [TestMethod] + public void CreateClient_WithDisableSslValidation_LogsWarningAndSetsCallback() + { + var loggerMock = new Moq.Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = true + }; + + var client = Cosmos.DataTransfer.CosmosExtension.CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + // Verify warning was logged + loggerMock.Verify( + l => l.Log( + Microsoft.Extensions.Logging.LogLevel.Warning, + Moq.It.IsAny(), + Moq.It.Is((v, t) => v.ToString().Contains("SSL certificate validation is DISABLED")), + Moq.It.IsAny(), + Moq.It.IsAny>()), + Moq.Times.AtLeastOnce); + + // Use reflection to get the ServerCertificateCustomValidationCallback + var optionsField = typeof(Microsoft.Azure.Cosmos.CosmosClient).GetField("clientOptions", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + var clientOptions = optionsField?.GetValue(client) as Microsoft.Azure.Cosmos.CosmosClientOptions; + Assert.IsNotNull(clientOptions, "CosmosClientOptions should not be null"); + Assert.IsNotNull(clientOptions.ServerCertificateCustomValidationCallback, "ServerCertificateCustomValidationCallback should be set"); + } private class TestableCosmosSettings : CosmosSettingsBase { From a654f9de1a05c159e861450a9aa974a5905df66c Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sat, 29 Nov 2025 08:45:35 -0800 Subject: [PATCH 31/31] Fix broken tests --- .../CertificateConfigurationTests.cs | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index 92dc142c..f88f080e 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -1,5 +1,5 @@ using System.ComponentModel.DataAnnotations; - +using Microsoft.Extensions.Logging; using Moq; namespace Cosmos.DataTransfer.CosmosExtension.UnitTests @@ -39,36 +39,61 @@ public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultVal Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false"); } - [TestMethod] - public void CreateClient_WithDisableSslValidation_LogsWarningAndSetsCallback() + [TestMethod] + public void CreateClient_WithDisableSslValidation_LogsWarningAndSetsCallback() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = true + }; + + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + // Verify warning was logged when DisableSslValidation is true + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")), + It.IsAny(), + It.IsAny>()), + Times.Once); + + // Verify client was created successfully + Assert.IsNotNull(client, "CosmosClient should be created"); + } + + [TestMethod] + public void CreateClient_WithoutDisableSslValidation_DoesNotLogWarning() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings { - var loggerMock = new Moq.Mock(); - var settings = new TestableCosmosSettings - { - ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...", - Database = "testDb", - Container = "testContainer", - DisableSslValidation = true - }; + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + DisableSslValidation = false + }; - var client = Cosmos.DataTransfer.CosmosExtension.CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); - // Verify warning was logged - loggerMock.Verify( - l => l.Log( - Microsoft.Extensions.Logging.LogLevel.Warning, - Moq.It.IsAny(), - Moq.It.Is((v, t) => v.ToString().Contains("SSL certificate validation is DISABLED")), - Moq.It.IsAny(), - Moq.It.IsAny>()), - Moq.Times.AtLeastOnce); + // Verify warning was NOT logged when DisableSslValidation is false + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")), + It.IsAny(), + It.IsAny>()), + Times.Never); - // Use reflection to get the ServerCertificateCustomValidationCallback - var optionsField = typeof(Microsoft.Azure.Cosmos.CosmosClient).GetField("clientOptions", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); - var clientOptions = optionsField?.GetValue(client) as Microsoft.Azure.Cosmos.CosmosClientOptions; - Assert.IsNotNull(clientOptions, "CosmosClientOptions should not be null"); - Assert.IsNotNull(clientOptions.ServerCertificateCustomValidationCallback, "ServerCertificateCustomValidationCallback should be set"); - } + // Verify client was created successfully + Assert.IsNotNull(client, "CosmosClient should be created"); + } private class TestableCosmosSettings : CosmosSettingsBase {