diff --git a/CHANGELOG.md b/CHANGELOG.md index 21819fc77c..a913ad8319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.2.2] - Unreleased +- Add ability to force primary keys at each batch of data when extracting to a database + ## [9.2.1] - 2026-04-07 - Update Sql Merge component to force destination column types - Update Sql merge to correctly match updates diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs index f65a81161b..a875b809f2 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs @@ -1310,7 +1310,6 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1, string v2 var destination = new DataTableUploadDestination(); destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; - var dt1 = new DataTable(); dt1.Columns.Add("TestedCol", typeof(string)); dt1.Rows.Add(new[] { v1 }); @@ -1329,6 +1328,7 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1, string v2 var tt = db.Server.GetQuerySyntaxHelper().TypeTranslater; var tbl = db.ExpectTable("DataTableUploadDestinationTests"); + try { destination.ProcessPipelineData(dt1, toConsole, token); diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 9ea6416957..b6fb9fb353 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -73,6 +73,9 @@ You must have either $a or $d [DemandsInitialization(DataTableUploadDestination.AlterTimeout_Description, DefaultValue = 300)] public int AlterTimeout { get; set; } + [DemandsInitialization("By applying the primary keys after writing the data, it ensures all data is extracted. Disabling this configuration may improve performance but will quickly raise issues with poorly keyed data.",DefaultValue =true)] + public bool WriteDataBeforeApplyingPrimaryKeys { get; set; } + [DemandsInitialization( "True to copy the column collations from the source database when creating the destination database. Only works if both the source and destination have the same DatabaseType. Excludes columns which feature a transform as part of extraction.", DefaultValue = false)] @@ -281,6 +284,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis _destination.AllowResizingColumnsAtUploadTime = true; _destination.AlterTimeout = AlterTimeout; + _destination.WriteDataBeforeApplyingPrimaryKeys = WriteDataBeforeApplyingPrimaryKeys; _destination.AppendDataIfTableExists = AppendDataIfTableExists; _destination.IncludeTimeStamp = IncludeTimeStamp; _destination.UseTrigger = AppendDataIfTableExists; diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs index 51b0535682..63633e9d25 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs @@ -84,6 +84,9 @@ public CultureInfo Culture set => _culture = value; } + [DemandsInitialization("By applying the primary keys after writing the data, it ensures all data is extracted. Disabling this configuration may improve performance but will quickly raise issues with poorly keyed data.")] + public bool WriteDataBeforeApplyingPrimaryKeys { get; set; } = true; + public string TargetTableName { get; private set; } /// @@ -181,7 +184,10 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener } } - ClearPrimaryKeyFromDataTableAndExplicitWriteTypes(toProcess); + if (WriteDataBeforeApplyingPrimaryKeys) + { + ClearPrimaryKeyFromDataTableAndExplicitWriteTypes(toProcess); + } StartAuditIfExists(TargetTableName); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index c997379eca..545efb3102 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -6,10 +6,10 @@ [assembly: AssemblyCompany("Health Informatics Centre, University of Dundee")] [assembly: AssemblyProduct("Research Data Management Platform (RDMP)")] -[assembly: AssemblyCopyright("Copyright (c) 2018 - 2025")] +[assembly: AssemblyCopyright("Copyright (c) 2018 - 2026")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.2.1")] -[assembly: AssemblyFileVersion("9.2.1")] -[assembly: AssemblyInformationalVersion("9.2.1")] +[assembly: AssemblyVersion("9.2.2")] +[assembly: AssemblyFileVersion("9.2.2")] +[assembly: AssemblyInformationalVersion("9.2.2")] diff --git a/rdmp-client.xml b/rdmp-client.xml index 4616b634e5..75064135e3 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 9.2.0.0 - https://github.com/HicServices/RDMP/releases/download/v9.2.1/rdmp-9.2.1-client.zip + 9.2.2.0 + https://github.com/HicServices/RDMP/releases/download/v9.2.2/rdmp-9.2.2-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true