Enable writing PKs per extraction batch (#2329)

* enable pk ordering

* add to mssql

* add space

* fix test

Author: JFriel

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.2.2] - Unreleased
+- Add ability to force primary keys at each batch of data when extracting to a database
+
 ## [9.2.1] - 2026-04-07
 - Update Sql Merge component to force destination column types
 - Update Sql merge to correctly match updates

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs

@@ -1310,7 +1310,6 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1, string v2
         var destination = new DataTableUploadDestination();
         destination.PreInitialize(null,db, toConsole);
         destination.AllowResizingColumnsAtUploadTime = true;
-
         var dt1 = new DataTable();
         dt1.Columns.Add("TestedCol", typeof(string));
         dt1.Rows.Add(new[] { v1 });
@@ -1329,6 +1328,7 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1, string v2
         var tt = db.Server.GetQuerySyntaxHelper().TypeTranslater;
         var tbl = db.ExpectTable("DataTableUploadDestinationTests");
 
+
         try
         {
             destination.ProcessPipelineData(dt1, toConsole, token);

Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs

@@ -73,6 +73,9 @@ You must have either $a or $d
     [DemandsInitialization(DataTableUploadDestination.AlterTimeout_Description, DefaultValue = 300)]
     public int AlterTimeout { get; set; }
 
+    [DemandsInitialization("By applying the primary keys after writing the data, it ensures all data is extracted. Disabling this configuration may improve performance but will quickly raise issues with poorly keyed data.",DefaultValue =true)]
+    public bool WriteDataBeforeApplyingPrimaryKeys { get; set; }
+
     [DemandsInitialization(
         "True to copy the column collations from the source database when creating the destination database.  Only works if both the source and destination have the same DatabaseType.  Excludes columns which feature a transform as part of extraction.",
         DefaultValue = false)]
@@ -281,6 +284,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis
 
         _destination.AllowResizingColumnsAtUploadTime = true;
         _destination.AlterTimeout = AlterTimeout;
+        _destination.WriteDataBeforeApplyingPrimaryKeys = WriteDataBeforeApplyingPrimaryKeys;
         _destination.AppendDataIfTableExists = AppendDataIfTableExists;
         _destination.IncludeTimeStamp = IncludeTimeStamp;
         _destination.UseTrigger = AppendDataIfTableExists;

Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs

@@ -84,6 +84,9 @@ public CultureInfo Culture
         set => _culture = value;
     }
 
+    [DemandsInitialization("By applying the primary keys after writing the data, it ensures all data is extracted. Disabling this configuration may improve performance but will quickly raise issues with poorly keyed data.")]
+    public bool WriteDataBeforeApplyingPrimaryKeys { get; set; } = true;
+
     public string TargetTableName { get; private set; }
 
     /// <summary>
@@ -181,7 +184,10 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
             }
         }
 
-        ClearPrimaryKeyFromDataTableAndExplicitWriteTypes(toProcess);
+        if (WriteDataBeforeApplyingPrimaryKeys)
+        {
+            ClearPrimaryKeyFromDataTableAndExplicitWriteTypes(toProcess);
+        }
 
         StartAuditIfExists(TargetTableName);
 

SharedAssemblyInfo.cs

@@ -6,10 +6,10 @@
 
 [assembly: AssemblyCompany("Health Informatics Centre, University of Dundee")]
 [assembly: AssemblyProduct("Research Data Management Platform (RDMP)")]
-[assembly: AssemblyCopyright("Copyright (c) 2018 - 2025")]
+[assembly: AssemblyCopyright("Copyright (c) 2018 - 2026")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
-[assembly: AssemblyVersion("9.2.1")]
-[assembly: AssemblyFileVersion("9.2.1")]
-[assembly: AssemblyInformationalVersion("9.2.1")]
+[assembly: AssemblyVersion("9.2.2")]
+[assembly: AssemblyFileVersion("9.2.2")]
+[assembly: AssemblyInformationalVersion("9.2.2")]

rdmp-client.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <item>
-    <version>9.2.0.0</version>
-    <url>https://github.com/HicServices/RDMP/releases/download/v9.2.1/rdmp-9.2.1-client.zip</url>
+    <version>9.2.2.0</version>
+    <url>https://github.com/HicServices/RDMP/releases/download/v9.2.2/rdmp-9.2.2-client.zip</url>
     <changelog>https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7</changelog>
     <mandatory>true</mandatory>
 </item>