Task/jira 255 catalogue status update

Directory.Packages.props

@@ -21,7 +21,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="NPOI" Version="2.7.2" />
     <PackageVersion Include="NLog" Version="5.4.0" />
-    <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.6" />
+    <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
     <PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="2.1.5" />
     <PackageVersion Include="Spectre.Console" Version="0.49.1" />
     <PackageVersion Include="SSH.NET" Version="2024.2.0" />
@@ -45,4 +45,4 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.4" />
   </ItemGroup>
-</Project>
+</Project>

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -129,11 +129,11 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
             };
             if (!isApiCall)
             {
-                yield return new ExecuteCommandChangeExtractability(_activator, c)
-                {
-                    Weight = -99.0010f,
-                    SuggestedCategory = Extraction
-                };
+                //yield return new ExecuteCommandChangeExtractability(_activator, c)
+                //{
+                //    Weight = -99.0010f,
+                //    SuggestedCategory = Extraction
+                //};
 
                 yield return c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)
                     ? new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c)

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs

@@ -65,7 +65,17 @@ public override void Execute()
             }
             else
             {
-                new ExtractableDataSet(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue);
+                var extractabilityRecord =
+                ((DataExportChildProvider)BasicActivator.CoreChildProvider).ExtractableDataSets.SingleOrDefault(ds =>
+                    ds.Catalogue_ID == _catalogue.ID);
+                if (extractabilityRecord != null) {
+                    extractabilityRecord.DisableExtraction = false;
+                    extractabilityRecord.SaveToDatabase();
+                }
+                else
+                {
+                    new ExtractableDataSet(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue);
+                }
                 Publish(_catalogue);
             }
         }
@@ -80,7 +90,9 @@ public override void Execute()
             }
             else
             {
-                extractabilityRecord.DeleteInDatabase();
+                extractabilityRecord.DisableExtraction = true;
+                extractabilityRecord.SaveToDatabase();
+                //extractabilityRecord.DeleteInDatabase();
                 Publish(_catalogue);
             }
         }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs

@@ -105,8 +105,8 @@ private void SetCatalogue(ICatalogue catalogue)
         if (status.IsProjectSpecific)
             SetImpossible("Catalogue is already Project Specific");
 
-        if (!status.IsExtractable)
-            SetImpossible("Catalogue must first be made Extractable");
+        //if (!status.IsExtractable)
+        //    SetImpossible("Catalogue must first be made Extractable");
 
         var ei = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any);
         if (!ei.Any())

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs

@@ -34,11 +34,11 @@ public ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(IBasicActivateItems
         _extractableDataSet = dataExportRepository.GetAllObjectsWithParent<ExtractableDataSet>(catalogue)
             .SingleOrDefault();
 
-        if (_extractableDataSet == null)
-        {
-            SetImpossible("Catalogue is not extractable");
-            return;
-        }
+        //if (_extractableDataSet == null)
+        //{
+        //    SetImpossible("Catalogue is not extractable");
+        //    return;
+        //}
 
         if (_extractableDataSet.Project_ID == null)
         {

Rdmp.Core/CommandExecution/BasicActivateItems.cs

@@ -380,19 +380,21 @@ protected virtual bool InteractiveDelete(IDeleteable deletable)
         {
             case Catalogue c:
                 {
-                    if (c.GetExtractabilityStatus(RepositoryLocator.DataExportRepository).IsExtractable)
+                    var extractableDataSets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", c.ID);
+                    if (extractableDataSets.Any())
                     {
-                        if (YesNo(
-                                "Catalogue must first be made non extractable before it can be deleted, mark non extractable?",
-                                "Make Non Extractable"))
+                        foreach(var ds in extractableDataSets)
                         {
-                            var cmd = new ExecuteCommandChangeExtractability(this, c);
-                            cmd.Execute();
-                        }
-                        else
-                        {
-                            return false;
+                            var selectedDatasets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere<SelectedDataSets>("ExtractableDataSet_ID", ds.ID);
+                            if (selectedDatasets.Any())
+                            {
+                                this.Show("Catalogue Is Used in a number of projects. Remove this catalogue from all projects to allow deletion");
+                                return false;
+                            }
                         }
+                        //not used anywhere, we can dlete it
+                        foreach (var ds in extractableDataSets) ds.DeleteInDatabase();
+
                     }
 
                     break;

Rdmp.Core/DataExport/Data/ExtractableDataSet.cs

@@ -153,7 +153,7 @@ public override void DeleteInDatabase()
     /// Returns an object indicating whether the dataset is project specific or not
     /// </summary>
     /// <returns></returns>
-    public CatalogueExtractabilityStatus GetCatalogueExtractabilityStatus() => new(true, Project_ID != null);
+    public CatalogueExtractabilityStatus GetCatalogueExtractabilityStatus() => new(!DisableExtraction, Project_ID != null);
 
     private Lazy<ICatalogue> _catalogue;
 

Rdmp.Core/Databases/CatalogueDatabase/up/091_UpdateCatalogueStatuses.sql

@@ -0,0 +1,7 @@
+--Version: 8.4.3
+--Description: Add new metadata fields for catalogues
+--update Catalogue
+--set IsInternalDataset=1
+--FROM RDMP_Catalogue.dbo.Catalogue as Catalogue
+--left join RDMP_DataExport.dbo.ExtractableDataSet as eds on Catalogue.ID = eds.Catalogue_ID
+--where eds.ID is NULL

Rdmp.Core/Providers/DataExportChildProvider.cs

@@ -184,10 +184,10 @@ public DataExportChildProvider(IRDMPPlatformRepositoryServiceLocator repositoryL
         var cataToEds = new Dictionary<int, ExtractableDataSet>(ExtractableDataSets.ToDictionary(k => k.Catalogue_ID));
 
         //inject extractability into Catalogues
-        foreach (var catalogue in AllCatalogues)
-            catalogue.InjectKnown(cataToEds.TryGetValue(catalogue.ID, out var result)
-                ? result.GetCatalogueExtractabilityStatus()
-                : new CatalogueExtractabilityStatus(false, false));
+        foreach (var catalogue in AllCatalogues) catalogue.GetExtractabilityStatus(repositoryLocator.DataExportRepository);
+            //catalogue.InjectKnown(cataToEds.TryGetValue(catalogue.ID, out var result)
+            //    ? result.GetCatalogueExtractabilityStatus()
+            //    : new CatalogueExtractabilityStatus(false, false));
 
         ReportProgress("Catalogue extractability injection");
 

Rdmp.Core/Rdmp.Core.csproj

@@ -256,6 +256,7 @@
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\078_AddLastLoadTimeToLoadMetadata.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\084_AddLoadDirectorySplit.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\085_AddTicketingReleaseStatuses.sql" />
+		<!--<EmbeddedResource Include="Databases\CatalogueDatabase\up\091_UpdateCatalogueStatuses.sql" />-->
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\090_UpdateCatalogueMetadata.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\089_AddDataLoadVersioning.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\088_AddRegexRedactionConfiguration.sql" />

Rdmp.Core/Repositories/DataExportRepository.cs

@@ -98,7 +98,15 @@ protected override IMapsDirectlyToDatabaseTable ConstructEntity(Type t, DbDataRe
     public CatalogueExtractabilityStatus GetExtractabilityStatus(ICatalogue c)
     {
         var eds = GetAllObjectsWithParent<ExtractableDataSet>(c).SingleOrDefault();
-        return eds == null ? new CatalogueExtractabilityStatus(false, false) : eds.GetCatalogueExtractabilityStatus();
+        if(eds is null)
+        {
+            //if there is not EDS for this catalogue, it must have bee non extractable
+            eds = new ExtractableDataSet(this, c, false);
+            eds.SaveToDatabase();
+            //todo need to publish this eds
+
+        }
+        return eds.GetCatalogueExtractabilityStatus();
     }
 
     public ISelectedDataSets[] GetSelectedDatasetsWithNoExtractionIdentifiers() =>

Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs

@@ -21,9 +21,7 @@ public CatalogueCollectionFilterUI()
 
         cbShowInternal.Checked = UserSettings.ShowInternalCatalogues;
         cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues;
-        cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues;
         cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues;
-        cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues;
 
         _loading = false;
     }
@@ -37,10 +35,7 @@ private void OnCheckboxChanged(object sender, EventArgs e)
 
         UserSettings.ShowInternalCatalogues = cbShowInternal.Checked;
         UserSettings.ShowDeprecatedCatalogues = cbShowDeprecated.Checked;
-        UserSettings.ShowColdStorageCatalogues = cbShowColdStorage.Checked;
         UserSettings.ShowProjectSpecificCatalogues = cbProjectSpecific.Checked;
-        UserSettings.ShowNonExtractableCatalogues = cbShowNonExtractable.Checked;
-
         FiltersChanged?.Invoke(this, EventArgs.Empty);
     }
 
@@ -49,15 +44,9 @@ public void EnsureVisible(Catalogue c)
         if (c.IsColdStorageDataset || c.IsDeprecated || c.IsInternalDataset)
         {
             //trouble is our flags might be hiding it so make sure it is visible
-            cbShowColdStorage.Checked = cbShowColdStorage.Checked || c.IsColdStorageDataset;
             cbShowDeprecated.Checked = cbShowDeprecated.Checked || c.IsDeprecated;
             cbShowInternal.Checked = cbShowInternal.Checked || c.IsInternalDataset;
         }
-
-        var isExtractable = c.GetExtractabilityStatus(null);
-
-        cbShowNonExtractable.Checked = cbShowNonExtractable.Checked || isExtractable == null ||
-                                       isExtractable.IsExtractable == false;
     }
 
     /// <summary>
@@ -72,13 +61,7 @@ public void CheckForChanges()
         if (cbShowDeprecated.Checked != UserSettings.ShowDeprecatedCatalogues)
             cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues;
 
-        if (cbShowColdStorage.Checked != UserSettings.ShowColdStorageCatalogues)
-            cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues;
-
         if (cbProjectSpecific.Checked != UserSettings.ShowProjectSpecificCatalogues)
             cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues;
-
-        if (cbShowNonExtractable.Checked != UserSettings.ShowNonExtractableCatalogues)
-            cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues;
     }
 }

Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs

@@ -31,7 +31,7 @@ public CatalogueCollectionFilter(ICoreChildProvider childProvider)
         _isDeprecated = UserSettings.ShowDeprecatedCatalogues;
         _isColdStorage = UserSettings.ShowColdStorageCatalogues;
         _isProjectSpecific = UserSettings.ShowProjectSpecificCatalogues;
-        _isNonExtractable = UserSettings.ShowNonExtractableCatalogues;
+        _isNonExtractable = true;
     }
 
     public bool Filter(object modelObject) => SearchablesMatchScorer.Filter(modelObject,

Rdmp.UI/MainFormUITabs/CatalogueUI.cs

@@ -123,7 +123,6 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab
     protected override void SetBindings(BinderWithErrorProviderFactory rules, Catalogue databaseObject)
     {
         base.SetBindings(rules, databaseObject);
-        Bind(cbColdStorage, "Checked", "IsColdStorageDataset", c => c.IsColdStorageDataset);
         Bind(cbDeprecated, "Checked", "IsDeprecated", c => c.IsDeprecated);
         Bind(cbInternal, "Checked", "IsInternalDataset", c => c.IsInternalDataset);
         Bind(editableCatalogueName, "TextValue", "Name", c => c.Name);

Rdmp.UI/PieCharts/CatalogueToDatasetLinkagePieChartUI.cs

@@ -68,13 +68,9 @@ private void SetupFlags()
         if (!_firstTime)
             return;
 
-        AddFlag("Non Extractable Catalogues", c => c.IncludeNonExtractableCatalogues,
-            (c, r) => c.IncludeNonExtractableCatalogues = r);
         AddFlag("Deprecated Catalogues", c => c.IncludeDeprecatedCatalogues,
             (c, r) => c.IncludeDeprecatedCatalogues = r);
         AddFlag("Internal Catalogues", c => c.IncludeInternalCatalogues, (c, r) => c.IncludeInternalCatalogues = r);
-        AddFlag("Cold Storage Catalogues", c => c.IncludeColdStorageCatalogues,
-            (c, r) => c.IncludeColdStorageCatalogues = r);
         AddFlag("Project Specific Catalogues", c => c.IncludeProjectSpecificCatalogues,
             (c, r) => c.IncludeProjectSpecificCatalogues = r);
 

Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs

@@ -68,13 +68,9 @@ private void SetupFlags()
         if (!firstTime)
             return;
 
-        AddFlag("Non Extractable Catalogues", c => c.IncludeNonExtractableCatalogues,
-            (c, r) => c.IncludeNonExtractableCatalogues = r);
         AddFlag("Deprecated Catalogues", c => c.IncludeDeprecatedCatalogues,
             (c, r) => c.IncludeDeprecatedCatalogues = r);
         AddFlag("Internal Catalogues", c => c.IncludeInternalCatalogues, (c, r) => c.IncludeInternalCatalogues = r);
-        AddFlag("Cold Storage Catalogues", c => c.IncludeColdStorageCatalogues,
-            (c, r) => c.IncludeColdStorageCatalogues = r);
         AddFlag("Project Specific Catalogues", c => c.IncludeProjectSpecificCatalogues,
             (c, r) => c.IncludeProjectSpecificCatalogues = r);
 

Rdmp.UI/SimpleDialogs/NewfindUI.cs

@@ -369,8 +369,6 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection)
                     v => UserSettings.ShowInternalCatalogues = v, "I", "Include Internal");
                 AddUserSettingCheckbox(() => UserSettings.ShowDeprecatedCatalogues,
                     v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated");
-                AddUserSettingCheckbox(() => UserSettings.ShowColdStorageCatalogues,
-                    v => UserSettings.ShowColdStorageCatalogues = v, "C", "Include Cold Storage");
                 AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues,
                     v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific");
                 AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues,