From 15c1e0ad37146d44ad3d58c139742406690c532e Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Tue, 20 Jan 2026 18:19:18 +0100 Subject: [PATCH 1/9] InvalidateLBIndexAndRowCache: Factor it out In a future commit we want to reuse the cache invalidation logic for a single logbook. And this also makes InvalidateLBIndexAndRowCaches_Impl superfluous. --- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 58125ae390..12e63b2a54 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -2017,25 +2017,28 @@ Function InvalidateLBIndexAndRowCaches() endfor Make/FREE/WAVE valuesWave = {GetNumericalResultsValues(), GetTextualResultsValues()} - InvalidateLBIndexAndRowCaches_Impl(valuesWave) + + for(WAVE values : valuesWave) + InvalidateLBIndexAndRowCache(values) + endfor End Function InvalidateLBIndexAndRowCachesForDevice(string device) Make/FREE/WAVE valuesWave = {GetLBNumericalValues(device), GetLBTextualValues(device)} - InvalidateLBIndexAndRowCaches_Impl(valuesWave) + for(WAVE values : valuesWave) + InvalidateLBIndexAndRowCache(values) + endfor End -static Function InvalidateLBIndexAndRowCaches_Impl(WAVE/WAVE valuesWave) +Function InvalidateLBIndexAndRowCache(WAVE values) string key - for(WAVE values : valuesWave) - Make/FREE/T keys = {CA_CreateLBIndexCacheKey(values), CA_CreateLBRowCacheKey(values)} + Make/FREE/T keys = {CA_CreateLBIndexCacheKey(values), CA_CreateLBRowCacheKey(values)} - for(key : keys) - CA_DeleteCacheEntry(key) - endfor + for(key : keys) + CA_DeleteCacheEntry(key) endfor End From 54549d2791a2dfa92343cc6cc4cfcc7419ec52b9 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Wed, 2 Apr 2025 15:50:19 +0200 Subject: [PATCH 2/9] ED: Add feature to ED_AddEntriesToLabnotebook for value insertion ED_AddEntriesToLabnotebook and further down functions were extended to add the feature that a given value is inserted into the LNB at the end of the sweepNumber/entrySourceType block. This row is also marked as postprocessed data. The row and index cache is invalidated when a row was inserted. Added tests for insertion in numerical and textual LNB. --- Packages/MIES/MIES_Constants.ipf | 3 +- .../MIES/MIES_ExperimentDocumentation.ipf | 106 +++++++++++-- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 2 +- .../labnotebook_numerical_description.itx | 3 +- .../MIES/labnotebook_textual_description.itx | 3 +- Packages/tests/Basic/UTF_Labnotebook.ipf | 147 ++++++++++++++++++ Packages/tests/UTF_DataGenerators.ipf | 14 ++ 7 files changed, 263 insertions(+), 15 deletions(-) diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index a9ac851924..451325199b 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,7 +39,7 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 85 +Constant LABNOTEBOOK_VERSION = 86 Constant RESULTS_VERSION = 4 ///@} @@ -1043,6 +1043,7 @@ StrConstant STIMSET_SIZE_KEY = "Stimset Size" StrConstant STIMSET_ERROR_KEY = "Wavebuilder Error" StrConstant AUTOBIAS_PERC_KEY = "Autobias %" StrConstant SWEEP_EPOCH_VERSION_ENTRY_KEY = "Epochs Version" +StrConstant POSTPROCESSED_ENTRY_KEY = "PostProcessed" Constant WAVEBUILDER_STATUS_ERROR = 1 diff --git a/Packages/MIES/MIES_ExperimentDocumentation.ipf b/Packages/MIES/MIES_ExperimentDocumentation.ipf index 214f80921b..d285207f39 100644 --- a/Packages/MIES/MIES_ExperimentDocumentation.ipf +++ b/Packages/MIES/MIES_ExperimentDocumentation.ipf @@ -62,14 +62,16 @@ /// It is recommended to gather all entries to be written in keys/values and call ED_AddEntriesToLabnotebook then once. /// /// @see ED_createTextNotes, ED_createWaveNote -Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType) +Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType, [variable insertAsPostProc]) + + insertAsPostProc = ParamIsDefault(insertAsPostProc) ? 0 : !!insertAsPostProc ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) else - ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) endif End @@ -79,9 +81,9 @@ Function ED_AddEntriesToResults(WAVE vals, WAVE/T keys, variable entrySourceType ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) else - ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) endif End @@ -109,6 +111,38 @@ static Function ED_CheckValuesAndKeys(WAVE vals, WAVE keys) endif End +static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row) + + variable col, unused + + ASSERT(DimSize(values, ROWS) >= row, "Row does not exist in LBN values wave") + + WAVE keys = GetLogbookKeysFromValues(values) + + if(IsTextWave(values)) + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesT + WAVE incomingValues = incomingValuesT + else + Make/FREE/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesNum + WAVE incomingValues = incomingValuesNum + endif + + Make/FREE/T/N=(3, 1) incomingKeys + incomingKeys[0] = POSTPROCESSED_ENTRY_KEY + incomingKeys[1] = LABNOTEBOOK_BINARY_UNIT + incomingKeys[2] = LABNOTEBOOK_NO_TOLERANCE + [WAVE indizes, unused] = ED_FindIndizesAndRedimension(incomingKeys, incomingValues, keys, values, LBT_LABNOTEBOOK) + ASSERT(WaveExists(indizes), "Missing indizes") + + col = indizes[0] + if(IsTextWave(values)) + WAVE/T valuesT = values + valuesT[row][col][] = "1" + else + values[row][col][] = 1 + endif +End + static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState) variable timestamp @@ -142,6 +176,29 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, endif End +/// @brief Inserts a row after the sweep block and returns the inserted row number +static Function ED_InsertRowAfterSweepBlock(WAVE values, variable sweepNo, variable entrySourceType) + + variable sweepCol, firstRow, lastRow, rowIndex, size + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweepNo, entrySourceType, firstRow, lastRow) + ASSERT(!IsNaN(firstRow) && !IsNaN(lastRow), "FindRange could not determine start and/or end of sweep block") + rowIndex = lastRow + 1 + InsertPoints/M=(ROWS) rowIndex, 1, values + + if(!IsTextWave(values)) + values[rowIndex][][] = NaN + endif + + size = GetNumberFromWaveNote(values, NOTE_INDEX) + SetNumberInWaveNote(values, NOTE_INDEX, size + 1) + + InvalidateLBIndexAndRowCache(values) + + return rowIndex +End + /// @brief Add textual entries to the logbook /// /// The text documentation wave will use layers to report the different headstages. @@ -156,9 +213,11 @@ End /// @param device [optional for logbookType LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType type of the logbook, one of @ref LogbookTypes -static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, i, lastValidIncomingLayer, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -173,8 +232,14 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) @@ -198,7 +263,9 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = NormalizeToEOL(incomingTextualValues[0][i][r], "\n"); AbortOnRTE endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End static Function ED_ParseHeadstageContingencyMode(string str) @@ -268,9 +335,11 @@ End /// @param device [optional for logbooktype LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType one of @ref LogbookTypes -static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, lastValidIncomingLayer, i, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -286,8 +355,14 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) @@ -303,7 +378,9 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = incomingNumericalValues[0][i][r]; AbortOnRTE endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End /// @brief Add custom entries to the numerical/textual labnotebook for the very last sweep acquired. @@ -715,6 +792,13 @@ static Function [WAVE colIndizes, variable rowIndex] ED_FindIndizesAndRedimensio LBN_SetDimensionLabels(key, values, start = ((rowIndex == 0) ? 0 : numKeyCols)) endif + // Notes on the LBN row expansion here: + // - Initialisation with NaN, "" takes only place when the wave is expanded for rows/cols. + // - It does not initialise the row returned to the caller. + // Thus, for the unlucky case that a previous LBN write attempt ended up to be partial (not seen happening so far) + // then the row returned might be partially filled. + // - This part is also called if a row is inserted in the LBN from postprocessing. The LBN wave size is increased if required. + // The new row at the end is not used by the insertion. It is still consistent as the NOTE_INDEX is not updated here. if(IsNumericWave(values)) EnsureLargeEnoughWave(values, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) if(numAdditions) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 12e63b2a54..30d84519ef 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1291,7 +1291,7 @@ End /// @param[in] entrySourceType type of the labnotebook entry, one of @ref DataAcqModes /// @param[out] first point index of the beginning of the range /// @param[out] last point index of the end of the range -threadsafe static Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) +threadsafe Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) variable numRows, i, j, sourceTypeCol, firstRow, lastRow, isNumeric, index, startRow, endRow diff --git a/Packages/MIES/labnotebook_numerical_description.itx b/Packages/MIES/labnotebook_numerical_description.itx index 16a7b9534e..c499424b9a 100644 --- a/Packages/MIES/labnotebook_numerical_description.itx +++ b/Packages/MIES/labnotebook_numerical_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(198,6) labnotebook_numerical_description +WAVES/T/N=(199,6) labnotebook_numerical_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "SweepNum" "" "-" "Sweep number: Non-repeating non-negative numeric identifier for sweep time series. Increments in the order of acquisition. Starts at zero." "ALL" "" @@ -199,6 +199,7 @@ BEGIN "Skip Sweeps source" "" "0.1" "Stores who is responsible for sweep skipping. Current values are 0x1 for the user and 0x2 for automatic/internal reasons." "INDEP" "IC;VC;I=0" "ADC Configuration bits" "" "0.1" "Supported bits (1: RSE, 2: NRSE, 4: Diff, 8: Pseudo-Diff)" "INDEP" "IC;VC;I=0" "Original data" "On/Off" "-" "Whether the acquired data is original data or from replay data" "INDEP" "IC;VC;I=0" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_numerical_description; SetScale/P y 0,1,"", labnotebook_numerical_description; SetScale d 0,0,"", labnotebook_numerical_description X Note labnotebook_numerical_description, "WAVE_LAYOUT_VERSION:2;" diff --git a/Packages/MIES/labnotebook_textual_description.itx b/Packages/MIES/labnotebook_textual_description.itx index 2f5b5eb0e2..030dba1270 100644 --- a/Packages/MIES/labnotebook_textual_description.itx +++ b/Packages/MIES/labnotebook_textual_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(78,6) labnotebook_textual_description +WAVES/T/N=(79,6) labnotebook_textual_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "TimeStamp" "s" "-" "Time Stamp: Seconds since Igor epoch (1/1/1904) in local time zone with millisecond precision. Written at time of labnotebook entry." "ALL" "" @@ -79,6 +79,7 @@ BEGIN "TTL Epochs Channel 5" "" "-" "Epochs of TTL Channel 5" "INDEP" "" "TTL Epochs Channel 6" "" "-" "Epochs of TTL Channel 6" "INDEP" "" "TTL Epochs Channel 7" "" "-" "Epochs of TTL Channel 7" "INDEP" "" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_textual_description; SetScale/P y 0,1,"", labnotebook_textual_description; SetScale d 0,0,"", labnotebook_textual_description X Note labnotebook_textual_description, "WAVE_LAYOUT_VERSION:1;" diff --git a/Packages/tests/Basic/UTF_Labnotebook.ipf b/Packages/tests/Basic/UTF_Labnotebook.ipf index 236e561e6c..326dda64da 100644 --- a/Packages/tests/Basic/UTF_Labnotebook.ipf +++ b/Packages/tests/Basic/UTF_Labnotebook.ipf @@ -1561,3 +1561,150 @@ Function EmptyLabnotebookWorks() WAVE/Z entries = GetLastSetting(textualValues, 0, "Sweep Number", UNKNOWN_MODE) CHECK_WAVE(entries, NULL_WAVE) End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForPostProcessingSweepIndexerText +static Function InsertRowForPostProcessingTextual([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col + string str + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + sizeBefore = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, EPOCHS_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[testHS], keyItem) + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[INDEP_HEADSTAGE], "1") + + FindValue/TEXT=keyItem/TXOP=4 textualValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + str = textualValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_STR(str, "1") +End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForPostProcessingSweepIndexerNum +static Function InsertRowForPostProcessingNumerical([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col, val + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, "DAC", DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[testHS], keyValue) + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[INDEP_HEADSTAGE], 1) + + FindValue/V=(keyValue) numericalValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + val = numericalValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_VAR(val, 1) +End + +static Function InsertRowForPostProcessingTextualUnknownSweep() + + variable sweepNo = 1337 + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + try + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + FAIL() + catch + PASS() + endtry +End + +static Function InsertRowForPostProcessingNumericalMultiple() + + variable sizeBefore, sizeAfter, sweepNo, index, beforeVal + + variable startSweep = 4 + variable endSweep = 15 + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + beforeVal = setting[index] + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + for(sweepNo = startSweep; sweepNo <= endSweep; sweepNo += 1) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + endfor + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1 + endSweep - startSweep, sizeAfter) + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(beforeVal, setting[index]) + +End diff --git a/Packages/tests/UTF_DataGenerators.ipf b/Packages/tests/UTF_DataGenerators.ipf index 4d04c8a40e..6cca2f3d25 100644 --- a/Packages/tests/UTF_DataGenerators.ipf +++ b/Packages/tests/UTF_DataGenerators.ipf @@ -2322,3 +2322,17 @@ static Function/WAVE FFI_ClampModeCases() return wv End + +static Function/WAVE InsertRowForPostProcessingSweepIndexerText() + + Make/FREE wv = {0, 1} + + return wv +End + +static Function/WAVE InsertRowForPostProcessingSweepIndexerNum() + + Make/FREE wv = {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + + return wv +End From 335753ea320dbb15bea1bb3da7f3ab2fa7e771a5 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Mon, 14 Apr 2025 23:38:27 +0200 Subject: [PATCH 3/9] LBN: Store LBN capabilities on LBN Upgrade - Added capability SupportsEntrySourceType - Added function to determine that support - Added function to get a LBN capability This should improve performance because capabilities stay constant through the lifetime of the LBN. The information about the EntrySourceType support is required for LBN row insertion. The insertion is done for a specific EntrySourceType at the end of the block defined by sweepNumber/EntrySourceType in the LBN. However, old LBNs do not feature the EntrySourceType column. In the LBN wave upgrade process this column is created but keeps the initialization values of NaN for UNKNOWN_MODE. Now if a row is inserted, we have to insert it with the same sweepNumber/EntrySourceType combination as the block for which FindRange determined the start/end row index. FindRange has a fallback for the old LBN properties. If we would blindly write the given EntrySourceType (which is typically not UNKNOWN_MODE) on insertion in such an old LBN, effectively a different sweepNumber/EntrySourceType combination is created that is then an own block. This new block makes the fallback of FindRange fail to determine the "old" LBN entries boundaries, making them non-retrievable. As the determination of EntrySourceType support is expensive and a static property of the LBN, it can be done in the LBN upgrade process and saved in the wavenote (of the LBN key wave). --- Packages/MIES/MIES_Constants.ipf | 10 ++++- .../MIES/MIES_ExperimentDocumentation.ipf | 14 +++--- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 44 +++++++++++++++++++ Packages/MIES/MIES_WaveDataFolderGetters.ipf | 31 ++++++++++++- Packages/tests/Basic/UTF_Labnotebook.ipf | 2 +- 5 files changed, 92 insertions(+), 9 deletions(-) diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index 451325199b..74e16ab3f7 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,8 +39,8 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 86 -Constant RESULTS_VERSION = 4 +Constant LABNOTEBOOK_VERSION = 87 +Constant RESULTS_VERSION = 5 ///@} /// @name Analysis function versions @@ -2725,6 +2725,12 @@ Constant TRACECOLORMAX = 21 StrConstant BSP_USER_DATA_SF_CONTENT_HASH = "SweepFormulaContentHash" +/// Labnotebook capabilities are stored in the key wave note +/// @anchor LabnotebookCapabilityKeys +///@{ +StrConstant LBN_CAP_SUPPORTS_ENTRYSOURCETYPE = "SupportsEntrySourceType" +///@} + Constant HM_HASHMAP_WAVE_VERSION = 1 /// @name Characters allowed in the SF PrepareFit operation to hold a coefficient diff --git a/Packages/MIES/MIES_ExperimentDocumentation.ipf b/Packages/MIES/MIES_ExperimentDocumentation.ipf index d285207f39..834405d586 100644 --- a/Packages/MIES/MIES_ExperimentDocumentation.ipf +++ b/Packages/MIES/MIES_ExperimentDocumentation.ipf @@ -143,7 +143,7 @@ static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row) endif End -static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState) +static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState, variable insertAsPostProc) variable timestamp string timestampStr @@ -151,7 +151,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, if(IsTextWave(values)) WAVE/T valuesT = values valuesT[rowIndex][0][] = num2istr(sweepNo) - valuesT[rowIndex][3][] = num2istr(entrySourceType) + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + valuesT[rowIndex][3][] = num2istr(entrySourceType) + endif valuesT[rowIndex][4][] = num2istr(acqState) @@ -163,7 +165,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, valuesT[rowIndex][2][] = timestampStr else values[rowIndex][0][] = sweepNo - values[rowIndex][3][] = entrySourceType + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + values[rowIndex][3][] = entrySourceType + endif values[rowIndex][4][] = acqState @@ -241,7 +245,7 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming ASSERT(WaveExists(indizes), "Missing indizes") rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS) @@ -364,7 +368,7 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming ASSERT(WaveExists(indizes), "Missing indizes") rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 30d84519ef..2e6cda7c07 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1717,6 +1717,50 @@ threadsafe Function ReverseEntrySourceTypeMapper(variable mapped) return ((mapped == 0) ? NaN : --mapped) End +/// @brief Tests if the LBN value wave supports the EntrySourceType column +/// Only used in UpgradeLabNotebook +/// returns 0 if not, 1 is yes +Function HasLBNEntrySourceTypeCapability(WAVE values) + + variable entryCol + + entryCol = FindDimLabel(values, COLS, "EntrySourceType") + if(entryCol == -2) + return 0 + endif + + if(!DimSize(values, ROWS)) + return 1 + endif + + if(IsNumericWave(values)) + WaveStats/Q/M=1/RMD=[][entryCol][] values + return !IsNaN(V_max) + endif + + Duplicate/FREE/RMD=[][entryCol][] values, entrySourceTypeValues + return HasOneValidEntry(entrySourceTypeValues) +End + +/// @brief Returns a labnotebook capability +/// +/// @param values LBN values wave +/// @param capabilityKey Capabilities key, one of @ref LabnotebookCapabilityKeys +/// +/// @returns capability value +threadsafe Function GetLBNCapability(WAVE values, string capabilityKey) + + variable cap + + WAVE/Z keys = GetLogbookKeysFromValues(values) + ASSERT_TS(WaveExists(keys), "Can not resolve LBN keys wave") + cap = GetNumberFromWaveNote(keys, capabilityKey) + // Triggers if LBN was not correctly upgraded in UpgradeLabNotebook + ASSERT_TS(!IsNaN(cap), "Requested LBN capability not found: " + capabilityKey) + + return cap +End + /// @brief Return labnotebook keys for patch seq analysis functions /// /// @param type One of @ref SpecialAnalysisFunctionTypes diff --git a/Packages/MIES/MIES_WaveDataFolderGetters.ipf b/Packages/MIES/MIES_WaveDataFolderGetters.ipf index 448e0d4a34..1ef702471e 100644 --- a/Packages/MIES/MIES_WaveDataFolderGetters.ipf +++ b/Packages/MIES/MIES_WaveDataFolderGetters.ipf @@ -1395,9 +1395,11 @@ End /// - Making dimension labels valid liberal object names /// - Extending the row dimension to 6 for the key waves including setting the dimension labels /// - Fixing empty column dimension labels in the columns of the value waves +/// - Add capabilities to wavenote Function UpgradeLabNotebook(string device) variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow + variable hasCapability string list, key // we only have to check the new place and name as we are called @@ -1687,6 +1689,18 @@ Function UpgradeLabNotebook(string device) endif // END add dimension labels for key waves + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalKeys, 87)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalValues) + SetNumberInWaveNote(numericalKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualKeys, 87)) + hasCapability = HasLBNEntrySourceTypeCapability(textualValues) + SetNumberInWaveNote(textualKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType + SetWaveVersion(numericalKeys, LABNOTEBOOK_VERSION) SetWaveVersion(textualkeys, LABNOTEBOOK_VERSION) End @@ -1761,6 +1775,7 @@ Function/WAVE GetLBTextualKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1815,6 +1830,7 @@ Function/WAVE GetLBNumericalKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1949,7 +1965,7 @@ End /// - Fixing empty column dimension labels in key waves static Function UpgradeResultsNotebook() - variable i, numCols + variable i, numCols, hasCapability DFREF dfr = GetResultsFolder() @@ -1995,6 +2011,17 @@ static Function UpgradeResultsNotebook() endfor endif // END fix missing column dimension labels in keyWaves + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalResultKeys, 5)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalResultValues) + SetNumberInWaveNote(numericalResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualResultKeys, 5)) + hasCapability = HasLBNEntrySourceTypeCapability(textualResultValues) + SetNumberInWaveNote(textualResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType End /// @brief Return a wave reference to the numeric labnotebook keys @@ -2064,6 +2091,7 @@ Function/WAVE GetNumericalResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -2159,6 +2187,7 @@ Function/WAVE GetTextualResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" diff --git a/Packages/tests/Basic/UTF_Labnotebook.ipf b/Packages/tests/Basic/UTF_Labnotebook.ipf index 326dda64da..cdd6d68dcf 100644 --- a/Packages/tests/Basic/UTF_Labnotebook.ipf +++ b/Packages/tests/Basic/UTF_Labnotebook.ipf @@ -1594,7 +1594,7 @@ static Function InsertRowForPostProcessingTextual([variable sweepNo]) WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) CHECK_EQUAL_STR(settings[INDEP_HEADSTAGE], "1") - FindValue/TEXT=keyItem/TXOP=4 textualValues + FindValue/TEXT=keyItem/TXOP=(TXOP_WHOLE_ELEM) textualValues CHECK_NEQ_VAR(V_value, -1) row = V_row col = FindDimlabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) From 290d5931809d3531c9d1028ab6d577613cb5dd42 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Tue, 15 Apr 2025 13:45:56 +0200 Subject: [PATCH 4/9] NWB: Recreate Epochs on export and insert this info into LBN - Epochs for DA channels get recreated. - Epoch recreation is an optional argument for NWB_ExportAllData and is by default off. - The location where the LNB is written to the NWB file through NWB_WriteLabnotebooksAndComments was moved to a spot after the LNB was adapted. Added tests for postProcessed epoch addition for vintage files. --- Packages/MIES/MIES_Epochs.ipf | 2 +- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 66 +++++++++ .../MIES/MIES_NeuroDataWithoutBorders.ipf | 17 ++- .../UTF_AttemptNWB2ExportOnOldData.ipf | 6 +- Packages/tests/UTF_HelperFunctions.ipf | 135 ++++++++++++++++++ 5 files changed, 221 insertions(+), 5 deletions(-) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index de78c5c9f4..7c57923352 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1795,7 +1795,7 @@ End /// @param sweepDFR single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) /// @param sweepNo sweep number /// @returns recreated 4D epoch wave -static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) +Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) STRUCT DataConfigurationResult s variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 2e6cda7c07..d1845387a6 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -2086,3 +2086,69 @@ Function InvalidateLBIndexAndRowCache(WAVE values) CA_DeleteCacheEntry(key) endfor End + +Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues, string device, variable sweepNo) + + string epochList + variable channelNumber, channelType, headstage, numChannelTypes, colCount, allocatedCols + variable assocCol = NaN + + DFREF deviceDFR = GetDeviceDataPath(device) + DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) + WAVE/Z recEpochs = EP_RecreateEpochsFromLoadedData(numericalValues, textualValues, sweepDFR, sweepNo) + if(!WaveExists(recEpochs)) + print "Could not recreate Epochs." + return NaN + endif + + Make/FREE channelTypes = {XOP_CHANNEL_TYPE_DAC, XOP_CHANNEL_TYPE_TTL} + + numChannelTypes = DimSize(channelTypes, ROWS) + allocatedCols = numChannelTypes * NUM_DA_TTL_CHANNELS + Make/FREE/T/N=(1, allocatedCols) keys + Make/FREE/T/N=(1, allocatedCols, LABNOTEBOOK_LAYER_COUNT) values + + for(channelType : channelTypes) + for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1) + // Currently only implemented for DAC channel type + if(channelType != XOP_CHANNEL_TYPE_DAC) + continue + endif + + epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType) + if(IsEmpty(epochList)) + continue + endif + + headstage = GetHeadstageForChannel(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE) + if(IsAssociatedChannel(headstage)) + if(IsNaN(assocCol)) + assocCol = colCount + colCount += 1 + keys[0][assocCol] = EPOCHS_ENTRY_KEY + endif + values[0][assocCol][headstage] = epochList + continue + endif + + values[0][colCount][INDEP_HEADSTAGE] = epochList + keys[0][colCount] = CreateLBNUnassocKey(EPOCHS_ENTRY_KEY, channelNumber, channelType) + colCount += 1 + endfor + endfor + if(!colCount) + // No Epochs could be recreated for any channel + return NaN + endif + + Redimension/N=(-1, colCount) keys + Redimension/N=(-1, colCount, -1) values + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + + Redimension/N=(-1, 1) keys + keys[0][0] = SWEEP_EPOCH_VERSION_ENTRY_KEY + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) valuesNum + FastOp valuesNum = (NaN) + valuesNum[0][0][INDEP_HEADSTAGE] = SWEEP_EPOCH_VERSION + ED_AddEntriesToLabnotebook(valuesNum, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) +End diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf index edc878742f..160929927d 100644 --- a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf +++ b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf @@ -485,7 +485,6 @@ static Function NWB_AddDeviceSpecificData(STRUCT NWBAsyncParameters &s, variable AddModificationTimeEntry(s.locationID, s.nwbVersion) NWB_AddDevice(s) - NWB_WriteLabnotebooksAndComments(s) NWB_WriteTestpulseData(s, writeStoredTestPulses) End @@ -594,12 +593,13 @@ End /// @param overwrite [optional, defaults to false] overwrite any existing NWB file with the same name, only /// used when overrideFilePath is passed /// @param verbose [optional, defaults to true] get diagnostic output to the command line +/// @param recreateEpochs [optional, defaults to false] when set to true, epoch information is recreated if the version of the data in the experiment is older than the latest version /// /// @return 0 on success, non-zero on failure -Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose]) +Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose, variable recreateEpochs]) string list, name, fileName - variable locationID, sweep, createdNewNWBFile, argCheck + variable locationID, sweep, createdNewNWBFile, argCheck, epochVersion string stimsetList = "" PerformSubsystemEntry() @@ -638,6 +638,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF verbose = !!verbose endif + recreateEpochs = ParamIsDefault(recreateEpochs) ? 0 : !!recreateEpochs + argCheck = ParamIsDefault(overrideFullFilePath) + ParamIsDefault(overrideFileTemplate) ASSERT(argCheck >= 1 && argCheck <= 2, "Either arg overrideFullFilePath or arg overrideFileTemplate must be given or none (auto-gen)") @@ -749,6 +751,13 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF WAVE s.DAQDataWave = TextSweepToWaveRef(sweepWave) WAVE s.DAQConfigWave = configWave + if(recreateEpochs) + epochVersion = GetLastSettingIndep(s.numericalValues, s.sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + if(IsNaN(epochVersion) || epochVersion < SWEEP_EPOCH_VERSION) + InsertRecreatedEpochsIntoLBN(s.numericalValues, s.textualValues, s.device, s.sweep) + endif + endif + NWB_AppendSweepLowLevel(s) stimsetList += AB_GetStimsetFromPanel(device, sweep) @@ -757,6 +766,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF endfor LOG_AddEntry(PACKAGE_MIES, "export", keys = {"size [MiB]"}, values = {num2str(NWB_GetExportedFileSize(device))}) + NWB_WriteLabnotebooksAndComments(s) + NWB_AppendStimset(nwbVersion, s.locationID, stimsetList, compressionMode) if(writeIgorHistory) diff --git a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf index 35dd9c4ce4..43c8d79f99 100644 --- a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf +++ b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf @@ -11,10 +11,14 @@ static Function TestExportingDataToNWB([string str]) LoadMIESFolderFromPXP("input:" + str) + WAVE devEpochVersionPre = GatherEpochVersions() + PathInfo home templateName = S_path + GetBaseName(str) // attempt export - NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1) + NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1, recreateEpochs = 1) + + CheckIfPostProcessedEpochsAreAdded(devEpochVersionPre) CHECK_NO_RTE() diff --git a/Packages/tests/UTF_HelperFunctions.ipf b/Packages/tests/UTF_HelperFunctions.ipf index 9006e88ed3..de3dd0d984 100644 --- a/Packages/tests/UTF_HelperFunctions.ipf +++ b/Packages/tests/UTF_HelperFunctions.ipf @@ -2097,3 +2097,138 @@ Function RunWithOpts([string testcase, string testsuite, variable allowDebug, va RunTest(testsuite, name = name, enableJU = enableJU, enableRegExp = enableRegExp, debugMode = debugMode, testcase = testcase, traceOptions = traceOptions, traceWinList = traceWinList, keepDataFolder = keepDataFolder, waveTrackingMode = waveTrackingMode, retry = IUTF_RETRY_FAILED_UNTIL_PASS) endif End + +Function/WAVE GatherEpochVersions() + + variable i + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + Make/FREE/WAVE/N=(DimSize(devicesWithContent, ROWS)) devEpochVersion + for(device : devicesWithContent) + WAVE numericalValues = GetLBNumericalValues(device) + DFREF dfr = GetDeviceDataPath(device) + + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + Make/FREE/N=(DimSize(sweepWaveNames, ROWS)) sweepNums + sweepNums[] = ExtractSweepNumber(sweepWaveNames[p]) + Make/FREE/D/N=(WaveMax(sweepNums) + 1) epochVersion + for(sweep : sweepNums) + epochVersion[sweep] = GetLastSettingIndep(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + endfor + devEpochVersion[i] = epochVersion + i += 1 + endfor + + return devEpochVersion +End + +/// @brief A simple function to find the last row of a sweep number/DATA_ACQUISITION_MODE block for a specific headstage in the LBN +/// Depending on the flag findAny there are two search modes +/// findAny 0 (default): a value val or string str must have been specified, the search finds the last entry with that value +/// (in case of text the comparison is case-insensitive) +/// findAny 1: The first non-empty/non-NaN entry is found +/// The search is backwards and returns the LBN row of the first match or NaN if nothing was found +/// +/// @param values LBN values wave +/// @param sweep sweep number +/// @param key key of the LBN column to search +/// @param headstage headstage (LBN layer) +/// @param val [optional] value to search for, use if LBN values wave is numeric +/// @param str [optional] string to search for, use if LBN values wave is text +/// @param findAny [optional, default 0] For default, search explicitly for a value (val/str), when set find the last row with any value +/// +/// @returns last LBN row found or NaN if not found +Function FindLastLBNRow(WAVE values, variable sweep, string key, variable headstage, [variable val, string str, variable findAny]) + + variable sweepCol, i, keyCol + variable firstValue, lastValue + + findAny = ParamIsDefault(findAny) ? 0 : !!findAny + if(!findAny) + REQUIRE_EQUAL_VAR(ParamIsDefault(val) + ParamIsDefault(str), 1) + endif + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweep, DATA_ACQUISITION_MODE, firstValue, lastValue) + REQUIRE_NEQ_VAR(firstValue, NaN) + REQUIRE_NEQ_VAR(lastValue, NaN) + REQUIRE_LE_VAR(firstValue, lastValue) + keyCol = FindDimLabel(values, COLS, key) + CHECK_NEQ_VAR(keyCol, -2) + if(IsTextWave(values)) + WAVE/T valuesT = values + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsEmpty(valuesT[i][keyCol][headstage])) + return i + endif + else + if(!CmpStr(valuesT[i][keyCol][headstage], str)) + return i + endif + endif + endfor + else + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsNaN(values[i][keyCol][headstage])) + return i + endif + else + if(values[i][keyCol][headstage] == val) + return i + endif + endif + endfor + endif + + return NaN +End + +Function CheckIfPostProcessedEpochsAreAdded(WAVE/WAVE devEpochVersionPre) + + variable devCnt, sweep, keyCol, row, i + + WAVE/WAVE devEpochVersionAfter = GatherEpochVersions() + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + for(device : devicesWithContent) + // check for postProcessed epoch info + WAVE numericalValues = GetLBNumericalValues(device) + WAVE/T textualValues = GetLBTextualValues(device) + + DFREF dfr = GetDeviceDataPath(device) + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + + WAVE epochVersionsPre = devEpochVersionPre[devCnt] + WAVE epochVersionsAfter = devEpochVersionAfter[devCnt] + + for(name : sweepWaveNames) + sweep = ExtractSweepNumber(name) + if(epochVersionsPre[sweep] < SWEEP_EPOCH_VERSION) + CHECK_EQUAL_VAR(SWEEP_EPOCH_VERSION, epochVersionsAfter[sweep]) + + row = FindLastLBNRow(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, INDEP_HEADSTAGE, val = SWEEP_EPOCH_VERSION) + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_VAR(numericalValues[row][keyCol][INDEP_HEADSTAGE], 1) + + for(i = 0; i < NUM_HEADSTAGES; i += 1) + row = FindLastLBNRow(textualValues, sweep, EPOCHS_ENTRY_KEY, i, findAny = 1) + if(!IsNaN(row)) + break + endif + endfor + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_STR(textualValues[row][keyCol][INDEP_HEADSTAGE], "1") + endif + + endfor + devCnt += 1 + endfor +End From 483b35ddb5b016f2db12fdabdad1d9fb95f79a24 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Thu, 10 Apr 2025 14:50:18 +0200 Subject: [PATCH 5/9] LBN PostProc: Also add TTL epochs to postproc LBN insertion on NWB export TTL epochs are now also inserted into the LBN. --- Packages/MIES/MIES_Epochs.ipf | 1 + Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index 7c57923352..af8d6f0868 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1811,6 +1811,7 @@ Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textu WAVE/T recEpochWave = GetEpochsWaveAsFree() EP_CollectEpochInfoDA(recEpochWave, s) + EP_CollectEpochInfoTTL(recEpochWave, s) EP_AddRecreatedUserEpochs(numericalValues, textualValues, sweepDFR, sweepNo, s, recEpochWave) WAVE/Z channelDA = GetDAQDataSingleColumnWaveNG(numericalValues, textualValues, sweepNo, sweepDFR, XOP_CHANNEL_TYPE_DAC, s.DACList[0]) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index d1845387a6..98e3d4f725 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -2110,16 +2110,19 @@ Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues for(channelType : channelTypes) for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1) - // Currently only implemented for DAC channel type - if(channelType != XOP_CHANNEL_TYPE_DAC) - continue - endif epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType) if(IsEmpty(epochList)) continue endif + if(channelType == XOP_CHANNEL_TYPE_TTL) + keys[0][colCount] = CreateTTLChannelLBNKey(EPOCHS_ENTRY_KEY, channelNumber) + values[0][colCount][INDEP_HEADSTAGE] = epochList + colCount += 1 + continue + endif + headstage = GetHeadstageForChannel(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE) if(IsAssociatedChannel(headstage)) if(IsNaN(assocCol)) From f8d2f684374d6bdf07feea67ef3bc8135a72b196 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Tue, 20 Jan 2026 20:28:26 +0100 Subject: [PATCH 6/9] HasLBNEntrySourceTypeCapability: Fix it We have a test LabnotebookUpgradeDoesNotModifyDefaultWaves which checks that the labnotebook waves are not changed with the upgrade procedure when they have cleared wave notes. This test now fails as HasLBNEntrySourceTypeCapability returns 0 for a values wave with only NaNs. But in this case it does support the entry source type capability. So let's fix it. --- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 98e3d4f725..80138d8cca 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1729,15 +1729,11 @@ Function HasLBNEntrySourceTypeCapability(WAVE values) return 0 endif - if(!DimSize(values, ROWS)) + // wave with no rows or only NaNs + if(!DimSize(values, ROWS) || !HasOneValidEntry(values)) return 1 endif - if(IsNumericWave(values)) - WaveStats/Q/M=1/RMD=[][entryCol][] values - return !IsNaN(V_max) - endif - Duplicate/FREE/RMD=[][entryCol][] values, entrySourceTypeValues return HasOneValidEntry(entrySourceTypeValues) End From 3d2621f6990baebe0dcc2c442ea0f922c1d19395 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 24 Jul 2026 16:47:29 +0200 Subject: [PATCH 7/9] HasLBNEntrySourceTypeCapability: Prefer GetLogbookSettingsColumn This is faster. It does return -1 with no hit, so we have to change the check in the early return as well. We prefer the canonical smaller zero here. --- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 80138d8cca..1c4edb1e69 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1724,8 +1724,8 @@ Function HasLBNEntrySourceTypeCapability(WAVE values) variable entryCol - entryCol = FindDimLabel(values, COLS, "EntrySourceType") - if(entryCol == -2) + entryCol = GetLogbookSettingsColumn(values, "EntrySourceType") + if(entryCol < 0) return 0 endif From 46b1dd6901cac54071e5d167d416fe7c5ba9a644 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Thu, 13 Aug 2026 22:13:34 +0200 Subject: [PATCH 8/9] EP: Add assert to EP_AddEpoch to have epoch begin always >= 0 --- Packages/MIES/MIES_Epochs.ipf | 1 + 1 file changed, 1 insertion(+) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index af8d6f0868..a160195988 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1199,6 +1199,7 @@ static Function EP_AddEpoch(WAVE/T epochWave, variable channel, variable channel ASSERT(!isNull(epTags), "Epoch name is null") ASSERT(!isEmpty(epTags), "Epoch name is empty") ASSERT(!isEmpty(epShortName), "Epoch short name is empty") + ASSERT(epBegin >= 0, "Epoch begin must be >= 0") ASSERT(epBegin <= epEnd, "Epoch end is <= epoch begin") ASSERT(epBegin < upperlimit, "Epoch begin is greater than upper limit") ASSERT(epEnd > lowerlimit, "Epoch end lesser than lower limit") From 1b631eb46773bbbded7acf3b24dd3eff6426614b Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Fri, 21 Aug 2026 16:52:59 +0200 Subject: [PATCH 9/9] Test: Add check for epoch limits to NWBv2 re-export check --- Packages/tests/UTF_TestNWBExportV2.ipf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Packages/tests/UTF_TestNWBExportV2.ipf b/Packages/tests/UTF_TestNWBExportV2.ipf index 8895c0e0d7..d92201e59a 100644 --- a/Packages/tests/UTF_TestNWBExportV2.ipf +++ b/Packages/tests/UTF_TestNWBExportV2.ipf @@ -313,7 +313,7 @@ End static Function TestTimeSeries(variable fileID, string filepath, string device, variable groupID, string channel, variable sweep, DFREF pxpSweepsDFR, WAVE/Z epochs) - variable channelGroupID, starting_time, session_start_time, actual, idx, index, GUIchannelNumber, ttlBit + variable channelGroupID, starting_time, session_start_time, actual, idx, index, GUIchannelNumber, ttlBit, col, timeSeriesEnd variable clampMode, gain, gain_ref, resolution, conversion, headstage, rate_ref, rate, samplingInterval, samplingInterval_ref string stimulus, stimulus_expected, channelName, str, path, neurodata_type string electrode_name, electrode_name_ref, key, unit_ref, unit, base_unit_ref @@ -520,6 +520,7 @@ static Function TestTimeSeries(variable fileID, string filepath, string device, // introduced in 18e1406b (Labnotebook: Add DA/AD ChannelType, 2019-02-15) [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", sweep, "DA ChannelType", params.channelNumber, params.channelType, DATA_ACQUISITION_MODE) + timeSeriesEnd = IndexToScale(loadedFromNWB, Inf, ROWS) // epochs if(((params.channelType == XOP_CHANNEL_TYPE_DAC \ && clampMode != I_EQUAL_ZERO_MODE \ @@ -539,6 +540,15 @@ static Function TestTimeSeries(variable fileID, string filepath, string device, CHECK_WAVE(epochsLBN, TEXT_WAVE) INFO("Channeltype: %s, GUI channel number %d, hardware channel number %d, TTL bit %d", s0 = StringFromList(params.channelType, CHANNEL_NAMES), n0 = GUIchannelNumber, n1 = params.channelNumber, n2 = ttlBit) CHECK_EQUAL_TEXTWAVES(epochsLBN, epochsSingleChannel) + + col = FindDimLabel(epochsLBN, COLS, "StartTime") + Duplicate/FREE/T/RMD=[][col] epochsLBN, startTimesT + Make/FREE/D/N=(DimSize(startTimesT, ROWS)) epTimes = str2num(startTimesT[p]) + CHECK_GE_VAR(WaveMin(epTimes), 0) + col = FindDimLabel(epochsLBN, COLS, "EndTime") + Duplicate/FREE/T/RMD=[][col] epochsLBN, EndTimesT + epTimes[] = str2num(EndTimesT[p]) + CHECK_LE_VAR(WaveMax(epTimes), timeSeriesEnd) endif End