Skip to content

Insert recreated epoch info on massexport #2385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MichaelHuth wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Insert recreated epoch info on massexport #2385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
379384b
InvalidateLBIndexAndRowCache: Factor it out
t-b Jan 20, 2026
9c83a25
ED: Add feature to ED_AddEntriesToLabnotebook for value insertion
MichaelHuth Apr 2, 2025
c734b86
LBN: Store LBN capabilities on LBN Upgrade
MichaelHuth Apr 14, 2025
6462895
NWB: Recreate Epochs on export and insert this info into LBN
MichaelHuth Apr 15, 2025
1a8e960
LBN PostProc: Also add TTL epochs to postproc LBN insertion on NWB ex…
MichaelHuth Apr 10, 2025
cbe1c35
HasLBNEntrySourceTypeCapability: Fix it
t-b Jan 20, 2026
e86433e
HasLBNEntrySourceTypeCapability: Prefer GetLogbookSettingsColumn
t-b Jul 24, 2026
36e9935
EP: Add assert to EP_AddEpoch to have epoch begin always >= 0
MichaelHuth Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions Packages/MIES/MIES_Constants.ipf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ Constant SWEEP_EPOCH_VERSION = 9
/// - New/Changed layers of entries
///
///@{
Constant LABNOTEBOOK_VERSION = 85
Constant RESULTS_VERSION = 4
Constant LABNOTEBOOK_VERSION = 87
Constant RESULTS_VERSION = 5
///@}

/// @name Analysis function versions
Expand Down Expand Up @@ -1043,6 +1043,7 @@ StrConstant STIMSET_SIZE_KEY = "Stimset Size"
StrConstant STIMSET_ERROR_KEY = "Wavebuilder Error"
StrConstant AUTOBIAS_PERC_KEY = "Autobias %"
StrConstant SWEEP_EPOCH_VERSION_ENTRY_KEY = "Epochs Version"
StrConstant POSTPROCESSED_ENTRY_KEY = "PostProcessed"
Comment thread
t-b marked this conversation as resolved.

Constant WAVEBUILDER_STATUS_ERROR = 1

Expand Down Expand Up @@ -2724,6 +2725,12 @@ Constant TRACECOLORMAX = 21

StrConstant BSP_USER_DATA_SF_CONTENT_HASH = "SweepFormulaContentHash"

/// Labnotebook capabilities are stored in the key wave note
/// @anchor LabnotebookCapabilityKeys
///@{
StrConstant LBN_CAP_SUPPORTS_ENTRYSOURCETYPE = "SupportsEntrySourceType"
///@}

Constant HM_HASHMAP_WAVE_VERSION = 1

/// @name Characters allowed in the SF PrepareFit operation to hold a coefficient
Expand Down
4 changes: 3 additions & 1 deletion Packages/MIES/MIES_Epochs.ipf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1199,6 +1199,7 @@ static Function EP_AddEpoch(WAVE/T epochWave, variable channel, variable channel
ASSERT(!isNull(epTags), "Epoch name is null")
ASSERT(!isEmpty(epTags), "Epoch name is empty")
ASSERT(!isEmpty(epShortName), "Epoch short name is empty")
ASSERT(epBegin >= 0, "Epoch begin must be >= 0")
ASSERT(epBegin <= epEnd, "Epoch end is <= epoch begin")
ASSERT(epBegin < upperlimit, "Epoch begin is greater than upper limit")
ASSERT(epEnd > lowerlimit, "Epoch end lesser than lower limit")
Expand Down Expand Up @@ -1795,7 +1796,7 @@ End
/// @param sweepDFR single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
/// @param sweepNo sweep number
/// @returns recreated 4D epoch wave
static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)

STRUCT DataConfigurationResult s
variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex
Expand All @@ -1811,6 +1812,7 @@ static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/

WAVE/T recEpochWave = GetEpochsWaveAsFree()
EP_CollectEpochInfoDA(recEpochWave, s)
EP_CollectEpochInfoTTL(recEpochWave, s)
EP_AddRecreatedUserEpochs(numericalValues, textualValues, sweepDFR, sweepNo, s, recEpochWave)

WAVE/Z channelDA = GetDAQDataSingleColumnWaveNG(numericalValues, textualValues, sweepNo, sweepDFR, XOP_CHANNEL_TYPE_DAC, s.DACList[0])
Expand Down
120 changes: 104 additions & 16 deletions Packages/MIES/MIES_ExperimentDocumentation.ipf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,16 @@
/// It is recommended to gather all entries to be written in keys/values and call ED_AddEntriesToLabnotebook then once.
///
/// @see ED_createTextNotes, ED_createWaveNote
Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType)
Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType, [variable insertAsPostProc])

insertAsPostProc = ParamIsDefault(insertAsPostProc) ? 0 : !!insertAsPostProc

ED_CheckValuesAndKeys(vals, keys)

if(IsTextWave(vals))
ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device)
ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device)
else
ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device)
ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device)
endif
End

Expand All @@ -79,9 +81,9 @@ Function ED_AddEntriesToResults(WAVE vals, WAVE/T keys, variable entrySourceType
ED_CheckValuesAndKeys(vals, keys)

if(IsTextWave(vals))
ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS)
ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0)
else
ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS)
ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0)
endif
End

Expand Down Expand Up @@ -109,15 +111,49 @@ static Function ED_CheckValuesAndKeys(WAVE vals, WAVE keys)
endif
End

Comment thread
t-b marked this conversation as resolved.
static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState)
static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row)

variable col, unused

ASSERT(DimSize(values, ROWS) >= row, "Row does not exist in LBN values wave")
Comment thread
t-b marked this conversation as resolved.

WAVE keys = GetLogbookKeysFromValues(values)

if(IsTextWave(values))
Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesT
WAVE incomingValues = incomingValuesT
else
Make/FREE/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesNum
WAVE incomingValues = incomingValuesNum
endif

Make/FREE/T/N=(3, 1) incomingKeys
incomingKeys[0] = POSTPROCESSED_ENTRY_KEY
incomingKeys[1] = LABNOTEBOOK_BINARY_UNIT
incomingKeys[2] = LABNOTEBOOK_NO_TOLERANCE
[WAVE indizes, unused] = ED_FindIndizesAndRedimension(incomingKeys, incomingValues, keys, values, LBT_LABNOTEBOOK)
ASSERT(WaveExists(indizes), "Missing indizes")

col = indizes[0]
if(IsTextWave(values))
WAVE/T valuesT = values
valuesT[row][col][] = "1"
else
values[row][col][] = 1
endif
End

static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState, variable insertAsPostProc)

variable timestamp
string timestampStr

if(IsTextWave(values))
WAVE/T valuesT = values
valuesT[rowIndex][0][] = num2istr(sweepNo)
valuesT[rowIndex][3][] = num2istr(entrySourceType)
if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE))
valuesT[rowIndex][3][] = num2istr(entrySourceType)
endif

valuesT[rowIndex][4][] = num2istr(acqState)

Expand All @@ -129,7 +165,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo,
valuesT[rowIndex][2][] = timestampStr
else
values[rowIndex][0][] = sweepNo
values[rowIndex][3][] = entrySourceType
if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE))
values[rowIndex][3][] = entrySourceType
endif

values[rowIndex][4][] = acqState

Expand All @@ -142,6 +180,29 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo,
endif
End

/// @brief Inserts a row after the sweep block and returns the inserted row number
Comment thread
t-b marked this conversation as resolved.
static Function ED_InsertRowAfterSweepBlock(WAVE values, variable sweepNo, variable entrySourceType)

variable sweepCol, firstRow, lastRow, rowIndex, size

sweepCol = GetSweepColumn(values)
FindRange(values, sweepCol, sweepNo, entrySourceType, firstRow, lastRow)
ASSERT(!IsNaN(firstRow) && !IsNaN(lastRow), "FindRange could not determine start and/or end of sweep block")
rowIndex = lastRow + 1
InsertPoints/M=(ROWS) rowIndex, 1, values

if(!IsTextWave(values))
values[rowIndex][][] = NaN
endif

size = GetNumberFromWaveNote(values, NOTE_INDEX)
SetNumberInWaveNote(values, NOTE_INDEX, size + 1)

InvalidateLBIndexAndRowCache(values)

return rowIndex
End
Comment on lines +184 to +204

/// @brief Add textual entries to the logbook
///
/// The text documentation wave will use layers to report the different headstages.
Expand All @@ -156,9 +217,11 @@ End
/// @param device [optional for logbookType LBT_RESULTS only] device
/// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes
/// @param logbookType type of the logbook, one of @ref LogbookTypes
static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device])
/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data
static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device])

variable rowIndex, numCols, i, lastValidIncomingLayer, state
variable addIndex, insertIndex

if(ParamIsDefault(device))
ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type")
Expand All @@ -173,10 +236,16 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming
state = ROVar(GetAcquisitionState(device))
endif

[WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType)
if(insertAsPostProc)
insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType)
ED_SetLabnotebookRowToPostProcessed(values, insertIndex)
endif

[WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType)
ASSERT(WaveExists(indizes), "Missing indizes")
rowIndex = insertAsPostProc ? insertIndex : addIndex

ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state)
ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc)

WAVE valuesDat = ExtractLogbookSliceTimeStamp(values)
EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS)
Expand All @@ -198,7 +267,9 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming
values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = NormalizeToEOL(incomingTextualValues[0][i][r], "\n"); AbortOnRTE
endfor

SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
if(!insertAsPostProc)
SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
endif
End

static Function ED_ParseHeadstageContingencyMode(string str)
Expand Down Expand Up @@ -268,9 +339,11 @@ End
/// @param device [optional for logbooktype LBT_RESULTS only] device
/// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes
/// @param logbookType one of @ref LogbookTypes
static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device])
/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data
static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device])

variable rowIndex, numCols, lastValidIncomingLayer, i, state
variable addIndex, insertIndex

if(ParamIsDefault(device))
ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type")
Expand All @@ -286,10 +359,16 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming
state = ROVar(GetAcquisitionState(device))
endif

[WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType)
if(insertAsPostProc)
insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType)
ED_SetLabnotebookRowToPostProcessed(values, insertIndex)
endif

[WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType)
ASSERT(WaveExists(indizes), "Missing indizes")
rowIndex = insertAsPostProc ? insertIndex : addIndex

ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state)
ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc)

WAVE valuesDat = ExtractLogbookSliceTimeStamp(values)
EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN)
Expand All @@ -303,7 +382,9 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming
values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = incomingNumericalValues[0][i][r]; AbortOnRTE
endfor

SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
if(!insertAsPostProc)
SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
endif
End

/// @brief Add custom entries to the numerical/textual labnotebook for the very last sweep acquired.
Expand Down Expand Up @@ -715,6 +796,13 @@ static Function [WAVE colIndizes, variable rowIndex] ED_FindIndizesAndRedimensio
LBN_SetDimensionLabels(key, values, start = ((rowIndex == 0) ? 0 : numKeyCols))
endif

// Notes on the LBN row expansion here:
// - Initialisation with NaN, "" takes only place when the wave is expanded for rows/cols.
// - It does not initialise the row returned to the caller.
// Thus, for the unlucky case that a previous LBN write attempt ended up to be partial (not seen happening so far)
// then the row returned might be partially filled.
// - This part is also called if a row is inserted in the LBN from postprocessing. The LBN wave size is increased if required.
// The new row at the end is not used by the insertion. It is still consistent as the NOTE_INDEX is not updated here.
if(IsNumericWave(values))
EnsureLargeEnoughWave(values, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN)
if(numAdditions)
Expand Down
Loading
Loading