Skip to content

Add configurable monitoring for stale Dali store with critical operator alerts #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Add configurable monitoring for stale Dali store with critical operator alerts #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7850ba1
Initial plan
Copilot Feb 17, 2026
2f89c55
Add configurable monitoring for stale Dali store files
Copilot Feb 17, 2026
43039da
Address code review feedback
Copilot Feb 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions dali/base/dasds.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5062,9 +5062,12 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface
unsigned lastSaveWriteTransactions = 0;
unsigned lastWarning = 0;
unsigned idlePeriodSecs, minimumSecsBetweenSaves, idleRate;
unsigned storeNotSavedWarningPeriodHours;
offset_t minDeltaSizeWarningThresholdKB;
Owned<IJlibDateTime> quietStartTime, quietEndTime;
CheckedCriticalSection crit;
IStoreHelper *iStoreHelper;
const StringAttr dataPath;

class CThreaded : public Thread
{
Expand All @@ -5076,11 +5079,13 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface
} threaded;
public:
IMPLEMENT_IINTERFACE;
CLightCoalesceThread(const IPropertyTree &config, IStoreHelper *_iStoreHelper) : iStoreHelper(_iStoreHelper)
CLightCoalesceThread(const IPropertyTree &config, IStoreHelper *_iStoreHelper, const char *_dataPath) : iStoreHelper(_iStoreHelper), dataPath(_dataPath)
{
idlePeriodSecs = config.getPropInt("@lCIdlePeriod", DEFAULT_LCIDLE_PERIOD);
minimumSecsBetweenSaves = config.getPropInt("@lCMinTime", DEFAULT_LCMIN_TIME);
idleRate = config.getPropInt("@lCIdleRate", DEFAULT_LCIDLE_RATE);
storeNotSavedWarningPeriodHours = config.getPropInt("@storeNotSavedWarningPeriod", STORENOTSAVE_WARNING_PERIOD);
minDeltaSizeWarningThresholdKB = config.getPropInt64("@minDeltaSizeWarningThreshold", 50000); // 50MB default
char const *quietStartTimeStr = config.queryProp("@lCQuietStartTime");
if (quietStartTimeStr)
{
Expand Down Expand Up @@ -5191,9 +5196,29 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface
else
{
t += idlePeriodSecs;
if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2)))
if (t/3600 >= storeNotSavedWarningPeriodHours && ((t-lastWarning)/3600>(storeNotSavedWarningPeriodHours/2)))
{
OERRLOG("Store has not been saved for %d hours", t/3600);
// Check if delta file size exceeds the threshold
StringBuffer deltaFilename(dataPath);
iStoreHelper->getCurrentDeltaFilename(deltaFilename);
OwnedIFile deltaIFile = createIFile(deltaFilename.str());
offset_t deltaSizeKB = 0;
if (deltaIFile->exists())
deltaSizeKB = deltaIFile->size() / 1024;

if (deltaSizeKB >= minDeltaSizeWarningThresholdKB)
{
// Issue critical operator error
VStringBuffer msg("Store has not been saved for %u hours and delta file has grown to %" I64F "u KB (threshold: %" I64F "u KB). "
"This indicates the sasha-coalescer may not be functioning properly. "
"The delta file will continue to grow until the store is saved.",
t/3600, deltaSizeKB, minDeltaSizeWarningThresholdKB);
LOG(MCoperatorError, "%s", msg.str());
}
else
{
OERRLOG("Store has not been saved for %u hours (delta file size: %" I64F "u KB)", t/3600, deltaSizeKB);
}
lastWarning = t;
}
}
Expand Down Expand Up @@ -6321,7 +6346,7 @@ CCovenSDSManager::CCovenSDSManager(ICoven &_coven, const char *_dataPath, const
iStoreHelper = createStoreHelper(storeBase, dataPath, remoteBackupLocation, configFlags, keepLastN, 100, &server.queryStopped(), saveBinary, saveAsync);
doTimeComparison = false;
if (config->getPropBool("@lightweightCoalesce", true))
coalesce.setown(new CLightCoalesceThread(*config, iStoreHelper));
coalesce.setown(new CLightCoalesceThread(*config, iStoreHelper, dataPath));

// if enabled (default), ensure the ext cache size is bigger than the pending delta writer cache
// to ensure that when the ext cache is full, there are transaction items to flush that aren't still
Expand Down
14 changes: 14 additions & 0 deletions initfiles/componentfiles/configxml/dali.xsd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,20 @@
</xs:appinfo>
</xs:annotation>
</xs:attribute>
<xs:attribute name="storeNotSavedWarningPeriod" type="xs:nonNegativeInteger" use="optional" default="72">
<xs:annotation>
<xs:appinfo>
<tooltip>Number of hours after which a critical operator error is issued if the store has not been saved</tooltip>
</xs:appinfo>
</xs:annotation>
</xs:attribute>
<xs:attribute name="minDeltaSizeWarningThreshold" type="xs:nonNegativeInteger" use="optional" default="50000">
<xs:annotation>
<xs:appinfo>
<tooltip>Minimum delta file size (in KB) to trigger critical operator error when combined with storeNotSavedWarningPeriod</tooltip>
</xs:appinfo>
</xs:annotation>
</xs:attribute>
<xs:attribute name="deltaSaveThresholdSecs" type="xs:nonNegativeInteger" use="optional" default="0">
<xs:annotation>
<xs:appinfo>
Expand Down