From 7850ba1378cc80f51649239d637c75fb5b1b4d5a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Feb 2026 17:13:58 +0000 Subject: [PATCH 1/3] Initial plan From 2f89c5535d26f4a09f8a858308dcfe7e9a981a07 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Feb 2026 17:18:35 +0000 Subject: [PATCH 2/3] Add configurable monitoring for stale Dali store files - Added storeNotSavedWarningPeriod config (default 72 hours) - Added minDeltaSizeWarningThreshold config (default 50MB) - Enhanced CLightCoalesceThread to check delta file size - Issue critical operator errors when both conditions are met - Provides early warning when sasha-coalescer may not be functioning Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- dali/base/dasds.cpp | 33 ++++++++++++++++++--- initfiles/componentfiles/configxml/dali.xsd | 14 +++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index 490501fb03f..b78672d1aae 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -5062,9 +5062,12 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface unsigned lastSaveWriteTransactions = 0; unsigned lastWarning = 0; unsigned idlePeriodSecs, minimumSecsBetweenSaves, idleRate; + unsigned storeNotSavedWarningPeriodHours; + offset_t minDeltaSizeWarningThresholdKB; Owned quietStartTime, quietEndTime; CheckedCriticalSection crit; IStoreHelper *iStoreHelper; + StringAttr dataPath; class CThreaded : public Thread { @@ -5076,11 +5079,13 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface } threaded; public: IMPLEMENT_IINTERFACE; - CLightCoalesceThread(const IPropertyTree &config, IStoreHelper *_iStoreHelper) : iStoreHelper(_iStoreHelper) + CLightCoalesceThread(const IPropertyTree &config, IStoreHelper *_iStoreHelper, const char *_dataPath) : iStoreHelper(_iStoreHelper), dataPath(_dataPath) { idlePeriodSecs = config.getPropInt("@lCIdlePeriod", DEFAULT_LCIDLE_PERIOD); minimumSecsBetweenSaves = config.getPropInt("@lCMinTime", DEFAULT_LCMIN_TIME); idleRate = config.getPropInt("@lCIdleRate", DEFAULT_LCIDLE_RATE); + storeNotSavedWarningPeriodHours = config.getPropInt("@storeNotSavedWarningPeriod", STORENOTSAVE_WARNING_PERIOD); + minDeltaSizeWarningThresholdKB = config.getPropInt64("@minDeltaSizeWarningThreshold", 50000); // 50MB default char const *quietStartTimeStr = config.queryProp("@lCQuietStartTime"); if (quietStartTimeStr) { @@ -5191,9 +5196,29 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface else { t += idlePeriodSecs; - if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2))) + if (t/3600 >= storeNotSavedWarningPeriodHours && ((t-lastWarning)/3600>(storeNotSavedWarningPeriodHours/2))) { - OERRLOG("Store has not been saved for %d hours", t/3600); + // Check if delta file size exceeds the threshold + StringBuffer deltaFilename(dataPath); + iStoreHelper->getCurrentDeltaFilename(deltaFilename); + OwnedIFile deltaIFile = createIFile(deltaFilename.str()); + offset_t deltaSizeKB = 0; + if (deltaIFile->exists()) + deltaSizeKB = deltaIFile->size() / 1024; + + if (deltaSizeKB >= minDeltaSizeWarningThresholdKB) + { + // Issue critical operator error + VStringBuffer msg("CRITICAL: Store has not been saved for %u hours and delta file has grown to %" I64F "u KB (threshold: %" I64F "u KB). " + "This indicates the sasha-coalescer may not be functioning properly. " + "The delta file will continue to grow until the store is saved.", + t/3600, deltaSizeKB, minDeltaSizeWarningThresholdKB); + LOG(MCoperatorError, "%s", msg.str()); + } + else + { + OERRLOG("Store has not been saved for %u hours (delta file size: %" I64F "u KB)", t/3600, deltaSizeKB); + } lastWarning = t; } } @@ -6321,7 +6346,7 @@ CCovenSDSManager::CCovenSDSManager(ICoven &_coven, const char *_dataPath, const iStoreHelper = createStoreHelper(storeBase, dataPath, remoteBackupLocation, configFlags, keepLastN, 100, &server.queryStopped(), saveBinary, saveAsync); doTimeComparison = false; if (config->getPropBool("@lightweightCoalesce", true)) - coalesce.setown(new CLightCoalesceThread(*config, iStoreHelper)); + coalesce.setown(new CLightCoalesceThread(*config, iStoreHelper, dataPath)); // if enabled (default), ensure the ext cache size is bigger than the pending delta writer cache // to ensure that when the ext cache is full, there are transaction items to flush that aren't still diff --git a/initfiles/componentfiles/configxml/dali.xsd b/initfiles/componentfiles/configxml/dali.xsd index 737055028b5..f18c13ec92f 100644 --- a/initfiles/componentfiles/configxml/dali.xsd +++ b/initfiles/componentfiles/configxml/dali.xsd @@ -288,6 +288,20 @@ + + + + Number of hours after which a critical operator error is issued if the store has not been saved + + + + + + + Minimum delta file size (in KB) to trigger critical operator error when combined with storeNotSavedWarningPeriod + + + From 43039da2a672f2c9884b40bd20d40a2e10d9a0a1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Feb 2026 17:19:57 +0000 Subject: [PATCH 3/3] Address code review feedback - Mark dataPath member as const since it's never modified - Remove redundant 'CRITICAL:' prefix from error message Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- dali/base/dasds.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index b78672d1aae..cfbd4bd0717 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -5067,7 +5067,7 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface Owned quietStartTime, quietEndTime; CheckedCriticalSection crit; IStoreHelper *iStoreHelper; - StringAttr dataPath; + const StringAttr dataPath; class CThreaded : public Thread { @@ -5209,7 +5209,7 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface if (deltaSizeKB >= minDeltaSizeWarningThresholdKB) { // Issue critical operator error - VStringBuffer msg("CRITICAL: Store has not been saved for %u hours and delta file has grown to %" I64F "u KB (threshold: %" I64F "u KB). " + VStringBuffer msg("Store has not been saved for %u hours and delta file has grown to %" I64F "u KB (threshold: %" I64F "u KB). " "This indicates the sasha-coalescer may not be functioning properly. " "The delta file will continue to grow until the store is saved.", t/3600, deltaSizeKB, minDeltaSizeWarningThresholdKB);