feat: consecutive-window alerts

packages/api/src/controllers/alerts.ts

@@ -155,6 +155,9 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial<IAlert> => {
     // Chart alerts
     dashboard: alert.dashboardId as unknown as ObjectId,
     tileId: alert.tileId,
+
+    // Multi-window alerting
+    windowsLookback: alert.windowsLookback,
   };
 };
 

packages/api/src/models/alert.ts

@@ -84,6 +84,9 @@ export interface IAlert {
     until: Date;
   };
 
+  // Multi-window alerting: fire only after N violations in M consecutive windows
+  windowsLookback?: number;
+
   // Errors recorded during the most recent execution
   executionErrors?: IAlertError[];
   createdAt: Date;
@@ -190,6 +193,11 @@ const AlertSchema = new Schema<IAlert>(
       type: String,
       required: false,
     },
+    windowsLookback: {
+      type: Number,
+      required: false,
+      min: 1,
+    },
     silenced: {
       required: false,
       type: {

packages/api/src/models/alertHistory.ts

@@ -12,6 +12,7 @@ export interface IAlertHistory {
   state: AlertState;
   lastValues: { startTime: Date; count: number }[];
   group?: string; // For group-by alerts, stores the group identifier
+  fired?: boolean;
 }
 
 const AlertHistorySchema = new Schema<IAlertHistory>({
@@ -45,6 +46,10 @@ const AlertHistorySchema = new Schema<IAlertHistory>({
     type: String,
     required: false,
   },
+  fired: {
+    type: Boolean,
+    required: false,
+  },
 });
 
 AlertHistorySchema.index(

packages/api/src/routers/api/alerts.ts

@@ -84,6 +84,7 @@ const formatAlertResponse = (
       'createdAt',
       'updatedAt',
       'executionErrors',
+      'windowsLookback',
     ]),
   };
 };

packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts

@@ -8421,6 +8421,135 @@ describe('checkAlerts', () => {
       );
       expect((await Alert.findById(details.alert.id))!.state).toBe('OK');
     });
+
+    describe('multi-window alerting (windowsLookback)', () => {
+      it('fires on the first violation when windowsLookback=1', async () => {
+        const { team, webhook, connection, source, savedSearch, teamWebhooksById, clickhouseClient } =
+          await setupSavedSearchAlertTest();
+
+        const details = await createAlertDetails(
+          team,
+          source,
+          {
+            source: AlertSource.SAVED_SEARCH,
+            channel: { type: 'webhook', webhookId: webhook._id.toString() },
+            interval: '5m',
+            thresholdType: AlertThresholdType.ABOVE,
+            threshold: 0,
+            savedSearchId: savedSearch.id,
+            windowsLookback: 1,
+          },
+          { taskType: AlertTaskType.SAVED_SEARCH, savedSearch },
+        );
+
+        await bulkInsertLogs([
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:05:00Z'), SeverityText: 'error', Body: 'err' },
+        ]);
+
+        await processAlertAtTime(new Date('2024-01-01T00:12:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+
+        const histories = await AlertHistory.find({ alert: details.alert.id });
+        expect(histories).toHaveLength(1);
+        expect(histories[0].state).toBe('ALERT');
+        expect(histories[0].fired).toBe(true);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
+      });
+
+      it('suppresses notification until all M windows have violated', async () => {
+        const { team, webhook, connection, source, savedSearch, teamWebhooksById, clickhouseClient } =
+          await setupSavedSearchAlertTest();
+
+        jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any);
+
+        const details = await createAlertDetails(
+          team,
+          source,
+          {
+            source: AlertSource.SAVED_SEARCH,
+            channel: { type: 'webhook', webhookId: webhook._id.toString() },
+            interval: '5m',
+            thresholdType: AlertThresholdType.ABOVE,
+            threshold: 0,
+            savedSearchId: savedSearch.id,
+            windowsLookback: 3,
+          },
+          { taskType: AlertTaskType.SAVED_SEARCH, savedSearch },
+        );
+
+        await bulkInsertLogs([
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:05:00Z'), SeverityText: 'error', Body: 'err' },
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:10:00Z'), SeverityText: 'error', Body: 'err' },
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:15:00Z'), SeverityText: 'error', Body: 'err' },
+        ]);
+
+        // don't fire on windows 1 or 2 since we need 3 consecutive violations to fire
+        await processAlertAtTime(new Date('2024-01-01T00:12:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0);
+        let histories = await AlertHistory.find({ alert: details.alert.id }).sort({ createdAt: 1 });
+        expect(histories).toHaveLength(1);
+        expect(histories[0].state).toBe('ALERT');
+        expect(histories[0].fired).toBeFalsy(); // shouldn't fire yet
+
+        await processAlertAtTime(new Date('2024-01-01T00:17:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0);
+        histories = await AlertHistory.find({ alert: details.alert.id }).sort({ createdAt: 1 });
+        expect(histories).toHaveLength(2);
+        expect(histories[1].state).toBe('ALERT');
+        expect(histories[1].fired).toBeFalsy(); // shouldn't fire yet
+
+        await processAlertAtTime(new Date('2024-01-01T00:22:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
+        histories = await AlertHistory.find({ alert: details.alert.id }).sort({ createdAt: 1 });
+        expect(histories).toHaveLength(3);
+        expect(histories[2].state).toBe('ALERT');
+        expect(histories[2].fired).toBe(true); // fires here b/c it's the third consecutive violation
+      });
+
+      it('does not send a resolution notification when no alert had previously fired', async () => {
+        const { team, webhook, connection, source, savedSearch, teamWebhooksById, clickhouseClient } =
+          await setupSavedSearchAlertTest();
+
+        jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any);
+
+        const details = await createAlertDetails(
+          team,
+          source,
+          {
+            source: AlertSource.SAVED_SEARCH,
+            channel: { type: 'webhook', webhookId: webhook._id.toString() },
+            interval: '5m',
+            thresholdType: AlertThresholdType.ABOVE,
+            threshold: 1, // threshold=1 so zero-fill (no data) produces OK, not ALERT
+            savedSearchId: savedSearch.id,
+            windowsLookback: 3,
+          },
+          { taskType: AlertTaskType.SAVED_SEARCH, savedSearch },
+        );
+
+        await bulkInsertLogs([
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:05:00Z'), SeverityText: 'error', Body: 'err' },
+          { ServiceName: 'api', Timestamp: new Date('2024-01-01T00:10:00Z'), SeverityText: 'error', Body: 'err' },
+        ]);
+
+        await processAlertAtTime(new Date('2024-01-01T00:12:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0);
+
+        await processAlertAtTime(new Date('2024-01-01T00:17:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0);
+
+        // window 3 is ok (no violation)
+        await processAlertAtTime(new Date('2024-01-01T00:22:00Z'), details, clickhouseClient, connection, alertProvider, teamWebhooksById);
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(0); // webhook should never fire in this case
+
+        const histories = await AlertHistory.find({ alert: details.alert.id }).sort({ createdAt: 1 });
+        expect(histories).toHaveLength(3);
+        expect(histories[0].state).toBe('ALERT');
+        expect(histories[0].fired).toBeFalsy();
+        expect(histories[1].state).toBe('ALERT');
+        expect(histories[1].fired).toBeFalsy();
+        expect(histories[2].state).toBe('OK');
+      });
+    });
   });
 
   describe('processAlert with materialized views', () => {

packages/api/src/tasks/checkAlerts/index.ts

@@ -998,13 +998,39 @@ export const processAlert = async (
       }
     };
 
+    // Fire an alert when a condition is met in M consecutive time windows
+    const shouldFireBasedOnConsecutiveWindows = async (
+      groupKey: string,
+    ): Promise<boolean> => {
+      const numWindowsToLookBack = alert.windowsLookback ?? 1;
+
+      if (numWindowsToLookBack <= 1) {
+        return true;
+      }
+
+      const groupFilter = groupKey ? { group: groupKey } : { group: null };
+      const alertHistory = await AlertHistory.find({
+        alert: new mongoose.Types.ObjectId(alert.id),
+        ...groupFilter,
+        createdAt: { $lt: nowInMinsRoundDown },
+      })
+        .sort({ createdAt: -1 })
+        .limit(numWindowsToLookBack - 1);
+
+      return (
+        alertHistory.length === numWindowsToLookBack - 1 &&
+        alertHistory.every(h => h.state === AlertState.ALERT)
+      );
+    };
+
     const sendNotificationIfResolved = async (
       previousHistory: AggregatedAlertHistory | undefined,
       currentHistory: IAlertHistory,
       groupKey: string,
     ) => {
       if (
         previousHistory?.state === AlertState.ALERT &&
+        previousHistory?.fired !== false &&
         currentHistory.state === AlertState.OK
       ) {
         const lastValue =
@@ -1042,12 +1068,15 @@ export const processAlert = async (
       if (doesExceedThreshold(alert, value)) {
         history.state = AlertState.ALERT;
         history.counts += 1;
-        await trySendNotification({
-          state: AlertState.ALERT,
-          group: '',
-          totalCount: value,
-          startTime: alertTimestamp,
-        });
+        if (await shouldFireBasedOnConsecutiveWindows('')) {
+          history.fired = true;
+          await trySendNotification({
+            state: AlertState.ALERT,
+            group: '',
+            totalCount: value,
+            startTime: alertTimestamp,
+          });
+        }
       }
 
       // Auto-resolve
@@ -1107,12 +1136,15 @@ export const processAlert = async (
           history.lastValues.push({ count: 0, startTime: bucketStart });
           history.state = AlertState.ALERT;
           history.counts += 1;
-          await trySendNotification({
-            state: AlertState.ALERT,
-            group: '',
-            totalCount: 0,
-            startTime: bucketStart,
-          });
+          if (await shouldFireBasedOnConsecutiveWindows('')) {
+            history.fired = true;
+            await trySendNotification({
+              state: AlertState.ALERT,
+              group: '',
+              totalCount: 0,
+              startTime: bucketStart,
+            });
+          }
         } else if (!hasGroupBy || !hasAlertsInPreviousMap) {
           // For grouped alerts, if there are alerts in the previous map,
           // we will handle creating a history as part of auto-resolve later
@@ -1141,15 +1173,17 @@ export const processAlert = async (
 
         if (doesExceedThreshold(alert, value)) {
           history.state = AlertState.ALERT;
-          await trySendNotification({
-            state: AlertState.ALERT,
-            group: groupKey,
-            totalCount: value,
-            startTime: bucketStart,
-            attributes,
-          });
-
           history.counts += 1;
+          if (await shouldFireBasedOnConsecutiveWindows(groupKey)) {
+            history.fired = true;
+            await trySendNotification({
+              state: AlertState.ALERT,
+              group: groupKey,
+              totalCount: value,
+              startTime: bucketStart,
+              attributes,
+            });
+          }
         } else {
           // TODO: if the alert was previously alerting (different bucket), should we set state to OK (plus auto-resolve)?
         }
@@ -1242,6 +1276,7 @@ export interface AggregatedAlertHistory {
   createdAt: Date;
   state: AlertState;
   group?: string;
+  fired?: boolean;
 }
 
 /**
@@ -1296,6 +1331,7 @@ export const getPreviousAlertHistories = async (
               },
               createdAt: { $first: '$createdAt' },
               state: { $first: '$state' },
+              fired: { $first: '$fired' },
             },
           },
           {
@@ -1304,6 +1340,7 @@ export const getPreviousAlertHistories = async (
               createdAt: 1,
               state: 1,
               group: '$_id.group',
+              fired: 1,
             },
           },
         ]);

packages/api/src/utils/zod.ts

@@ -620,6 +620,7 @@ export const alertSchema = z
     name: z.string().min(1).max(512).nullish(),
     message: z.string().min(1).max(4096).nullish(),
     note: alertNoteSchema,
+    windowsLookback: z.number().int().min(1).optional(),
   })
   .and(zSavedSearchAlert.or(zTileAlert))
   .superRefine(validateAlertScheduleOffsetMinutes)

packages/app/src/DBSearchPageAlertModal.tsx

@@ -77,6 +77,7 @@ const SavedSearchAlertFormSchema = z
     scheduleStartAt: scheduleStartAtSchema,
     thresholdType: z.nativeEnum(AlertThresholdType),
     channel: zAlertChannel,
+    windowsLookback: z.number().int().min(1).optional(),
   })
   .passthrough()
   .superRefine(validateAlertScheduleOffsetMinutes)
@@ -150,6 +151,7 @@ const AlertForm = ({
   const groupByValue = useWatch({ control, name: 'groupBy' });
   const threshold = useWatch({ control, name: 'threshold' });
   const thresholdMax = useWatch({ control, name: 'thresholdMax' });
+  const windowsLookback = useWatch({ control, name: 'windowsLookback' });
   const maxScheduleOffsetMinutes = Math.max(
     intervalToMinutes(interval ?? '5m') - 1,
     0,
@@ -241,6 +243,31 @@ const AlertForm = ({
                 />
               )}
             />
+            <Text size="sm" opacity={0.7}>
+              for
+            </Text>
+            <Controller
+              control={control}
+              name="windowsLookback"
+              render={({ field }) => (
+                <NumberInput
+                  {...field}
+                  value={field.value ?? 1}
+                  onChange={v => {
+                    const num = typeof v === 'number' ? v : 1;
+                    field.onChange(num > 1 ? num : undefined);
+                  }}
+                  min={1}
+                  size="xs"
+                  w={70}
+                />
+              )}
+            />
+            <Text size="sm" opacity={0.7}>
+              {(windowsLookback ?? 1) === 1
+                ? 'window'
+                : 'consecutive windows'}
+            </Text>
             <Text size="sm" opacity={0.7}>
               via
             </Text>