tests(health api): extract specific assertions from list validator

Author: yaauie

.buildkite/scripts/health-report-tests/bootstrap.py

@@ -24,7 +24,7 @@ def __init__(self) -> None:
         logstash_branch = os.environ.get("LS_BRANCH")
         if logstash_branch is None:
             # version is not specified, use the main branch, no need to git checkout
-            print(f"LS_BRANCH is not specified, using main branch.")
+            print(f"LS_BRANCH is not specified, using HEAD.")
         else:
             # LS_BRANCH accepts major latest as a major.x or specific branch as X.Y
             if logstash_branch.find(".x") == -1:

.buildkite/scripts/health-report-tests/scenario_executor.py

@@ -2,6 +2,7 @@
 A class to execute the given scenario for Logstash Health Report integration test
 """
 import time
+import re
 from logstash_health_report import LogstashHealthReport
 
 
@@ -11,39 +12,38 @@ class ScenarioExecutor:
     def __init__(self):
         pass
 
-    def __has_intersection(self, expects, results):
-        # TODO: this logic is aligned on current Health API response
-        #   there is no guarantee that method correctly runs if provided multi expects and results
-        # we expect expects to be existing in results
-        for expect in expects:
-            for result in results:
-                if result.get('help_url') and "health-report-pipeline-" not in result.get('help_url'):
-                    return False
-                if not all(key in result and result[key] == value for key, value in expect.items()):
-                    return False
-        return True
+    def __get_difference(self, expect: any(), actual: any(), path: str | None = None) -> list:
 
-    def __get_difference(self, differences: list, expectations: dict, reports: dict) -> dict:
-        for key in expectations.keys():
+        path = path or ""
+        differences = []
 
-            if type(expectations.get(key)) != type(reports.get(key)):
-                differences.append(f"Scenario expectation and Health API report structure differs for {key}.")
-                return differences
+        match expect:
+            case {"$include": inclusion} if isinstance(expect, dict) and len(expect) == 1 and isinstance(actual, str):
+                if inclusion not in actual:
+                    differences.append(f"Value at path `{path}` does not include:`{inclusion}`; got:`{actual}`")
+            case dict():
+                if not isinstance(actual, dict):
+                    differences.append(f"Structure differs at `{path}`, expected:`{expect}` got:`{actual}`")
+                else:
+                    for key in expect.keys():
+                        differences.extend(self.__get_difference(expect.get(key), actual.get(key), f"{path}.{key}"))
+            case list():
+                if not isinstance(actual, list):
+                    differences.append(f"Structure differs at `{path}`, expected:`{expect}` got:`{actual}`")
+                else:
+                    for index, (expectEntry, actualEntry) in enumerate(zip(expect, actual)):
+                        differences.extend(self.__get_difference(expectEntry, actualEntry, f"{path}[{index}]"))
+                    if len(actual) < len(expect):
+                        differences.append(f"Missing entries at path `{path}`, expected:`{len(expect)}`, got:`{len(actual)}`")
+            case _:
+                if expect != actual:
+                    differences.append(f"Value not match at path `{path}`; expected:`{expect}`, got:`{actual}`")
 
-            if isinstance(expectations.get(key), str):
-                if expectations.get(key) != reports.get(key):
-                    differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
-                continue
-            elif isinstance(expectations.get(key), dict):
-                self.__get_difference(differences, expectations.get(key), reports.get(key))
-            elif isinstance(expectations.get(key), list):
-                if not self.__has_intersection(expectations.get(key), reports.get(key)):
-                    differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
         return differences
 
     def __is_expected(self, expectations: dict) -> None:
         reports = self.logstash_health_report_api.get()
-        differences = self.__get_difference([], expectations, reports)
+        differences = self.__get_difference(expect=expectations, actual=reports)
         if differences:
             print("Differences found in 'expectation' section between YAML content and stats:")
             for diff in differences:

.buildkite/scripts/health-report-tests/tests/abnormal-termination.yaml

@@ -24,6 +24,7 @@ expectation:
           diagnosis:
             - cause: "pipeline is not running, likely because it has encountered an error"
               action: "view logs to determine the cause of abnormal pipeline shutdown"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - description: "the pipeline is not currently processing"
               impact_areas: ["pipeline_execution"]

.buildkite/scripts/health-report-tests/tests/backpressure-1m.yaml

@@ -25,6 +25,7 @@ expectation:
             - id: "logstash:health:pipeline:flow:worker_utilization:diagnosis:1m-blocked"
               cause: "pipeline workers have been completely blocked for at least one minute"
               action: "address bottleneck or add resources"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - id: "logstash:health:pipeline:flow:impact:blocked_processing"
               severity: 2

.buildkite/scripts/health-report-tests/tests/backpressure-5m.yaml

@@ -9,7 +9,7 @@ config:
     pipeline.batch.size: 1
 conditions:
   full_start_required: true
-  wait_seconds: 310 # give more seconds to make sure time is over the threshold, 1m in this case
+  wait_seconds: 315 # give more seconds to make sure time is over the threshold, 5m in this case
 expectation:
   status: "red"
   symptom: "1 indicator is unhealthy (`pipelines`)"
@@ -25,6 +25,7 @@ expectation:
             - id: "logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-blocked"
               cause: "pipeline workers have been completely blocked for at least five minutes"
               action: "address bottleneck or add resources"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - id: "logstash:health:pipeline:flow:impact:blocked_processing"
               severity: 1

.buildkite/scripts/health-report-tests/tests/multipipeline.yaml

@@ -37,6 +37,7 @@ expectation:
           diagnosis:
             - cause: "pipeline is loading"
               action: "if pipeline does not come up quickly, you may need to check the logs to see if it is stalled"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - impact_areas: ["pipeline_execution"]
           details:
@@ -48,6 +49,7 @@ expectation:
           diagnosis:
             - cause: "pipeline has finished running because its inputs have been closed and events have been processed"
               action: "if you expect this pipeline to run indefinitely, you will need to configure its inputs to continue receiving or fetching events"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - impact_areas: [ "pipeline_execution" ]
           details:
@@ -59,6 +61,7 @@ expectation:
           diagnosis:
             - cause: "pipeline is not running, likely because it has encountered an error"
               action: "view logs to determine the cause of abnormal pipeline shutdown"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - description: "the pipeline is not currently processing"
               impact_areas: [ "pipeline_execution" ]

.buildkite/scripts/health-report-tests/tests/normal-termination.yaml

@@ -23,6 +23,7 @@ expectation:
           diagnosis:
             - cause: "pipeline has finished running because its inputs have been closed and events have been processed"
               action: "if you expect this pipeline to run indefinitely, you will need to configure its inputs to continue receiving or fetching events"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - impact_areas: ["pipeline_execution"]
           details:

.buildkite/scripts/health-report-tests/tests/slow-start.yaml

@@ -24,6 +24,7 @@ expectation:
           diagnosis:
             - cause: "pipeline is loading"
               action: "if pipeline does not come up quickly, you may need to check the logs to see if it is stalled"
+              help_url: { $include: "health-report-pipeline-" }
           impacts:
             - impact_areas: ["pipeline_execution"]
           details:

.buildkite/scripts/health-report-tests/util.py

@@ -23,13 +23,17 @@ def git_check_out_branch(branch_name: str) -> None:
     run_or_raise_error(["git", "checkout", branch_name],
                        "Error occurred while checking out the " + branch_name + " branch")
 
-
 def run_or_raise_error(commands: list, error_message):
     f"""
     Executes the {list} commands and raises an {Exception} if operation fails.
     """
-    result = subprocess.run(commands, env=os.environ.copy(), universal_newlines=True, stdout=subprocess.PIPE)
+    myEnv = os.environ.copy()
+    result = subprocess.run(commands, env=myEnv, universal_newlines=True, stdout=subprocess.PIPE)
     if result.returncode != 0:
+        print("ENV")
+        for k, v in myEnv.items():
+            print(f"  {k}={v}")
+        print("---")
         full_error_message = (error_message + ", output: " + result.stdout) \
             if result.stdout else error_message
         raise Exception(f"{full_error_message}")