feat(webui): Add log-ingestor and S3 auth config to deployment templates.

components/clp-package-utils/clp_package_utils/controller.py

@@ -794,12 +794,28 @@ def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict:
             server_settings_json_updates["PrestoHost"] = None
             server_settings_json_updates["PrestoPort"] = None
 
+        if self._clp_config.log_ingestor is not None:
+            server_settings_json_updates["LogIngestorHost"] = (
+                container_clp_config.log_ingestor.host
+            )
+            server_settings_json_updates["LogIngestorPort"] = (
+                container_clp_config.log_ingestor.port
+            )
+        else:
+            server_settings_json_updates["LogIngestorHost"] = None
+            server_settings_json_updates["LogIngestorPort"] = None
+
         if StorageType.FS == self._clp_config.logs_input.type:
             client_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR)
             server_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR)
+            server_settings_json_updates["LogsInputS3AwsAuthType"] = None
+            server_settings_json_updates["LogsInputS3AwsProfile"] = None
         else:
             client_settings_json_updates["LogsInputRootDir"] = None
             server_settings_json_updates["LogsInputRootDir"] = None
+            s3_auth = self._clp_config.logs_input.aws_authentication
+            server_settings_json_updates["LogsInputS3AwsAuthType"] = s3_auth.type
+            server_settings_json_updates["LogsInputS3AwsProfile"] = s3_auth.profile
 
         resolved_client_settings_json_path = resolve_host_path_in_container(
             client_settings_json_path
@@ -915,34 +931,27 @@ def _read_and_update_settings_json(
         """
         with open(settings_file_path, "r") as settings_json_file:
             settings_object = json.loads(settings_json_file.read())
-        self._update_settings_object("", settings_object, updates)
+        self._update_settings_object(settings_object, updates)
 
         return settings_object
 
     def _update_settings_object(
         self,
-        parent_key_prefix: str,
         settings: dict[str, Any],
         updates: dict[str, Any],
     ) -> None:
         """
-        Recursively updates the given settings object with the values from `updates`.
+        Updates the given settings object with the values from `updates`.
 
-        :param parent_key_prefix: The prefix for keys at this level in the settings dictionary.
         :param settings: The settings to update.
         :param updates: The updates.
         :raise ValueError: If a key in `updates` doesn't exist in `settings`.
         """
         for key, value in updates.items():
             if key not in settings:
-                error_msg = (
-                    f"{parent_key_prefix}{key} is not a valid configuration key for the webui."
-                )
+                error_msg = f"{key} is not a valid configuration key for the webui."
                 raise ValueError(error_msg)
-            if isinstance(value, dict):
-                self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value)
-            else:
-                settings[key] = value
+            settings[key] = value
 
 
 _DEPLOYMENT_TYPE_TO_COMPOSE_FILE: MappingProxyType[DeploymentType, str] = MappingProxyType(
@@ -1000,6 +1009,18 @@ def set_up_env(self) -> None:
                     ),
                 }
 
+        if self._clp_config.logs_input.type == StorageType.S3:
+            logs_input_aws_auth = self._clp_config.logs_input.aws_authentication
+            if logs_input_aws_auth.type == AwsAuthType.credentials:
+                env_vars |= {
+                    "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID": (
+                        logs_input_aws_auth.credentials.access_key_id
+                    ),
+                    "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY": (
+                        logs_input_aws_auth.credentials.secret_access_key
+                    ),
+                }
+
         # Identity config
         env_vars |= {
             "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID,

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -780,6 +780,9 @@ class LogIngestor(BaseModel):
     port: Port = 3002
     logging_level: LoggingLevelRust = "INFO"
 
+    def transform_for_container(self):
+        self.host = LOG_INGESTOR_COMPONENT_NAME
+
 
 class Presto(BaseModel):
     DEFAULT_PORT: ClassVar[int] = 8080
@@ -1091,6 +1094,8 @@ def transform_for_container(self):
         self.results_cache.transform_for_container(BundledService.RESULTS_CACHE in self.bundled)
         self.query_scheduler.transform_for_container()
         self.reducer.transform_for_container()
+        if self.log_ingestor is not None:
+            self.log_ingestor.transform_for_container()
         if self.package.query_engine == QueryEngine.PRESTO and self.presto is not None:
             self.presto.transform_for_container()
 

components/webui/server/.env

@@ -18,3 +18,7 @@ PRESTO_SCHEMA=default
 # Security
 RATE_LIMIT=1000
 
+# S3 Logs Input
+CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID=
+CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY=
+

components/webui/server/src/plugins/external/env.ts

@@ -13,6 +13,8 @@ declare module "fastify" {
             PRESTO_CATALOG: string;
             PRESTO_SCHEMA: string;
             RATE_LIMIT: number;
+            CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: string;
+            CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: string;
         };
     }
 }
@@ -72,6 +74,16 @@ const schema = {
             default: 1_000,
             minimum: 1,
         },
+
+        // S3 Logs Input
+        CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: {
+            type: "string",
+            default: "",
+        },
+        CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: {
+            type: "string",
+            default: "",
+        },
     },
 };
 

tools/deployment/package-helm/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.2.1-dev.6"
+version: "0.2.1-dev.7"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
-appVersion: "0.10.1-dev"
+appVersion: "0.11.1-dev"
 home: "https://github.com/y-scope/clp"
 icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png"
 sources: ["https://github.com/y-scope/clp"]

tools/deployment/package-helm/templates/configmap.yaml

@@ -283,10 +283,27 @@ data:
         {{ .Values.clpConfig.results_cache.stream_collection_name | quote }},
       "ClientDir": "/opt/clp/var/www/webui/client",
       "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer",
+      {{- if .Values.clpConfig.log_ingestor }}
+      "LogIngestorHost": "{{ include "clp.fullname" . }}-log-ingestor",
+      "LogIngestorPort": 3002,
+      {{- else }}
+      "LogIngestorHost": null,
+      "LogIngestorPort": null,
+      {{- end }}
       {{- if eq .Values.clpConfig.logs_input.type "fs" }}
       "LogsInputRootDir": "/mnt/logs",
+      "LogsInputS3AwsAuthType": null,
+      "LogsInputS3AwsProfile": null,
       {{- else }}
       "LogsInputRootDir": null,
+      {{- with .Values.clpConfig.logs_input.aws_authentication }}
+      "LogsInputS3AwsAuthType": {{ .type | quote }},
+      {{- if .profile }}
+      "LogsInputS3AwsProfile": {{ .profile | quote }},
+      {{- else }}
+      "LogsInputS3AwsProfile": null,
+      {{- end }}
+      {{- end }}
       {{- end }}
       {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
       "StreamFilesDir": "/var/data/streams",

tools/deployment/package-helm/templates/webui-deployment.yaml

@@ -64,6 +64,15 @@ spec:
             {{- end }}{{/* if and (eq .type "s3")
                                   (eq .s3_config.aws_authentication.type "credentials") */}}
             {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}}
+            {{- with .Values.clpConfig.logs_input }}
+            {{- if and (eq .type "s3") (eq .aws_authentication.type "credentials") }}
+            - name: "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID"
+              value: {{ .aws_authentication.credentials.access_key_id | quote }}
+            - name: "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY"
+              value: {{ .aws_authentication.credentials.secret_access_key | quote }}
+            {{- end }}{{/* if and (eq .type "s3")
+                                  (eq .aws_authentication.type "credentials") */}}
+            {{- end }}{{/* with .Values.clpConfig.logs_input */}}
           ports:
             - name: "webui"
               containerPort: 4000

tools/deployment/package/docker-compose-all.yaml

@@ -370,6 +370,8 @@ services:
       AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}"
       CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
       CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
+      CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: "${CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID:-}"
+      CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: "${CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY:-}"
       HOST: "0.0.0.0"
       NODE_ENV: "production"
       NODE_PATH: "/opt/clp/var/www/webui/server/node_modules"