diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d7a0c0b99e..8b0ddfd4ae 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -794,12 +794,27 @@ def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict: server_settings_json_updates["PrestoHost"] = None server_settings_json_updates["PrestoPort"] = None + if ( + self._clp_config.log_ingestor is not None + and self._clp_config.logs_input.type == StorageType.S3 + ): + server_settings_json_updates["LogIngestorHost"] = container_clp_config.log_ingestor.host + server_settings_json_updates["LogIngestorPort"] = container_clp_config.log_ingestor.port + else: + server_settings_json_updates["LogIngestorHost"] = None + server_settings_json_updates["LogIngestorPort"] = None + if StorageType.FS == self._clp_config.logs_input.type: client_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR) server_settings_json_updates["LogsInputRootDir"] = str(CONTAINER_INPUT_LOGS_ROOT_DIR) + server_settings_json_updates["LogsInputS3AwsAuthType"] = None + server_settings_json_updates["LogsInputS3AwsProfile"] = None else: client_settings_json_updates["LogsInputRootDir"] = None server_settings_json_updates["LogsInputRootDir"] = None + s3_auth = self._clp_config.logs_input.aws_authentication + server_settings_json_updates["LogsInputS3AwsAuthType"] = s3_auth.type + server_settings_json_updates["LogsInputS3AwsProfile"] = s3_auth.profile resolved_client_settings_json_path = resolve_host_path_in_container( client_settings_json_path @@ -1000,6 +1015,18 @@ def set_up_env(self) -> None: ), } + if self._clp_config.logs_input.type == StorageType.S3: + logs_input_aws_auth = self._clp_config.logs_input.aws_authentication + if logs_input_aws_auth.type == AwsAuthType.credentials: + env_vars |= { + "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID": ( + logs_input_aws_auth.credentials.access_key_id + ), + "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY": ( + logs_input_aws_auth.credentials.secret_access_key + ), + } + # Identity config env_vars |= { "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 404d7b0ab9..9b52517e3f 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -776,10 +776,16 @@ class ApiServer(BaseModel): class LogIngestor(BaseModel): + DEFAULT_PORT: ClassVar[int] = 3002 + host: DomainStr = "localhost" - port: Port = 3002 + port: Port = DEFAULT_PORT logging_level: LoggingLevelRust = "INFO" + def transform_for_container(self): + self.host = LOG_INGESTOR_COMPONENT_NAME + self.port = self.DEFAULT_PORT + class Presto(BaseModel): DEFAULT_PORT: ClassVar[int] = 8080 @@ -1091,6 +1097,8 @@ def transform_for_container(self): self.results_cache.transform_for_container(BundledService.RESULTS_CACHE in self.bundled) self.query_scheduler.transform_for_container() self.reducer.transform_for_container() + if self.log_ingestor is not None: + self.log_ingestor.transform_for_container() if self.package.query_engine == QueryEngine.PRESTO and self.presto is not None: self.presto.transform_for_container() diff --git a/components/webui/server/.env b/components/webui/server/.env index b51df3a791..607ef681e5 100644 --- a/components/webui/server/.env +++ b/components/webui/server/.env @@ -18,3 +18,7 @@ PRESTO_SCHEMA=default # Security RATE_LIMIT=1000 +# S3 +CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID= +CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY= + diff --git a/components/webui/server/src/plugins/external/env.ts b/components/webui/server/src/plugins/external/env.ts index 311df0c886..30fdebb684 100644 --- a/components/webui/server/src/plugins/external/env.ts +++ b/components/webui/server/src/plugins/external/env.ts @@ -13,6 +13,8 @@ declare module "fastify" { PRESTO_CATALOG: string; PRESTO_SCHEMA: string; RATE_LIMIT: number; + CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: string; + CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: string; }; } } @@ -72,6 +74,16 @@ const schema = { default: 1_000, minimum: 1, }, + + // S3 + CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: { + type: "string", + default: "", + }, + CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: { + type: "string", + default: "", + }, }, }; diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 31ed1f0dff..d3e23108f4 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.3.1-dev.0" +version: "0.3.1-dev.1" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.11.1-dev" diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 700b23f7a1..fdcf76ab6e 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -283,10 +283,28 @@ data: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + {{- if and .Values.clpConfig.log_ingestor (eq .Values.clpConfig.logs_input.type "s3") }} + "LogIngestorHost": "{{ include "clp.fullname" . }}-log-ingestor", + "LogIngestorPort": 3002, + {{- else }} + "LogIngestorHost": null, + "LogIngestorPort": null, + {{- end }}{{/* if and .Values.clpConfig.log_ingestor + (eq .Values.clpConfig.logs_input.type "s3") */}} {{- if eq .Values.clpConfig.logs_input.type "fs" }} "LogsInputRootDir": "/mnt/logs", + "LogsInputS3AwsAuthType": null, + "LogsInputS3AwsProfile": null, {{- else }} "LogsInputRootDir": null, + {{- with .Values.clpConfig.logs_input.aws_authentication }} + "LogsInputS3AwsAuthType": {{ .type | quote }}, + {{- if .profile }} + "LogsInputS3AwsProfile": {{ .profile | quote }}, + {{- else }} + "LogsInputS3AwsProfile": null, + {{- end }}{{/* if .profile */}} + {{- end }}{{/* with .Values.clpConfig.logs_input.aws_authentication */}} {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} "StreamFilesDir": "/var/data/streams", diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 71edbc6e05..ed4b29f405 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -64,6 +64,15 @@ spec: {{- end }}{{/* if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") */}} {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} + {{- with .Values.clpConfig.logs_input }} + {{- if and (eq .type "s3") (eq .aws_authentication.type "credentials") }} + - name: "CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID" + value: {{ .aws_authentication.credentials.access_key_id | quote }} + - name: "CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY" + value: {{ .aws_authentication.credentials.secret_access_key | quote }} + {{- end }}{{/* if and (eq .type "s3") + (eq .aws_authentication.type "credentials") */}} + {{- end }}{{/* with .Values.clpConfig.logs_input */}} ports: - name: "webui" containerPort: 4000 diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 31756ccbd5..cd813a0873 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -370,6 +370,8 @@ services: AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:-clp-user}" + CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID: "${CLP_LOGS_INPUT_AWS_ACCESS_KEY_ID:-}" + CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY: "${CLP_LOGS_INPUT_AWS_SECRET_ACCESS_KEY:-}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules"