Tvvinblade

.gitignore

@@ -11,3 +11,5 @@ tmp/
 *.code-workspace
 .DS_Store
 .idea/
+.venv/
+venv/

docker-compose.yml

@@ -103,8 +103,10 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_CDC_USER: ${POSTGRES_CDC_USER:-cdc_reader}
       POSTGRES_CDC_PASSWORD: ${POSTGRES_CDC_PASSWORD:-cdc_reader_pwd}
+      POSTGRES_HOST_AUTH_METHOD: scram-sha-256
+      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
     ports:
-      - "5432:5432"
+      - "0.0.0.0:50125:5432"
     volumes:
       - pg-data:/var/lib/postgresql/data
     healthcheck:
@@ -143,6 +145,8 @@ services:
       KAFKA_CFG_ADVERTISED_LISTENERS: ${KAFKA_ADVERTISED_LISTENERS}
       KAFKA_CFG_CONTROLLER_LISTENER_NAMES: ${KAFKA_CONTROLLER_LISTENER_NAMES}
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: ${KAFKA_AUTO_CREATE_TOPICS_ENABLE}
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: ${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP}
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: ${KAFKA_INTER_BROKER_LISTENER_NAME}
     ports:
       - "9092:9092"
     volumes:
@@ -264,7 +268,7 @@ services:
       resources:
         limits:
           cpus: '1.00'
-          memory: 1024M
+          memory: 768M
         reservations:
           cpus: '0.50'
           memory: 512M
@@ -305,11 +309,11 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1.00'
-          memory: 2048M
+          cpus: '0.75'
+          memory: 768M
         reservations:
           cpus: '0.50'
-          memory: 1024M
+          memory: 512M
     depends_on:
       spark-master:
         condition: service_healthy
@@ -343,11 +347,11 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1.00'
-          memory: 2048M
+          cpus: '0.75'
+          memory: 768M
         reservations:
           cpus: '0.50'
-          memory: 1024M
+          memory: 512M
     depends_on:
       spark-master:
         condition: service_healthy
@@ -485,8 +489,8 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1.00'
-          memory: 1024M
+          cpus: '0.75'
+          memory: 768M
         reservations:
           cpus: '0.50'
           memory: 512M
@@ -512,8 +516,8 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1.00'
-          memory: 1024M
+          cpus: '0.75'
+          memory: 768M
         reservations:
           cpus: '0.50'
           memory: 512M

infra/airflow/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ARG SPARK_VERSION=3.5.6
 RUN set -eux; \
     curl -fSL --connect-timeout 20 --max-time 900 --retry 5 --retry-connrefused \
-    -o /tmp/spark.tgz "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
+    -o /tmp/spark.tgz "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
     tar -xzf /tmp/spark.tgz -C /opt; \
     mv "/opt/spark-${SPARK_VERSION}-bin-hadoop3" /opt/spark; \
     rm /tmp/spark.tgz

infra/airflow/dags/_spark_common.py

@@ -84,6 +84,7 @@ def spark_env_vars() -> dict[str, str]:
     """Return environment variables required for Spark submissions."""
 
     return {
+        "HOME": os.getenv("HOME", "/opt/bitnami/spark"),
         "AWS_REGION": os.getenv("AWS_REGION", "us-east-1"),
         "AWS_DEFAULT_REGION": os.getenv("AWS_DEFAULT_REGION", "us-east-1"),
         "AWS_ACCESS_KEY_ID": os.getenv("MINIO_ROOT_USER", "minio"),

infra/schema-registry/Dockerfile

@@ -1,7 +1,7 @@
 FROM confluentinc/cp-schema-registry:latest
 
 USER root
-RUN yum install -y --allowerasing curl jq && yum clean all
+#RUN dnf install -y curl jq && dnf clean all
 
 COPY schemas/ /opt/schemas/
 COPY init-schemas.sh /opt/init-schemas.sh