cromper (third time's the charm)

.env

@@ -1,5 +1,4 @@
 DEBUG=on
-DUMMY_COMPILER=on
 SECRET_KEY=django-insecure-nm#!8%zhc0wwi#m_*l9l)=m*6gs4&o_^-e5b5vj*k05&yaqc1
 DATABASE_URL=sqlite:///dev.db
 API_BASE=http://127.0.0.1:8000/api

.github/workflows/chores.yaml

@@ -14,10 +14,10 @@ jobs:
     if: github.repository == 'decompme/decomp.me'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.14"
 

.github/workflows/ci.yml

@@ -1,191 +1,190 @@
-name: PR
 on:
   push:
     branches:
       - main
   pull_request:
+
+name: "CI"
+
 jobs:
   backend_test_docker_dev:
     name: backend tests (docker dev container)
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
-      - name: Build decompme_backend image
+      - uses: actions/checkout@v7
+      - name: Build backend development image
+        run: docker build backend --target dev -t decompme_backend
+      - name: Run backend tests
         run: |-
-          docker build backend --target dev -t decompme_backend
-      - name: Run tests
-        run: |-
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
-          # NOTE: CI does not run as 'ubuntu' user
           export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          for dir in backend/.venv sandbox local_files compilers libraries; do
-            mkdir -p "$dir"
-            sudo chown "${USER_1000}:${USER_1000}" "$dir"
-          done
+          mkdir -p backend/.venv
+          sudo chown "${USER_1000}:${USER_1000}" backend/.venv
           docker run \
             -v $(pwd):/decomp.me \
-            -v $(pwd)/local_files:/local_files \
-            -v $(pwd)/compilers:/compilers \
-            -v $(pwd)/libraries:/libraries \
-            --security-opt apparmor=unconfined \
-            --security-opt seccomp=unconfined \
-            --cap-drop all \
-            --cap-add setuid \
-            --cap-add setgid \
-            --cap-add setfcap \
-            --tmpfs /sandbox/tmp:exec,uid=1000,gid=1000,size=64M,mode=0700 \
             --entrypoint /bin/bash \
-            -e COMPILER_BASE_PATH=/compilers \
-            -e LIBRARY_BASE_PATH=/libraries \
-            -e LOCAL_FILE_DIR=/local_files \
-            -e USE_SANDBOX_JAIL=on \
-            -e SANDBOX_DISABLE_PROC=true \
-            -e TIMEOUT_SCALE_FACTOR=10 \
-            -e DUMMY_COMPILER=1 \
+            -e DATABASE_URL=sqlite:///:memory: \
+            -e SECRET_KEY=secret-key-secret-key-secret-key-secret-key-secret-key-secret-key \
             decompme_backend \
-              -c 'cd /decomp.me/backend && \
-              uv sync --locked && \
-              uv run compilers/download.py --compilers-dir ${COMPILER_BASE_PATH} && \
-              uv run libraries/download.py --libraries-dir ${LIBRARY_BASE_PATH} && \
-              uv run python manage.py test'
+              -c 'cd /decomp.me/backend && uv sync --locked && uv run python manage.py test'
 
   backend_test_docker_prod:
     name: backend tests (docker prod container)
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
-      - name: Build decompme_backend image
-        run: |-
-          docker build backend --target prod -t decompme_backend
-      - name: Fetch compilers and libraries
+      - uses: actions/checkout@v7
+      - name: Build backend production image
+        run: docker build backend --target prod -t decompme_backend
+      - name: Run backend tests
         run: |-
-          python3 -m pip install requests
-          python3 backend/compilers/download.py
-          python3 backend/libraries/download.py
-          export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          sudo chown -R ${USER_1000}:${USER_1000} backend/compilers
-          sudo chown -R ${USER_1000}:${USER_1000} backend/libraries
-      - name: Run tests
+          docker run \
+            --entrypoint /bin/bash \
+            -e DATABASE_URL=sqlite:///:memory: \
+            -e SECRET_KEY=secret-key-secret-key-secret-key-secret-key-secret-key-secret-key \
+            decompme_backend \
+              -c 'uv run python manage.py test'
+
+  cromper_test:
+    name: cromper tests (docker dev container)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - name: Build cromper development image
+        run: docker build cromper --target dev -t decompme_cromper
+      - name: Run cromper tests
         run: |-
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
           export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          mkdir -p local_files && sudo chown ${USER_1000}:${USER_1000} local_files
-          docker run \
-            -v $(pwd)/local_files:/local_files \
-            -v $(pwd)/backend/compilers:/compilers \
-            -v $(pwd)/backend/libraries:/libraries \
+          for dir in cromper/.venv cromper/compilers cromper/libraries; do
+            mkdir -p "$dir"
+            sudo chown -R "$USER_1000:$USER_1000" "$dir"
+          done
+          docker run --rm \
+            -v $(pwd)/cromper:/cromper \
             --security-opt apparmor=unconfined \
             --security-opt seccomp=unconfined \
             --cap-drop all \
             --cap-add setuid \
             --cap-add setgid \
             --cap-add setfcap \
-            --tmpfs /sandbox/tmp:exec,uid=1000,gid=1000,size=64M,mode=0700 \
+            --tmpfs /tmp/sandbox:exec,uid=1000,gid=1000,size=256M,mode=0700 \
             --entrypoint /bin/bash \
-            -e LOCAL_FILE_DIR=/local_files \
-            -e COMPILER_BASE_PATH=/compilers \
-            -e LIBRARY_BASE_PATH=/libraries \
-            -e USE_SANDBOX_JAIL=on \
+            -e COMPILER_BASE_PATH=/cromper/compilers \
+            -e LIBRARY_BASE_PATH=/cromper/libraries \
+            -e USE_SANDBOX_JAIL=true \
             -e SANDBOX_DISABLE_PROC=true \
-            -e TIMEOUT_SCALE_FACTOR=10 \
-            -e DUMMY_COMPILER=1 \
-            -e DATABASE_URL=sqlite:///:memory: \
-            -e SECRET_KEY=secret-key-secret-key-secret-key-secret-key-secret-key-secret-key \
-            decompme_backend \
-              -c 'uv run python manage.py test'
+            decompme_cromper \
+              -c 'cd /cromper && \
+              uv sync --locked --all-extras && \
+              uv run --locked compilers/download.py --compilers-dir $COMPILER_BASE_PATH && \
+              uv run --locked libraries/download.py --libraries-dir $LIBRARY_BASE_PATH && \
+              uv run --locked pytest'
 
   docker_compose_test:
     name: test docker compose
     runs-on: ubuntu-latest
-    timeout-minutes: 10
-
+    timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
+
+      - name: Allow nsjail user namespaces
+        run: |
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
 
       - name: Fetch compilers and libraries
         run: |
           python3 -m pip install requests
-          python3 backend/compilers/download.py
-          python3 backend/libraries/download.py
+          python3 cromper/compilers/download.py
+          python3 cromper/libraries/download.py
           export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          sudo chown -R ${USER_1000}:${USER_1000} backend/compilers
-          sudo chown -R ${USER_1000}:${USER_1000} backend/libraries
+          sudo chown -R "${USER_1000}:${USER_1000}" cromper/compilers cromper/libraries
 
-      - name: Setup dummy production env
+      - name: Setup dummy production environment
         run: |
           echo 'POSTGRES_USER=decompme' >> docker.prod.env
           echo 'POSTGRES_PASSWORD=decompme' >> docker.prod.env
           echo 'SECRET_KEY=secret-key-secret-key-secret-key-secret-key-secret-key-secret-key' >> docker.prod.env
           echo 'DATABASE_URL=psql://decompme:decompme@postgres:5432/decompme' >> docker.prod.env
           echo 'CONN_MAX_AGE=60' >> docker.prod.env
           echo 'CONN_HEALTH_CHECKS="true"' >> docker.prod.env
-          echo 'SANDBOX_DISABLE_PROC="true"' >> docker.prod.env
           echo 'ALLOWED_HOSTS="backend,backend-green,backend-blue,localhost,127.0.0.1"' >> docker.prod.env
+          echo 'CROMPER_URL=http://cromper:8888' >> docker.prod.env
+          echo 'CROMPER_PORT=8888' >> docker.prod.env
           echo 'USE_SANDBOX_JAIL="on"' >> docker.prod.env
+          echo 'SANDBOX_DISABLE_PROC="true"' >> docker.prod.env
           echo 'CI=true' >> docker.prod.env
           echo 'ACTIVE_SLOT=blue' >> .deploy.env
           echo 'BLUE_TAG=latest' >> .deploy.env
           echo 'GREEN_TAG=latest' >> .deploy.env
           echo 'NGINX_TAG=latest' >> .deploy.env
+          echo 'CROMPER_TAG=latest' >> .deploy.env
 
       - name: Comment out SSL server configuration from nginx
         run: |
           sed -i '/{{HTTPS_SERVER_BLOCK_START}}/,/{{HTTPS_SERVER_BLOCK_END}}/s/^/#/' nginx/production/default.conf
 
       - name: Prepare production runtime config
         run: |
-          cp ./nginx/production/runtime/geo.conf.example      ./nginx/production/runtime/geo.conf
+          cp ./nginx/production/runtime/geo.conf.example ./nginx/production/runtime/geo.conf
           cp ./nginx/production/runtime/upstream.conf.example ./nginx/production/runtime/upstream.conf
 
       - name: Build production images
         run: |
-          docker compose -f docker-compose.prod.yaml build nginx backend-blue frontend-blue
+          docker compose -f docker-compose.prod.yaml build cromper nginx backend-blue frontend-blue
 
       - name: Ensure production services
         run: |
-          timeout 60s python3 deploy.py ensure || true
+          timeout 120s python3 deploy.py ensure
 
       - name: View docker compose logs (blue)
+        if: always()
         run: |
           timeout 15s docker compose -f docker-compose.prod.yaml logs -f || true
 
       - name: Check docker compose status (blue)
-        run: |
-          docker compose -f docker-compose.prod.yaml ps
+        run: docker compose -f docker-compose.prod.yaml ps
 
       - name: Deploy green slot
-        run: |
-          python3 deploy.py deploy --no-pull latest green
+        run: python3 deploy.py deploy --no-pull latest green
 
       - name: View docker compose logs (green)
+        if: always()
         run: |
           timeout 15s docker compose -f docker-compose.prod.yaml logs -f || true
 
       - name: Check docker compose status (green)
+        run: docker compose -f docker-compose.prod.yaml ps
+
+      - name: Run integration tests
         run: |
-          docker compose -f docker-compose.prod.yaml ps
+          docker compose -f docker-compose.prod.yaml run --rm --no-deps \
+            -v "$PWD/integration_tests:/integration_tests" \
+            -v "$PWD/backend/pyproject.toml:/backend/pyproject.toml:ro" \
+            -v "$PWD/cromper:/cromper" \
+            --entrypoint /bin/bash \
+            backend-blue \
+            -lc 'export UV_PROJECT_ENVIRONMENT=/tmp/integration-venv && cd /integration_tests && uv sync --locked --all-extras && uv run pytest -v'
 
       - name: Shut everything down
         if: always()
-        run: |
-          docker compose -f docker-compose.prod.yaml down
+        run: docker compose -f docker-compose.prod.yaml down
 
-      - name: Print out image sizes
+      - name: Print image sizes
         if: always()
-        run: |
-          docker image ls
+        run: docker image ls
 
   frontend_lint:
-    name: biome
+    name: frontend lint and tests
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Setup Node.js 24
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: 24
           cache: yarn
@@ -203,39 +202,44 @@ jobs:
           cd frontend
           yarn test
 
-  mypy:
-    name: mypy
+  backend_lint:
+    name: backend lint and type check
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Install uv
-        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
+        uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
       - name: Setup Python 3.14
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.14"
       - run: |-
           cd backend
-          uv sync --locked
+          uv sync --locked --all-extras
           uv run mypy
+          uv run ruff check
+          uv run ruff format --check
 
-  ruff:
-    name: ruff
+  cromper_lint:
+    name: cromper lint and type check
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Install uv
-        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
+        uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
       - name: Setup Python 3.14
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.14"
       - run: |-
-          cd backend
-          uv sync --locked
-          uv run ruff check .
-          uv run ruff format --check .
+          cd cromper
+          uv sync --locked --all-extras
+          uv run --locked mypy .
+          uv run --locked ruff check
+          uv run --locked ruff format --check

.github/workflows/ghcr.yaml

@@ -21,6 +21,9 @@ jobs:
           - name: backend
             context: backend
             target: prod
+          - name: cromper
+            context: cromper
+            target: prod
           - name: frontend
             context: frontend
             target: prod
@@ -29,7 +32,7 @@ jobs:
 
     steps:
       - name: Check out repo
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Log in to GHCR
         uses: docker/login-action@v4

backend/.dockerignore

@@ -8,10 +8,7 @@ __pycache__/
 Dockerfile*
 docker.*.env
 
-# compilers
-compilers/*/**
-!compilers/download.py
 
-# libraries
-libraries/*/**
-!libraries/download.py
+# TEMPORARY: because i'm switching between main/cromper branches
+compilers/**
+libraries/**

backend/.gitignore

@@ -0,0 +1,3 @@
+# TEMPORARY: because i'm switching between main/cromper branches
+compilers/**
+libraries/**