AP-638: Implements e2e tests using Playwright

README.md

@@ -53,6 +53,28 @@ Airflow keeps ephemeral information in Redis but persists virtually all state (a
 docker compose down -v --remove-orphans
 ```
 
+## Testing
+
+With the stack running, execute the tests by exec-ing pytest in one of the airflow containers (cli works well). A full rundown of pytest flags is out-of-scope for this README, but here are some common use-cases to get you going:
+
+```sh
+# Run all the tests
+docker compose exec airflow-cli python3 -m pytest
+
+# Run tests with a specific marker
+# Example: Only run the end-to-end (browser) tests
+docker compose exec airflow-cli python3 -m pytest -m e2e
+
+# Run a specific test file / folder
+# Example: Only run examples in the ./tests sub-folder
+docker compose exec airflow-cli python3 -m pytest test/tests
+
+# Run a test by name
+docker compose exec airflow-cli python3 -m pytest -k test_dags_load_with_no_errors
+```
+
+Test results / reports are written to the `./artifacts/pytest` directory.
+
 ## Configuration
 
 Airflow's configuration is propagated by environment variables [defined upstream by Airflow](https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html) and that are defined and handled in the base image. You can use a `.env` file or pass them.

docker-compose.yml

@@ -237,6 +237,12 @@ services:
     volumes:
       - ./keycloak/config:/config
 
+  playwright:
+    build: ./playwright
+    ports:
+      - 127.0.0.1:3000:3000
+    restart: unless-stopped
+
   postgres:
     environment:
       POSTGRES_USER: airflow

example.env

@@ -33,3 +33,12 @@ AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-blah-blah-blah
 AWS_MODEL_ID=us.anthropic.claude-haiku-4-5-20251001-v1:0
 AWS_MODEL_LABEL=Claude Haiku 4.5
 AWS_MODEL_PROVIDER=anthropic
+
+# --- E2E browser test configuration (test/e2e/) ---
+# To run pytest from your host workstation (not `docker compose exec`),
+# uncomment and override the WS endpoint to reach the sidecar via its
+# published port:
+#PYTEST_PLAYWRIGHT_WS_ENDPOINT=
+# Set these to run against non-localhost airflow or keycloak instances.
+#PYTEST_AIRFLOW_BASE_URL=
+#PYTEST_KEYCLOAK_BASE_URL=

playwright/Dockerfile

@@ -0,0 +1,26 @@
+# The Playwright server (here) and client (from requirements.txt)
+# must match down to the minor version. If they differ, expect to
+# see a clear error when running the e2e tests.
+ARG PLAYWRIGHT_VERSION=1.58.2
+
+FROM mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-jammy
+
+# socat runs per-port TCP forwarders that translate localhost:8080 and
+# localhost:8180 inside this container to airflow-apiserver:8080 and
+# keycloak:8180 on the compose network. This lets Chromium drive the
+# Airflow UI via the exact same localhost URLs a host-browser developer
+# uses, so webserver_config.py and Keycloak's KC_HOSTNAME stay prod-parity.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends socat \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY entrypoint.sh /usr/local/bin/playwright-sidecar-entrypoint.sh
+RUN chmod +x /usr/local/bin/playwright-sidecar-entrypoint.sh
+
+# Inherits from the top-level ARG
+ARG PLAYWRIGHT_VERSION
+ENV PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION
+RUN npm install -g playwright@${PLAYWRIGHT_VERSION}
+
+ENTRYPOINT ["/usr/local/bin/playwright-sidecar-entrypoint.sh"]
+CMD ["playwright", "run-server", "--port", "3000", "--host", "0.0.0.0", "--path", "/ws"]

playwright/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+# Port forwarders: make localhost:<port> inside this container transparently
+# reach the named compose services. See this container's Dockerfile for why.
+socat TCP-LISTEN:8080,fork,reuseaddr TCP:airflow-apiserver:8080 &
+socat TCP-LISTEN:8180,fork,reuseaddr TCP:keycloak:8180 &
+
+exec "$@"

pyproject.toml

@@ -26,6 +26,10 @@ license = "MIT"
 [project.optional-dependencies]
 test = [
     "pytest",
+    # Keep the playwright client aligned with the minor version
+    # of the playwright service in the Compose file.
+    "playwright ~= 1.58.0",
+    "pytest-playwright",
 ]
 lint = [
     "mypy ~= 1.17.1",
@@ -56,4 +60,14 @@ disable = ["expression-not-assigned"]
 
 [tool.pytest]
 minversion = "9.0"
-addopts = ["-v", "-s", "--junit-xml=artifacts/pytest/airflow_test.xml"]
+addopts = [
+    "-v",
+    "-s",
+    "--junit-xml=artifacts/pytest/airflow_test.xml",
+    "--tracing=retain-on-failure",
+    "--screenshot=only-on-failure",
+    "--output=artifacts/pytest/e2e",
+]
+markers = [
+    "e2e: end-to-end browser tests (require playwright sidecar and running Airflow stack)",
+]

requirements.txt

@@ -521,6 +521,7 @@ greenlet==3.3.2 \
     --hash=sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9
     # via
     #   greenback
+    #   playwright
     #   sqlalchemy
 grpcio==1.80.0 \
     --hash=sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1 \
@@ -1398,6 +1399,18 @@ pendulum==3.2.0 \
     # via
     #   apache-airflow-core
     #   apache-airflow-task-sdk
+playwright==1.58.0 \
+    --hash=sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117 \
+    --hash=sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99 \
+    --hash=sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b \
+    --hash=sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71 \
+    --hash=sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa \
+    --hash=sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606 \
+    --hash=sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8 \
+    --hash=sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b
+    # via
+    #   mokelumne (pyproject.toml)
+    #   pytest-playwright
 pluggy==1.6.0 \
     --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \
     --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746
@@ -1595,6 +1608,10 @@ pydantic-settings==2.13.1 \
     --hash=sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025 \
     --hash=sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237
     # via fastapi
+pyee==13.0.1 \
+    --hash=sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8 \
+    --hash=sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228
+    # via playwright
 pygments==2.20.0 \
     --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \
     --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176
@@ -1621,6 +1638,17 @@ pymarc==5.3.1 \
 pytest==9.0.3 \
     --hash=sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9 \
     --hash=sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c
+    # via
+    #   mokelumne (pyproject.toml)
+    #   pytest-base-url
+    #   pytest-playwright
+pytest-base-url==2.1.0 \
+    --hash=sha256:02748589a54f9e63fcbe62301d6b0496da0d10231b753e950c63e03aee745d45 \
+    --hash=sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6
+    # via pytest-playwright
+pytest-playwright==0.7.2 \
+    --hash=sha256:247b61123b28c7e8febb993a187a07e54f14a9aa04edc166f7a976d88f04c770 \
+    --hash=sha256:8084e015b2b3ecff483c2160f1c8219b38b66c0d4578b23c0f700d1b0240ea38
     # via mokelumne (pyproject.toml)
 python-daemon==3.1.2 \
     --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \
@@ -1648,7 +1676,9 @@ python-multipart==0.0.24 \
 python-slugify==8.0.4 \
     --hash=sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8 \
     --hash=sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856
-    # via apache-airflow-core
+    # via
+    #   apache-airflow-core
+    #   pytest-playwright
 pyyaml==6.0.3 \
     --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \
     --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \
@@ -1741,6 +1771,7 @@ requests==2.33.1 \
     #   apache-airflow-core
     #   langsmith
     #   opentelemetry-exporter-otlp-proto-http
+    #   pytest-base-url
     #   python-tind-client
     #   requests-toolbelt
 requests-toolbelt==1.0.0 \
@@ -2133,6 +2164,7 @@ typing-extensions==4.15.0 \
     #   pydantic
     #   pydantic-core
     #   pydantic-extra-types
+    #   pyee
     #   referencing
     #   rich-toolkit
     #   sqlalchemy

test/e2e/conftest.py

@@ -0,0 +1,116 @@
+"""
+conftest.py
+
+Fixtures and helpers for e2e tests.
+"""
+
+import os
+import re
+
+from collections.abc import Generator
+
+import pytest
+
+from playwright.sync_api import Browser, BrowserContext, Page, Playwright
+
+
+# Defaults match a host-browser developer's URLs. Chromium inside the
+# Playwright sidecar reaches these via localhost TCP forwarders installed
+# by playwright/entrypoint.sh, so the OAuth flow hits the same code paths
+# as a human driving the UI from their workstation. Host devs running
+# pytest from their shell only need to override PYTEST_PLAYWRIGHT_WS_ENDPOINT.
+AIRFLOW_BASE_URL = os.environ.get("PYTEST_AIRFLOW_BASE_URL", "http://localhost:8080")
+KEYCLOAK_BASE_URL = os.environ.get("PYTEST_KEYCLOAK_BASE_URL", "http://localhost:8180")
+PLAYWRIGHT_WS_ENDPOINT = os.environ.get("PYTEST_PLAYWRIGHT_WS_ENDPOINT", "ws://playwright:3000/ws")
+
+
+def airflow_url(path: str = ".*") -> re.Pattern[str]:
+    """
+    Generates AirFlow URL matchers for tests
+
+    Use this when you want to assert that you're on AirFlow. Use `path`
+    to narrow down the expected path.
+
+    Ex.
+       page.wait_for_url(airflow_url('/security/users.*'))
+    """
+    return re.compile(f"^{re.escape(AIRFLOW_BASE_URL)}{path}")
+
+
+def airflow_login_url() -> re.Pattern[str]:
+    """
+    Shorthand for matching AirFlow's login page
+    """
+    return airflow_url("/auth/login.*")
+
+
+def keycloak_url(path: str = ".*") -> re.Pattern[str]:
+    """
+    Generates KeyCloak URL matchers for tests
+
+    Use this when you want to assert that you're on KeyCloak.
+    """
+    return re.compile(f"^{re.escape(KEYCLOAK_BASE_URL)}{path}")
+
+
+def keycloak_login_url() -> re.Pattern[str]:
+    """
+    Shorthand for matching KeyCloak's mocked CalNet login page
+    """
+    return keycloak_url("/realms/berkeley-local/protocol/openid-connect/auth.*")
+
+
+def login(page: Page, username: str, password: str) -> None:
+    """Walk a fresh page through the Airflow -> Keycloak OIDC login flow.
+
+    Leaves the page on whatever Airflow returns after the post-login redirect —
+    the caller is responsible for asserting the authorized/denied outcome.
+    """
+    page.goto("/")
+    page.wait_for_url(airflow_login_url())
+    page.locator("#btn-signin-keycloak").click()
+    page.wait_for_url(keycloak_login_url())
+    page.locator("#username").fill(username)
+    page.locator("#password").fill(password)
+    page.locator("#kc-login").click()
+    page.wait_for_url(airflow_url())
+
+
+@pytest.fixture
+def context(context) -> Generator[BrowserContext, None, None]:
+    """
+    Shorten pytest-playwright's default timeouts to 5s
+    """
+    context.set_default_timeout(5000)
+    context.set_default_navigation_timeout(5000)
+    yield context
+
+
+@pytest.fixture
+def page_as_testpublic(page: Page) -> Page:
+    login(page, 'testpublic', 'testpublic')
+    return page
+
+
+@pytest.fixture
+def page_as_testuser(page: Page) -> Page:
+    login(page, 'testuser', 'testuser')
+    return page
+
+
+@pytest.fixture
+def page_as_testadmin(page: Page) -> Page:
+    login(page, 'testadmin', 'testadmin')
+    return page
+
+
+@pytest.fixture(scope="session")
+def browser(playwright: Playwright) -> Generator[Browser, None, None]:
+    browser = playwright.chromium.connect(PLAYWRIGHT_WS_ENDPOINT)
+    yield browser
+    browser.close()
+
+
+@pytest.fixture(scope="session")
+def browser_context_args(browser_context_args: dict) -> dict:
+    return {**browser_context_args, "base_url": AIRFLOW_BASE_URL}

test/e2e/test_auth.py

@@ -0,0 +1,50 @@
+"""
+test_auth.py
+
+Playwright-driven e2e tests for the OIDC login flow.
+"""
+
+import pytest
+
+from playwright.sync_api import Page, expect
+
+from .conftest import airflow_login_url, airflow_url
+
+pytestmark = pytest.mark.e2e
+
+
+# Matches the main div of the homepage. Absent from the login page.
+HOMEPAGE_MARKER = '[data-testid="main-content"]'
+
+# Matches the Admin tab in the left navbar.
+ADMIN_MARKER = 'button[aria-label="Admin"]'
+
+
+def test_airflow_requires_login(page: Page) -> None:
+    page.goto("/")
+    expect(page).to_have_url(airflow_login_url())
+    expect(page.locator("#btn-signin-keycloak")).to_be_visible()
+
+
+def test_public_user_cannot_view_homepage(page_as_testpublic: Page) -> None:
+    page_as_testpublic.goto('/')
+    expect(page_as_testpublic).to_have_url(airflow_login_url())
+    expect(page_as_testpublic.locator(HOMEPAGE_MARKER)).not_to_be_visible()
+
+
+def test_regular_user_can_view_homepage(page_as_testuser: Page) -> None:
+    page_as_testuser.goto('/')
+    expect(page_as_testuser).to_have_url(airflow_url("/"))
+    expect(page_as_testuser.locator(HOMEPAGE_MARKER)).to_be_visible()
+
+
+def test_regular_user_cannot_view_admin_tab(page_as_testuser: Page) -> None:
+    page_as_testuser.goto('/')
+    expect(page_as_testuser).to_have_url(airflow_url("/"))
+    expect(page_as_testuser.locator(ADMIN_MARKER)).not_to_be_visible()
+
+
+def test_admin_user_can_view_admin_tab(page_as_testadmin: Page) -> None:
+    page_as_testadmin.goto('/')
+    expect(page_as_testadmin).to_have_url(airflow_url("/"))
+    expect(page_as_testadmin.locator(ADMIN_MARKER)).to_be_visible()