From 2d8952e1d573bf246c9d84f629ba6f55c4a5bd05 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Tue, 14 Apr 2026 12:56:28 -0700 Subject: [PATCH] AP-638: Implement end-to-end testing w/Playwright - Implements end-to-end (e2e) browser-based tests using Playwright. - Resolves issues running the stack as a human developer vs. tests from the container by using `socat` in the playwright container to forward localhost: to the respective service. - Adds an initial set of tests for the OIDC login flow (admin, user, public, non-auth'd). --- README.md | 22 ++++++++ docker-compose.yml | 6 ++ example.env | 9 +++ playwright/Dockerfile | 26 +++++++++ playwright/entrypoint.sh | 9 +++ pyproject.toml | 16 +++++- requirements.txt | 34 +++++++++++- test/e2e/__init__.py | 0 test/e2e/conftest.py | 116 +++++++++++++++++++++++++++++++++++++++ test/e2e/test_auth.py | 50 +++++++++++++++++ 10 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 playwright/Dockerfile create mode 100644 playwright/entrypoint.sh create mode 100644 test/e2e/__init__.py create mode 100644 test/e2e/conftest.py create mode 100644 test/e2e/test_auth.py diff --git a/README.md b/README.md index 2f5c7b4..54e0b3a 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,28 @@ Airflow keeps ephemeral information in Redis but persists virtually all state (a docker compose down -v --remove-orphans ``` +## Testing + +With the stack running, execute the tests by exec-ing pytest in one of the airflow containers (cli works well). A full rundown of pytest flags is out-of-scope for this README, but here are some common use-cases to get you going: + +```sh +# Run all the tests +docker compose exec airflow-cli python3 -m pytest + +# Run tests with a specific marker +# Example: Only run the end-to-end (browser) tests +docker compose exec airflow-cli python3 -m pytest -m e2e + +# Run a specific test file / folder +# Example: Only run examples in the ./tests sub-folder +docker compose exec airflow-cli python3 -m pytest test/tests + +# Run a test by name +docker compose exec airflow-cli python3 -m pytest -k test_dags_load_with_no_errors +``` + +Test results / reports are written to the `./artifacts/pytest` directory. + ## Configuration Airflow's configuration is propagated by environment variables [defined upstream by Airflow](https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html) and that are defined and handled in the base image. You can use a `.env` file or pass them. diff --git a/docker-compose.yml b/docker-compose.yml index 2a0db2f..c3a9bf1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -237,6 +237,12 @@ services: volumes: - ./keycloak/config:/config + playwright: + build: ./playwright + ports: + - 127.0.0.1:3000:3000 + restart: unless-stopped + postgres: environment: POSTGRES_USER: airflow diff --git a/example.env b/example.env index 427c1e1..52dd42d 100644 --- a/example.env +++ b/example.env @@ -33,3 +33,12 @@ AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-blah-blah-blah AWS_MODEL_ID=us.anthropic.claude-haiku-4-5-20251001-v1:0 AWS_MODEL_LABEL=Claude Haiku 4.5 AWS_MODEL_PROVIDER=anthropic + +# --- E2E browser test configuration (test/e2e/) --- +# To run pytest from your host workstation (not `docker compose exec`), +# uncomment and override the WS endpoint to reach the sidecar via its +# published port: +#PYTEST_PLAYWRIGHT_WS_ENDPOINT= +# Set these to run against non-localhost airflow or keycloak instances. +#PYTEST_AIRFLOW_BASE_URL= +#PYTEST_KEYCLOAK_BASE_URL= diff --git a/playwright/Dockerfile b/playwright/Dockerfile new file mode 100644 index 0000000..e794b11 --- /dev/null +++ b/playwright/Dockerfile @@ -0,0 +1,26 @@ +# The Playwright server (here) and client (from requirements.txt) +# must match down to the minor version. If they differ, expect to +# see a clear error when running the e2e tests. +ARG PLAYWRIGHT_VERSION=1.58.2 + +FROM mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-jammy + +# socat runs per-port TCP forwarders that translate localhost:8080 and +# localhost:8180 inside this container to airflow-apiserver:8080 and +# keycloak:8180 on the compose network. This lets Chromium drive the +# Airflow UI via the exact same localhost URLs a host-browser developer +# uses, so webserver_config.py and Keycloak's KC_HOSTNAME stay prod-parity. +RUN apt-get update \ + && apt-get install -y --no-install-recommends socat \ + && rm -rf /var/lib/apt/lists/* + +COPY entrypoint.sh /usr/local/bin/playwright-sidecar-entrypoint.sh +RUN chmod +x /usr/local/bin/playwright-sidecar-entrypoint.sh + +# Inherits from the top-level ARG +ARG PLAYWRIGHT_VERSION +ENV PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION +RUN npm install -g playwright@${PLAYWRIGHT_VERSION} + +ENTRYPOINT ["/usr/local/bin/playwright-sidecar-entrypoint.sh"] +CMD ["playwright", "run-server", "--port", "3000", "--host", "0.0.0.0", "--path", "/ws"] diff --git a/playwright/entrypoint.sh b/playwright/entrypoint.sh new file mode 100644 index 0000000..c432809 --- /dev/null +++ b/playwright/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +# Port forwarders: make localhost: inside this container transparently +# reach the named compose services. See this container's Dockerfile for why. +socat TCP-LISTEN:8080,fork,reuseaddr TCP:airflow-apiserver:8080 & +socat TCP-LISTEN:8180,fork,reuseaddr TCP:keycloak:8180 & + +exec "$@" diff --git a/pyproject.toml b/pyproject.toml index 921d00b..2579c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,10 @@ license = "MIT" [project.optional-dependencies] test = [ "pytest", + # Keep the playwright client aligned with the minor version + # of the playwright service in the Compose file. + "playwright ~= 1.58.0", + "pytest-playwright", ] lint = [ "mypy ~= 1.17.1", @@ -56,4 +60,14 @@ disable = ["expression-not-assigned"] [tool.pytest] minversion = "9.0" -addopts = ["-v", "-s", "--junit-xml=artifacts/pytest/airflow_test.xml"] +addopts = [ + "-v", + "-s", + "--junit-xml=artifacts/pytest/airflow_test.xml", + "--tracing=retain-on-failure", + "--screenshot=only-on-failure", + "--output=artifacts/pytest/e2e", +] +markers = [ + "e2e: end-to-end browser tests (require playwright sidecar and running Airflow stack)", +] diff --git a/requirements.txt b/requirements.txt index d3bbeae..813c42c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -521,6 +521,7 @@ greenlet==3.3.2 \ --hash=sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9 # via # greenback + # playwright # sqlalchemy grpcio==1.80.0 \ --hash=sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1 \ @@ -1398,6 +1399,18 @@ pendulum==3.2.0 \ # via # apache-airflow-core # apache-airflow-task-sdk +playwright==1.58.0 \ + --hash=sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117 \ + --hash=sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99 \ + --hash=sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b \ + --hash=sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71 \ + --hash=sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa \ + --hash=sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606 \ + --hash=sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8 \ + --hash=sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b + # via + # mokelumne (pyproject.toml) + # pytest-playwright pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 @@ -1595,6 +1608,10 @@ pydantic-settings==2.13.1 \ --hash=sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025 \ --hash=sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237 # via fastapi +pyee==13.0.1 \ + --hash=sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8 \ + --hash=sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228 + # via playwright pygments==2.20.0 \ --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 @@ -1621,6 +1638,17 @@ pymarc==5.3.1 \ pytest==9.0.3 \ --hash=sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9 \ --hash=sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c + # via + # mokelumne (pyproject.toml) + # pytest-base-url + # pytest-playwright +pytest-base-url==2.1.0 \ + --hash=sha256:02748589a54f9e63fcbe62301d6b0496da0d10231b753e950c63e03aee745d45 \ + --hash=sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6 + # via pytest-playwright +pytest-playwright==0.7.2 \ + --hash=sha256:247b61123b28c7e8febb993a187a07e54f14a9aa04edc166f7a976d88f04c770 \ + --hash=sha256:8084e015b2b3ecff483c2160f1c8219b38b66c0d4578b23c0f700d1b0240ea38 # via mokelumne (pyproject.toml) python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ @@ -1648,7 +1676,9 @@ python-multipart==0.0.24 \ python-slugify==8.0.4 \ --hash=sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8 \ --hash=sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856 - # via apache-airflow-core + # via + # apache-airflow-core + # pytest-playwright pyyaml==6.0.3 \ --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ @@ -1741,6 +1771,7 @@ requests==2.33.1 \ # apache-airflow-core # langsmith # opentelemetry-exporter-otlp-proto-http + # pytest-base-url # python-tind-client # requests-toolbelt requests-toolbelt==1.0.0 \ @@ -2133,6 +2164,7 @@ typing-extensions==4.15.0 \ # pydantic # pydantic-core # pydantic-extra-types + # pyee # referencing # rich-toolkit # sqlalchemy diff --git a/test/e2e/__init__.py b/test/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/e2e/conftest.py b/test/e2e/conftest.py new file mode 100644 index 0000000..7e8e355 --- /dev/null +++ b/test/e2e/conftest.py @@ -0,0 +1,116 @@ +""" +conftest.py + +Fixtures and helpers for e2e tests. +""" + +import os +import re + +from collections.abc import Generator + +import pytest + +from playwright.sync_api import Browser, BrowserContext, Page, Playwright + + +# Defaults match a host-browser developer's URLs. Chromium inside the +# Playwright sidecar reaches these via localhost TCP forwarders installed +# by playwright/entrypoint.sh, so the OAuth flow hits the same code paths +# as a human driving the UI from their workstation. Host devs running +# pytest from their shell only need to override PYTEST_PLAYWRIGHT_WS_ENDPOINT. +AIRFLOW_BASE_URL = os.environ.get("PYTEST_AIRFLOW_BASE_URL", "http://localhost:8080") +KEYCLOAK_BASE_URL = os.environ.get("PYTEST_KEYCLOAK_BASE_URL", "http://localhost:8180") +PLAYWRIGHT_WS_ENDPOINT = os.environ.get("PYTEST_PLAYWRIGHT_WS_ENDPOINT", "ws://playwright:3000/ws") + + +def airflow_url(path: str = ".*") -> re.Pattern[str]: + """ + Generates AirFlow URL matchers for tests + + Use this when you want to assert that you're on AirFlow. Use `path` + to narrow down the expected path. + + Ex. + page.wait_for_url(airflow_url('/security/users.*')) + """ + return re.compile(f"^{re.escape(AIRFLOW_BASE_URL)}{path}") + + +def airflow_login_url() -> re.Pattern[str]: + """ + Shorthand for matching AirFlow's login page + """ + return airflow_url("/auth/login.*") + + +def keycloak_url(path: str = ".*") -> re.Pattern[str]: + """ + Generates KeyCloak URL matchers for tests + + Use this when you want to assert that you're on KeyCloak. + """ + return re.compile(f"^{re.escape(KEYCLOAK_BASE_URL)}{path}") + + +def keycloak_login_url() -> re.Pattern[str]: + """ + Shorthand for matching KeyCloak's mocked CalNet login page + """ + return keycloak_url("/realms/berkeley-local/protocol/openid-connect/auth.*") + + +def login(page: Page, username: str, password: str) -> None: + """Walk a fresh page through the Airflow -> Keycloak OIDC login flow. + + Leaves the page on whatever Airflow returns after the post-login redirect — + the caller is responsible for asserting the authorized/denied outcome. + """ + page.goto("/") + page.wait_for_url(airflow_login_url()) + page.locator("#btn-signin-keycloak").click() + page.wait_for_url(keycloak_login_url()) + page.locator("#username").fill(username) + page.locator("#password").fill(password) + page.locator("#kc-login").click() + page.wait_for_url(airflow_url()) + + +@pytest.fixture +def context(context) -> Generator[BrowserContext, None, None]: + """ + Shorten pytest-playwright's default timeouts to 5s + """ + context.set_default_timeout(5000) + context.set_default_navigation_timeout(5000) + yield context + + +@pytest.fixture +def page_as_testpublic(page: Page) -> Page: + login(page, 'testpublic', 'testpublic') + return page + + +@pytest.fixture +def page_as_testuser(page: Page) -> Page: + login(page, 'testuser', 'testuser') + return page + + +@pytest.fixture +def page_as_testadmin(page: Page) -> Page: + login(page, 'testadmin', 'testadmin') + return page + + +@pytest.fixture(scope="session") +def browser(playwright: Playwright) -> Generator[Browser, None, None]: + browser = playwright.chromium.connect(PLAYWRIGHT_WS_ENDPOINT) + yield browser + browser.close() + + +@pytest.fixture(scope="session") +def browser_context_args(browser_context_args: dict) -> dict: + return {**browser_context_args, "base_url": AIRFLOW_BASE_URL} diff --git a/test/e2e/test_auth.py b/test/e2e/test_auth.py new file mode 100644 index 0000000..236a227 --- /dev/null +++ b/test/e2e/test_auth.py @@ -0,0 +1,50 @@ +""" +test_auth.py + +Playwright-driven e2e tests for the OIDC login flow. +""" + +import pytest + +from playwright.sync_api import Page, expect + +from .conftest import airflow_login_url, airflow_url + +pytestmark = pytest.mark.e2e + + +# Matches the main div of the homepage. Absent from the login page. +HOMEPAGE_MARKER = '[data-testid="main-content"]' + +# Matches the Admin tab in the left navbar. +ADMIN_MARKER = 'button[aria-label="Admin"]' + + +def test_airflow_requires_login(page: Page) -> None: + page.goto("/") + expect(page).to_have_url(airflow_login_url()) + expect(page.locator("#btn-signin-keycloak")).to_be_visible() + + +def test_public_user_cannot_view_homepage(page_as_testpublic: Page) -> None: + page_as_testpublic.goto('/') + expect(page_as_testpublic).to_have_url(airflow_login_url()) + expect(page_as_testpublic.locator(HOMEPAGE_MARKER)).not_to_be_visible() + + +def test_regular_user_can_view_homepage(page_as_testuser: Page) -> None: + page_as_testuser.goto('/') + expect(page_as_testuser).to_have_url(airflow_url("/")) + expect(page_as_testuser.locator(HOMEPAGE_MARKER)).to_be_visible() + + +def test_regular_user_cannot_view_admin_tab(page_as_testuser: Page) -> None: + page_as_testuser.goto('/') + expect(page_as_testuser).to_have_url(airflow_url("/")) + expect(page_as_testuser.locator(ADMIN_MARKER)).not_to_be_visible() + + +def test_admin_user_can_view_admin_tab(page_as_testadmin: Page) -> None: + page_as_testadmin.goto('/') + expect(page_as_testadmin).to_have_url(airflow_url("/")) + expect(page_as_testadmin.locator(ADMIN_MARKER)).to_be_visible()