Add upload features and tests

.devcontainer/devenv.yaml

@@ -23,10 +23,12 @@ forwardPorts:
   - 9005
   - 9006
   - 9007
+  - 9008
   - 9009
   - 9010
   - 9011
   - 9012
+  - 4566
   - 6080
 
 # Optional: Mount directories from host to container

.devcontainer/shared/devcontainer.json

@@ -25,10 +25,12 @@
     9005,
     9006,
     9007,
+    9008,
     9009,
     9010,
     9011,
     9012,
+    4566,
     6080
   ],
   "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04",

.devcontainer/user/devcontainer.json

@@ -25,10 +25,12 @@
     9005,
     9006,
     9007,
+    9008,
     9009,
     9010,
     9011,
     9012,
+    4566,
     6080
   ],
   "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04",

common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala

@@ -226,7 +226,7 @@ object S3Ops {
       .credentialsProvider(config.awsCredentials)
       .region(config.awsRegion)
 
-    config.awsLocalEndpointUri match {
+    config.awsLocalPresigningEndpointUri match {
       case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint)
         .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build()
       case _ => builder.build()

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala

@@ -30,6 +30,16 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui
   val awsLocalEndpoint: Option[String] = if(isDev) stringOpt("aws.local.endpoint").filter(_.nonEmpty) else None
   override val awsLocalEndpointUri: Option[URI] = awsLocalEndpoint.map(new URI(_))
 
+  // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint`
+  // (used by the S3 client for container-internal calls) because the browser cannot resolve the
+  // container-network host. Falls back to `aws.local.endpoint` when unset.
+  val awsLocalPresigningEndpointUri: Option[URI] =
+    if (isDev) stringOpt("aws.local.presigningEndpoint")
+      .filter(_.nonEmpty)
+      .orElse(awsLocalEndpoint)
+      .map(new URI(_))
+    else None
+
   val useLocalAuth: Boolean = isDev && boolean("auth.useLocal")
 
   val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean

e2e-tests/README.md

@@ -55,7 +55,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses
 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded
    buckets, provisioned into LocalStack;
 4. per-service config, generated by `dev/script/generate-config/service-config.js`;
-5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all eight Play services.
+5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services.
 
 Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is
 exposed to the tests. `globalTeardown` stops everything and cleans up.

e2e-tests/global-setup.ts

@@ -3,10 +3,10 @@
  *
  * Boots the full local Grid stack with Testcontainers:
  *   1. a shared network,
- *   2. Elasticsearch + LocalStack (infrastructure),
+ *   2. Elasticsearch + LocalStack + imgops (infrastructure),
  *   3. the CloudFormation core stack + seeded buckets (provisioning),
  *   4. generated per-service config (reusing dev/script/generate-config),
- *   5. the pre-built `grid-e2e-ci` image running all eight services.
+ *   5. the pre-built `grid-e2e-ci` image running the Grid services under test.
  *
  * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started
  * containers are stashed for `global-teardown.ts`.
@@ -23,13 +23,19 @@ import {
   ELASTICSEARCH_IMAGE,
   GRID_ALIAS,
   GRID_IMAGE,
+  IMGOPS_ALIAS,
+  IMGOPS_CONTEXT,
+  IMGOPS_IMAGE,
+  IMGOPS_NGINX_CONF,
+  IMGOPS_PORT,
   KAHUNA_PORT,
   LOCALSTACK_ALIAS,
   LOCALSTACK_IMAGE,
   LOCALSTACK_PORT,
   MEDIA_API_PORT,
   PROXY_IMAGE,
   REGION,
+  REPO_ROOT,
   SERVICE_PORTS,
   URLS_FILE,
 } from './testcontainers/constants';
@@ -95,6 +101,8 @@ function buildCaddyfile(coreStackProps: Record<string, string>): string {
   const appServices: Record<string, number> = {
     [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna,
     [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'],
+    [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'],
+    [`loader-projection.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'],
     [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper,
     [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall,
     [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'],
@@ -106,7 +114,7 @@ function buildCaddyfile(coreStackProps: Record<string, string>): string {
   // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended.
   const imageBuckets: Record<string, string> = {
     [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket,
-    [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket,
+    [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket
   };
 
   const blocks: string[] = [];
@@ -126,6 +134,11 @@ function buildCaddyfile(coreStackProps: Record<string, string>): string {
     `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`,
   );
 
+  // On-the-fly image resizing (optimised / full-screen views) -> the imgops container.
+  blocks.push(
+    `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`,
+  );
+
   return `${blocks.join('\n\n')}\n`;
 }
 
@@ -171,13 +184,28 @@ async function globalSetup(): Promise<void> {
     .start();
   started.push(localstack);
 
+  // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to
+  // the `localstack` alias on 4566, so it shares this network.
+  const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, {
+    deleteOnExit: false,
+  });
+  const imgops = await imgopsImage
+    .withNetwork(network)
+    .withNetworkAliases(IMGOPS_ALIAS)
+    .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }])
+    .withExposedPorts({ container: 80, host: IMGOPS_PORT })
+    .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200))
+    .withStartupTimeout(120_000)
+    .start();
+  started.push(imgops);
+
   // Provisioning + config generation
   const coreStackProps = await provisionCoreStack(localstack.getConnectionUri());
 
   const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-'));
   generateServiceConfig(configDir, coreStackProps);
 
-  // All eight Grid services run inside this single container and talk to each
+  // All Grid services under test run inside this single container and talk to each
   // other over its localhost. Each is published on the fixed host port its
   // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's
   // dev-nginx routes the https://*.media.<domain> domains straight into this
@@ -192,6 +220,9 @@ async function globalSetup(): Promise<void> {
       // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both.
       { source: configDir, target: '/root/.grid', mode: 'ro' },
       { source: configDir, target: '/etc/grid', mode: 'ro' },
+      // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo
+      // over /build so host edits recompile live.
+      ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]),
     ])
     .withEnvironment({
       AWS_ACCESS_KEY_ID: 'test',
@@ -200,7 +231,11 @@ async function globalSetup(): Promise<void> {
       AWS_DEFAULT_REGION: REGION,
       AWS_CBOR_DISABLE: 'true',
     })
-    .withWaitStrategy(Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200))
+    .withWaitStrategy(
+      Wait.forAll(Object.values(SERVICE_PORTS).map(port =>
+        Wait.forHttp('/management/healthcheck', port).forStatusCode(200),
+      ))
+    )
     .withStartupTimeout(startupTimeoutMs);
 
   if (process.env.GRID_DEBUG) {

e2e-tests/images/Dockerfile

@@ -1,14 +1,14 @@
 # syntax=docker/dockerfile:1
 
-# Single-container image that builds and runs eight Grid Play services
-# (auth, collections, cropper, kahuna, leases, media-api, metadata-editor,
-# thrall) together with Kahuna's frontend.
+# Single-container image that builds and runs all Grid Play services under test
+# (auth, collections, cropper, image-loader, kahuna, leases, media-api,
+# metadata-editor, thrall) together with Kahuna's frontend.
 #
 # This one Dockerfile produces two images, selected with `--target`:
 #
-#   * ci (grid-e2e-ci)  — CI/production style. Stages pre-compiled artefacts
-#                           and runs them under a slim JRE.
-#       docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci .
+#   * ci (grid-e2e-ci)  — CI/production style. Stages pre-compiled artefacts and
+#                           runs them under a slim JRE. docker build --target ci
+#       -f e2e-tests/images/Dockerfile -t grid-e2e-ci .
 #
 #   * dev (grid-e2e-dev)      — local development. Runs the services under
 #                           `sbt <service>/run` (Play dev mode) with the repo
@@ -75,6 +75,7 @@ RUN sbt \
       "auth/stage" \
       "collections/stage" \
       "cropper/stage" \
+      "image-loader/stage" \
       "kahuna/stage" \
       "leases/stage" \
       "media-api/stage" \
@@ -97,14 +98,15 @@ RUN set -eux; \
 COPY --from=backend /build/auth/target/universal/stage            /usr/share/auth
 COPY --from=backend /build/collections/target/universal/stage     /usr/share/collections
 COPY --from=backend /build/cropper/target/universal/stage         /usr/share/cropper
+COPY --from=backend /build/image-loader/target/universal/stage    /usr/share/image-loader
 COPY --from=backend /build/kahuna/target/universal/stage          /usr/share/kahuna
 COPY --from=backend /build/leases/target/universal/stage          /usr/share/leases
 COPY --from=backend /build/media-api/target/universal/stage       /usr/share/media-api
 COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/metadata-editor
 COPY --from=backend /build/thrall/target/universal/stage          /usr/share/thrall
 
 RUN set -eux; \
-    for service in auth collections cropper kahuna leases media-api metadata-editor thrall; do \
+    for service in auth collections cropper image-loader kahuna leases media-api metadata-editor thrall; do \
       mkdir -p "/var/log/$service"; \
     done
 
@@ -115,8 +117,8 @@ RUN chmod +x /usr/local/bin/entrypoint.ci.sh
 # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility.
 ENV AWS_CBOR_DISABLE=true
 
-# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases
-EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012
+# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases
+EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.ci.sh"]
 
@@ -173,9 +175,9 @@ RUN chmod +x /usr/local/bin/entrypoint.dev.sh
 # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility.
 ENV AWS_CBOR_DISABLE=true
 
-# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases
+# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases
 # plus the JDWP debug port used when GRID_DEBUG is set.
-EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005
+EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 5005
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"]
 

e2e-tests/images/README.md

@@ -1,7 +1,7 @@
 c# Grid E2E test images
 
-A single `Dockerfile` produces two single-container images that run eight
-Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter:
+A single `Dockerfile` produces two single-container images that run all
+Grid Play services under test, plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter:
 - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness.
 - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt <svc>/run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below.
 
@@ -24,7 +24,7 @@ from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config with
 ```bash
 docker run --rm \
   -v "$PWD/my-config:/etc/grid:ro" \
-  -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \
+  -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9005:9005 -p 9006:9006 \
   -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \
   grid-e2e-[ci|dev]
 ```

e2e-tests/images/entrypoint.common.sh

@@ -9,6 +9,7 @@
 declare -A PORTS=(
   [media-api]=9001
   [thrall]=9002
+  [image-loader]=9003
   [kahuna]=9005
   [cropper]=9006
   [metadata-editor]=9007
@@ -17,7 +18,7 @@ declare -A PORTS=(
   [leases]=9012
 )
 
-export SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall"
+export SERVICES="auth collections cropper image-loader kahuna leases media-api metadata-editor thrall"
 
 # Install a TERM/INT trap that kills the given child pids, waits for them, and
 # exits cleanly. Pass the pids to tear down as arguments, e.g.

e2e-tests/package.json

@@ -7,7 +7,8 @@
     "test": "bddgen && playwright test",
     "test:headed": "bddgen && playwright test --headed",
     "test:report": "playwright show-report",
-    "test:ui": "bddgen && playwright test --ui-port 6080"
+    "test:ui": "bddgen && playwright test --ui-port 6080",
+    "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080"
   },
   "keywords": [],
   "author": "",

e2e-tests/testcontainers/config.ts

@@ -34,13 +34,20 @@ function rewriteEndpoints(conf: string): string {
   const guardianLocalstackUrl = `https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`;
   const legacyLocalstackUrl = 'http://localhost:4576';
   const localLocalstackUrl = `http://localhost:${LOCALSTACK_PORT}`;
-  return conf
+
+  const rewrittenConf = conf
     .split(guardianLocalstackUrl)
     .join(localstackUrl)
     .split(legacyLocalstackUrl)
     .join(localstackUrl)
     .split(localLocalstackUrl)
     .join(localstackUrl);
+
+  // The S3 client reaches LocalStack over the container network (localstack:4566), but
+  // presigned URLs are handed to the browser, which can only reach LocalStack via the
+  // `localstack.media.<domain>` vanity domain (dev-nginx locally, the Caddy proxy in CI).
+  // Sign against that host so the URLs resolve outside the container network.
+  return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`;
 }
 
 /**
@@ -89,6 +96,7 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr
     if (!conf) {
       throw new Error(`service-config.js did not produce config for '${service}'`);
     }
+
     fs.writeFileSync(path.join(configDir, `${service}.conf`), rewriteEndpoints(conf));
   }
 }

e2e-tests/testcontainers/constants.ts

@@ -22,6 +22,18 @@ export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0';
 /** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */
 export const PROXY_IMAGE = 'caddy:2.8-alpine';
 
+/**
+ * imgops: standalone nginx on-the-fly image resizer, built from dev/imgops at setup time.
+ * Its nginx.conf proxies to the `localstack` alias on 4566, so it shares the stack network.
+ * Published on IMGOPS_PORT, the fixed host port dev-nginx maps `media-imgops` to.
+ */
+export const IMGOPS_ALIAS = 'imgops';
+export const IMGOPS_PORT = 9008;
+export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops');
+export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf');
+/** Stable tag for the imgops image so it persists across runs and Docker reuses cached layers. */
+export const IMGOPS_IMAGE = 'grid-e2e-imgops';
+
 /** Network aliases the app container uses to reach the infrastructure containers. */
 export const ELASTICSEARCH_ALIAS = 'elasticsearch';
 export const LOCALSTACK_ALIAS = 'localstack';
@@ -33,6 +45,7 @@ export const GRID_ALIAS = 'grid-e2e-ci';
 export const SERVICE_PORTS: Record<string, number> = {
   'media-api': 9001,
   thrall: 9002,
+  'image-loader': 9003,
   kahuna: 9005,
   cropper: 9006,
   'metadata-editor': 9007,