diff --git a/.github/workflows/akto-guardrails-service.yml b/.github/workflows/akto-guardrails-service.yml index 1106447394..b53e90bb01 100644 --- a/.github/workflows/akto-guardrails-service.yml +++ b/.github/workflows/akto-guardrails-service.yml @@ -43,6 +43,16 @@ jobs: chmod +x setup-netrc.sh GITHUB_TOKEN=${{ secrets.AKTO_GITHUB_TOKEN }} ./setup-netrc.sh + - name: Unit tests + env: + GOPRIVATE: github.com/akto-api-security/* + GONOSUMDB: github.com/akto-api-security/* + run: | + cp apps/guardrails-service/container/.netrc ~/.netrc + chmod 600 ~/.netrc + cd apps/guardrails-service/container/src + go test ./... + - name: Build and push to DockerHub and ECR uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 494004baa5..5050a4851d 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -304,13 +304,20 @@ jobs: ECR_REGISTRY: aktosecurity IMAGE_TAG: ${{ steps.docker_tag.outputs.IMAGE_TAG }} GAR_ENABLED: ${{ vars.GCP_PROJECT_ID != '' }} + GOPRIVATE: github.com/akto-api-security/* + GONOSUMDB: github.com/akto-api-security/* run: | echo $IMAGE_TAG >> $GITHUB_STEP_SUMMARY - docker buildx create --use cd apps/guardrails-service/container chmod +x setup-netrc.sh # token expires in 1 year [2026-12-17] GITHUB_TOKEN=${{ secrets.AKTO_GITHUB_TOKEN }} ./setup-netrc.sh + cp .netrc ~/.netrc + chmod 600 ~/.netrc + cd src + go test ./... + cd .. + docker buildx create --use GAR_TAG="" if [ "$GAR_ENABLED" = "true" ]; then GAR_TAG="-t $GAR_REGISTRY/guardrails-service:$IMAGE_TAG" diff --git a/apps/guardrails-service/container/.env.example b/apps/guardrails-service/container/.env.example index a82522c7fa..9f251a4fc6 100644 --- a/apps/guardrails-service/container/.env.example +++ b/apps/guardrails-service/container/.env.example @@ -22,6 +22,14 @@ THREAT_DETECTION_API_URL=http://localhost:9090/api/threat_detection/record_malic THREAT_BACKEND_URL=https://tbs.akto.io THREAT_BACKEND_TOKEN=your-akto-api-token-here +# --- PII dummy redaction (before scanner / LLM / semantic cache) ------------ +# Normalizes PII to dummy placeholders (e.g. alice@x.com -> aaaa@aaaa.com) so +# cache keys and downstream scanners see stable text. Implemented in mcp-endpoint-shield. +# GUARDRAILS_PII_ENGINE=hyperscan # default; sync PII policy rules + dummy redaction; needs -tags hyperscan + libvectorscan +# GUARDRAILS_PII_ENGINE=regexp # force sequential Go regexp (no native lib; local dev without vectorscan) +# GUARDRAILS_SKIP_PII_REDACTION=true # load-test only: skip dummy redaction before scanners/cache +# GUARDRAILS_SKIP_PII_SYNC=true # load-test only: skip sync piiFilter policy rules (block/redact/warn) + # Logging LOG_LEVEL=info GIN_MODE=release diff --git a/apps/guardrails-service/container/DEPLOY.md b/apps/guardrails-service/container/DEPLOY.md index 3a4663b708..ff684b1453 100644 --- a/apps/guardrails-service/container/DEPLOY.md +++ b/apps/guardrails-service/container/DEPLOY.md @@ -5,6 +5,18 @@ GITHUB_TOKEN=xxx ./setup-netrc.sh docker buildx build --platform linux/amd64 -t guardrails-service:1.0.0_local . ``` +The image links **Vectorscan** (`libvectorscan5`) for Hyperscan-accelerated PII redaction. Runtime is `debian:bookworm-slim` (not distroless static) because CGO requires glibc + native libs. + +Unit tests run in CI before `docker build` (not inside the image) so each platform only compiles the Hyperscan binary once. BuildKit cache mounts (`go mod`, `go-build`) speed up rebuilds. + +# Local PII redaction smoke test (no go.mod bump) +```bash +cd apps/guardrails-service/container +AKTO_GATEWAY_ROOT=/path/to/akto-gateway ./scripts/local-test-pii.sh +``` + +See `README.md` → Development for full local build options (`GUARDRAILS_PII_ENGINE`, `link-local-akto-gateway.sh`). + # Push image to cloudflare ```bash npx wrangler containers push guardrails-service:1.0.0_local diff --git a/apps/guardrails-service/container/Dockerfile b/apps/guardrails-service/container/Dockerfile index bd97f32959..3a48124ba4 100644 --- a/apps/guardrails-service/container/Dockerfile +++ b/apps/guardrails-service/container/Dockerfile @@ -1,63 +1,53 @@ -# Build stage - use native platform for building (avoids QEMU emulation issues) -FROM --platform=$BUILDPLATFORM golang:1.24-alpine AS builder +# syntax=docker/dockerfile:1 +# Build stage — TARGETPLATFORM required for CGO + native Vectorscan per arch. +FROM --platform=$TARGETPLATFORM golang:1.24-bookworm AS builder -# Build arguments for cross-compilation (automatically set by BuildKit) ARG TARGETOS ARG TARGETARCH -# Install build dependencies -RUN apk add --no-cache git make +RUN apt-get update && apt-get install -y --no-install-recommends \ + git make gcc pkg-config libvectorscan-dev \ + && rm -rf /var/lib/apt/lists/* -# Set working directory WORKDIR /app -# Set GOPRIVATE to skip checksum verification for private repos ENV GOPRIVATE=github.com/akto-api-security/* ENV GONOSUMDB=github.com/akto-api-security/* +ENV CGO_ENABLED=1 -# Copy .netrc for GitHub authentication (create one with your token first) -# Run: echo -e "machine github.com\nlogin x-access-token\npassword YOUR_TOKEN" > .netrc COPY --chmod=600 .netrc /root/.netrc -# Copy go mod files from src directory COPY src/go.mod src/go.sum ./ -# Copy local replacement modules referenced by replace directives in go.mod -# (required so `go mod download` can resolve them) COPY src/internal/stub/ ./internal/stub/ -# Download dependencies -RUN go mod download +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download -# Clean up credentials after download RUN rm -f /root/.netrc -# Copy source code from src directory COPY src/ . -# Run unit tests before building the release binary. -RUN go test ./... +# Unit tests run in CI (see staging.yml). In-image tests doubled compile time because +# they ran without -tags hyperscan while the binary build requires CGO + hyperscan. +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -tags hyperscan -ldflags="-w -s" -o guardrails-service . -# Build the application with cross-compilation -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -installsuffix cgo -ldflags="-w -s" -o guardrails-service . +FROM debian:bookworm-slim -# Final stage - use distroless for smaller image and better security -FROM gcr.io/distroless/static-debian11:nonroot +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libvectorscan5 \ + && rm -rf /var/lib/apt/lists/* \ + && useradd -u 65532 -r -s /usr/sbin/nologin nonroot -# Copy ca-certificates from builder -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - -# Set working directory WORKDIR /app -# Copy the binary from builder COPY --from=builder /app/guardrails-service . -# Create directory for mcp-endpoint-shield data USER nonroot:nonroot -# Expose port EXPOSE 8080 -# Run the application ENTRYPOINT ["/app/guardrails-service"] diff --git a/apps/guardrails-service/container/README.md b/apps/guardrails-service/container/README.md index 16da530605..5fcbff5422 100644 --- a/apps/guardrails-service/container/README.md +++ b/apps/guardrails-service/container/README.md @@ -461,9 +461,52 @@ cd src go test ./... ``` -Tests also run automatically during the Docker image build (`docker build`). +Unit tests run in CI before the Docker image build (not inside `docker build`), so the image layer only compiles once with `-tags hyperscan`. ### Building for Production + +**VM / Docker (Hyperscan/Vectorscan — recommended)** + +The container Dockerfile builds with `CGO_ENABLED=1`, `-tags hyperscan`, and links `libvectorscan5` at runtime (Debian bookworm). No extra flags needed for `docker build`. + +**Local binary with Hyperscan (macOS / Linux)** + +```bash +# macOS +brew install vectorscan + +# Debian/Ubuntu +sudo apt install libvectorscan-dev pkg-config build-essential + +cd src +CGO_ENABLED=1 go build -tags hyperscan -o guardrails-service . +export GUARDRAILS_PII_ENGINE=hyperscan +./guardrails-service +``` + +**Local binary without native lib (regexp fallback)** + +```bash +cd src +CGO_ENABLED=0 go build -o guardrails-service . +export GUARDRAILS_PII_ENGINE=regexp +./guardrails-service +``` + +**Test against unpublished akto-gateway changes** + +```bash +cd apps/guardrails-service/container +./scripts/link-local-akto-gateway.sh /path/to/akto-gateway +./scripts/local-test-pii.sh +# when done: +./scripts/unlink-local-akto-gateway.sh +``` + +On startup, confirm the engine in logs: `[PIIFilterHandler] PII redact engine engine=hyperscan` (or `regexp` if Vectorscan is unavailable). + +**Legacy static build (no Hyperscan)** + ```bash cd src CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o guardrails-service . diff --git a/apps/guardrails-service/container/src/go.mod b/apps/guardrails-service/container/src/go.mod index dec89c53c2..96409971bd 100644 --- a/apps/guardrails-service/container/src/go.mod +++ b/apps/guardrails-service/container/src/go.mod @@ -3,7 +3,7 @@ module github.com/akto-api-security/guardrails-service go 1.24.11 require ( - github.com/akto-api-security/akto-endpoint-shield v0.0.0-20260710064642-ccde9ac87ed9 + github.com/akto-api-security/akto-endpoint-shield v0.0.0-20260710085344-f0d691e8f9d3 github.com/gin-gonic/gin v1.11.0 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/redis/go-redis/v9 v9.20.0 @@ -39,6 +39,7 @@ require ( github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/semgroup v1.2.0 // indirect + github.com/flier/gohs v1.2.3 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect github.com/gin-contrib/sse v1.1.0 // indirect @@ -133,6 +134,6 @@ require ( ) replace ( - github.com/akto-api-security/akto-endpoint-shield => github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710064642-ccde9ac87ed9 + github.com/akto-api-security/akto-endpoint-shield => github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710085344-f0d691e8f9d3 github.com/shoenig/go-m1cpu => ./internal/stub/go-m1cpu ) diff --git a/apps/guardrails-service/container/src/go.sum b/apps/guardrails-service/container/src/go.sum index 99feacfb9b..23279ecda9 100644 --- a/apps/guardrails-service/container/src/go.sum +++ b/apps/guardrails-service/container/src/go.sum @@ -29,8 +29,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710064642-ccde9ac87ed9 h1:Ik+ycNTdo+mMFMrDP9f9eqJwEaI94pQBo8k9Cl/QamA= -github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710064642-ccde9ac87ed9/go.mod h1:xygh4yYStZhELBqnSAgC/eflshcDFacvKQz2ZF5bFwY= +github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710085344-f0d691e8f9d3 h1:k71z57skkyo0LS/jqqUB2SBgpPj/xM1FRjbJ+CyCzOI= +github.com/akto-api-security/akto-gateway/mcp-endpoint-shield v0.0.0-20260710085344-f0d691e8f9d3/go.mod h1:EBsWKTBehxxc0NhsY07NPVYGeVIeUjBDm49O7ufaqCg= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -82,6 +82,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg= github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8= +github.com/flier/gohs v1.2.3 h1:GlsPhGTLfhLFQ6ZzNbXojyzIADldmC5OPGcvNd1Pteo= +github.com/flier/gohs v1.2.3/go.mod h1:MJr+IUI8QKDiE8lrDE4OhA++wRctvD9+UQB6GbOXf1c= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -147,6 +149,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -171,6 +175,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= @@ -276,6 +282,10 @@ github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/i github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=