From d8b68c102f1a07bbecb26333737c20b5338ea0c6 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 29 Jul 2026 10:01:22 -0700 Subject: [PATCH 1/9] docs: reorganize inference gateway guides Fixes: #714 Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 7 +++--- ...=> llmisvc-inference-gateway-extension.md} | 6 +++-- .../llmisvc/llmisvc-inference-gateways.md | 22 +++++++++++++++++++ docusaurus.config.ts | 4 ++++ sidebars.ts | 8 +++++-- 5 files changed, 40 insertions(+), 7 deletions(-) rename docs/model-serving/generative-inference/llmisvc/{llmisvc-envoy-ai-gateway.md => llmisvc-inference-gateway-extension.md} (99%) create mode 100644 docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index d7851ffc8..0acd948a6 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -1,9 +1,9 @@ --- -title: LLMInferenceService with AgentGateway -description: How to integrate KServe LLMInferenceService with AgentGateway for LLM-aware routing, token tracking, and GenAI telemetry +title: LLMInferenceService with agentgateway +description: How to integrate KServe LLMInferenceService with agentgateway for LLM-aware routing, token tracking, and GenAI telemetry --- -# LLMInferenceService with AgentGateway +# LLMInferenceService with agentgateway This guide walks through integrating a KServe LLMInferenceService with [AgentGateway](https://agentgateway.dev) to enable LLM-aware routing with token tracking, GenAI telemetry (OpenTelemetry semantic conventions), and token-based rate limiting. The key mechanism is overriding the HTTPRoute `backendRef` via `LLMInferenceServiceConfig` so that KServe's auto-generated routes point to an `AgentgatewayBackend` instead of a plain `InferencePool` or `Service`. @@ -383,6 +383,7 @@ This approach is generic. Any gateway that supports custom `backendRef` kinds vi ## Next Steps +- Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). - Explore the [AgentGateway documentation](https://agentgateway.dev/docs) for advanced features like content filtering and cost tracking. - Learn more about [LLMInferenceServiceConfig composition](./llmisvc-config-composition.md) for managing configurations across multiple services. - See the [LLMInferenceService Configuration Guide](./llmisvc-configuration.md) for the full `spec.router.route.http` reference. diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md similarity index 99% rename from docs/model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway.md rename to docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md index ef122405b..119305659 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md @@ -1,9 +1,9 @@ --- -title: LLMInferenceService with Inference Gateway Extension (IGW) +title: Gateway API Inference Extension with Envoy AI Gateway description: How to integrate KServe LLMInferenceService with Envoy AI Gateway to manage LLM traffic and usage-based rate limits --- -# LLMInferenceService with Inference Gateway Extension (IGW) +# Gateway API Inference Extension with Envoy AI Gateway This tutorial walks through deploying a KServe LLMInferenceService that wraps [llm-d](https://llm-d.ai/) — which implements the [Gateway API Inference Extension](https://gateway-api-inference-extension.sigs.k8s.io/) (the llm-d router and inference pool) — and fronts it with Envoy AI Gateway to provide OpenAI-compatible routing, token usage accounting, and usage-based rate limiting. KServe integrates with llm-d via a Kubernetes-native custom resource, LLMInferenceService, which provisions the router and inference pool. You will create a Gateway and an AIGatewayRoute that forward requests to the KServe InferencePool, enable automatic token metering (input, output, and total) via llmRequestCosts, and enforce per-user, per-model quotas using a BackendTrafficPolicy. KServe can run behind the AI Gateway in the same cluster or a different one; for clarity, this guide uses a single-cluster setup. @@ -636,6 +636,8 @@ done Now that you've tested the basic setup, you can: +- Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). + - Explore more rate limiter-related configuration at the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/capabilities/usage-based-ratelimiting). - Explore the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/) to learn more about the features and capabilities. diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md new file mode 100644 index 000000000..88e81ee98 --- /dev/null +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md @@ -0,0 +1,22 @@ +--- +title: Inference Gateways +description: Reference inference gateway integrations for KServe LLMInferenceService +--- + +# Inference Gateways + +KServe `LLMInferenceService` integrates with Kubernetes Gateway API +implementations to expose models, route inference requests, and apply +gateway-specific traffic policies. Choose a reference integration based on the +gateway and LLM traffic-management features you want to use. + +## Reference integrations + +| Gateway | Integration | Use this guide for | +| --- | --- | --- | +| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [Gateway API Inference Extension with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | +| [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | LLM-aware routing through an `AgentgatewayBackend`, GenAI telemetry, token tracking, and token-based policies | + +Both guides use Gateway API resources and KServe's configurable router +templates. Their gateway-specific custom resources and supported policies are +different, so follow the guide for the gateway installed in your cluster. diff --git a/docusaurus.config.ts b/docusaurus.config.ts index 2e60799fc..a855aad51 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -538,6 +538,10 @@ const config: Config = { from: '/latest/admin/ai-gateway_integration/', to: '/docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway', }, + { + from: '/docs/next/model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway', + to: '/docs/next/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension', + }, // Predictive Inference Section { diff --git a/sidebars.ts b/sidebars.ts index 6ba20d6e1..581a5a195 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -141,9 +141,13 @@ const sidebars: SidebarsConfig = { "model-serving/generative-inference/llmisvc/llmisvc-dependencies", { type: 'category', - label: 'Integrations', + label: 'Inference Gateways', + link: { + type: 'doc', + id: 'model-serving/generative-inference/llmisvc/llmisvc-inference-gateways', + }, items: [ - "model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway", + "model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension", "model-serving/generative-inference/llmisvc/llmisvc-agentgateway", ], }, From 4386ff1c1ff2286d9dca42d4688d9c2282da6a59 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 29 Jul 2026 15:00:14 -0700 Subject: [PATCH 2/9] docs: lowercase agentgateway references Fixes: #714 Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index 0acd948a6..614978b47 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -5,21 +5,21 @@ description: How to integrate KServe LLMInferenceService with agentgateway for L # LLMInferenceService with agentgateway -This guide walks through integrating a KServe LLMInferenceService with [AgentGateway](https://agentgateway.dev) to enable LLM-aware routing with token tracking, GenAI telemetry (OpenTelemetry semantic conventions), and token-based rate limiting. The key mechanism is overriding the HTTPRoute `backendRef` via `LLMInferenceServiceConfig` so that KServe's auto-generated routes point to an `AgentgatewayBackend` instead of a plain `InferencePool` or `Service`. +This guide walks through integrating a KServe LLMInferenceService with [agentgateway](https://agentgateway.dev) to enable LLM-aware routing with token tracking, GenAI telemetry (OpenTelemetry semantic conventions), and token-based rate limiting. The key mechanism is overriding the HTTPRoute `backendRef` via `LLMInferenceServiceConfig` so that KServe's auto-generated routes point to an `AgentgatewayBackend` instead of a plain `InferencePool` or `Service`. -## AgentGateway Overview +## agentgateway Overview -[AgentGateway](https://github.com/agentgateway/agentgateway) is a Rust-based proxy under the [AI Agent Infrastructure Foundation (AAIF)](https://aaif.io) at the Linux Foundation. It implements the Kubernetes Gateway API but is LLM-aware: it parses OpenAI chat completion requests and responses, extracts token usage, emits OpenTelemetry GenAI semantic conventions, and enforces token-based rate limits and policies. Key custom resources: +[agentgateway](https://github.com/agentgateway/agentgateway) is a Rust-based proxy under the [AI Agent Infrastructure Foundation (AAIF)](https://aaif.io) at the Linux Foundation. It implements the Kubernetes Gateway API but is LLM-aware: it parses OpenAI chat completion requests and responses, extracts token usage, emits OpenTelemetry GenAI semantic conventions, and enforces token-based rate limits and policies. Key custom resources: - **`AgentgatewayBackend`**: Declares a backend as an LLM provider so the gateway activates its LLM pipeline (token parsing, model tracking, GenAI telemetry). - **`AgentgatewayPolicy`**: Attaches governance policies such as token-based rate limiting. -- **`HTTPRoute`**: Standard Gateway API routing — AgentGateway supports `AgentgatewayBackend` as a `backendRef` kind. +- **`HTTPRoute`**: Standard Gateway API routing — agentgateway supports `AgentgatewayBackend` as a `backendRef` kind. -For more information, see the [AgentGateway documentation](https://agentgateway.dev/docs). +For more information, see the [agentgateway documentation](https://agentgateway.dev/docs). ## Why This Integration Matters -When KServe generates HTTPRoutes for an LLMInferenceService, the `backendRef` defaults to `InferencePool` (or `Service` for catch-all routes). AgentGateway treats these as generic HTTP traffic — it routes requests correctly but cannot activate LLM-aware features because it doesn't know the backend serves LLM traffic. +When KServe generates HTTPRoutes for an LLMInferenceService, the `backendRef` defaults to `InferencePool` (or `Service` for catch-all routes). agentgateway treats these as generic HTTP traffic — it routes requests correctly but cannot activate LLM-aware features because it doesn't know the backend serves LLM traffic. By overriding the `backendRef` to use `AgentgatewayBackend`, the gateway recognizes the backend as an LLM provider and activates: @@ -42,7 +42,7 @@ KServe supports [distributed tracing](https://github.com/kserve/kserve/pull/5481 Before you begin, ensure you have the following components installed and configured: - A Kubernetes cluster with [KServe with Gateway API enabled](../../../admin-guide/kubernetes-deployment.md) -- [AgentGateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster +- [agentgateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster - [Gateway API CRDs](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) installed - [LLMInferenceService dependencies](./llmisvc-dependencies.md) installed - The `kubectl` command-line tool installed and configured to access your cluster @@ -58,7 +58,7 @@ kubectl create namespace kserve-test ### Create Gateway -Create an AgentGateway Gateway resource: +Create an agentgateway Gateway resource: ```yaml apiVersion: gateway.networking.k8s.io/v1 @@ -115,7 +115,7 @@ kubectl wait --for=condition=Ready llminferenceservice/my-model \ ### Step 1: Create AgentgatewayBackend -Create an `AgentgatewayBackend` that declares the KServe predictor Service as an LLM provider. This tells AgentGateway to activate its LLM pipeline for traffic to this backend: +Create an `AgentgatewayBackend` that declares the KServe predictor Service as an LLM provider. This tells agentgateway to activate its LLM pipeline for traffic to this backend: ```yaml apiVersion: agentgateway.dev/v1alpha1 @@ -343,7 +343,7 @@ curl -s "$GATEWAY_URL/v1/chat/completions" \ ### Verify LLM-Aware Processing -Check the AgentGateway logs to confirm the LLM pipeline is active: +Check the agentgateway logs to confirm the LLM pipeline is active: ```shell kubectl logs -n kserve-test deploy/agentgateway --tail=10 @@ -375,7 +375,7 @@ The default [LLMInferenceServiceConfig route template](https://github.com/kserve By overriding `spec.router.route.http` — either directly on the `LLMInferenceService` or via a reusable `LLMInferenceServiceConfig` — you change the `backendRef` to `AgentgatewayBackend` (kind: `AgentgatewayBackend`, group: `agentgateway.dev`). This leverages Gateway API's support for [arbitrary backendRef kinds (GEP-1742)](https://gateway-api.sigs.k8s.io/geps/gep-1742/). -AgentGateway recognizes `AgentgatewayBackend` references and activates its LLM pipeline: parsing OpenAI request/response payloads, extracting token usage, emitting GenAI telemetry, and enforcing token-based policies. +agentgateway recognizes `AgentgatewayBackend` references and activates its LLM pipeline: parsing OpenAI request/response payloads, extracting token usage, emitting GenAI telemetry, and enforcing token-based policies. :::note This approach is generic. Any gateway that supports custom `backendRef` kinds via Gateway API can use the same `LLMInferenceServiceConfig` override mechanism. Replace `AgentgatewayBackend` with your gateway's backend CRD. @@ -384,7 +384,7 @@ This approach is generic. Any gateway that supports custom `backendRef` kinds vi ## Next Steps - Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). -- Explore the [AgentGateway documentation](https://agentgateway.dev/docs) for advanced features like content filtering and cost tracking. +- Explore the [agentgateway documentation](https://agentgateway.dev/docs) for advanced features like content filtering and cost tracking. - Learn more about [LLMInferenceServiceConfig composition](./llmisvc-config-composition.md) for managing configurations across multiple services. - See the [LLMInferenceService Configuration Guide](./llmisvc-configuration.md) for the full `spec.router.route.http` reference. - Follow the discussion in [kserve/kserve#5729](https://github.com/kserve/kserve/issues/5729) and [agentgateway/agentgateway#2323](https://github.com/agentgateway/agentgateway/issues/2323) for ongoing integration improvements. From d7ade198f916ff09cd9a269c66c2e30eeff74ff3 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 29 Jul 2026 15:00:49 -0700 Subject: [PATCH 3/9] docs: list agentgateway first Fixes: #714 Signed-off-by: Daneyon Hansen --- .../generative-inference/llmisvc/llmisvc-inference-gateways.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md index 88e81ee98..633ccbb52 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md @@ -14,8 +14,8 @@ gateway and LLM traffic-management features you want to use. | Gateway | Integration | Use this guide for | | --- | --- | --- | -| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [Gateway API Inference Extension with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | | [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | LLM-aware routing through an `AgentgatewayBackend`, GenAI telemetry, token tracking, and token-based policies | +| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [Gateway API Inference Extension with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | Both guides use Gateway API resources and KServe's configurable router templates. Their gateway-specific custom resources and supported policies are From 8cdb73d31fb6424d869ec54204cead7d6ab7e2c7 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 29 Jul 2026 15:11:38 -0700 Subject: [PATCH 4/9] docs: clarify agentgateway backend options Fixes: #714 Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 95 +++++++++++-------- .../llmisvc/llmisvc-inference-gateways.md | 2 +- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index 614978b47..e9548c30a 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -1,40 +1,42 @@ --- title: LLMInferenceService with agentgateway -description: How to integrate KServe LLMInferenceService with agentgateway for LLM-aware routing, token tracking, and GenAI telemetry +description: How to use KServe LLMInferenceService with agentgateway as an Inference Gateway and optionally apply AI policies --- # LLMInferenceService with agentgateway -This guide walks through integrating a KServe LLMInferenceService with [agentgateway](https://agentgateway.dev) to enable LLM-aware routing with token tracking, GenAI telemetry (OpenTelemetry semantic conventions), and token-based rate limiting. The key mechanism is overriding the HTTPRoute `backendRef` via `LLMInferenceServiceConfig` so that KServe's auto-generated routes point to an `AgentgatewayBackend` instead of a plain `InferencePool` or `Service`. +This guide walks through using [agentgateway](https://agentgateway.dev) as an +Inference Gateway for KServe `LLMInferenceService`. agentgateway supports the +Gateway API Inference Extension, so KServe's generated `HTTPRoute` can route +directly to the standard `InferencePool` backend. An `AgentgatewayBackend` is +only needed when you want to apply AI policies that require LLM-aware +processing, such as token-based rate limiting. ## agentgateway Overview [agentgateway](https://github.com/agentgateway/agentgateway) is a Rust-based proxy under the [AI Agent Infrastructure Foundation (AAIF)](https://aaif.io) at the Linux Foundation. It implements the Kubernetes Gateway API but is LLM-aware: it parses OpenAI chat completion requests and responses, extracts token usage, emits OpenTelemetry GenAI semantic conventions, and enforces token-based rate limits and policies. Key custom resources: -- **`AgentgatewayBackend`**: Declares a backend as an LLM provider so the gateway activates its LLM pipeline (token parsing, model tracking, GenAI telemetry). +- **`InferencePool`**: Standard Gateway API Inference Extension backend supported directly by agentgateway. +- **`AgentgatewayBackend`**: Optional backend that declares an LLM provider so the gateway can apply LLM-aware processing and AI policies. - **`AgentgatewayPolicy`**: Attaches governance policies such as token-based rate limiting. -- **`HTTPRoute`**: Standard Gateway API routing — agentgateway supports `AgentgatewayBackend` as a `backendRef` kind. +- **`HTTPRoute`**: Standard Gateway API routing that can reference either an `InferencePool` or an `AgentgatewayBackend`. -For more information, see the [agentgateway documentation](https://agentgateway.dev/docs). +For more information, see the [agentgateway KServe integration guide](https://agentgateway.dev/docs/kubernetes/main/integrations/kserve/) and the [Gateway API Inference Extension implementation list](https://gateway-api-inference-extension.sigs.k8s.io/implementations/gateways/#gateway-implementations). -## Why This Integration Matters +## Choose a Backend Type -When KServe generates HTTPRoutes for an LLMInferenceService, the `backendRef` defaults to `InferencePool` (or `Service` for catch-all routes). agentgateway treats these as generic HTTP traffic — it routes requests correctly but cannot activate LLM-aware features because it doesn't know the backend serves LLM traffic. +KServe generates `HTTPRoute` resources with an `InferencePool` backend by +default. agentgateway supports this standard backend without any route +override. Use `AgentgatewayBackend` only when an AI policy needs agentgateway +to parse the LLM request or response. -By overriding the `backendRef` to use `AgentgatewayBackend`, the gateway recognizes the backend as an LLM provider and activates: - -| Capability | Plain Service / InferencePool | AgentgatewayBackend | -|---|---|---| -| Routing | Yes | Yes | -| Protocol detection | `http` (generic) | `llm` (LLM-aware) | -| Token parsing | No | Yes (extracts `usage.input_tokens`, `usage.output_tokens` from response body) | -| Model tracking | No | Yes (`gen_ai.request.model`, `gen_ai.response.model`) | -| GenAI semantic conventions (OTel) | No | Yes (`gen_ai.*` attributes per [OTel GenAI spec](https://opentelemetry.io/docs/specs/semconv/gen-ai/)) | -| Token-based rate limiting | No | Yes | -| Cost tracking | No | Yes | +| Backend | Use it for | Route override required | +| --- | --- | --- | +| `InferencePool` | Standard inference routing through the Gateway API Inference Extension | No | +| `AgentgatewayBackend` | AI policies such as token-based rate limiting, plus LLM-aware telemetry and model tracking | Yes | :::note -KServe supports [distributed tracing](https://github.com/kserve/kserve/pull/5481) natively via `spec.tracing`, which provides request-level spans and traces. The GenAI semantic conventions listed above are complementary — they add LLM-specific attributes (token counts, model name, operation type) at the gateway level. +KServe supports [distributed tracing](https://github.com/kserve/kserve/pull/5481) natively via `spec.tracing`, which provides request-level spans and traces. The LLM-aware telemetry available through `AgentgatewayBackend` is complementary — it adds LLM-specific attributes such as token counts, model name, and operation type at the gateway level. ::: ## Prerequisites @@ -111,7 +113,27 @@ kubectl wait --for=condition=Ready llminferenceservice/my-model \ -n kserve-test --timeout=300s ``` -## Configure LLM-Aware Routing +## Use the Default InferencePool Backend + +The default [LLMInferenceServiceConfig route template](https://github.com/kserve/kserve/blob/master/config/llmisvcconfig/config-llm-router-route.yaml) +generates an `HTTPRoute` that references an `InferencePool`. agentgateway +supports this backend directly, so no `AgentgatewayBackend` or route override +is required for standard inference routing. + +Verify the generated backend reference: + +```shell +kubectl get httproute my-model-llminferenceservice-route \ + -n kserve-test -o jsonpath='{.spec.rules[*].backendRefs[*]}' +``` + +If you do not need AI policies, continue to [Configure the gateway URL](#configure-gateway-url). + +## Optional: Configure an AgentgatewayBackend for AI Policies + +To use AI policies that require LLM-aware request or response processing, +create an `AgentgatewayBackend` and override the generated route to reference +it. ### Step 1: Create AgentgatewayBackend @@ -279,7 +301,7 @@ spec: -### Step 3: Attach Token-Based Rate Limiting (Optional) +### Step 3: Attach Token-Based Rate Limiting Apply an `AgentgatewayPolicy` to enforce token-based rate limits on the route: @@ -301,7 +323,7 @@ spec: unit: Hours ``` -## Configure $GATEWAY_URL +## Configure $GATEWAY_URL {#configure-gateway-url} Check if your Gateway has an external IP address assigned: @@ -341,9 +363,10 @@ curl -s "$GATEWAY_URL/v1/chat/completions" \ }' | jq . ``` -### Verify LLM-Aware Processing +### Verify AI Policy Processing -Check the agentgateway logs to confirm the LLM pipeline is active: +If you configured an `AgentgatewayBackend`, check the agentgateway logs to +confirm the LLM pipeline is active: ```shell kubectl logs -n kserve-test deploy/agentgateway --tail=10 @@ -362,24 +385,18 @@ gen_ai.usage.input_tokens=12 gen_ai.usage.output_tokens=15 ``` -Without `AgentgatewayBackend` (plain Service or InferencePool backendRef), the same request would only show: - -```text -protocol=http -http.status=200 -``` - ## How It Works -The default [LLMInferenceServiceConfig route template](https://github.com/kserve/kserve/blob/master/config/llmisvcconfig/config-llm-router-route.yaml) generates HTTPRoutes with `backendRef` pointing to `InferencePool` (kind: `InferencePool`, group: `inference.networking.x-k8s.io`). +For standard inference routing, KServe generates an `HTTPRoute` whose +`backendRef` points to `InferencePool` (kind: `InferencePool`, group: +`inference.networking.x-k8s.io`). agentgateway implements the Gateway API +Inference Extension and routes this traffic through the pool's endpoint picker. -By overriding `spec.router.route.http` — either directly on the `LLMInferenceService` or via a reusable `LLMInferenceServiceConfig` — you change the `backendRef` to `AgentgatewayBackend` (kind: `AgentgatewayBackend`, group: `agentgateway.dev`). This leverages Gateway API's support for [arbitrary backendRef kinds (GEP-1742)](https://gateway-api.sigs.k8s.io/geps/gep-1742/). - -agentgateway recognizes `AgentgatewayBackend` references and activates its LLM pipeline: parsing OpenAI request/response payloads, extracting token usage, emitting GenAI telemetry, and enforcing token-based policies. - -:::note -This approach is generic. Any gateway that supports custom `backendRef` kinds via Gateway API can use the same `LLMInferenceServiceConfig` override mechanism. Replace `AgentgatewayBackend` with your gateway's backend CRD. -::: +For AI policies, overriding `spec.router.route.http` changes the `backendRef` +to `AgentgatewayBackend` (kind: `AgentgatewayBackend`, group: +`agentgateway.dev`). agentgateway then recognizes the upstream as an LLM +provider and can parse OpenAI request and response payloads, extract token +usage, emit GenAI telemetry, and enforce token-based policies. ## Next Steps diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md index 633ccbb52..fe28b645c 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md @@ -14,7 +14,7 @@ gateway and LLM traffic-management features you want to use. | Gateway | Integration | Use this guide for | | --- | --- | --- | -| [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | LLM-aware routing through an `AgentgatewayBackend`, GenAI telemetry, token tracking, and token-based policies | +| [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | Standard `InferencePool` routing with optional `AgentgatewayBackend` configuration for AI policies | | [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [Gateway API Inference Extension with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | Both guides use Gateway API resources and KServe's configurable router From 85766f955ffe2820a37347943f73947281d24a78 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 30 Jul 2026 12:43:27 -0700 Subject: [PATCH 5/9] docs: fix agentgateway backend examples Fixes: #714 Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 94 ++++++++++++++----- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index e9548c30a..7694d7c92 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -10,7 +10,8 @@ Inference Gateway for KServe `LLMInferenceService`. agentgateway supports the Gateway API Inference Extension, so KServe's generated `HTTPRoute` can route directly to the standard `InferencePool` backend. An `AgentgatewayBackend` is only needed when you want to apply AI policies that require LLM-aware -processing, such as token-based rate limiting. +processing, such as token-based rate limiting. It can wrap the generated +`InferencePool` so that endpoint selection remains available. ## agentgateway Overview @@ -25,10 +26,10 @@ For more information, see the [agentgateway KServe integration guide](https://ag ## Choose a Backend Type -KServe generates `HTTPRoute` resources with an `InferencePool` backend by -default. agentgateway supports this standard backend without any route -override. Use `AgentgatewayBackend` only when an AI policy needs agentgateway -to parse the LLM request or response. +When the managed scheduler is enabled, KServe generates `HTTPRoute` resources +with an `InferencePool` backend. agentgateway supports this standard backend +without any route override. Use `AgentgatewayBackend` only when an AI policy +needs agentgateway to parse the LLM request or response. | Backend | Use it for | Route override required | | --- | --- | --- | @@ -44,12 +45,23 @@ KServe supports [distributed tracing](https://github.com/kserve/kserve/pull/5481 Before you begin, ensure you have the following components installed and configured: - A Kubernetes cluster with [KServe with Gateway API enabled](../../../admin-guide/kubernetes-deployment.md) -- [agentgateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster +- [agentgateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster with [Inference Extension support](https://agentgateway.dev/docs/kubernetes/latest/inference/) enabled - [Gateway API CRDs](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) installed - [LLMInferenceService dependencies](./llmisvc-dependencies.md) installed - The `kubectl` command-line tool installed and configured to access your cluster - Basic understanding of [KServe concepts](../../../concepts/index.md) and [LLMInferenceService](./llmisvc-overview.md) +Set `AGENTGATEWAY_VERSION` to the installed chart version, then enable +Inference Extension support in the agentgateway Helm release: + +```shell +helm upgrade -i -n agentgateway-system agentgateway \ + oci://cr.agentgateway.dev/charts/agentgateway \ + --version $AGENTGATEWAY_VERSION \ + --set inferenceExtension.enabled=true \ + --reuse-values +``` + ## Deploy LLMInferenceService ### Create Namespace @@ -96,6 +108,7 @@ spec: replicas: 1 router: route: {} + scheduler: {} template: containers: - name: main @@ -113,18 +126,20 @@ kubectl wait --for=condition=Ready llminferenceservice/my-model \ -n kserve-test --timeout=300s ``` -## Use the Default InferencePool Backend +## Use the Standard InferencePool Backend -The default [LLMInferenceServiceConfig route template](https://github.com/kserve/kserve/blob/master/config/llmisvcconfig/config-llm-router-route.yaml) -generates an `HTTPRoute` that references an `InferencePool`. agentgateway +The managed scheduler and default +[LLMInferenceServiceConfig route template](https://github.com/kserve/kserve/blob/master/config/llmisvcconfig/config-llm-router-route.yaml) +generate an `InferencePool` and an `HTTPRoute` that references it. agentgateway supports this backend directly, so no `AgentgatewayBackend` or route override is required for standard inference routing. Verify the generated backend reference: ```shell -kubectl get httproute my-model-llminferenceservice-route \ - -n kserve-test -o jsonpath='{.spec.rules[*].backendRefs[*]}' +kubectl get httproute my-model-kserve-route \ + -n kserve-test \ + -o jsonpath='{.spec.rules[?(@.name=="v1-chat-completions-path")].backendRefs[0]}' ``` If you do not need AI policies, continue to [Configure the gateway URL](#configure-gateway-url). @@ -137,7 +152,9 @@ it. ### Step 1: Create AgentgatewayBackend -Create an `AgentgatewayBackend` that declares the KServe predictor Service as an LLM provider. This tells agentgateway to activate its LLM pipeline for traffic to this backend: +Create an `AgentgatewayBackend` that wraps the `InferencePool` generated by +KServe. This tells agentgateway to activate its LLM pipeline while retaining +the pool's endpoint selection: ```yaml apiVersion: agentgateway.dev/v1alpha1 @@ -148,17 +165,23 @@ metadata: spec: ai: provider: - openai: + custom: + backendRef: + group: inference.networking.k8s.io + kind: InferencePool + name: my-model-inference-pool model: Qwen/Qwen2.5-0.5B-Instruct - host: my-model-kserve-workload-svc.kserve-test.svc.cluster.local - port: 8000 + formats: + - type: Completions + path: /v1/chat/completions ``` :::tip -The `host` should point to the KServe workload Service. The naming convention is `{llminferenceservice-name}-kserve-workload-svc`. Verify with: +The generated pool name is `{llminferenceservice-name}-inference-pool`. +Verify it with: ```shell -kubectl get svc -n kserve-test | grep workload +kubectl get inferencepool -n kserve-test ``` ::: @@ -186,6 +209,7 @@ spec: name: Qwen/Qwen2.5-0.5B-Instruct replicas: 1 router: + scheduler: {} route: http: spec: @@ -288,6 +312,8 @@ spec: replicas: 1 baseRefs: - name: agentgateway-route-config + router: + scheduler: {} template: containers: - name: main @@ -315,7 +341,7 @@ spec: targetRefs: - group: gateway.networking.k8s.io kind: HTTPRoute - name: my-model-llminferenceservice-route + name: my-model-kserve-route traffic: rateLimit: local: @@ -352,10 +378,29 @@ kubectl get svc -n kserve-test -l gateway.networking.k8s.io/gateway-name=agentga ## Testing the Integration +Set the request path for the backend you selected: + + + + +```shell +export GATEWAY_PATH="/kserve-test/my-model/v1/chat/completions" +``` + + + + +```shell +export GATEWAY_PATH="/v1/chat/completions" +``` + + + + Send a test request: ```shell -curl -s "$GATEWAY_URL/v1/chat/completions" \ +curl -s "$GATEWAY_URL$GATEWAY_PATH" \ -H "Content-Type: application/json" \ -d '{ "model": "Qwen/Qwen2.5-0.5B-Instruct", @@ -375,7 +420,7 @@ kubectl logs -n kserve-test deploy/agentgateway --tail=10 With `AgentgatewayBackend`, you should see GenAI fields in the log: ```text -route=my-model-llminferenceservice-route +route=kserve-test/my-model-kserve-route http.status=200 protocol=llm gen_ai.operation.name=chat @@ -389,14 +434,15 @@ gen_ai.usage.output_tokens=15 For standard inference routing, KServe generates an `HTTPRoute` whose `backendRef` points to `InferencePool` (kind: `InferencePool`, group: -`inference.networking.x-k8s.io`). agentgateway implements the Gateway API +`inference.networking.k8s.io`). agentgateway implements the Gateway API Inference Extension and routes this traffic through the pool's endpoint picker. For AI policies, overriding `spec.router.route.http` changes the `backendRef` to `AgentgatewayBackend` (kind: `AgentgatewayBackend`, group: -`agentgateway.dev`). agentgateway then recognizes the upstream as an LLM -provider and can parse OpenAI request and response payloads, extract token -usage, emit GenAI telemetry, and enforce token-based policies. +`agentgateway.dev`). The `AgentgatewayBackend` wraps the generated +`InferencePool`, so agentgateway can parse OpenAI request and response payloads, +extract token usage, emit GenAI telemetry, and enforce token-based policies +without bypassing the endpoint picker. ## Next Steps From 1ad5c06ad1e80cd946a156d02494dac5ae0485a5 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 30 Jul 2026 12:48:28 -0700 Subject: [PATCH 6/9] docs: link llm-d gateway guides Fixes: #714 Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 6 ++-- .../llmisvc/llmisvc-dependencies.md | 10 +++++-- .../llmisvc-inference-gateway-extension.md | 28 +++++++++++++++---- .../llmisvc/llmisvc-inference-gateways.md | 7 +++++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index 7694d7c92..87152bd4d 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -22,7 +22,7 @@ processing, such as token-based rate limiting. It can wrap the generated - **`AgentgatewayPolicy`**: Attaches governance policies such as token-based rate limiting. - **`HTTPRoute`**: Standard Gateway API routing that can reference either an `InferencePool` or an `AgentgatewayBackend`. -For more information, see the [agentgateway KServe integration guide](https://agentgateway.dev/docs/kubernetes/main/integrations/kserve/) and the [Gateway API Inference Extension implementation list](https://gateway-api-inference-extension.sigs.k8s.io/implementations/gateways/#gateway-implementations). +For more information, see the [agentgateway KServe integration guide](https://agentgateway.dev/docs/kubernetes/main/integrations/kserve/), the [llm-d agentgateway guide](https://llm-d.ai/docs/infrastructure/gateway/agentgateway), and the [Gateway API Inference Extension implementation list](https://gateway-api-inference-extension.sigs.k8s.io/implementations/gateways/#gateway-implementations). ## Choose a Backend Type @@ -46,7 +46,7 @@ Before you begin, ensure you have the following components installed and configu - A Kubernetes cluster with [KServe with Gateway API enabled](../../../admin-guide/kubernetes-deployment.md) - [agentgateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster with [Inference Extension support](https://agentgateway.dev/docs/kubernetes/latest/inference/) enabled -- [Gateway API CRDs](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) installed +- [Gateway API and Gateway API Inference Extension CRDs](https://llm-d.ai/docs/infrastructure/gateway/install-crds) installed - [LLMInferenceService dependencies](./llmisvc-dependencies.md) installed - The `kubectl` command-line tool installed and configured to access your cluster - Basic understanding of [KServe concepts](../../../concepts/index.md) and [LLMInferenceService](./llmisvc-overview.md) @@ -447,6 +447,8 @@ without bypassing the endpoint picker. ## Next Steps - Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). +- Follow the [llm-d agentgateway guide](https://llm-d.ai/docs/infrastructure/gateway/agentgateway) for the llm-d Router and gateway infrastructure. +- Explore the [llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) for production deployment patterns. - Explore the [agentgateway documentation](https://agentgateway.dev/docs) for advanced features like content filtering and cost tracking. - Learn more about [LLMInferenceServiceConfig composition](./llmisvc-config-composition.md) for managing configurations across multiple services. - See the [LLMInferenceService Configuration Guide](./llmisvc-configuration.md) for the full `spec.router.route.http` reference. diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md index 938953439..32d827f73 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md @@ -58,7 +58,14 @@ Extends Gateway API with **LLM-specific scheduling and load balancing** capabili LLMInferenceService creates GIE resources when scheduler is enabled. -**Key Point**: InferencePool has `extensionRef` pointing to the EPP (Endpoint Picker) service, which runs the scheduling logic. +**Key Point**: `InferencePool` has an `endpointPickerRef` that points to the +llm-d Router service, which runs the scheduling logic. + +For current production guidance, see the +[llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths), the +[llm-d Router gateway guides](https://llm-d.ai/docs/infrastructure/gateway), +and the +[Gateway API CRD installation guide](https://llm-d.ai/docs/infrastructure/gateway/install-crds). --- @@ -306,4 +313,3 @@ curl http://localhost:8080/v1/completions ... | **Prefill-Decode** | ✅ | ✅ (required) | ❌ (single-node) or ✅ (multi-node) | ✅ | | **DP+EP** | ✅ | ✅ (scheduler) | ✅ | ✅ | | **No Scheduler** | ✅ | ❌ | ❌ | ✅ | - diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md index 119305659..d61bab406 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md @@ -5,7 +5,18 @@ description: How to integrate KServe LLMInferenceService with Envoy AI Gateway t # Gateway API Inference Extension with Envoy AI Gateway -This tutorial walks through deploying a KServe LLMInferenceService that wraps [llm-d](https://llm-d.ai/) — which implements the [Gateway API Inference Extension](https://gateway-api-inference-extension.sigs.k8s.io/) (the llm-d router and inference pool) — and fronts it with Envoy AI Gateway to provide OpenAI-compatible routing, token usage accounting, and usage-based rate limiting. KServe integrates with llm-d via a Kubernetes-native custom resource, LLMInferenceService, which provisions the router and inference pool. You will create a Gateway and an AIGatewayRoute that forward requests to the KServe InferencePool, enable automatic token metering (input, output, and total) via llmRequestCosts, and enforce per-user, per-model quotas using a BackendTrafficPolicy. KServe can run behind the AI Gateway in the same cluster or a different one; for clarity, this guide uses a single-cluster setup. +This tutorial walks through deploying a KServe LLMInferenceService that uses +the [llm-d Router](https://llm-d.ai/docs/infrastructure/gateway) with +[Gateway API Inference Extension](https://gateway-api-inference-extension.sigs.k8s.io/) +resources, and fronts it with Envoy AI Gateway to provide OpenAI-compatible +routing, token usage accounting, and usage-based rate limiting. KServe +integrates with llm-d via a Kubernetes-native custom resource, +LLMInferenceService, which provisions the router and inference pool. You will +create a Gateway and an AIGatewayRoute that forward requests to the KServe +InferencePool, enable automatic token metering (input, output, and total) via +llmRequestCosts, and enforce per-user, per-model quotas using a +BackendTrafficPolicy. KServe can run behind the AI Gateway in the same cluster +or a different one; for clarity, this guide uses a single-cluster setup. ## AI Gateway Overview @@ -21,7 +32,7 @@ For more information, see the [Envoy AI Gateway documentation](https://aigateway ## llm-d Overview -[llm-d](https://llm-d.ai/) is a Kubernetes-native distributed inference serving stack, providing well-lit paths for anyone to serve large generative AI models at scale, with the fastest time-to-value and competitive performance per dollar for most models across most hardware accelerators. +[llm-d](https://llm-d.ai/) is a Kubernetes-native distributed inference serving stack, providing [well-lit paths](https://llm-d.ai/docs/well-lit-paths) for anyone to serve large generative AI models at scale, with the fastest time-to-value and competitive performance per dollar for most models across most hardware accelerators. KServe's generative inference leverages llm-d components to scale and schedule traffic efficiently: @@ -39,7 +50,7 @@ In this tutorial you'll deploy an `LLMInferenceService` that creates a router an Before you begin, ensure you have the following components installed and configured: - A Kubernetes cluster with [KServe with Gateway API Enabled](../../../admin-guide/kubernetes-deployment.md) -- [Gateway API Inference Extension](https://gateway-api-inference-extension.sigs.k8s.io/guides/) installed in your cluster +- [Gateway API and Gateway API Inference Extension CRDs](https://llm-d.ai/docs/infrastructure/gateway/install-crds) installed in your cluster - [Envoy Gateway with Inference Pool support enabled prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites) installed in your cluster - [Envoy AI Gateway](https://aigateway.envoyproxy.io/docs/getting-started/installation) installed in your cluster - [LeaderWorkerSet (LWS)](https://lws.sigs.k8s.io/docs/installation/) installed in your cluster @@ -98,7 +109,11 @@ spec: ### Create EndpointPickerConfig -The Endpoint Picker (EPP) or scheduler is a core component of the Gateway API Inference Extension. It is responsible for selecting the best backend endpoint (pod) from the InferencePool for each request. You can customize the scheduling behavior by defining various plugins for scoring, filtering, and picking endpoints based on your requirements. +The llm-d Router, formerly called the Endpoint Picker (EPP), is the scheduler +used with the Gateway API Inference Extension. It selects the best backend +endpoint (pod) from the InferencePool for each request. You can customize the +scheduling behavior by defining plugins for scoring, filtering, and picking +endpoints. About the configuration: @@ -150,7 +165,7 @@ In this step, you’ll define an LLMInferenceServiceConfig — a reusable templa In this example, we will configure: - vLLM worker defaults: image, command/args that pass the served model name, port 8000, logging level, HF cache path, liveness/readiness probes, secure pod settings, and volumes for /home, /dev/shm, model cache, and TLS certs. -- Router and scheduler defaults: an inference scheduler (gRPC + metrics ports) configured for secure serving and wired to the EndpointPickerConfig (from the ConfigMap above) to score/pick endpoints; the pool targets port 8000 and references an internal EPP service. +- Router and scheduler defaults: an inference scheduler (gRPC + metrics ports) configured for secure serving and wired to the EndpointPickerConfig (from the ConfigMap above) to score/pick endpoints; the pool targets port 8000 and references an internal llm-d Router service. - Operational safeguards: conservative timeouts and termination grace, plus readiness/liveness for safe rollouts. @@ -637,6 +652,9 @@ done Now that you've tested the basic setup, you can: - Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). +- Review the [llm-d gateway guides](https://llm-d.ai/docs/infrastructure/gateway). +- Follow the [llm-d Envoy AI Gateway guide](https://llm-d.ai/docs/infrastructure/gateway/envoy-ai-gateway). +- Explore the [llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) for production deployment patterns. - Explore more rate limiter-related configuration at the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/capabilities/usage-based-ratelimiting). diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md index fe28b645c..31ea4c38f 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md @@ -20,3 +20,10 @@ gateway and LLM traffic-management features you want to use. Both guides use Gateway API resources and KServe's configurable router templates. Their gateway-specific custom resources and supported policies are different, so follow the guide for the gateway installed in your cluster. + +For production llm-d Router deployment patterns, see the +[llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) and +[gateway guides](https://llm-d.ai/docs/infrastructure/gateway). The llm-d +documentation also provides provider-specific infrastructure guides for +[agentgateway](https://llm-d.ai/docs/infrastructure/gateway/agentgateway) and +[Envoy AI Gateway](https://llm-d.ai/docs/infrastructure/gateway/envoy-ai-gateway). From 19cb2482353915f539ac6d76757be39a39e63b18 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 30 Jul 2026 14:02:08 -0700 Subject: [PATCH 7/9] docs: align inference gateways with llm-d Router Update GAIE v1 resources, llm-d Router artifacts, and the tested agentgateway and KServe installation order. Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-agentgateway.md | 105 ++++++++++++++---- .../llmisvc/llmisvc-config-composition.md | 6 +- .../llmisvc/llmisvc-dependencies.md | 11 +- .../llmisvc-inference-gateway-extension.md | 77 +++++++------ 4 files changed, 132 insertions(+), 67 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index 87152bd4d..0f6bd777b 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -45,23 +45,77 @@ KServe supports [distributed tracing](https://github.com/kserve/kserve/pull/5481 Before you begin, ensure you have the following components installed and configured: - A Kubernetes cluster with [KServe with Gateway API enabled](../../../admin-guide/kubernetes-deployment.md) -- [agentgateway](https://agentgateway.dev/docs/kubernetes/latest/getting-started/) installed in your cluster with [Inference Extension support](https://agentgateway.dev/docs/kubernetes/latest/inference/) enabled -- [Gateway API and Gateway API Inference Extension CRDs](https://llm-d.ai/docs/infrastructure/gateway/install-crds) installed +- The [Gateway API CRDs](https://gateway-api.sigs.k8s.io/guides/) installed - [LLMInferenceService dependencies](./llmisvc-dependencies.md) installed - The `kubectl` command-line tool installed and configured to access your cluster - Basic understanding of [KServe concepts](../../../concepts/index.md) and [LLMInferenceService](./llmisvc-overview.md) -Set `AGENTGATEWAY_VERSION` to the installed chart version, then enable -Inference Extension support in the agentgateway Helm release: +Configure the KServe `LLMInferenceService` controller to attach generated +routes to the shared agentgateway Gateway. During the GAIE v1 migration, KServe +also installs transitional CRDs that its controller uses for compatibility: ```shell -helm upgrade -i -n agentgateway-system agentgateway \ +export KSERVE_VERSION=v0.20.0-rc0 + +helm upgrade -i kserve-llmisvc-resources \ + oci://ghcr.io/kserve/charts/kserve-llmisvc-resources \ + --version $KSERVE_VERSION \ + --namespace kserve \ + --set kserve.controller.deploymentMode=Standard \ + --set kserve.controller.gateway.ingressGateway.enableGatewayApi=true \ + --set kserve.controller.gateway.ingressGateway.createGateway=false \ + --set kserve.controller.gateway.ingressGateway.kserveGateway=kserve/kserve-ingress-gateway \ + --set kserve.controller.gateway.ingressGateway.className=agentgateway \ + --set kserve.controller.gateway.disableIstioVirtualHost=true \ + --set kserve.controller.gateway.disableIngressCreation=false \ + --set kserve.controller.knativeAddressableResolver.enabled=false \ + --set kserve.controller.gateway.localGateway.gateway="" \ + --set kserve.controller.gateway.localGateway.gatewayService="" +``` + +Wait for the updated controller, then apply the final GAIE v1.5.0 CRD bundle. +Applying the bundle after the KServe chart updates the stable API definitions +and retains KServe's transitional CRDs: + +```shell +kubectl rollout status deployment/llmisvc-controller-manager \ + --namespace kserve \ + --timeout=240s + +kubectl apply --server-side -f \ + https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.5.0/manifests.yaml +``` + +Install or upgrade agentgateway after the GAIE CRDs so that its controller +discovers `InferencePool`, then install the matching KServe runtime +configuration: + +```shell +export AGENTGATEWAY_VERSION=v1.4.1 + +helm upgrade -i agentgateway-crds \ + oci://cr.agentgateway.dev/charts/agentgateway-crds \ + --version $AGENTGATEWAY_VERSION \ + --namespace agentgateway-system \ + --create-namespace + +helm upgrade -i agentgateway \ oci://cr.agentgateway.dev/charts/agentgateway \ --version $AGENTGATEWAY_VERSION \ - --set inferenceExtension.enabled=true \ - --reuse-values + --namespace agentgateway-system \ + --set inferenceExtension.enabled=true + +helm upgrade -i kserve-runtime-configs \ + oci://ghcr.io/kserve/charts/kserve-runtime-configs \ + --version $KSERVE_VERSION \ + --namespace kserve \ + --set kserve.llmisvcConfigs.enabled=true ``` +KServe creates the `InferencePool` and deploys the llm-d Router endpoint picker +from its runtime configuration. Do not install the llm-d Router Helm chart +separately for this workflow. + ## Deploy LLMInferenceService ### Create Namespace @@ -72,14 +126,15 @@ kubectl create namespace kserve-test ### Create Gateway -Create an agentgateway Gateway resource: +Create a shared agentgateway Gateway resource in the `kserve` namespace. +Routes from model namespaces can attach to this Gateway: ```yaml apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: - name: agentgateway - namespace: kserve-test + name: kserve-ingress-gateway + namespace: kserve spec: gatewayClassName: agentgateway listeners: @@ -88,7 +143,10 @@ spec: port: 80 allowedRoutes: namespaces: - from: Same + from: All + infrastructure: + labels: + serving.kserve.io/gateway: kserve-ingress-gateway ``` ### Deploy Your Model @@ -96,7 +154,7 @@ spec: Deploy an LLMInferenceService. This example uses a small model for demonstration; replace with your model of choice: ```yaml -apiVersion: serving.kserve.io/v1alpha1 +apiVersion: serving.kserve.io/v1alpha2 kind: LLMInferenceService metadata: name: my-model @@ -198,7 +256,7 @@ import TabItem from '@theme/TabItem'; Override the route configuration on an individual `LLMInferenceService` using `spec.router.route.http`: ```yaml -apiVersion: serving.kserve.io/v1alpha1 +apiVersion: serving.kserve.io/v1alpha2 kind: LLMInferenceService metadata: name: my-model @@ -216,8 +274,8 @@ spec: parentRefs: - group: gateway.networking.k8s.io kind: Gateway - name: agentgateway - namespace: kserve-test + name: kserve-ingress-gateway + namespace: kserve rules: - backendRefs: - group: agentgateway.dev @@ -257,7 +315,7 @@ spec: Create an `LLMInferenceServiceConfig` that overrides the route template. This can be referenced by multiple `LLMInferenceService` resources via `baseRefs`: ```yaml -apiVersion: serving.kserve.io/v1alpha1 +apiVersion: serving.kserve.io/v1alpha2 kind: LLMInferenceServiceConfig metadata: name: agentgateway-route-config @@ -270,8 +328,8 @@ spec: parentRefs: - group: gateway.networking.k8s.io kind: Gateway - name: agentgateway - namespace: kserve-test + name: kserve-ingress-gateway + namespace: kserve rules: - backendRefs: - group: agentgateway.dev @@ -300,7 +358,7 @@ spec: Then reference it in your LLMInferenceService: ```yaml -apiVersion: serving.kserve.io/v1alpha1 +apiVersion: serving.kserve.io/v1alpha2 kind: LLMInferenceService metadata: name: my-model @@ -354,7 +412,8 @@ spec: Check if your Gateway has an external IP address assigned: ```shell -kubectl get svc -n kserve-test -l gateway.networking.k8s.io/gateway-name=agentgateway +kubectl get svc -n kserve \ + -l gateway.networking.k8s.io/gateway-name=kserve-ingress-gateway ``` @@ -362,7 +421,7 @@ kubectl get svc -n kserve-test -l gateway.networking.k8s.io/gateway-name=agentga If the EXTERNAL-IP shows an actual IP address (not <pending>): ```shell - export GATEWAY_URL="http://$(kubectl get gateway -n kserve-test agentgateway \ + export GATEWAY_URL="http://$(kubectl get gateway -n kserve kserve-ingress-gateway \ -o jsonpath='{.status.addresses[0].value}')" ``` @@ -371,7 +430,7 @@ kubectl get svc -n kserve-test -l gateway.networking.k8s.io/gateway-name=agentga ```shell export GATEWAY_URL="http://localhost:8080" - kubectl port-forward -n kserve-test svc/agentgateway 8080:80 + kubectl port-forward -n kserve svc/kserve-ingress-gateway 8080:80 ``` @@ -414,7 +473,7 @@ If you configured an `AgentgatewayBackend`, check the agentgateway logs to confirm the LLM pipeline is active: ```shell -kubectl logs -n kserve-test deploy/agentgateway --tail=10 +kubectl logs -n kserve deploy/kserve-ingress-gateway --tail=10 ``` With `AgentgatewayBackend`, you should see GenAI fields in the log: diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-config-composition.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-config-composition.md index 62f058240..297dc549a 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-config-composition.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-config-composition.md @@ -100,7 +100,7 @@ The sources that get merged (well-known configs are auto-injected, baseRef is re 
-{"router:\n  scheduler:\n    pool:\n      spec:\n        selector: [...]\n        targetPort: 8000\n    template:\n      containers:\n        - name: epp\n          image: llm-d-inference-scheduler\n          ports: [9002]\n        - name: tokenizer\n          image: llm-d-uds-tokenizer\n          ports: [8082]"}
+{"router:\n  scheduler:\n    pool:\n      spec:\n        selector: [...]\n        targetPorts:\n          - number: 8000\n    template:\n      containers:\n        - name: epp\n          image: llm-d-router-endpoint-picker\n          ports: [9002]\n        - name: tokenizer\n          image: llm-d-uds-tokenizer\n          ports: [8082]"}
@@ -199,10 +199,10 @@ The sources that get merged (well-known configs are auto-injected, baseRef is re {" template:\n"} {" containers:\n"} {" - name: epp\n"} -{" image: llm-d-inference-scheduler\n"} +{" image: llm-d-router-endpoint-picker\n"} {" ports: [9002]\n"} {" - name: tokenizer\n"} -{" image: llm-d-inference-scheduler\n"} +{" image: llm-d-router-endpoint-picker\n"} {" ports: [8082]\n"} {" route:\n"} {" http:\n"} diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md index 32d827f73..b3cc4904e 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-dependencies.md @@ -50,7 +50,6 @@ Extends Gateway API with **LLM-specific scheduling and load balancing** capabili ### Why Required - **Intelligent Routing**: Routes requests to optimal pods based on KV cache, load, and prefill-decode separation - **InferencePool**: Represents a pool of inference pods with custom scheduling logic -- **InferenceModel**: Defines model metadata and criticality for scheduling decisions ### How LLMInferenceService Uses It @@ -152,7 +151,7 @@ Gateway API is a **specification**, not an implementation. The actual traffic ro 1. **Gateway Provider Initialization**: When a Gateway Provider (e.g., Envoy Gateway, Istio) starts, it scans for available CRDs to determine which extensions to support. -2. **GIE Support**: If GIE CRDs are installed **after** the Gateway Provider, the provider won't know about `InferencePool` and `InferenceModel` resources. +2. **GIE Support**: If GIE CRDs are installed **after** the Gateway Provider, the provider might not detect `InferencePool` resources. 3. **Operator Restart Required**: Installing GIE CRDs later requires restarting the Gateway Provider operator to detect the new CRDs. @@ -160,17 +159,17 @@ Gateway API is a **specification**, not an implementation. The actual traffic ro ```bash # Step 1: Install cert-manager (required by LWS) -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.0/cert-manager.yaml +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.2/cert-manager.yaml # Step 2: Install Gateway API CRDs -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml +kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.0/standard-install.yaml # Step 3: Install GIE CRDs (BEFORE Gateway Provider!) -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v0.3.0/install.yaml +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.5.0/manifests.yaml # Step 4: Install Gateway Provider (Envoy Gateway example) helm install eg oci://docker.io/envoyproxy/gateway-helm \ - --version v1.2.4 \ + --version v1.8.3 \ -n envoy-gateway-system --create-namespace # Step 5: Install LWS Operator (if using multi-node) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md index d61bab406..7a8f54984 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md @@ -109,11 +109,10 @@ spec: ### Create EndpointPickerConfig -The llm-d Router, formerly called the Endpoint Picker (EPP), is the scheduler -used with the Gateway API Inference Extension. It selects the best backend -endpoint (pod) from the InferencePool for each request. You can customize the -scheduling behavior by defining plugins for scoring, filtering, and picking -endpoints. +The llm-d Router provides the Endpoint Picker (EPP) used with the Gateway API +Inference Extension. The EPP selects the best backend endpoint from the +`InferencePool` for each request. You can customize its behavior with plugins +for scoring, filtering, and selecting endpoints. About the configuration: @@ -124,13 +123,11 @@ About the configuration: Common plugins used in this guide: -- single-profile-handler (Profile Handler): Always selects a single, primary profile. Parameters: none. -- prefix-cache-scorer (Scorer): Increases score for pods likely to contain more of the prompt in their KV cache, improving latency and throughput. Parameters: - - hashBlockSize: Block size for prompt hashing (default: 64). - - maxPrefixBlocksToMatch: Maximum number of prefix blocks to match (default: 256). - - lruCapacityPerServer: LRU index capacity per server/pod (default: 31250). -- load-aware-scorer (Scorer): Scores candidates based on current load; lower load yields a higher score. Parameters may include sensitivity controls such as threshold (example below uses threshold: 100). -- max-score-picker (Picker): Chooses the candidate with the highest aggregate score. +- queue-scorer: Prefers endpoints with shorter request queues. +- kv-cache-utilization-scorer: Accounts for each endpoint's KV-cache usage. +- prefix-cache-scorer: Prefers endpoints that already cache more of the request prefix. +- metrics-data-source: Reads model server metrics used by scoring plugins. +- core-metrics-extractor: Extracts the core vLLM metrics used by the router. ```yaml apiVersion: v1 @@ -140,23 +137,27 @@ metadata: namespace: kserve-test data: endpoint-picker-config.yaml: | - apiVersion: inference.networking.x-k8s.io/v1alpha1 + apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - - type: single-profile-handler + - type: queue-scorer + - type: kv-cache-utilization-scorer - type: prefix-cache-scorer - - type: load-aware-scorer + - type: metrics-data-source parameters: - threshold: 100 - - type: max-score-picker + scheme: http + path: /metrics + insecureSkipVerify: true + - type: core-metrics-extractor schedulingProfiles: - name: default plugins: + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 - pluginRef: prefix-cache-scorer - weight: 2.0 - - pluginRef: load-aware-scorer - weight: 1.0 - - pluginRef: max-score-picker + weight: 3 ``` ### Create LLMInferenceServiceConfig @@ -257,13 +258,16 @@ spec: scheduler: pool: spec: - extensionRef: + endpointPickerRef: failureMode: FailOpen kind: Service name: |- {{ ChildName .ObjectMeta.Name `-epp-service` }} + port: + number: 9002 selector: { } - targetPortNumber: 8000 + targetPorts: + - number: 8000 template: containers: - name: main @@ -277,13 +281,13 @@ spec: - containerPort: 9090 name: metrics protocol: TCP - image: ghcr.io/llm-d/llm-d-inference-scheduler:v0.2.0 + image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 grpc: port: 9003 - service: envoy.service.ext_proc.v3.ExternalProcessor + service: liveness initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 @@ -292,26 +296,29 @@ spec: failureThreshold: 3 grpc: port: 9003 - service: envoy.service.ext_proc.v3.ExternalProcessor + service: readiness initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 - args: - - --poolName + command: + - /app/epp + - --pool-name - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --poolNamespace + - --pool-namespace - "{{ .ObjectMeta.Namespace }}" - --zap-encoder - json - - --grpcPort + - --grpc-port - "9002" - - --grpcHealthPort + - --grpc-health-port - "9003" - - --secureServing - - --certPath + - --secure-serving=true + - --enable-cert-reload=true + - --model-server-metrics-scheme=https + - --cert-path - "/etc/ssl/certs" - - --configFile + - --config-file - "/etc/config/endpoint-picker-config.yaml" resources: requests: @@ -414,7 +421,7 @@ spec: name: x-ai-eg-model value: Qwen/Qwen2.5-0.5B-Instruct backendRefs: - - group: inference.networking.x-k8s.io + - group: inference.networking.k8s.io kind: InferencePool name: qwen-instruct-inference-pool # Route to the InferencePool created by the LLMInferenceService timeouts: From 9632a3e33cc8fd9b34136b743b28e0558857a9ab Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 30 Jul 2026 16:58:39 -0700 Subject: [PATCH 8/9] docs: align inference gateway titles Signed-off-by: Daneyon Hansen --- .../llmisvc/llmisvc-inference-gateway-extension.md | 4 ++-- .../llmisvc/llmisvc-inference-gateways.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md index 7a8f54984..459edf17f 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md @@ -1,9 +1,9 @@ --- -title: Gateway API Inference Extension with Envoy AI Gateway +title: LLMInferenceService with Envoy AI Gateway description: How to integrate KServe LLMInferenceService with Envoy AI Gateway to manage LLM traffic and usage-based rate limits --- -# Gateway API Inference Extension with Envoy AI Gateway +# LLMInferenceService with Envoy AI Gateway This tutorial walks through deploying a KServe LLMInferenceService that uses the [llm-d Router](https://llm-d.ai/docs/infrastructure/gateway) with diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md index 31ea4c38f..afd613c75 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md @@ -15,7 +15,7 @@ gateway and LLM traffic-management features you want to use. | Gateway | Integration | Use this guide for | | --- | --- | --- | | [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | Standard `InferencePool` routing with optional `AgentgatewayBackend` configuration for AI policies | -| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [Gateway API Inference Extension with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | +| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [LLMInferenceService with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | Both guides use Gateway API resources and KServe's configurable router templates. Their gateway-specific custom resources and supported policies are From 3d356300dbc5d0ecde08117e2baf9b1cad7b4865 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 10 Aug 2026 14:20:54 -0700 Subject: [PATCH 9/9] docs: consolidate inference gateway guidance Fixes: #714 Signed-off-by: Daneyon Hansen --- docs/getting-started/genai-first-isvc.md | 2 +- docs/intro.mdx | 2 +- .../ai-gateway/envoy-ai-gateway.md | 423 ------------------ .../ai-gateway/inference-gateways.md | 43 ++ .../llmisvc/llmisvc-agentgateway.md | 2 +- .../llmisvc-inference-gateway-extension.md | 2 +- .../llmisvc/llmisvc-inference-gateways.md | 29 -- .../generative-inference/overview.md | 2 +- .../tasks/embedding/embedding.md | 2 +- .../tasks/reranking/rerank.md | 2 +- .../tasks/text-generation/text-generation.md | 2 +- .../text2text-generation.md | 2 +- docs/reference/integrations.md | 2 +- docusaurus.config.ts | 8 + sidebars.ts | 3 +- 15 files changed, 62 insertions(+), 464 deletions(-) delete mode 100644 docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md create mode 100644 docs/model-serving/generative-inference/ai-gateway/inference-gateways.md delete mode 100644 docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md diff --git a/docs/getting-started/genai-first-isvc.md b/docs/getting-started/genai-first-isvc.md index 5325ded96..d97c74cf4 100644 --- a/docs/getting-started/genai-first-isvc.md +++ b/docs/getting-started/genai-first-isvc.md @@ -320,4 +320,4 @@ Now that you have successfully deployed a generative AI service using KServe, yo - 📖 **[Autoscaling](../model-serving/generative-inference/autoscaling/autoscaling.md)**: Automatically scale your service based on traffic and resource usage / metrics. - 📖 **[KV Cache Offloading](../model-serving/generative-inference/kvcache-offloading/kvcache-offloading.md)** - Learn how to offload key-value caches to external storage for improved performance and reduced latency. - 📖 **[Model Caching](../model-serving/generative-inference/modelcache/localmodel.md)** - Learn how to cache models for faster startup time. -- 📖 **[Token Rate Limiting](../model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md)** - Rate limit users based on token usage. +- 📖 **[Inference Gateways](../model-serving/generative-inference/ai-gateway/inference-gateways.md)** - Compare gateway integrations for `LLMInferenceService` routing and AI policies. diff --git a/docs/intro.mdx b/docs/intro.mdx index 85340041a..b6f70bbc5 100644 --- a/docs/intro.mdx +++ b/docs/intro.mdx @@ -48,7 +48,7 @@ KServe eliminates the complexity of productionizing AI models. Whether you're a | **LocalModelCache** | Reduce LLM startup time from 15–20 minutes to ~1 minute | | **KV Cache Offloading** | Optimized memory management for long conversations and large contexts | | **Multi-node Inference** | Distributed LLM serving across multiple nodes | -| **Envoy AI Gateway** | Enterprise-grade API management and routing for AI workloads | +| **Inference Gateways** | Envoy AI Gateway and agentgateway options for LLMInferenceService | | **Metric-based Autoscaling** | Scale on token throughput, queue depth, and GPU utilization | | **Canary Deployments** | A/B testing and canary rollouts for LLM experiments | diff --git a/docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md b/docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md deleted file mode 100644 index a7b0b4c25..000000000 --- a/docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md +++ /dev/null @@ -1,423 +0,0 @@ ---- -title: Rate Limiting with AI Gateway -description: How to integrate KServe with the Envoy AI Gateway for managing LLM/AI traffic ---- - -# Integrating KServe with Envoy AI Gateway - -This guide explains how to integrate KServe with the Envoy AI Gateway. KServe can be integrated with the AI Gateway either within the same cluster or across different clusters. In this example, we will demonstrate how to integrate KServe with the Envoy AI Gateway within the same cluster. - -## AI Gateway Overview - -The Envoy AI Gateway was created to address the complexity of connecting applications to GenAI services by leveraging Envoy's flexibility and Kubernetes-native features. It provides a secure, scalable, and efficient way to manage LLM/AI traffic, with backend rate limiting and policy control.Key features of the AI Gateway include: - -- **Request Routing**: Directs API requests to appropriate GenAI services. -- **Authentication and Authorization**: Implements API key validation to secure communication. -- **Backend Security Policy**: Introduces fine-grained access control for backend services, controlling LLM/AI backend usage using token-per-second (TPS) policies to prevent overuse. -- **Multi-Upstream Provider Support for LLM/AI Services**: The ability to receive requests in the format of one LLM provider and route them to different upstream providers, ensuring compatibility with their expected formats. This is achieved through built-in transformation capabilities that adapt requests and responses accordingly. -- **AWS Request Signing**: Supports external processing for secure communication with AWS-hosted LLM/AI services. - -For more information, see the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/). - -## Prerequisites - -Before you begin, ensure you have the following components installed and configured: - -- A Kubernetes cluster with [KServe with Gateway API Enabled](../../../admin-guide/kubernetes-deployment.md) -- [Envoy Gateway](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites) installed in your cluster -- [Envoy AI Gateway](https://aigateway.envoyproxy.io/docs/getting-started/installation) installed in your cluster -- Hugging Face token for accessing the LLM models (you can get a token from [Hugging Face](https://huggingface.co/settings/tokens)) -- The `kubectl` command-line tool installed and configured to access your cluster -- Basic understanding of Kubernetes concepts and [KServe](../../../concepts/index.md) - -## Deploy InferenceService - -### Create Namespace - -Create a namespace for the InferenceService. - -```shell -kubectl create namespace kserve-test -``` - -### Create a Hugging Face Secret -Create a secret with the Hugging Face token in the `kserve-test` namespace. This token will be used to access the Hugging Face models. - -```yaml title="hf-secret.yaml" -apiVersion: v1 -kind: Secret -metadata: - name: hf-secret - namespace: kserve-test -type: Opaque -stringData: - HF_TOKEN: -``` - -### Create an InferenceService - -Create an InferenceService in the `kserve-test` namespace with OpenAI route prefix disabled. The following example creates an InferenceService with the 'llama3.2-1B' model from Hugging Face. - -```yaml -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - name: llama3-1b - namespace: kserve-test -spec: - predictor: - model: - modelFormat: - name: huggingface - args: - - --model_name=llama3-1b - - --model_id=meta-llama/Llama-3.2-1B-Instruct - env: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-secret - key: HF_TOKEN - optional: false - - - name: KSERVE_OPENAI_ROUTE_PREFIX # Disable OpenAI Route Prefix - value: "" - resources: - requests: - cpu: "4" - memory: 12Gi - nvidia.com/gpu: "1" - limits: - cpu: "6" - memory: 12Gi - nvidia.com/gpu: "1" -``` - -## Create BackendSecurityPolicy - -You can configure the BackendSecurityPolicy for authentication and authorization with the InferenceService. For example, you can create a BackendSecurityPolicy to secure communication with the InferenceService using an API key. - -But for simplicity, we will ignore the authentication and authorization for this example. - -```yaml -apiVersion: aigateway.envoyproxy.io/v1alpha1 -kind: BackendSecurityPolicy -metadata: - name: envoy-ai-gateway-openai-kserve-apikey - namespace: default -spec: - type: APIKey - apiKey: - secretRef: - name: envoy-ai-gateway-openai-kserve-apikey - namespace: default -``` - -## Create BackendTLSPolicy - -If the InferenceService is using TLS, you can create a BackendTLSPolicy to configure the TLS settings for the InferenceService. - -For this example, we will ignore the TLS settings. - -```yaml -apiVersion: gateway.networking.k8s.io/v1alpha3 -kind: BackendTLSPolicy -metadata: - name: envoy-ai-gateway-basic-openai-tls - namespace: default -spec: - targetRefs: - - group: "" - kind: Service - name: llama3-1b-predictor - validation: - wellKnownCACertificates: "System" - hostname: "llama3-1b-kserve-test.example.com" -``` - -## Create AIServiceBackend - -Create an AIServiceBackend for the InferenceService created in the previous step. You can uncomment the `backendSecurityPolicyRef` field to use the BackendSecurityPolicy if you have configured it. - -```yaml -apiVersion: aigateway.envoyproxy.io/v1alpha1 -kind: AIServiceBackend -metadata: - name: envoy-ai-gateway-openai-kserve - namespace: default -spec: - schema: - name: OpenAI - backendRef: - name: llama3-1b-predictor - namespace: kserve-test - kind: Service - group: "" - port: 80 -# backendSecurityPolicyRef: -# name: envoy-ai-gateway-openai-kserve-apikey -# kind: BackendSecurityPolicy -# group: aigateway.envoyproxy.io -``` - -## Create ReferenceGrant - -Since the InferenceService is in the `kserve-test` namespace and the AIServiceBacked in the `default` namespace, we need to create a `ReferenceGrant` to allow the `AIServiceBackend` to reference the InferenceService. - -```yaml -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: ReferenceGrant -metadata: - name: envoy-gateway-ref-grant - namespace: kserve-test -spec: - from: - - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: default - to: - - group: "" - kind: Service -``` - -## Create Gateway for the AI Gateway - -Create a Gateway for the AI Gateway to route the traffic to different LLM providers. - -```yaml -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: envoy-ai-gateway-basic - namespace: default -spec: - gatewayClassName: envoy-ai-gateway-basic - listeners: - - name: http - protocol: HTTP - port: 80 -``` - -## Create AIGatewayRoute with LLM Request Costs - -Create an `AIGatewayRoute` and configure KServe as the LLM service provider for the model `llama3-1b` using the `AIServiceBackend` created in the previous step. AI Gateway automatically tracks token usage for each request. We will configure `AIGatewayRoute` to track InputToken, OutputToken, and TotalToken usage. - -```yaml -apiVersion: aigateway.envoyproxy.io/v1alpha1 -kind: AIGatewayRoute -metadata: - name: envoy-ai-gateway - namespace: default -spec: - schema: - name: OpenAI - targetRefs: - - name: envoy-ai-gateway-basic - kind: Gateway - group: gateway.networking.k8s.io - rules: - - matches: - - headers: - - type: Exact - name: x-ai-eg-model - value: llama3-1b - backendRefs: - - name: envoy-ai-gateway-openai-kserve # AIServiceBackend Name - weight: 100 - timeouts: - request: 60s - llmRequestCosts: - - metadataKey: llm_input_token - type: InputToken # Counts tokens in the request - - metadataKey: llm_output_token - type: OutputToken # Counts tokens in the response - - metadataKey: llm_total_token - type: TotalToken # Tracks combined usage -``` - -The traffic from the Envoy AI Gateway will be routed to the InferenceService based on the `x-ai-eg-model` header value. This header is automatically set by the AI Gateway by reading the model name in the payload request. - -## Configure Rate Limiting - -AI Gateway uses Envoy Gateway's Global Rate Limit API to configure rate limits. Rate limits should be defined using a combination of user and model identifiers to properly control costs at the model level. We will configure a rate limit of 1000 total tokens per hour per user for the model `llama3-1b` using `BackendTrafficPolicy`. - -```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: BackendTrafficPolicy -metadata: - name: model-specific-token-limit-policy - namespace: default -spec: - targetRefs: - - name: envoy-ai-gateway-basic - kind: Gateway - group: gateway.networking.k8s.io - rateLimit: - type: Global - global: - rules: - # Rate limit rule for llama3-1b: 1000 total tokens per hour per user - - clientSelectors: - - headers: - - name: x-user-id - type: Distinct - - name: x-ai-eg-model - type: Exact - value: llama3-1b - limit: - requests: 1000 # 1000 total tokens per hour - unit: Hour - cost: - request: - from: Number - number: 0 # Set to 0 so only token usage counts - response: - from: Metadata - metadata: - namespace: io.envoy.ai_gateway - key: llm_total_token # Uses total tokens from the responses -``` - -:::warning -When configuring rate limits: - -1. Always set the request cost number to 0 to ensure only token usage counts towards the limit. -2. Set appropriate limits for different models based on their costs and capabilities. -3. Ensure both user and model identifiers are used in rate limiting rules. -::: - -## Configure $GATEWAY_URL - -First, check if your Gateway has an external IP address assigned: - -```shell -kubectl get svc -n envoy-gateway-system \ - --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic -``` - -You'll see output similar to this: - -``` -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) -eg-envoy-ai-gateway LoadBalancer 10.96.61.234 80:31234/TCP -``` - -Choose one of these options based on the EXTERNAL-IP status: - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - If the EXTERNAL-IP shows an actual IP address (not <pending>), you can access the gateway directly: - - First, save the external IP and set the gateway URL: - - ```shell - export GATEWAY_URL=$(kubectl get gateway/envoy-ai-gateway-basic -o jsonpath='{.status.addresses[0].value}') - ``` - - - If the EXTERNAL-IP shows <pending> or your cluster doesn't support LoadBalancer services, use port forwarding. - - First, set the gateway URL: - - ```shell - export GATEWAY_URL="http://localhost:8080" - ``` - Then set up port forwarding (this will block the terminal): - - ```shell - export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system \ - --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic \ - -o jsonpath='{.items[0].metadata.name}') - - kubectl port-forward -n envoy-gateway-system svc/$ENVOY_SERVICE 8080:80 - ``` - - - -## Testing the Gateway - -Verify the URL is available: - -```shell -echo $GATEWAY_URL -``` - -:::tip -If you're opening a new terminal, you'll need to set the GATEWAY_URL variable again. -For proper cost control and rate limiting, requests must include: -- x-user-id: Identifies the user making the request -::: - -Send a test request to the AI Gateway as user _user123_ using the GATEWAY_URL we set up: -```shell -curl -v -H "Content-Type: application/json" -H "x-user-id: user123" -d '{ - "model": "llama3-1b", - "messages": [ - { - "role": "system", - "content": "You are a poet." - }, - { - "role": "user", - "content": "Write a poem about clouds." - } - ] - }' localhost:8080/v1/chat/completions -``` - -:::tip[Expected Output] -The response should be similar to the following: - ```json - { - "id": "chatcmpl-7e49ea73-8e9a-4790-9fdd-c6e551ae14b0", - "object": "chat.completion", - "created": 1742223214, - "model": "llama3-1b", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "reasoning_content": null, - "content": "Majestic silhouettes drifting by,\n Shapes that morph, a constant sigh,\nTheir whispers echo, yet remain unseen,\nGently drifting, leaving memories unseen.\n\nSoft puffs of white, like cotton tufts,\nFleeting wisps, a wistful route,\nAcross the sky, a canvas so grand,\nA symphony of shapes, at the clouds' command.\n\nTheir shadows dance, on walls so bright,\nA silver glow, in the moon's pale light,\nTheir silence is a reassuring sound,\nAs clouds resurface, without a bound.\n\nIn their depths, a world is spun,\nA realm of wonder, where dreams have begun,\nTheir wispy tendrils, reaching high,\nA veil of mystery, touched by the sky.\n\nAnd when they fade, into the ground,\nTheir memory remains, a poet's sound,\nFor in their fleeting, ephemeral kiss,\nLies a beauty, that forever bliss.\n\nTheir ethereal voice, a lullaby sweet,\nEchoes still, a wonder to the heart hecat,\nFor in the clouds, our souls ascend,\nAnd the magic of the heavens, will forever transcend.", - "tool_calls": [] - }, - "logprobs": null, - "finish_reason": "stop", - "stop_reason": null - } - ], - "usage": { - "prompt_tokens": 46, - "total_tokens": 277, - "completion_tokens": 231, - "prompt_tokens_details": null - }, - "prompt_logprobs": null - } - ``` -::: - -Once the token limit is reached, you will receive a 429 error response with the message `Too Many Requests`. For example: -``` -< HTTP/1.1 429 Too Many Requests -< x-envoy-ratelimited: true -< x-ratelimit-limit: 1000, 1000;w=3600 -< x-ratelimit-remaining: 0 -< x-ratelimit-reset: 1715 -< date: Mon, 17 Mar 2025 14:31:24 GMT -< content-length: 0 -< -* Connection #0 to host localhost left intact -``` - -## Next Steps - -Now that you've tested the basic setup, you can: - -- For more rate limiter related configuration, see the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/capabilities/usage-based-ratelimiting). - -- Explore the [Envoy AI Gateway documentation](https://aigateway.envoyproxy.io/docs/) to learn more about the features and capabilities. diff --git a/docs/model-serving/generative-inference/ai-gateway/inference-gateways.md b/docs/model-serving/generative-inference/ai-gateway/inference-gateways.md new file mode 100644 index 000000000..fd0a49e9a --- /dev/null +++ b/docs/model-serving/generative-inference/ai-gateway/inference-gateways.md @@ -0,0 +1,43 @@ +--- +title: Inference Gateways +description: Choose an inference gateway integration for KServe LLMInferenceService +--- + +# Inference Gateways + +KServe `LLMInferenceService` integrates with Kubernetes Gateway API +implementations to expose models, route inference requests, and apply +gateway-specific traffic policies. For generative inference, use +`LLMInferenceService` instead of the general-purpose `InferenceService` API so +KServe can provision an `InferencePool` and the llm-d Router for +inference-aware endpoint selection. + +The common request path is: + +```text +Client -> Inference Gateway -> InferencePool -> llm-d Router -> Model server +``` + +Choose either agentgateway or Envoy AI Gateway for the gateway layer. KServe +manages the `LLMInferenceService`, `InferencePool`, router, and model-serving +workloads in both integrations. + +## Reference integrations + +| Gateway | Integration | Use this guide for | +| --- | --- | --- | +| [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](../llmisvc/llmisvc-agentgateway.md) | Standard `InferencePool` routing with optional `AgentgatewayBackend` configuration for AI policies | +| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [LLMInferenceService with Envoy AI Gateway](../llmisvc/llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | + +Both guides use Gateway API resources and KServe's configurable router +templates. Their gateway-specific custom resources and supported policies are +different, so follow the guide for the gateway installed in your cluster. Do +not combine resources from the two guides unless you are intentionally running +both gateway implementations. + +For production llm-d Router deployment patterns, see the +[llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) and +[gateway guides](https://llm-d.ai/docs/infrastructure/gateway). The llm-d +documentation also provides provider-specific infrastructure guides for +[agentgateway](https://llm-d.ai/docs/infrastructure/gateway/agentgateway) and +[Envoy AI Gateway](https://llm-d.ai/docs/infrastructure/gateway/envoy-ai-gateway). diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md index 0f6bd777b..fb2d35dba 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-agentgateway.md @@ -505,7 +505,7 @@ without bypassing the endpoint picker. ## Next Steps -- Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). +- Compare the other [inference gateway integrations](../ai-gateway/inference-gateways.md). - Follow the [llm-d agentgateway guide](https://llm-d.ai/docs/infrastructure/gateway/agentgateway) for the llm-d Router and gateway infrastructure. - Explore the [llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) for production deployment patterns. - Explore the [agentgateway documentation](https://agentgateway.dev/docs) for advanced features like content filtering and cost tracking. diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md index 459edf17f..f6e73c19c 100644 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md +++ b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension.md @@ -658,7 +658,7 @@ done Now that you've tested the basic setup, you can: -- Compare the other [inference gateway integrations](./llmisvc-inference-gateways.md). +- Compare the other [inference gateway integrations](../ai-gateway/inference-gateways.md). - Review the [llm-d gateway guides](https://llm-d.ai/docs/infrastructure/gateway). - Follow the [llm-d Envoy AI Gateway guide](https://llm-d.ai/docs/infrastructure/gateway/envoy-ai-gateway). - Explore the [llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) for production deployment patterns. diff --git a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md b/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md deleted file mode 100644 index afd613c75..000000000 --- a/docs/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Inference Gateways -description: Reference inference gateway integrations for KServe LLMInferenceService ---- - -# Inference Gateways - -KServe `LLMInferenceService` integrates with Kubernetes Gateway API -implementations to expose models, route inference requests, and apply -gateway-specific traffic policies. Choose a reference integration based on the -gateway and LLM traffic-management features you want to use. - -## Reference integrations - -| Gateway | Integration | Use this guide for | -| --- | --- | --- | -| [agentgateway](https://agentgateway.dev/) | [LLMInferenceService with agentgateway](./llmisvc-agentgateway.md) | Standard `InferencePool` routing with optional `AgentgatewayBackend` configuration for AI policies | -| [Envoy AI Gateway](https://aigateway.envoyproxy.io/) | [LLMInferenceService with Envoy AI Gateway](./llmisvc-inference-gateway-extension.md) | Routing to an `InferencePool` with an endpoint picker, OpenAI-compatible routing, token accounting, and usage-based rate limiting | - -Both guides use Gateway API resources and KServe's configurable router -templates. Their gateway-specific custom resources and supported policies are -different, so follow the guide for the gateway installed in your cluster. - -For production llm-d Router deployment patterns, see the -[llm-d well-lit paths](https://llm-d.ai/docs/well-lit-paths) and -[gateway guides](https://llm-d.ai/docs/infrastructure/gateway). The llm-d -documentation also provides provider-specific infrastructure guides for -[agentgateway](https://llm-d.ai/docs/infrastructure/gateway/agentgateway) and -[Envoy AI Gateway](https://llm-d.ai/docs/infrastructure/gateway/envoy-ai-gateway). diff --git a/docs/model-serving/generative-inference/overview.md b/docs/model-serving/generative-inference/overview.md index 910335472..17f196994 100644 --- a/docs/model-serving/generative-inference/overview.md +++ b/docs/model-serving/generative-inference/overview.md @@ -176,7 +176,7 @@ The Hugging Face runtime supports several advanced features to enhance model ser - [**Model Caching**](./modelcache/localmodel.md): Cache models on local storage for faster loading and reduced latency. This is particularly useful for large models that are frequently accessed. - [**KV Cache Offloading**](./kvcache-offloading/kvcache-offloading.md): Offload key-value caches to CPU memory to reduce GPU memory usage, allowing larger models to be served on GPUs with limited memory. - [**Distributed LLM Serving**](./multi-node/multi-node.md): Scale model serving across multiple nodes and GPUs for high throughput and low latency. This is useful for serving large models or handling high request volumes. -- [**AI Gateway**](./ai-gateway/envoy-ai-gateway.md): Use the AI Gateway to manage rate-limiting based on tokens and route requests to different models, providing a unified API for various generative tasks. +- [**Inference Gateways**](./ai-gateway/inference-gateways.md): Use Envoy AI Gateway or agentgateway with `LLMInferenceService` for inference-aware routing and gateway-specific AI policies. ## Environment Variables diff --git a/docs/model-serving/generative-inference/tasks/embedding/embedding.md b/docs/model-serving/generative-inference/tasks/embedding/embedding.md index 856f21f97..da59cd42f 100644 --- a/docs/model-serving/generative-inference/tasks/embedding/embedding.md +++ b/docs/model-serving/generative-inference/tasks/embedding/embedding.md @@ -264,6 +264,6 @@ Once you've successfully deployed your embedding model, consider: - **Exploring other inference tasks** such as [text-to-text generation](../text2text-generation/text2text-generation.md) and [reranking](../reranking/rerank.md) - **Optimizing performance** with features like [model caching](../../modelcache/localmodel.md) and [KV cache offloading](../../kvcache-offloading/kvcache-offloading.md) - **Auto-scaling** your inference services based on traffic patterns using [KServe's auto-scaling capabilities](../../autoscaling/autoscaling.md) -- **Token based rate limiting** to control usage with [AI Gateway](../../ai-gateway/envoy-ai-gateway.md) for serving models. +- **Inference gateway policies** with [Envoy AI Gateway or agentgateway](../../ai-gateway/inference-gateways.md) for `LLMInferenceService` workloads. For more information on KServe's capabilities for generative AI, see the [generative inference overview](../../overview.md). diff --git a/docs/model-serving/generative-inference/tasks/reranking/rerank.md b/docs/model-serving/generative-inference/tasks/reranking/rerank.md index e39307358..e1eb30f6e 100644 --- a/docs/model-serving/generative-inference/tasks/reranking/rerank.md +++ b/docs/model-serving/generative-inference/tasks/reranking/rerank.md @@ -223,6 +223,6 @@ Once you've successfully deployed your reranker model, consider: - **Exploring other inference tasks** such as [text-to-text generation](../text2text-generation/text2text-generation.md) and [embedding](../embedding/embedding.md) - **Optimizing performance** with features like [model caching](../../modelcache/localmodel.md) and [KV cache offloading](../../kvcache-offloading/kvcache-offloading.md) - **Auto-scaling** your inference services based on traffic patterns using [KServe's auto-scaling capabilities](../../autoscaling/autoscaling.md) -- **Token based rate limiting** to control usage with [AI Gateway](../../ai-gateway/envoy-ai-gateway.md) for serving models. +- **Inference gateway policies** with [Envoy AI Gateway or agentgateway](../../ai-gateway/inference-gateways.md) for `LLMInferenceService` workloads. For more information on KServe's capabilities for generative AI, see the [generative inference overview](../../overview.md). diff --git a/docs/model-serving/generative-inference/tasks/text-generation/text-generation.md b/docs/model-serving/generative-inference/tasks/text-generation/text-generation.md index 4d80d1929..3c040d39e 100644 --- a/docs/model-serving/generative-inference/tasks/text-generation/text-generation.md +++ b/docs/model-serving/generative-inference/tasks/text-generation/text-generation.md @@ -391,6 +391,6 @@ Once you've successfully deployed your text generation model, consider: - **Exploring other inference tasks** such as [text-to-text generation](../text2text-generation/text2text-generation.md) and [embedding](../embedding/embedding.md) - **Optimizing performance** with features like [model caching](../../modelcache/localmodel.md) and [KV cache offloading](../../kvcache-offloading/kvcache-offloading.md) - **Auto-scaling** your inference services based on traffic patterns using [KServe's auto-scaling capabilities](../../autoscaling/autoscaling.md) -- **Token based rate limiting** to control usage with [AI Gateway](../../ai-gateway/envoy-ai-gateway.md) for serving models. +- **Inference gateway policies** with [Envoy AI Gateway or agentgateway](../../ai-gateway/inference-gateways.md) for `LLMInferenceService` workloads. For more information on KServe's capabilities for generative AI, see the [generative inference overview](../../overview.md). diff --git a/docs/model-serving/generative-inference/tasks/text2text-generation/text2text-generation.md b/docs/model-serving/generative-inference/tasks/text2text-generation/text2text-generation.md index a18699142..26c7e0498 100644 --- a/docs/model-serving/generative-inference/tasks/text2text-generation/text2text-generation.md +++ b/docs/model-serving/generative-inference/tasks/text2text-generation/text2text-generation.md @@ -204,6 +204,6 @@ Once you've successfully deployed your text generation model, consider: - **Exploring other inference tasks** such as [reranking](../reranking/rerank.md) and [embedding](../embedding/embedding.md) - **Optimizing performance** with features like [model caching](../../modelcache/localmodel.md) and [KV cache offloading](../../kvcache-offloading/kvcache-offloading.md) - **Auto-scaling** your inference services based on traffic patterns using [KServe's auto-scaling capabilities](../../autoscaling/autoscaling.md) -- **Token based rate limiting** to control usage with [AI Gateway](../../ai-gateway/envoy-ai-gateway.md) for serving models. +- **Inference gateway policies** with [Envoy AI Gateway or agentgateway](../../ai-gateway/inference-gateways.md) for `LLMInferenceService` workloads. For more information on KServe's capabilities for generative AI, see the [generative inference overview](../../overview.md). diff --git a/docs/reference/integrations.md b/docs/reference/integrations.md index 548d81f78..85a21fae3 100644 --- a/docs/reference/integrations.md +++ b/docs/reference/integrations.md @@ -173,7 +173,7 @@ spec: - **AI Specialization**: Built-in support for LLM-specific requirements - **Enterprise Features**: Advanced security and monitoring capabilities -For detailed Envoy integration, see the [AI Gateway Integration Guide](../model-serving/generative-inference/ai-gateway/envoy-ai-gateway.md). +For LLM gateway integrations, see the [Inference Gateways guide](../model-serving/generative-inference/ai-gateway/inference-gateways.md). ## ML Framework Integrations diff --git a/docusaurus.config.ts b/docusaurus.config.ts index a855aad51..24bc0d627 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -538,6 +538,14 @@ const config: Config = { from: '/latest/admin/ai-gateway_integration/', to: '/docs/model-serving/generative-inference/ai-gateway/envoy-ai-gateway', }, + { + from: '/docs/next/model-serving/generative-inference/ai-gateway/envoy-ai-gateway', + to: '/docs/next/model-serving/generative-inference/ai-gateway/inference-gateways', + }, + { + from: '/docs/next/model-serving/generative-inference/llmisvc/llmisvc-inference-gateways', + to: '/docs/next/model-serving/generative-inference/ai-gateway/inference-gateways', + }, { from: '/docs/next/model-serving/generative-inference/llmisvc/llmisvc-envoy-ai-gateway', to: '/docs/next/model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension', diff --git a/sidebars.ts b/sidebars.ts index 581a5a195..77f8e5459 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -144,7 +144,7 @@ const sidebars: SidebarsConfig = { label: 'Inference Gateways', link: { type: 'doc', - id: 'model-serving/generative-inference/llmisvc/llmisvc-inference-gateways', + id: 'model-serving/generative-inference/ai-gateway/inference-gateways', }, items: [ "model-serving/generative-inference/llmisvc/llmisvc-inference-gateway-extension", @@ -179,7 +179,6 @@ const sidebars: SidebarsConfig = { "model-serving/generative-inference/modelcache/localmodel", "model-serving/generative-inference/autoscaling/autoscaling", "model-serving/generative-inference/multi-node/multi-node", - "model-serving/generative-inference/ai-gateway/envoy-ai-gateway", ], }, {