Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions internal/embed/embed_crd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -741,6 +741,56 @@ func bindingHasSubject(doc map[string]any, name, namespace string) bool {
return false
}

func TestX402VerifierRBAC_CanReadAgentAPISecrets(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/x402.yaml")
if err != nil {
t.Fatalf("ReadInfrastructureFile: %v", err)
}
docs := multiDoc(data)

role := findDocByName(docs, "ClusterRole", "x402-verifier")
if role == nil {
t.Fatal("no ClusterRole 'x402-verifier' found")
}
rules, ok := role["rules"].([]any)
if !ok {
t.Fatal("x402-verifier ClusterRole has no rules")
}

for _, r := range rules {
rm := r.(map[string]any)
if !stringSet(rm["apiGroups"])[""] || !stringSet(rm["resources"])["secrets"] {
continue
}
verbs := stringSet(rm["verbs"])
if !verbs["get"] || !verbs["list"] || !verbs["watch"] {
continue
}
names := stringSet(rm["resourceNames"])
if !names["litellm-secrets"] {
t.Fatal("x402-verifier secret rule lost litellm-secrets")
}
if !names["hermes-api-server"] {
t.Fatal("x402-verifier secret rule must include hermes-api-server for agent upstream auth")
}
return
}

t.Fatal("x402-verifier ClusterRole missing scoped secret get/list/watch rule")
}

func TestX402VerifierImage_CarriesAgentAuthFix(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/x402.yaml")
if err != nil {
t.Fatalf("ReadInfrastructureFile: %v", err)
}

const ref = "ghcr.io/obolnetwork/x402-verifier:46e63fd@sha256:a8cd7946884c9a702b5cfcfad28d1f5eac1037899303eb4e0157e3ffab7a572c"
if !strings.Contains(string(data), "image: "+ref) {
t.Fatalf("x402-verifier image must carry agent upstream auth fix: %s", ref)
}
}

func TestAgentRBAC_NoOverlyBroadPermissions(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/obol-agent-monetize-rbac.yaml")
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/embed/infrastructure/base/templates/x402.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["litellm-secrets"]
resourceNames: ["litellm-secrets", "hermes-api-server"]
verbs: ["get", "list", "watch"]

---
Expand Down Expand Up @@ -234,7 +234,7 @@ spec:
type: RuntimeDefault
containers:
- name: verifier
image: ghcr.io/obolnetwork/x402-verifier:b13254e@sha256:a8a7aa0ca4c35b0ddf6983fa6e3e5f8a3f64e44d8e506ebfd55e39de2bc0342d
image: ghcr.io/obolnetwork/x402-verifier:46e63fd@sha256:a8cd7946884c9a702b5cfcfad28d1f5eac1037899303eb4e0157e3ffab7a572c
imagePullPolicy: IfNotPresent
# PSS Restricted: per-container hardening. Verifier is a Go binary
# reading two RO ConfigMaps; no writeable rootfs paths required.
Expand Down
10 changes: 10 additions & 0 deletions internal/serviceoffercontroller/agent_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ func (c *Controller) resolveAgentOffer(ctx context.Context, offer *monetizeapi.S
if ref.Name == "" || ref.Namespace == "" {
return false, fmt.Errorf("type=agent offer %s/%s missing spec.agent.ref", offer.Namespace, offer.Name)
}
if ref.Namespace != offer.Namespace {
// Confused-deputy guard: the verifier route source injects the
// hermes-api-server API_SERVER_KEY from ref.Namespace into the
// outbound Authorization header. Allowing a cross-namespace ref
// would let any principal with serviceoffers write in namespace A
// expose Hermes /api in namespace B as an x402-gated route under
// attacker-controlled path and payTo, granting paying buyers
// authenticated proxy access to the victim agent.
return false, fmt.Errorf("type=agent offer %s/%s: spec.agent.ref.namespace %q must equal offer namespace", offer.Namespace, offer.Name, ref.Namespace)
}

raw, err := c.agents.Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
Expand Down
49 changes: 49 additions & 0 deletions internal/serviceoffercontroller/agent_resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func TestResolveAgentOffer_PopulatesFromReadyAgent(t *testing.T) {
c := newResolverTestController(t, agent)

offer := &monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Namespace: "agent-quant"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Expand Down Expand Up @@ -83,6 +84,7 @@ func TestResolveAgentOffer_NotReadyAgentClearsResolution(t *testing.T) {
c := newResolverTestController(t, agent)

offer := &monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Namespace: "agent-quant"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Expand Down Expand Up @@ -113,6 +115,7 @@ func TestResolveAgentOffer_MissingAgentReturnsNotReady(t *testing.T) {
c := newResolverTestController(t)

offer := &monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Namespace: "agent-missing"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Expand Down Expand Up @@ -147,6 +150,52 @@ func TestResolveAgentOffer_RejectsMissingRef(t *testing.T) {
}
}

// TestResolveAgentOffer_RejectsCrossNamespaceRef guards the confused-deputy
// invariant: an offer in namespace A must not be allowed to reference an agent
// in namespace B, because the verifier route source injects ref.Namespace's
// hermes-api-server API_SERVER_KEY as the upstream Authorization. Allowing a
// cross-namespace ref would let any principal with serviceoffers write expose
// another tenant's Hermes /api as an x402-gated route under attacker-controlled
// path + payTo.
func TestResolveAgentOffer_RejectsCrossNamespaceRef(t *testing.T) {
agent := &monetizeapi.Agent{
TypeMeta: metav1.TypeMeta{APIVersion: "obol.org/v1alpha1", Kind: "Agent"},
ObjectMeta: metav1.ObjectMeta{Name: "victim", Namespace: "agent-victim"},
Status: monetizeapi.AgentStatus{
Phase: monetizeapi.AgentPhaseReady,
Endpoint: "http://hermes.agent-victim.svc.cluster.local:8642",
},
}
c := newResolverTestController(t, agent)

offer := &monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Name: "spoof", Namespace: "attacker-ns"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Ref: monetizeapi.ServiceOfferAgentRef{Name: "victim", Namespace: "agent-victim"},
},
},
}
status := monetizeapi.ServiceOfferStatus{
AgentResolution: &monetizeapi.ServiceOfferAgentResolution{Model: "stale"},
}

ok, err := c.resolveAgentOffer(context.Background(), offer, &status)
if err == nil {
t.Fatal("expected error for cross-namespace spec.agent.ref")
}
if ok {
t.Fatal("expected ok=false for cross-namespace ref")
}
if status.AgentResolution == nil || status.AgentResolution.Model != "stale" {
// Guard fires before touching status: the caller is responsible for
// the failure-mode condition update, and we should not silently wipe
// a prior AgentResolution.
t.Errorf("guard must reject without mutating status.AgentResolution; got %+v", status.AgentResolution)
}
}

func newResolverTestController(t *testing.T, agents ...*monetizeapi.Agent) *Controller {
t.Helper()
objs := make([]runtime.Object, 0, len(agents))
Expand Down
64 changes: 47 additions & 17 deletions internal/x402/serviceoffer_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"k8s.io/client-go/tools/cache"
)

// WatchServiceOffers runs the ServiceOffer + litellm-secrets informers and
// WatchServiceOffers runs the ServiceOffer + upstream-auth Secret informers and
// pushes rendered RouteRules to apply on every change. The optional
// onFirstApply callback is invoked exactly once after the post-cache-sync
// refresh succeeds; it is the signal that the route source has produced its
Expand All @@ -33,14 +33,20 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
}

offerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, nil)
secretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
litellmSecretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("metadata.name", "litellm-secrets").String()
})
hermesSecretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("metadata.name", "hermes-api-server").String()
})
offers := offerFactory.ForResource(monetizeapi.ServiceOfferGVR).Informer()
secrets := secretFactory.ForResource(monetizeapi.SecretGVR).Informer()
litellmSecrets := litellmSecretFactory.ForResource(monetizeapi.SecretGVR).Informer()
hermesSecrets := hermesSecretFactory.ForResource(monetizeapi.SecretGVR).Informer()

refresh := func() (ok bool) {
routes, err := routesFromStore(offers.GetStore().List(), secrets.GetStore().List())
secretItems := append([]any{}, litellmSecrets.GetStore().List()...)
secretItems = append(secretItems, hermesSecrets.GetStore().List()...)
routes, err := routesFromStore(offers.GetStore().List(), secretItems)
if err != nil {
log.Printf("x402-serviceoffer-source: render routes: %v", err)
return false
Expand All @@ -59,11 +65,13 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
DeleteFunc: func(any) { refresh() },
}
offers.AddEventHandler(handler)
secrets.AddEventHandler(handler)
litellmSecrets.AddEventHandler(handler)
hermesSecrets.AddEventHandler(handler)

go offers.Run(ctx.Done())
go secrets.Run(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), offers.HasSynced, secrets.HasSynced) {
go litellmSecrets.Run(ctx.Done())
go hermesSecrets.Run(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), offers.HasSynced, litellmSecrets.HasSynced, hermesSecrets.HasSynced) {
return fmt.Errorf("wait for serviceoffer informer sync")
}

Expand All @@ -75,7 +83,7 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
}

func routesFromStore(offerItems, secretItems []any) ([]RouteRule, error) {
upstreamAuthByNamespace, err := upstreamAuthByNamespace(secretItems)
litellmAuthByNamespace, hermesAuthByNamespace, err := upstreamAuthByNamespace(secretItems)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -103,7 +111,11 @@ func routesFromStore(offerItems, secretItems []any) ([]RouteRule, error) {
continue
}

rule, err := routeRuleFromOffer(&offer, upstreamAuthByNamespace[offer.EffectiveNamespace()])
upstreamAuth := litellmAuthByNamespace[offer.EffectiveNamespace()]
if offer.IsAgent() {
upstreamAuth = hermesAuthByNamespace[offer.Spec.Agent.Ref.Namespace]
}
rule, err := routeRuleFromOffer(&offer, upstreamAuth)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -194,36 +206,54 @@ func effectivePrice(offer *monetizeapi.ServiceOffer) (price, priceModel, perMTok
}
}

func upstreamAuthByNamespace(items []any) (map[string]string, error) {
result := make(map[string]string)
func upstreamAuthByNamespace(items []any) (map[string]string, map[string]string, error) {
litellmAuth := make(map[string]string)
hermesAuth := make(map[string]string)
for _, item := range items {
obj, ok := item.(*unstructured.Unstructured)
if !ok || obj.GetName() != "litellm-secrets" {
if !ok {
continue
}
dataKey := ""
switch obj.GetName() {
case "litellm-secrets":
dataKey = "LITELLM_MASTER_KEY"
case "hermes-api-server":
dataKey = "API_SERVER_KEY"
default:
continue
}

value, found, err := unstructured.NestedString(obj.Object, "data", "LITELLM_MASTER_KEY")
value, found, err := unstructured.NestedString(obj.Object, "data", dataKey)
if err != nil {
return nil, err
return nil, nil, err
}
if !found || value == "" {
continue
}

decoded, err := base64.StdEncoding.DecodeString(value)
if err != nil {
return nil, err
return nil, nil, err
}
token := strings.TrimSpace(string(decoded))
if token == "" {
continue
}
result[obj.GetNamespace()] = "Bearer " + token
switch obj.GetName() {
case "litellm-secrets":
litellmAuth[obj.GetNamespace()] = "Bearer " + token
case "hermes-api-server":
hermesAuth[obj.GetNamespace()] = "Bearer " + token
}
}
return result, nil
return litellmAuth, hermesAuth, nil
}

func effectiveUpstreamAuth(offer *monetizeapi.ServiceOffer, upstreamAuth string) string {
if offer.IsAgent() {
return upstreamAuth
}
if !strings.EqualFold(offer.Spec.Upstream.Service, "litellm") {
return ""
}
Expand Down
Loading
Loading