feat: supoort kruise custom api for gateway

.github/workflows/e2e-e2b-2.4.1.yaml

@@ -21,12 +21,22 @@ env:
   CONTROLLER_IMG: agent-sandbox-controller:latest
   MANAGER_IMG: sandbox-manager:latest
   RUNTIME_IMG: agent-runtime:latest
+  GATEWAY_IMG: sandbox-gateway:latest
   CODE_INTERPRETER_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6
   INPLACE_UPDATE_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6-new
 
 jobs:
   sandbox:
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: without sandbox-gateway
+            with_gateway: false
+          - name: with sandbox-gateway
+            with_gateway: true
+    name: sandbox (${{ matrix.name }})
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
@@ -91,15 +101,31 @@ jobs:
           docker rmi ${CODE_INTERPRETER_IMG}
           kind load docker-image --name=${KIND_CLUSTER_NAME} ${INPLACE_UPDATE_IMG} || { echo >&2 "kind not installed or error loading image: ${INPLACE_UPDATE_IMG}"; exit 1; }
           docker rmi ${INPLACE_UPDATE_IMG}
+
+      - name: Build sandbox-gateway image
+        if: matrix.with_gateway
+        run: |
+          make docker-build-sandbox-gateway
+          kind load docker-image --name=${KIND_CLUSTER_NAME} ${GATEWAY_IMG} || { echo >&2 "kind not installed or error loading image: ${GATEWAY_IMG}"; exit 1; }
+          docker rmi ${GATEWAY_IMG}
+
       - name: Install Kruise Agents
         run: |
           make deploy-agent-sandbox-controller
           make deploy-sandbox-manager
           bash hack/wait-agent-sandbox-controller.sh
+
+      - name: Deploy sandbox-gateway
+        if: matrix.with_gateway
+        run: make deploy-sandbox-gateway
+
       - name: Run E2E Tests
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           export E2B_DOMAIN=localhost
           export E2B_API_KEY=some-api-key
-          bash hack/run-e2b-e2e-test.sh --e2b-version 2.8.1 --sdk-version 2.4.1
-
+          if [ "${{ matrix.with_gateway }}" = "true" ]; then
+            bash hack/run-e2b-e2e-test.sh --e2b-version 2.8.1 --sdk-version 2.4.1 --with-gateway
+          else
+            bash hack/run-e2b-e2e-test.sh --e2b-version 2.8.1 --sdk-version 2.4.1
+          fi

.github/workflows/e2e-e2b-latest.yaml

@@ -21,12 +21,22 @@ env:
   CONTROLLER_IMG: agent-sandbox-controller:latest
   MANAGER_IMG: sandbox-manager:latest
   RUNTIME_IMG: agent-runtime:latest
+  GATEWAY_IMG: sandbox-gateway:latest
   CODE_INTERPRETER_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6
   INPLACE_UPDATE_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6-new
 
 jobs:
   sandbox:
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: without sandbox-gateway
+            with_gateway: false
+          - name: with sandbox-gateway
+            with_gateway: true
+    name: sandbox (${{ matrix.name }})
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
@@ -92,15 +102,31 @@ jobs:
            docker rmi ${CODE_INTERPRETER_IMG}
            kind load docker-image --name=${KIND_CLUSTER_NAME} ${INPLACE_UPDATE_IMG} || { echo >&2 "kind not installed or error loading image: ${INPLACE_UPDATE_IMG}"; exit 1; }
            docker rmi ${INPLACE_UPDATE_IMG}
+
+      - name: Build sandbox-gateway image
+        if: matrix.with_gateway
+        run: |
+           make docker-build-sandbox-gateway
+           kind load docker-image --name=${KIND_CLUSTER_NAME} ${GATEWAY_IMG} || { echo >&2 "kind not installed or error loading image: ${GATEWAY_IMG}"; exit 1; }
+           docker rmi ${GATEWAY_IMG}
+
       - name: Install Kruise Agents
         run: |
           make deploy-agent-sandbox-controller
           make deploy-sandbox-manager
           bash hack/wait-agent-sandbox-controller.sh
+
+      - name: Deploy sandbox-gateway
+        if: matrix.with_gateway
+        run: make deploy-sandbox-gateway
+
       - name: Run E2E Tests
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           export E2B_DOMAIN=localhost
           export E2B_API_KEY=some-api-key
-          bash hack/run-e2b-e2e-test.sh
-
+          if [ "${{ matrix.with_gateway }}" = "true" ]; then
+            bash hack/run-e2b-e2e-test.sh --with-gateway
+          else
+            bash hack/run-e2b-e2e-test.sh
+          fi

.github/workflows/e2e-e2b-mysql-latest.yaml

@@ -21,6 +21,7 @@ env:
   CONTROLLER_IMG: agent-sandbox-controller:latest
   MANAGER_IMG: sandbox-manager:latest
   RUNTIME_IMG: agent-runtime:latest
+  GATEWAY_IMG: sandbox-gateway:latest
   CODE_INTERPRETER_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6
   INPLACE_UPDATE_IMG: registry-ap-southeast-1.ack.aliyuncs.com/acs/code-interpreter:v1.6-new
   MYSQL_DATABASE: e2b_keys
@@ -29,6 +30,15 @@ env:
 jobs:
   sandbox:
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: without sandbox-gateway
+            with_gateway: false
+          - name: with sandbox-gateway
+            with_gateway: true
+    name: sandbox (${{ matrix.name }})
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
@@ -96,6 +106,13 @@ jobs:
            kind load docker-image --name=${KIND_CLUSTER_NAME} ${INPLACE_UPDATE_IMG} || { echo >&2 "kind not installed or error loading image: ${INPLACE_UPDATE_IMG}"; exit 1; }
            docker rmi ${INPLACE_UPDATE_IMG}
 
+      - name: Build sandbox-gateway image
+        if: matrix.with_gateway
+        run: |
+           make docker-build-sandbox-gateway
+           kind load docker-image --name=${KIND_CLUSTER_NAME} ${GATEWAY_IMG} || { echo >&2 "kind not installed or error loading image: ${GATEWAY_IMG}"; exit 1; }
+           docker rmi ${GATEWAY_IMG}
+
       - name: Deploy MySQL in Kind cluster
         env:
           MYSQL_DATABASE: ${{ env.MYSQL_DATABASE }}
@@ -127,9 +144,17 @@ jobs:
           make deploy-sandbox-manager
           bash hack/wait-agent-sandbox-controller.sh
 
+      - name: Deploy sandbox-gateway
+        if: matrix.with_gateway
+        run: make deploy-sandbox-gateway
+
       - name: Run E2E Tests
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           export E2B_DOMAIN=localhost
           export E2B_API_KEY=some-api-key
-          bash hack/run-e2b-e2e-test.sh
+          if [ "${{ matrix.with_gateway }}" = "true" ]; then
+            bash hack/run-e2b-e2e-test.sh --with-gateway
+          else
+            bash hack/run-e2b-e2e-test.sh
+          fi

Makefile

@@ -162,10 +162,18 @@ deploy-sandbox-manager: kustomize
 deploy-agent-sandbox-controller: kustomize
 	$(KUSTOMIZE) build config/default/ | kubectl apply -f -
 
+.PHONY: deploy-sandbox-gateway
+deploy-sandbox-gateway: kustomize
+	$(KUSTOMIZE) build config/sandbox-gateway/ | kubectl apply -f -
+
 .PHONY: undeploy-sandbox-manager
 undeploy-sandbox-manager: kustomize
 	$(KUSTOMIZE) build config/sandbox-manager/ | kubectl delete -f -
 
+.PHONY: undeploy-sandbox-gateway
+undeploy-sandbox-gateway: kustomize
+	$(KUSTOMIZE) build config/sandbox-gateway/ | kubectl delete -f -
+
 .PHONY: undeploy-agent-sandbox-controller
 undeploy-agent-sandbox-controller: kustomize
 	$(KUSTOMIZE) build config/undeploy/ | kubectl delete -f -

config/sandbox-gateway/configmap.yaml

@@ -19,29 +19,52 @@ data:
           address:
             socket_address:
               address: 0.0.0.0
-              port_value: 10000
+              port_value: 7788
           filter_chains:
             - filters:
                 - name: envoy.filters.network.http_connection_manager
                   typed_config:
                     "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                     stat_prefix: ingress_http
+                    stream_idle_timeout: 600s 
+                    upgrade_configs:
+                      - upgrade_type: websocket
                     access_log:
-                      - name: envoy.access_loggers.stdout
+                      - name: envoy.access_loggers.file
                         typed_config:
-                          "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
-                          log_format:
-                            text_format: "[%START_TIME%] %REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL% %RESPONSE_CODE% %BYTES_SENT% %DURATION%ms req_id=%REQ(X-REQUEST-ID)%\n"
+                          "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+                          path: "/dev/stdout"
+                          json_format:
+                            start_time: "%START_TIME%"
+                            method: "%REQ(:METHOD)%"
+                            path: "%REQ(:PATH)%"
+                            protocol: "%PROTOCOL%"
+                            response_code: "%RESPONSE_CODE%"
+                            response_flags: "%RESPONSE_FLAGS%"
+                            bytes_received: "%BYTES_RECEIVED%"
+                            bytes_sent: "%BYTES_SENT%"
+                            duration_ms: "%DURATION%"
+                            upstream_host: "%UPSTREAM_HOST%"
+                            downstream_remote_address: "%DOWNSTREAM_REMOTE_ADDRESS%"
+                            user_agent: "%REQ(USER-AGENT)%"
+                            request_id: "%REQ(X-REQUEST-ID)%"
+                            authority: "%REQ(:AUTHORITY)%"
+                            upstream_local_address: "%UPSTREAM_LOCAL_ADDRESS%"
                     route_config:
                       name: local_route
                       virtual_hosts:
                         - name: backend
                           domains: ["*"]
                           routes:
+                            - match:
+                                prefix: "/kruise/api"
+                              route:
+                                cluster: manager_cluster
                             - match:
                                 prefix: "/"
                               route:
                                 cluster: original_dst_cluster
+                                timeout: 600s
                     http_filters:
                       - name: envoy.filters.http.golang
                         typed_config:
@@ -60,10 +83,39 @@ data:
                         typed_config:
                           "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
 
+        - name: prometheus_listener
+          address:
+            socket_address:
+              address: 0.0.0.0
+              port_value: 9090
+          filter_chains:
+            - filters:
+                - name: envoy.filters.network.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                    stat_prefix: prometheus
+                    route_config:
+                      name: prometheus_route
+                      virtual_hosts:
+                        - name: prometheus
+                          domains: ["*"]
+                          routes:
+                            - match:
+                                prefix: "/stats/prometheus"
+                              route:
+                                cluster: envoy_admin
+                                prefix_rewrite: "/stats/prometheus"
+                    http_filters:
+                      - name: envoy.filters.http.router
+                        typed_config:
+                          "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
       clusters:
         - name: original_dst_cluster
           type: ORIGINAL_DST
           lb_policy: CLUSTER_PROVIDED
+          common_http_protocol_options:
+            max_requests_per_connection: 1
           connect_timeout: 5s
           original_dst_lb_config:
             metadata_key:
@@ -77,3 +129,29 @@ data:
               max_pending_requests: 32768
               max_requests: 65536
               max_retries: 5
+        - name: manager_cluster
+          type: STRICT_DNS
+          lb_policy: ROUND_ROBIN
+          connect_timeout: 5s
+          load_assignment:
+            cluster_name: manager_cluster
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: sandbox-manager.sandbox-system.svc.cluster.local
+                          port_value: 7788
+        - name: envoy_admin
+          type: STATIC
+          lb_policy: ROUND_ROBIN
+          connect_timeout: 5s
+          load_assignment:
+            cluster_name: envoy_admin
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: 127.0.0.1
+                          port_value: 9901

config/sandbox-gateway/deployment.yaml

@@ -25,36 +25,6 @@ spec:
         component: sandbox-manager
     spec:
       serviceAccountName: sandbox-gateway
-      initContainers:
-        - name: init-sysctl
-          image: "registry.cn-hangzhou.aliyuncs.com/acs/busybox:v1.29.2"
-          imagePullPolicy: IfNotPresent
-          command:
-            - /bin/sh
-            - -c
-            - |-
-              if [ "$POD_IP" != "$HOST_IP" ]; then
-              mount -o remount rw /proc/sys
-              sysctl -w net.core.somaxconn=65535
-              sysctl -w net.ipv4.ip_local_port_range="1024 65535"
-              sysctl -w net.ipv4.tcp_tw_reuse=1
-              sysctl -w fs.file-max=1048576
-              fi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-              add:
-                - SYS_ADMIN
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: HOST_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.hostIP
       containers:
         - name: sandbox-gateway
           image: "sandbox-gateway:latest"
@@ -68,6 +38,17 @@ spec:
             - warn
             - --concurrency
             - "2"
+            - --drain-time-s
+            - "30"
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - sh
+                - -c
+                - |
+                  curl -s -X POST http://localhost:9901/drain_listeners?graceful
+                  sleep 30
           env:
             - name: GOMAXPROCS
               value: "2"
@@ -77,18 +58,21 @@ spec:
               value: "component=sandbox-manager"
           ports:
             - name: http
-              containerPort: 10000
+              containerPort: 7788
+              protocol: TCP
+            - name: metrics
+              containerPort: 9090
               protocol: TCP
           livenessProbe:
             tcpSocket:
-              port: 10000
+              port: 7788
             initialDelaySeconds: 10
             periodSeconds: 10
             timeoutSeconds: 5
             failureThreshold: 3
           readinessProbe:
             tcpSocket:
-              port: 10000
+              port: 7788
             initialDelaySeconds: 5
             periodSeconds: 5
             timeoutSeconds: 3