Bootstrap cattery agent on runners

docs/configuration.md

@@ -116,17 +116,49 @@ Provider-specific fields:
 #### trayTypes
 Defines one or more tray "profiles" that the Tray Manager can maintain.
 
-| Key                 | Type               | Required | Description                                                                    |
-|---------------------|--------------------|----------|--------------------------------------------------------------------------------|
-| name                | string             | yes      | Unique name for the tray type. Also used as the runner scale set name/label.   |
-| provider            | string             | yes      | Name of a provider defined in `providers`.                                     |
-| runnerGroupId       | int                | yes      | GitHub Runner Group ID to register runners into.                               |
-| githubOrg           | string             | yes      | The GitHub org key, matching one of the entries under `github`.                |
-| shutdown            | bool               | no       | Whether instances should self-terminate when the job completes.                |
-| maxTrays            | int                | no       | Maximum number of concurrent trays of this type.                               |
-| maxParallelCreation | int                | no       | Maximum number of trays to create in parallel. Defaults to 10.                 |
-| extraMetadata       | map[string]string  | no       | Extra key-value metadata passed to the provider (e.g., GCE instance metadata). |
-| config              | provider-dependent | yes      | Provider-specific configuration for how to create a tray (see below).          |
+| Key                 | Type               | Required | Description                                                                                                |
+|---------------------|--------------------|----------|------------------------------------------------------------------------------------------------------------|
+| name                | string             | yes      | Unique name for the tray type. Also used as the runner scale set name/label.                               |
+| provider            | string             | yes      | Name of a provider defined in `providers`.                                                                 |
+| runnerGroupId       | int                | yes      | GitHub Runner Group ID to register runners into.                                                           |
+| githubOrg           | string             | yes      | The GitHub org key, matching one of the entries under `github`.                                            |
+| shutdown            | bool               | no       | Whether instances should self-terminate when the job completes.                                            |
+| maxTrays            | int                | no       | Maximum number of concurrent trays of this type.                                                           |
+| maxParallelCreation | int                | no       | Maximum number of trays to create in parallel. Defaults to 10.                                             |
+| runnerVersion       | string             | no       | Pin the GitHub Actions runner version the agent downloads. Empty -> latest from GH releases.               |
+| bootstrap           | object             | no       | Provider-injected agent bootstrap (see below). Enabled by default; set `bootstrap.enabled: false` to opt out. |
+| extraMetadata       | map[string]string  | no       | Extra key-value metadata passed to the provider (e.g., GCE instance metadata).                             |
+| config              | provider-dependent | yes      | Provider-specific configuration for how to create a tray (see below).                                      |
+
+#### bootstrap
+
+When enabled, the provider injects a script into the spawned tray that
+downloads the cattery agent binary from `<advertiseUrl>/agent/download` and
+starts it. The agent in turn downloads the GitHub Actions runner if it is not
+already present on disk.
+
+This means a fresh VM image only needs the OS plus whatever heavy tooling the
+user wants (Docker, language runtimes, security agents). Cattery handles
+installing itself and the runner.
+
+| Key          | Type   | Required | Description                                                                                                            |
+|--------------|--------|----------|------------------------------------------------------------------------------------------------------------------------|
+| enabled      | bool   | no       | Master switch. Defaults to `true`. Set `false` for legacy pre-baked images that already start the agent themselves.    |
+| os           | string | no       | Selects the built-in script template. Default: `linux`.                                                                |
+| agentFolder  | string | no       | Where to download the cattery binary on the tray. Default: `/opt/cattery`.                                             |
+| runnerFolder | string | no       | Where to install the GH Actions runner. Default: `/opt/cattery/actions-runner`. Passed to the agent as `--runner-folder`. |
+| user         | string | no       | OS user to run the agent as. Default: empty (script runs as whatever user the provider's delivery mechanism uses).     |
+| script       | string | no       | Override the built-in template. Treated as a Go `text/template` with `{{.ServerURL}}`, `{{.AgentID}}`, `{{.AgentFolder}}`, `{{.RunnerFolder}}`, `{{.User}}` available. |
+
+Provider delivery:
+
+- **gce**: script is set as the `startup-script` instance metadata key.
+- **docker**: script is piped to `/bin/sh -s` as the container's entrypoint stdin.
+
+**Migration note**: If you previously relied on a pre-baked image with its own
+systemd unit (e.g. `cattery.service` + `install-agent.sh`) starting the agent,
+add `bootstrap: { enabled: false }` to those tray types after upgrading.
+Otherwise the injected startup script will spawn a second agent.
 
 Provider-specific config under trayType.config:
 

examples/example-config.yaml

@@ -32,28 +32,68 @@ providers:
     credentialsFile: path/to/credentials.json
 
 trayTypes:
-  - name: cattery-tiny
-    provider: docker-local
-    shutdown: false
-    runnerGroupId: 3
-    githubOrg: My-Github-Org
-    config:
-      image: cattery-runner-tiny:latest
+  # Bootstrap is enabled by default for every tray type. The provider injects
+  # a script that downloads the cattery agent from /agent/download and runs it.
+  # The agent itself downloads the GH Actions runner if not present on disk
+  # (latest release by default, or pinned via runnerVersion).
 
-  - name: cattery-gce
+  - name: cattery-gce-default
+    provider: gce
+    githubOrg: My-Github-Org
+    runnerGroupId: 3
+    shutdown: true
+    maxTrays: 3
     config:
       instanceTemplate: global/instanceTemplates/<instance-template>
       machineType: e2-standard-2
       project: my-gcp-project
       zones:
         - us-west1-a
         - us-west1-b
-    extraMetadata:
-      # can be: version (0.0.2), server (to download binary from server) or a commit hash
-      cattery-agent-version: 0.0.4
-      # cattery-agent-version: 13d197aa1e73db09514772e55794b3a0f9b7952b
+
+  - name: cattery-gce-custom
+    provider: gce
     githubOrg: My-Github-Org
-    maxTrays: 3 # max number of VMs of this type
+    runnerGroupId: 3
+    shutdown: true
+    runnerVersion: "2.332.0"          # pin runner version (default: latest)
+    bootstrap:
+      user: cattery                   # run agent as this user (default: root)
+      agentFolder: /home/cattery
+      runnerFolder: /home/cattery/actions-runner
+      # script: |                     # optional template override
+      #   #!/bin/bash
+      #   curl -sSfL {{.ServerURL}}/agent/download -o /tmp/cattery
+      #   ...
+    config:
+      instanceTemplate: global/instanceTemplates/<instance-template>
+      machineType: e2-standard-2
+      project: my-gcp-project
+      zones:
+        - us-west1-a
+
+  # Opt out: legacy pre-baked image where the agent is already installed and
+  # started by your own systemd unit (e.g. the cattery.service chain).
+  - name: cattery-gce-legacy
     provider: gce
-    runnerGroupId: 3 # check in github org settings -> Runner groups
+    githubOrg: My-Github-Org
+    runnerGroupId: 3
     shutdown: true
+    bootstrap:
+      enabled: false
+    extraMetadata:
+      cattery-agent-version: 0.0.4    # consumed by your image's install-agent.sh
+    config:
+      instanceTemplate: global/instanceTemplates/legacy-image
+      machineType: e2-standard-2
+      project: my-gcp-project
+      zones:
+        - us-west1-a
+
+  - name: cattery-tiny
+    provider: docker-local
+    shutdown: false
+    runnerGroupId: 3
+    githubOrg: My-Github-Org
+    config:
+      image: ubuntu:24.04             # plain image -- bootstrap installs the agent

src/agent/agent.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"cattery/agent/catteryClient"
 	"cattery/agent/githubListener"
+	"cattery/agent/runner"
 	"cattery/agent/tools"
 	"cattery/lib/agents"
 	"cattery/lib/messages"
@@ -41,13 +42,15 @@ type CatteryAgent struct {
 	agent         *agents.Agent
 	agentId       string
 
+	runnerFolder     string
 	listenerExecPath string
 }
 
 func NewCatteryAgent(runnerFolder string, catteryServerUrl string, agentId string) *CatteryAgent {
 	return &CatteryAgent{
 		logger:           log.WithFields(log.Fields{"name": "agent", "agentId": agentId}),
 		catteryClient:    catteryClient.NewCatteryClient(catteryServerUrl, agentId),
+		runnerFolder:     runnerFolder,
 		listenerExecPath: path.Join(runnerFolder, "bin", "Runner.Listener"),
 		agentId:          agentId,
 	}
@@ -56,12 +59,20 @@ func NewCatteryAgent(runnerFolder string, catteryServerUrl string, agentId strin
 func (a *CatteryAgent) Start() {
 	a.logger.Info("Starting Cattery Agent")
 
-	agent, jitConfig, err := a.catteryClient.RegisterAgent(a.agentId)
+	resp, err := a.catteryClient.RegisterAgent(a.agentId)
 	if err != nil {
 		a.logger.Errorf("Failed to register agent: %v", err)
 		return
 	}
-	a.agent = agent
+	a.agent = &resp.Agent
+	jitConfig := &resp.JitConfig
+
+	// Ensure the GH Actions runner distribution is present on disk before we
+	// try to launch Runner.Listener. No-op when the runner is pre-baked.
+	if err := runner.EnsureRunner(a.runnerFolder, resp.RunnerVersion); err != nil {
+		a.logger.Errorf("Failed to ensure runner: %v", err)
+		return
+	}
 
 	a.logger.Info("Agent registered, starting Listener")
 

src/agent/catteryClient/client.go

@@ -29,42 +29,43 @@ func NewCatteryClient(baseURL string, agentId string) *CatteryClient {
 	}
 }
 
-// RegisterAgent request just-in-time runner configuration from the Cattery server
-// and returns the configuration as a base64 encoded string
+// RegisterAgent requests just-in-time runner configuration from the Cattery server.
+// Returns the full RegisterResponse so callers can read the JIT config, agent info,
+// and runner version (used for runner bootstrap).
 //
 // https://docs.github.com/en/rest/actions/self-hosted-runners?apiVersion=2022-11-28#create-configuration-for-a-just-in-time-runner-for-an-organization
-func (c *CatteryClient) RegisterAgent(id string) (*agents.Agent, *string, error) {
+func (c *CatteryClient) RegisterAgent(id string) (*messages.RegisterResponse, error) {
 
 	client := c.httpClient
 
 	requestUrl, err := url.JoinPath(c.baseURL, "/agent", "register/", id)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	request, err := http.NewRequest("GET", requestUrl, nil)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create request: %w", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 	response, err := client.Do(request)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
 		bodyBytes, _ := io.ReadAll(response.Body)
-		return nil, nil, fmt.Errorf("response status code: %s body: %s", response.Status, string(bodyBytes))
+		return nil, fmt.Errorf("response status code: %s body: %s", response.Status, string(bodyBytes))
 	}
 
 	registerResponse := &messages.RegisterResponse{}
 	err = json.NewDecoder(response.Body).Decode(registerResponse)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	return &registerResponse.Agent, &registerResponse.JitConfig, nil
+	return registerResponse, nil
 }
 
 // UnregisterAgent sends a POST request to the Cattery server to unregister the agent