From 9b47913165af56c48d18515f81b771ef6b410feb Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 28 Aug 2026 15:31:47 +0200 Subject: [PATCH] docs: add device provisioning documentation Document the platform-agnostic provisioning contract in docs/concepts/provisioning.md: the Device resource, lifecycle phases, HTTP endpoints, source validation, the ProvisioningProvider hook interface, and password handling. Document Cisco NX-OS specifics (POAP, install all, password hash formats) in docs/concepts/ztp-nxos.md, with the reference POAP boot script at hack/ztp/nxos.py. Signed-off-by: Sebastian Wagner --- docs/.vitepress/config.mts | 2 + docs/concepts/index.md | 2 + docs/concepts/provisioning.md | 212 ++++++++++++++++ docs/concepts/ztp-nxos.md | 37 +++ hack/ztp/nxos.py | 453 ++++++++++++++++++++++++++++++++++ 5 files changed, 706 insertions(+) create mode 100644 docs/concepts/provisioning.md create mode 100644 docs/concepts/ztp-nxos.md create mode 100644 hack/ztp/nxos.py diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index b92092f59..7d7ed60ba 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -82,6 +82,8 @@ export default withMermaid({ { text: 'Config Backups', link: '/concepts/config-backup' }, { text: 'Pausing Reconciliation', link: '/concepts/pausing' }, { text: 'Numbered Resources', link: '/concepts/numbered-resources' }, + { text: 'Device Provisioning', link: '/concepts/provisioning' }, + { text: 'ZTP for Cisco NX-OS', link: '/concepts/ztp-nxos' }, ], }, { diff --git a/docs/concepts/index.md b/docs/concepts/index.md index 9ce6bae9d..a798cfb12 100644 --- a/docs/concepts/index.md +++ b/docs/concepts/index.md @@ -5,3 +5,5 @@ This section covers the core concepts behind the Network Operator. - [Pausing Reconciliation](./pausing.md) — Temporarily prevent controllers from reconciling resources. - [Interface Neighbor Validation](./cabling.md) — Validate physical cabling via LLDP. - [Numbered Resource Allocation](./numbered-resources.md) — Allocate indices, IP addresses, and IP prefixes from managed pools using Claims. +- [Device Provisioning](./provisioning.md): Bootstrap a device on first boot through the platform-agnostic provisioning contract. +- [Zero-Touch Provisioning for Cisco NX-OS](./ztp-nxos.md): Cisco NX-OS specifics (POAP) for device provisioning. diff --git a/docs/concepts/provisioning.md b/docs/concepts/provisioning.md new file mode 100644 index 000000000..9062d7fa6 --- /dev/null +++ b/docs/concepts/provisioning.md @@ -0,0 +1,212 @@ +# Device Provisioning + +The network operator can bootstrap a device automatically on first boot, from initial network configuration through OS image upgrade and final handoff to the operator. This page describes the platform-agnostic provisioning contract: the `Device` resource, the lifecycle phases, the HTTP endpoints a device talks to, and the provider hook interface a new platform must implement. + +Platform-specific mechanics (how the device obtains and runs the boot script, which on-device commands perform the upgrade, which password hash formats are supported) live on the per-platform pages. For Cisco NX-OS see [Zero-Touch Provisioning for Cisco NX-OS](ztp-nxos.md). + +## Overview + +Provisioning is driven by a small boot script that runs on the device early in its boot sequence, before it has any operator-managed configuration. The script speaks HTTP(S) to the operator's provisioning server: it identifies the device by serial number, receives an image URL, credentials and a token, then downloads and installs the image and reports progress back. Once the device reboots onto the target image and becomes reachable, the operator takes over and reconciles the device's configuration resources. + +Because any platform can supply its own boot script and its own provider implementation, the operator side of this contract stays the same across vendors. What differs per platform is packaged behind the provider hook interface described below. + +The design rests on an explicit assumption: provisioning works on any vendor's device that can (a) run a boot script early in its boot sequence which is able to configure the device, and (b) reach the operator over HTTP(S). Platforms that meet these two requirements can plug in to the operator-side contract. + +Provisioning is currently implemented only by the Cisco NX-OS provider (see [Zero-Touch Provisioning for Cisco NX-OS](ztp-nxos.md)). The contract described here is the interface a future platform provider would implement. + +The overall flow, independent of platform, looks like this: + +```mermaid +sequenceDiagram + participant SW as Device + participant DHCP as DHCP Server + participant OP as Network Operator + participant IMG as Image Server + + SW->>DHCP: Request network settings + DHCP-->>SW: IP, DNS, NTP, boot-script location + Note over SW: Fetch and run boot script + SW->>OP: GET /provisioning/config?serial= + OP-->>SW: image URL, checksum, credentials, token + SW->>IMG: Download image + IMG-->>SW: image + SW->>OP: PUT /provisioning/status-report (DownloadingImage, UpgradeStarting, RebootingDevice) + Note over SW: Install image and reboot + SW->>OP: Reachable on new image + OP-->>SW: Reconcile configuration +``` + +The DHCP server must hand out, at minimum, a management IP address, DNS and NTP servers, and a pointer to where the boot script can be fetched (for example a TFTP path). The exact option used to convey the boot-script location and the protocol used to fetch it are platform specific. + +## Device resource + +A `Device` opts into provisioning by setting `spec.provisioning`: + +```yaml +apiVersion: core.network-operator.io/v1alpha1 +kind: Device +metadata: + name: spine-01 + namespace: fabric + labels: + networking.metal.ironcore.dev/device-serial: "9vt9ohzbc3h" +spec: + endpoint: + address: "192.0.2.10:830" + secretRef: + name: spine-01-credentials + provisioning: + image: + url: "http://image-server.example.com/image.bin" + checksum: "d41d8cd98f00b204e9800998ecf8427e" + checksumType: MD5 + bootScript: + configMapRef: + name: boot-script + key: script +``` + +The `provisioning.image` section tells the operator which image the device should run. When the boot script contacts the operator, this is what gets returned in the config response. + +The `provisioning.bootScript` holds the boot script the device runs, supplied inline or from a referenced Secret or ConfigMap. The operator ships an optional built-in TFTP server (beta) that serves this script directly: the device requests a filename encoding its serial (`serial-` or `.`), the server resolves the matching `Device`, reads `spec.provisioning.bootScript`, and returns its contents as-is. With source validation enabled it also checks that the client IP matches the device endpoint and the serial matches `status.serialNumber`. If you already run your own TFTP infrastructure, delivery can stay external to the operator instead. + +If `spec.provisioning` is omitted, the device skips straight from `Pending` to `Running`. + +## Lifecycle phases + +```mermaid +stateDiagram-v2 + [*] --> Pending + Pending --> Provisioning : device contacts operator + Provisioning --> Provisioned : upgrade complete, rebooting + Provisioned --> Running : device reachable on new image + Provisioning --> Failed : download or install error + Provisioned --> Failed : reboot timeout / unreachable + Running --> [*] + + Pending --> Running : no provisioning spec +``` + +| Phase | Description | +|-------|-------------| +| `Pending` | Device resource created, waiting for the device to contact the operator | +| `Provisioning` | Boot sequence in progress (image download, upgrade, reboot) | +| `Provisioned` | Upgrade complete, device rebooting into new image | +| `Running` | Device reachable on new image; operator is applying configuration | +| `Failed` | Provisioning failed; see `status.provisioning[].error` | + +The `status.provisioning` list records each provisioning attempt with its start time, reboot time, and any error message. + +## HTTP provisioning API + +The operator exposes an HTTP provisioning server with four endpoints. Every call carries a `serial` query parameter identifying the device (see [Device identification](#device-identification) below). + +### Device identification + +Provisioning is keyed on the device serial number. Every provisioning request supplies the serial, and the operator resolves it to a `Device` by matching it against the `networking.metal.ironcore.dev/device-serial` **label**. This means: + +- The serial must be present as the `networking.metal.ironcore.dev/device-serial` label on the `Device`; a `Device` without it cannot be matched to an incoming provisioning request. +- The serial must be **unique** across the cluster. If two `Device` objects carry the same serial label the request is rejected, since the operator cannot tell which one to provision. +- The label is set automatically by the device controller from `status.serialNumber` once the operator has observed the device. For a first-boot device that the operator has never reached, set the `networking.metal.ironcore.dev/device-serial` label on the `Device` up front so the initial provisioning request can be resolved. + +### Source validation + +Source validation is optional and off by default. It is enabled per server via controller-manager flags: + +- `--provisioning-http-validate-source-ip` for the HTTP server. +- `--tftp-validate-source` for the built-in TFTP server. + +When enabled, the operator additionally verifies that a request genuinely originates from the device's known management address: + +- HTTP endpoints: the request's client IP must match the host portion of the device's `spec.endpoint.address`. A mismatch is rejected with `403`. +- Built-in TFTP server: the client IP must match the device endpoint IP, and the serial encoded in the requested filename must match `status.serialNumber`. + +Enable it only once devices reach the operator from their known management addresses; otherwise legitimate first-boot requests behind NAT or on a different source address will be rejected. + +### `GET /provisioning/config` + +Called first by the boot script. The operator looks up the device by serial, mints a provisioning token on the first call, reads the admin credential from the device's endpoint Secret, and responds with: + +```json +{ + "provisioningToken": "", + "image": { + "url": "http://image-server.example.com/image.bin", + "checksum": "d41d8cd98f00b204e9800998ecf8427e", + "checksumType": "MD5" + }, + "userAccounts": [ + { + "username": "admin", + "hashedPassword": "", + "hashAlgorithm": "" + } + ], + "hostname": "" +} +``` + +The `hostname` is the `Device` resource name. The `provisioningToken` is used by the device for all subsequent status-report calls so the operator can authenticate progress updates. See [Password handling](#password-handling) for how `hashedPassword` and `hashAlgorithm` are produced. + +### `PUT /provisioning/status-report` + +The device reports progress. Authenticated with the token from the config response: + +``` +PUT /provisioning/status-report?serial= +Authorization: Bearer +``` + +The operator understands the following status values: + +| Status | Meaning | +|--------|---------| +| `DownloadingImage` | Image download in progress | +| `UpgradeStarting` | Checksum verified, install command running | +| `RebootingDevice` | Install complete, rebooting | +| `ImageDownloadFailed` | Download or checksum failure | +| `UpgradeFailed` | Install command failed | + +### `GET /provisioning/mtls-client-ca` and `GET /provisioning/device-certificate` + +Both are optional. They let the device fetch the operator's mTLS client CA and a per-device certificate so the operator can later connect over an authenticated channel. A device that does not need certificates can ignore these; the operator returns `404` when there is nothing to hand out, which the boot script treats as "skip". + +## Provisioning provider interface + +A platform plugs into provisioning by implementing `ProvisioningProvider` (`internal/provider/provider.go`): + +```go +type ProvisioningProvider interface { + Reprovision(context.Context, *deviceutil.Connection) error + HashProvisioningPassword(password string) (hash string, algorithm string, err error) + VerifyProvisioned(context.Context, *deviceutil.Connection, *v1alpha1.Device) bool +} +``` + +- `Reprovision` resets the device and re-enables its provisioning mechanism so it can run through the sequence again. +- `HashProvisioningPassword` turns a plaintext password into a device-native hash plus an algorithm label (see below). +- `VerifyProvisioned` checks whether the device has finished provisioning and is running the expected image. + +## Password handling + +The operator never sends a plaintext password to the device over the provisioning channel. That channel may be unauthenticated or otherwise not fully trusted (the device has not yet been secured, and the operator does not verify the device's identity at this stage), so shipping a cleartext credential over it would be unsafe. + +Instead the operator reads the admin credential from the device's endpoint Secret and passes the plaintext to the provider's `HashProvisioningPassword` hook. The hook returns two things: a hash in the device's native on-box format, and an algorithm label identifying which hash format it is. + +Both are placed in the config response as `userAccounts[].hashedPassword` and `userAccounts[].hashAlgorithm`. The boot script maps the algorithm label to the platform's corresponding on-device password type and configures the account with the pre-hashed value; the plaintext never leaves the operator. + +## Observing provisioning progress + +```bash +# Watch the device phase +kubectl get device spine-01 -w + +# Full status including provisioning history +kubectl get device spine-01 -o yaml | yq .status + +# Conditions only +kubectl get device spine-01 -o yaml | yq .status.conditions + +# Events +kubectl describe device spine-01 +``` diff --git a/docs/concepts/ztp-nxos.md b/docs/concepts/ztp-nxos.md new file mode 100644 index 000000000..69ec78555 --- /dev/null +++ b/docs/concepts/ztp-nxos.md @@ -0,0 +1,37 @@ +# Zero-Touch Provisioning for Cisco NX-OS + +Zero-Touch Provisioning (ZTP), referred to as POAP (Power-On Auto Provisioning) on Cisco NX-OS, lets a switch bootstrap itself automatically on first boot, without manual intervention. + +This page documents only the Cisco NX-OS specifics. For the platform-agnostic model (the end-to-end flow, DHCP, the `Device` resource, lifecycle phases, HTTP endpoints, source validation, and the provider hook interface) see [Device Provisioning](provisioning.md). + +## Boot script + +The network operator ships its own POAP boot script at [`hack/ztp/nxos.py`](https://github.com/ironcore-dev/network-operator/blob/main/hack/ztp/nxos.py). It is not Cisco's reference POAP script; it talks to the operator's provisioning API and uses `install all` (see below). Site-specific values like the provisioning server URL have been replaced with placeholders and must be adapted to your environment before use. + +A switch enters POAP when it boots with no startup configuration (`write erase` + `reload`). NX-OS expects the boot-script location in DHCP option 67 (`bootfile-name`) and fetches the script over TFTP. The script reads the switch serial from `show version` (`proc_board_id`) and uses it as the `serial` parameter for the operator handshake. + +## Image download and upgrade + +The boot script downloads the NX-OS image from the URL provided by the operator and verifies the checksum. Cisco's reference POAP script uses `boot nxos`, which is no longer recommended in current Cisco documentation. The network operator uses `install all` instead: + +``` +install all nxos +``` + +`install all` runs compatibility checks and also updates BIOS firmware when the new image contains a newer version. Since NX-OS 10.5(3) the EPLD firmware is bundled into the `.bin`, so `install all` upgrades it too when required (with an exception for switches affected by the Secure Boot vulnerability, which need a manual `install epld`). + +Driving `install all` from within POAP has some quirks the operator's boot script works around: + +- `install all` refuses to run while `boot poap enable` is set. The script disables POAP (`no boot poap enable`) and saves the running config before invoking it. +- The script runs `install all nxos no-reload` so it can stage the device configuration before the reboot, rather than letting `install all` reboot immediately. + +## Applying configuration across the reboot + +POAP applies configuration through the reboot rather than on the running system, which has its own quirks the operator's boot script works around: + +- Configuration meant to apply after the upgrade is staged via NX-OS `scheduled-config`. Writing `scheduled-config` directly does not reliably survive the POAP reboot; the script instead writes the config to a file on `bootflash:` and copies it into `scheduled-config`, which is the only approach found to work consistently. +- POAP completing without a reboot (via script exit codes, as Cisco's [reference POAP script](https://github.com/CiscoSE/Cisco-POAP/blob/master/poap.py#L3-L9) suggests) could not be made to work and is not documented by Cisco, so the operator's boot script always relies on the reboot to apply the staged configuration. + +## Password hash formats + +The operator never sends a plaintext admin password to the switch (see [Password handling](provisioning.md#password-handling) for why). The NX-OS provider's `HashProvisioningPassword` hook hashes the password with scrypt (NX-OS type 9) by default, using a 10-byte zero-free random salt and Cisco's custom base64 alphabet, and returns the algorithm label `scrypt`. diff --git a/hack/ztp/nxos.py b/hack/ztp/nxos.py new file mode 100644 index 000000000..780052ded --- /dev/null +++ b/hack/ztp/nxos.py @@ -0,0 +1,453 @@ +#!/isan/bin/python +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +# SPDX-License-Identifier: Apache-2.0 + +# +# Example POAP (Power-On Auto Provisioning) boot script for Cisco NX-OS. +# +# This script is delivered to the switch via TFTP during POAP (see +# docs/concepts/ztp-nxos.md). It contacts the network operator's +# provisioning endpoint, downloads and verifies the NX-OS image, installs +# certificates, applies base configuration and reboots into the new image. +# +# It is provided as a reference only. Site-specific values (provisioning +# server URLs, management subnets, credentials) have been replaced with +# placeholders and must be adapted to your environment before use. +import base64 +import os +import glob +import logging +import logging.handlers +from typing import List +import requests +import signal +import secrets +import time +import traceback +from datetime import datetime + +from cli import cli, clid, json +from cisco import vrf + +# Network operator provisioning endpoint. Replace with your own URL. +PROVISIONING_SERVER = "https://network-operator.example.com" + +VERIFY_TLS = True +LOG_CONFIG = { + "level": logging.INFO, + "format": "%(asctime)s: %(name)s %(levelname)s - %(message)s", + "file": True, +} + +LOCAL_IMAGE_DIR = "bootflash:///" + +STATIC_CONFIG = [ + "feature grpc", + "grpc port 9339", + "feature nxapi", + "nxapi https port 443", + "nxapi ssl protocols TLSv1.3", + "feature bash-shell", + "ssh key ecdsa 256 force", + "no password strength-check", + # Replace with your own hashed admin password. + "username admin password 5 role network-admin", + "hardware access-list tcam region ing-racl 1792", + "hardware access-list tcam region ing-flow-redirect 512", +] + +CLIENT_CA_CONFIG = [ + "grpc client root certificate network-operator", +] + +DEVICE_CERT_CONFIG = [ + "grpc certificate device", + "nxapi certificate trustpoint device", +] + +SCRIPT_EXECUTION_STARTED = "ScriptExecutionStarted" +SCRIPT_EXECUTION_FAILED = "ScriptExecutionFailed" +INSTALLING_CERTIFICATES = "InstallingCertificates" +DOWNLOADING_IMAGE = "DownloadingImage" +IMAGE_DOWNLOAD_FAILED = "ImageDownloadFailed" +UPGRADE_STARTING = "UpgradeStarting" +UPGRADE_FAILED = "UpgradeFailed" +REBOOTING_DEVICE = "RebootingDevice" +EXECUTION_FINISHED_WITHOUT_REBOOT = "ExecutionFinishedWithoutReboot" + +ALL_STATUSES = [ + SCRIPT_EXECUTION_STARTED, SCRIPT_EXECUTION_FAILED, INSTALLING_CERTIFICATES, DOWNLOADING_IMAGE, + IMAGE_DOWNLOAD_FAILED, UPGRADE_STARTING, UPGRADE_FAILED, REBOOTING_DEVICE, EXECUTION_FINISHED_WITHOUT_REBOOT +] + +# These global variables will be initialized once the script starts. +# They are needed in a lot of places. +TOKEN = None +SERIAL = None + +vrf.set_global_vrf("management") +LOG = logging.getLogger("poap-nxos") +SCHEDEDULED_CONFIG = "" + + +def setup_logging(): + streamHandler = logging.StreamHandler() + formatter = logging.Formatter(LOG_CONFIG["format"]) + streamHandler.setFormatter(formatter) + LOG.addHandler(streamHandler) + syslogHandler = logging.handlers.SysLogHandler(address="/dev/log") + syslogHandler.setFormatter(formatter) + LOG.addHandler(syslogHandler) + if LOG_CONFIG["file"]: + iso_date = datetime.now().isoformat(timespec="seconds").replace(":", "-") + fileHandler = logging.FileHandler(f"/bootflash/poap-{iso_date}.log") + fileHandler.setFormatter(formatter) + LOG.addHandler(fileHandler) + LOG.setLevel(LOG_CONFIG["level"]) + + +def cli_json(command: str): + return json.loads(clid(command)) + + +def os_command(command: str): + """Execute an OS (linux) command and return its output.""" + result = os.popen(command).read() + return result + + +def configure(config_lines: list, running=False, scheduled=True): + """Configure the device with the provided configuration lines.""" + config_lines = [line for line in config_lines if line] + if running: + conf_string = "configure terminal ; " + " ; ".join(config_lines) + " ; end" + cli(conf_string) + if scheduled: + global SCHEDEDULED_CONFIG + SCHEDEDULED_CONFIG += "\n".join(config_lines) + "\n" + + +def to_posix_path(path: str) -> str: + """Convert a Cisco-style path to a POSIX-style path.""" + ppath = [] + tokens = path.split("/") + if tokens[0] in ["bootflash:", "flash:", "nvram:", "usb2:", "usb1:", "volatile:"]: + ppath.append('/' + tokens[0].replace(":", "")) + for token in tokens[1:]: + if token: + ppath.append(token) + return "/".join(ppath) + else: + raise ValueError(f"Unknown path type: {path}") + + +def get_provisioning_data() -> dict: + url = f"{PROVISIONING_SERVER}/provisioning/config" + response = requests.get(url, params={"serial": SERIAL}, timeout=10, verify=VERIFY_TLS) + response.raise_for_status() + return response.json() + + +def report_status(status: str, message: str = ""): + if status not in ALL_STATUSES: + raise ValueError(f"Cannot report status: {status}") + payload = { + "status": status, + } + if message: + payload["detail"] = ';'.join(message.split('\n')) + try: + headers = {"Authorization": f"Bearer {TOKEN}"} + response = requests.put(PROVISIONING_SERVER + "/provisioning/status-report", + json=payload, + params={"serial": SERIAL}, + headers=headers, + timeout=5, + verify=VERIFY_TLS) + response.raise_for_status() + except requests.RequestException as e: + logging.error(f"Failed to report status: {e}") + + +def get_client_ca() -> str: + headers = {"Authorization": f"Bearer {TOKEN}"} + response = requests.get(PROVISIONING_SERVER + "/provisioning/mtls-client-ca", + headers=headers, params={"serial": SERIAL}, + timeout=5, verify=VERIFY_TLS) + if response.status_code == 404: + return None + response.raise_for_status() + return response.text + + +def configure_ca_certificates(ca_data: str = None): + crt_path = 'volatile:///network-operator-ca.crt' + p7_path = 'bootflash:///network-operator-ca.p7' + if os.path.exists(to_posix_path(p7_path)): + os.remove(to_posix_path(p7_path)) + if os.path.exists(to_posix_path(crt_path)): + os.remove(to_posix_path(crt_path)) + with open(to_posix_path(crt_path), 'w') as f: + f.write(ca_data) + os_command(f"openssl crl2pkcs7 -nocrl -certfile {to_posix_path(crt_path)} -out {to_posix_path(p7_path)}") + configure([ + 'crypto ca trustpoint network-operator', + 'crypto ca import network-operator pkcs7 {} force'.format(p7_path), + 'crypto ca cabundle network-operator',]) + + +def get_device_certificates(): + headers = {"Authorization": f"Bearer {TOKEN}"} + response = requests.get(PROVISIONING_SERVER + "/provisioning/device-certificate", + headers=headers, + params={"serial": SERIAL}, + timeout=5, + verify=VERIFY_TLS) + if response.status_code == 404: + return None + return response.json() + + +def configure_device_certitificates(cert_data: dict = None): + key_path = 'volatile:///device-tls.key' + crt_path = 'volatile:///device-tls.crt' + ca_path = 'volatile:///device-ca.crt' + p12_path = 'bootflash:///device-cert.p12' + for path in [key_path, crt_path, ca_path, p12_path]: + if os.path.exists(to_posix_path(path)): + os.remove(to_posix_path(path)) + with open(to_posix_path(key_path), 'w') as f: + f.write(cert_data["privateKey"]) + with open(to_posix_path(crt_path), 'w') as f: + f.write(cert_data["certificate"]) + with open(to_posix_path(ca_path), 'w') as f: + f.write(cert_data["caCertificate"]) + password = secrets.token_hex(16) + os_command(f"openssl pkcs12 -export -in {to_posix_path(crt_path)} -inkey {to_posix_path(key_path)} " + f"-certfile {to_posix_path(ca_path)} -out {to_posix_path(p12_path)} " + "-passout pass:'{}'".format(password)) + os.remove(to_posix_path(key_path)) + os.remove(to_posix_path(crt_path)) + os.remove(to_posix_path(ca_path)) + configure(['crypto ca trustpoint device', + 'crypto ca import device pkcs12 {} {}'.format(p12_path, password)]) + + +def verify_checksum(checksum: str, checksum_type: str, path: str = None) -> bool: + if checksum_type not in ["MD5", "SHA256", "SHA512"]: + raise ValueError(f"Unsupported checksum type: {checksum_type}") + nxos_checksum_type = checksum_type.lower() + "sum" + + if not path: + path = cli_json('show version')['nxos_file_name'] + + test_checksum = cli(f"show file {path} {nxos_checksum_type}").strip() + return test_checksum == checksum + + +def clean_old_images(): + s = os.statvfs("/bootflash/") + before = s.f_bavail * s.f_frsize + + bin_files = glob.glob("/bootflash/*.bin", recursive=False) + LOG.info(f"Found .bin files: {bin_files}") + + booted_image = cli_json('show version')['nxos_file_name'] + booted_image_path = to_posix_path(booted_image) + for bin_file in bin_files: + if bin_file == booted_image_path: + LOG.info(f"Skipping booted image file: {bin_file}") + continue + try: + LOG.info(f"Removing file: {bin_file}") + os.remove(bin_file) + except OSError as e: + LOG.error(f"Failed to remove file {bin_file}: {e}") + + s = os.statvfs("/bootflash/") + after = s.f_bavail * s.f_frsize + LOG.info(f"Cleaned old images, freed {(before - after)/1024**2} MB of space.") + + +def download_image(image_url: str, target_path: str): + LOG.info(f"Downloading image from {image_url} to {target_path}") + response = requests.get(image_url, timeout=60, stream=True, verify=VERIFY_TLS) + response.raise_for_status() + written = 0 + last_written = time.time() + with open(to_posix_path(target_path), "wb") as f: + for data in response.iter_content(chunk_size=50 * 1024**2): + f.write(data) + elapsed = time.time() - last_written + chunk_speed = len(data) / elapsed / (1024 * 1024) + written += len(data) + last_written = time.time() + msg = f"Downloaded {written / (1024 * 1024):.2f} MB (avg speed: {chunk_speed:.1f} MB/s)" + LOG.info(msg) + report_status(DOWNLOADING_IMAGE, msg) + + +def set_firmware(image_path: str = None): + if not image_path: + image_path = cli_json('show version')['nxos_file_name'] + LOG.info('Setting boot image to: ' + image_path) + configure([f'boot nxos {image_path}'], running=True) + cli('copy running-config startup-config') + + +def upgrade_firmware(target_path: str): + report_status(UPGRADE_STARTING, "Starting firmware upgrade.") + # Use `install all` rather than just `boot nxos ...`: it runs compatibility + # checks and, since NX-OS 10.5(3), upgrades the bundled BIOS and EPLD images + # when required. It refuses to run while `boot poap enable` is set, so POAP + # must be disabled around the command. + # Note: on switches affected by the Secure Boot vulnerability + # (cisco-sa-20190513-secureboot), `install all` skips the EPLD upgrade from + # 10.6(1)F onward; those require a manual `install epld` post-provisioning. + configure(['no boot poap enable'], running=True, scheduled=False) + cli('copy running-config startup-config') + cli(f'install all nxos {target_path} no-reload') + + +def configure_management_networking(): + dhcp_config = cli("show run | sec '(interface mgmt0|vrf context management)'").split('\n') + dhcp_config = [line for line in dhcp_config if line] + configure(dhcp_config) + + +def configure_user_accounts(user_accounts: List[dict]): + config = [] + hash_algo_map = { + "Encrypt": 5, + "Pbkdf2": 8, + "scrypt": 9 + } + for a in user_accounts: + if a.get("hashedPassword") is None or a.get("hashAlgorithm") is None: + config.append(f'username {a["username"]} role network-admin') + continue + algo = hash_algo_map.get(a["hashAlgorithm"]) + if not algo: + LOG.warning(f"Unsupported hash algorithm {a['hashAlgorithm']} for user {a['username']}, skipping.") + continue + config.append(f'username {a["username"]} password {algo} {a["hashedPassword"]} role network-admin') + if len(config) == 0: + LOG.error("No valid user accounts to configure.") + report_status(SCRIPT_EXECUTION_FAILED, "No valid user accounts to configure.") + exit(-1) + configure(config) + + +def main(): + setup_logging() + signal.signal(signal.SIGTERM, lambda s: LOG.info(f"Received signal {s}, exiting...")) + + global TOKEN, SERIAL + LOG.info(f"Using provisioning server: {PROVISIONING_SERVER}") + SERIAL = cli_json('show version')['proc_board_id'] + LOG.info(f"Starting POAP process for device with serial: {SERIAL}") + try: + provisioning_data = get_provisioning_data() + except requests.RequestException as e: + LOG.error(f"Failed to fetch provisioning data: {e}") + exit(-1) + TOKEN = provisioning_data.get("provisioningToken") + if not TOKEN: + LOG.error("Provisioning token not found in provisioning data.") + exit(-1) + + report_status(SCRIPT_EXECUTION_STARTED, "Boot script initialized.") + + clean_old_images() + + LOG.info("Checking if firmware upgrade is needed.") + image = provisioning_data.get("image") + if not image: + LOG.error("No image information provided in provisioning data.") + report_status(UPGRADE_FAILED, "No image information for firmware upgrade.") + exit(-1) + + # Determine if upgrade is needed and download if necessary + need_upgrade = not verify_checksum(checksum=image["checksum"], checksum_type=image["checksumType"]) + LOG.info(f"Firmware upgrade needed: {need_upgrade}") + if need_upgrade: + need_download = True + target_path = f'{LOCAL_IMAGE_DIR}{os.path.basename(image["url"])}' + if os.path.exists(to_posix_path(target_path)): + LOG.info(f"Image already exists at {target_path}, verifying checksum.") + match = verify_checksum(checksum=image["checksum"], checksum_type=image["checksumType"], path=target_path) + if match: + LOG.info("Existing image checksum matches, skipping download.") + need_download = False + else: + LOG.warning("Existing image checksum does not match, will re-download the image.") + os.remove(to_posix_path(target_path)) + if need_download: + report_status(DOWNLOADING_IMAGE, "Downloading firmware image.") + try: + download_image(image["url"], f'{LOCAL_IMAGE_DIR}{os.path.basename(image["url"])}') + except requests.RequestException as e: + LOG.error(f"Failed to download or verify image: {e}") + report_status(IMAGE_DOWNLOAD_FAILED, f"Image download or verification failed: {e}") + exit(-1) + match = verify_checksum(checksum=image["checksum"], checksum_type=image["checksumType"], path=target_path) + if not match: + report_status(IMAGE_DOWNLOAD_FAILED, "Downloaded image checksum does not match.") + LOG.error("Downloaded image checksum does not match expected value.") + exit(-1) + + configure([f'hostname {provisioning_data["hostname"]}']) + configure_management_networking() + configure_user_accounts(provisioning_data.get("userAccounts", [])) + configure(STATIC_CONFIG) + + if ca_data := get_client_ca(): + report_status(INSTALLING_CERTIFICATES, "Installing CA certificates.") + configure_ca_certificates(ca_data) + configure(CLIENT_CA_CONFIG) + + if cert := get_device_certificates(): + report_status(INSTALLING_CERTIFICATES, "Installing device certificates.") + configure_device_certitificates(cert) + configure(DEVICE_CERT_CONFIG) + + if need_upgrade: + upgrade_firmware(target_path) + else: + set_firmware() + + # Writing the scheduled-config directly does not reliably survive the POAP + # reboot. Writing it to a file first and then copying it into + # scheduled-config is the only approach found to work consistently. + + cfg_file = "bootflash:///scheduled-config.cfg" + with open(to_posix_path(cfg_file), 'w') as f: + f.write(SCHEDEDULED_CONFIG) + if os.path.exists(to_posix_path('bootflash:///scheduled-config')): + os.remove(to_posix_path('bootflash:///scheduled-config')) + cli(f'copy {cfg_file} scheduled-config') + os.remove(to_posix_path(cfg_file)) + + # Some older reference scripts indicate POAP can complete without a reboot + # using exit codes, but this could not be made to work and there is no + # vendor documentation on the required behaviour. + # reference: https://github.com/CiscoSE/Cisco-POAP/blob/master/poap.py#L3-L9 + if need_upgrade: + report_status(REBOOTING_DEVICE, "Rebooting device to complete firmware upgrade.") + else: + report_status(REBOOTING_DEVICE, "Rebooting device.") + + +if __name__ == "__main__": + try: + main() + except Exception as e: + e = traceback.format_exc() + stack = traceback.format_stack() + LOG.error(f"An uncaught error occurred: {e}") + if TOKEN: + report_status(SCRIPT_EXECUTION_FAILED, f"Uncaught error: {'; '.join(e.splitlines())}") + LOG.error("Stack trace:") + for line in stack: + LOG.error(line) + exit(-1)