feat(tls): verify against the OS trust store by default (closes #2004)

CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- APM now verifies HTTPS against the OS trust store by default (via
+  `truststore`), so it works out-of-the-box behind a corporate CA or a
+  TLS-inspecting proxy -- matching the behaviour of `git`/`curl` on the same
+  host instead of failing against the bundled `certifi` set. This covers the
+  Python-based paths: `apm install` (in-process) and the Python-based `llm`
+  child runtime spawned by `apm run`, into whose venv APM installs `truststore`
+  and a self-contained `.pth` bootstrap at setup time. The frozen binary honours
+  the system store as well, with `certifi` as a genuine fallback. An explicitly
+  set `REQUESTS_CA_BUNDLE` / `CURL_CA_BUNDLE` still wins, and
+  `APM_DISABLE_TRUSTSTORE=1` restores the previous certifi-only behaviour.
+  Node-based (Copilot) and Rust-based (Codex) child runtimes are not yet covered
+  and continue to use their own default trust; tracked in #2034. (closes #2004)
+  (#2005)
+
 ## [0.24.0] - 2026-07-05
 
 ### Added

build/apm.spec

@@ -88,6 +88,13 @@ entry_point = repo_root / 'src' / 'apm_cli' / 'cli.py'
 datas = [
     (str(repo_root / 'scripts' / 'runtime'), 'scripts/runtime'),  # Bundle runtime setup scripts
     (str(repo_root / 'pyproject.toml'), '.'),  # Bundle pyproject.toml for version reading
+    # Child-runtime TLS trust bootstrap: ships at apm_cli/core/_child_tls/ so
+    # ensure_child_tls_bootstrap() can copy the self-contained .pth bootstrap
+    # into a child runtime venv's site-packages from the frozen binary.
+    (str(repo_root / 'src' / 'apm_cli' / 'core' / '_child_tls' / '_apm_tls_bootstrap.py'),
+     'apm_cli/core/_child_tls'),
+    (str(repo_root / 'src' / 'apm_cli' / 'core' / '_child_tls' / '_apm_tls.pth'),
+     'apm_cli/core/_child_tls'),
 ]
 
 # Bundle platform-appropriate token helper
@@ -188,6 +195,7 @@ hiddenimports = [
     'frontmatter',
     'requests',
     'certifi',  # CA certificate bundle for SSL verification in frozen binary
+    'truststore',  # OS trust-store verification (corporate CA / TLS proxy support)
     # Rich modules (lazily imported, must be explicitly included)
     'rich',
     'rich.console',

build/hooks/runtime_hook_ssl_certs.py

@@ -32,6 +32,11 @@ def _configure_ssl_certs() -> None:
         ca_bundle = certifi.where()
         if os.path.isfile(ca_bundle):
             os.environ["SSL_CERT_FILE"] = ca_bundle
+            # Marker: record that WE set this default (vs a genuine user
+            # SSL_CERT_FILE). apm_cli.core.tls_trust reads this to know it may
+            # pop SSL_CERT_FILE before truststore injection so the OS store is
+            # used, restoring it only if injection fails (certifi fallback).
+            os.environ["APM_SSL_CERT_FILE_IS_BUNDLED_DEFAULT"] = "1"
     except Exception:
         # certifi unavailable or broken -- fall through to system defaults.
         pass

docs/src/content/docs/enterprise/registry-proxy.md

@@ -278,6 +278,7 @@ and `apm cache clean`.
 | `ERROR: ... locked to direct VCS hosts` | Lockfile predates the proxy | `apm install --update` |
 | HTTP 401/403 from the proxy | Missing or invalid `PROXY_REGISTRY_TOKEN` | Verify the token has read on the upstream repo path |
 | `git clone` hangs through the proxy | `HTTPS_PROXY` not set in the env that runs `git` | Export it in the shell that invokes `apm install`; CI secrets often miss this |
+| `TLS verification failed` | Corporate proxy CA is not trusted by the OS store | Install the CA into the OS trust store, or set `REQUESTS_CA_BUNDLE`; see [SSL / TLS issues](../troubleshooting/ssl-issues/) |
 | `DeprecationWarning: ARTIFACTORY_BASE_URL is deprecated` | Legacy env names | Rename to `PROXY_REGISTRY_*` |
 | Plaintext-token warning on proxy startup | Token sent over `http://` | Use `https://`, or set `PROXY_REGISTRY_ALLOW_HTTP=1` if the link is internal-only |
 | `Invalid zip archive` with a body that starts `<!DOCTYPE html>` and is ~17KB | Upstream returned a sign-in page; proxy cached the HTML | Configure upstream credentials on the registry remote, purge the cache, then refetch |
@@ -293,5 +294,6 @@ connected host with `apm pack` and restore offline. See
 
 - [Authentication](../consumer/authentication/) -- token resolution order
 - [Private and org packages](../consumer/private-and-org-packages/) -- per-host PAT scoping
+- [SSL / TLS issues](../troubleshooting/ssl-issues/) -- corporate CA and TLS proxy trust
 - [Pack and distribute](../producer/pack-a-bundle/) -- air-gapped bundle delivery
 - [Governance deep-dive](./governance-guide/) -- policy-cache offline story

docs/src/content/docs/reference/environment-variables.md

@@ -40,6 +40,16 @@ Controls how APM clones packages from Git hosts. These settings can also be pers
 | `APM_GIT_PROTOCOL` | Preferred clone protocol for shorthand (`owner/repo`) dependencies. Accepted values: `ssh`, `https`. | unset | Equivalent to `--ssh` / `--https` flag. Resolution: CLI flag → env var → `prefer-ssh` key in `~/.apm/config.json` → git `insteadOf` rules → HTTPS. |
 | `APM_ALLOW_PROTOCOL_FALLBACK` | Set to `1` (or `true`/`yes`/`on`) to enable the legacy cross-protocol fallback chain. When enabled, a failed clone is retried with the opposite protocol. | unset | Equivalent to `--allow-protocol-fallback`. Resolution: CLI flag → env var → `allow-protocol-fallback` key in `~/.apm/config.json` → `false`. |
 
+## TLS trust
+
+APM verifies HTTPS against the operating-system trust store by default. For the full troubleshooting flow, see [SSL / TLS issues](../troubleshooting/ssl-issues/).
+
+| Variable | Purpose | Default | Notes |
+|---|---|---|---|
+| `REQUESTS_CA_BUNDLE` | PEM bundle for APM's Python HTTP requests. | unset | Explicit override; wins over OS trust-store injection. Use for a per-shell corporate CA bundle. |
+| `CURL_CA_BUNDLE` | PEM bundle fallback honoured by `requests`. | unset | Explicit override; wins over OS trust-store injection when `REQUESTS_CA_BUNDLE` is unset. |
+| `APM_DISABLE_TRUSTSTORE` | Set to `1` (or `true`/`yes`/`on`) to disable OS trust-store injection. | unset | Escape hatch that restores the legacy bundled-`certifi` verification path. |
+
 ## Registry (MCP and proxy)
 
 | Variable | Purpose | Default | Notes |

docs/src/content/docs/troubleshooting/common-errors.md

@@ -241,14 +241,15 @@ See also: [./install-failures/](./install-failures/)
 ### `TLS verification failed`
 
 ```
-TLS verification failed -- if you're behind a corporate proxy or
-firewall, set the REQUESTS_CA_BUNDLE environment variable to the
-path of your organisation's CA bundle (a PEM file) and retry.
+TLS verification failed -- APM uses the system trust store by default.
+If you're behind a corporate proxy or firewall, make sure your
+organisation's CA is installed in the OS trust store, or set
+REQUESTS_CA_BUNDLE to a readable PEM bundle and retry.
 ```
 
 Cause: Python's TLS stack rejected the server certificate. Almost always a corporate proxy doing TLS interception with a CA that is not in the system trust store.
 
-Fix: export `REQUESTS_CA_BUNDLE=/path/to/corporate-ca.pem` and retry. Do not disable TLS verification.
+Fix: install the corporate CA into the OS trust store and retry. For a per-shell override, export `REQUESTS_CA_BUNDLE=/path/to/corporate-ca.pem`; `SSL_CERT_FILE` alone is not a reliable requests override. Do not disable TLS verification.
 
 See also: [./ssl-issues/](./ssl-issues/)
 

docs/src/content/docs/troubleshooting/install-failures.md

@@ -110,7 +110,7 @@ For end-to-end auth setup see [Authentication](../getting-started/authentication
 [!] TLS verification failed
 ```
 
-Behind a corporate proxy, set `REQUESTS_CA_BUNDLE` to your org's CA bundle (PEM file). Full walkthrough: [SSL / TLS issues](./ssl-issues/).
+APM verifies HTTPS against the OS trust store by default. Behind a corporate proxy, install your org's CA into the OS trust store; for a per-shell override, set `REQUESTS_CA_BUNDLE` to a readable PEM bundle. Full walkthrough: [SSL / TLS issues](./ssl-issues/).
 
 ### Timeouts and proxies
 

docs/src/content/docs/troubleshooting/ssl-issues.md

@@ -14,8 +14,10 @@ Related: [environment variables](../reference/environment-variables/), [install
 Typical errors APM surfaces or passes through from the underlying HTTP/git stack:
 
 ```text
-[!] TLS verification failed -- if you're behind a corporate proxy or
-    firewall, set REQUESTS_CA_BUNDLE to your organisation's CA bundle.
+[!] TLS verification failed -- APM uses the system trust store by default.
+    If you're behind a corporate proxy or firewall, make sure your
+    organisation's CA is installed in the OS trust store, or set
+    REQUESTS_CA_BUNDLE to a readable PEM bundle and retry.
 ```
 
 ```text
@@ -52,6 +54,25 @@ Re-run the failing command with `--verbose` to see the underlying exception and
 apm install --verbose
 ```
 
+## Default behaviour: the OS trust store
+
+APM verifies HTTPS against the **operating-system trust store** by default (via [`truststore`](https://pypi.org/project/truststore/)), the same source `git` and `curl` use. This applies to `apm install` (in-process) and to the Python-based `llm` child runtime that `apm run` spawns: `apm runtime setup llm` installs `truststore` into the runtime's virtual environment and drops a self-contained bootstrap into it, so that interpreter also verifies against the OS store. So if your corporate CA or TLS-proxy root is installed in the OS trust store (Keychain on macOS, `update-ca-certificates`/`update-ca-trust` on Linux, the Trusted Root store on Windows), APM picks it up with **no configuration** -- importing the CA at the OS level is the recommended fix. The standalone frozen binary honours the system store as well, falling back to its bundled `certifi` set only when the OS store is unavailable.
+
+**Scope caveat:** only the Python-based paths are covered. The Node-based (Copilot) and Rust-based (Codex) child runtimes are **not yet** wired to the OS store (tracked in #2034). Behind a TLS-proxy today, export `NODE_EXTRA_CA_CERTS=/path/to/org-ca-bundle.pem` for the Node runtime (and configure the Codex/Rust runtime's own trust) until #2034 lands.
+
+You only need the steps below when the CA is *not* in the OS store, or you want to pin a specific bundle:
+
+- Setting `REQUESTS_CA_BUNDLE` or `CURL_CA_BUNDLE` makes APM's HTTP layer verify against that bundle instead of the OS store. (`SSL_CERT_FILE` configures the stdlib `ssl` layer but is *not* read by `requests`, so on its own it does not override the HTTP path -- use `REQUESTS_CA_BUNDLE` for that.)
+- `APM_DISABLE_TRUSTSTORE=1` restores the legacy behaviour (verify against APM's bundled `certifi` set only).
+
+### Known limitations
+
+- The Node-based (Copilot) and Rust-based (Codex) child runtimes are **not yet covered** by OS-trust propagation and continue to use their own default trust; behind a TLS-inspecting proxy, configure those runtimes' own trust or export `REQUESTS_CA_BUNDLE`/`NODE_EXTRA_CA_CERTS` for them. Tracked in #2034.
+- The `llm` child runtime's OS-trust bootstrap needs the runtime venv's interpreter to be **Python 3.10+** (the `truststore` library requires 3.10). On systems where `apm runtime setup llm` builds the venv from a stock **Python 3.9** (for example Apple's `/usr/bin/python3`), `truststore` cannot install and the `llm` child silently falls back to its bundled `certifi` set behind a proxy. Use a Python 3.10+ `python3` on your `PATH` before running setup.
+- The initial `pip install` run *during* `apm runtime setup llm` uses pip's **own** certificate resolution, not APM's OS-trust path. Behind a MITM proxy, `pip` may fail to fetch `llm`/`truststore` before the bootstrap is even in place. Export `PIP_CERT=/path/to/org-ca-bundle.pem` (or run `pip config set global.cert /path/to/org-ca-bundle.pem`) before running setup so pip trusts your proxy CA.
+- An additive `APM_EXTRA_CA_BUNDLE` (trust the OS store *and* an extra bundle) is not yet available -- use `REQUESTS_CA_BUNDLE` to pin a single bundle for now.
+- On Windows, the `schannel` backend has trust caveats.
+
 ## Configure trust
 
 APM uses `requests` for HTTP and shells out to `git` for repository operations. Both honour standard environment variables. Set them at the shell or in your profile (`~/.zshrc`, `~/.bashrc`, or the Windows user environment).
@@ -60,12 +81,9 @@ APM uses `requests` for HTTP and shells out to `git` for repository operations.
 
 ```bash
 export REQUESTS_CA_BUNDLE=/path/to/ca-bundle.pem
-# or, more general:
-export SSL_CERT_FILE=/path/to/ca-bundle.pem
-export SSL_CERT_DIR=/etc/ssl/certs
 ```
 
-`REQUESTS_CA_BUNDLE` wins for `requests`. `SSL_CERT_FILE` / `SSL_CERT_DIR` cover the rest of the Python TLS stack.
+`REQUESTS_CA_BUNDLE` wins for `requests`. `SSL_CERT_FILE` / `SSL_CERT_DIR` cover parts of the stdlib TLS stack, but on their own they are not reliable overrides for the `requests` HTTP path APM uses.
 
 ### Git operations
 
@@ -170,6 +188,7 @@ unset GIT_SSL_NO_VERIFY PYTHONHTTPSVERIFY
 
 [>] Re-run with `--verbose` and capture the full exception chain.
 [>] Check `curl -v https://<host>` from the same shell - if it fails, the problem is the system trust store, not APM.
-[>] Confirm `REQUESTS_CA_BUNDLE` and `GIT_SSL_CAINFO` point at a readable PEM file (`openssl x509 -in $REQUESTS_CA_BUNDLE -noout -subject` should print a subject line).
+[>] Confirm `REQUESTS_CA_BUNDLE` and `GIT_SSL_CAINFO` point at a readable PEM file (`openssl x509 -in $REQUESTS_CA_BUNDLE -noout -subject` should print a subject line). Note `REQUESTS_CA_BUNDLE` *replaces* the OS store rather than augmenting it (like `git`'s `http.sslCAInfo` and `curl --cacert`), so a bundle missing your proxy root will still fail even though the OS store has it.
+[>] If `git`/`curl` succeed but `apm` does not, suspect a **stale `REQUESTS_CA_BUNDLE`** (or `CURL_CA_BUNDLE`) pinning APM to an old bundle that predates the OS store. `unset REQUESTS_CA_BUNDLE CURL_CA_BUNDLE` and retry to let APM fall back to the OS trust store.
 [>] If only one host fails, see [GHES and GitLab self-managed](#ghes-and-gitlab-self-managed) and the per-host `git config` recipe above.
 [>] If the install proceeds past TLS but then fails, continue at [install failures](./install-failures/).

packages/apm-guide/.apm/skills/apm-usage/troubleshooting.md

@@ -6,6 +6,7 @@
 | Authentication errors (401/403) | Set the correct token. Run `apm install --verbose` to see which token source is used. See [Authentication](./authentication.md). |
 | File collision on install | A local file conflicts with a dependency file. Use `--force` to overwrite, or rename the local file. |
 | Stale dependencies | Run `apm install --update` to refresh to latest refs. |
+| TLS verification failed | Install your corporate CA into the OS trust store. For a per-shell override, set `REQUESTS_CA_BUNDLE=/path/to/ca-bundle.pem`; `SSL_CERT_FILE` alone is not a reliable requests override. |
 | Orphaned packages in lockfile | Run `apm prune` to remove packages no longer in apm.yml. |
 | Security findings block install | Run `apm audit` to review findings, then `apm install --force` if acceptable. |
 | Compilation not picking up changes | Run `apm compile --clean` to remove orphaned output, or `apm compile --watch` for auto-regeneration. |

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "colorama>=0.4.6",
     "pyyaml>=6.0.0",
     "requests>=2.31.0",
+    "truststore>=0.10.0",
     "python-frontmatter>=1.0.0",
     "llm>=0.28",
     "llm-github-models>=0.18.0",
@@ -62,6 +63,12 @@ apm = "apm_cli.cli:main"
 where = ["src"]
 include = ["apm_cli*"]
 
+[tool.setuptools.package-data]
+# Belt-and-suspenders: ship the child-TLS .pth alongside the bootstrap module.
+# Delivery no longer depends on this (ensure_child_tls_bootstrap generates the
+# .pth inline), but packaging it keeps the source tree and wheel in sync.
+"apm_cli.core._child_tls" = ["*.pth"]
+
 [tool.ruff]
 line-length = 100
 target-version = "py310"

scripts/runtime/setup-llm.ps1

@@ -38,6 +38,18 @@ function Install-Llm {
     & $pipExe install --upgrade pip
     & $pipExe install llm
 
+    # Install truststore so the llm venv verifies HTTPS against the OS trust
+    # store (corporate CA / TLS-proxy support). APM drops a self-contained .pth
+    # bootstrap into this venv's site-packages after setup; that bootstrap
+    # depends only on truststore, which must be present here.
+    Write-Info "Installing truststore for OS-trust HTTPS verification..."
+    try {
+        & $pipExe install "truststore>=0.10.0"
+        if ($LASTEXITCODE -ne 0) { throw "pip exited $LASTEXITCODE" }
+    } catch {
+        Write-WarningText "truststore install failed (needs Python 3.10+); llm child will fall back to bundled CAs behind a proxy"
+    }
+
     # Install GitHub Models plugin in non-vanilla mode
     if (-not $Vanilla) {
         Write-Info "Installing GitHub Models plugin for APM defaults..."

scripts/runtime/setup-llm.sh

@@ -49,6 +49,13 @@ setup_llm() {
     log_info "Installing LLM library..."
     "$llm_venv/bin/pip" install --upgrade pip
     "$llm_venv/bin/pip" install llm
+
+    # Install truststore so the llm venv verifies HTTPS against the OS trust
+    # store (corporate CA / TLS-proxy support). APM drops a self-contained .pth
+    # bootstrap into this venv's site-packages after setup; that bootstrap
+    # depends only on truststore, which must be present here.
+    log_info "Installing truststore for OS-trust HTTPS verification..."
+    "$llm_venv/bin/pip" install "truststore>=0.10.0" || log_warning "truststore install failed (needs Python 3.10+; stock macOS python3 is 3.9); llm child will fall back to bundled CAs behind a proxy"
 
     # Install GitHub Models plugin in non-vanilla mode
     if [[ "$VANILLA_MODE" == "false" ]]; then

src/apm_cli/cli.py

@@ -3,6 +3,8 @@
 Thin wiring layer  -- all command logic lives in ``apm_cli.commands.*`` modules.
 """
 
+# ruff: noqa: E402
+
 import ctypes
 import logging
 import os
@@ -11,6 +13,22 @@
 
 import click
 
+from apm_cli.core.tls_trust import configure_tls_trust
+
+_TLS_TRUST_CONFIGURED = False
+
+
+def _configure_process_tls_trust() -> None:
+    """Configure process-wide TLS trust before network clients are imported."""
+    global _TLS_TRUST_CONFIGURED
+    if _TLS_TRUST_CONFIGURED:
+        return
+    configure_tls_trust()
+    _TLS_TRUST_CONFIGURED = True
+
+
+_configure_process_tls_trust()
+
 from apm_cli.commands._helpers import (
     ERROR,
     RESET,
@@ -333,6 +351,7 @@ def main():
     """Main entry point for the CLI."""
     _configure_logging()  # honours APM_LOG_LEVEL env var; --verbose upgrades in cli()
     _configure_encoding()
+    _configure_process_tls_trust()
     try:
         cli(obj={})
     except Exception as e:

src/apm_cli/core/_child_tls/__init__.py

@@ -0,0 +1,15 @@
+"""Child-process TLS trust bootstrap package.
+
+Holds the two artifacts copied into a child runtime venv's site-packages by
+``ensure_child_tls_bootstrap`` in ``apm_cli.core.tls_trust``:
+
+* ``_apm_tls_bootstrap.py`` -- a self-contained OS-trust bootstrap with NO
+  ``apm_cli`` dependency (it only needs ``truststore``).
+* ``_apm_tls.pth`` -- a one-line path-config file (``import _apm_tls_bootstrap``)
+  that Python executes at interpreter startup, so trust is delivered at
+  venv-setup time instead of by mutating the child's ``PYTHONPATH`` at spawn
+  time (which would shadow a user/corporate ``sitecustomize.py``).
+
+The ``__init__`` exists only so the directory ships as package data; the
+bootstrap files are copied by path, not imported as submodules of this package.
+"""

src/apm_cli/core/_child_tls/_apm_tls.pth

@@ -0,0 +1 @@
+import _apm_tls_bootstrap