fix: llama-server without OpenMP on Linux, fail fast, 2 threads on 2 vCPUs

.github/workflows/llama.yml

@@ -0,0 +1,115 @@
+name: llama.cpp Linux build
+
+# Upstream's Ubuntu release links OpenMP, so llama-server does not start on a
+# system without libgomp.so.1 (Ubuntu 26.04 cloud images ship without it).
+# A llama-b<build>-noomp tag rebuilds the pinned llama.cpp tag for Linux with
+# GGML_OPENMP=OFF and publishes it as a prerelease that
+# internal/provision/provision.go pins by sha256. Every other flag mirrors
+# upstream's ubuntu-cpu job at that tag.
+#
+# Prerelease + --latest=false: install.sh and `camne update` read
+# /releases/latest, which must keep pointing at a camne release.
+on:
+  push:
+    tags: ["llama-b*-noomp"]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        include:
+          - build: x64
+            os: ubuntu-22.04
+          - build: arm64
+            os: ubuntu-24.04-arm
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    steps:
+      - name: Pinned llama.cpp tag from the camne tag
+        run: |
+          TAG="${GITHUB_REF_NAME}"
+          UPSTREAM="${TAG#llama-}"
+          echo "TAG=$TAG" >> "$GITHUB_ENV"
+          echo "UPSTREAM=${UPSTREAM%-noomp}" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: ggml-org/llama.cpp
+          ref: ${{ env.UPSTREAM }}
+
+      # Upstream's own workaround: the arm64 CPU variants need GCC 14.
+      - name: Toolchain (GCC 14)
+        if: matrix.build == 'arm64'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-14 g++-14
+          echo "CC=gcc-14" >> "$GITHUB_ENV"
+          echo "CXX=g++-14" >> "$GITHUB_ENV"
+
+      # Differences from upstream: GGML_OPENMP=OFF (the fix), LLAMA_OPENSSL=OFF
+      # (camne never has llama-server fetch over HTTPS, and libssl is one more
+      # library a minimal image can lack), LLAMA_BUILD_UI=OFF (camne passes
+      # --no-webui).
+      - name: Build
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+            -DGGML_BACKEND_DL=ON \
+            -DGGML_NATIVE=OFF \
+            -DGGML_CPU_ALL_VARIANTS=ON \
+            -DGGML_OPENMP=OFF \
+            -DLLAMA_OPENSSL=OFF \
+            -DLLAMA_BUILD_UI=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_TOOLS=ON \
+            -DLLAMA_BUILD_SERVER=ON
+          cmake --build build --config Release -j "$(nproc)"
+
+      # The whole point of this build: fail if anything still needs libgomp
+      # or libssl, or if the server does not start.
+      - name: Check dependencies
+        run: |
+          cd build/bin
+          if readelf -d llama-server *.so* | grep -E 'NEEDED.*(gomp|ssl|crypto)'; then
+            echo "llama-server still needs a library this build removes" >&2
+            exit 1
+          fi
+          ./llama-server --version
+
+      - name: Pack
+        run: |
+          cp LICENSE build/bin/
+          tar -czf "$TAG-bin-ubuntu-${{ matrix.build }}.tar.gz" --transform "s,^\.,$TAG," -C build/bin .
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ubuntu-${{ matrix.build }}
+          path: ${{ env.TAG }}-bin-ubuntu-${{ matrix.build }}.tar.gz
+
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+      - name: Publish prerelease
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          sha256sum *.tar.gz | tee checksums.txt
+          stat -c '%s %n' *.tar.gz
+          gh release create "$GITHUB_REF_NAME" \
+            --title "$GITHUB_REF_NAME" \
+            --notes "llama.cpp ${GITHUB_REF_NAME#llama-} rebuilt for Linux with GGML_OPENMP=OFF, so llama-server needs no libgomp. Downloaded by camne, pinned by sha256 in internal/provision/provision.go. Not a camne release." \
+            --prerelease --latest=false \
+            *.tar.gz checksums.txt

cmd/camne/install.go

@@ -119,6 +119,8 @@ func installServer(asset provision.Asset, serverPath string) error {
 	if _, err := os.Stat(serverPath); err != nil {
 		return fmt.Errorf("unpacking finished but llama-server is not where it should be — delete the folder %s, then run camne again", filepath.Dir(serverPath))
 	}
+	// Only after the new server is in place, so a failed upgrade keeps the old.
+	provision.RemoveOtherServers(serverPath)
 	return nil
 }
 

internal/engine/engine.go

@@ -47,6 +47,10 @@ const idleSleep = "1800"
 type Client struct {
 	base string // URL prefix; a placeholder host when dialing a unix socket
 	hc   *http.Client
+	// exited closes when a server this process started stops. nil for a
+	// server that was already running.
+	exited  <-chan struct{}
+	logPath string
 }
 
 // runDir is the 0700 directory holding the socket, pidfile and server log.
@@ -71,15 +75,19 @@ func pidPath(rd string) string { return filepath.Join(rd, "llama.pid") }
 
 // threads: decoding is memory-bandwidth-bound, not compute-bound, so half the
 // cores capped at 4 is the whatisit-validated default for the target box.
-func threads() string {
-	n := runtime.NumCPU() / 2
+func threads() string { return strconv.Itoa(threadsFor(runtime.NumCPU())) }
+
+// threadsFor floors at 2 when the machine has 2 logical CPUs. Half of 2 is 1,
+// and on a 2-vCPU t3.large -t 2 measured 3.6 s per query against 5.8 s at -t 1.
+func threadsFor(cpus int) int {
+	n := cpus / 2
+	if n < 2 {
+		n = min(cpus, 2)
+	}
 	if n < 1 {
 		n = 1
 	}
-	if n > 4 {
-		n = 4
-	}
-	return strconv.Itoa(n)
+	return min(n, 4)
 }
 
 // Connect returns a client for the resident llama-server, starting one
@@ -110,7 +118,8 @@ func Connect(serverPath, modelPath string) (c *Client, cold bool, err error) {
 	}, hostArgs...)
 	cmd := exec.Command(serverPath, args...)
 	// llama-server chatter goes to a log file, never to the user's terminal.
-	if logf, lerr := os.OpenFile(filepath.Join(rd, "llama.log"),
+	c.logPath = filepath.Join(rd, "llama.log")
+	if logf, lerr := os.OpenFile(c.logPath,
 		os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600); lerr == nil {
 		cmd.Stdout, cmd.Stderr = logf, logf
 		defer logf.Close()
@@ -120,7 +129,12 @@ func Connect(serverPath, modelPath string) (c *Client, cold bool, err error) {
 		return nil, false, fmt.Errorf("could not start llama-server — run `camne doctor` to see what is missing: %w", err)
 	}
 	os.WriteFile(pidPath(rd), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600)
-	cmd.Process.Release()
+	// A server that dies at start (a missing shared library, a bad model
+	// file) must fail in seconds, not after the whole WaitReady timeout.
+	// Wait never blocks camne's own exit; the server is in its own session.
+	exited := make(chan struct{})
+	go func() { cmd.Wait(); close(exited) }()
+	c.exited = exited
 	return c, true, nil
 }
 
@@ -148,13 +162,29 @@ func (c *Client) WaitReady(timeout time.Duration) error {
 		if ready, _ := c.health(); ready {
 			return nil
 		}
+		select {
+		case <-c.exited: // a nil channel never fires
+			return fmt.Errorf("llama-server stopped while starting: %s — run `camne doctor`, then ask again. If it keeps happening, open an issue at https://github.com/officialdad/camne/issues with this message", lastLine(c.logPath))
+		default:
+		}
 		if time.Now().After(deadline) {
 			return errors.New("the model is still not ready — run `camne stop`, then ask again. If it keeps happening, `camne doctor` will show what is missing")
 		}
 		time.Sleep(300 * time.Millisecond)
 	}
 }
 
+// lastLine returns the last non-empty line of the server log, stripped to
+// printable ASCII because it reaches the terminal.
+func lastLine(path string) string {
+	b, _ := os.ReadFile(path)
+	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
+	if l := strings.TrimSpace(printable(lines[len(lines)-1])); l != "" {
+		return l
+	}
+	return "no error was recorded"
+}
+
 type completionReq struct {
 	Prompt      string   `json:"prompt"`
 	NPredict    int      `json:"n_predict"`

internal/engine/engine_test.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -47,6 +49,42 @@ func TestWaitReadyTimeout(t *testing.T) {
 	}
 }
 
+func TestThreadsFor(t *testing.T) {
+	for _, tt := range []struct{ cpus, want int }{
+		{0, 1}, {1, 1}, {2, 2}, {3, 2}, {4, 2}, {6, 3}, {8, 4}, {16, 4},
+	} {
+		if got := threadsFor(tt.cpus); got != tt.want {
+			t.Errorf("threadsFor(%d) = %d, want %d", tt.cpus, got, tt.want)
+		}
+	}
+}
+
+func TestWaitReadyServerExited(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close()
+
+	log := filepath.Join(t.TempDir(), "llama.log")
+	os.WriteFile(log, []byte("llama-server: error while loading shared libraries: libgomp.so.1\x1b[0m\n\n"), 0o600)
+	exited := make(chan struct{})
+	close(exited)
+	c := testClient(srv)
+	c.exited, c.logPath = exited, log
+
+	start := time.Now()
+	err := c.WaitReady(time.Minute)
+	if err == nil {
+		t.Fatal("expected error for an exited server, got nil")
+	}
+	if time.Since(start) > 5*time.Second {
+		t.Errorf("WaitReady took %v, want it to stop as soon as the server exits", time.Since(start))
+	}
+	if !strings.Contains(err.Error(), "libgomp.so.1") || strings.Contains(err.Error(), "\x1b") {
+		t.Errorf("error = %q, want the log's last line with control bytes stripped", err)
+	}
+}
+
 func TestComplete(t *testing.T) {
 	var got completionReq
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

internal/provision/provision.go

@@ -29,6 +29,12 @@ const (
 	ModelSize int64 = 986048032
 )
 
+// linuxBuild is llamaBuild rebuilt by .github/workflows/llama.yml with
+// GGML_OPENMP=OFF. Upstream's Ubuntu build needs libgomp.so.1, which minimal
+// images (Ubuntu 26.04 cloud) lack, and llama-server then exits at start.
+// macOS and Windows keep upstream's builds.
+const linuxBuild = llamaBuild + "-noomp"
+
 // Asset is one llama.cpp release file for a specific platform.
 type Asset struct {
 	Name   string // release asset file name
@@ -38,15 +44,27 @@ type Asset struct {
 
 // URL is the direct release-asset download URL for the pinned build.
 func (a Asset) URL() string {
+	if strings.HasPrefix(a.Name, "llama-"+linuxBuild+"-") {
+		return "https://github.com/officialdad/camne/releases/download/llama-" + linuxBuild + "/" + a.Name
+	}
 	return "https://github.com/ggml-org/llama.cpp/releases/download/" + llamaBuild + "/" + a.Name
 }
 
+// dir is the versioned directory under bin/ the asset unpacks to: the name up
+// to "-bin-", which is also the tar.gz entry prefix. A new build gets a new
+// directory, so an install of the old one is replaced, never trusted.
+func (a Asset) dir() string {
+	d, _, _ := strings.Cut(a.Name, "-bin-")
+	return d
+}
+
 // llamaAssets maps GOOS/GOARCH to the release asset for the pinned build.
-// Digests and sizes come from the GitHub release API for that tag, hardcoded
+// Digests and sizes come from the GitHub release API for that tag (Linux: the
+// llama-b10333-noomp release on this repo), hardcoded
 // so provisioning never depends on the rate-limited API at runtime.
 var llamaAssets = map[string]Asset{
-	"linux/amd64":   {"llama-" + llamaBuild + "-bin-ubuntu-x64.tar.gz", "936ce04d98abe2a977e9dd2ff92659bb96947e136acee8f2bc3e21d8eaebbf23", 16507165},
-	"linux/arm64":   {"llama-" + llamaBuild + "-bin-ubuntu-arm64.tar.gz", "95da1a0f7538340f625b0301593ee63c046ec0a155c74f21444ea4d43bca79a1", 13377770},
+	"linux/amd64":   {"llama-" + linuxBuild + "-bin-ubuntu-x64.tar.gz", "5041ff232f7e5d19f5748fe93417f69c10ea0adb2e8c34ce10d0be80eaeeae88", 16266714},
+	"linux/arm64":   {"llama-" + linuxBuild + "-bin-ubuntu-arm64.tar.gz", "ecb4e32cb37ea8aac1404c3b7b36ac9cd86aa713bdbc9b8770d91738863f48ef", 13137461},
 	"darwin/amd64":  {"llama-" + llamaBuild + "-bin-macos-x64.tar.gz", "6ffd9e0b9b2e3ab6ccfba332a74b968b0fef891f01bd4747d3d75bc7393877ea", 11290712},
 	"darwin/arm64":  {"llama-" + llamaBuild + "-bin-macos-arm64.tar.gz", "e5d67c5264107e3c14d3bf2aee349365bb2b85ae99bb077a0cb974a1c4c2741a", 11015270},
 	"windows/amd64": {"llama-" + llamaBuild + "-bin-win-cpu-x64.zip", "563ab97cd003cf7cd4843677e6fea2f6162aee0c9580589c8d9e5816923076d5", 18399512},
@@ -88,7 +106,23 @@ func ServerPath() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return filepath.Join(d, "bin", "llama-"+llamaBuild, serverBinary()), nil
+	a, err := LlamaAsset(runtime.GOOS, runtime.GOARCH)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(d, "bin", a.dir(), serverBinary()), nil
+}
+
+// RemoveOtherServers deletes every directory under bin/ except the one holding
+// serverPath. Without it each llama.cpp upgrade strands ~40 MB on disk.
+func RemoveOtherServers(serverPath string) {
+	keep := filepath.Dir(serverPath)
+	names, _ := filepath.Glob(filepath.Join(filepath.Dir(keep), "*"))
+	for _, n := range names {
+		if n != keep {
+			os.RemoveAll(n)
+		}
+	}
 }
 
 // ModelPath is where the GGUF lives once downloaded.

internal/provision/provision_test.go

@@ -1,14 +1,19 @@
 package provision
 
-import "testing"
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
 
 func TestLlamaAsset(t *testing.T) {
 	tests := []struct {
 		goos, goarch string
 		want         string // asset name, "" = expect error
 	}{
-		{"linux", "amd64", "llama-" + llamaBuild + "-bin-ubuntu-x64.tar.gz"},
-		{"linux", "arm64", "llama-" + llamaBuild + "-bin-ubuntu-arm64.tar.gz"},
+		{"linux", "amd64", "llama-" + linuxBuild + "-bin-ubuntu-x64.tar.gz"},
+		{"linux", "arm64", "llama-" + linuxBuild + "-bin-ubuntu-arm64.tar.gz"},
 		{"darwin", "amd64", "llama-" + llamaBuild + "-bin-macos-x64.tar.gz"},
 		{"darwin", "arm64", "llama-" + llamaBuild + "-bin-macos-arm64.tar.gz"},
 		{"windows", "amd64", "llama-" + llamaBuild + "-bin-win-cpu-x64.zip"},
@@ -34,6 +39,35 @@ func TestLlamaAsset(t *testing.T) {
 		if len(a.SHA256) != 64 || a.Size <= 0 {
 			t.Errorf("%s/%s: incomplete pin: sha256 len %d, size %d", tt.goos, tt.goarch, len(a.SHA256), a.Size)
 		}
+		// The tar.gz entry prefix must equal the directory ServerPath uses,
+		// and the Linux rebuild comes from camne's release, not upstream's.
+		wantDir, wantRepo := "llama-"+llamaBuild, "ggml-org/llama.cpp"
+		if tt.goos == "linux" {
+			wantDir, wantRepo = "llama-"+linuxBuild, "officialdad/camne"
+		}
+		if a.dir() != wantDir {
+			t.Errorf("%s/%s: dir %q, want %q", tt.goos, tt.goarch, a.dir(), wantDir)
+		}
+		if !strings.HasPrefix(a.URL(), "https://github.com/"+wantRepo+"/releases/download/") || !strings.HasSuffix(a.URL(), "/"+a.Name) {
+			t.Errorf("%s/%s: URL %q, want a %s release asset", tt.goos, tt.goarch, a.URL(), wantRepo)
+		}
+	}
+}
+
+func TestRemoveOtherServers(t *testing.T) {
+	bin := t.TempDir()
+	current := filepath.Join(bin, "llama-"+linuxBuild, "llama-server")
+	old := filepath.Join(bin, "llama-"+llamaBuild, "llama-server")
+	for _, p := range []string{current, old} {
+		os.MkdirAll(filepath.Dir(p), 0o755)
+		os.WriteFile(p, []byte("x"), 0o755)
+	}
+	RemoveOtherServers(current)
+	if _, err := os.Stat(current); err != nil {
+		t.Errorf("current server removed: %v", err)
+	}
+	if _, err := os.Stat(filepath.Dir(old)); !os.IsNotExist(err) {
+		t.Errorf("old server directory still present: %v", err)
 	}
 }