diff --git a/.github/workflows/llama.yml b/.github/workflows/llama.yml new file mode 100644 index 0000000..65e507f --- /dev/null +++ b/.github/workflows/llama.yml @@ -0,0 +1,115 @@ +name: llama.cpp Linux build + +# Upstream's Ubuntu release links OpenMP, so llama-server does not start on a +# system without libgomp.so.1 (Ubuntu 26.04 cloud images ship without it). +# A llama-b-noomp tag rebuilds the pinned llama.cpp tag for Linux with +# GGML_OPENMP=OFF and publishes it as a prerelease that +# internal/provision/provision.go pins by sha256. Every other flag mirrors +# upstream's ubuntu-cpu job at that tag. +# +# Prerelease + --latest=false: install.sh and `camne update` read +# /releases/latest, which must keep pointing at a camne release. +on: + push: + tags: ["llama-b*-noomp"] + +permissions: + contents: read + +jobs: + build: + strategy: + matrix: + include: + - build: x64 + os: ubuntu-22.04 + - build: arm64 + os: ubuntu-24.04-arm + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - name: Pinned llama.cpp tag from the camne tag + run: | + TAG="${GITHUB_REF_NAME}" + UPSTREAM="${TAG#llama-}" + echo "TAG=$TAG" >> "$GITHUB_ENV" + echo "UPSTREAM=${UPSTREAM%-noomp}" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: ggml-org/llama.cpp + ref: ${{ env.UPSTREAM }} + + # Upstream's own workaround: the arm64 CPU variants need GCC 14. + - name: Toolchain (GCC 14) + if: matrix.build == 'arm64' + run: | + sudo apt-get update + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + # Differences from upstream: GGML_OPENMP=OFF (the fix), LLAMA_OPENSSL=OFF + # (camne never has llama-server fetch over HTTPS, and libssl is one more + # library a minimal image can lack), LLAMA_BUILD_UI=OFF (camne passes + # --no-webui). + - name: Build + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_UI=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON + cmake --build build --config Release -j "$(nproc)" + + # The whole point of this build: fail if anything still needs libgomp + # or libssl, or if the server does not start. + - name: Check dependencies + run: | + cd build/bin + if readelf -d llama-server *.so* | grep -E 'NEEDED.*(gomp|ssl|crypto)'; then + echo "llama-server still needs a library this build removes" >&2 + exit 1 + fi + ./llama-server --version + + - name: Pack + run: | + cp LICENSE build/bin/ + tar -czf "$TAG-bin-ubuntu-${{ matrix.build }}.tar.gz" --transform "s,^\.,$TAG," -C build/bin . + + - uses: actions/upload-artifact@v4 + with: + name: ubuntu-${{ matrix.build }} + path: ${{ env.TAG }}-bin-ubuntu-${{ matrix.build }}.tar.gz + + release: + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + - name: Publish prerelease + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + sha256sum *.tar.gz | tee checksums.txt + stat -c '%s %n' *.tar.gz + gh release create "$GITHUB_REF_NAME" \ + --title "$GITHUB_REF_NAME" \ + --notes "llama.cpp ${GITHUB_REF_NAME#llama-} rebuilt for Linux with GGML_OPENMP=OFF, so llama-server needs no libgomp. Downloaded by camne, pinned by sha256 in internal/provision/provision.go. Not a camne release." \ + --prerelease --latest=false \ + *.tar.gz checksums.txt diff --git a/cmd/camne/install.go b/cmd/camne/install.go index f52c4c6..e8e3f66 100644 --- a/cmd/camne/install.go +++ b/cmd/camne/install.go @@ -119,6 +119,8 @@ func installServer(asset provision.Asset, serverPath string) error { if _, err := os.Stat(serverPath); err != nil { return fmt.Errorf("unpacking finished but llama-server is not where it should be — delete the folder %s, then run camne again", filepath.Dir(serverPath)) } + // Only after the new server is in place, so a failed upgrade keeps the old. + provision.RemoveOtherServers(serverPath) return nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 7bf7ab3..4c9cbb9 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -47,6 +47,10 @@ const idleSleep = "1800" type Client struct { base string // URL prefix; a placeholder host when dialing a unix socket hc *http.Client + // exited closes when a server this process started stops. nil for a + // server that was already running. + exited <-chan struct{} + logPath string } // runDir is the 0700 directory holding the socket, pidfile and server log. @@ -71,15 +75,19 @@ func pidPath(rd string) string { return filepath.Join(rd, "llama.pid") } // threads: decoding is memory-bandwidth-bound, not compute-bound, so half the // cores capped at 4 is the whatisit-validated default for the target box. -func threads() string { - n := runtime.NumCPU() / 2 +func threads() string { return strconv.Itoa(threadsFor(runtime.NumCPU())) } + +// threadsFor floors at 2 when the machine has 2 logical CPUs. Half of 2 is 1, +// and on a 2-vCPU t3.large -t 2 measured 3.6 s per query against 5.8 s at -t 1. +func threadsFor(cpus int) int { + n := cpus / 2 + if n < 2 { + n = min(cpus, 2) + } if n < 1 { n = 1 } - if n > 4 { - n = 4 - } - return strconv.Itoa(n) + return min(n, 4) } // Connect returns a client for the resident llama-server, starting one @@ -110,7 +118,8 @@ func Connect(serverPath, modelPath string) (c *Client, cold bool, err error) { }, hostArgs...) cmd := exec.Command(serverPath, args...) // llama-server chatter goes to a log file, never to the user's terminal. - if logf, lerr := os.OpenFile(filepath.Join(rd, "llama.log"), + c.logPath = filepath.Join(rd, "llama.log") + if logf, lerr := os.OpenFile(c.logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600); lerr == nil { cmd.Stdout, cmd.Stderr = logf, logf defer logf.Close() @@ -120,7 +129,12 @@ func Connect(serverPath, modelPath string) (c *Client, cold bool, err error) { return nil, false, fmt.Errorf("could not start llama-server — run `camne doctor` to see what is missing: %w", err) } os.WriteFile(pidPath(rd), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600) - cmd.Process.Release() + // A server that dies at start (a missing shared library, a bad model + // file) must fail in seconds, not after the whole WaitReady timeout. + // Wait never blocks camne's own exit; the server is in its own session. + exited := make(chan struct{}) + go func() { cmd.Wait(); close(exited) }() + c.exited = exited return c, true, nil } @@ -148,6 +162,11 @@ func (c *Client) WaitReady(timeout time.Duration) error { if ready, _ := c.health(); ready { return nil } + select { + case <-c.exited: // a nil channel never fires + return fmt.Errorf("llama-server stopped while starting: %s — run `camne doctor`, then ask again. If it keeps happening, open an issue at https://github.com/officialdad/camne/issues with this message", lastLine(c.logPath)) + default: + } if time.Now().After(deadline) { return errors.New("the model is still not ready — run `camne stop`, then ask again. If it keeps happening, `camne doctor` will show what is missing") } @@ -155,6 +174,17 @@ func (c *Client) WaitReady(timeout time.Duration) error { } } +// lastLine returns the last non-empty line of the server log, stripped to +// printable ASCII because it reaches the terminal. +func lastLine(path string) string { + b, _ := os.ReadFile(path) + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + if l := strings.TrimSpace(printable(lines[len(lines)-1])); l != "" { + return l + } + return "no error was recorded" +} + type completionReq struct { Prompt string `json:"prompt"` NPredict int `json:"n_predict"` diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index f787d02..91e534b 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -47,6 +49,42 @@ func TestWaitReadyTimeout(t *testing.T) { } } +func TestThreadsFor(t *testing.T) { + for _, tt := range []struct{ cpus, want int }{ + {0, 1}, {1, 1}, {2, 2}, {3, 2}, {4, 2}, {6, 3}, {8, 4}, {16, 4}, + } { + if got := threadsFor(tt.cpus); got != tt.want { + t.Errorf("threadsFor(%d) = %d, want %d", tt.cpus, got, tt.want) + } + } +} + +func TestWaitReadyServerExited(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + log := filepath.Join(t.TempDir(), "llama.log") + os.WriteFile(log, []byte("llama-server: error while loading shared libraries: libgomp.so.1\x1b[0m\n\n"), 0o600) + exited := make(chan struct{}) + close(exited) + c := testClient(srv) + c.exited, c.logPath = exited, log + + start := time.Now() + err := c.WaitReady(time.Minute) + if err == nil { + t.Fatal("expected error for an exited server, got nil") + } + if time.Since(start) > 5*time.Second { + t.Errorf("WaitReady took %v, want it to stop as soon as the server exits", time.Since(start)) + } + if !strings.Contains(err.Error(), "libgomp.so.1") || strings.Contains(err.Error(), "\x1b") { + t.Errorf("error = %q, want the log's last line with control bytes stripped", err) + } +} + func TestComplete(t *testing.T) { var got completionReq srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/provision/provision.go b/internal/provision/provision.go index b06f31c..2e78f2f 100644 --- a/internal/provision/provision.go +++ b/internal/provision/provision.go @@ -29,6 +29,12 @@ const ( ModelSize int64 = 986048032 ) +// linuxBuild is llamaBuild rebuilt by .github/workflows/llama.yml with +// GGML_OPENMP=OFF. Upstream's Ubuntu build needs libgomp.so.1, which minimal +// images (Ubuntu 26.04 cloud) lack, and llama-server then exits at start. +// macOS and Windows keep upstream's builds. +const linuxBuild = llamaBuild + "-noomp" + // Asset is one llama.cpp release file for a specific platform. type Asset struct { Name string // release asset file name @@ -38,15 +44,27 @@ type Asset struct { // URL is the direct release-asset download URL for the pinned build. func (a Asset) URL() string { + if strings.HasPrefix(a.Name, "llama-"+linuxBuild+"-") { + return "https://github.com/officialdad/camne/releases/download/llama-" + linuxBuild + "/" + a.Name + } return "https://github.com/ggml-org/llama.cpp/releases/download/" + llamaBuild + "/" + a.Name } +// dir is the versioned directory under bin/ the asset unpacks to: the name up +// to "-bin-", which is also the tar.gz entry prefix. A new build gets a new +// directory, so an install of the old one is replaced, never trusted. +func (a Asset) dir() string { + d, _, _ := strings.Cut(a.Name, "-bin-") + return d +} + // llamaAssets maps GOOS/GOARCH to the release asset for the pinned build. -// Digests and sizes come from the GitHub release API for that tag, hardcoded +// Digests and sizes come from the GitHub release API for that tag (Linux: the +// llama-b10333-noomp release on this repo), hardcoded // so provisioning never depends on the rate-limited API at runtime. var llamaAssets = map[string]Asset{ - "linux/amd64": {"llama-" + llamaBuild + "-bin-ubuntu-x64.tar.gz", "936ce04d98abe2a977e9dd2ff92659bb96947e136acee8f2bc3e21d8eaebbf23", 16507165}, - "linux/arm64": {"llama-" + llamaBuild + "-bin-ubuntu-arm64.tar.gz", "95da1a0f7538340f625b0301593ee63c046ec0a155c74f21444ea4d43bca79a1", 13377770}, + "linux/amd64": {"llama-" + linuxBuild + "-bin-ubuntu-x64.tar.gz", "5041ff232f7e5d19f5748fe93417f69c10ea0adb2e8c34ce10d0be80eaeeae88", 16266714}, + "linux/arm64": {"llama-" + linuxBuild + "-bin-ubuntu-arm64.tar.gz", "ecb4e32cb37ea8aac1404c3b7b36ac9cd86aa713bdbc9b8770d91738863f48ef", 13137461}, "darwin/amd64": {"llama-" + llamaBuild + "-bin-macos-x64.tar.gz", "6ffd9e0b9b2e3ab6ccfba332a74b968b0fef891f01bd4747d3d75bc7393877ea", 11290712}, "darwin/arm64": {"llama-" + llamaBuild + "-bin-macos-arm64.tar.gz", "e5d67c5264107e3c14d3bf2aee349365bb2b85ae99bb077a0cb974a1c4c2741a", 11015270}, "windows/amd64": {"llama-" + llamaBuild + "-bin-win-cpu-x64.zip", "563ab97cd003cf7cd4843677e6fea2f6162aee0c9580589c8d9e5816923076d5", 18399512}, @@ -88,7 +106,23 @@ func ServerPath() (string, error) { if err != nil { return "", err } - return filepath.Join(d, "bin", "llama-"+llamaBuild, serverBinary()), nil + a, err := LlamaAsset(runtime.GOOS, runtime.GOARCH) + if err != nil { + return "", err + } + return filepath.Join(d, "bin", a.dir(), serverBinary()), nil +} + +// RemoveOtherServers deletes every directory under bin/ except the one holding +// serverPath. Without it each llama.cpp upgrade strands ~40 MB on disk. +func RemoveOtherServers(serverPath string) { + keep := filepath.Dir(serverPath) + names, _ := filepath.Glob(filepath.Join(filepath.Dir(keep), "*")) + for _, n := range names { + if n != keep { + os.RemoveAll(n) + } + } } // ModelPath is where the GGUF lives once downloaded. diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go index 64b687d..d156c4e 100644 --- a/internal/provision/provision_test.go +++ b/internal/provision/provision_test.go @@ -1,14 +1,19 @@ package provision -import "testing" +import ( + "os" + "path/filepath" + "strings" + "testing" +) func TestLlamaAsset(t *testing.T) { tests := []struct { goos, goarch string want string // asset name, "" = expect error }{ - {"linux", "amd64", "llama-" + llamaBuild + "-bin-ubuntu-x64.tar.gz"}, - {"linux", "arm64", "llama-" + llamaBuild + "-bin-ubuntu-arm64.tar.gz"}, + {"linux", "amd64", "llama-" + linuxBuild + "-bin-ubuntu-x64.tar.gz"}, + {"linux", "arm64", "llama-" + linuxBuild + "-bin-ubuntu-arm64.tar.gz"}, {"darwin", "amd64", "llama-" + llamaBuild + "-bin-macos-x64.tar.gz"}, {"darwin", "arm64", "llama-" + llamaBuild + "-bin-macos-arm64.tar.gz"}, {"windows", "amd64", "llama-" + llamaBuild + "-bin-win-cpu-x64.zip"}, @@ -34,6 +39,35 @@ func TestLlamaAsset(t *testing.T) { if len(a.SHA256) != 64 || a.Size <= 0 { t.Errorf("%s/%s: incomplete pin: sha256 len %d, size %d", tt.goos, tt.goarch, len(a.SHA256), a.Size) } + // The tar.gz entry prefix must equal the directory ServerPath uses, + // and the Linux rebuild comes from camne's release, not upstream's. + wantDir, wantRepo := "llama-"+llamaBuild, "ggml-org/llama.cpp" + if tt.goos == "linux" { + wantDir, wantRepo = "llama-"+linuxBuild, "officialdad/camne" + } + if a.dir() != wantDir { + t.Errorf("%s/%s: dir %q, want %q", tt.goos, tt.goarch, a.dir(), wantDir) + } + if !strings.HasPrefix(a.URL(), "https://github.com/"+wantRepo+"/releases/download/") || !strings.HasSuffix(a.URL(), "/"+a.Name) { + t.Errorf("%s/%s: URL %q, want a %s release asset", tt.goos, tt.goarch, a.URL(), wantRepo) + } + } +} + +func TestRemoveOtherServers(t *testing.T) { + bin := t.TempDir() + current := filepath.Join(bin, "llama-"+linuxBuild, "llama-server") + old := filepath.Join(bin, "llama-"+llamaBuild, "llama-server") + for _, p := range []string{current, old} { + os.MkdirAll(filepath.Dir(p), 0o755) + os.WriteFile(p, []byte("x"), 0o755) + } + RemoveOtherServers(current) + if _, err := os.Stat(current); err != nil { + t.Errorf("current server removed: %v", err) + } + if _, err := os.Stat(filepath.Dir(old)); !os.IsNotExist(err) { + t.Errorf("old server directory still present: %v", err) } }