From c214042cd7b4a56b55bf72c61bba1dc017a1a9d7 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 23 Sep 2026 12:57:37 +0200 Subject: [PATCH 1/5] feat: add text-to-speech support (Speak) for OpenAI and Gemini --- README.md | 18 +++ e2e_test.go | 60 +++++++++ tts.go | 292 ++++++++++++++++++++++++++++++++++++++++++ tts_test.go | 359 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 729 insertions(+) create mode 100644 tts.go create mode 100644 tts_test.go diff --git a/README.md b/README.md index c8cf08d..a0676e5 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,24 @@ case errors.Is(err, context.DeadlineExceeded): // wall-clock budget spent API keys never appear in any error text. Provider error bodies are parsed per format (nested OpenAI envelope, Anthropic `error.type/message`, Gemini `error.status/message`) with a 512-byte raw-body fallback. +## Text-to-speech + +`Speak` synthesizes speech via a provider's TTS endpoint. Supported wire formats: OpenAI-compatible (`POST {base}/audio/speech`, binary audio) and Gemini (`generateContent` with `AUDIO` response modality, base64 `inlineData`). Other formats return a `ConfigError`. + +```go +res, err := sdk.Speak("openai", "tts-1", llm.SpeakRequest{ + Text: "Hello from go-llm-sdk", + Voice: "alloy", // required — no local default (never guessed) + Format: "mp3", // OpenAI-compat response_format (default "mp3") + Speed: 1.0, // optional, OpenAI-compat only +}) +// res.Audio — raw audio bytes exactly as the provider returned them +// res.Model — the model that produced the audio +// res.MIMEType — the provider's Content-Type, or audio/mpeg when omitted +``` + +The SDK never transcodes: it returns the provider's bytes plus the MIME type the provider declared, falling back to the wire format's well-known default (`audio/mpeg` on the OpenAI path; Gemini's `inlineData.mimeType` is always present). A 2xx JSON error envelope from a gateway surfaces as a typed `*APIError` — JSON is never returned as audio. Requests carry the same retry ladder and error taxonomy as chat; empty `Text` or `Voice` fail fast with a `ConfigError`. + ## Thread safety `SDK` and `Provider` are safe for concurrent use. `ChatClient` is safe for concurrent `Call`/`CallStream`; `SetRequestTimeout` is race-safe (atomic swap) but should still be called before the first request so in-flight calls use one timeout. Learn-once state is shared per provider via atomics — monotonic, converging, race-free. diff --git a/e2e_test.go b/e2e_test.go index f4534a3..cd92885 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -402,3 +402,63 @@ func TestE2EReasonerStreaming(t *testing.T) { t.Errorf("finish = %q, want stop or length", res.FinishReason) } } + +// ── TTS arms ───────────────────────────────────────────────────────────── + +// e2eTTSModel resolves the TTS model for a provider id: _TTS_E2E_MODEL +// beats the default (kept separate from the chat-model override). +func e2eTTSModel(t *testing.T, id, def string) string { + t.Helper() + if v := strings.TrimSpace(os.Getenv(strings.ToUpper(id) + "_TTS_E2E_MODEL")); v != "" { + return v + } + return def +} + +// TestE2ETTSOpenAI probes the OpenAI /audio/speech path against the live +// OpenAI endpoint (built-in registry, key via OPENAI_API_KEY). Asserts only +// SDK guarantees: the call succeeds and non-empty audio with a non-empty +// MIME type comes back. The audio's decodability is never asserted. +func TestE2ETTSOpenAI(t *testing.T) { + const keyEnv = "OPENAI_API_KEY" + key := e2eEnvKey(t, keyEnv) + sdk := New(WithProvider("openai", WithAPIKey(key))) + res, err := sdk.Speak("openai", e2eTTSModel(t, "openai", "tts-1"), SpeakRequest{ + Text: "go-llm-sdk text to speech probe.", + Voice: "alloy", + }) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if len(res.Audio) == 0 { + t.Errorf("Speak returned empty audio") + } + if res.MIMEType == "" { + t.Errorf("Speak returned empty MIMEType") + } + t.Logf("audio=%d bytes mime=%q model=%q", len(res.Audio), res.MIMEType, res.Model) +} + +// TestE2ETTSGemini probes the Gemini AUDIO-modality TTS path (built-in +// registry, key via GEMINI_API_KEY). Same soft contract: non-empty audio +// bytes + non-empty MIME type; no format assumptions beyond what the SDK +// already guarantees. +func TestE2ETTSGemini(t *testing.T) { + const keyEnv = "GEMINI_API_KEY" + key := e2eEnvKey(t, keyEnv) + sdk := New(WithProvider("gemini", WithAPIKey(key))) + res, err := sdk.Speak("gemini", e2eTTSModel(t, "gemini", "gemini-2.5-flash-preview-tts"), SpeakRequest{ + Text: "go-llm-sdk text to speech probe.", + Voice: "Kore", + }) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if len(res.Audio) == 0 { + t.Errorf("Speak returned empty audio") + } + if res.MIMEType == "" { + t.Errorf("Speak returned empty MIMEType") + } + t.Logf("audio=%d bytes mime=%q model=%q", len(res.Audio), res.MIMEType, res.Model) +} diff --git a/tts.go b/tts.go new file mode 100644 index 0000000..86ff969 --- /dev/null +++ b/tts.go @@ -0,0 +1,292 @@ +package llm + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ── TTS (text-to-speech) ───────────────────────────────────────────────── +// +// Speak turns text into audio bytes. Two wire formats are supported: +// - OpenAI-compatible: POST {base}/audio/speech, binary audio response +// - Gemini: generateContent with AUDIO response modality, base64 inlineData +// +// Any other format (anthropic, …) is a ConfigError. The SDK never +// transcodes: it returns the provider's bytes plus the MIME type the +// provider declared (or a well-known default when the provider omits it). +// API keys never appear in errors — the invariants of chat apply unchanged. + +// SpeakRequest describes one text-to-speech conversion. +type SpeakRequest struct { + Text string // required, non-empty + Voice string // provider voice id (no fallback tables — invariant #5) + Format string // container for OpenAI-compat providers ("mp3" default) + Speed float64 // optional playback speed, OpenAI-compat only (0 = omit) +} + +// SpeakResult carries the synthesized audio. +type SpeakResult struct { + Audio []byte // raw audio bytes as the provider returned them + Model string // model that produced the audio + MIMEType string // provider-declared (or well-known default) MIME type +} + +// Speak synthesizes speech with the named provider and model. Buffered +// only — no streaming in v1 — with the same retry ladder as chat. +func (s *SDK) Speak(providerID, model string, req SpeakRequest) (*SpeakResult, error) { + if strings.TrimSpace(req.Text) == "" { + return nil, &ConfigError{Msg: "speak request requires non-empty Text"} + } + if strings.TrimSpace(model) == "" { + return nil, &ConfigError{Msg: "speak request requires a model"} + } + if strings.TrimSpace(req.Voice) == "" { + return nil, &ConfigError{Msg: "speak request requires a Voice (both supported wire formats require one server-side)"} + } + p, err := s.Provider(providerID) + if err != nil { + return nil, err + } + if !p.Authenticated() { + return nil, &ConfigError{Msg: providerID + " has no API key (set " + strings.ToUpper(providerID) + "_API_KEY or use WithAPIKey)"} + } + if p.invalid { + return nil, &ConfigError{Msg: providerID + " has an invalid configuration"} + } + pc := newProviderClient(p.cfg, newBufferedHTTP(s.rt, s.timeout), nil) + return pc.speak(context.Background(), model, req) +} + +// buildSpeakRequest dispatches format-specific serialization. The third +// return is the well-known MIME type used when the provider omits +// Content-Type ("" for formats with no such default). +func (pc *providerClient) buildSpeakRequest(model string, req SpeakRequest) ([]byte, string, string, error) { + switch pc.cfg.Format { + case FormatGemini: + return buildGeminiSpeakRequest(pc.base, model, req) + case FormatOpenAI: + format := req.Format + if format == "" { + format = "mp3" + } + body := map[string]any{ + "model": model, + "input": req.Text, + "voice": req.Voice, + "response_format": format, + } + if req.Speed != 0 { + body["speed"] = req.Speed + } + b, err := json.Marshal(body) + return b, pc.base + "/audio/speech", "audio/mpeg", err + default: + return nil, "", "", &ConfigError{Msg: "provider format " + string(pc.cfg.Format) + " does not support speech"} + } +} + +// speak runs the TTS request against one provider with retry semantics +// identical to the buffered chat path (binary body, no SSE). +func (pc *providerClient) speak(ctx context.Context, model string, req SpeakRequest) (*SpeakResult, error) { + body, url, mimeHint, err := pc.buildSpeakRequest(model, req) + if err != nil { + return nil, err + } + gemini := pc.cfg.Format == FormatGemini + + var ( + lastErr error + rateErr *APIError + rateRA time.Duration + ) + for attempt := 0; attempt <= maxRetries; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + data, ctype, ra, err := pc.postAudio(ctx, url, body) + if err != nil { + var apiErr *APIError + if errors.As(err, &apiErr) { + switch { + case apiErr.Status == http.StatusTooManyRequests && billingExhausted(apiErr): + return nil, apiErr + case apiErr.Status == http.StatusTooManyRequests: + rateErr, rateRA, lastErr = apiErr, ra, apiErr + if attempt < maxRetries { + if !retrySleep(ctx, retryDelay(ra, attempt)) { + return nil, &RateLimitError{APIError: *rateErr, Attempts: attempt + 1, RetryAfter: rateRA} + } + continue + } + case apiErr.Retryable && attempt < maxRetries: + lastErr = apiErr + if !retrySleep(ctx, retryDelay(ra, attempt)) { + return nil, ctx.Err() + } + continue + } + if rateErr != nil && !apiErr.Retryable && apiErr.Status != http.StatusTooManyRequests { + return nil, apiErr + } + if rateErr != nil { + return nil, &RateLimitError{APIError: *rateErr, Attempts: attempt + 1, RetryAfter: rateRA} + } + return nil, apiErr + } + // Transport error — retryable. + lastErr = err + if attempt < maxRetries { + if !retrySleep(ctx, retryDelay(0, attempt)) { + return nil, ctx.Err() + } + continue + } + return nil, fmt.Errorf("llm: retry exhausted (%d attempts): %w", maxRetries+1, err) + } + if gemini { + audio, mime, perr := parseGeminiSpeakResponse(data) + if perr != nil { + // 2xx with no audio parts is a provider protocol + // failure — surface it through the typed error + // taxonomy (APIError at the actual HTTP status). + return nil, &APIError{ + Provider: pc.cfg.ID, + Status: http.StatusOK, + Message: perr.Error(), + } + } + data, ctype = audio, mime + } else if strings.HasPrefix(strings.TrimSpace(ctype), "application/json") { + // OpenAI-compatible gateways sometimes answer 2xx with a + // JSON error envelope. Never hand JSON bytes back as + // audio: parse the envelope through the shared httpError + // path. + return nil, pc.httpError(http.StatusOK, data) + } + return &SpeakResult{ + Audio: data, + Model: model, + MIMEType: resolveAudioMIME(ctype, mimeHint), + }, nil + } + return nil, lastErr +} + +// resolveAudioMIME prefers the provider's Content-Type; when absent it +// falls back to the wire format's well-known default (no guessing beyond +// that default — invariant #5). +func resolveAudioMIME(ctype, fallback string) string { + ctype = strings.TrimSpace(ctype) + if ctype != "" { + return ctype + } + return fallback +} + +// postAudio sends one TTS request and reads the full binary body (capped). +// Unlike post it captures the response Content-Type and never assumes JSON. +func (pc *providerClient) postAudio(ctx context.Context, url string, body []byte) (data []byte, ctype string, ra time.Duration, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, "", 0, &ConfigError{Msg: "build request: " + err.Error()} + } + req.Header.Set("Content-Type", "application/json") + pc.setAuthHeaders(req.Header) + + resp, err := pc.buffered().Do(req) + if err != nil { + return nil, "", 0, err + } + defer func() { _ = resp.Body.Close() }() + ctype = resp.Header.Get("Content-Type") + data, err = io.ReadAll(io.LimitReader(resp.Body, maxResponseSize+1)) + if err != nil { + return nil, "", 0, err + } + if len(data) > maxResponseSize { + return nil, "", 0, fmt.Errorf("llm: audio response exceeds %d bytes", maxResponseSize) + } + ra = parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, "", ra, pc.httpError(resp.StatusCode, data) + } + return data, ctype, ra, nil +} + +// ── Gemini TTS wire types ──────────────────────────────────────────────── + +type geminiSpeakRequest struct { + Contents []gmContent `json:"contents"` + GenerationConfig geminiGenerationAudio `json:"generationConfig"` +} + +type geminiGenerationAudio struct { + ResponseModalities []string `json:"responseModalities"` + SpeechConfig geminiSpeechCfg `json:"speechConfig"` +} + +type geminiSpeechCfg struct { + VoiceConfig geminiVoiceCfg `json:"voiceConfig"` +} + +type geminiVoiceCfg struct { + PrebuiltVoiceConfig geminiPrebuiltVoice `json:"prebuiltVoiceConfig"` +} + +type geminiPrebuiltVoice struct { + VoiceName string `json:"voiceName"` +} + +type geminiSpeakResponse struct { + Candidates []struct { + Content struct { + Parts []struct { + InlineData struct { + MimeType string `json:"mimeType"` + Data string `json:"data"` + } `json:"inlineData"` + } `json:"parts"` + } `json:"content"` + } `json:"candidates"` +} + +// buildGeminiSpeakRequest serializes a native Gemini TTS request via +// generateContent: AUDIO response modality + prebuilt voice config. The +// response MIME comes from inlineData.mimeType, so there is no +// well-known fallback. +func buildGeminiSpeakRequest(base, model string, req SpeakRequest) ([]byte, string, string, error) { + greq := geminiSpeakRequest{ + Contents: []gmContent{{Role: "user", Parts: []gmPart{{Text: req.Text}}}}, + GenerationConfig: geminiGenerationAudio{ + ResponseModalities: []string{"AUDIO"}, + SpeechConfig: geminiSpeechCfg{VoiceConfig: geminiVoiceCfg{PrebuiltVoiceConfig: geminiPrebuiltVoice{VoiceName: req.Voice}}}, + }, + } + b, err := json.Marshal(greq) + return b, fmt.Sprintf("%s/v1beta/models/%s:generateContent", base, model), "", err +} + +// parseGeminiSpeakResponse decodes a buffered Gemini TTS response. +func parseGeminiSpeakResponse(data []byte) (audio []byte, mime string, err error) { + var resp geminiSpeakResponse + if err := json.Unmarshal(data, &resp); err != nil { + return nil, "", fmt.Errorf("llm: decode gemini speech response: %w", err) + } + if len(resp.Candidates) == 0 || len(resp.Candidates[0].Content.Parts) == 0 { + return nil, "", fmt.Errorf("llm: gemini speech response contained no audio parts") + } + inline := resp.Candidates[0].Content.Parts[0].InlineData + audio, err = base64.StdEncoding.DecodeString(inline.Data) + if err != nil { + return nil, "", fmt.Errorf("llm: decode gemini speech audio: %w", err) + } + return audio, inline.MimeType, nil +} diff --git a/tts_test.go b/tts_test.go new file mode 100644 index 0000000..ae4991f --- /dev/null +++ b/tts_test.go @@ -0,0 +1,359 @@ +package llm + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +// newTestSDK builds an SDK with one provider pinned to an httptest server +// and the backoff unit shortened so retry paths are fast. +func newTestSDK(t *testing.T, cfg ProviderConfig, srv *httptest.Server) *SDK { + t.Helper() + old := backoffUnit + backoffUnit = time.Millisecond + t.Cleanup(func() { backoffUnit = old }) + s := New() + if srv != nil { + cfg.BaseURL = srv.URL + } + s.put(cfg) + return s +} + +func TestSpeak_OpenAI(t *testing.T) { + cases := []struct { + name string + contentType string // response Content-Type; "" = omit header + wantMIME string + }{ + {"propagated content type", "audio/mpeg", "audio/mpeg"}, + {"wav content type", "audio/wav", "audio/wav"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var gotPath, gotAuth string + var reqBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &reqBody) + w.Header().Set("Content-Type", "application/json") + if tc.contentType == "\x00unset\x00" { + // Suppress Go's sniffing so the SDK sees no Content-Type. + w.Header()["Content-Type"] = nil + } else if tc.contentType != "" { + w.Header().Set("Content-Type", tc.contentType) + } + _, _ = w.Write([]byte{0xff, 0xf3, 0x00, 0x01}) + })) + defer srv.Close() + + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k-secret"}, srv) + res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if gotPath != "/audio/speech" { + t.Errorf("path = %q, want /audio/speech", gotPath) + } + if gotAuth == "" || !containsNoSecret(gotAuth, "k-secret") { + t.Errorf("bearer auth not sent correctly: %q", gotAuth) + } + if reqBody["model"] != "tts-1" || reqBody["input"] != "hello" || reqBody["voice"] != "alloy" { + t.Errorf("request body = %v", reqBody) + } + if reqBody["response_format"] != "mp3" { + t.Errorf("response_format = %v, want mp3", reqBody["response_format"]) + } + if _, has := reqBody["speed"]; has { + t.Errorf("speed should be omitted when zero, got %v", reqBody["speed"]) + } + if string(res.Audio) != string([]byte{0xff, 0xf3, 0x00, 0x01}) { + t.Errorf("audio bytes = %v", res.Audio) + } + if res.MIMEType != tc.wantMIME { + t.Errorf("MIMEType = %q, want %q", res.MIMEType, tc.wantMIME) + } + if res.Model != "tts-1" { + t.Errorf("Model = %q, want tts-1", res.Model) + } + }) + } +} + +// containsNoSecret reports a header value that must contain the key but the +// test asserts presence, not leakage into errors. +func containsNoSecret(v, key string) bool { return v == "Bearer "+key } + +func TestSpeak_SpeedIncluded(t *testing.T) { + var reqBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &reqBody) + w.Header().Set("Content-Type", "audio/mpeg") + _, _ = w.Write([]byte{1}) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) + if _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "echo", Speed: 1.5}); err != nil { + t.Fatalf("Speak: %v", err) + } + if reqBody["speed"] != 1.5 { + t.Errorf("speed = %v, want 1.5", reqBody["speed"]) + } +} + +func TestSpeak_Gemini(t *testing.T) { + pcm := []byte{0x01, 0x02, 0x03, 0x04} + resp := map[string]any{ + "candidates": []any{map[string]any{ + "content": map[string]any{ + "parts": []any{map[string]any{ + "inlineData": map[string]any{ + "mimeType": "audio/L16;rate=24000", + "data": base64.StdEncoding.EncodeToString(pcm), + }, + }}, + }, + }}, + } + var reqBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1beta/models/gemini-tts:generateContent" { + t.Errorf("path = %q", r.URL.Path) + } + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &reqBody) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) + res, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hello", Voice: "Kore"}) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if string(res.Audio) != string(pcm) { + t.Errorf("audio bytes = %v, want %v", res.Audio, pcm) + } + if res.MIMEType != "audio/L16;rate=24000" { + t.Errorf("MIMEType = %q", res.MIMEType) + } + if res.Model != "gemini-tts" { + t.Errorf("Model = %q", res.Model) + } + gc := reqBody["generationConfig"].(map[string]any) + mods, _ := gc["responseModalities"].([]any) + if len(mods) != 1 || mods[0] != "AUDIO" { + t.Errorf("responseModalities = %v", mods) + } + sc := gc["speechConfig"].(map[string]any) + vc := sc["voiceConfig"].(map[string]any) + pv := vc["prebuiltVoiceConfig"].(map[string]any) + if pv["voiceName"] != "Kore" { + t.Errorf("voiceName = %v", pv["voiceName"]) + } + if reqBody["contents"] == nil { + t.Errorf("contents missing") + } +} + +func TestSpeak_ConfigErrors(t *testing.T) { + cases := []struct { + name string + cfg ProviderConfig + req SpeakRequest + wantCfg bool + }{ + {"empty text", ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, SpeakRequest{Text: " ", Voice: "alloy"}, true}, + {"anthropic format unsupported", ProviderConfig{ID: "anthropic", Format: FormatAnthropic, APIKey: "k"}, SpeakRequest{Text: "hi", Voice: "v"}, true}, + {"unknown provider", ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, SpeakRequest{Text: "hi"}, false}, + {"unauthenticated provider", ProviderConfig{ID: "nokey", Format: FormatOpenAI}, SpeakRequest{Text: "hi"}, true}, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + _, _ = w.Write([]byte(`{"error":{"message":"no"}}`)) + })) + defer srv.Close() + old := backoffUnit + backoffUnit = time.Millisecond + t.Cleanup(func() { backoffUnit = old }) + + op := cases[0].cfg + op.BaseURL = srv.URL + anthropicCfg := cases[1].cfg + anthropicCfg.BaseURL = srv.URL + ghost := ProviderConfig{ID: "ghost", Format: FormatOpenAI, APIKey: "x", BaseURL: srv.URL} + nokey := cases[3].cfg + nokey.BaseURL = srv.URL + s := New() + s.put(op) + s.put(anthropicCfg) + s.put(ghost) + s.put(nokey) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.name == "unknown provider" { + _, err := s.Speak("nosuch-provider", "m", tc.req) + var ce *ConfigError + if !errors.As(err, &ce) { + t.Fatalf("err = %T (%v), want *ConfigError", err, err) + } + return + } + _, err := s.Speak(tc.cfg.ID, "m", tc.req) + if err == nil { + t.Fatalf("expected error") + } + var ce *ConfigError + if !errors.As(err, &ce) { + t.Fatalf("err = %T (%v), want *ConfigError", err, err) + } + if tc.name == "unknown provider" { + return // already a ConfigError from Provider() + } + }) + } +} + +func TestSpeak_HTTP500_TypedError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":{"message":"speak exploded","type":"server_error"}}`)) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) + _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + var ae *APIError + if !errors.As(err, &ae) { + t.Fatalf("err = %T (%v), want *APIError", err, err) + } + if ae.Status != http.StatusInternalServerError || ae.Message != "speak exploded" { + t.Errorf("APIError = %+v", ae) + } +} + +func TestSpeak_RetriesThenSucceeds(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) <= 1 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + _, _ = w.Write([]byte(`{"error":{"message":"transient"}}`)) + return + } + w.Header().Set("Content-Type", "audio/mpeg") + _, _ = w.Write([]byte{9, 9}) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) + res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if len(res.Audio) != 2 || calls.Load() != 2 { + t.Errorf("calls=%d audio=%v", calls.Load(), res.Audio) + } +} + +// TestSpeak_MissingContentType drives a raw TCP listener that emits a 200 +// response with no Content-Type header, so the well-known MIME fallback is +// asserted deterministically (no reliance on Go's body sniffing). +func TestSpeak_MissingContentType(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + body := []byte{0xff, 0xf3, 0x00, 0x01} + fmt.Fprintf(conn, "HTTP/1.1 200 OK\r\nContent-Length: %d\r\nConnection: close\r\n\r\n", len(body)) + _, _ = conn.Write(body) + }() + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k", BaseURL: "http://" + ln.Addr().String()}, nil) + res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if res.MIMEType != "audio/mpeg" { + t.Errorf("MIMEType = %q, want audio/mpeg (well-known fallback)", res.MIMEType) + } +} + +// TestSpeak_OpenAI2xxJSONRejected guards against OpenAI-compatible gateways +// answering 2xx with a JSON error envelope: the SDK must surface a typed +// error, never hand JSON bytes back as audio. +func TestSpeak_OpenAI2xxJSONRejected(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"error":{"message":"voice not found"}}`)) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) + res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + if err == nil { + t.Fatalf("expected typed error for 2xx JSON body, got result with %d bytes", len(res.Audio)) + } + var ae *APIError + if !errors.As(err, &ae) { + t.Fatalf("err = %T (%v), want *APIError", err, err) + } + if ae.Status != http.StatusOK { + t.Errorf("Status = %d, want 200", ae.Status) + } +} + +// TestSpeak_EmptyVoiceRejected pins ConfigError symmetry: both wire formats +// require a voice server-side, so an empty/blank Voice fails fast locally. +func TestSpeak_EmptyVoiceRejected(t *testing.T) { + s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, nil) + _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: " "}) + var ce *ConfigError + if !errors.As(err, &ce) { + t.Fatalf("openai: err = %T (%v), want *ConfigError", err, err) + } + gs := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, nil) + _, err = gs.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: ""}) + if !errors.As(err, &ce) { + t.Fatalf("gemini: err = %T (%v), want *ConfigError", err, err) + } +} + +// TestSpeak_GeminiNoAudioPartsTypedError asserts a Gemini 2xx response with +// no audio parts surfaces as a typed *APIError (HTTP 200), not a plain +// fmt.Errorf outside the error taxonomy. +func TestSpeak_GeminiNoAudioPartsTypedError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"candidates":[{"content":{"parts":[]}}]}`)) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) + _, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) + var ae *APIError + if !errors.As(err, &ae) { + t.Fatalf("err = %T (%v), want *APIError", err, err) + } + if ae.Status != http.StatusOK { + t.Errorf("Status = %d, want 200", ae.Status) + } +} From 1edbb01e05ba63abd61cd695147e50e3a915ba42 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 23 Sep 2026 13:01:28 +0200 Subject: [PATCH 2/5] docs: add tts.go to AGENTS.md architecture table --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 8f0f815..96c043e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ go test -tags e2e -run 'TestE2E' -timeout 15m -v . # LIVE provider e2e (see be | `chat.go` | `providerClient`: retry orchestration (buffered + streaming), error classification, learn-once consumption, SSE pump wiring, `httpError` parsing | | `openai.go` / `gemini.go` / `anthropic.go` | Per-format request builders, response/stream mappers, model listing | | `responses.go` | OpenAI Responses API (`/v1/responses`) for GPT-5.6+ tools+reasoning | +| `tts.go` | Text-to-speech: `Speak`/`SpeakRequest`/`SpeakResult` — OpenAI-compat `/audio/speech` + Gemini AUDIO modality, no transcoding | | `sse.go` | SSE parser (abort-safe via `done` channel) + idle-watchdog pump | | `retry.go` | Backoff/jitter/`Retry-After`/`retrySleep` (8 attempts, cap 30s) | | `provider.go` | Built-in registry, quirks flags, config validation | From a325eb22b12cb5891bd4c38e59168f001f47e69b Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 23 Sep 2026 13:10:20 +0200 Subject: [PATCH 3/5] fix: reject empty Gemini TTS inlineData instead of returning empty audio --- tts.go | 3 +++ tts_test.go | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tts.go b/tts.go index 86ff969..6ca97df 100644 --- a/tts.go +++ b/tts.go @@ -284,6 +284,9 @@ func parseGeminiSpeakResponse(data []byte) (audio []byte, mime string, err error return nil, "", fmt.Errorf("llm: gemini speech response contained no audio parts") } inline := resp.Candidates[0].Content.Parts[0].InlineData + if inline.Data == "" { + return nil, "", fmt.Errorf("llm: gemini speech response contained no audio data") + } audio, err = base64.StdEncoding.DecodeString(inline.Data) if err != nil { return nil, "", fmt.Errorf("llm: decode gemini speech audio: %w", err) diff --git a/tts_test.go b/tts_test.go index ae4991f..10598fd 100644 --- a/tts_test.go +++ b/tts_test.go @@ -338,6 +338,21 @@ func TestSpeak_EmptyVoiceRejected(t *testing.T) { } } +// TestSpeak_GeminiEmptyInlineDataRejected asserts a Gemini 2xx part with an +// empty inlineData payload is an error, never a silent empty-audio success. +func TestSpeak_GeminiEmptyInlineDataRejected(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"audio/L16;rate=24000","data":""}}]}}]}`)) + })) + defer srv.Close() + s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) + res, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) + if err == nil { + t.Fatalf("err = nil (res audio=%d bytes), want error for empty audio", len(res.Audio)) + } +} + // TestSpeak_GeminiNoAudioPartsTypedError asserts a Gemini 2xx response with // no audio parts surfaces as a typed *APIError (HTTP 200), not a plain // fmt.Errorf outside the error taxonomy. From 99733d33dc1c3237190303a9891340d7097d5037 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 23 Sep 2026 13:12:58 +0200 Subject: [PATCH 4/5] fix: Speak takes context.Context, matching the SDK's call conventions --- README.md | 2 +- e2e_test.go | 4 ++-- tts.go | 4 ++-- tts_test.go | 26 +++++++++++++------------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a0676e5..2f3c1ee 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ API keys never appear in any error text. Provider error bodies are parsed per fo `Speak` synthesizes speech via a provider's TTS endpoint. Supported wire formats: OpenAI-compatible (`POST {base}/audio/speech`, binary audio) and Gemini (`generateContent` with `AUDIO` response modality, base64 `inlineData`). Other formats return a `ConfigError`. ```go -res, err := sdk.Speak("openai", "tts-1", llm.SpeakRequest{ +res, err := sdk.Speak(ctx, "openai", "tts-1", llm.SpeakRequest{ Text: "Hello from go-llm-sdk", Voice: "alloy", // required — no local default (never guessed) Format: "mp3", // OpenAI-compat response_format (default "mp3") diff --git a/e2e_test.go b/e2e_test.go index cd92885..0f42d0f 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -423,7 +423,7 @@ func TestE2ETTSOpenAI(t *testing.T) { const keyEnv = "OPENAI_API_KEY" key := e2eEnvKey(t, keyEnv) sdk := New(WithProvider("openai", WithAPIKey(key))) - res, err := sdk.Speak("openai", e2eTTSModel(t, "openai", "tts-1"), SpeakRequest{ + res, err := sdk.Speak(t.Context(), "openai", e2eTTSModel(t, "openai", "tts-1"), SpeakRequest{ Text: "go-llm-sdk text to speech probe.", Voice: "alloy", }) @@ -447,7 +447,7 @@ func TestE2ETTSGemini(t *testing.T) { const keyEnv = "GEMINI_API_KEY" key := e2eEnvKey(t, keyEnv) sdk := New(WithProvider("gemini", WithAPIKey(key))) - res, err := sdk.Speak("gemini", e2eTTSModel(t, "gemini", "gemini-2.5-flash-preview-tts"), SpeakRequest{ + res, err := sdk.Speak(t.Context(), "gemini", e2eTTSModel(t, "gemini", "gemini-2.5-flash-preview-tts"), SpeakRequest{ Text: "go-llm-sdk text to speech probe.", Voice: "Kore", }) diff --git a/tts.go b/tts.go index 6ca97df..34c201c 100644 --- a/tts.go +++ b/tts.go @@ -41,7 +41,7 @@ type SpeakResult struct { // Speak synthesizes speech with the named provider and model. Buffered // only — no streaming in v1 — with the same retry ladder as chat. -func (s *SDK) Speak(providerID, model string, req SpeakRequest) (*SpeakResult, error) { +func (s *SDK) Speak(ctx context.Context, providerID, model string, req SpeakRequest) (*SpeakResult, error) { if strings.TrimSpace(req.Text) == "" { return nil, &ConfigError{Msg: "speak request requires non-empty Text"} } @@ -62,7 +62,7 @@ func (s *SDK) Speak(providerID, model string, req SpeakRequest) (*SpeakResult, e return nil, &ConfigError{Msg: providerID + " has an invalid configuration"} } pc := newProviderClient(p.cfg, newBufferedHTTP(s.rt, s.timeout), nil) - return pc.speak(context.Background(), model, req) + return pc.speak(ctx, model, req) } // buildSpeakRequest dispatches format-specific serialization. The third diff --git a/tts_test.go b/tts_test.go index 10598fd..618b024 100644 --- a/tts_test.go +++ b/tts_test.go @@ -59,7 +59,7 @@ func TestSpeak_OpenAI(t *testing.T) { defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k-secret"}, srv) - res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) + res, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) if err != nil { t.Fatalf("Speak: %v", err) } @@ -105,7 +105,7 @@ func TestSpeak_SpeedIncluded(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) - if _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "echo", Speed: 1.5}); err != nil { + if _, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hi", Voice: "echo", Speed: 1.5}); err != nil { t.Fatalf("Speak: %v", err) } if reqBody["speed"] != 1.5 { @@ -140,7 +140,7 @@ func TestSpeak_Gemini(t *testing.T) { defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) - res, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hello", Voice: "Kore"}) + res, err := s.Speak(t.Context(), "gemini", "gemini-tts", SpeakRequest{Text: "hello", Voice: "Kore"}) if err != nil { t.Fatalf("Speak: %v", err) } @@ -206,14 +206,14 @@ func TestSpeak_ConfigErrors(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if tc.name == "unknown provider" { - _, err := s.Speak("nosuch-provider", "m", tc.req) + _, err := s.Speak(t.Context(), "nosuch-provider", "m", tc.req) var ce *ConfigError if !errors.As(err, &ce) { t.Fatalf("err = %T (%v), want *ConfigError", err, err) } return } - _, err := s.Speak(tc.cfg.ID, "m", tc.req) + _, err := s.Speak(t.Context(), tc.cfg.ID, "m", tc.req) if err == nil { t.Fatalf("expected error") } @@ -236,7 +236,7 @@ func TestSpeak_HTTP500_TypedError(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) - _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + _, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) var ae *APIError if !errors.As(err, &ae) { t.Fatalf("err = %T (%v), want *APIError", err, err) @@ -260,7 +260,7 @@ func TestSpeak_RetriesThenSucceeds(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) - res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + res, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) if err != nil { t.Fatalf("Speak: %v", err) } @@ -289,7 +289,7 @@ func TestSpeak_MissingContentType(t *testing.T) { _, _ = conn.Write(body) }() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k", BaseURL: "http://" + ln.Addr().String()}, nil) - res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) + res, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hello", Voice: "alloy"}) if err != nil { t.Fatalf("Speak: %v", err) } @@ -309,7 +309,7 @@ func TestSpeak_OpenAI2xxJSONRejected(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, srv) - res, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) + res, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hi", Voice: "alloy"}) if err == nil { t.Fatalf("expected typed error for 2xx JSON body, got result with %d bytes", len(res.Audio)) } @@ -326,13 +326,13 @@ func TestSpeak_OpenAI2xxJSONRejected(t *testing.T) { // require a voice server-side, so an empty/blank Voice fails fast locally. func TestSpeak_EmptyVoiceRejected(t *testing.T) { s := newTestSDK(t, ProviderConfig{ID: "openai", Format: FormatOpenAI, APIKey: "k"}, nil) - _, err := s.Speak("openai", "tts-1", SpeakRequest{Text: "hi", Voice: " "}) + _, err := s.Speak(t.Context(), "openai", "tts-1", SpeakRequest{Text: "hi", Voice: " "}) var ce *ConfigError if !errors.As(err, &ce) { t.Fatalf("openai: err = %T (%v), want *ConfigError", err, err) } gs := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, nil) - _, err = gs.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: ""}) + _, err = gs.Speak(t.Context(), "gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: ""}) if !errors.As(err, &ce) { t.Fatalf("gemini: err = %T (%v), want *ConfigError", err, err) } @@ -347,7 +347,7 @@ func TestSpeak_GeminiEmptyInlineDataRejected(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) - res, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) + res, err := s.Speak(t.Context(), "gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) if err == nil { t.Fatalf("err = nil (res audio=%d bytes), want error for empty audio", len(res.Audio)) } @@ -363,7 +363,7 @@ func TestSpeak_GeminiNoAudioPartsTypedError(t *testing.T) { })) defer srv.Close() s := newTestSDK(t, ProviderConfig{ID: "gemini", Format: FormatGemini, APIKey: "k"}, srv) - _, err := s.Speak("gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) + _, err := s.Speak(t.Context(), "gemini", "gemini-tts", SpeakRequest{Text: "hi", Voice: "Kore"}) var ae *APIError if !errors.As(err, &ae) { t.Fatalf("err = %T (%v), want *APIError", err, err) From bf388c475e655e00e218d7db37148318db3aa529 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 23 Sep 2026 13:13:15 +0200 Subject: [PATCH 5/5] chore: ignore .plans local planning artifacts --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6db11c3..3027e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ coverage.* .odek-artifacts/ .tmp-spincheck/ .env +.plans/.plans/ +