// Package llm is a minimal client for a local Ollama server's chat API.
package llm
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
)
const (
// numCtx must match what the chunk budget in internal/tldr assumes.
numCtx = 8192
numPredict = 550
temperature = 0.3
)
type Client struct {
baseURL string
model string
threads int
http *http.Client
}
// New builds a client. threads caps how many CPU threads the model may use
// (0 lets Ollama decide). Always set it on a shared box: Ollama otherwise
// starts one thread per core and they busy-wait, which both starves other
// services and, under a container CPU quota, throttles inference to a crawl.
func New(baseURL, model string, threads int) *Client {
return &Client{
baseURL: strings.TrimRight(baseURL, "/"),
model: model,
threads: threads,
// No client-level timeout: CPU inference can legitimately take
// minutes, so callers bound it with the context instead.
http: &http.Client{},
}
}
type chatMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type chatRequest struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Stream bool `json:"stream"`
Think *bool `json:"think,omitempty"`
Options map[string]any `json:"options"`
}
type chatResponse struct {
Message chatMessage `json:"message"`
Done bool `json:"done"`
Error string `json:"error"`
}
var thinkTags = regexp.MustCompile(`(?s).*?`)
// answer strips any inline reasoning (some models still emit it) so the user
// only gets the reply.
func answer(s string) string {
return strings.TrimSpace(thinkTags.ReplaceAllString(s, ""))
}
// visible is answer for text that is still being written: an unfinished
// block is hidden from where it starts.
func visible(s string) string {
s = thinkTags.ReplaceAllString(s, "")
if i := strings.Index(s, ""); i >= 0 {
s = s[:i]
}
return strings.TrimSpace(s)
}
func statusError(status int, body []byte) error {
return fmt.Errorf("ollama returned %d: %s", status, truncate(string(body), 300))
}
// post sends one request and returns the raw response.
func (c *Client) post(ctx context.Context, system, user string, stream bool, think *bool) (*http.Response, error) {
options := map[string]any{
"num_ctx": numCtx,
"num_predict": numPredict,
"temperature": temperature,
}
if c.threads > 0 {
options["num_thread"] = c.threads
}
body, err := json.Marshal(chatRequest{
Model: c.model,
Messages: []chatMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
},
Stream: stream,
Think: think,
Options: options,
})
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/chat", bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("ollama unreachable: %w", err)
}
return resp, nil
}
// open returns a 200 response ready to read. Thinking is switched off because
// a summary has no use for a reasoning preamble and on CPU it can cost most of
// the time budget. Models that don't support the flag reject it, in which
// case the request is repeated without it.
func (c *Client) open(ctx context.Context, system, user string, stream bool) (*http.Response, error) {
noThink := false
resp, err := c.post(ctx, system, user, stream, &noThink)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusBadRequest {
raw, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if !strings.Contains(strings.ToLower(string(raw)), "think") {
return nil, statusError(resp.StatusCode, raw)
}
if resp, err = c.post(ctx, system, user, stream, nil); err != nil {
return nil, err
}
}
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, statusError(resp.StatusCode, raw)
}
return resp, nil
}
// Chat sends one system+user exchange and returns the model's whole reply.
func (c *Client) Chat(ctx context.Context, system, user string) (string, error) {
resp, err := c.open(ctx, system, user, false)
if err != nil {
return "", err
}
defer resp.Body.Close()
raw, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
var parsed chatResponse
if err := json.Unmarshal(raw, &parsed); err != nil {
return "", fmt.Errorf("decoding ollama response: %w", err)
}
if parsed.Error != "" {
return "", fmt.Errorf("ollama error: %s", parsed.Error)
}
out := answer(parsed.Message.Content)
if out == "" {
return "", fmt.Errorf("ollama returned an empty reply")
}
return out, nil
}
// ChatStream is Chat, but calls onText with the reply so far each time the
// model writes more, so a caller can show it growing instead of waiting for
// the end. onText may be nil. The returned string is the complete reply.
func (c *Client) ChatStream(ctx context.Context, system, user string, onText func(string)) (string, error) {
resp, err := c.open(ctx, system, user, true)
if err != nil {
return "", err
}
defer resp.Body.Close()
// Ollama streams one JSON object per line.
r := bufio.NewReader(resp.Body)
var acc strings.Builder
lastShown := ""
for {
line, readErr := r.ReadBytes('\n')
if len(bytes.TrimSpace(line)) > 0 {
var parsed chatResponse
if err := json.Unmarshal(line, &parsed); err != nil {
return "", fmt.Errorf("decoding ollama stream: %w", err)
}
if parsed.Error != "" {
return "", fmt.Errorf("ollama error: %s", parsed.Error)
}
if parsed.Message.Content != "" {
acc.WriteString(parsed.Message.Content)
if shown := visible(acc.String()); onText != nil && shown != "" && shown != lastShown {
lastShown = shown
onText(shown)
}
}
if parsed.Done {
break
}
}
if readErr != nil {
if readErr == io.EOF {
break
}
return "", fmt.Errorf("ollama stream interrupted: %w", readErr)
}
}
out := answer(acc.String())
if out == "" {
return "", fmt.Errorf("ollama returned an empty reply")
}
return out, nil
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}