Add /tldr: a private overview of what you missed in a channel
/tldr [since] summarizes the messages in the channel it is run in that the caller has not been summarized yet, using a small local model on Ollama, and replies ephemerally. The answer streams into the reply as it is written. - Per-user, per-channel cursor (tldr_cursors) so each run picks up where the last left off; `since:` (e.g. 6h, 2d, max 7d) peeks without moving it. First use covers the last 24h, and a stale cursor never reaches back more than 7 days. - Only summarizes channels both the caller and the bot can read. - The transcript drops bots, system messages, reactions and filler, turns links, GIFs, images and files into short tags ([link: site], [image], [file: name], preview titles), marks replies, and writes a date marker where the chat paused for 3h or more so the model can keep topics apart. - The prompt asks for one bullet per conversation, oldest first, with a worked example; it was tuned against a synthetic chat and a real export with qwen3:4b-instruct-2507 (the default model). - One model call at a time across the bot, one /tldr per user. - docker-compose: Ollama service (CPU only, 8g memory cap: 6g killed the runner mid-request), model pull on startup. Needs the tldr_cursors migration (velox-dashboard-api 000016). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
FernandoJVideiraandClaude Opus 5.5 committed
1 parent
b1ea5f17e3
commit
7ae1fa5747
21 files changed
+2423
-2
No files matched your search
@@ -0,0 +1,237 @@
|
||||
// Package llm is a minimal client for a local Ollama server's chat API.
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// numCtx must match what the chunk budget in internal/tldr assumes.
|
||||
numCtx = 8192
|
||||
numPredict = 550
|
||||
temperature = 0.3
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
model string
|
||||
threads int
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// New builds a client. threads caps how many CPU threads the model may use
|
||||
// (0 lets Ollama decide). Always set it on a shared box: Ollama otherwise
|
||||
// starts one thread per core and they busy-wait, which both starves other
|
||||
// services and, under a container CPU quota, throttles inference to a crawl.
|
||||
func New(baseURL, model string, threads int) *Client {
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
model: model,
|
||||
threads: threads,
|
||||
// No client-level timeout: CPU inference can legitimately take
|
||||
// minutes, so callers bound it with the context instead.
|
||||
http: &http.Client{},
|
||||
}
|
||||
}
|
||||
|
||||
type chatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type chatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []chatMessage `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
Think *bool `json:"think,omitempty"`
|
||||
Options map[string]any `json:"options"`
|
||||
}
|
||||
|
||||
type chatResponse struct {
|
||||
Message chatMessage `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
var thinkTags = regexp.MustCompile(`(?s)<think>.*?</think>`)
|
||||
|
||||
// answer strips any inline reasoning (some models still emit it) so the user
|
||||
// only gets the reply.
|
||||
func answer(s string) string {
|
||||
return strings.TrimSpace(thinkTags.ReplaceAllString(s, ""))
|
||||
}
|
||||
|
||||
// visible is answer for text that is still being written: an unfinished
|
||||
// <think> block is hidden from where it starts.
|
||||
func visible(s string) string {
|
||||
s = thinkTags.ReplaceAllString(s, "")
|
||||
if i := strings.Index(s, "<think>"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
func statusError(status int, body []byte) error {
|
||||
return fmt.Errorf("ollama returned %d: %s", status, truncate(string(body), 300))
|
||||
}
|
||||
|
||||
// post sends one request and returns the raw response.
|
||||
func (c *Client) post(ctx context.Context, system, user string, stream bool, think *bool) (*http.Response, error) {
|
||||
options := map[string]any{
|
||||
"num_ctx": numCtx,
|
||||
"num_predict": numPredict,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if c.threads > 0 {
|
||||
options["num_thread"] = c.threads
|
||||
}
|
||||
|
||||
body, err := json.Marshal(chatRequest{
|
||||
Model: c.model,
|
||||
Messages: []chatMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
},
|
||||
Stream: stream,
|
||||
Think: think,
|
||||
Options: options,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/chat", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ollama unreachable: %w", err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// open returns a 200 response ready to read. Thinking is switched off because
|
||||
// a summary has no use for a reasoning preamble and on CPU it can cost most of
|
||||
// the time budget. Models that don't support the flag reject it, in which
|
||||
// case the request is repeated without it.
|
||||
func (c *Client) open(ctx context.Context, system, user string, stream bool) (*http.Response, error) {
|
||||
noThink := false
|
||||
resp, err := c.post(ctx, system, user, stream, &noThink)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusBadRequest {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.Contains(strings.ToLower(string(raw)), "think") {
|
||||
return nil, statusError(resp.StatusCode, raw)
|
||||
}
|
||||
if resp, err = c.post(ctx, system, user, stream, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, statusError(resp.StatusCode, raw)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Chat sends one system+user exchange and returns the model's whole reply.
|
||||
func (c *Client) Chat(ctx context.Context, system, user string) (string, error) {
|
||||
resp, err := c.open(ctx, system, user, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var parsed chatResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return "", fmt.Errorf("decoding ollama response: %w", err)
|
||||
}
|
||||
if parsed.Error != "" {
|
||||
return "", fmt.Errorf("ollama error: %s", parsed.Error)
|
||||
}
|
||||
|
||||
out := answer(parsed.Message.Content)
|
||||
if out == "" {
|
||||
return "", fmt.Errorf("ollama returned an empty reply")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ChatStream is Chat, but calls onText with the reply so far each time the
|
||||
// model writes more, so a caller can show it growing instead of waiting for
|
||||
// the end. onText may be nil. The returned string is the complete reply.
|
||||
func (c *Client) ChatStream(ctx context.Context, system, user string, onText func(string)) (string, error) {
|
||||
resp, err := c.open(ctx, system, user, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Ollama streams one JSON object per line.
|
||||
r := bufio.NewReader(resp.Body)
|
||||
var acc strings.Builder
|
||||
lastShown := ""
|
||||
|
||||
for {
|
||||
line, readErr := r.ReadBytes('\n')
|
||||
if len(bytes.TrimSpace(line)) > 0 {
|
||||
var parsed chatResponse
|
||||
if err := json.Unmarshal(line, &parsed); err != nil {
|
||||
return "", fmt.Errorf("decoding ollama stream: %w", err)
|
||||
}
|
||||
if parsed.Error != "" {
|
||||
return "", fmt.Errorf("ollama error: %s", parsed.Error)
|
||||
}
|
||||
if parsed.Message.Content != "" {
|
||||
acc.WriteString(parsed.Message.Content)
|
||||
if shown := visible(acc.String()); onText != nil && shown != "" && shown != lastShown {
|
||||
lastShown = shown
|
||||
onText(shown)
|
||||
}
|
||||
}
|
||||
if parsed.Done {
|
||||
break
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return "", fmt.Errorf("ollama stream interrupted: %w", readErr)
|
||||
}
|
||||
}
|
||||
|
||||
out := answer(acc.String())
|
||||
if out == "" {
|
||||
return "", fmt.Errorf("ollama returned an empty reply")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestChatRequestAndReply(t *testing.T) {
|
||||
var got chatRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/chat" || r.Method != http.MethodPost {
|
||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&got)
|
||||
_, _ = w.Write([]byte(`{"message":{"role":"assistant","content":"<think>hmm</think>\n the summary \n"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
out, err := New(srv.URL+"/", "some-model", 6).Chat(context.Background(), "sys", "usr")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out != "the summary" {
|
||||
t.Errorf("reply = %q (think block and whitespace should be stripped)", out)
|
||||
}
|
||||
if got.Model != "some-model" || got.Stream || len(got.Messages) != 2 ||
|
||||
got.Messages[0].Role != "system" || got.Messages[1].Content != "usr" {
|
||||
t.Errorf("request = %+v", got)
|
||||
}
|
||||
if got.Options["num_ctx"].(float64) != numCtx {
|
||||
t.Errorf("num_ctx = %v", got.Options["num_ctx"])
|
||||
}
|
||||
if got.Options["num_thread"].(float64) != 6 {
|
||||
t.Errorf("num_thread = %v, want 6", got.Options["num_thread"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatOmitsThreadsWhenUnset(t *testing.T) {
|
||||
var got chatRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&got)
|
||||
_, _ = w.Write([]byte(`{"message":{"content":"ok"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := New(srv.URL, "m", 0).Chat(context.Background(), "s", "u"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, set := got.Options["num_thread"]; set {
|
||||
t.Error("num_thread must be left to Ollama when threads is 0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatErrors(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
status int
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
"http error": {http.StatusNotFound, `model not found`, "404"},
|
||||
"api error": {http.StatusOK, `{"error":"out of memory"}`, "out of memory"},
|
||||
"empty reply": {http.StatusOK, `{"message":{"content":"<think>x</think>"}}`, "empty"},
|
||||
"bad json": {http.StatusOK, `nope`, "decoding"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(c.status)
|
||||
_, _ = w.Write([]byte(c.body))
|
||||
}))
|
||||
_, err := New(srv.URL, "m", 0).Chat(context.Background(), "s", "u")
|
||||
srv.Close()
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: err = %v, want containing %q", name, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := New("http://127.0.0.1:1", "m", 0).Chat(context.Background(), "s", "u"); err == nil || !strings.Contains(err.Error(), "unreachable") {
|
||||
t.Errorf("unreachable server: err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatDisablesThinkingAndFallsBack(t *testing.T) {
|
||||
var seen []bool // whether each request carried think:false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req chatRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
hasThink := req.Think != nil && !*req.Think
|
||||
seen = append(seen, hasThink)
|
||||
if hasThink {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":"\"some-model\" does not support thinking"}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"message":{"content":"fine"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
out, err := New(srv.URL, "some-model", 0).Chat(context.Background(), "s", "u")
|
||||
if err != nil || out != "fine" {
|
||||
t.Fatalf("out=%q err=%v", out, err)
|
||||
}
|
||||
if len(seen) != 2 || !seen[0] || seen[1] {
|
||||
t.Errorf("want one request with think:false then a retry without it, got %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatDoesNotRetryOtherBadRequests(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
calls++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":"model is required"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := New(srv.URL, "", 0).Chat(context.Background(), "s", "u"); err == nil || calls != 1 {
|
||||
t.Errorf("err=%v calls=%d, want an error after exactly one call", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
// streamServer replies to /api/chat with the given NDJSON lines, flushing
|
||||
// after each so the client sees them as separate reads.
|
||||
func streamServer(t *testing.T, gotReq *chatRequest, lines ...string) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if gotReq != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(gotReq)
|
||||
}
|
||||
for _, l := range lines {
|
||||
_, _ = w.Write([]byte(l + "\n"))
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
func TestChatStreamReportsGrowingTextAndHidesThinking(t *testing.T) {
|
||||
var req chatRequest
|
||||
srv := streamServer(t, &req,
|
||||
`{"message":{"content":"<think>pl"},"done":false}`,
|
||||
`{"message":{"content":"an</think>Olá"},"done":false}`,
|
||||
`{"message":{"content":" mundo"},"done":false}`,
|
||||
`{"message":{"content":""},"done":true}`,
|
||||
)
|
||||
defer srv.Close()
|
||||
|
||||
var seen []string
|
||||
out, err := New(srv.URL, "m", 6).ChatStream(context.Background(), "s", "u", func(s string) { seen = append(seen, s) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out != "Olá mundo" {
|
||||
t.Errorf("final = %q", out)
|
||||
}
|
||||
// The first chunk is only an unfinished think block, so nothing is shown for it.
|
||||
if strings.Join(seen, "|") != "Olá|Olá mundo" {
|
||||
t.Errorf("callbacks = %v", seen)
|
||||
}
|
||||
if !req.Stream {
|
||||
t.Error("request must ask for stream:true")
|
||||
}
|
||||
if req.Think == nil || *req.Think {
|
||||
t.Error("request must carry think:false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatStreamWorksWithoutCallback(t *testing.T) {
|
||||
srv := streamServer(t, nil, `{"message":{"content":"ok"},"done":true}`)
|
||||
defer srv.Close()
|
||||
if out, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err != nil || out != "ok" {
|
||||
t.Errorf("out=%q err=%v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatStreamErrors(t *testing.T) {
|
||||
srv := streamServer(t, nil, `{"message":{"content":"par"},"done":false}`, `{"error":"model crashed"}`)
|
||||
if _, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err == nil || !strings.Contains(err.Error(), "model crashed") {
|
||||
t.Errorf("mid-stream error: %v", err)
|
||||
}
|
||||
srv.Close()
|
||||
|
||||
srv = streamServer(t, nil, `not json`)
|
||||
if _, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err == nil || !strings.Contains(err.Error(), "decoding") {
|
||||
t.Errorf("bad json: %v", err)
|
||||
}
|
||||
srv.Close()
|
||||
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = w.Write([]byte("model not found"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
if _, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err == nil || !strings.Contains(err.Error(), "404") {
|
||||
t.Errorf("bad status: %v", err)
|
||||
}
|
||||
|
||||
empty := streamServer(t, nil, `{"message":{"content":"<think>only</think>"},"done":true}`)
|
||||
defer empty.Close()
|
||||
if _, err := New(empty.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err == nil || !strings.Contains(err.Error(), "empty") {
|
||||
t.Errorf("empty reply: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatStreamKeepsWhatArrivedIfTheStreamEndsEarly(t *testing.T) {
|
||||
srv := streamServer(t, nil, `{"message":{"content":"half an ans"},"done":false}`) // no done:true
|
||||
defer srv.Close()
|
||||
if out, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil); err != nil || out != "half an ans" {
|
||||
t.Errorf("out=%q err=%v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatStreamFallsBackWhenThinkingUnsupported(t *testing.T) {
|
||||
var seen []bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req chatRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
hasThink := req.Think != nil
|
||||
seen = append(seen, hasThink)
|
||||
if hasThink {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":"\"m\" does not support thinking"}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"message":{"content":"fine"},"done":true}` + "\n"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
out, err := New(srv.URL, "m", 0).ChatStream(context.Background(), "s", "u", nil)
|
||||
if err != nil || out != "fine" || len(seen) != 2 || !seen[0] || seen[1] {
|
||||
t.Errorf("out=%q err=%v seen=%v", out, err, seen)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user