Add a local benchmark for /tldr with a dashboard
`go test -tags bench ./internal/tldr -run TestBench` runs the real /tldr pipeline against one or more Ollama models and prompt variants on a chat dump from disk (text copied from Discord, "Author: message" lines, or a DiscordChatExporter JSON export) and writes a self-contained HTML dashboard plus a JSON of the results. Scoring uses an optional facts file (a fact passes only when one bullet holds all its details) and automatic checks: numbers not in the chat, copying the chat, an English intro, cut-off output. TestFactsSkeleton proposes candidate facts from the chat to prune by hand. Chat dumps and results stay local: bench-out/ and the private folders are git-ignored. See internal/tldr/tldrbench/README.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
7ae1fa5747
commit
17548bbf7d
10 files changed
+1495
No files matched your search
@@ -0,0 +1,140 @@
|
||||
package tldrbench
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var (
|
||||
// Things a good overview has to get right: figures, times, money, days.
|
||||
figureRe = regexp.MustCompile(`\d+(?:[.,:]\d+)*(?:h\d*|€|%|k|gb|mb|min)?`)
|
||||
weekdayRe = regexp.MustCompile(`(?i)\b(segunda|ter[çc]a|quarta|quinta|sexta|s[áa]bado|domingo|monday|tuesday|wednesday|thursday|friday|saturday|sunday|amanh[ãa]|tomorrow|hoje|today)\b`)
|
||||
moneyRe = regexp.MustCompile(`(?i)€|\$|\beur\b|\beuros?\b`)
|
||||
wordRe = regexp.MustCompile(`[\p{L}][\p{L}'-]{5,}`)
|
||||
|
||||
// Common long words that make poor topic keywords.
|
||||
stopwords = map[string]bool{
|
||||
"porque": true, "também": true, "tambem": true, "depois": true, "ontem": true, "ainda": true, "estava": true,
|
||||
"podemos": true, "devíamos": true, "devemos": true, "fazer": true, "mesmo": true, "sempre": true, "ficou": true,
|
||||
"because": true, "should": true, "would": true, "could": true, "really": true, "maybe": true, "already": true,
|
||||
}
|
||||
)
|
||||
|
||||
// signals counts how many different "this matters" markers a line has.
|
||||
func signals(line string) int {
|
||||
n := 0
|
||||
if figureRe.MatchString(line) {
|
||||
n++
|
||||
}
|
||||
if weekdayRe.MatchString(line) {
|
||||
n++
|
||||
}
|
||||
if moneyRe.MatchString(line) {
|
||||
n++
|
||||
}
|
||||
if strings.Contains(line, "[link") || strings.Contains(line, "(preview:") {
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// SuggestFacts proposes candidate facts from transcript lines ("Author: text").
|
||||
// It does not understand the chat: it picks the lines that carry figures,
|
||||
// times, days, money or links, and for each suggests patterns (the figures and
|
||||
// days in it, plus one topic word). You are meant to prune and merge them into
|
||||
// the 10 to 20 facts that matter, then check the patterns against a real
|
||||
// overview. At most max candidates are returned, chronologically.
|
||||
func SuggestFacts(lines []string, max int) []Fact {
|
||||
type scored struct {
|
||||
idx, signals int
|
||||
line string
|
||||
}
|
||||
var cands []scored
|
||||
for i, ln := range lines {
|
||||
ln = strings.TrimSpace(ln)
|
||||
if ln == "" {
|
||||
continue
|
||||
}
|
||||
if s := signals(ln); s > 0 {
|
||||
cands = append(cands, scored{i, s, ln})
|
||||
}
|
||||
}
|
||||
// Keep the strongest, then restore chat order.
|
||||
sort.SliceStable(cands, func(a, b int) bool { return cands[a].signals > cands[b].signals })
|
||||
if max > 0 && len(cands) > max {
|
||||
cands = cands[:max]
|
||||
}
|
||||
sort.Slice(cands, func(a, b int) bool { return cands[a].idx < cands[b].idx })
|
||||
|
||||
var facts []Fact
|
||||
for n, c := range cands {
|
||||
facts = append(facts, Fact{
|
||||
Name: fmt.Sprintf("fact %d: %s", n+1, excerpt(c.line, 48)),
|
||||
All: suggestPatterns(c.line),
|
||||
Note: c.line,
|
||||
})
|
||||
}
|
||||
return facts
|
||||
}
|
||||
|
||||
func suggestPatterns(line string) []string {
|
||||
// Drop the "Author: " prefix so names do not become patterns.
|
||||
text := line
|
||||
if i := strings.Index(line, ": "); i > 0 && i <= 32 {
|
||||
text = line[i+2:]
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var pats []string
|
||||
add := func(p string) {
|
||||
key := strings.ToLower(p)
|
||||
if p == "" || len(pats) >= 3 {
|
||||
return
|
||||
}
|
||||
for have := range seen {
|
||||
if strings.Contains(have, key) || strings.Contains(key, have) {
|
||||
return // already covered, e.g. "segunda" next to "segunda-feira"
|
||||
}
|
||||
}
|
||||
seen[key] = true
|
||||
pats = append(pats, p)
|
||||
}
|
||||
|
||||
// A topic word first, so the fact stays tied to its subject.
|
||||
best := ""
|
||||
for _, w := range wordRe.FindAllString(text, -1) {
|
||||
lw := strings.ToLower(w)
|
||||
if stopwords[lw] || strings.ContainsAny(lw, "0123456789") {
|
||||
continue
|
||||
}
|
||||
if len([]rune(w)) > len([]rune(best)) {
|
||||
best = w
|
||||
}
|
||||
}
|
||||
if best != "" {
|
||||
add(regexp.QuoteMeta(strings.ToLower(trimWord(best))))
|
||||
}
|
||||
for _, d := range weekdayRe.FindAllString(text, -1) {
|
||||
add(regexp.QuoteMeta(strings.ToLower(d)))
|
||||
}
|
||||
for _, f := range figureRe.FindAllString(text, -1) {
|
||||
add(regexp.QuoteMeta(strings.ToLower(f)))
|
||||
}
|
||||
return pats
|
||||
}
|
||||
|
||||
func trimWord(w string) string {
|
||||
return strings.TrimFunc(w, func(r rune) bool { return !unicode.IsLetter(r) })
|
||||
}
|
||||
|
||||
func excerpt(s string, n int) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
Reference in new issue
Block a user