Add a local benchmark for /tldr with a dashboard
`go test -tags bench ./internal/tldr -run TestBench` runs the real /tldr pipeline against one or more Ollama models and prompt variants on a chat dump from disk (text copied from Discord, "Author: message" lines, or a DiscordChatExporter JSON export) and writes a self-contained HTML dashboard plus a JSON of the results. Scoring uses an optional facts file (a fact passes only when one bullet holds all its details) and automatic checks: numbers not in the chat, copying the chat, an English intro, cut-off output. TestFactsSkeleton proposes candidate facts from the chat to prune by hand. Chat dumps and results stay local: bench-out/ and the private folders are git-ignored. See internal/tldr/tldrbench/README.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
FernandoJVideiraandClaude Opus 5.5 committed
1 parent
7ae1fa5747
commit
17548bbf7d
10 files changed
+1495
No files matched your search
@@ -0,0 +1,261 @@
|
||||
package tldrbench
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"html/template"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ChatInfo describes the chat that was summarized, without containing any of it.
|
||||
type ChatInfo struct {
|
||||
Source string `json:"source"` // file name only
|
||||
Messages int `json:"messages"`
|
||||
Kept int `json:"kept"`
|
||||
Chunks int `json:"chunks"`
|
||||
InputChars int `json:"inputChars"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
|
||||
// Run is one model writing one overview with one prompt.
|
||||
type Run struct {
|
||||
Model string `json:"model"`
|
||||
Variant string `json:"variant"`
|
||||
N int `json:"n"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
FirstText float64 `json:"firstText"` // seconds until the first streamed text, 0 if never
|
||||
Output string `json:"output,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Score
|
||||
}
|
||||
|
||||
// Report is everything the dashboard shows.
|
||||
type Report struct {
|
||||
When time.Time `json:"when"`
|
||||
Chat ChatInfo `json:"chat"`
|
||||
Threads int `json:"threads"`
|
||||
Facts []string `json:"facts,omitempty"`
|
||||
Models []string `json:"models"`
|
||||
Variants []string `json:"variants"`
|
||||
Runs []Run `json:"runs"`
|
||||
}
|
||||
|
||||
// WriteJSON saves the raw results so they can be re-rendered or compared later.
|
||||
func (r *Report) WriteJSON(path string) error {
|
||||
raw, err := json.MarshalIndent(r, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, raw, 0o600)
|
||||
}
|
||||
|
||||
// ReadReport loads a report written by WriteJSON.
|
||||
func ReadReport(path string) (*Report, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r Report
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// Cell aggregates the runs of one model with one prompt.
|
||||
type Cell struct {
|
||||
Runs, Failed int
|
||||
Seconds, FirstText, Facts float64
|
||||
Invented float64
|
||||
Copied, English, CutOff int
|
||||
HitCount map[string]int
|
||||
HasFacts bool
|
||||
Class string // ok, mid, no, na
|
||||
SecondsMin, SecondsMax float64
|
||||
FactsTotal int
|
||||
FactsPct int
|
||||
Flags []string
|
||||
}
|
||||
|
||||
func aggregate(runs []Run) Cell {
|
||||
c := Cell{HitCount: map[string]int{}, SecondsMin: -1}
|
||||
var ok int
|
||||
for _, r := range runs {
|
||||
c.Runs++
|
||||
if r.Error != "" {
|
||||
c.Failed++
|
||||
continue
|
||||
}
|
||||
ok++
|
||||
c.Seconds += r.Seconds
|
||||
c.FirstText += r.FirstText
|
||||
c.Facts += float64(r.FactsHit)
|
||||
c.Invented += float64(len(r.Invented))
|
||||
c.FactsTotal = r.FactsTotal
|
||||
if r.Copied {
|
||||
c.Copied++
|
||||
}
|
||||
if r.EnglishIntro {
|
||||
c.English++
|
||||
}
|
||||
if r.CutOff {
|
||||
c.CutOff++
|
||||
}
|
||||
for _, h := range r.Hit {
|
||||
c.HitCount[h]++
|
||||
}
|
||||
if c.SecondsMin < 0 || r.Seconds < c.SecondsMin {
|
||||
c.SecondsMin = r.Seconds
|
||||
}
|
||||
if r.Seconds > c.SecondsMax {
|
||||
c.SecondsMax = r.Seconds
|
||||
}
|
||||
}
|
||||
if ok > 0 {
|
||||
n := float64(ok)
|
||||
c.Seconds /= n
|
||||
c.FirstText /= n
|
||||
c.Facts /= n
|
||||
c.Invented /= n
|
||||
}
|
||||
c.HasFacts = c.FactsTotal > 0
|
||||
c.Class = "na"
|
||||
if ok > 0 && c.HasFacts {
|
||||
c.FactsPct = int(100*c.Facts/float64(c.FactsTotal) + 0.5)
|
||||
switch {
|
||||
case c.FactsPct >= 80:
|
||||
c.Class = "ok"
|
||||
case c.FactsPct >= 55:
|
||||
c.Class = "mid"
|
||||
default:
|
||||
c.Class = "no"
|
||||
}
|
||||
}
|
||||
if c.Copied > 0 {
|
||||
c.Flags = append(c.Flags, "copies the chat")
|
||||
}
|
||||
if c.English > 0 {
|
||||
c.Flags = append(c.Flags, "English intro")
|
||||
}
|
||||
if c.CutOff > 0 {
|
||||
c.Flags = append(c.Flags, "may be cut off")
|
||||
}
|
||||
if c.Invented > 0 {
|
||||
c.Flags = append(c.Flags, "numbers not in the chat")
|
||||
}
|
||||
if c.Failed > 0 {
|
||||
c.Flags = append(c.Flags, "failed runs")
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type viewRow struct {
|
||||
Variant string
|
||||
Cells []Cell
|
||||
}
|
||||
|
||||
type viewModelFacts struct {
|
||||
Model string
|
||||
Rows []struct {
|
||||
Variant string
|
||||
Hits []int
|
||||
Runs int
|
||||
}
|
||||
}
|
||||
|
||||
type view struct {
|
||||
Report
|
||||
Rows []viewRow
|
||||
FactGrid []viewModelFacts
|
||||
BestModel string
|
||||
BestVar string
|
||||
HasText bool
|
||||
RunsSorted []Run
|
||||
}
|
||||
|
||||
// WriteHTML renders a self-contained dashboard: inline styles, no scripts, no
|
||||
// external requests.
|
||||
func (r *Report) WriteHTML(w io.Writer) error {
|
||||
v := view{Report: *r}
|
||||
cell := map[[2]string][]Run{}
|
||||
for _, run := range r.Runs {
|
||||
k := [2]string{run.Model, run.Variant}
|
||||
cell[k] = append(cell[k], run)
|
||||
if run.Output != "" {
|
||||
v.HasText = true
|
||||
}
|
||||
}
|
||||
best := -1.0
|
||||
for _, variant := range r.Variants {
|
||||
row := viewRow{Variant: variant}
|
||||
for _, model := range r.Models {
|
||||
c := aggregate(cell[[2]string{model, variant}])
|
||||
row.Cells = append(row.Cells, c)
|
||||
if c.HasFacts && c.Runs > c.Failed {
|
||||
score := c.Facts - c.Seconds/1000 // ties go to the faster one
|
||||
if score > best {
|
||||
best, v.BestModel, v.BestVar = score, model, variant
|
||||
}
|
||||
}
|
||||
}
|
||||
v.Rows = append(v.Rows, row)
|
||||
}
|
||||
if len(r.Facts) > 0 {
|
||||
for _, model := range r.Models {
|
||||
mf := viewModelFacts{Model: model}
|
||||
for _, variant := range r.Variants {
|
||||
c := aggregate(cell[[2]string{model, variant}])
|
||||
row := struct {
|
||||
Variant string
|
||||
Hits []int
|
||||
Runs int
|
||||
}{Variant: variant, Runs: c.Runs - c.Failed}
|
||||
for _, f := range r.Facts {
|
||||
row.Hits = append(row.Hits, c.HitCount[f])
|
||||
}
|
||||
mf.Rows = append(mf.Rows, row)
|
||||
}
|
||||
v.FactGrid = append(v.FactGrid, mf)
|
||||
}
|
||||
}
|
||||
v.RunsSorted = append(v.RunsSorted, r.Runs...)
|
||||
sort.SliceStable(v.RunsSorted, func(i, j int) bool {
|
||||
a, b := v.RunsSorted[i], v.RunsSorted[j]
|
||||
if a.Model != b.Model {
|
||||
return indexOf(r.Models, a.Model) < indexOf(r.Models, b.Model)
|
||||
}
|
||||
if a.Variant != b.Variant {
|
||||
return indexOf(r.Variants, a.Variant) < indexOf(r.Variants, b.Variant)
|
||||
}
|
||||
return a.N < b.N
|
||||
})
|
||||
return page.Execute(w, v)
|
||||
}
|
||||
|
||||
func indexOf(list []string, s string) int {
|
||||
for i, x := range list {
|
||||
if x == s {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return len(list)
|
||||
}
|
||||
|
||||
var page = template.Must(template.New("page").Funcs(template.FuncMap{
|
||||
"inc": func(n int) int { return n + 1 },
|
||||
"f1": func(f float64) string { return fmtFloat(f, 1) },
|
||||
"sec": func(f float64) string { return fmtFloat(f, 0) },
|
||||
"hitClass": func(hits, runs int) string {
|
||||
switch {
|
||||
case runs == 0:
|
||||
return "h0"
|
||||
case hits == runs:
|
||||
return "h2"
|
||||
case hits > 0:
|
||||
return "h1"
|
||||
}
|
||||
return "h0"
|
||||
},
|
||||
}).Parse(pageHTML))
|
||||
Reference in new issue
Block a user