Add a local benchmark for /tldr with a dashboard
`go test -tags bench ./internal/tldr -run TestBench` runs the real /tldr pipeline against one or more Ollama models and prompt variants on a chat dump from disk (text copied from Discord, "Author: message" lines, or a DiscordChatExporter JSON export) and writes a self-contained HTML dashboard plus a JSON of the results. Scoring uses an optional facts file (a fact passes only when one bullet holds all its details) and automatic checks: numbers not in the chat, copying the chat, an English intro, cut-off output. TestFactsSkeleton proposes candidate facts from the chat to prune by hand. Chat dumps and results stay local: bench-out/ and the private folders are git-ignored. See internal/tldr/tldrbench/README.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
FernandoJVideiraandClaude Opus 5.5 committed
1 parent
7ae1fa5747
commit
17548bbf7d
10 files changed
+1495
No files matched your search
@@ -0,0 +1,64 @@
|
||||
package tldrbench
|
||||
|
||||
import "strconv"
|
||||
|
||||
func fmtFloat(f float64, prec int) string { return strconv.FormatFloat(f, 'f', prec, 64) }
|
||||
|
||||
const pageHTML = `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>/tldr benchmark</title>
|
||||
<style>
|
||||
:root{--bg:#f4f6f8;--surface:#fff;--surface-2:#eceff3;--fg:#16202b;--muted:#5b6876;--line:#d9dfe6;--accent:#2f5fd0;--good:#1b7f4b;--good-bg:#e0f3e8;--warn:#a35a00;--warn-bg:#fbecd2;--bad:#b3261e;--bad-bg:#fbe2df;color-scheme:light}
|
||||
@media (prefers-color-scheme:dark){:root{--bg:#0f151c;--surface:#17202a;--surface-2:#1f2a36;--fg:#e6ecf2;--muted:#93a1b0;--line:#2a3745;--accent:#7fa2ff;--good:#5fd391;--good-bg:#153a28;--warn:#f0b45a;--warn-bg:#3d2d12;--bad:#ff8d84;--bad-bg:#44201d;color-scheme:dark}}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif;padding:32px max(16px,env(safe-area-inset-right,0px)) 64px max(16px,env(safe-area-inset-left,0px))}
|
||||
.wrap{max-width:1100px;margin:0 auto;display:flex;flex-direction:column;gap:28px}
|
||||
h1{font-size:1.9rem;margin:0;letter-spacing:-.02em}h2{font-size:1.1rem;margin:0 0 6px}
|
||||
.eyebrow{font:500 .74rem ui-monospace,Menlo,monospace;letter-spacing:.08em;text-transform:uppercase;color:var(--muted)}
|
||||
.sub,.note{color:var(--muted)}.note{font-size:.85rem}
|
||||
.panel{background:var(--surface);border:1px solid var(--line);border-radius:14px;padding:18px 20px;min-width:0}
|
||||
.tablewrap{overflow-x:auto}
|
||||
table{border-collapse:collapse;width:100%;font-size:.92rem}
|
||||
th{text-align:left;font:500 .76rem ui-monospace,Menlo,monospace;letter-spacing:.05em;text-transform:uppercase;color:var(--muted);padding:8px 12px;border-bottom:1px solid var(--line);white-space:nowrap}
|
||||
td{padding:10px 12px;border-bottom:1px solid var(--line);vertical-align:top}
|
||||
tr:last-child td{border-bottom:0}
|
||||
td.c,th.c{text-align:center}
|
||||
tr.group td{background:var(--surface-2);font:.78rem ui-monospace,Menlo,monospace;color:var(--muted);padding-block:6px}
|
||||
.pill{display:inline-block;font:500 .78rem ui-monospace,Menlo,monospace;padding:2px 10px;border-radius:999px;white-space:nowrap}
|
||||
.pill.ok{background:var(--good-bg);color:var(--good)}.pill.mid{background:var(--warn-bg);color:var(--warn)}.pill.no{background:var(--bad-bg);color:var(--bad)}.pill.na{background:var(--surface-2);color:var(--muted)}
|
||||
.h2{background:var(--good-bg);color:var(--good)}.h1{background:var(--warn-bg);color:var(--warn)}.h0{background:var(--bad-bg);color:var(--bad)}
|
||||
.mono{font-family:ui-monospace,Menlo,monospace;font-size:.84rem}
|
||||
details{background:var(--surface);border:1px solid var(--line);border-radius:10px;padding:0 16px;margin-top:8px}
|
||||
summary{cursor:pointer;padding:12px 0;font-weight:500}
|
||||
summary .meta{color:var(--muted);font:400 .78rem ui-monospace,Menlo,monospace;margin-left:8px}
|
||||
pre{margin:0 0 14px;padding:12px 14px;background:var(--surface-2);border-radius:8px;overflow-x:auto;white-space:pre-wrap;word-break:break-word;font:.8rem/1.6 ui-monospace,Menlo,monospace}
|
||||
.best{border-left:3px solid var(--accent);padding-left:12px}
|
||||
</style></head><body><div class="wrap">
|
||||
<div><span class="eyebrow">/tldr benchmark · local · {{.When.Format "2006-01-02 15:04"}}</span>
|
||||
<h1>{{len .Models}} model{{if ne (len .Models) 1}}s{{end}}, {{len .Variants}} prompt{{if ne (len .Variants) 1}}s{{end}}</h1>
|
||||
<p class="sub">Chat: <span class="mono">{{.Chat.Source}}</span>, {{.Chat.Messages}} messages, {{.Chat.Kept}} kept after trimming, {{.Chat.InputChars}} characters in {{.Chat.Chunks}} chunk{{if ne .Chat.Chunks 1}}s{{end}}{{if .Chat.Truncated}} (the transcript was truncated by the size limit){{end}}. {{.Threads}} threads. This page is self-contained and makes no network requests.</p></div>
|
||||
|
||||
{{if .BestModel}}<div class="panel best"><span class="eyebrow">Best on this chat</span><p><strong>{{.BestModel}}</strong> with prompt <strong>{{.BestVar}}</strong> got the most facts, preferring the faster one on a tie.</p></div>{{end}}
|
||||
|
||||
<section><h2>Summary</h2><p class="sub">Per cell: facts found{{if .Facts}} (out of {{len .Facts}}){{else}} (no facts file, so only the checks below){{end}}, average seconds to finish, seconds to the first text, and flags.</p>
|
||||
<div class="panel"><div class="tablewrap"><table>
|
||||
<thead><tr><th>Prompt</th>{{range .Models}}<th class="c">{{.}}</th>{{end}}</tr></thead><tbody>
|
||||
{{range .Rows}}<tr><td class="mono">{{.Variant}}</td>
|
||||
{{range .Cells}}<td class="c">{{if eq .Runs 0}}<span class="note">not run</span>{{else if eq .Runs .Failed}}<span class="pill no">failed</span>{{else}}
|
||||
{{if .HasFacts}}<span class="pill {{.Class}}">{{f1 .Facts}} of {{.FactsTotal}}</span>{{else}}<span class="pill na">{{f1 .Invented}} invented</span>{{end}}<br>
|
||||
<span class="note">{{sec .Seconds}} s{{if gt .FirstText 0.0}} · first text {{sec .FirstText}} s{{end}}</span>{{if .Flags}}<br><span class="note">{{range $i, $f := .Flags}}{{if $i}}, {{end}}{{$f}}{{end}}</span>{{end}}{{end}}</td>{{end}}</tr>{{end}}
|
||||
</tbody></table></div></div></section>
|
||||
|
||||
{{if .FactGrid}}<section><h2>Which facts each prompt got</h2><p class="sub">Each cell is how many of that cell's runs got the fact right.</p>
|
||||
<div class="panel"><div class="tablewrap"><table>
|
||||
<thead><tr><th>Prompt</th>{{range .Facts}}<th class="c">{{.}}</th>{{end}}</tr></thead><tbody>
|
||||
{{range .FactGrid}}<tr class="group"><td colspan="{{inc (len $.Facts)}}">{{.Model}}</td></tr>
|
||||
{{range .Rows}}<tr><td class="mono">{{.Variant}}</td>{{$runs := .Runs}}{{range .Hits}}<td class="c mono {{hitClass . $runs}}">{{.}}</td>{{end}}</tr>{{end}}{{end}}
|
||||
</tbody></table></div></div></section>{{end}}
|
||||
|
||||
{{if .HasText}}<section><h2>Every run</h2><p class="sub">Outputs exactly as returned.</p>
|
||||
{{range .RunsSorted}}<details><summary>{{.Model}} · {{.Variant}} · run {{.N}}<span class="meta">{{if .Error}}failed{{else}}{{sec .Seconds}} s{{if .Missed}} · missed: {{range $i, $m := .Missed}}{{if $i}}, {{end}}{{$m}}{{end}}{{end}}{{if .Invented}} · numbers not in the chat: {{range $i, $m := .Invented}}{{if $i}}, {{end}}{{$m}}{{end}}{{end}}{{end}}</span></summary>
|
||||
<pre>{{if .Error}}{{.Error}}{{else}}{{.Output}}{{end}}</pre></details>{{end}}</section>{{end}}
|
||||
|
||||
<p class="note">"Numbers not in the chat" lists figures the overview wrote that appear nowhere in the transcript: a cheap check for invented details. "May be cut off" means the text ends without punctuation, usually the token limit. Facts pass only when one bullet holds all of the fact's details.</p>
|
||||
</div></body></html>`
|
||||
Reference in new issue
Block a user