velox-bot/internal/tldr/tldrbench/suggest.go

package tldrbench

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

var (
	// Things a good overview has to get right: figures, times, money, days.
	figureRe  = regexp.MustCompile(`\d+(?:[.,:]\d+)*(?:h\d*|€|%|k|gb|mb|min)?`)
	weekdayRe = regexp.MustCompile(`(?i)\b(segunda|ter[çc]a|quarta|quinta|sexta|s[áa]bado|domingo|monday|tuesday|wednesday|thursday|friday|saturday|sunday|amanh[ãa]|tomorrow|hoje|today)\b`)
	moneyRe   = regexp.MustCompile(`(?i)€|\$|\beur\b|\beuros?\b`)
	wordRe    = regexp.MustCompile(`[\p{L}][\p{L}'-]{5,}`)

	// Common long words that make poor topic keywords.
	stopwords = map[string]bool{
		"porque": true, "também": true, "tambem": true, "depois": true, "ontem": true, "ainda": true, "estava": true,
		"podemos": true, "devíamos": true, "devemos": true, "fazer": true, "mesmo": true, "sempre": true, "ficou": true,
		"because": true, "should": true, "would": true, "could": true, "really": true, "maybe": true, "already": true,
	}
)

// signals counts how many different "this matters" markers a line has.
func signals(line string) int {
	n := 0
	if figureRe.MatchString(line) {
		n++
	}
	if weekdayRe.MatchString(line) {
		n++
	}
	if moneyRe.MatchString(line) {
		n++
	}
	if strings.Contains(line, "[link") || strings.Contains(line, "(preview:") {
		n++
	}
	return n
}

// SuggestFacts proposes candidate facts from transcript lines ("Author: text").
// It does not understand the chat: it picks the lines that carry figures,
// times, days, money or links, and for each suggests patterns (the figures and
// days in it, plus one topic word). You are meant to prune and merge them into
// the 10 to 20 facts that matter, then check the patterns against a real
// overview. At most max candidates are returned, chronologically.
func SuggestFacts(lines []string, max int) []Fact {
	type scored struct {
		idx, signals int
		line         string
	}
	var cands []scored
	for i, ln := range lines {
		ln = strings.TrimSpace(ln)
		if ln == "" {
			continue
		}
		if s := signals(ln); s > 0 {
			cands = append(cands, scored{i, s, ln})
		}
	}
	// Keep the strongest, then restore chat order.
	sort.SliceStable(cands, func(a, b int) bool { return cands[a].signals > cands[b].signals })
	if max > 0 && len(cands) > max {
		cands = cands[:max]
	}
	sort.Slice(cands, func(a, b int) bool { return cands[a].idx < cands[b].idx })

	var facts []Fact
	for n, c := range cands {
		facts = append(facts, Fact{
			Name: fmt.Sprintf("fact %d: %s", n+1, excerpt(c.line, 48)),
			All:  suggestPatterns(c.line),
			Note: c.line,
		})
	}
	return facts
}

func suggestPatterns(line string) []string {
	// Drop the "Author: " prefix so names do not become patterns.
	text := line
	if i := strings.Index(line, ": "); i > 0 && i <= 32 {
		text = line[i+2:]
	}

	seen := map[string]bool{}
	var pats []string
	add := func(p string) {
		key := strings.ToLower(p)
		if p == "" || len(pats) >= 3 {
			return
		}
		for have := range seen {
			if strings.Contains(have, key) || strings.Contains(key, have) {
				return // already covered, e.g. "segunda" next to "segunda-feira"
			}
		}
		seen[key] = true
		pats = append(pats, p)
	}

	// A topic word first, so the fact stays tied to its subject.
	best := ""
	for _, w := range wordRe.FindAllString(text, -1) {
		lw := strings.ToLower(w)
		if stopwords[lw] || strings.ContainsAny(lw, "0123456789") {
			continue
		}
		if len([]rune(w)) > len([]rune(best)) {
			best = w
		}
	}
	if best != "" {
		add(regexp.QuoteMeta(strings.ToLower(trimWord(best))))
	}
	for _, d := range weekdayRe.FindAllString(text, -1) {
		add(regexp.QuoteMeta(strings.ToLower(d)))
	}
	for _, f := range figureRe.FindAllString(text, -1) {
		add(regexp.QuoteMeta(strings.ToLower(f)))
	}
	return pats
}

func trimWord(w string) string {
	return strings.TrimFunc(w, func(r rune) bool { return !unicode.IsLetter(r) })
}

func excerpt(s string, n int) string {
	s = strings.Join(strings.Fields(s), " ")
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return string(r[:n]) + "…"
}