package tldrbench import ( "fmt" "regexp" "sort" "strings" "unicode" ) var ( // Things a good overview has to get right: figures, times, money, days. figureRe = regexp.MustCompile(`\d+(?:[.,:]\d+)*(?:h\d*|€|%|k|gb|mb|min)?`) weekdayRe = regexp.MustCompile(`(?i)\b(segunda|ter[çc]a|quarta|quinta|sexta|s[áa]bado|domingo|monday|tuesday|wednesday|thursday|friday|saturday|sunday|amanh[ãa]|tomorrow|hoje|today)\b`) moneyRe = regexp.MustCompile(`(?i)€|\$|\beur\b|\beuros?\b`) wordRe = regexp.MustCompile(`[\p{L}][\p{L}'-]{5,}`) // Common long words that make poor topic keywords. stopwords = map[string]bool{ "porque": true, "também": true, "tambem": true, "depois": true, "ontem": true, "ainda": true, "estava": true, "podemos": true, "devíamos": true, "devemos": true, "fazer": true, "mesmo": true, "sempre": true, "ficou": true, "because": true, "should": true, "would": true, "could": true, "really": true, "maybe": true, "already": true, } ) // signals counts how many different "this matters" markers a line has. func signals(line string) int { n := 0 if figureRe.MatchString(line) { n++ } if weekdayRe.MatchString(line) { n++ } if moneyRe.MatchString(line) { n++ } if strings.Contains(line, "[link") || strings.Contains(line, "(preview:") { n++ } return n } // SuggestFacts proposes candidate facts from transcript lines ("Author: text"). // It does not understand the chat: it picks the lines that carry figures, // times, days, money or links, and for each suggests patterns (the figures and // days in it, plus one topic word). You are meant to prune and merge them into // the 10 to 20 facts that matter, then check the patterns against a real // overview. At most max candidates are returned, chronologically. func SuggestFacts(lines []string, max int) []Fact { type scored struct { idx, signals int line string } var cands []scored for i, ln := range lines { ln = strings.TrimSpace(ln) if ln == "" { continue } if s := signals(ln); s > 0 { cands = append(cands, scored{i, s, ln}) } } // Keep the strongest, then restore chat order. sort.SliceStable(cands, func(a, b int) bool { return cands[a].signals > cands[b].signals }) if max > 0 && len(cands) > max { cands = cands[:max] } sort.Slice(cands, func(a, b int) bool { return cands[a].idx < cands[b].idx }) var facts []Fact for n, c := range cands { facts = append(facts, Fact{ Name: fmt.Sprintf("fact %d: %s", n+1, excerpt(c.line, 48)), All: suggestPatterns(c.line), Note: c.line, }) } return facts } func suggestPatterns(line string) []string { // Drop the "Author: " prefix so names do not become patterns. text := line if i := strings.Index(line, ": "); i > 0 && i <= 32 { text = line[i+2:] } seen := map[string]bool{} var pats []string add := func(p string) { key := strings.ToLower(p) if p == "" || len(pats) >= 3 { return } for have := range seen { if strings.Contains(have, key) || strings.Contains(key, have) { return // already covered, e.g. "segunda" next to "segunda-feira" } } seen[key] = true pats = append(pats, p) } // A topic word first, so the fact stays tied to its subject. best := "" for _, w := range wordRe.FindAllString(text, -1) { lw := strings.ToLower(w) if stopwords[lw] || strings.ContainsAny(lw, "0123456789") { continue } if len([]rune(w)) > len([]rune(best)) { best = w } } if best != "" { add(regexp.QuoteMeta(strings.ToLower(trimWord(best)))) } for _, d := range weekdayRe.FindAllString(text, -1) { add(regexp.QuoteMeta(strings.ToLower(d))) } for _, f := range figureRe.FindAllString(text, -1) { add(regexp.QuoteMeta(strings.ToLower(f))) } return pats } func trimWord(w string) string { return strings.TrimFunc(w, func(r rune) bool { return !unicode.IsLetter(r) }) } func excerpt(s string, n int) string { s = strings.Join(strings.Fields(s), " ") r := []rune(s) if len(r) <= n { return s } return string(r[:n]) + "…" }