Spaces:
Sleeping
Sleeping
| """Score the phrasebook short-circuit against configs/eval/phrasebook_eval.json. | |
| For every (input, lang) pair the outcome is one of: | |
| correct — matched the expected row | |
| wrong — matched a DIFFERENT row (worst case: user hears a wrong phrase) | |
| false_hit — expected the LLM to handle it, but the phrasebook fired | |
| miss — expected a row, nothing matched (falls through to LLM — safe) | |
| correct_miss — open-ended input correctly left to the LLM | |
| python scripts/eval_phrasebook.py # summary | |
| python scripts/eval_phrasebook.py -v # list every non-correct item | |
| python scripts/eval_phrasebook.py --threshold 0.85 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| from src.llm.phrasebook import DEFAULT_THRESHOLD, lookup # noqa: E402 | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--file", default=str(ROOT / "configs" / "eval" / "phrasebook_eval.json")) | |
| ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD) | |
| ap.add_argument("-v", "--verbose", action="store_true") | |
| args = ap.parse_args() | |
| data = json.loads(Path(args.file).read_text(encoding="utf-8")) | |
| groups = data.get("groups", {}) | |
| per_lang: dict[str, Counter] = {"bam": Counter(), "ful": Counter()} | |
| problems: list[str] = [] | |
| for it in data["items"]: | |
| langs = ["bam", "ful"] if it["lang"] == "both" else [it["lang"]] | |
| exp = it["expect"] | |
| ok_sources = set(groups.get(exp, [exp])) if exp else set() | |
| for lang in langs: | |
| hit = lookup(it["q"], lang, threshold=args.threshold) | |
| got = hit["source"] if hit else None | |
| if exp is None: | |
| outcome = "false_hit" if hit else "correct_miss" | |
| elif hit is None: | |
| outcome = "miss" | |
| else: | |
| outcome = "correct" if got in ok_sources else "wrong" | |
| per_lang[lang][outcome] += 1 | |
| if outcome not in ("correct", "correct_miss"): | |
| problems.append(f" [{lang}] {outcome:9} {it['q']!r:42} expected={exp!r} got={got!r}" | |
| + (f" ({hit['score']})" if hit else "")) | |
| print(f"threshold={args.threshold}") | |
| for lang, c in per_lang.items(): | |
| n = sum(c.values()) | |
| good = c["correct"] + c["correct_miss"] | |
| print(f"{lang}: {good}/{n} = {good / n:.0%} right | correct={c['correct']} " | |
| f"correct_miss={c['correct_miss']} miss={c['miss']} " | |
| f"wrong={c['wrong']} false_hit={c['false_hit']}") | |
| if args.verbose and problems: | |
| print("\nNon-correct items:") | |
| print("\n".join(problems)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |