"""Score the phrasebook short-circuit against configs/eval/phrasebook_eval.json.

For every (input, lang) pair the outcome is one of:
    correct    — matched the expected row
    wrong      — matched a DIFFERENT row (worst case: user hears a wrong phrase)
    false_hit  — expected the LLM to handle it, but the phrasebook fired
    miss       — expected a row, nothing matched (falls through to LLM — safe)
    correct_miss — open-ended input correctly left to the LLM

    python scripts/eval_phrasebook.py              # summary
    python scripts/eval_phrasebook.py -v           # list every non-correct item
    python scripts/eval_phrasebook.py --threshold 0.85
"""
from __future__ import annotations

import argparse
import json
import sys
from collections import Counter
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from src.llm.phrasebook import DEFAULT_THRESHOLD, lookup  # noqa: E402


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--file", default=str(ROOT / "configs" / "eval" / "phrasebook_eval.json"))
    ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
    ap.add_argument("-v", "--verbose", action="store_true")
    args = ap.parse_args()

    data = json.loads(Path(args.file).read_text(encoding="utf-8"))
    groups = data.get("groups", {})
    per_lang: dict[str, Counter] = {"bam": Counter(), "ful": Counter()}
    problems: list[str] = []

    for it in data["items"]:
        langs = ["bam", "ful"] if it["lang"] == "both" else [it["lang"]]
        exp = it["expect"]
        ok_sources = set(groups.get(exp, [exp])) if exp else set()
        for lang in langs:
            hit = lookup(it["q"], lang, threshold=args.threshold)
            got = hit["source"] if hit else None
            if exp is None:
                outcome = "false_hit" if hit else "correct_miss"
            elif hit is None:
                outcome = "miss"
            else:
                outcome = "correct" if got in ok_sources else "wrong"
            per_lang[lang][outcome] += 1
            if outcome not in ("correct", "correct_miss"):
                problems.append(f"  [{lang}] {outcome:9} {it['q']!r:42} expected={exp!r} got={got!r}"
                                + (f" ({hit['score']})" if hit else ""))

    print(f"threshold={args.threshold}")
    for lang, c in per_lang.items():
        n = sum(c.values())
        good = c["correct"] + c["correct_miss"]
        print(f"{lang}: {good}/{n} = {good / n:.0%} right | correct={c['correct']} "
              f"correct_miss={c['correct_miss']} miss={c['miss']} "
              f"wrong={c['wrong']} false_hit={c['false_hit']}")
    if args.verbose and problems:
        print("\nNon-correct items:")
        print("\n".join(problems))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())