Download code/text.py from oddadmix/tiny-voca-arabic: direct link, hf CLI and curl.
- Browser
- Download file 5.01 kB
-
https://huggingface.co/oddadmix/tiny-voca-arabic/resolve/main/code/text.py
- Command line
-
hf download hf://oddadmix/tiny-voca-arabic/code/text.py
-
curl -L -o text.py https://huggingface.co/oddadmix/tiny-voca-arabic/resolve/main/code/text.py
5.01 kB
| """Modern Standard Arabic front end: text -> phoneme tokens with a stress feature. | |
| Same interface as the English front end it replaces (text_english.py), so the model, annotation | |
| and training code are unchanged. The pipeline is in ``arabic_frontend``: | |
| undiacritized text -> phrases (split at punctuation) -> CATT diacritizer -> MSA rules -> phonemes | |
| * A token is one phoneme; geminates are two identical consonant tokens. | |
| * Stress (0/1) is a parallel feature from MSA syllable weight, not a token. | |
| * Punctuation, word boundaries and the utterance edges are tokens that may last zero frames. | |
| * ``Phonemized.optional`` marks case endings inside a phrase. The aligner keeps one only when | |
| the recording has it; inference always says them (formal MSA). | |
| Diacritizations can be served from a cache (``TINY_VOCA_DIACRITIZED`` = jsonl of {"p", "d"}) so | |
| annotation and training use exactly the vowels that were aligned. | |
| """ | |
| from dataclasses import dataclass, field | |
| import hashlib | |
| import json | |
| import os | |
| from . import arabic_frontend as af | |
| FRONTEND_VERSION = "ar-1" | |
| FRONTENDS = ("arabic",) | |
| FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND" | |
| _frontend = "arabic" | |
| PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>" | |
| SPECIALS = (PAD, BOS, EOS, SP, UNK) | |
| PUNCTUATION = (",", ".", "?", "!", ";", ":", "…") | |
| _PUNCT_MAP = {"،": ",", ",": ",", ".": ".", "؟": "?", "?": "?", "!": "!", "؛": ";", ";": ";", ":": ":", "…": "…"} | |
| CONSONANTS = tuple(af.CONSONANTS) | |
| VOWELS = tuple(af.VOWELS) | |
| PHONES = CONSONANTS + VOWELS | |
| # Append-only: changing the order or removing a symbol invalidates checkpoints. | |
| SYMBOLS = SPECIALS + PUNCTUATION + PHONES | |
| SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)} | |
| PHONE_SET = frozenset(PHONES) | |
| PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]} | |
| VOICELESS = frozenset(af.VOICELESS) | |
| VOWEL_SET = frozenset(VOWELS) | |
| class NothingToPronounce(ValueError): | |
| """The text has no Arabic word to say.""" | |
| class Phonemized: | |
| symbols: tuple | |
| ids: tuple | |
| stress: tuple | |
| unknown: tuple | |
| optional: tuple = field(default=()) | |
| diacritized: tuple = field(default=()) | |
| def __len__(self): | |
| return len(self.ids) | |
| def frontend(): | |
| return _frontend | |
| def set_frontend(name): | |
| if name not in FRONTENDS: | |
| raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}") | |
| def frontend_identity(): | |
| return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tiny_voca.arabic_frontend", | |
| "diacritizer": "catt-tashkeel ed_model v2", "symbols": list(SYMBOLS)} | |
| def frontend_runtime(): | |
| return {"phonemizer": hashlib.sha256(open(af.__file__, "rb").read()).hexdigest()} | |
| def frontend_fingerprint(): | |
| return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest() | |
| _cache = None | |
| def _diacritized(phrases): | |
| global _cache | |
| if _cache is None: | |
| _cache = {} | |
| path = os.environ.get("TINY_VOCA_DIACRITIZED") | |
| if path and os.path.exists(path): | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| r = json.loads(line); _cache[r["p"]] = r["d"] | |
| missing = [p for p in phrases if p not in _cache] | |
| if missing: | |
| for p, d in zip(missing, af.diacritize(missing)): | |
| _cache[p] = d | |
| return [_cache[p] for p in phrases] | |
| def phonemize(text): | |
| if not isinstance(text, str) or not text.strip(): | |
| raise ValueError("text must not be empty") | |
| parts = af.split_phrases(text) | |
| if not parts: | |
| raise NothingToPronounce("the text has nothing to pronounce") | |
| res = af.phonemize(text, _diacritized([p for p, _ in parts])) | |
| symbols, stress, optional = [], [], [] | |
| for x in res.phones: | |
| if x.p == "^": s = BOS | |
| elif x.p == "$": s = EOS | |
| elif x.p == " ": s = SP | |
| elif x.word < 0: s = _PUNCT_MAP.get(x.p, ",") | |
| else: s = x.p | |
| if s in PUNCTUATION and symbols and symbols[-1] in PUNCTUATION: | |
| continue | |
| if s in PUNCTUATION and symbols and symbols[-1] == SP: | |
| symbols.pop(); stress.pop(); optional.pop() | |
| symbols.append(s); stress.append(x.stress); optional.append(int(x.optional)) | |
| if not any(s in PHONE_SET for s in symbols): | |
| raise NothingToPronounce("the text has nothing to pronounce") | |
| return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), (), | |
| tuple(optional), tuple(res.phrases)) | |
| def drop(ph, remove): | |
| """Copy of ``ph`` without the token positions in ``remove`` (dropped optional case endings).""" | |
| remove = set(remove) | |
| keep = [i for i in range(len(ph.symbols)) if i not in remove] | |
| pick = lambda seq: tuple(seq[i] for i in keep) if seq else seq | |
| return Phonemized(pick(ph.symbols), pick(ph.ids), pick(ph.stress), ph.unknown, pick(ph.optional), ph.diacritized) | |
| def is_phone(token_ids): | |
| return [int(i) in PHONE_IDS for i in token_ids] | |