| { |
| "model": "oruk/oruk-loop-153M", |
| "stored_parameters": 153202638, |
| "parameters_by_part": { |
| "audio_encoder_loop": 41213952, |
| "decoder_loop": 67886080, |
| "token_table": 28932430, |
| "kept_qwen3_asr_modules": 15170176 |
| }, |
| "teacher": { |
| "model": "Qwen/Qwen3-ASR-1.7B-hf", |
| "stored_parameters": 2038052480 |
| }, |
| "scoring": "Whisper EnglishTextNormalizer on reference and hypothesis; empty / \"ignore time segment in scoring\" references dropped; corpus WER per set; audio >30 s split at the quietest 100 ms frame in the last 5 s before each 30 s limit", |
| "test_sets": { |
| "leaderboard": "hf-audio/open-asr-leaderboard (original transcripts)", |
| "tedlium": "sanchit-gandhi/tedlium-data test" |
| }, |
| "evaluation_pipeline": { |
| "beam4": { |
| "decoding": "beam search, 4 beams, length_penalty 1.0, bf16", |
| "sets": { |
| "ami": { |
| "wer_pct": 11.222, |
| "reference_words": 89564, |
| "utterances": 11653, |
| "dropped": 990, |
| "hours": 8.68 |
| }, |
| "earnings22": { |
| "wer_pct": 9.804, |
| "reference_words": 48899, |
| "utterances": 2737, |
| "dropped": 4, |
| "hours": 5.43 |
| }, |
| "gigaspeech": { |
| "wer_pct": 10.455, |
| "reference_words": 403656, |
| "utterances": 19898, |
| "dropped": 33, |
| "hours": 35.36 |
| }, |
| "ls_clean": { |
| "wer_pct": 2.399, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4 |
| }, |
| "ls_other": { |
| "wer_pct": 5.953, |
| "reference_words": 52882, |
| "utterances": 2939, |
| "dropped": 0, |
| "hours": 5.34 |
| }, |
| "spgispeech": { |
| "wer_pct": 2.476, |
| "reference_words": 969328, |
| "utterances": 39341, |
| "dropped": 0, |
| "hours": 100.0 |
| }, |
| "tedlium": { |
| "wer_pct": 3.772, |
| "reference_words": 28207, |
| "utterances": 1155, |
| "dropped": 314, |
| "hours": 3.06 |
| }, |
| "voxpopuli": { |
| "wer_pct": 7.546, |
| "reference_words": 44079, |
| "utterances": 1842, |
| "dropped": 0, |
| "hours": 4.93 |
| } |
| }, |
| "mean_7_leaderboard_sets": 7.122, |
| "mean_8_incl_tedlium": 6.703 |
| }, |
| "greedy": { |
| "decoding": "greedy, bf16", |
| "sets": { |
| "ami": { |
| "wer_pct": 11.575, |
| "reference_words": 89564, |
| "utterances": 11653, |
| "dropped": 990, |
| "hours": 8.68 |
| }, |
| "earnings22": { |
| "wer_pct": 10.289, |
| "reference_words": 48899, |
| "utterances": 2737, |
| "dropped": 4, |
| "hours": 5.43 |
| }, |
| "earnings22_cleaned_aa": { |
| "wer_pct": 10.916, |
| "reference_words": 15418, |
| "utterances": 6, |
| "dropped": 0, |
| "hours": 1.92 |
| }, |
| "gigaspeech": { |
| "wer_pct": 10.75, |
| "reference_words": 403656, |
| "utterances": 19898, |
| "dropped": 33, |
| "hours": 35.36 |
| }, |
| "ls_clean": { |
| "wer_pct": 2.529, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4 |
| }, |
| "ls_other": { |
| "wer_pct": 6.322, |
| "reference_words": 52882, |
| "utterances": 2939, |
| "dropped": 0, |
| "hours": 5.34 |
| }, |
| "spgispeech": { |
| "wer_pct": 2.607, |
| "reference_words": 969328, |
| "utterances": 39341, |
| "dropped": 0, |
| "hours": 100.0 |
| }, |
| "tedlium": { |
| "wer_pct": 3.793, |
| "reference_words": 28207, |
| "utterances": 1155, |
| "dropped": 314, |
| "hours": 3.06 |
| }, |
| "voxpopuli": { |
| "wer_pct": 7.698, |
| "reference_words": 44079, |
| "utterances": 1842, |
| "dropped": 0, |
| "hours": 4.93 |
| }, |
| "voxpopuli_cleaned_aa": { |
| "wer_pct": 4.611, |
| "reference_words": 17784, |
| "utterances": 628, |
| "dropped": 0, |
| "hours": 1.98 |
| } |
| }, |
| "mean_7_leaderboard_sets": 7.396, |
| "mean_8_incl_tedlium": 6.945 |
| }, |
| "note_cleaned_sets": "earnings22_cleaned_aa / voxpopuli_cleaned_aa: ArtificialAnalysis re-transcribed versions of the same test audio (ArtificialAnalysis/Earnings22-Cleaned-AA, hf-audio voxpopuli_cleaned_aa); not part of the leaderboard average" |
| }, |
| "out_of_domain_beam4": { |
| "note": "fleursu = google/fleurs en_us test (647); cvu = Common Voice test from hf-audio/open-asr-leaderboard, fixed random 4000 (seed 20261014); edacc = edinburghcstr/edacc test; peoples = MLCommons/peoples_speech clean test, fixed random 4000 (3,999 scored)", |
| "sets": { |
| "ls_clean": { |
| "wer_pct": 2.399, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4 |
| }, |
| "fleursu": { |
| "wer_pct": 9.882, |
| "reference_words": 14279, |
| "utterances": 647, |
| "dropped": 0, |
| "hours": 1.77 |
| }, |
| "cvu": { |
| "wer_pct": 17.148, |
| "reference_words": 38629, |
| "utterances": 4000, |
| "dropped": 0, |
| "hours": 6.62 |
| }, |
| "edacc": { |
| "wer_pct": 20.158, |
| "reference_words": 157116, |
| "utterances": 8537, |
| "dropped": 752, |
| "hours": 16.94 |
| }, |
| "peoples": { |
| "wer_pct": 20.154, |
| "reference_words": 70861, |
| "utterances": 3999, |
| "dropped": 1, |
| "hours": 6.93 |
| } |
| } |
| }, |
| "frontier_analysis": { |
| "snapshot": "hf-audio/open_asr_leaderboard English results, fetched 2026-10-14", |
| "previous_frontier_points": [ |
| [ |
| "nvidia/parakeet-tdt_ctc-110m", |
| 0.11, |
| 7.977 |
| ], |
| [ |
| "nvidia/canary-180m-flash", |
| 0.18, |
| 7.484 |
| ], |
| [ |
| "usefulsensors/moonshine-streaming-medium", |
| 0.24, |
| 7.181 |
| ], |
| [ |
| "nvidia/parakeet-tdt-0.6b-v2", |
| 0.6, |
| 6.436 |
| ] |
| ], |
| "previous_frontier_at_153M_pct": 7.645, |
| "gap_at_153M_pct_points": 0.524, |
| "size_where_previous_frontier_reaches_7.12_B": 0.2587, |
| "parameter_ratio_at_equal_accuracy": 1.69, |
| "method": "linear interpolation of 7-set mean WER in log10(size) between neighbouring frontier models", |
| "note": "gap uses the exact beam-4 mean 7.122" |
| }, |
| "release_package_rescore": { |
| "code": "this repository: AutoModel.from_pretrained(trust_remote_code=True) + model.transcribe, batch 64 (beam: 32), one A100 40GB per job", |
| "greedy": { |
| "sets": { |
| "ami": { |
| "wer_pct": 11.548, |
| "reference_words": 89564, |
| "utterances": 11653, |
| "dropped": 990, |
| "hours": 8.68, |
| "seconds": 308 |
| }, |
| "earnings22": { |
| "wer_pct": 10.27, |
| "reference_words": 48899, |
| "utterances": 2737, |
| "dropped": 4, |
| "hours": 5.43, |
| "seconds": 162 |
| }, |
| "tedlium": { |
| "wer_pct": 3.907, |
| "reference_words": 28207, |
| "utterances": 1155, |
| "dropped": 314, |
| "hours": 3.06, |
| "seconds": 120 |
| }, |
| "ls_clean": { |
| "wer_pct": 2.542, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4, |
| "seconds": 129 |
| }, |
| "ls_other": { |
| "wer_pct": 6.431, |
| "reference_words": 52882, |
| "utterances": 2939, |
| "dropped": 0, |
| "hours": 5.34, |
| "seconds": 152 |
| }, |
| "voxpopuli": { |
| "wer_pct": 7.709, |
| "reference_words": 44079, |
| "utterances": 1842, |
| "dropped": 0, |
| "hours": 4.93, |
| "seconds": 103 |
| }, |
| "gigaspeech": { |
| "wer_pct": 10.762, |
| "reference_words": 403656, |
| "utterances": 19898, |
| "dropped": 33, |
| "hours": 35.36, |
| "seconds": 1021 |
| }, |
| "spgispeech": { |
| "wer_pct": 2.607, |
| "reference_words": 969328, |
| "utterances": 39341, |
| "dropped": 0, |
| "hours": 100.0, |
| "seconds": 2221 |
| } |
| }, |
| "mean_7_leaderboard_sets": 7.41, |
| "mean_8_incl_tedlium": 6.972 |
| }, |
| "beam4": { |
| "sets": { |
| "ami": { |
| "wer_pct": 11.224, |
| "reference_words": 89564, |
| "utterances": 11653, |
| "dropped": 990, |
| "hours": 8.68, |
| "seconds": 635 |
| }, |
| "earnings22": { |
| "wer_pct": 9.802, |
| "reference_words": 48899, |
| "utterances": 2737, |
| "dropped": 4, |
| "hours": 5.43, |
| "seconds": 374 |
| }, |
| "tedlium": { |
| "wer_pct": 3.772, |
| "reference_words": 28207, |
| "utterances": 1155, |
| "dropped": 314, |
| "hours": 3.06, |
| "seconds": 238 |
| }, |
| "ls_clean": { |
| "wer_pct": 2.404, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4, |
| "seconds": 267 |
| }, |
| "ls_other": { |
| "wer_pct": 5.942, |
| "reference_words": 52882, |
| "utterances": 2939, |
| "dropped": 0, |
| "hours": 5.34, |
| "seconds": 293 |
| }, |
| "voxpopuli": { |
| "wer_pct": 7.543, |
| "reference_words": 44079, |
| "utterances": 1842, |
| "dropped": 0, |
| "hours": 4.93, |
| "seconds": 222 |
| }, |
| "gigaspeech": { |
| "wer_pct": 10.446, |
| "reference_words": 403656, |
| "utterances": 19898, |
| "dropped": 33, |
| "hours": 35.36, |
| "seconds": 2156 |
| }, |
| "spgispeech": { |
| "wer_pct": 2.475, |
| "reference_words": 969328, |
| "utterances": 39341, |
| "dropped": 0, |
| "hours": 100.0, |
| "seconds": 4312 |
| } |
| }, |
| "mean_7_leaderboard_sets": 7.119, |
| "mean_8_incl_tedlium": 6.701 |
| } |
| }, |
| "teacher_on_our_harness": { |
| "model": "Qwen/Qwen3-ASR-1.7B-hf", |
| "decoding": "greedy, bf16", |
| "sets": { |
| "spgispeech": { |
| "wer_pct": 2.835, |
| "reference_words": 969328, |
| "utterances": 39341, |
| "dropped": 0, |
| "hours": 100.0, |
| "seconds": 1392 |
| }, |
| "tedlium": { |
| "wer_pct": 2.28, |
| "reference_words": 28207, |
| "utterances": 1155, |
| "dropped": 314, |
| "hours": 3.06, |
| "seconds": 58 |
| }, |
| "ls_clean": { |
| "wer_pct": 1.618, |
| "reference_words": 53027, |
| "utterances": 2620, |
| "dropped": 0, |
| "hours": 5.4, |
| "seconds": 85 |
| }, |
| "ami": { |
| "wer_pct": 10.481, |
| "reference_words": 89564, |
| "utterances": 11653, |
| "dropped": 990, |
| "hours": 8.68, |
| "seconds": 420 |
| }, |
| "earnings22": { |
| "wer_pct": 10.062, |
| "reference_words": 48899, |
| "utterances": 2737, |
| "dropped": 4, |
| "hours": 5.43, |
| "seconds": 101 |
| }, |
| "gigaspeech": { |
| "wer_pct": 8.738, |
| "reference_words": 403656, |
| "utterances": 19898, |
| "dropped": 33, |
| "hours": 35.36, |
| "seconds": 632 |
| }, |
| "ls_other": { |
| "wer_pct": 3.364, |
| "reference_words": 52882, |
| "utterances": 2939, |
| "dropped": 0, |
| "hours": 5.34, |
| "seconds": 92 |
| }, |
| "voxpopuli": { |
| "wer_pct": 6.237, |
| "reference_words": 44079, |
| "utterances": 1842, |
| "dropped": 0, |
| "hours": 4.93, |
| "seconds": 68 |
| } |
| }, |
| "mean_7_leaderboard_sets": 6.191, |
| "mean_8_incl_tedlium": 5.702, |
| "leaderboard_snapshot_7set_mean": 6.25 |
| } |
| } |
