| { | |
| "model": "Qev-9B", | |
| "revision": "v0.3.0", | |
| "full_suite": { | |
| "decision_dev": { | |
| "correct": 1296, | |
| "total": 1468, | |
| "accuracy": 0.8828337874659401, | |
| "rejected": 0 | |
| }, | |
| "transfer_dev": { | |
| "correct": 629, | |
| "total": 764, | |
| "accuracy": 0.8232984293193717, | |
| "rejected": 0 | |
| }, | |
| "mmlupro": { | |
| "correct": 565, | |
| "total": 1000, | |
| "accuracy": 0.565, | |
| "rejected": 0 | |
| }, | |
| "semif": { | |
| "correct": 243, | |
| "total": 252, | |
| "accuracy": 0.9642857142857143, | |
| "rejected": 0 | |
| }, | |
| "scienthoon": { | |
| "correct": 653, | |
| "total": 873, | |
| "accuracy": 0.7479954180985109, | |
| "rejected": 0 | |
| }, | |
| "wanli": { | |
| "correct": 192, | |
| "total": 256, | |
| "accuracy": 0.75, | |
| "rejected": 0 | |
| }, | |
| "public": { | |
| "correct": 187, | |
| "total": 231, | |
| "accuracy": 0.8095238095238095, | |
| "rejected": 0 | |
| } | |
| }, | |
| "matched_subsets": { | |
| "decision_dev_all": { | |
| "correct": 1296, | |
| "total": 1468 | |
| }, | |
| "decision_dev_clean": { | |
| "correct": 1107, | |
| "total": 1264 | |
| }, | |
| "transfer_dev_all": { | |
| "correct": 629, | |
| "total": 764 | |
| }, | |
| "transfer_dev_clean": { | |
| "correct": 549, | |
| "total": 656 | |
| }, | |
| "mmlupro": { | |
| "correct": 565, | |
| "total": 1000 | |
| }, | |
| "scienthoon": { | |
| "correct": 653, | |
| "total": 873 | |
| }, | |
| "wanli": { | |
| "correct": 192, | |
| "total": 256 | |
| }, | |
| "semif_handwritten": { | |
| "correct": 135, | |
| "total": 144 | |
| }, | |
| "jevbench_public": { | |
| "correct": 187, | |
| "total": 231 | |
| } | |
| } | |
| } | |