| { |
| "temperature": { |
| "choice": 1.160062551498413, |
| "noul": 0.9890053272247314, |
| "by_options": { |
| "choice:2": 1.1245635747909546, |
| "choice:3-5": 1.1677029132843018, |
| "noul:2": 0.9890053272247314 |
| } |
| }, |
| "test_sets": { |
| "test_exist": { |
| "questions": 1120, |
| "accuracy": 0.9518, |
| "nll": 0.1888, |
| "ece": 0.0705, |
| "accuracy_mismatched": 0.5054 |
| }, |
| "test_vqa_choice": { |
| "questions": 1453, |
| "accuracy": 0.905, |
| "nll": 0.2511, |
| "ece": 0.0265, |
| "accuracy_mismatched": 0.373 |
| }, |
| "test_size": { |
| "questions": 1198, |
| "accuracy": 0.8765, |
| "nll": 0.4159, |
| "ece": 0.1535, |
| "accuracy_mismatched": 0.5109 |
| }, |
| "bench_pope": { |
| "questions": 9000, |
| "accuracy": 0.8607, |
| "nll": 0.3615, |
| "ece": 0.0285, |
| "accuracy_mismatched": 0.5177 |
| }, |
| "test_gqa": { |
| "questions": 986, |
| "accuracy": 0.8448, |
| "nll": 0.3462, |
| "ece": 0.0291, |
| "accuracy_mismatched": 0.5578 |
| }, |
| "test_position": { |
| "questions": 1276, |
| "accuracy": 0.9412, |
| "nll": 0.1818, |
| "ece": 0.0407, |
| "accuracy_mismatched": 0.5 |
| }, |
| "test_vqa_yesno": { |
| "questions": 1000, |
| "accuracy": 0.721, |
| "nll": 0.5673, |
| "ece": 0.0622, |
| "accuracy_mismatched": 0.501 |
| }, |
| "test_relation": { |
| "questions": 2366, |
| "accuracy": 0.9121, |
| "nll": 0.2263, |
| "ece": 0.0281, |
| "accuracy_mismatched": 0.4721 |
| }, |
| "test_vsr": { |
| "questions": 902, |
| "accuracy": 0.7184, |
| "nll": 0.5901, |
| "ece": 0.1022, |
| "accuracy_mismatched": 0.469 |
| }, |
| "test_v7w": { |
| "questions": 1000, |
| "accuracy": 0.777, |
| "nll": 0.5846, |
| "ece": 0.0407, |
| "accuracy_mismatched": 0.446 |
| }, |
| "bench_countbench": { |
| "questions": 491, |
| "accuracy": 0.5153, |
| "nll": 1.2606, |
| "ece": 0.0791, |
| "accuracy_mismatched": 0.1059 |
| }, |
| "test_position_flip": { |
| "questions": 640, |
| "accuracy": 0.9406, |
| "nll": 0.1774, |
| "ece": 0.0337, |
| "accuracy_mismatched": 0.5219 |
| }, |
| "test_relation_flip": { |
| "questions": 1786, |
| "accuracy": 0.9227, |
| "nll": 0.2098, |
| "ece": 0.0196, |
| "accuracy_mismatched": 0.4905 |
| }, |
| "bench_whatsup": { |
| "questions": 820, |
| "accuracy": 0.5183, |
| "nll": 1.1743, |
| "ece": 0.096, |
| "accuracy_mismatched": 0.2329 |
| } |
| }, |
| "laya_vision": { |
| "vqav2_yesno": { |
| "questions": 4887, |
| "laya_vision": 0.7172, |
| "devision": 0.7493, |
| "difference": 0.0321, |
| "ci95": [ |
| 0.0174, |
| 0.0469 |
| ] |
| }, |
| "aokvqa": { |
| "questions": 1138, |
| "laya_vision": 0.5984, |
| "devision": 0.6274, |
| "difference": 0.029, |
| "ci95": [ |
| -0.0026, |
| 0.0624 |
| ] |
| }, |
| "scienceqa": { |
| "questions": 2097, |
| "laya_vision": 0.824, |
| "devision": 0.8207, |
| "difference": -0.0033, |
| "ci95": [ |
| -0.0234, |
| 0.0153 |
| ] |
| }, |
| "scienceqa_natural_needs_picture": { |
| "questions": 323, |
| "laya_vision": 0.6997, |
| "devision": 0.6006, |
| "difference": -0.0991, |
| "ci95": [ |
| -0.1641, |
| -0.0341 |
| ] |
| }, |
| "pope": { |
| "random": { |
| "laya_vision_published": 0.836, |
| "devision": 0.8943 |
| }, |
| "popular": { |
| "laya_vision_published": 0.819, |
| "devision": 0.867 |
| }, |
| "adversarial": { |
| "laya_vision_published": 0.777, |
| "devision": 0.8207 |
| } |
| } |
| }, |
| "cpu_latency": { |
| "p50": 301, |
| "p95": 321, |
| "threads": 4, |
| "machine": "Linux x86_64", |
| "source": "release/v0.5/cpu_latency.json" |
| } |
| } |
|
|
