{ "schema": 1, "suite": "public-benchmark", "date": "2026-09-30", "machine": "rtx-5090", "device": "cuda", "servers": { "ollaya": "0.8.0 (pre-release build of the runtime that shipped as 0.8.0)", "ollama": "0.35.0" }, "protocol": { "benchmark": "Bespoke Labs' public decision benchmark: 13 subsets of 11 human-labeled public datasets, 3,880 questions", "source": "https://github.com/bespokelabsai/nimble/blob/main/docs/PUBLIC_BENCHMARKS.md", "runner": "Bespoke's own evaluate_public_jev, through convert/ollaya_convert/bench_public.py", "client": "one request (one question) at a time over HTTP to /v1/systemone, timed by the client", "accuracy": "macro: the mean of the 13 subsets; a rejected request counts as wrong", "ece": "ten-bin expected calibration error of the top probability; pooled over all 3,880 questions" }, "note": "Ollama's Nimble run is the rerun on an idle machine (same numbers as the first run within 1 ms). Ollama runs Nimble and Tev1 from its own registry (Q8_0 GGUF on llama.cpp).", "results": [ { "server": "ollaya", "model": "winnow:12b", "records": 3880, "errors": 0, "macro_accuracy": 0.7726, "micro_accuracy": 0.7778, "macro_ece": 0.1458, "pooled_ece": 0.1415, "brier": 0.3623, "median_ms": 59.6, "p95_ms": 115.2, "by_type": { "choice": 0.8, "noul": 0.8535, "score": 0.5922 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7546, "ece": 0.1907 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8657, "ece": 0.1059 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8514, "ece": 0.1019 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8733, "ece": 0.0898 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.8763, "ece": 0.085 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.832, "ece": 0.1102 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8161, "ece": 0.1348 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.87, "ece": 0.0818 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.816, "ece": 0.1456 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.4056, "ece": 0.3895 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4958, "ece": 0.1986 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.875, "ece": 0.087 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.712, "ece": 0.1749 } } }, { "server": "ollaya", "model": "decider:4b", "records": 3880, "errors": 0, "macro_accuracy": 0.7565, "micro_accuracy": 0.7649, "macro_ece": 0.0887, "pooled_ece": 0.0427, "brier": 0.3279, "median_ms": 188.4, "p95_ms": 1485.8, "by_type": { "choice": 0.8169, "noul": 0.8203, "score": 0.5495 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7412, "ece": 0.1472 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8629, "ece": 0.041 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8429, "ece": 0.0278 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8667, "ece": 0.0359 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7893, "ece": 0.0429 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.768, "ece": 0.1219 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.9298, "ece": 0.0138 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8733, "ece": 0.0565 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.804, "ece": 0.0467 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.4498, "ece": 0.1422 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.3792, "ece": 0.063 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8194, "ece": 0.3299 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.708, "ece": 0.0844 } } }, { "server": "ollaya", "model": "kev:9b", "records": 3880, "errors": 0, "macro_accuracy": 0.7526, "micro_accuracy": 0.7588, "macro_ece": 0.1059, "pooled_ece": 0.0583, "brier": 0.3563, "median_ms": 229.1, "p95_ms": 542.1, "by_type": { "choice": 0.816, "noul": 0.8073, "score": 0.5558 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7346, "ece": 0.1563 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8457, "ece": 0.0484 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8371, "ece": 0.0662 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8467, "ece": 0.0724 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7425, "ece": 0.1521 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.804, "ece": 0.1216 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.9064, "ece": 0.0375 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8433, "ece": 0.0558 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.8, "ece": 0.0844 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3855, "ece": 0.1351 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4625, "ece": 0.0714 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8194, "ece": 0.3039 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.756, "ece": 0.0717 } } }, { "server": "ollama", "model": "nimble", "records": 3880, "errors": 0, "macro_accuracy": 0.7488, "micro_accuracy": 0.7588, "macro_ece": 0.1375, "pooled_ece": 0.1216, "brier": 0.3567, "median_ms": 209.9, "p95_ms": 372.1, "by_type": { "choice": 0.8253, "noul": 0.7901, "score": 0.5526 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7863, "ece": 0.1358 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.84, "ece": 0.0813 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8286, "ece": 0.0827 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.86, "ece": 0.0787 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7425, "ece": 0.1801 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.744, "ece": 0.1841 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8997, "ece": 0.0475 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.78, "ece": 0.0867 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.824, "ece": 0.1042 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3494, "ece": 0.3441 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.475, "ece": 0.1859 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8333, "ece": 0.1347 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.772, "ece": 0.1418 } } }, { "server": "ollaya", "model": "nimble:9b", "records": 3880, "errors": 0, "macro_accuracy": 0.7476, "micro_accuracy": 0.7562, "macro_ece": 0.0996, "pooled_ece": 0.0225, "brier": 0.3389, "median_ms": 309.5, "p95_ms": 640, "by_type": { "choice": 0.7997, "noul": 0.8263, "score": 0.5295 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7679, "ece": 0.065 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.84, "ece": 0.0826 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8086, "ece": 0.1012 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8533, "ece": 0.0304 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.8227, "ece": 0.0238 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.8, "ece": 0.0762 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8261, "ece": 0.0679 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8433, "ece": 0.1198 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.812, "ece": 0.0499 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3414, "ece": 0.159 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4, "ece": 0.081 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8472, "ece": 0.3789 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.756, "ece": 0.0589 } } }, { "server": "ollama", "model": "tev1:4b", "records": 3880, "errors": 0, "macro_accuracy": 0.7472, "micro_accuracy": 0.7552, "macro_ece": 0.1014, "pooled_ece": 0.0749, "brier": 0.3503, "median_ms": 406.4, "p95_ms": 738.5, "by_type": { "choice": 0.8199, "noul": 0.7917, "score": 0.5519 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7462, "ece": 0.1414 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8571, "ece": 0.0549 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8286, "ece": 0.069 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8533, "ece": 0.0756 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7692, "ece": 0.1184 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.824, "ece": 0.0875 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.9197, "ece": 0.0192 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.74, "ece": 0.0724 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.772, "ece": 0.0801 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3655, "ece": 0.2427 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4917, "ece": 0.0982 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.7986, "ece": 0.167 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.748, "ece": 0.0917 } } }, { "server": "ollaya", "model": "kev:4b", "records": 3880, "errors": 0, "macro_accuracy": 0.7448, "micro_accuracy": 0.7528, "macro_ece": 0.1129, "pooled_ece": 0.062, "brier": 0.3595, "median_ms": 195.5, "p95_ms": 447.4, "by_type": { "choice": 0.8153, "noul": 0.8161, "score": 0.5082 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7212, "ece": 0.1563 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8629, "ece": 0.1604 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.86, "ece": 0.2062 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8533, "ece": 0.0639 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7826, "ece": 0.0622 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.816, "ece": 0.0608 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.9164, "ece": 0.0429 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8167, "ece": 0.102 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.812, "ece": 0.0751 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3414, "ece": 0.0989 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.3708, "ece": 0.0372 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8125, "ece": 0.3662 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.716, "ece": 0.0359 } } }, { "server": "ollaya", "model": "winnow:e4b", "records": 3880, "errors": 0, "macro_accuracy": 0.7343, "micro_accuracy": 0.7371, "macro_ece": 0.1023, "pooled_ece": 0.0714, "brier": 0.3802, "median_ms": 43.1, "p95_ms": 77.1, "by_type": { "choice": 0.7762, "noul": 0.798, "score": 0.5582 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6962, "ece": 0.1696 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8514, "ece": 0.0237 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.8086, "ece": 0.0505 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8167, "ece": 0.0998 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.8161, "ece": 0.0595 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.808, "ece": 0.0437 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8328, "ece": 0.0387 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.7533, "ece": 0.048 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.796, "ece": 0.0872 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3775, "ece": 0.3159 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.45, "ece": 0.0731 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8472, "ece": 0.2233 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.692, "ece": 0.0974 } } }, { "server": "ollaya", "model": "decider:2b", "records": 3880, "errors": 0, "macro_accuracy": 0.7033, "micro_accuracy": 0.7106, "macro_ece": 0.1075, "pooled_ece": 0.0872, "brier": 0.4021, "median_ms": 97.8, "p95_ms": 642.4, "by_type": { "choice": 0.7669, "noul": 0.7732, "score": 0.4808 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6361, "ece": 0.213 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.82, "ece": 0.0687 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.7943, "ece": 0.0752 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8033, "ece": 0.0836 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7826, "ece": 0.0687 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.72, "ece": 0.1475 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8562, "ece": 0.0431 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.84, "ece": 0.0376 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.72, "ece": 0.1476 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.4257, "ece": 0.0904 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.35, "ece": 0.0486 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.6667, "ece": 0.3138 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.728, "ece": 0.0591 } } }, { "server": "ollaya", "model": "decider:0.8b", "records": 3880, "errors": 0, "macro_accuracy": 0.6687, "micro_accuracy": 0.6799, "macro_ece": 0.1237, "pooled_ece": 0.0612, "brier": 0.421, "median_ms": 87.5, "p95_ms": 561, "by_type": { "choice": 0.7449, "noul": 0.7187, "score": 0.4585 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6778, "ece": 0.1041 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8171, "ece": 0.0707 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.7829, "ece": 0.0651 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7933, "ece": 0.0326 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7224, "ece": 0.085 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.592, "ece": 0.1471 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8227, "ece": 0.0313 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.87, "ece": 0.0308 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.616, "ece": 0.1876 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3976, "ece": 0.0895 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.1375, "ece": 0.2462 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8403, "ece": 0.44 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.624, "ece": 0.0775 } } }, { "server": "ollama", "model": "tev1:0.8b", "records": 3880, "errors": 0, "macro_accuracy": 0.6391, "micro_accuracy": 0.6505, "macro_ece": 0.1613, "pooled_ece": 0.1288, "brier": 0.5009, "median_ms": 60.9, "p95_ms": 101.2, "by_type": { "choice": 0.7073, "noul": 0.6929, "score": 0.4357 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6878, "ece": 0.1326 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.7914, "ece": 0.0985 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.6943, "ece": 0.1121 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7767, "ece": 0.074 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.709, "ece": 0.0821 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.64, "ece": 0.1717 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.7592, "ece": 0.0223 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.7667, "ece": 0.0764 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.572, "ece": 0.1491 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.3293, "ece": 0.3355 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.1375, "ece": 0.6458 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8403, "ece": 0.1007 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.604, "ece": 0.0954 } } }, { "server": "ollaya", "model": "jevk5:4b", "records": 3880, "errors": 700, "macro_accuracy": 0.6205, "micro_accuracy": 0.599, "macro_ece": 0.0702, "pooled_ece": 0.04, "brier": 0.373, "median_ms": 25.6, "p95_ms": 60.2, "by_type": { "choice": 0.4539, "noul": 0.8175, "score": 0.5698 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6828, "ece": 0.0886 }, "massive-en-US": { "n": 350, "errors": 350, "accuracy": 0, "ece": null }, "massive-de-DE": { "n": 350, "errors": 350, "accuracy": 0, "ece": null }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8467, "ece": 0.03 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.7993, "ece": 0.0277 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.808, "ece": 0.0481 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8428, "ece": 0.0489 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8333, "ece": 0.0376 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.8, "ece": 0.1031 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.4177, "ece": 0.1289 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4375, "ece": 0.0566 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.8542, "ece": 0.1034 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.744, "ece": 0.0989 } } }, { "server": "ollaya", "model": "kev:0.8b", "records": 3880, "errors": 0, "macro_accuracy": 0.6106, "micro_accuracy": 0.6314, "macro_ece": 0.128, "pooled_ece": 0.0345, "brier": 0.4651, "median_ms": 105.6, "p95_ms": 208.2, "by_type": { "choice": 0.6637, "noul": 0.7288, "score": 0.3251 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6628, "ece": 0.0735 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.7486, "ece": 0.207 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.66, "ece": 0.1629 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7933, "ece": 0.0597 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.6622, "ece": 0.0162 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.66, "ece": 0.0669 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.7993, "ece": 0.0515 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8367, "ece": 0.137 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.692, "ece": 0.0893 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.2892, "ece": 0.1086 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.1375, "ece": 0.2744 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.5486, "ece": 0.2256 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.448, "ece": 0.1915 } } }, { "server": "ollaya", "model": "decision:eos", "records": 3880, "errors": 0, "macro_accuracy": 0.5905, "micro_accuracy": 0.6206, "macro_ece": 0.1158, "pooled_ece": 0.087, "brier": 0.478, "median_ms": 96.3, "p95_ms": 180.7, "by_type": { "choice": 0.6674, "noul": 0.7139, "score": 0.2565 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.6244, "ece": 0.1027 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.8171, "ece": 0.0573 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.7543, "ece": 0.0582 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.6833, "ece": 0.1294 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.8729, "ece": 0.0609 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.672, "ece": 0.2006 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.7692, "ece": 0.0477 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.6533, "ece": 0.1218 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.688, "ece": 0.048 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.257, "ece": 0.2262 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.2208, "ece": 0.1048 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.2917, "ece": 0.1242 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.372, "ece": 0.2242 } } }, { "server": "ollaya", "model": "laya:multilingual", "records": 3880, "errors": 27, "macro_accuracy": 0.5787, "micro_accuracy": 0.6121, "macro_ece": 0.178, "pooled_ece": 0.1548, "brier": 0.5365, "median_ms": 13.6, "p95_ms": 14.9, "by_type": { "choice": 0.639, "noul": 0.7287, "score": 0.2282 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7629, "ece": 0.0748 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.6229, "ece": 0.124 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.4429, "ece": 0.2858 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7533, "ece": 0.1551 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.602, "ece": 0.3104 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.768, "ece": 0.0761 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8462, "ece": 0.0962 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.92, "ece": 0.0679 }, "aegis2": { "n": 250, "errors": 1, "accuracy": 0.6, "ece": 0.258 }, "helpsteer2": { "n": 249, "errors": 26, "accuracy": 0.261, "ece": 0.1265 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.125, "ece": 0.3258 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.2986, "ece": 0.14 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.52, "ece": 0.2738 } } }, { "server": "ollaya", "model": "laya:en", "records": 3880, "errors": 453, "macro_accuracy": 0.5321, "micro_accuracy": 0.5853, "macro_ece": 0.1765, "pooled_ece": 0.1063, "brier": 0.48, "median_ms": 16.9, "p95_ms": 18, "by_type": { "choice": 0.652, "noul": 0.6792, "score": 0.0869 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7896, "ece": 0.129 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.6857, "ece": 0.1375 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.4086, "ece": 0.2871 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.8367, "ece": 0.0807 }, "squad2": { "n": 299, "errors": 19, "accuracy": 0.6187, "ece": 0.1819 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.576, "ece": 0.2128 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.8763, "ece": 0.0579 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.9367, "ece": 0.1714 }, "aegis2": { "n": 250, "errors": 8, "accuracy": 0.428, "ece": 0.3384 }, "helpsteer2": { "n": 249, "errors": 144, "accuracy": 0.1566, "ece": 0.0894 }, "summeval-relevance": { "n": 240, "errors": 170, "accuracy": 0.0625, "ece": 0.1428 }, "summeval-consistency": { "n": 144, "errors": 101, "accuracy": 0.0417, "ece": 0.1958 }, "pubmedqa": { "n": 250, "errors": 11, "accuracy": 0.5, "ece": 0.27 } } }, { "server": "ollaya", "model": "von:1.1", "records": 3880, "errors": 0, "macro_accuracy": 0.4853, "micro_accuracy": 0.526, "macro_ece": 0.1815, "pooled_ece": 0.0979, "brier": 0.6248, "median_ms": 15.7, "p95_ms": 21.8, "by_type": { "choice": 0.5164, "noul": 0.6365, "score": 0.1815 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.7613, "ece": 0.2366 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.6914, "ece": 0.2641 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.2657, "ece": 0.5944 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7167, "ece": 0.1275 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.5017, "ece": 0.1688 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.576, "ece": 0.0393 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.3278, "ece": 0.1417 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.76, "ece": 0.144 }, "aegis2": { "n": 250, "errors": 0, "accuracy": 0.628, "ece": 0.0563 }, "helpsteer2": { "n": 249, "errors": 0, "accuracy": 0.1446, "ece": 0.1433 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.3167, "ece": 0.0643 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.0833, "ece": 0.2673 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.536, "ece": 0.1112 } } }, { "server": "ollaya", "model": "nli:deberta-v3-large", "records": 3880, "errors": 326, "macro_accuracy": 0.4583, "micro_accuracy": 0.4665, "macro_ece": 0.2092, "pooled_ece": 0.1956, "brier": 0.6352, "median_ms": 14.3, "p95_ms": 63.9, "by_type": { "choice": 0.4433, "noul": 0.6639, "score": 0.1405 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.2604, "ece": 0.1569 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.6, "ece": 0.41 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.5457, "ece": 0.4059 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.7467, "ece": 0.0586 }, "squad2": { "n": 299, "errors": 19, "accuracy": 0.4749, "ece": 0.4351 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.596, "ece": 0.0537 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.3144, "ece": 0.2782 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.89, "ece": 0.0672 }, "aegis2": { "n": 250, "errors": 7, "accuracy": 0.612, "ece": 0.2001 }, "helpsteer2": { "n": 249, "errors": 124, "accuracy": 0.1687, "ece": 0.1401 }, "summeval-relevance": { "n": 240, "errors": 115, "accuracy": 0.225, "ece": 0.1025 }, "summeval-consistency": { "n": 144, "errors": 53, "accuracy": 0.0278, "ece": 0.3603 }, "pubmedqa": { "n": 250, "errors": 8, "accuracy": 0.496, "ece": 0.0508 } } }, { "server": "ollaya", "model": "gliclass:large", "records": 3880, "errors": 28, "macro_accuracy": 0.4336, "micro_accuracy": 0.4263, "macro_ece": 0.3018, "pooled_ece": 0.2838, "brier": 0.8105, "median_ms": 14.2, "p95_ms": 25.8, "by_type": { "choice": 0.3658, "noul": 0.6001, "score": 0.2689 }, "per_subset": { "vitaminc-dev": { "n": 599, "errors": 0, "accuracy": 0.1202, "ece": 0.6138 }, "massive-en-US": { "n": 350, "errors": 0, "accuracy": 0.5743, "ece": 0.1701 }, "massive-de-DE": { "n": 350, "errors": 0, "accuracy": 0.4943, "ece": 0.1679 }, "boolq": { "n": 300, "errors": 0, "accuracy": 0.58, "ece": 0.3773 }, "squad2": { "n": 299, "errors": 0, "accuracy": 0.505, "ece": 0.3296 }, "paws": { "n": 250, "errors": 0, "accuracy": 0.544, "ece": 0.077 }, "multinli": { "n": 299, "errors": 0, "accuracy": 0.3244, "ece": 0.5065 }, "civil_comments": { "n": 300, "errors": 0, "accuracy": 0.8033, "ece": 0.045 }, "aegis2": { "n": 250, "errors": 1, "accuracy": 0.568, "ece": 0.3016 }, "helpsteer2": { "n": 249, "errors": 27, "accuracy": 0.2651, "ece": 0.0413 }, "summeval-relevance": { "n": 240, "errors": 0, "accuracy": 0.4583, "ece": 0.272 }, "summeval-consistency": { "n": 144, "errors": 0, "accuracy": 0.0833, "ece": 0.6608 }, "pubmedqa": { "n": 250, "errors": 0, "accuracy": 0.316, "ece": 0.3602 } } } ] }