{
 "schema": 1,
 "suite": "latency-triage",
 "date": "2026-10-01T11:18:51Z",
 "ollaya": "0.8.0",
 "device": "cuda",
 "protocol": {
  "preset": "triage",
  "questions": 5,
  "states": 30,
  "warm": 5,
  "n": 20,
  "client": "one request at a time over HTTP to /api/decide, timed by the client; each request a different short message, in turn"
 },
 "note": "",
 "results": [
  {
   "model": "clm:8b",
   "questions": "triage",
   "load_ms": 6261.1,
   "first_request_ms": 6494.3,
   "answered_by": "clm:8b",
   "p50_ms": 100.1,
   "p90_ms": 101.3,
   "min_ms": 97.6,
   "eval_p50_ms": 99.1,
   "input_tokens": 337,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "cygnet:12b",
   "questions": "triage",
   "load_ms": 4610.6,
   "first_request_ms": 4777.3,
   "answered_by": "cygnet:12b",
   "p50_ms": 168.7,
   "p90_ms": 169.5,
   "min_ms": 165.6,
   "eval_p50_ms": 167.6,
   "input_tokens": 746,
   "device": "cuda:0",
   "precision": "Q8_0"
  },
  {
   "model": "decider:0.8b",
   "questions": "triage",
   "load_ms": 13585.2,
   "first_request_ms": 13924.9,
   "answered_by": "decider:0.8b",
   "p50_ms": 176.8,
   "p90_ms": 178,
   "min_ms": 175.7,
   "eval_p50_ms": 175.7,
   "input_tokens": 552,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "decider:2b",
   "questions": "triage",
   "load_ms": 23933.3,
   "first_request_ms": 24235.9,
   "answered_by": "decider:2b",
   "p50_ms": 207.9,
   "p90_ms": 211.3,
   "min_ms": 205.7,
   "eval_p50_ms": 206.8,
   "input_tokens": 544,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "decider:2b-vision",
   "questions": "triage",
   "load_ms": 36921.1,
   "first_request_ms": 37135.1,
   "answered_by": "decider:2b-vision",
   "p50_ms": 150.5,
   "p90_ms": 153.6,
   "min_ms": 148.2,
   "eval_p50_ms": 149.4,
   "input_tokens": 366,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "decider:4b",
   "questions": "triage",
   "load_ms": 27913.3,
   "first_request_ms": 28495.5,
   "answered_by": "decider:4b",
   "p50_ms": 439.5,
   "p90_ms": 442.9,
   "min_ms": 429.4,
   "eval_p50_ms": 438.4,
   "input_tokens": 544,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "decision:eos",
   "questions": "triage",
   "load_ms": 12937.1,
   "first_request_ms": 13253.3,
   "answered_by": "decision:eos",
   "p50_ms": 214.5,
   "p90_ms": 219.8,
   "min_ms": 212.7,
   "eval_p50_ms": 213.2,
   "input_tokens": 690,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "gliclass:large",
   "questions": "triage",
   "load_ms": 8444.4,
   "first_request_ms": 8463,
   "answered_by": "gliclass:large",
   "p50_ms": 15.7,
   "p90_ms": 16.4,
   "min_ms": 15.4,
   "eval_p50_ms": 14.8,
   "input_tokens": 278,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "jeb:4b",
   "questions": "triage",
   "load_ms": 11658.1,
   "first_request_ms": 11756.6,
   "answered_by": "jeb:4b",
   "p50_ms": 98,
   "p90_ms": 99.6,
   "min_ms": 95.8,
   "eval_p50_ms": 97,
   "input_tokens": 559,
   "device": "cuda:0",
   "precision": "Q8_0"
  },
  {
   "model": "jeb:9b",
   "questions": "triage",
   "load_ms": 21288.6,
   "first_request_ms": 21416.5,
   "answered_by": "jeb:9b",
   "p50_ms": 113,
   "p90_ms": 115.9,
   "min_ms": 109.3,
   "eval_p50_ms": 111.9,
   "input_tokens": 569,
   "device": "cuda:0",
   "precision": "Q8_0"
  },
  {
   "model": "jeb:27b",
   "questions": "triage",
   "load_ms": 34368.1,
   "first_request_ms": 34634.8,
   "answered_by": "jeb:27b",
   "p50_ms": 260.6,
   "p90_ms": 267.5,
   "min_ms": 248.4,
   "eval_p50_ms": 259.5,
   "input_tokens": 554,
   "device": "cuda:0",
   "precision": "Q4_K - MEDIUM"
  },
  {
   "model": "jeeves:9b",
   "questions": "triage",
   "load_ms": 46432.3,
   "first_request_ms": 47309.2,
   "answered_by": "jeeves:9b",
   "p50_ms": 668.2,
   "p90_ms": 670.9,
   "min_ms": 648.4,
   "eval_p50_ms": 667.2,
   "input_tokens": 503,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "jevk5:4b",
   "questions": "triage",
   "load_ms": 9867.2,
   "first_request_ms": 9976.4,
   "answered_by": "jevk5:4b",
   "p50_ms": 106.2,
   "p90_ms": 107,
   "min_ms": 105.1,
   "eval_p50_ms": 105.1,
   "input_tokens": 737,
   "device": "cuda:0",
   "precision": "Q8_0"
  },
  {
   "model": "kev:0.8b",
   "questions": "triage",
   "load_ms": 12546,
   "first_request_ms": 12771.7,
   "answered_by": "kev:0.8b",
   "p50_ms": 145,
   "p90_ms": 146,
   "min_ms": 144.1,
   "eval_p50_ms": 143.9,
   "input_tokens": 288,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "kev:4b",
   "questions": "triage",
   "load_ms": 28892.4,
   "first_request_ms": 29329.9,
   "answered_by": "kev:4b",
   "p50_ms": 326.7,
   "p90_ms": 328.3,
   "min_ms": 325.6,
   "eval_p50_ms": 325.6,
   "input_tokens": 278,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "kev:9b",
   "questions": "triage",
   "load_ms": 47523.9,
   "first_request_ms": 48041.2,
   "answered_by": "kev:9b",
   "p50_ms": 375.2,
   "p90_ms": 406.3,
   "min_ms": 373.7,
   "eval_p50_ms": 374.2,
   "input_tokens": 288,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "laya:en",
   "questions": "triage",
   "load_ms": 4381.2,
   "first_request_ms": 4416.9,
   "answered_by": "laya:en",
   "p50_ms": 16.3,
   "p90_ms": 17.2,
   "min_ms": 15.4,
   "eval_p50_ms": 15.4,
   "input_tokens": 349,
   "device": "cuda:0",
   "precision": "F16"
  },
  {
   "model": "laya:multilingual",
   "questions": "triage",
   "load_ms": 4264.7,
   "first_request_ms": 4296.8,
   "answered_by": "laya:multilingual",
   "p50_ms": 13.9,
   "p90_ms": 14.4,
   "min_ms": 13.1,
   "eval_p50_ms": 13,
   "input_tokens": 356,
   "device": "cuda:0",
   "precision": "F16"
  },
  {
   "model": "nimble:9b",
   "questions": "triage",
   "load_ms": 51106.7,
   "first_request_ms": 53076.1,
   "answered_by": "nimble:9b",
   "p50_ms": 1789.4,
   "p90_ms": 1890,
   "min_ms": 1646.5,
   "eval_p50_ms": 1788.4,
   "input_tokens": 2973,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "nli:deberta-v3-large",
   "questions": "triage",
   "load_ms": 7892.9,
   "first_request_ms": 7913.1,
   "answered_by": "nli:deberta-v3-large",
   "p50_ms": 17.6,
   "p90_ms": 17.9,
   "min_ms": 17.3,
   "eval_p50_ms": 16.7,
   "input_tokens": 619,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "nli:modernbert-large",
   "questions": "triage",
   "load_ms": 6073.9,
   "first_request_ms": 6092.3,
   "answered_by": "nli:modernbert-large",
   "p50_ms": 16.5,
   "p90_ms": 16.8,
   "min_ms": 16.1,
   "eval_p50_ms": 15.7,
   "input_tokens": 395,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "qwen3guard:0.6b",
   "questions": "builtin (4)",
   "load_ms": 0,
   "first_request_ms": 35.3,
   "answered_by": "qwen3guard:0.6b",
   "p50_ms": 34.7,
   "p90_ms": 35.2,
   "min_ms": 33.6,
   "eval_p50_ms": 33.6,
   "input_tokens": 642,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "von:1.1",
   "questions": "triage",
   "load_ms": 6828.3,
   "first_request_ms": 6851.3,
   "answered_by": "von:1.1",
   "p50_ms": 21.4,
   "p90_ms": 21.7,
   "min_ms": 20.9,
   "eval_p50_ms": 20.2,
   "input_tokens": 374,
   "device": "cuda:0",
   "precision": "F32"
  },
  {
   "model": "winnow:e4b",
   "questions": "triage",
   "load_ms": 16644,
   "first_request_ms": 16781.2,
   "answered_by": "winnow:e4b",
   "p50_ms": 121.2,
   "p90_ms": 123.2,
   "min_ms": 119,
   "eval_p50_ms": 120.2,
   "input_tokens": 686,
   "device": "cuda:0",
   "precision": "Q8_0"
  },
  {
   "model": "winnow:12b",
   "questions": "triage",
   "load_ms": 25483.6,
   "first_request_ms": 25680.7,
   "answered_by": "winnow:12b",
   "p50_ms": 156.7,
   "p90_ms": 158.4,
   "min_ms": 144.5,
   "eval_p50_ms": 155.6,
   "input_tokens": 706,
   "device": "cuda:0",
   "precision": "Q8_0"
  }
 ],
 "machine": "rtx-5090",
 "machine_detail": {
  "id": "rtx-5090",
  "cpu": "AMD Ryzen Threadripper 3960X 24-Core Processor",
  "threads": 48,
  "gpus": [
   {
    "name": "NVIDIA GeForce RTX 5090",
    "driver": "596.21",
    "memory": "32607 MiB"
   }
  ],
  "os": "Linux 6.6.87.2-microsoft-standard-WSL2"
 }
}