{
 "site": "https://benchmarks.site",
 "built": "2026-11-17",
 "note": "All result tables from benchmarks.site. Values are strings exactly as published. See /methodology/ for caveats.",
 "tables": [
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "1. Test of 2026-09-22: clean speech synthesis, Polish + English in one recording",
   "columns": [
    "model",
    "platform",
    "WER PL clean",
    "WER EN",
    "mix clean, no language hint",
    "mix noisy",
    "runs with English preserved (verbatim)",
    "latency, 20–48 s clip",
    "45-min file in one request"
   ],
   "rows": [
    [
     "Qwen/Qwen3-ASR-1.7B",
     "DeepInfra",
     "2.5%",
     "0%",
     "6.4% (verbatim)",
     "12.8%",
     "17/17",
     "1.8–9 s",
     "15–27 s, 42/42 English sentences preserved"
    ],
    [
     "Qwen/Qwen3-ASR-0.6B",
     "DeepInfra",
     "18.0%",
     "0%",
     "8.5%",
     "21–23%",
     "12/12",
     "1.6–5.5 s",
     "not tested"
    ],
    [
     "openai/whisper-large-v3-turbo",
     "DeepInfra",
     "4.9% (drops words at the 30 s window boundary)",
     "0%",
     "2.1% when correct, 45–55% when hallucinating",
     "2–4%",
     "10/22",
     "1.5–3.9 s (up to 10 s)",
     "13.5 s, only 21/42 English sentences preserved; 88 MB WAV file accepted"
    ],
    [
     "openai/whisper-large-v3",
     "DeepInfra",
     "6.6%",
     "0%",
     "44.7% (English sentence translated into Polish every time)",
     "46.8% (English omitted)",
     "0/12",
     "5.5–23 s",
     "not tested"
    ],
    [
     "mistralai/Voxtral-Small-24B-2507",
     "DeepInfra",
     "77% without a language hint (translates into English); 4.9% with Polish specified, but wrong month („maja” (\"of May\"))",
     "0%",
     "44.7% (English omitted)",
     "2.1% or 44.7% (inconsistent)",
     "0/10",
     "3–8.8 s",
     "not tested"
    ],
    [
     "mistralai/Voxtral-Mini-3B-2507",
     "DeepInfra",
     "64.8% without a language hint (output in Russian); 9% with a hint (output in LaTeX)",
     "1.2%",
     "76.6% (everything in English)",
     "94–96%",
     "0/10",
     "1.5–3.3 s",
     "not tested"
    ],
    [
     "nvidia/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b",
     "DeepInfra",
     "47.5% (40% with Polish specified)",
     "0%",
     "23.4% (writes in Cyrillic)",
     "38%",
     "6/12 (only with the language specified)",
     "1.6–5.6 s",
     "not tested"
    ],
    [
     "google/gemini-3.1-flash-lite (chat model with audio input)",
     "DeepInfra → Google",
     "0–1%",
     "0%",
     "0% (verbatim)",
     "0%",
     "3/3",
     "2.5–3.4 s; 10 min in 10.5 s",
     "46 min without streaming: 300 s timeout exceeded"
    ],
    [
     "google/gemini-3.5-flash (chat model with audio input)",
     "DeepInfra → Google",
     "approx. 1%",
     "n/a",
     "n/a",
     "0%",
     "1/1",
     "2.7–3.6 s",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-01.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "2. Degraded recordings, 2026-09-29 (single models, no text correction)",
   "columns": [
    "model",
    "platform",
    "clean: WER (facts/109)",
    "classroom: WER (facts/109)",
    "back: WER (facts/109)",
    "notes"
   ],
   "rows": [
    [
     "Qwen/Qwen3-ASR-1.7B",
     "DeepInfra",
     "10.9% (80)",
     "25.8% (61)",
     "63.8% (20)",
     "deterministic; keeps English verbatim; in reverb loses about two thirds of the words"
    ],
    [
     "Qwen/Qwen3-ASR-0.6B",
     "DeepInfra",
     "26.8% (60)",
     "n/a",
     "88.1% (0)",
     "unfit for Polish speech in reverb"
    ],
    [
     "openai/whisper-large-v3",
     "DeepInfra",
     "13.3% (89)*",
     "15.8% (82)*",
     "34.4% (51) in one run, 47.4% (50) in the second",
     "*one 30-s window lost; non-deterministic; translates English sentences into Polish; once every few runs turns a whole window into gibberish or drops it (topic \"history\" 55–71% WER even on clean)"
    ],
    [
     "openai/whisper-large-v3-turbo",
     "DeepInfra",
     "n/a",
     "n/a",
     "41.9% (47)",
     "one run"
    ],
    [
     "MiMo-V2.6-Pro and MiMo-V2.6-Flash (chat model, audio in the message)",
     "DeepInfra",
     "n/a",
     "n/a",
     "> 800%",
     "loops („w tym samym roku…” (\"in the same year…\"))"
    ],
    [
     "Inkling, Inkling-Small, gemma-4-31B-it",
     "DeepInfra",
     "n/a",
     "n/a",
     "n/a",
     "the interface rejects audio input (HTTP 400)"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-02.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "Earlier measurement, 2026-09-23 (one deliberately degraded recording)",
   "columns": [
    "model",
    "platform",
    "WER PL",
    "WER PL+mix (average)",
    "English preserved",
    "notes"
   ],
   "rows": [
    [
     "Qwen/Qwen3-ASR-1.7B, raw audio",
     "DeepInfra",
     "19.0%",
     "26.0%",
     "71%",
     ""
    ],
    [
     "openai/whisper-large-v3-turbo",
     "DeepInfra",
     "23.3%",
     "n/a",
     "n/a",
     ""
    ],
    [
     "openai/whisper-large-v3",
     "DeepInfra",
     "24.8%",
     "n/a",
     "n/a",
     ""
    ],
    [
     "Voxtral Mini and Small",
     "DeepInfra",
     "≥ 90%",
     "n/a",
     "n/a",
     "unfit for Polish"
    ],
    [
     "Nemotron-Diarization (used as ASR)",
     "DeepInfra",
     "≥ 90%",
     "n/a",
     "n/a",
     "a speaker-recognition model, not a transcription model"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-03.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "3. Language lessons: Polish and a foreign language in one sentence, 2026-09-29/30",
   "columns": [
    "model",
    "level",
    "WER overall",
    "WER of Polish fragments",
    "foreign preserved",
    "translated",
    "garbled",
    "lost"
   ],
   "rows": [
    [
     "Qwen3-ASR-1.7B",
     "clean",
     "15.4%",
     "16.0%",
     "86.1%",
     "0.5%",
     "5.8%",
     "7.7%"
    ],
    [
     "whisper-large-v3",
     "clean",
     "34.3%",
     "10.8%",
     "41.6%",
     "30.7%",
     "7.9%",
     "19.8%"
    ],
    [
     "Qwen3-ASR-1.7B",
     "classroom",
     "35.6%",
     "31.7%",
     "62.1%",
     "10.9%",
     "10.9%",
     "16.1%"
    ],
    [
     "whisper-large-v3",
     "classroom",
     "33.4%",
     "29.9%",
     "68.4%",
     "19.3%",
     "8.4%",
     "4.0%"
    ],
    [
     "Qwen3-ASR-1.7B",
     "back",
     "65.5%",
     "67.6%",
     "40.5%",
     "7.7%",
     "13.4%",
     "38.4%"
    ],
    [
     "whisper-large-v3",
     "back",
     "56.5%",
     "60.1%",
     "56.1%",
     "7.8%",
     "7.3%",
     "28.8%"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-04.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "3. Language lessons: Polish and a foreign language in one sentence, 2026-09-29/30",
   "columns": [
    "language",
    "WER classroom",
    "foreign preserved / translated, classroom",
    "WER back",
    "foreign preserved / translated, back"
   ],
   "rows": [
    [
     "Spanish",
     "39.1%",
     "60.1% / 27.2%",
     "63.9%",
     "44.3% / 4.5%"
    ],
    [
     "English",
     "14.5%",
     "90.3% / 2.6%",
     "32.1%",
     "85.1% / 0.0%"
    ],
    [
     "German",
     "41.7%",
     "61.2% / 21.4%",
     "67.3%",
     "48.5% / 23.3%"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-05.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "4. External providers on the same recordings, 2026-11-06",
   "columns": [
    "model",
    "platform",
    "WER %",
    "facts out of 109",
    "numbers out of 79",
    "names out of 54",
    "compute time, s per minute of recording",
    "word timestamps"
   ],
   "rows": [
    [
     "Soniox stt-async-v6",
     "Soniox",
     "2.4 / 3.4 / 8.9",
     "103 / 97 / 87",
     "77 / 77 / 77",
     "52 / 50 / 42",
     "8.6–14.8",
     "yes"
    ],
    [
     "gpt-transcribe",
     "OpenAI",
     "1.2 / 4.7 / 14.0",
     "107 / 104 / 92",
     "78 / 77 / 75",
     "54 / 53 / 43",
     "2.8–4.3",
     "no"
    ],
    [
     "gpt-4o-transcribe",
     "OpenAI",
     "16.7 / 16.8 / 30.7",
     "79 / 76 / 65",
     "67 / 67 / 67",
     "27 / 29 / 19",
     "3.5–5.6",
     "no"
    ],
    [
     "gpt-4o-mini-transcribe",
     "OpenAI",
     "16.4 / 19.8 / 61.5",
     "82 / 82 / 51",
     "68 / 70 / 55",
     "29 / 39 / 15",
     "2.3–2.7",
     "no"
    ],
    [
     "gpt-4o-transcribe-diarize (classroom level only)",
     "OpenAI",
     "11.2",
     "88",
     "77",
     "44",
     "29.9",
     "segments"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-06.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "5. Live speech (streaming), 2026-11-05/06",
   "columns": [
    "model",
    "platform",
    "WER clean (hist./math/biol.)",
    "WER classroom (hist./math/biol.)"
   ],
   "rows": [
    [
     "Nemotron 3.5 ASR Multilingual, language specified (20 s chunks)",
     "DeepInfra",
     "26.5 / 29.5 / 31.4%",
     "43.7 / 39.9 / 46.3%"
    ],
    [
     "Qwen3-ASR (20 s chunks)",
     "DeepInfra",
     "17.2 / 11.3 / 69.9%*",
     "28.5 / 16.4 / 24.2%"
    ],
    [
     "Soniox stt-rt-v3",
     "Soniox",
     "9.3 / 6.4 / 6.9%",
     "10.7 / 6.4 / 8.2%"
    ],
    [
     "Soniox stt-rt-v6",
     "Soniox",
     "8.2 / 6.4 / 6.6%",
     "7.6 / 6.2 / 7.4%"
    ],
    [
     "OpenAI gpt-live-transcribe",
     "OpenAI",
     "9.0 / 10.2 / 10.9%",
     "12.7 / 9.4 / 12.8%"
    ],
    [
     "OpenAI gpt-realtime-whisper",
     "OpenAI",
     "9.3 / 10.5 / 10.4%",
     "11.5 / 10.2 / 9.8%"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-07.csv"
  },
  {
   "benchmark": "asr-polish",
   "title": "ASR: Polish speech recognition",
   "area": "Speech",
   "section": "5. Live speech (streaming), 2026-11-05/06",
   "columns": [
    "model",
    "first partial result",
    "end of utterance → result (min / median / max)",
    "WER at pauses",
    "numbers / names from the lesson"
   ],
   "rows": [
    [
     "Soniox stt-rt-v3, PCM",
     "961–1009 ms",
     "280 / 499 / 1709 ms",
     "4.0%",
     "168/202, 65/68"
    ],
    [
     "Soniox stt-rt-v6, PCM",
     "1254–1316 ms",
     "254 / 461 / 691 ms",
     "3.1%",
     "167/202, 64/68"
    ],
    [
     "OpenAI gpt-live-transcribe",
     "861–3296 ms under load; 1268–1337 ms at pauses",
     "185 / 484 / 1643 ms (last text)",
     "5.0%",
     "141/202, 68/68"
    ],
    [
     "OpenAI gpt-realtime-whisper",
     "1503–7157 ms under load; 1673–1769 ms at pauses",
     "598 / 837 / 1310 ms (last text)",
     "6.6%",
     "138/202, 68/68"
    ]
   ],
   "url": "https://benchmarks.site/asr-polish/",
   "csv": "https://benchmarks.site/data/asr-polish-08.csv"
  },
  {
   "benchmark": "diarization",
   "title": "Diarization: who is speaking",
   "area": "Speech",
   "section": "1. nvidia/Nemotron-3-Diarization (DeepInfra), 2026-09-30",
   "columns": [
    "recording",
    "level",
    "DER",
    "groups / speakers",
    "several people at once: recall / precision (frames)",
    "\"overlap\" events detected"
   ],
   "rows": [
    [
     "history",
     "classroom",
     "6.8%",
     "3 / 4",
     "94% / 94%",
     "5/5"
    ],
    [
     "biology",
     "classroom",
     "6.2%",
     "3 / 5",
     "98% / 90%",
     "4/4"
    ],
    [
     "mathematics",
     "classroom",
     "11.5%",
     "3 / 5",
     "82% / 88%",
     "3/4"
    ],
    [
     "English",
     "classroom",
     "11.7%",
     "4 / 5",
     "75% / 88%",
     "4/5"
    ],
    [
     "history",
     "back",
     "8.3% (5.5%)",
     "3 / 4",
     "84% / 87%",
     "4/5"
    ],
    [
     "biology",
     "back",
     "6.5% (4.4%)",
     "3 / 5",
     "98% / 88%",
     "4/4"
    ],
    [
     "mathematics",
     "back",
     "13.5% (9.6%)",
     "2 / 5",
     "85% / 92%",
     "3/4"
    ],
    [
     "English",
     "back",
     "11.1% (7.0%)",
     "4 / 5",
     "91% / 87%",
     "4/5"
    ]
   ],
   "url": "https://benchmarks.site/diarization/",
   "csv": "https://benchmarks.site/data/diarization-01.csv"
  },
  {
   "benchmark": "diarization",
   "title": "Diarization: who is speaking",
   "area": "Speech",
   "section": "2. Comparison with external providers, 2026-11-06",
   "columns": [
    "model",
    "platform",
    "DER average",
    "several people at once detected",
    "notes"
   ],
   "rows": [
    [
     "nvidia/Nemotron-3-Diarization",
     "DeepInfra",
     "9.4% (classroom 9.1%)",
     "31 of 36",
     ""
    ],
    [
     "Soniox stt-async-v6 (diarization in file mode)",
     "Soniox",
     "13.6% (classroom 12.7%)",
     "0 of 36",
     "at the level of the \"one speaker\" answer; does not capture speech by several people"
    ],
    [
     "OpenAI gpt-4o-transcribe-diarize (classroom only)",
     "OpenAI",
     "18.1%",
     "0 of 18",
     "provides labeled segments; processing time 29.9 s per minute of recording"
    ]
   ],
   "url": "https://benchmarks.site/diarization/",
   "csv": "https://benchmarks.site/data/diarization-02.csv"
  },
  {
   "benchmark": "omni-video-audio",
   "title": "Omni models: video and audio input",
   "area": "Speech",
   "section": "1. Does the model hear the audio from a video file?",
   "columns": [
    "model",
    "board read",
    "audio from video_url"
   ],
   "rows": [
    [
     "google/gemma-4-31B-it (video + audio)",
     "yes",
     "no audio"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Flash (video + audio)",
     "yes",
     "no audio"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Pro (video + audio)",
     "yes",
     "no audio"
    ],
    [
     "Qwen/Qwen3.5-35B-A3B",
     "yes",
     "no audio"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "yes",
     "no audio"
    ],
    [
     "zai-org/GLM-5.3-Flash",
     "yes",
     "no audio"
    ]
   ],
   "url": "https://benchmarks.site/omni-video-audio/",
   "csv": "https://benchmarks.site/data/omni-video-audio-01.csv"
  },
  {
   "benchmark": "omni-video-audio",
   "title": "Omni models: video and audio input",
   "area": "Speech",
   "section": "2. How many frames does the model see?",
   "columns": [
    "model",
    "tokens per video (any length)",
    "tokens per 1 full image",
    "frames"
   ],
   "rows": [
    [
     "google/gemma-4-31B-it",
     "2421",
     "266",
     "32"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "approx. 4400",
     "883",
     "approx. 32"
    ],
    [
     "Qwen/Qwen3.5-35B-A3B",
     "approx. 11720",
     "883",
     "approx. 32"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Flash",
     "approx. 14250",
     "882",
     "16"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Pro",
     "14254",
     "n/a",
     "16"
    ],
    [
     "zai-org/GLM-5.3-Flash",
     "9732",
     "n/a",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/omni-video-audio/",
   "csv": "https://benchmarks.site/data/omni-video-audio-02.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "1. Test from 2026-09-22: 4 photos, 19 models",
   "columns": [
    "model",
    "CER",
    "diacW",
    "math",
    "nums",
    "date",
    "seq (s)",
    "conc (s)",
    "inTok"
   ],
   "rows": [
    [
     "google/gemini-3.1-flash-lite",
     "0.018",
     "1.00",
     "1.00",
     "0.97",
     "N (2025)",
     "4.3 / 2.8",
     "4.2",
     "1305"
    ],
    [
     "ByteDance/Seed-2.0-mini",
     "0.019",
     "0.99",
     "0.98",
     "1.00",
     "Y",
     "10.9 / 4.9",
     "7.4",
     "1515"
    ],
    [
     "google/gemma-4-31B-it-turbo",
     "0.019",
     "0.99",
     "0.98",
     "0.89",
     "N (2020)",
     "48–112 / 3–7",
     "21.6",
     "490"
    ],
    [
     "google/gemma-4-31B-it",
     "0.019",
     "0.99",
     "0.98",
     "0.89",
     "N (2023)",
     "n/a",
     "22.8",
     "490"
    ],
    [
     "Qwen/Qwen3.6-35B-A3B",
     "0.022",
     "0.99",
     "1.00",
     "0.89",
     "Y",
     "80 / 55",
     "58.8",
     "1169"
    ],
    [
     "Qwen/Qwen3.5-35B-A3B",
     "0.025",
     "0.96",
     "1.00",
     "0.89",
     "Y",
     "6.4 / 2.1",
     "4.3",
     "1170"
    ],
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "0.026",
     "0.97",
     "1.00",
     "0.89",
     "N (2025)",
     "9.5 / 2.6",
     "9.8",
     "1189"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "0.028",
     "0.93",
     "1.00",
     "0.89",
     "N (2025)",
     "8.7 / 2.2",
     "7.9",
     "2446"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "0.029",
     "0.95",
     "1.00",
     "0.89",
     "N (2025)",
     "32 / 2.7",
     "42.7",
     "2430"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Flash",
     "0.033",
     "0.93",
     "0.98",
     "0.87",
     "skipped",
     "4.7 / 2.1",
     "4.6",
     "2430"
    ],
    [
     "Qwen/Qwen3.8-27B",
     "0.034",
     "0.89",
     "0.98",
     "0.83",
     "N",
     "n/a",
     "26.1",
     "2438"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
     "0.038",
     "0.89",
     "0.95",
     "0.88",
     "Y",
     "n/a",
     "15.7",
     "549"
    ],
    [
     "Qwen/Qwen3-VL-30B-A3B-Instruct",
     "0.045",
     "0.92",
     "1.00",
     "0.89",
     "N (2025)",
     "43 / 2.3",
     "22.3",
     "2430"
    ],
    [
     "google/gemma-3-27b-it",
     "0.047",
     "0.92",
     "0.87",
     "0.81",
     "Y",
     "27 / 2.8",
     "17.8",
     "472"
    ],
    [
     "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
     "0.056",
     "0.84",
     "0.96",
     "0.87",
     "skipped",
     "5.9 / 2.1",
     "6.9",
     "2677"
    ],
    [
     "meta-llama/Llama-4-Scout-17B-16E-Instruct",
     "0.063",
     "0.82",
     "0.93",
     "0.82",
     "skipped",
     "n/a",
     "7.9",
     "2677"
    ],
    [
     "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
     "0.065",
     "0.81",
     "0.94",
     "0.85",
     "skipped",
     "n/a",
     "8.5",
     "2631"
    ],
    [
     "google/gemma-3-12b-it",
     "0.078",
     "0.91",
     "0.91",
     "0.82",
     "N (22.07)",
     "n/a",
     "15.1",
     "472"
    ],
    [
     "Qwen/Qwen3.5-9B (reasoning not disabled)",
     "0.369",
     "0.70",
     "0.79",
     "0.77",
     "Y",
     "n/a",
     "47",
     "1167"
    ],
    [
     "google/gemini-3.5-flash (reference, cursive only)",
     "0.023",
     "1.00",
     "1.00",
     "1.00",
     "–",
     "n/a",
     "5.5",
     "1305"
    ],
    [
     "google/gemma-4-26B-A4B-it",
     "0.023",
     "1.00",
     "0.91",
     "0.83",
     "–",
     "n/a",
     "98–180 (2 of 4 timed out)",
     "490"
    ],
    [
     "zai-org/GLM-5.3-Flash",
     "–",
     "–",
     "–",
     "–",
     "–",
     "all 4 calls with a photo timed out (180–400 s)",
     "–",
     "–"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-01.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "2. \"Hard\" test from 2026-09-29/30: 14 photos, handwriting and small numbers",
   "columns": [
    "model",
    "CER (10 pages)",
    "facts (10 pages)",
    "CER / facts (4 pages with numbers)",
    "time per photo, s",
    "measurement date"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "4.1%",
     "95.2%",
     "5.7% / 84%",
     "5–11",
     "2026-09-29"
    ],
    [
     "Qwen/Qwen3.5-35B-A3B",
     "3.1%",
     "95.2%",
     "2.5% / 94%",
     "4",
     "2026-09-29"
    ],
    [
     "moonshotai/Kimi-K2.6",
     "2.5%",
     "97.1%",
     "2.3% / 91%",
     "11–19",
     "2026-09-29"
    ],
    [
     "moonshotai/Kimi-K2.6 (separate measurement, 2 runs)",
     "2.8%",
     "96.7% (203/210)",
     "1.6% / 92.0%",
     "11.4",
     "2026-09-30"
    ],
    [
     "Qwen/Qwen3.5-397B-A17B",
     "2.8%",
     "94.3%",
     "1.1% / 86%",
     "7–9",
     "2026-09-29"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Pro",
     "2.7%",
     "95.2%",
     "3.5% / 86%",
     "7–8",
     "2026-09-29"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "3.4%",
     "95.2%",
     "2.4% / 91%",
     "16–26",
     "2026-09-29"
    ],
    [
     "Qwen/Qwen3.8-27B",
     "3.4%",
     "92.4%",
     "n/a",
     "7",
     "2026-09-29"
    ],
    [
     "MiniMaxAI/MiniMax-M3",
     "6.4%",
     "92.4%",
     "n/a",
     "10",
     "2026-09-29"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-02.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "Three hard photos of Polish notes, 2026-10-04",
   "columns": [
    "model",
    "CER",
    "facts",
    "time per whole page"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "4.5%",
     "90%",
     "about 5 s"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "7.8%",
     "82.5%",
     "25–265 s"
    ],
    [
     "Qwen/Qwen3.5-397B-A17B (low reasoning)",
     "1.5–1.9%",
     "88–92%",
     "1–4 min"
    ],
    [
     "Qwen/Qwen3.8-27B",
     "5.3–5.8%",
     "85–90%",
     "about 20 s"
    ],
    [
     "moonshotai/Kimi-K3 (variant described in the source as \"off\")",
     "0.4%",
     "100%",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-03.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "3. Textbook pages (print, scans and photos), 2026-10-05",
   "columns": [
    "model",
    "CER scan",
    "CER photo",
    "CER hard",
    "facts",
    "printed page number",
    "table as table",
    "s/page"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "0.10%",
     "0.13%",
     "0.38%",
     "99%",
     "100%",
     "3/3",
     "6.8"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "0.08%",
     "0.11%",
     "0.26%",
     "100%",
     "100%",
     "3/3",
     "8.0"
    ],
    [
     "Qwen/Qwen3.5-35B-A3B",
     "1.39%",
     "1.27%",
     "0.73%",
     "98%",
     "100%",
     "3/3",
     "5.4"
    ],
    [
     "Qwen/Qwen3.8-27B",
     "0.22%",
     "0.20%",
     "0.36%",
     "96%",
     "100%",
     "3/3",
     "12.1"
    ],
    [
     "Qwen/Qwen3.5-397B-A17B",
     "0.03%",
     "0.06%",
     "0.00%",
     "99%",
     "100%",
     "3/3",
     "12.6"
    ],
    [
     "moonshotai/Kimi-K2.6",
     "0.04%",
     "0.04%",
     "0.06%",
     "100%",
     "100%",
     "3/3",
     "30.2"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-04.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "Cheaper models, 2026-10-06 (the same 21 readings)",
   "columns": [
    "model (requested)",
    "responded",
    "CER scan",
    "CER photo",
    "CER hard",
    "facts",
    "page number"
   ],
   "rows": [
    [
     "Qwen/Qwen3.5-0.8B",
     "Qwen/Qwen3.5-9B",
     "0.31%",
     "0.64%",
     "0.51%",
     "98%",
     "90%"
    ],
    [
     "Qwen/Qwen3.5-2B",
     "Qwen/Qwen3.5-9B",
     "0.31%",
     "0.61%",
     "0.53%",
     "98%",
     "90%"
    ],
    [
     "Qwen/Qwen3.5-4B",
     "Qwen/Qwen3.5-9B",
     "0.31%",
     "0.64%",
     "0.53%",
     "98%",
     "90%"
    ],
    [
     "Qwen/Qwen3.5-9B",
     "Qwen/Qwen3.5-9B",
     "0.31%",
     "0.64%",
     "0.53%",
     "98%",
     "90%"
    ],
    [
     "deepseek-ai/DeepSeek-OCR",
     "google/gemma-4-31B-it",
     "0.55%",
     "0.48%",
     "1.12%",
     "96%",
     "100%"
    ],
    [
     "allenai/olmOCR-2-7B-1025",
     "google/gemma-4-31B-it",
     "0.55%",
     "0.73%",
     "1.01%",
     "96%",
     "100%"
    ],
    [
     "microsoft/Phi-4-multimodal-instruct",
     "google/gemma-3-12b-it",
     "100%",
     "100%",
     "100%",
     "0%",
     "0%"
    ],
    [
     "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
     "Mistral-Small-3.2-24B",
     "0.95%",
     "0.98%",
     "0.91%",
     "99%",
     "100%"
    ],
    [
     "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
     "(the same)",
     "0.30%",
     "0.98%",
     "0.94%",
     "98%",
     "100%"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-05.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "305-page scan and loops on leader dots, 2026-10-08",
   "columns": [
    "page",
    "Ling-3.0-flash-VL",
    "DeepSeek-V4.1-Flash"
   ],
   "rows": [
    [
     "regular (701–1467 output tokens)",
     "8–62 s, sometimes 100–185 s (platform queue)",
     "7–27 s"
    ],
    [
     "exercises with dotted lines",
     "8–100 s, passes",
     "response judged as looped (15 identical lines of dots)"
    ],
    [
     "table of contents with dots",
     "loop „. . . .” until the 6144-token limit (88–149 s), 3/3 attempts",
     "loop until 6144 tokens in one of the attempts (older prompt), 490 tokens in another"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-06.csv"
  },
  {
   "benchmark": "ocr-handwriting-boards-textbooks",
   "title": "OCR: handwritten notes, boards and textbook pages",
   "area": "Vision & OCR",
   "section": "4. Describing illustrations (charts, diagrams, maps), 2026-10-06",
   "columns": [
    "model",
    "facts",
    "clean",
    "photo crop",
    "full descriptions (of 18)",
    "logo = \"decoration\"",
    "s"
   ],
   "rows": [
    [
     "Qwen/Qwen3.5-397B-A17B",
     "98.2%",
     "100%",
     "96%",
     "16",
     "Y",
     "23.8"
    ],
    [
     "Qwen/Qwen3.6-27B",
     "97.6%",
     "99%",
     "96%",
     "14",
     "Y",
     "146.6"
    ],
    [
     "ByteDance/Seed-2.0-pro",
     "96.3%",
     "95%",
     "98%",
     "13",
     "N",
     "7.0"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "95.7%",
     "99%",
     "93%",
     "14",
     "Y",
     "15.2"
    ],
    [
     "thinkingmachines/Inkling",
     "95.1%",
     "96%",
     "94%",
     "11",
     "N",
     "5.2"
    ],
    [
     "XiaomiMiMo/MiMo-V2.6-Pro",
     "95.1%",
     "98%",
     "93%",
     "13",
     "Y",
     "8.5"
    ],
    [
     "meta-models/Muse-Glimmer-30B",
     "94.5%",
     "96%",
     "93%",
     "11",
     "Y",
     "33.3"
    ],
    [
     "MiniMaxAI/MiniMax-M3",
     "93.9%",
     "94%",
     "94%",
     "13",
     "Y",
     "12.6"
    ],
    [
     "inclusionAI/Ling-3.0-flash-VL",
     "93.3%",
     "95%",
     "91%",
     "12",
     "N",
     "10.1"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
     "92.7%",
     "95%",
     "90%",
     "12",
     "N",
     "4.9"
    ]
   ],
   "url": "https://benchmarks.site/ocr-handwriting-boards-textbooks/",
   "csv": "https://benchmarks.site/data/ocr-handwriting-boards-textbooks-07.csv"
  },
  {
   "benchmark": "ocr-exam-sheets",
   "title": "OCR: exam-sheet photos with handwritten answers",
   "area": "Vision & OCR",
   "section": "1. Reading a single handwritten word (Gemini 3.1 Pro)",
   "columns": [
    "conditions",
    "date",
    "attempts",
    "„opracował” (correct)",
    "„opanował” (word from print)"
   ],
   "rows": [
    [
     "prompt variant 1",
     "up to 2026-09-23",
     "18",
     "18/18",
     "0"
    ],
    [
     "prompt variant 1, whole-page reading",
     "2026-10-13",
     "20 readings (10 runs of 2)",
     "19/20",
     "1/20"
    ],
    [
     "prompt variant 2 (different description of the tasks on the page)",
     "2026-10-13",
     "10 readings (5 runs of 2)",
     "5/10",
     "5/10"
    ],
    [
     "prompt variant 3 (with a request to correct spelling only)",
     "2026-10-13",
     "3",
     "0/3",
     "3/3"
    ]
   ],
   "url": "https://benchmarks.site/ocr-exam-sheets/",
   "csv": "https://benchmarks.site/data/ocr-exam-sheets-01.csv"
  },
  {
   "benchmark": "ocr-exam-sheets",
   "title": "OCR: exam-sheet photos with handwritten answers",
   "area": "Vision & OCR",
   "section": "3. Blank exam-sheet page",
   "columns": [
    "model",
    "data",
    "date",
    "attempts",
    "result"
   ],
   "rows": [
    [
     "google/gemini-3.1-pro (full page reading)",
     "4 photos of exam-sheet pages with no handwriting at all, 1500×2000 px",
     "2026-10-13",
     "12 (3 per page; the agent reported 15 in total)",
     "12/12 with no invented answers"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct (full page reading; model reconstructed)",
     "rendered blank mathematics exam sheet, 2 pages",
     "2026-09-04",
     "2 runs",
     "10 invented answers on 2 pages, described as confidently as real ones; the second run repeated the same error"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct (closed question: is there handwriting on the page)",
     "the same blank sheet and the same pages with marks drawn in",
     "2026-09-04",
     "several (exact number unknown)",
     "correct: \"no handwriting\" on blank pages, \"handwriting present\" on pages with marks"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct (the same closed question)",
     "photo from section 1 (page with handwriting)",
     "2026-10-13",
     "40",
     "40/40 \"handwriting present\""
    ]
   ],
   "url": "https://benchmarks.site/ocr-exam-sheets/",
   "csv": "https://benchmarks.site/data/ocr-exam-sheets-02.csv"
  },
  {
   "benchmark": "ocr-exam-sheets",
   "title": "OCR: exam-sheet photos with handwritten answers",
   "area": "Vision & OCR",
   "section": "4. Identifying the exam sheet from photos",
   "columns": [
    "task",
    "model",
    "attempts",
    "result"
   ],
   "rows": [
    [
     "give the exam-sheet code visible in the photos",
     "Qwen/Qwen3-VL-235B-A22B-Instruct, google/gemini-3.1-pro (combined)",
     "6",
     "0/6 correct, including 2 times an English-language exam sheet was indicated. Qwen repeated the example values given in the prompt, Gemini indicated an exam sheet from a different year"
    ],
    [
     "transcribe a few sentences of print from each of 4 photos",
     "both models",
     "7 photo sets",
     "print transcribed usefully in all 7 sets (7/7; no CER measurement)"
    ]
   ],
   "url": "https://benchmarks.site/ocr-exam-sheets/",
   "csv": "https://benchmarks.site/data/ocr-exam-sheets-03.csv"
  },
  {
   "benchmark": "ocr-exam-sheets",
   "title": "OCR: exam-sheet photos with handwritten answers",
   "area": "Vision & OCR",
   "section": "5. Cost and time of a single call",
   "columns": [
    "model",
    "task",
    "date",
    "sample",
    "cost per call",
    "time per call"
   ],
   "rows": [
    [
     "google/gemini-3.1-pro (reasoning medium)",
     "full reading of a page with handwriting",
     "2026-09-17",
     "5 photos of one exam sheet",
     "approx. 12.3 gr",
     "n/a"
    ],
    [
     "google/gemini-3.1-pro",
     "full reading of a blank page",
     "2026-09-17",
     "same as above",
     "approx. 3.2 gr",
     "n/a"
    ],
    [
     "google/gemini-3.1-pro",
     "average per page (mix of blank and written pages)",
     "2026-09-17",
     "same as above",
     "approx. 6.9 gr",
     "n/a"
    ],
    [
     "google/gemini-3.1-pro",
     "full reading of a blank page (1500×2000 px)",
     "2026-10-13",
     "12",
     "n/a",
     "6.3–19.7 s (typically 15–20 s)"
    ],
    [
     "google/gemini-3.1-pro",
     "full reading of a page with handwriting (1500×2000 px)",
     "2026-10-13",
     "13",
     "n/a",
     "approx. 25–30 s (indirect estimate, a single call was not measured separately)"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "closed question about the page (one-word answer)",
     "2026-10",
     "n/a",
     "approx. 0.1 gr",
     "a few seconds (estimate)"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "transcribing fragments of print from 4 photos",
     "2026-09-22",
     "7 sets",
     "approx. 0.50 gr",
     "n/a"
    ],
    [
     "google/gemini-3.1-pro",
     "transcribing fragments of print from 4 photos",
     "2026-09-22",
     "7 sets",
     "approx. 4.1 gr",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/ocr-exam-sheets/",
   "csv": "https://benchmarks.site/data/ocr-exam-sheets-04.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "1. Lesson subject classification, 2026-09-22",
   "columns": [
    "model",
    "latency",
    "valid JSON",
    "correct without explanations (of 5)",
    "correct with explanations (of 6)"
   ],
   "rows": [
    [
     "openai/gpt-oss-20b",
     "1.0–1.9 s (one 3.6 s)",
     "100% (also without an enforced schema)",
     "4/5 (EDB → Biologia)",
     "6/6"
    ],
    [
     "openai/gpt-oss-120b",
     "1.9–3.4 s",
     "100%",
     "4/5 (EDB → WOS)",
     "6/6"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash",
     "1.3–2.8 s (one 8.3 s)",
     "100%",
     "5/5",
     "6/6"
    ],
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "1.8–5 s, one 180 s timeout",
     "100%",
     "n/a",
     "6/6"
    ],
    [
     "Qwen/Qwen3-Next-80B-A3B",
     "0.9–2.5 s",
     "100%",
     "4/5",
     "5/6 (art → „Język polski”, confidence 0.3; in json_object mode it returned a label outside the list: „Sztuka” – \"Art\")"
    ],
    [
     "google/gemma-3-27b-it",
     "1.9–4 s",
     "100% with schema; without schema wraps the JSON in  json `",
     "4/5",
     "5/6 (art → Język polski)"
    ],
    [
     "meta-llama/Llama-4-Scout-17B-16E-Instruct",
     "1.2–2.4 s",
     "100% with schema; without schema adds a comment",
     "4/5",
     "6/6 (typo „Resusytacja”)"
    ],
    [
     "Qwen/Qwen3-32B",
     "17–22 s with thinking; 2.1–6.8 s with it disabled",
     "100%",
     "4/5 (EDB → Wychowanie fizyczne)",
     "EDB correct"
    ],
    [
     "zai-org/GLM-5.3-Flash",
     "6–12 s; later 75 s and a 180 s timeout",
     "100%",
     "5/5",
     "EDB correct"
    ],
    [
     "google/gemma-4-26B-A4B-it",
     "17–32 s (1.4–4.5 s in a later attempt)",
     "100%",
     "4/5",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-01.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "2. Lesson summary (description), 2026-09-22",
   "columns": [
    "model",
    "first token",
    "total",
    "output tokens",
    "quality assessment"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "1.35–1.5 s",
     "16–37 s",
     "570–650",
     "best: faithful, complete, no invented facts, clean Polish"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash",
     "1.7 s",
     "15–21 s",
     "about 500",
     "faithful, more concise"
    ],
    [
     "Qwen/Qwen3.5-397B-A17B (thinking disabled)",
     "0.8–1.1 s",
     "17 s",
     "about 600",
     "good; used the term „faza ciemna” (\"dark phase\") although the teacher advised against it; added one invented line about the next lesson; LaTeX"
    ],
    [
     "Qwen/Qwen3-235B-A22B-2507",
     "1.0–1.2 s",
     "30–40 s",
     "n/a",
     "invented facts („imię z 44 liter” – \"a name of 44 letters\", „np. Geniusz stróż” – \"e.g. Guardian Genius\")"
    ],
    [
     "moonshotai/Kimi-K2.6 (default thinking)",
     "none",
     "79 s",
     "3000 reasoning tokens, 0 content",
     "empty result (token limit used up on reasoning)"
    ],
    [
     "moonshotai/Kimi-K2.6 (thinking disabled)",
     "1.2 s",
     "23 s",
     "n/a",
     "good; one grammatical error"
    ],
    [
     "deepseek-ai/DeepSeek-V3.2",
     "5.3 s",
     "103 s (about 7 tokens/s)",
     "n/a",
     "faithful"
    ],
    [
     "google/gemma-4-31B-it",
     "3.9 s",
     "107 s",
     "n/a",
     "faithful; LaTeX"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-02.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "3. \"Lossless\" summaries: how much lesson knowledge survived, 2026-09-29",
   "columns": [
    "model",
    "present (share of items)",
    "output tokens",
    "time, s"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "99.6–99.7%",
     "4070",
     "56–99"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Pro-0813",
     "98.7% (v2 and v3)",
     "3950",
     "45–53"
    ],
    [
     "moonshotai/Kimi-K2.6",
     "98.3%",
     "4500",
     "91"
    ],
    [
     "MiniMax-M3",
     "95.7%",
     "3850",
     "180"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash",
     "93.9% (v2), 90.0% (v3)",
     "3400",
     "86–98"
    ],
    [
     "Qwen/Qwen3.5-397B-A17B",
     "92.6%",
     "4400",
     "240"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-03.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "DeepSeek-V4-Pro vs V4.1-Flash on lesson description, 2026-11-05",
   "columns": [
    "model",
    "present",
    "present + partial/2",
    "output tokens",
    "time, s"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4-Pro",
     "97.6% (biology 94.7; history 100; mathematics 98.1)",
     "97.6%",
     "3204",
     "50–70"
    ],
    [
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "98.8% (100; 98.3; 100)",
     "99.4%",
     "3698",
     "22–26"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-04.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "4. Chat with tools, 2026-09-22",
   "columns": [
    "model",
    "parallel calls",
    "arguments",
    "turn 1: first token / first tool call",
    "turn 2: first text / total",
    "reasoning"
   ],
   "rows": [
    [
     "DeepSeek-V4.1-Flash",
     "Y",
     "in pieces",
     "1.0–2.3 s / 1.3–2.5 s",
     "1.0–1.5 s / 3.1–4.6 s",
     "none by default"
    ],
    [
     "DeepSeek-V4-Flash",
     "Y",
     "in pieces",
     "n/a / 1.1–1.8 s",
     "1.7–2.5 s / 5.9–10 s",
     "none"
    ],
    [
     "Qwen3.5-397B (thinking enabled)",
     "Y",
     "in pieces",
     "0.7 s / 4.2 s",
     "3.0 s / 9 s",
     "Y (separate field)"
    ],
    [
     "Qwen3.5-397B (thinking disabled)",
     "Y",
     "in pieces",
     "n/a / 1.0–1.1 s",
     "0.8 s / 6.3–7.2 s",
     "none"
    ],
    [
     "Kimi-K2.6 (default)",
     "Y",
     "in pieces",
     "2.9 s / 11 s (once 115 s / 172 s)",
     "28.7 s / 35.9 s",
     "Y"
    ],
    [
     "Kimi-K2.6 (thinking disabled)",
     "Y",
     "in pieces",
     "n/a / 1.1–2.6 s",
     "1.0–1.2 s / 5.5–8.4 s",
     "none"
    ],
    [
     "Kimi-K3",
     "Y",
     "whole in one chunk",
     "18 s / 27 s",
     "1.5 s / 6.8 s",
     "Y"
    ],
    [
     "GLM-5.3",
     "Y",
     "in pieces",
     "1.0–1.1 s / 1.4–1.5 s",
     "1.1–1.25 s / 2.7–3.2 s",
     "Y"
    ],
    [
     "DeepSeek-V3.2",
     "Y",
     "whole in one chunk",
     "n/a / 9.7–19.6 s",
     "1.2–1.7 s / 11–28 s",
     "none"
    ],
    [
     "gpt-oss-120b",
     "N (one call per round; 3 rounds)",
     "whole in one chunk",
     "0.7–0.8 s / 1.5–3.8 s",
     "1.1 s / 9.5 s",
     "Y"
    ],
    [
     "DeepSeek-V4-Pro",
     "Y",
     "in pieces",
     "n/a / 1.2 s",
     "3.2 s / 7.7 s",
     "none"
    ],
    [
     "Qwen3.8-Flash",
     "Y",
     "in pieces",
     "1.6 s / 2.7 s",
     "4.0 s / 7.4 s",
     "Y, cannot be disabled"
    ],
    [
     "GLM-5.3-Flash",
     "Y",
     "in pieces",
     "n/a / 147.6 s",
     "182 s / 206 s",
     "n/a"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-05.csv"
  },
  {
   "benchmark": "llm-classification-summaries-chat",
   "title": "Language models: classification, summaries, chat with tools",
   "area": "Language models",
   "section": "Whiteboard photo in chat (1 photo, 1024×1024)",
   "columns": [
    "model",
    "result",
    "first token / total",
    "input tokens"
   ],
   "rows": [
    [
     "DeepSeek-V4.1-Flash",
     "correct",
     "1.85 s / 5.0 s",
     "720"
    ],
    [
     "Qwen3.5-397B (thinking disabled)",
     "correct",
     "2.5 s / 6.4 s",
     "1039"
    ],
    [
     "Kimi-K2.6 (default thinking)",
     "correct",
     "18.3 s / 20.3 s",
     "1459"
    ],
    [
     "google/gemma-4-31B-it",
     "correct",
     "3.2 s / 14.6 s",
     "333"
    ],
    [
     "DeepSeek-V4-Flash, Qwen3.8-Flash",
     "HTTP 405 – the model does not accept images",
     "–",
     "–"
    ]
   ],
   "url": "https://benchmarks.site/llm-classification-summaries-chat/",
   "csv": "https://benchmarks.site/data/llm-classification-summaries-chat-06.csv"
  },
  {
   "benchmark": "llm-short-answers-grading-cost",
   "title": "Language models: short answers, grading text, time and cost",
   "area": "Language models",
   "section": "1. DeepSeek-V4-Flash: undated name vs dated name (2026-10-13)",
   "columns": [
    "model",
    "task",
    "attempts",
    "result",
    "time per call",
    "cost per call"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4-Flash (undated)",
     "A",
     "12",
     "7/12 rejected (HTTP 429 engine_overloaded, \"Model busy, retry later\")",
     "8–13 s (those that went through)",
     "1.26·10⁻⁶ $"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-0731",
     "A",
     "6",
     "6/6",
     "1.2–2.3 s",
     "1.02·10⁻⁶ $"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Pro-0813",
     "A",
     "4",
     "4/4",
     "0.5–1.3 s",
     "n/a"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash (undated)",
     "B",
     "approx. 8 successful",
     "some requests rejected as above",
     "7.3–15.8 s",
     "5.24·10⁻⁴ $ without cache, 1.19·10⁻⁴ $ with a cache hit"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-0731",
     "B",
     "approx. 17",
     "all went through",
     "5.2–12.6 s (typically 6–9 s)",
     "3.5·10⁻⁴ $ without cache, 9.8·10⁻⁵ $ with a cache hit (5632 of 5763 tokens from cache)"
    ]
   ],
   "url": "https://benchmarks.site/llm-short-answers-grading-cost/",
   "csv": "https://benchmarks.site/data/llm-short-answers-grading-cost-01.csv"
  },
  {
   "benchmark": "reasoning-effort",
   "title": "Reasoning level (reasoning_effort): what models actually do",
   "area": "Language models",
   "section": "1. The minimal value on different models (2026-09-03)",
   "columns": [
    "model",
    "platform",
    "minimal accepted?",
    "reasons on minimal?",
    "notes"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4-Pro (-0813)",
     "DeepInfra",
     "Y",
     "Y",
     "the only model in the test that actually reasoned on minimal"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash (-0731)",
     "DeepInfra",
     "Y",
     "N",
     "response longer than without the parameter, but 0 reasoning tokens; reasons only from low"
    ],
    [
     "gpt-5",
     "OpenAI",
     "Y",
     "N (0 reasoning tokens)",
     "lowest level with real reasoning: low"
    ],
    [
     "gpt-5.6 (variants terra, sol)",
     "OpenAI",
     "N (value rejected)",
     "–",
     "lowest accepted level: low"
    ],
    [
     "anthropic/claude-opus-5-5",
     "DeepInfra",
     "per Anthropic documentation, N (lowest is low)",
     "–",
     "not measured live (on 2026-10-16 the account returned 402)"
    ]
   ],
   "url": "https://benchmarks.site/reasoning-effort/",
   "csv": "https://benchmarks.site/data/reasoning-effort-01.csv"
  },
  {
   "benchmark": "reasoning-effort",
   "title": "Reasoning level (reasoning_effort): what models actually do",
   "area": "Language models",
   "section": "2. Cost of reasoning",
   "columns": [
    "model",
    "task",
    "date",
    "sample",
    "result"
   ],
   "rows": [
    [
     "deepseek-ai/DeepSeek-V4-Flash",
     "hint for an exam task, with reasoning (low) vs without",
     "2026-09-03",
     "a few calls",
     "with reasoning 7.3× more expensive than without"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Pro",
     "the same task, with reasoning, compared with DeepSeek-V4-Flash without reasoning",
     "2026-09-03",
     "a few calls",
     "approx. 40× more expensive"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Pro (model reconstructed from change history)",
     "grading a short open answer worth 2 points, with reasoning",
     "2026-09-03",
     "1 task",
     "1914 reasoning tokens, 3.38 gr (1 gr = 0.01 PLN), i.e. roughly the same as grading an essay worth 20 points"
    ]
   ],
   "url": "https://benchmarks.site/reasoning-effort/",
   "csv": "https://benchmarks.site/data/reasoning-effort-02.csv"
  },
  {
   "benchmark": "reasoning-effort",
   "title": "Reasoning level (reasoning_effort): what models actually do",
   "area": "Language models",
   "section": "3. Reasoning and the output token limit",
   "columns": [
    "model",
    "conditions",
    "date",
    "result"
   ],
   "rows": [
    [
     "gpt-5.6-terra / gpt-5.6-sol",
     "lowest accepted reasoning level, output limit of 2000 tokens, task with a photo",
     "2026-09-03",
     "empty response: reasoning used up the whole limit before the model wrote a single character; a limit of around 6000–8000 was needed"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "reasoning low, reading a page from a photo",
     "2026-09-13",
     "reasoning tokens count toward the same limit as the response, so the limit has to be raised (approx. +1500 tokens was adopted); this is an assumption based on the behavior of other models, an empty response from Qwen was not recorded"
    ]
   ],
   "url": "https://benchmarks.site/reasoning-effort/",
   "csv": "https://benchmarks.site/data/reasoning-effort-03.csv"
  },
  {
   "benchmark": "embeddings-polish",
   "title": "Embeddings: retrieval accuracy in Polish",
   "area": "Embeddings",
   "section": "Data",
   "columns": [
    "model",
    "dimensions",
    "latency per batch",
    "graded test „równania kwadratowe”: score / margin",
    "10-query test"
   ],
   "rows": [
    [
     "BAAI/bge-m3",
     "1024",
     "1.5–1.6 s",
     "0.632 / −0.063",
     "0.95 (9/10 in first place)"
    ],
    [
     "Qwen/Qwen3-Embedding-0.6B",
     "1024",
     "1.5–3 s (one cold 26 s)",
     "0.953 / +0.179 without instruction; 0.935 / +0.187 with instruction",
     "0.925 without instruction; 1.00 with instruction"
    ],
    [
     "Qwen/Qwen3-Embedding-4B",
     "2560",
     "1.5–5.8 s",
     "0.953 / +0.078; with instruction margin +0.127; at 1024 dimensions 0.888 / +0.104",
     "1.00"
    ],
    [
     "Qwen/Qwen3-Embedding-8B",
     "4096",
     "10–31 s",
     "0.953 / +0.048; with instruction margin +0.097",
     "1.00"
    ],
    [
     "multilingual-e5-large",
     "1024 (512-token limit)",
     "1.6–29 s",
     "0.751 / −0.030 (similarities clustered in the 0.73–0.88 range)",
     "1.00"
    ],
    [
     "embeddinggemma-300m",
     "768",
     "0.9–1.6 s",
     "0.817 / −0.036",
     "1.00"
    ]
   ],
   "url": "https://benchmarks.site/embeddings-polish/",
   "csv": "https://benchmarks.site/data/embeddings-polish-01.csv"
  },
  {
   "benchmark": "platforms-availability-latency-limits",
   "title": "Platforms: availability, latency and limits",
   "area": "Platforms",
   "section": "DeepInfra",
   "columns": [
    "observation",
    "date",
    "notes"
   ],
   "rows": [
    [
     "Queue/overload of Qwen3-ASR-1.7B: for approx. 25 min 429 Model busy responses, then for approx. half an hour a queue of 114–186 s",
     "2026-09-29",
     "a single episode"
    ],
    [
     "Whisper and other models: latency depends on load, e.g. whisper-large-v3 up to 23 s for a 48 s clip; Qwen3-ASR 4–22 s for 10 min of audio and 15–27 s for 46 min",
     "2026-09-22",
     ""
    ],
    [
     "Qwen3-ASR, 20 s chunk: 50–77 s under platform load (Nemotron 1.4–3.5 s)",
     "2026-11-05",
     "normal mode and priority gave similar times"
    ],
    [
     "flex tier (lowest priority): on a short query, time to first response for DeepSeek-V4-Pro went from 1.3 s to 13–20 s, for DeepSeek-V4.1-Flash from 0.8–1.0 s to 3.6–4.8 s; according to a later assessment, flex latency was 8–40 s higher than in normal mode even without a queue",
     "2026-09-30",
     "speech models (Qwen3-ASR, Whisper, Nemotron) accepted this field with no effect"
    ],
    [
     "Chat models under load: Kimi-K3 HTTP 429 engine_overloaded with 4 retries over approx. 30 s; GLM-5.3-Flash 75–206 s to first token and 180 s timeouts; Kimi-K2.6 once 115 s to first token; DeepSeek-V4.1-Flash one 180 s timeout in approx. 25 calls",
     "2026-09-22",
     ""
    ],
    [
     "Retired models are silently redirected to a replacement (replaced_by), so \"DeepSeek-OCR\" and \"olmOCR\" responded as google/gemma-4-31B-it, small Qwen3.5 models (0.8B–4B) as Qwen3.5-9B, and Phi-4-multimodal as gemma-3-12b",
     "2026-10-06",
     "results require checking the model field in the response"
    ],
    [
     "The cost field in the native ASR interface has inconsistent units: cents for Whisper and Voxtral, dollars for Qwen and Nemotron",
     "2026-09-22",
     ""
    ],
    [
     "Images: MIME image/jpg and image/heic – HTTP 422 with a misleading message; HEIC bytes labeled as jpeg – HTTP 400 on models hosted by DeepInfra, while Gemini and Seed accept them; the documented 20 MB limit is not enforced (a 21.8 MB PNG was accepted); ByteDance Seed-2.0-mini has its own limits of 10 MiB and 36 Mpx (reported as HTTP 500)",
     "2026-09-22",
     ""
    ],
    [
     "Video: the video_url part passes only the image to the model (see video-audio); audio input has a limit of 600 s per part",
     "2026-09-23",
     ""
    ],
    [
     "Prompt caching: a repeated query with the same image and prompt is sometimes billed cheaper (automatic cache hits, varying between models)",
     "2026-09-22",
     ""
    ],
    [
     "Claude (Sonnet, Opus) via DeepInfra: no reported cache hits",
     "2026-11-06",
     "see LLM text, section 6"
    ],
    [
     "Gemini models are forwarded to Google (partner flag); gemma-4-31B-it-Ultra is also a partner model",
     "2026-09-23",
     ""
    ]
   ],
   "url": "https://benchmarks.site/platforms-availability-latency-limits/",
   "csv": "https://benchmarks.site/data/platforms-availability-latency-limits-01.csv"
  },
  {
   "benchmark": "platforms-errors-billing-prices",
   "title": "Platforms: errors, reasoning billing and prices",
   "area": "Platforms",
   "section": "DeepInfra",
   "columns": [
    "observation",
    "date",
    "notes"
   ],
   "rows": [
    [
     "The Qwen/Qwen3-VL-235B-A22B-Thinking variant was served by the Instruct model: 0 reasoning tokens in the response, at a rate approx. 2× higher than for Qwen3-VL-235B-A22B-Instruct",
     "2026-09-03",
     "worth checking the model field and the number of reasoning tokens in the response"
    ],
    [
     "With missing balance, every call (including partner models, e.g. Gemini, and a request for 1 token) ends with HTTP 402 You need positive balance to do inference. Please add balance manually or setup top-up",
     "2026-10-13 … 2026-10-16",
     "the state lasted at least 3 days; the balance ran out during a series of approx. 140 test calls (cause unconfirmed)"
    ],
    [
     "A model name that does not exist gives HTTP 404 model_not_found even with zero balance, while an existing name gives 402; so the response code tells you whether the model exists",
     "2026-10-16",
     "3 names checked"
    ],
    [
     "/v1/openai/models returned 180 models; google/gemma-4-E4B-it was not on the list, but the name was recognized (402, not 404), and in September the model responded normally",
     "2026-10-16",
     "the list is not a complete inventory of callable models"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash (undated name): approx. every second request HTTP 429 engine_overloaded \"Model busy, retry later\", a retry after a second usually went through; deepseek-ai/DeepSeek-V4-Flash-0731 (dated) responded every time in 1.2–2.3 s",
     "2026-10-13",
     "12 + 6 calls; details in LLM text, section 1"
    ],
    [
     "429 with engine_overloaded / \"Model busy\" in the body means overload of the model pool, not an account limit; 429 with a body about the request rate limit, and 402, are account limits",
     "2026-10-13",
     "distinguishable only by the error body"
    ],
    [
     "Gemini (google/gemini-3.1-pro) bills reasoning tokens that are not in completion_tokens; the cost from the token price list came out approx. 13× too low compared with estimated_cost",
     "2026-09-17",
     "reading pages from photos, 5 photos"
    ],
    [
     "Automatic prompt caching: with a repeated prompt of approx. 5.7 thousand tokens, DeepSeek-V4-Flash-0731 returned cached_tokens 5632 and an approx. 3.6× lower cost",
     "2026-10-13",
     "without any parameter in the request"
    ],
    [
     "Gemini and Claude (anthropic/*) models are available through the same DeepInfra key (forwarded to Google / Anthropic)",
     "2026-09-13, 2026-10-16",
     ""
    ],
    [
     "The anthropic/claude-opus-5-5 model page does not document support for temperature, reasoning_effort or response_format in the OpenAI-compatible API (the example contains only max_tokens)",
     "2026-10-16",
     "live behavior not checked (402)"
    ]
   ],
   "url": "https://benchmarks.site/platforms-errors-billing-prices/",
   "csv": "https://benchmarks.site/data/platforms-errors-billing-prices-01.csv"
  },
  {
   "benchmark": "platforms-errors-billing-prices",
   "title": "Platforms: errors, reasoning billing and prices",
   "area": "Platforms",
   "section": "Prices from /v1/openai/models (2026-10-16)",
   "columns": [
    "model",
    "input",
    "output",
    "cache",
    "tags"
   ],
   "rows": [
    [
     "anthropic/claude-fable-5",
     "10.00",
     "50.00",
     "–",
     "chat, vlm, vision, reasoning_effort, reasoning"
    ],
    [
     "anthropic/claude-opus-5-5",
     "4.00",
     "20.00",
     "0.20",
     "chat, vlm, vision, prompt_cache, reasoning_effort, reasoning; context 1000000"
    ],
    [
     "anthropic/claude-sonnet-5-5",
     "2.00",
     "10.00",
     "–",
     "chat, vlm, vision, reasoning_effort, reasoning"
    ],
    [
     "moonshotai/Kimi-K3",
     "2.85",
     "14.25",
     "0.285",
     "chat, vlm, vision, prompt_cache, reasoning"
    ],
    [
     "google/gemini-3.1-pro",
     "2.00",
     "12.00",
     "–",
     "chat, vlm, vision, reasoning_effort, reasoning; context 1000000"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Pro-0813",
     "1.30",
     "2.60",
     "0.10",
     "chat, prompt_cache"
    ],
    [
     "google/gemini-3.5-flash",
     "1.50",
     "9.00",
     "–",
     "context 1000000 (tags not read)"
    ],
    [
     "google/gemini-3.7-flash",
     "0.75",
     "3.75",
     "–",
     "chat, vlm, vision, reasoning_effort, reasoning"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
     "0.44",
     "1.32",
     "0.014",
     "chat, vlm, vision, prompt_cache, reasoning"
    ],
    [
     "Qwen/Qwen3-VL-235B-A22B-Instruct",
     "0.20",
     "0.88",
     "0.11",
     "chat, vlm, vision, prompt_cache, reasoning_effort"
    ],
    [
     "google/gemma-4-31B-it",
     "0.20",
     "0.40",
     "–",
     "chat, vlm, vision, reasoning"
    ],
    [
     "Qwen/Qwen3.6-35B-A3B",
     "0.10",
     "0.95",
     "0.10",
     "chat, vlm, vision, prompt_cache, reasoning_effort, reasoning"
    ],
    [
     "google/gemma-4-26B-A4B-it",
     "0.07",
     "0.34",
     "–",
     "chat, vlm, vision, reasoning; context 262144"
    ],
    [
     "deepseek-ai/DeepSeek-V4-Flash-0731",
     "0.06",
     "0.18",
     "0.015",
     "chat, prompt_cache, reasoning"
    ]
   ],
   "url": "https://benchmarks.site/platforms-errors-billing-prices/",
   "csv": "https://benchmarks.site/data/platforms-errors-billing-prices-02.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "",
   "columns": [
    "Part",
    "Passed",
    "Total",
    "Rate"
   ],
   "rows": [
    [
     "Official AMTSO Security Features Check + eicar.org files",
     "11",
     "11",
     "100%"
    ],
    [
     "EICAR in container formats (must be detected)",
     "57",
     "57",
     "100%"
    ],
    [
     "Clean twins of the same containers without EICAR (must not alarm)",
     "57",
     "57",
     "100%"
    ],
    [
     "SHA-256 hashes from Objective-See reports 2019–2024 (database lookup only)",
     "8",
     "33",
     "24.2%"
    ],
    [
     "Mac malware families 2019–2025 with a rule named after the family",
     "57",
     "83",
     "68.7%"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-01.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "1. Official test files: AMTSO Security Features Check + eicar.org",
   "columns": [
    "Test",
    "Expected",
    "Verdict",
    "OK",
    "Detecting engines (signature names)"
   ],
   "rows": [
    [
     "eicar.org eicar.com",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "eicar.org eicar.com.txt",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "eicar.org eicar_com.zip",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, MalwareBazaar:Generic.zip, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "eicar.org eicar_com2.zip (zip in zip)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, MalwareBazaar:Generic.zip, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "AMTSO download of malware (eicar.com)",
     "malicious",
     "MALICIOUS",
     "Y",
     "YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test, YARA-ext:TRELLIX_ARC_Malw_Eicar"
    ],
    [
     "AMTSO compressed malware (eicar.zip)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "AMTSO compressed malware (eicar.zipx)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "AMTSO compressed malware (eicar.exe, self-extracting)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:PE.Toolchain.WinRAR-SFX, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "AMTSO compressed malware (eicar_zip.exe, self-extracting)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Fuzzy:TLSH.njrat, Heur:PE.Toolchain.WinRAR-SFX, XProtect:EICAR"
    ],
    [
     "AMTSO potentially unwanted application",
     "pua",
     "MALICIOUS",
     "Y",
     "ClamAV:PUA.Win.Trojan.Agent-1350158, ClamAV:Win.Tool.EICAR-9917185-0, Heur:PE.Checksum.Mismatch"
    ],
    [
     "AMTSO cloud lookup (CloudCar)",
     "detected",
     "MALICIOUS",
     "Y",
     "ClamAV:Win.Trojan.11584714-1, Heur:PE.Checksum.Mismatch"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-02.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "2. EICAR in 57 container formats",
   "columns": [
    "Test",
    "Expected",
    "Verdict",
    "OK",
    "Detecting engines (signature names)"
   ],
   "rows": [
    [
     "plain .com",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "plain + CRLF (AMTSO style)",
     "malicious",
     "MALICIOUS",
     "Y",
     "YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test, YARA-ext:TRELLIX_ARC_Malw_Eicar"
    ],
    [
     "plain .txt",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "plain, no extension",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip (deflate)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip (stored)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip (bzip2)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip (lzma)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip in folder inside zip",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip nested 2×",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip nested 3×",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip nested 5×",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip nested 8×",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "gzip",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "bzip2",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "xz",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "lzma (alone)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "tar",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "tar.gz",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "tar.bz2",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "tar.xz",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "cpio (newc)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "ar",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "deb (ar + tar.gz)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "jar",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "docx (OOXML part)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Office.EmbeddedExecutable, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "xlsx (OOXML part)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Office.EmbeddedExecutable, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "odt (ODF)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "epub",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "crx (Chrome extension)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "nupkg",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "asar (Electron)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "e-mail attachment (base64)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Email.ExecutableAttachment, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "e-mail with zip attachment",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Email.ArchivedExecutable, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "mbox",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Email.ExecutableAttachment, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "PDF attachment (FlateDecode)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:PDF.EmbeddedExecutable, Heur:PDF.EmbeddedFile, XProtect:EICAR"
    ],
    [
     "base64 file",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "Unix compress (.Z)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zstd",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "tar.zst",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "lz4",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "7z",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "7z, password 'infected', names encrypted",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Archive.Encrypted, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "zip, password 'infected' (ZipCrypto)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Archive.Encrypted, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "zip, password 'infected' (AES-256)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Archive.Encrypted, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "zip, password a year (2024)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, Heur:Archive.Encrypted, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test"
    ],
    [
     "xar",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "macOS .pkg (pkgbuild)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG UDZO",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG UDBZ",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG ULFO",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG ULMO",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG UDRO",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "ISO 9660",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "zip inside DMG",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "app bundle (Contents/Resources)",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ],
    [
     "DMG inside zip",
     "malicious",
     "MALICIOUS",
     "Y",
     "ClamAV:Eicar-Test-Signature, XProtect:EICAR, YARA-ext:BINARYALERT_Eicar_Av_Test, YARA-ext:BINARYALERT_Eicar_Substring_Test"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-03.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "Clean twins (near-miss files)",
   "columns": [
    "Test",
    "Expected",
    "Verdict",
    "OK",
    "Detecting engines (signature names)"
   ],
   "rows": [
    [
     "plain .com",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "plain + CRLF (AMTSO style)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "plain .txt",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "plain, no extension",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip (deflate)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip (stored)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip (bzip2)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip (lzma)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip in folder inside zip",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip nested 2×",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip nested 3×",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip nested 5×",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip nested 8×",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "gzip",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "bzip2",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "xz",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "lzma (alone)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "tar",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "tar.gz",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "tar.bz2",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "tar.xz",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "cpio (newc)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "ar",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "deb (ar + tar.gz)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "jar",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "docx (OOXML part)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "xlsx (OOXML part)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "odt (ODF)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "epub",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "crx (Chrome extension)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "nupkg",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "asar (Electron)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "e-mail attachment (base64)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "e-mail with zip attachment",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "mbox",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "PDF attachment (FlateDecode)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "base64 file",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "Unix compress (.Z)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zstd",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "tar.zst",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "lz4",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "7z",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "7z, password 'infected', names encrypted",
     "clean",
     "INFO",
     "Y",
     "Heur:Archive.Encrypted"
    ],
    [
     "zip, password 'infected' (ZipCrypto)",
     "clean",
     "INFO",
     "Y",
     "Heur:Archive.Encrypted"
    ],
    [
     "zip, password 'infected' (AES-256)",
     "clean",
     "INFO",
     "Y",
     "Heur:Archive.Encrypted"
    ],
    [
     "zip, password a year (2024)",
     "clean",
     "INFO",
     "Y",
     "Heur:Archive.Encrypted"
    ],
    [
     "xar",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "macOS .pkg (pkgbuild)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG UDZO",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG UDBZ",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG ULFO",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG ULMO",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG UDRO",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "ISO 9660",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "zip inside DMG",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "app bundle (Contents/Resources)",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ],
    [
     "DMG inside zip",
     "clean",
     "CLEAN",
     "Y",
     "—"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-04.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "3. SHA-256 hashes from Objective-See reports 2019–2024",
   "columns": [
    "Sample (year, family / file)",
    "SHA-256",
    "Expected",
    "Verdict (database match)",
    "OK"
   ],
   "rows": [
    [
     "2019 Siggen (HEYgiNb.py)",
     "f5808e9b9d204f646e33bbc4279b98b97b34086ffc3e9fb2ac828a8161099ee8",
     "known",
     "none",
     "N"
    ],
    [
     "2019 NewTab",
     "9237dc996a9e2c939f74414c87350009f11b147f1703823062f790921be30be9",
     "known",
     "none",
     "N"
    ],
    [
     "2020 watchcat (NukeSpeed)",
     "3bb96bfaf492782b38985f4bd6b7e7f9dc22c1332b42bb74b16041298fd31f93",
     "known",
     "none",
     "N"
    ],
    [
     "2022 SysJoker",
     "1a9a5c797777f37463b44de2b49a7f95abca786db3977dcdac0f79da739c08ac",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 SysJoker",
     "fe99db3268e058e1204aff679e0726dc77fd45d06757a5fda9eafc6a28cfb8df",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 SysJoker",
     "d0febda3a3d2d68b0374c26784198dc4309dbe4a8978e44bb7584fd832c325f0",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 DazzleSpy",
     "f9ad42a9bd9ade188e997845cae1b0587bf496a35c3bffacd20fefe07860a348",
     "known",
     "none",
     "N"
    ],
    [
     "2022 CoinMiner",
     "fe3700a52e86e250a9f38b7a5a48397196e7832fd848a7da3cc02fe52f49cdcf",
     "known",
     "none",
     "N"
    ],
    [
     "2022 CoinMiner",
     "fabe0b41fb5bce6bda8812197ffd74571fc9e8a5a51767bcceef37458e809c5c",
     "known",
     "none",
     "N"
    ],
    [
     "2022 CoinMiner",
     "a2909754783bb5c4fd6955bcebc356e9d6eda94f298ed3e66c7e13511275fbc4",
     "known",
     "none",
     "N"
    ],
    [
     "2022 Gimmick",
     "2a9296ac999e78f6c0bee8aca8bfa4d4638aa30d9c8ccc65124b1cbfc9caab5f",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 oRAT",
     "ee07dfd6443af8f20f5f11effb9cbcec07e125697a28aee78718caeed17f1407",
     "known",
     "none",
     "N"
    ],
    [
     "2022 Pymafka",
     "b117f042fe9bac7c7d39eab98891c2465ef45612f5355beea8d3c4ebd0665b45",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 VPN Trojan (Covid)",
     "7831806172857a563d7b4789acddc98fc11763aaf3cedf937630b4a9dce31419",
     "known",
     "none",
     "N"
    ],
    [
     "2022 VPN Trojan (Covid)",
     "d9bba1cfca6b1d20355ce08eda37d6d0bca8cb8141073b699000d05025510dcc",
     "known",
     "none",
     "N"
    ],
    [
     "2022 CloudMensis",
     "317ce26cae14dc9a5e4d4667f00fee771b4543e91c944580bbb136e7fe339427",
     "known",
     "none",
     "N"
    ],
    [
     "2022 CloudMensis",
     "b8a61adccefb13b7058e47edcd10a127c483403cf38f7ece126954e95e86f2bd",
     "known",
     "none",
     "N"
    ],
    [
     "2022 rShell",
     "3a9e72b3810b320fa6826a1273732fee7a8e2b2e5c0fd95b8c36bbab970e830a",
     "known",
     "none",
     "N"
    ],
    [
     "2022 rShell",
     "8c3be245cbbe9206a5d146017c14b8f965ab7045268033d70811d5bcc4b796ec",
     "known",
     "none",
     "N"
    ],
    [
     "2022 Insekt (exploit)",
     "ef130f1941077ffe383fe90e241620dde771cd0dd496dad29d2048d5fc478faf",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2022 Insekt (payload.so)",
     "ae9f370c89f0191492ed9c17a224d9c41778b47ca2768f732b4de6ee7d0d1459",
     "known",
     "none",
     "N"
    ],
    [
     "2022 KeySteal (ResignTool)",
     "410da3923ea30d5fdd69b9ae69716b094d276cc609f76590369ff254f71c65da",
     "known",
     "none",
     "N"
    ],
    [
     "2022 KeySteal",
     "f5b4a388fee4183dfa46908000c5c50dceb4bf8025c4cfcb4d478c5d03833202",
     "known",
     "none",
     "N"
    ],
    [
     "2023 LockBit macOS",
     "3e4bbd21756ae30c24ff7d6942656be024139f8180b7bddd4e5c62a9dfbd8c79",
     "known",
     "Generic.macho",
     "Y"
    ],
    [
     "2023 GoSorry (archive)",
     "c80c8c0e961e1692c5afdb8b3dd48e84fb6bb24911ad449ef75976cbf5bf1199",
     "known",
     "none",
     "N"
    ],
    [
     "2023 GoSorry (liboptimizer.dylib)",
     "17b84ed0f8bdbb6619d64bf686a6b990f497b378e3b70792e2cd896c2abda14d",
     "known",
     "none",
     "N"
    ],
    [
     "2024 fake MiroTalk.dmg",
     "9abf6b93eafb797a3556bea1fe8a3b7311d2864d5a9a3687fce84bc1ec4a428c",
     "known",
     "Generic.dmg",
     "Y"
    ],
    [
     "2024 trojanised ultraedit.dmg",
     "9eb7bda5ffbb1a7549b1e481b1a6ed6efe2e28d0463370c87630fed74eee6228",
     "known",
     "none",
     "N"
    ],
    [
     "2024 libConfigurer64.dylib (ultraedit)",
     "ce40829673687b48d68defa3176c8ab59a2a50ee9c658fe46a5de7692fbc112d",
     "known",
     "none",
     "N"
    ],
    [
     "2024 .fseventsd (ultraedit)",
     "1b2d50cdacfd39205c3caff2925eb35b59312dbe099bd3a98ae3b2f2f909ab17",
     "known",
     "none",
     "N"
    ],
    [
     "2024 Khepri build (.test)",
     "21457a89317e4c6b8aaee5e461a6b4e444c736243d550efd0e681eda05b97007",
     "known",
     "none",
     "N"
    ],
    [
     "2024 vShell (Cloudflare Security Authenticator.dmg)",
     "c5686b85efb3ebf2ce07dba4192195c3dac7c335a371b7bcfbf52d5fb15cb507",
     "known",
     "none",
     "N"
    ],
    [
     "2024 SNOWLIGHT-like",
     "d1b7f3d88ed8d37774b229cc46df2c08c95067736cf418b4a76a8403cae4e2a6",
     "known",
     "none",
     "N"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-05.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "4. Mac malware families 2019–2025 with a family-named rule",
   "columns": [
    "Year",
    "Families with a named rule",
    "Total",
    "Rate"
   ],
   "rows": [
    [
     "2019",
     "6",
     "8",
     "75.0%"
    ],
    [
     "2020",
     "5",
     "7",
     "71.4%"
    ],
    [
     "2021",
     "7",
     "8",
     "87.5%"
    ],
    [
     "2022",
     "8",
     "12",
     "66.7%"
    ],
    [
     "2023",
     "15",
     "18",
     "83.3%"
    ],
    [
     "2024",
     "11",
     "18",
     "61.1%"
    ],
    [
     "2025",
     "5",
     "12",
     "41.7%"
    ],
    [
     "All",
     "57",
     "83",
     "68.7%"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-06.csv"
  },
  {
   "benchmark": "malware-detection-safe",
   "title": "Malware detection: AMTSO, EICAR containers, Objective-See hashes, Mac malware families",
   "area": "Security (macOS malware scanning)",
   "section": "4. Mac malware families 2019–2025 with a family-named rule",
   "columns": [
    "Family (year)",
    "Expected",
    "Sources with a family-named signature",
    "Hash only",
    "OK"
   ],
   "rows": [
    [
     "2019 BirdMiner / LoudMiner",
     "named signature",
     "ClamAV",
     "N",
     "Y"
    ],
    [
     "2019 CookieMiner",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2019 GMERA",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2019 Mokes",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2019 NetWire",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2019 Siggen",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2019 Yort",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2019 Lazarus MacLoader / AppleJeus",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2020 GravityRAT",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2020 IPStorm",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2020 Dacls",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2020 EvilQuest / ThiefQuest",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2020 FinSpy",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2020 WatchCat",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2020 XCSSET",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2021 ElectroRAT",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2021 ElectrumStealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2021 Macma / CDDS",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2021 Silver Sparrow",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2021 XcodeSpy / EggShell",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2021 ZuRu",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2021 WildPressure (Milum)",
     "named signature",
     "YARA+ClamAV+hash",
     "N",
     "Y"
    ],
    [
     "2021 XLoader",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2022 CloudMensis",
     "named signature",
     "YARA+hash",
     "N",
     "Y"
    ],
    [
     "2022 CrateDepression",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2022 DazzleSpy",
     "named signature",
     "YARA+hash",
     "N",
     "Y"
    ],
    [
     "2022 Gimmick",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2022 Insekt",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2022 KeySteal",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2022 oRAT",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2022 Pymafka",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2022 rShell",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2022 SentinelSneak",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2022 SysJoker",
     "named signature",
     "YARA+ClamAV+hash",
     "N",
     "Y"
    ],
    [
     "2022 VPN Trojan (Covid)",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 SmoothOperator (3CX)",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 Atomic Stealer (AMOS)",
     "named signature",
     "YARA+ClamAV+hash",
     "N",
     "Y"
    ],
    [
     "2023 Geacon",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 GoSorry",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2023 JaskaGO",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 JokerSpy",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 KandyKorn",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2023 LockBit (macOS)",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 MacStealer",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2023 MetaStealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2023 NokNok",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 ObjCShellz",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 PureLand",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 Realst",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2023 RustBucket",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 SparkRAT",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2023 Turtle",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2023 WSClient",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 Activator",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 Banshee",
     "named signature",
     "YARA+ClamAV",
     "N",
     "Y"
    ],
    [
     "2024 BeaverTail",
     "named signature",
     "YARA+ClamAV+hash",
     "N",
     "Y"
    ],
    [
     "2024 CloudChat",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 Cthulhu",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 Hidden Risk",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 HZ Rat",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 InletDrift",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 LightSpy",
     "named signature",
     "YARA+ClamAV+hash",
     "N",
     "Y"
    ],
    [
     "2024 NotLockBit",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 Poseidon",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 PyStealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 RustDoor",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 RustyAttr",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2024 SnowLight",
     "named signature",
     "hash",
     "Y",
     "N"
    ],
    [
     "2024 SpectralBlur",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 ToDoSwift",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2024 VShell",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2025 ChillyHell",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2025 DigitStealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 FlexibleFerret",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2025 Frigid Stealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 Kitty Stealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 Koi Stealer",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 MacSync",
     "named signature",
     "YARA+hash",
     "N",
     "Y"
    ],
    [
     "2025 NightPaw",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 Paradox",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2025 PasivRobber",
     "named signature",
     "YARA",
     "N",
     "Y"
    ],
    [
     "2025 Phexia",
     "named signature",
     "none",
     "N",
     "N"
    ],
    [
     "2025 RN Loader / RN Stealer",
     "named signature",
     "none",
     "N",
     "N"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-safe/",
   "csv": "https://benchmarks.site/data/malware-detection-safe-07.csv"
  },
  {
   "benchmark": "malware-detection-held-out",
   "title": "Malware detection: test samples vs held-out clean files",
   "area": "Security (macOS malware scanning)",
   "section": "Runs",
   "columns": [
    "Run",
    "Date (run end)",
    "Files scanned",
    "Malicious test samples",
    "Clean test samples",
    "Clean (tuning)",
    "Clean (held-out)",
    "Scan time (s)"
   ],
   "rows": [
    [
     "20261103",
     "2026-11-03 13:43",
     "5824",
     "528",
     "14",
     "2309",
     "2973",
     "462"
    ],
    [
     "20261103-fix",
     "2026-11-03 14:01",
     "5824",
     "528",
     "14",
     "2309",
     "2973",
     "827"
    ],
    [
     "20261109-speed",
     "2026-11-09 13:38",
     "5431",
     "528",
     "14",
     "2077",
     "2812",
     "3558"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-held-out/",
   "csv": "https://benchmarks.site/data/malware-detection-held-out-01.csv"
  },
  {
   "benchmark": "malware-detection-held-out",
   "title": "Malware detection: test samples vs held-out clean files",
   "area": "Security (macOS malware scanning)",
   "section": "Run 20261103",
   "columns": [
    "Threshold",
    "Detection (test samples)",
    "False alarms: held-out clean (never tuned)",
    "False alarms: clean used for tuning",
    "False alarms: clean test samples"
   ],
   "rows": [
    [
     "INFO+",
     "519/528 (98.30%)",
     "119/2973 (4.00%)",
     "32/2309 (1.39%)",
     "1/14 (7.14%)"
    ],
    [
     "SUSPICIOUS+",
     "479/528 (90.72%)",
     "0/2973 (0.00%)",
     "0/2309 (0.00%)",
     "0/14 (0.00%)"
    ],
    [
     "MALICIOUS",
     "318/528 (60.23%)",
     "0/2973 (0.00%)",
     "0/2309 (0.00%)",
     "0/14 (0.00%)"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-held-out/",
   "csv": "https://benchmarks.site/data/malware-detection-held-out-02.csv"
  },
  {
   "benchmark": "malware-detection-held-out",
   "title": "Malware detection: test samples vs held-out clean files",
   "area": "Security (macOS malware scanning)",
   "section": "Run 20261103-fix",
   "columns": [
    "Threshold",
    "Detection (test samples)",
    "False alarms: held-out clean (never tuned)",
    "False alarms: clean used for tuning",
    "False alarms: clean test samples"
   ],
   "rows": [
    [
     "INFO+",
     "519/528 (98.30%)",
     "10/2973 (0.34%)",
     "4/2309 (0.17%)",
     "1/14 (7.14%)"
    ],
    [
     "SUSPICIOUS+",
     "479/528 (90.72%)",
     "0/2973 (0.00%)",
     "0/2309 (0.00%)",
     "0/14 (0.00%)"
    ],
    [
     "MALICIOUS",
     "318/528 (60.23%)",
     "0/2973 (0.00%)",
     "0/2309 (0.00%)",
     "0/14 (0.00%)"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-held-out/",
   "csv": "https://benchmarks.site/data/malware-detection-held-out-03.csv"
  },
  {
   "benchmark": "malware-detection-held-out",
   "title": "Malware detection: test samples vs held-out clean files",
   "area": "Security (macOS malware scanning)",
   "section": "Run 20261109-speed",
   "columns": [
    "Threshold",
    "Detection (test samples)",
    "False alarms: held-out clean (never tuned)",
    "False alarms: clean used for tuning",
    "False alarms: clean test samples"
   ],
   "rows": [
    [
     "INFO+",
     "519/528 (98.30%)",
     "9/2812 (0.32%)",
     "4/2077 (0.19%)",
     "1/14 (7.14%)"
    ],
    [
     "SUSPICIOUS+",
     "479/528 (90.72%)",
     "0/2812 (0.00%)",
     "0/2077 (0.00%)",
     "0/14 (0.00%)"
    ],
    [
     "MALICIOUS",
     "318/528 (60.23%)",
     "0/2812 (0.00%)",
     "0/2077 (0.00%)",
     "0/14 (0.00%)"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-held-out/",
   "csv": "https://benchmarks.site/data/malware-detection-held-out-04.csv"
  },
  {
   "benchmark": "malware-detection-held-out",
   "title": "Malware detection: test samples vs held-out clean files",
   "area": "Security (macOS malware scanning)",
   "section": "2. Held-out clean files by category (INFO+ findings)",
   "columns": [
    "Category",
    "Description",
    "20261103",
    "20261103-fix",
    "20261109-speed"
   ],
   "rows": [
    [
     "app_frameworks_macho",
     "Mach-O binaries inside application frameworks",
     "0/725 (0.00%)",
     "0/725 (0.00%)",
     "0/641 (0.00%)"
    ],
    [
     "app_resources_scripts",
     "Scripts from application resource folders",
     "3/500 (0.60%)",
     "3/500 (0.60%)",
     "2/464 (0.43%)"
    ],
    [
     "homebrew_cellar",
     "Files from installed Homebrew packages",
     "2/800 (0.25%)",
     "2/800 (0.25%)",
     "2/800 (0.25%)"
    ],
    [
     "python_libs",
     "Installed Python library files",
     "0/400 (0.00%)",
     "0/400 (0.00%)",
     "0/400 (0.00%)"
    ],
    [
     "user_documents",
     "User document files",
     "0/22 (0.00%)",
     "0/22 (0.00%)",
     "0/22 (0.00%)"
    ],
    [
     "installers_archives",
     "Downloaded installers and archives",
     "5/30 (16.67%)",
     "2/30 (6.67%)",
     "2/30 (6.67%)"
    ],
    [
     "app_support",
     "Application support data",
     "109/492 (22.15%)",
     "3/492 (0.61%)",
     "3/451 (0.67%)"
    ],
    [
     "project_deps",
     "Dependencies inside software projects",
     "0/4 (0.00%)",
     "0/4 (0.00%)",
     "0/4 (0.00%)"
    ],
    [
     "All held-out",
     "",
     "119/2973 (4.00%)",
     "10/2973 (0.34%)",
     "9/2812 (0.32%)"
    ]
   ],
   "url": "https://benchmarks.site/malware-detection-held-out/",
   "csv": "https://benchmarks.site/data/malware-detection-held-out-05.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "",
   "columns": [
    "Run",
    "Date",
    "Description"
   ],
   "rows": [
    [
     "before",
     "2026-11-02",
     "Heuristics before the fixes."
    ],
    [
     "after",
     "2026-11-02",
     "After the fixes, including an extraction of embedded scripts from Mach-O binaries."
    ],
    [
     "after-final",
     "2026-11-02",
     "The after run with the records it flagged re-scanned after reverting the Mach-O embedded-script extraction. Removing an analyzer path can only remove detections; the two later script rules were checked separately: 0 matches on all 2380 clean text files."
    ],
    [
     "bench-20261103",
     "2026-11-03",
     "Re-measurement the next day, on the day of the first fair benchmark run."
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-01.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "1. Clean files: summary",
   "columns": [
    "Run",
    "Files",
    "False alarms (SUSPICIOUS+)",
    "Of which MALICIOUS",
    "INFO findings",
    "Summed scan time (s)"
   ],
   "rows": [
    [
     "before",
     "2945",
     "6 (0.20%)",
     "0",
     "61 (2.07%)",
     "4149.2"
    ],
    [
     "after",
     "2944",
     "2 (0.07%)",
     "1",
     "61 (2.07%)",
     "4954.6"
    ],
    [
     "after-final",
     "2944",
     "0 (0.00%)",
     "0",
     "62 (2.11%)",
     "4954.6 (from after)"
    ],
    [
     "bench-20261103",
     "2910",
     "0 (0.00%)",
     "0",
     "46 (1.58%)",
     "1044.6"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-02.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "2. Clean files by category",
   "columns": [
    "Category",
    "Description",
    "before",
    "after",
    "after-final",
    "bench-20261103"
   ],
   "rows": [
    [
     "apps_plist",
     "Info.plist files of installed apps",
     "0 / 0 / 36",
     "0 / 0 / 36",
     "0 / 0 / 36",
     "0 / 0 / 36"
    ],
    [
     "apps_helper",
     "Helper executables inside app bundles (Mach-O)",
     "4 / 0 / 90",
     "0 / 0 / 90",
     "0 / 0 / 90",
     "0 / 0 / 90"
    ],
    [
     "apps_main",
     "Main executables of installed apps",
     "1 / 2 / 41",
     "1 / 1 / 41",
     "0 / 2 / 41",
     "0 / 2 / 41"
    ],
    [
     "apps_script",
     "Scripts inside app bundles",
     "1 / 0 / 84",
     "0 / 1 / 84",
     "0 / 1 / 84",
     "0 / 1 / 84"
    ],
    [
     "homebrew_bin",
     "Executables installed by Homebrew",
     "0 / 0 / 495",
     "1 / 0 / 495",
     "0 / 0 / 495",
     "0 / 1 / 495"
    ],
    [
     "extra_script",
     "Additional script files (JavaScript, Python, shell, text, PowerShell, Perl)",
     "0 / 46 / 477",
     "0 / 46 / 477",
     "0 / 46 / 477",
     "0 / 29 / 459"
    ],
    [
     "launchd_plist",
     "launchd job definitions (plist)",
     "0 / 0 / 131",
     "0 / 0 / 131",
     "0 / 0 / 131",
     "0 / 0 / 131"
    ],
    [
     "sysapps_plist",
     "Info.plist files of macOS system apps",
     "0 / 0 / 65",
     "0 / 0 / 65",
     "0 / 0 / 65",
     "0 / 0 / 65"
    ],
    [
     "sysapps_main",
     "Main executables of macOS system apps",
     "0 / 0 / 66",
     "0 / 0 / 66",
     "0 / 0 / 66",
     "0 / 0 / 66"
    ],
    [
     "docs_own",
     "Source and config files of the author's own projects",
     "0 / 7 / 660",
     "0 / 7 / 659",
     "0 / 7 / 659",
     "0 / 7 / 643"
    ],
    [
     "docs_deps",
     "Dependency files inside those projects",
     "0 / 6 / 549",
     "0 / 6 / 549",
     "0 / 6 / 549",
     "0 / 6 / 549"
    ],
    [
     "usr_bin_script",
     "macOS system command-line scripts (Perl, shell, Ruby)",
     "0 / 0 / 251",
     "0 / 0 / 251",
     "0 / 0 / 251",
     "0 / 0 / 251"
    ],
    [
     "All",
     "",
     "6 / 61 / 2945",
     "2 / 61 / 2944",
     "0 / 62 / 2944",
     "0 / 46 / 2910"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-03.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "before (6 false alarms, all SUSPICIOUS)",
   "columns": [
    "Detection",
    "Severity",
    "Files",
    "Categories"
   ],
   "rows": [
    [
     "Heur:MachO.CertificateRevoked",
     "HIGH",
     "5",
     "apps_helper 4, apps_main 1 (together with Heur:Context.DownloadedUnsigned)"
    ],
    [
     "Heur:SHELL.RemoveQuarantine + Heur:SHELL.KeychainAccess",
     "MEDIUM + MEDIUM",
     "1",
     "apps_script 1"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-04.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "after (2 false alarms)",
   "columns": [
    "Category",
    "Verdict",
    "Score",
    "Detections"
   ],
   "rows": [
    [
     "apps_main",
     "MALICIOUS",
     "231",
     "Own:OWN_Script_Bash_CurlPipeShell_RawIpv4, Heur:SHELL.PipeToShell, Heur:SHELL.DecodePipe, Heur:SHELL.KeychainExfil, Heur:SHELL.DownloadExecute, Heur:SHELL.Exfil, Heur:MachO.Downloader, Heur:MachO.AdhocSigned"
    ],
    [
     "homebrew_bin",
     "SUSPICIOUS",
     "53",
     "Heur:SHELL.ReverseShellDevTcp, Heur:MachO.Downloader, Heur:MachO.AdhocSigned"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-05.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "4. Fixtures: detection on test samples",
   "columns": [
    "Run",
    "Mode",
    "INFO+",
    "SUSPICIOUS+",
    "MALICIOUS"
   ],
   "rows": [
    [
     "before",
     "heur",
     "168/528 (31.82%)",
     "107/528 (20.27%)",
     "31/528 (5.87%)"
    ],
    [
     "after",
     "heur",
     "197/528 (37.31%)",
     "135/528 (25.57%)",
     "43/528 (8.14%)"
    ],
    [
     "bench-20261103",
     "heur",
     "197/528 (37.31%)",
     "135/528 (25.57%)",
     "43/528 (8.14%)"
    ],
    [
     "before",
     "yara",
     "519/528 (98.30%)",
     "466/528 (88.26%)",
     "296/528 (56.06%)"
    ],
    [
     "after",
     "yara",
     "519/528 (98.30%)",
     "479/528 (90.72%)",
     "319/528 (60.42%)"
    ],
    [
     "bench-20261103",
     "yara",
     "519/528 (98.30%)",
     "479/528 (90.72%)",
     "319/528 (60.42%)"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-06.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "By sample group, heur mode (INFO+ / SUSPICIOUS+ / MALICIOUS)",
   "columns": [
    "Group",
    "Samples",
    "before",
    "after",
    "bench-20261103"
   ],
   "rows": [
    [
     "documents",
     "15",
     "15 / 15 / 8",
     "15 / 15 / 8",
     "15 / 15 / 8"
    ],
    [
     "eicar",
     "2",
     "0 / 0 / 0",
     "0 / 0 / 0",
     "0 / 0 / 0"
    ],
    [
     "scripts",
     "16",
     "16 / 16 / 5",
     "16 / 16 / 6",
     "16 / 16 / 6"
    ],
    [
     "yara/gen",
     "107",
     "37 / 28 / 7",
     "55 / 45 / 14",
     "55 / 45 / 14"
    ],
    [
     "yara/macos",
     "60",
     "33 / 17 / 5",
     "41 / 26 / 8",
     "41 / 26 / 8"
    ],
    [
     "yara/native",
     "16",
     "4 / 3 / 0",
     "4 / 3 / 0",
     "4 / 3 / 0"
    ],
    [
     "yara/scripts_web",
     "312",
     "63 / 28 / 6",
     "66 / 30 / 7",
     "66 / 30 / 7"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-07.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "By sample group, yara mode (INFO+ / SUSPICIOUS+ / MALICIOUS)",
   "columns": [
    "Group",
    "Samples",
    "before",
    "after",
    "bench-20261103"
   ],
   "rows": [
    [
     "documents",
     "15",
     "15 / 15 / 10",
     "15 / 15 / 10",
     "15 / 15 / 10"
    ],
    [
     "eicar",
     "2",
     "2 / 2 / 2",
     "2 / 2 / 2",
     "2 / 2 / 2"
    ],
    [
     "scripts",
     "16",
     "16 / 16 / 9",
     "16 / 16 / 9",
     "16 / 16 / 9"
    ],
    [
     "yara/gen",
     "107",
     "107 / 55 / 21",
     "107 / 68 / 35",
     "107 / 68 / 35"
    ],
    [
     "yara/macos",
     "60",
     "60 / 60 / 54",
     "60 / 60 / 60",
     "60 / 60 / 60"
    ],
    [
     "yara/native",
     "16",
     "7 / 6 / 4",
     "7 / 6 / 4",
     "7 / 6 / 4"
    ],
    [
     "yara/scripts_web",
     "312",
     "312 / 312 / 196",
     "312 / 312 / 199",
     "312 / 312 / 199"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-08.csv"
  },
  {
   "benchmark": "malware-static-false-positives",
   "title": "Malware scanning: static-heuristic false alarms before and after fixes",
   "area": "Security (macOS malware scanning)",
   "section": "Neutral archives (48 files, no expected verdict)",
   "columns": [
    "Run",
    "Mode",
    "CLEAN",
    "INFO",
    "SUSPICIOUS"
   ],
   "rows": [
    [
     "before",
     "heur",
     "30",
     "10",
     "8"
    ],
    [
     "before",
     "yara",
     "30",
     "8",
     "10"
    ],
    [
     "after",
     "heur",
     "30",
     "10",
     "8"
    ],
    [
     "after",
     "yara",
     "30",
     "8",
     "10"
    ],
    [
     "bench-20261103",
     "heur",
     "30",
     "10",
     "8"
    ],
    [
     "bench-20261103",
     "yara",
     "30",
     "8",
     "10"
    ]
   ],
   "url": "https://benchmarks.site/malware-static-false-positives/",
   "csv": "https://benchmarks.site/data/malware-static-false-positives-09.csv"
  },
  {
   "benchmark": "ml-tlsh-macho-classifiers",
   "title": "ML classifiers on TLSH fingerprints of Mach-O files",
   "area": "Security (macOS malware scanning)",
   "section": "Dataset",
   "columns": [
    "Set",
    "Count"
   ],
   "rows": [
    [
     "Malicious Mach-O files (positives)",
     "1436"
    ],
    [
     "Clean Mach-O files (negatives)",
     "4353"
    ],
    [
     "Benchmark samples (bench)",
     "547"
    ],
    [
     "Malware families",
     "23"
    ]
   ],
   "url": "https://benchmarks.site/ml-tlsh-macho-classifiers/",
   "csv": "https://benchmarks.site/data/ml-tlsh-macho-classifiers-01.csv"
  },
  {
   "benchmark": "ml-tlsh-macho-classifiers",
   "title": "ML classifiers on TLSH fingerprints of Mach-O files",
   "area": "Security (macOS malware scanning)",
   "section": "1. Main results (out-of-fold, threshold at 0.1% FP)",
   "columns": [
    "Model",
    "AUC test",
    "OOF AUC",
    "Recall at 0.1% FP",
    "FP at threshold",
    "Precision",
    "Params",
    "Training (s)"
   ],
   "rows": [
    [
     "GBDT 600× d6 (600 trees, depth 6)",
     "0.967",
     "0.966",
     "39.3% (564/1436)",
     "4/4353",
     "0.993",
     "76200",
     "568.2"
    ],
    [
     "GBDT 300× d4 (300 trees, depth 4)",
     "0.963",
     "0.962",
     "36.5% (524/1436)",
     "4/4353",
     "0.992",
     "9300",
     "159.4"
    ],
    [
     "MLP 1M (561-1024-512-1)",
     "0.943",
     "0.923",
     "23.8% (342/1436)",
     "4/4353",
     "0.988",
     "1100801",
     "307.0"
    ],
    [
     "MLP 0.2M (561-256-256-1)",
     "0.942",
     "0.925",
     "18.0% (259/1436)",
     "4/4353",
     "0.985",
     "209921",
     "67.6"
    ],
    [
     "Logistic regression (linear)",
     "0.896",
     "0.894",
     "9.5% (137/1436)",
     "4/4353",
     "0.972",
     "562",
     "1.9"
    ]
   ],
   "url": "https://benchmarks.site/ml-tlsh-macho-classifiers/",
   "csv": "https://benchmarks.site/data/ml-tlsh-macho-classifiers-02.csv"
  },
  {
   "benchmark": "ml-tlsh-macho-classifiers",
   "title": "ML classifiers on TLSH fingerprints of Mach-O files",
   "area": "Security (macOS malware scanning)",
   "section": "2. Per-fold thresholds (summed over the 5 test folds)",
   "columns": [
    "Model",
    "Recall test (mean over folds)",
    "Malicious detected (sum)",
    "False positives (sum)",
    "Precision",
    "AUC train",
    "Recall train"
   ],
   "rows": [
    [
     "GBDT 600× d6",
     "36.0%",
     "498/1436",
     "2/4353",
     "0.996",
     "1.000",
     "100.0%"
    ],
    [
     "GBDT 300× d4",
     "32.8%",
     "479/1436",
     "2/4353",
     "0.996",
     "1.000",
     "100.0%"
    ],
    [
     "MLP 1M (561-1024-512-1)",
     "21.4%",
     "378/1436",
     "3/4353",
     "0.992",
     "1.000",
     "100.0%"
    ],
    [
     "MLP 0.2M (561-256-256-1)",
     "21.7%",
     "296/1436",
     "2/4353",
     "0.993",
     "1.000",
     "100.0%"
    ],
    [
     "Logistic regression (linear)",
     "4.7%",
     "72/1436",
     "5/4353",
     "0.935",
     "0.992",
     "50.7%"
    ]
   ],
   "url": "https://benchmarks.site/ml-tlsh-macho-classifiers/",
   "csv": "https://benchmarks.site/data/ml-tlsh-macho-classifiers-03.csv"
  },
  {
   "benchmark": "ml-tlsh-macho-classifiers",
   "title": "ML classifiers on TLSH fingerprints of Mach-O files",
   "area": "Security (macOS malware scanning)",
   "section": "3. Baseline: TLSH nearest-neighbour rule (no model)",
   "columns": [
    "Rule",
    "Recall",
    "False positives",
    "Precision",
    "AUC test",
    "Params"
   ],
   "rows": [
    [
     "TLSH-NN ≤ 30",
     "4.7% (68/1436)",
     "1/4353 (0.023%)",
     "0.986",
     "0.582",
     "0"
    ],
    [
     "TLSH-NN ≤ 50",
     "13.7% (197/1436)",
     "7/4353 (0.161%)",
     "0.966",
     "0.582",
     "0"
    ]
   ],
   "url": "https://benchmarks.site/ml-tlsh-macho-classifiers/",
   "csv": "https://benchmarks.site/data/ml-tlsh-macho-classifiers-04.csv"
  },
  {
   "benchmark": "ml-tlsh-macho-classifiers",
   "title": "ML classifiers on TLSH fingerprints of Mach-O files",
   "area": "Security (macOS malware scanning)",
   "section": "4. Additional recall figures recorded by the script",
   "columns": [
    "Model",
    "oof_far_recall",
    "oof_union_recall"
   ],
   "rows": [
    [
     "GBDT 600× d6",
     "30.9% (383/1239)",
     "40.4% (580/1436)"
    ],
    [
     "GBDT 300× d4",
     "28.7% (355/1239)",
     "38.4% (552/1436)"
    ],
    [
     "MLP 1M (561-1024-512-1)",
     "19.8% (245/1239)",
     "30.8% (442/1436)"
    ],
    [
     "MLP 0.2M (561-256-256-1)",
     "12.0% (149/1239)",
     "24.1% (346/1436)"
    ],
    [
     "Logistic regression (linear)",
     "9.2% (114/1239)",
     "21.7% (311/1436)"
    ]
   ],
   "url": "https://benchmarks.site/ml-tlsh-macho-classifiers/",
   "csv": "https://benchmarks.site/data/ml-tlsh-macho-classifiers-05.csv"
  },
  {
   "benchmark": "scanner-performance",
   "title": "Scanner performance: database loading, re-scans, TLSH in SQL, Aho-Corasick",
   "area": "Security (macOS malware scanning)",
   "section": "Ours (own engine)",
   "columns": [
    "Measurement",
    "Date",
    "Before",
    "After",
    "Result"
   ],
   "rows": [
    [
     "Database load time",
     "2026-11-09",
     "60–77 s",
     "~11 s",
     "about 5.5–7× faster"
    ],
    [
     "Scan memory: re-scan of Numbers.app (33819 files)",
     "2026-11-11",
     "131.7 s",
     "24.7 s",
     "about 5.3× faster"
    ],
    [
     "TLSH candidate restriction in SQL (tlsh_candidates_within)",
     "2026-11-09",
     "—",
     "—",
     "718/718 identical results, ~1.6× faster"
    ]
   ],
   "url": "https://benchmarks.site/scanner-performance/",
   "csv": "https://benchmarks.site/data/scanner-performance-01.csv"
  },
  {
   "benchmark": "scanner-performance",
   "title": "Scanner performance: database loading, re-scans, TLSH in SQL, Aho-Corasick",
   "area": "Security (macOS malware scanning)",
   "section": "External libraries",
   "columns": [
    "Library / approach",
    "Date",
    "Measurement",
    "Result"
   ],
   "rows": [
    [
     "pyahocorasick (save/load of a prebuilt automaton, pickle)",
     "2026-11-09",
     "load time",
     "76 s"
    ],
    [
     "ahocorasick_rs (build the automaton)",
     "2026-11-09",
     "build time",
     "~25 s"
    ],
    [
     "Building the automaton in threads (sharding)",
     "2026-11-09",
     "build time",
     "no speedup (GIL)"
    ]
   ],
   "url": "https://benchmarks.site/scanner-performance/",
   "csv": "https://benchmarks.site/data/scanner-performance-02.csv"
  }
 ],
 "public_leaderboards": [
  {
   "benchmark": "humanitys-last-exam-hle",
   "title": "Humanity's Last Exam (HLE)",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "Epoch AI (Benchmarking Hub)",
    "url": "https://epoch.ai/data/benchmark_data.zip",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Accuracy (%)",
    "Calibration error (%)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "GPT 6 Astra",
     "OpenAI",
     "54.8",
     "39.0",
     "2026-09-03"
    ],
    [
     "2",
     "Fable 5.1 (xhigh)",
     "Anthropic",
     "46.5",
     "20.0",
     "2026-09-01"
    ],
    [
     "3",
     "gemini-3.1-pro-preview (thinking high)",
     "Google DeepMind",
     "46.4",
     "51.0",
     "2026-02-19"
    ],
    [
     "4",
     "Sonnet 5.5 (xhigh)",
     "Anthropic",
     "46.1",
     "",
     "2026-09-28"
    ],
    [
     "5",
     "Gemini 3.8 Flash",
     "Google DeepMind",
     "44.5",
     "51.0",
     "2026-09-02"
    ],
    [
     "6",
     "gpt-5.4-pro-2026-03-05",
     "OpenAI",
     "44.3",
     "38.0",
     "2026-03-05"
    ],
    [
     "7",
     "Muse Spark",
     "Meta AI",
     "40.6",
     "50.0",
     "2026-04-08"
    ],
    [
     "8",
     "gemini-3-pro-preview",
     "Google DeepMind",
     "37.5",
     "57.0",
     "2025-11-18"
    ],
    [
     "9",
     "gpt-5.4-2026-03-05 (xhigh thinking)",
     "OpenAI",
     "36.2",
     "42.0",
     "2026-03-05"
    ],
    [
     "10",
     "claude-opus-4-7",
     "Anthropic",
     "36.2",
     "47.0",
     "2026-04-16"
    ],
    [
     "11",
     "claude-opus-4-6-thinking-max",
     "Anthropic",
     "34.4",
     "46.0",
     "2026-02-05"
    ],
    [
     "12",
     "gpt-5-pro-2025-10-06",
     "OpenAI",
     "31.6",
     "49.0",
     "2025-10-07"
    ],
    [
     "13",
     "Claude Haiku 5.5",
     "Anthropic",
     "30.1",
     "",
     "2026-10-07"
    ],
    [
     "14",
     "gpt-5.2-2025-12-11",
     "OpenAI",
     "27.8",
     "45.0",
     "2025-12-11"
    ],
    [
     "15",
     "gpt-5-2025-08-07",
     "OpenAI",
     "25.3",
     "50.0",
     "2025-08-07"
    ],
    [
     "16",
     "gpt-5-2025-08-07",
     "OpenAI",
     "25.3",
     "50.0",
     "2025-08-07"
    ],
    [
     "17",
     "claude-opus-4-5-20251101-thinking",
     "Anthropic",
     "25.2",
     "55.0",
     "2025-11-24"
    ],
    [
     "18",
     "kimi-k2.5",
     "Moonshot",
     "24.4",
     "67.0",
     "2026-01-27"
    ],
    [
     "19",
     "gpt-5.1-thinking",
     "OpenAI",
     "23.7",
     "55.0",
     "2025-11-13"
    ],
    [
     "20",
     "gemini-2.5-pro-preview-06-05",
     "Google DeepMind",
     "21.6",
     "72.0",
     "2025-06-05"
    ],
    [
     "21",
     "o3 (high) (April 2025)",
     "OpenAI",
     "20.3",
     "34.0",
     "2025-04-16"
    ],
    [
     "22",
     "gpt-5-mini-2025-08-07",
     "OpenAI",
     "19.4",
     "65.0",
     "2025-08-07"
    ],
    [
     "23",
     "o3 (medium) (April 2025)",
     "OpenAI",
     "19.2",
     "39.0",
     "2025-04-16"
    ],
    [
     "24",
     "claude-opus-4-6 (Non-Thinking)",
     "Anthropic",
     "19.0",
     "44.0",
     "2026-02-05"
    ],
    [
     "25",
     "Gemini 2.5 Pro Experimental (March 2025)",
     "Google DeepMind",
     "18.2",
     "71.0",
     "2025-03-25"
    ]
   ],
   "url": "https://benchmarks.site/internet/humanitys-last-exam-hle/",
   "csv": "https://benchmarks.site/data/internet/humanitys-last-exam-hle.csv"
  },
  {
   "benchmark": "arc-agi-arc-prize",
   "title": "ARC-AGI (ARC Prize)",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "ARC Prize Foundation",
    "url": "https://arcprize.org/media/data/leaderboard/v3.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-16",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "System",
    "Organization",
    "ARC-AGI-3 score (%)",
    "Total cost (USD)"
   ],
   "rows": [
    [
     "1",
     "GPT-6 Astra - Provider Adapter (High)",
     "OpenAI",
     "99.9",
     "18816.63"
    ],
    [
     "2",
     "Gemini 4 Carbon - Provider Adapter (Medium)",
     "Google",
     "99.1",
     "4559.50"
    ],
    [
     "3",
     "GPT-6 Astra - Provider Adapter (Max)",
     "OpenAI",
     "98.6",
     "17331.95"
    ],
    [
     "4",
     "GPT-6 Astra - Provider Adapter (XHigh)",
     "OpenAI",
     "98.4",
     "18147.19"
    ],
    [
     "5",
     "GPT-6 Astra - Provider Adapter (Medium)",
     "OpenAI",
     "98.4",
     "19284.78"
    ],
    [
     "6",
     "GPT-6 Astra - Provider Adapter (Low)",
     "OpenAI",
     "98.0",
     "21297.92"
    ],
    [
     "7",
     "Claude Fable 5.2 - Provider Adapter (High)",
     "Anthropic",
     "97.7",
     "18624.91"
    ],
    [
     "8",
     "Gemini 4 Carbon - Provider Adapter (Low)",
     "Google",
     "97.7",
     "4350.87"
    ],
    [
     "9",
     "Gemini 4 Argon - Provider Adapter (Low)",
     "Google",
     "97.5",
     "4770.15"
    ],
    [
     "10",
     "Gemini 4 Argon - Provider Adapter (Medium)",
     "Google",
     "97.3",
     "4981.20"
    ],
    [
     "11",
     "Gemini 4 Argon - Provider Adapter (High)",
     "Google",
     "97.3",
     "4625.84"
    ],
    [
     "12",
     "Claude Fable 5.2 - Provider Adapter (XHigh)",
     "Anthropic",
     "97.3",
     "18251.68"
    ],
    [
     "13",
     "Gemini 4 Carbon - Provider Adapter (High)",
     "Google",
     "96.9",
     "4350.03"
    ],
    [
     "14",
     "Claude Fable 5.2 - Provider Adapter (Medium)",
     "Anthropic",
     "96.8",
     "21662.29"
    ],
    [
     "15",
     "GPT-6 Astra - Provider Adapter (None)",
     "OpenAI",
     "96.7",
     "23456.66"
    ],
    [
     "16",
     "GPT-6.1 Sol - Provider Adapter (XHigh)",
     "OpenAI",
     "96.4",
     "4360.21"
    ],
    [
     "17",
     "GPT-6.1 Sol - Provider Adapter (Max)",
     "OpenAI",
     "96.2",
     "3816.80"
    ],
    [
     "18",
     "GPT-6.1 Sol - Provider Adapter (High)",
     "OpenAI",
     "95.0",
     "5403.98"
    ],
    [
     "19",
     "Claude Fable 5.2 - Provider Adapter (Max)",
     "Anthropic",
     "94.9",
     "17875.84"
    ],
    [
     "20",
     "Claude Fable 5.2 - Provider Adapter (Low)",
     "Anthropic",
     "92.3",
     "27377.33"
    ],
    [
     "21",
     "GPT-6.1 Sol - Provider Adapter (Medium)",
     "OpenAI",
     "91.0",
     "5834.91"
    ],
    [
     "22",
     "GPT-6.1 Sol - Provider Adapter (Low)",
     "OpenAI",
     "82.8",
     "6622.20"
    ],
    [
     "23",
     "Claude Fable 5.2 (XHigh)",
     "Anthropic",
     "81.2",
     "21948.83"
    ],
    [
     "24",
     "Claude Fable 5.2 (High)",
     "Anthropic",
     "79.3",
     "25173.94"
    ],
    [
     "25",
     "Claude Fable 5.2 (Medium)",
     "Anthropic",
     "73.3",
     "29034.48"
    ]
   ],
   "url": "https://benchmarks.site/internet/arc-agi-arc-prize/",
   "csv": "https://benchmarks.site/data/internet/arc-agi-arc-prize.csv"
  },
  {
   "benchmark": "simplebench",
   "title": "SimpleBench",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "Epoch AI (Benchmarking Hub)",
    "url": "https://epoch.ai/data/benchmark_data.zip",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Score AVG@5 (%)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2 (unknown)",
     "Anthropic",
     "94.9",
     "2026-11-09"
    ],
    [
     "2",
     "Gemini 4 Carbon (unknown)",
     "Google DeepMind",
     "92.6",
     "2026-11-10"
    ],
    [
     "3",
     "Gemini 4 Argon (unknown)",
     "Google DeepMind",
     "91.7",
     "2026-10-13"
    ],
    [
     "4",
     "Claude Opus 5.5 (unknown)",
     "Anthropic",
     "88.4",
     "2026-09-22"
    ],
    [
     "5",
     "Claude Fable 5.1 (unknown)",
     "Anthropic",
     "86.6",
     "2026-09-01"
    ],
    [
     "6",
     "GPT-6 Astra (pro, max)",
     "OpenAI",
     "86.5",
     "2026-09-03"
    ],
    [
     "7",
     "GPT-6 Astra (unknown thinking)",
     "OpenAI",
     "83.6",
     "2026-09-03"
    ],
    [
     "8",
     "GPT-6.1 Sol (unknown thinking)",
     "OpenAI",
     "82.9",
     "2026-09-29"
    ],
    [
     "9",
     "Gemini 3.8 Flash (unknown)",
     "Google DeepMind",
     "82.4",
     "2026-09-02"
    ],
    [
     "10",
     "Claude Fable 5 (max)",
     "Anthropic",
     "81.9",
     "2026-06-09"
    ],
    [
     "11",
     "Muse Spark 1.3 (unknown)",
     "Meta AI",
     "81.8",
     "2026-09-02"
    ],
    [
     "12",
     "Claude Opus 5 (unknown)",
     "Anthropic",
     "80.6",
     "2026-07-24"
    ],
    [
     "13",
     "Gemini 3.1 Pro Preview",
     "Google DeepMind",
     "79.6",
     "2026-02-19"
    ],
    [
     "14",
     "GPT-5.5 Pro (unknown thinking)",
     "OpenAI",
     "76.9",
     "2026-04-23"
    ],
    [
     "15",
     "GPT-5.5 Pro (high)",
     "OpenAI",
     "76.9",
     "2026-04-23"
    ],
    [
     "16",
     "Gemini 3.5 Flash",
     "Google DeepMind",
     "76.7",
     "2026-05-19"
    ],
    [
     "17",
     "Gemini 3 Pro Preview",
     "Google DeepMind",
     "76.4",
     "2025-11-18"
    ],
    [
     "18",
     "Grok 4.6 (unknown)",
     "xAI",
     "75.9",
     "2026-08-12"
    ],
    [
     "19",
     "Claude Sonnet 5.5 (unknown)",
     "Anthropic",
     "75.9",
     "2026-09-28"
    ],
    [
     "20",
     "Muse Spark 1.2",
     "Meta AI",
     "74.5",
     "2026-08-05"
    ],
    [
     "21",
     "GPT-5.4 Pro",
     "OpenAI",
     "74.1",
     "2026-03-05"
    ],
    [
     "22",
     "GPT-6 Sol (unknown thinking)",
     "OpenAI",
     "73.1",
     "2026-09-22"
    ],
    [
     "23",
     "Muse Ember (unknown)",
     "Meta AI",
     "72.7",
     "2026-11-09"
    ],
    [
     "24",
     "GPT-6.1 Terra (unknown thinking)",
     "OpenAI",
     "72.2",
     "2026-10-26"
    ],
    [
     "25",
     "GPT-5.6 Sol (pro, xhigh)",
     "OpenAI",
     "71.7",
     "2026-07-09"
    ]
   ],
   "url": "https://benchmarks.site/internet/simplebench/",
   "csv": "https://benchmarks.site/data/internet/simplebench.csv"
  },
  {
   "benchmark": "livebench",
   "title": "LiveBench",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "LiveBench team (Abacus.AI and collaborators)",
    "url": "https://livebench.ai/data/20261106/table.csv",
    "as_of": "2026-11-06",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model",
    "Overall (computed)",
    "Long Agentic",
    "Reasoning",
    "Coding"
   ],
   "rows": [
    [
     "1",
     "claude-fable-5.2",
     "84.9",
     "82.9",
     "88.8",
     "79.2"
    ],
    [
     "2",
     "gpt-6-astra",
     "84.4",
     "75.6",
     "92.7",
     "87.5"
    ],
    [
     "3",
     "claude-fable-5.1",
     "84.0",
     "74.2",
     "91.7",
     "89.7"
    ],
    [
     "4",
     "gemini-4-carbon",
     "83.5",
     "72.7",
     "90.6",
     "91.0"
    ],
    [
     "5",
     "claude-opus-5.5",
     "83.4",
     "72.8",
     "92.2",
     "87.1"
    ],
    [
     "6",
     "gpt-6.1-sol",
     "81.4",
     "65.6",
     "92.6",
     "82.1"
    ],
    [
     "7",
     "gpt-6.1-terra",
     "78.8",
     "56.2",
     "88.7",
     "89.2"
    ],
    [
     "8",
     "gemini-4-argon",
     "77.7",
     "58.5",
     "95.1",
     "85.7"
    ],
    [
     "9",
     "gemini-4-hydrogen",
     "77.0",
     "61.8",
     "88.6",
     "78.5"
    ],
    [
     "10",
     "claude-sonnet-5.5",
     "77.0",
     "63.4",
     "91.6",
     "77.2"
    ],
    [
     "11",
     "qwen4-max",
     "73.3",
     "47.5",
     "92.2",
     "69.8"
    ],
    [
     "12",
     "muse-wonder-1.0",
     "70.7",
     "46.2",
     "90.6",
     "74.9"
    ],
    [
     "13",
     "deepseek-v4.1-flash-1017",
     "70.3",
     "38.8",
     "89.1",
     "80.4"
    ],
    [
     "14",
     "claude-haiku-5.5",
     "68.9",
     "47.9",
     "82.4",
     "77.6"
    ],
    [
     "15",
     "kimi-k3.1-pro",
     "68.7",
     "38.7",
     "91.1",
     "67.8"
    ],
    [
     "16",
     "muse-spark-1.3",
     "68.6",
     "47.9",
     "89.7",
     "84.4"
    ],
    [
     "17",
     "grok-4.7",
     "67.9",
     "45.3",
     "82.7",
     "73.1"
    ],
    [
     "18",
     "gpt-5.6-terra",
     "67.2",
     "25.9",
     "90.6",
     "84.2"
    ],
    [
     "19",
     "mimo-v2.6-pro",
     "66.1",
     "36.4",
     "79.6",
     "64.5"
    ],
    [
     "20",
     "kimi-k3",
     "65.0",
     "36.0",
     "90.7",
     "53.1"
    ],
    [
     "21",
     "minimax-m3.1-flash",
     "63.7",
     "25.7",
     "82.9",
     "68.8"
    ],
    [
     "22",
     "grok-4.6",
     "63.5",
     "18.7",
     "91.2",
     "76.7"
    ],
    [
     "23",
     "qwen3.8-max",
     "60.9",
     "26.6",
     "88.2",
     "60.9"
    ],
    [
     "24",
     "glm-5.3-flash",
     "60.7",
     "20.7",
     "77.6",
     "70.9"
    ],
    [
     "25",
     "gpt-6-luna",
     "60.4",
     "24.7",
     "81.8",
     "70.8"
    ]
   ],
   "url": "https://benchmarks.site/internet/livebench/",
   "csv": "https://benchmarks.site/data/internet/livebench.csv"
  },
  {
   "benchmark": "critpt",
   "title": "CritPt",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "Epoch AI (Benchmarking Hub)",
    "url": "https://epoch.ai/data/benchmark_data.zip",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Accuracy (%)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "Gemini 4 Carbon (High)",
     "Google DeepMind",
     "36.0",
     "2026-11-10"
    ],
    [
     "2",
     "Claude Fable 5.2 (Xhigh, Default Fallback)",
     "Anthropic",
     "35.7",
     "2026-11-09"
    ],
    [
     "3",
     "Claude Fable 5.2 (Max, Default Fallback)",
     "Anthropic",
     "35.4",
     "2026-11-09"
    ],
    [
     "4",
     "Claude Fable 5.2 (High, Default Fallback)",
     "Anthropic",
     "32.9",
     "2026-11-09"
    ],
    [
     "5",
     "GPT-5.6 Sol (Max)",
     "OpenAI",
     "32.3",
     "2026-07-09"
    ],
    [
     "6",
     "GPT-6 Astra (Max)",
     "OpenAI",
     "31.7",
     "2026-09-03"
    ],
    [
     "7",
     "Claude Opus 5.5 (Max, Default Fallback)",
     "Anthropic",
     "31.7",
     "2026-09-22"
    ],
    [
     "8",
     "Claude Opus 5.5 (Xhigh, Default Fallback)",
     "Anthropic",
     "31.7",
     "2026-09-22"
    ],
    [
     "9",
     "GPT-6.1 Sol (Max)",
     "OpenAI",
     "31.7",
     "2026-09-29"
    ],
    [
     "10",
     "GPT-6.1 Sol (Xhigh)",
     "OpenAI",
     "31.7",
     "2026-09-29"
    ],
    [
     "11",
     "GPT-6 Astra (Xhigh)",
     "OpenAI",
     "31.4",
     "2026-09-03"
    ],
    [
     "12",
     "Claude Sonnet 5.5 (Max, Default Fallback)",
     "Anthropic",
     "31.4",
     "2026-09-28"
    ],
    [
     "13",
     "Claude Fable 5.1 (Xhigh, Default Fallback)",
     "Anthropic",
     "31.1",
     "2026-09-01"
    ],
    [
     "14",
     "Claude Sonnet 5.5 (Xhigh, Default Fallback)",
     "Anthropic",
     "31.1",
     "2026-09-28"
    ],
    [
     "15",
     "GPT-6 Sol (Max)",
     "OpenAI",
     "30.9",
     "2026-09-22"
    ],
    [
     "16",
     "Claude Opus 5.5 (High, Default Fallback)",
     "Anthropic",
     "30.9",
     "2026-09-22"
    ],
    [
     "17",
     "Claude Fable 5.2 (Medium, Default Fallback)",
     "Anthropic",
     "30.9",
     "2026-11-09"
    ],
    [
     "18",
     "GPT-5.5 Pro (Xhigh)",
     "OpenAI",
     "30.6",
     "2026-04-23"
    ],
    [
     "19",
     "Claude Fable 5.1 (High, Default Fallback)",
     "Anthropic",
     "30.3",
     "2026-09-01"
    ],
    [
     "20",
     "GPT-5.4 Pro (Xhigh)",
     "OpenAI",
     "30.0",
     "2026-03-05"
    ],
    [
     "21",
     "GPT-5.6 Terra (Max)",
     "OpenAI",
     "30.0",
     "2026-07-09"
    ],
    [
     "22",
     "GPT-6.1 Sol (High)",
     "OpenAI",
     "30.0",
     "2026-09-29"
    ],
    [
     "23",
     "Claude Fable 5.1 (Max, Default Fallback)",
     "Anthropic",
     "29.7",
     "2026-09-01"
    ],
    [
     "24",
     "Claude Fable 5.2 (Low, Default Fallback)",
     "Anthropic",
     "29.4",
     "2026-11-09"
    ],
    [
     "25",
     "Claude Opus 5 (Adaptive Reasoning, Max Effort)",
     "Anthropic",
     "29.1",
     "2026-07-24"
    ]
   ],
   "url": "https://benchmarks.site/internet/critpt/",
   "csv": "https://benchmarks.site/data/internet/critpt.csv"
  },
  {
   "benchmark": "eq-bench",
   "title": "EQ-Bench",
   "area": "General knowledge & reasoning",
   "section": "Published results",
   "source": {
    "name": "EQ-Bench (Sam Paech)",
    "url": "https://eqbench.com/eqbench4/eqbench4_data.js",
    "as_of": "2026-10-24",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model",
    "Elo",
    "95% CI"
   ],
   "rows": [
    [
     "1",
     "claude-opus-5",
     "1364.9",
     "1349.5 - 1378.4"
    ],
    [
     "2",
     "claude-opus-5-5",
     "1339.5",
     "1329.4 - 1349.6"
    ],
    [
     "3",
     "muse-wonder",
     "1324.9",
     "1312.5 - 1339.0"
    ],
    [
     "4",
     "kimi-k3",
     "1319.2",
     "1308.7 - 1333.0"
    ],
    [
     "5",
     "claude-fable-5",
     "1312.5",
     "1302.8 - 1323.6"
    ],
    [
     "6",
     "gpt-5.5",
     "1295.4",
     "1285.0 - 1305.6"
    ],
    [
     "7",
     "claude-opus-4-7",
     "1293.2",
     "1283.6 - 1303.4"
    ],
    [
     "8",
     "gpt-6.1-sol",
     "1272.4",
     "1262.9 - 1282.7"
    ],
    [
     "9",
     "claude-opus-4-8",
     "1267.8",
     "1258.6 - 1277.0"
    ],
    [
     "10",
     "gpt-5.4",
     "1257.7",
     "1250.5 - 1265.3"
    ],
    [
     "11",
     "muse-spark-1.1",
     "1244.4",
     "1230.8 - 1256.2"
    ],
    [
     "12",
     "gemini-4-argon",
     "1236.3",
     "1225.0 - 1248.1"
    ],
    [
     "13",
     "gpt-5.6-sol",
     "1236.1",
     "1225.4 - 1247.7"
    ],
    [
     "14",
     "claude-sonnet-5",
     "1221.2",
     "1211.6 - 1233.5"
    ],
    [
     "15",
     "gpt-5.6-terra",
     "1220.1",
     "1209.9 - 1229.9"
    ],
    [
     "16",
     "Inkling",
     "1212.1",
     "1201.2 - 1222.0"
    ],
    [
     "17",
     "claude-opus-4-6",
     "1210.0",
     "1198.5 - 1219.6"
    ],
    [
     "18",
     "glm-5.2",
     "1208.1",
     "1197.1 - 1216.9"
    ],
    [
     "19",
     "mimo-v2.5-pro",
     "1197.7",
     "1185.0 - 1212.5"
    ],
    [
     "20",
     "claude-sonnet-4-6",
     "1196.8",
     "1187.9 - 1204.9"
    ],
    [
     "21",
     "kimi-k2.6",
     "1191.4",
     "1181.4 - 1200.8"
    ],
    [
     "22",
     "deepseek-v4-pro",
     "1156.3",
     "1144.7 - 1166.7"
    ],
    [
     "23",
     "gpt-5.6-luna",
     "1145.9",
     "1134.2 - 1156.6"
    ],
    [
     "24",
     "minimax-m3",
     "1140.1",
     "1128.8 - 1152.1"
    ],
    [
     "25",
     "gemini-3.1-pro-preview",
     "1132.3",
     "1118.9 - 1144.4"
    ]
   ],
   "url": "https://benchmarks.site/internet/eq-bench/",
   "csv": "https://benchmarks.site/data/internet/eq-bench.csv"
  },
  {
   "benchmark": "frontiermath",
   "title": "FrontierMath",
   "area": "Math",
   "section": "Published results",
   "source": {
    "name": "Epoch AI (Benchmarking Hub)",
    "url": "https://epoch.ai/data/benchmark_data.zip",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Tiers 1-3 (%)",
    "Tier 4 (%)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2 (max)",
     "Anthropic",
     "95.1",
     "100.0",
     "2026-11-09"
    ],
    [
     "2",
     "GPT-6.1 Sol (max)",
     "OpenAI",
     "93.7",
     "100.0",
     "2026-09-29"
    ],
    [
     "3",
     "GPT-6 Astra (max)",
     "OpenAI",
     "93.7",
     "97.6",
     "2026-09-03"
    ],
    [
     "4",
     "Claude Opus 5.5 (max)",
     "Anthropic",
     "91.2",
     "95.0",
     "2026-09-22"
    ],
    [
     "5",
     "GPT-6.1 Terra (max)",
     "OpenAI",
     "90.9",
     "92.7",
     "2026-10-26"
    ],
    [
     "6",
     "Gemini 4 Carbon (high)",
     "Google DeepMind",
     "90.5",
     "95.1",
     "2026-11-10"
    ],
    [
     "7",
     "Claude Fable 5.1 (max)",
     "Anthropic",
     "90.2",
     "87.8",
     "2026-09-01"
    ],
    [
     "8",
     "GPT-6 Sol (max)",
     "OpenAI",
     "89.8",
     "90.0",
     "2026-09-22"
    ],
    [
     "9",
     "GPT-5.6 Sol (max)",
     "OpenAI",
     "89.1",
     "82.9",
     "2026-07-09"
    ],
    [
     "10",
     "Claude Sonnet 5.5 (max)",
     "Anthropic",
     "88.8",
     "80.5",
     "2026-09-28"
    ],
    [
     "11",
     "GPT-5.5 Pro (xhigh)",
     "OpenAI",
     "87.7",
     "78.0",
     "2026-04-23"
    ],
    [
     "12",
     "Claude Fable 5 (max)",
     "Anthropic",
     "87.0",
     "90.2",
     "2026-06-09"
    ],
    [
     "13",
     "Gemini 4 Argon (high)",
     "Google DeepMind",
     "86.0",
     "78.0",
     "2026-10-13"
    ],
    [
     "14",
     "GPT-5.6 Terra (max)",
     "OpenAI",
     "86.0",
     "70.7",
     "2026-07-09"
    ],
    [
     "15",
     "Claude Opus 5 (max)",
     "Anthropic",
     "85.6",
     "73.2",
     "2026-07-24"
    ],
    [
     "16",
     "GPT-5.5 (xhigh)",
     "OpenAI",
     "85.3",
     "72.5",
     "2026-04-23"
    ],
    [
     "17",
     "GPT-5.4 Pro (xhigh)",
     "OpenAI",
     "82.5",
     "58.5",
     "2026-03-05"
    ],
    [
     "18",
     "Muse Wonder (max)",
     "Meta AI",
     "82.1",
     "70.7",
     "2026-10-12"
    ],
    [
     "19",
     "GPT-5.6 Luna (max)",
     "OpenAI",
     "82.1",
     "61.0",
     "2026-07-09"
    ],
    [
     "20",
     "Claude Opus 4.8",
     "Anthropic",
     "80.0",
     "56.1",
     "2026-05-28"
    ],
    [
     "21",
     "GPT-6 Luna (max)",
     "OpenAI",
     "78.9",
     "56.1",
     "2026-09-22"
    ],
    [
     "22",
     "GPT-5.4 (xhigh)",
     "OpenAI",
     "78.6",
     "49.0",
     "2026-03-05"
    ],
    [
     "23",
     "Kimi K3.1 Pro (Together, max)",
     "Moonshot",
     "78.2",
     "65.9",
     "2026-10-23"
    ],
    [
     "24",
     "Claude Haiku 5.5 (max)",
     "Anthropic",
     "75.1",
     "46.3",
     "2026-10-07"
    ],
    [
     "25",
     "Qwen3.8 Max (xhigh)",
     "Alibaba",
     "74.7",
     "46.3",
     "2026-08-02"
    ]
   ],
   "url": "https://benchmarks.site/internet/frontiermath/",
   "csv": "https://benchmarks.site/data/internet/frontiermath.csv"
  },
  {
   "benchmark": "matharena",
   "title": "MathArena",
   "area": "Math",
   "section": "Published results",
   "source": {
    "name": "MathArena (SRI Lab ETH Zurich / INSAIT)",
    "url": "https://matharena.ai/",
    "as_of": "2026-11-13",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model",
    "Provider",
    "Accuracy (± 95% CI)",
    "Cost"
   ],
   "rows": [
    [
     "1",
     "Gemini 4 Carbon (high)",
     "Google",
     "94.28% ± 3.64%",
     "$5.08"
    ],
    [
     "2",
     "GPT-6.1 Sol (max)",
     "OpenAI",
     "92.81% ± 3.10%",
     "$0.96"
    ],
    [
     "3",
     "Gemini 4 Argon (high)",
     "Google",
     "84.39% ± 4.65%",
     "$3.74"
    ],
    [
     "4",
     "Claude-Opus-5.5 (high)",
     "Anthropic",
     "82.61% ± 4.72%",
     "$1.93"
    ],
    [
     "5",
     "Kimi K3.1 Pro (max)",
     "Moonshot AI",
     "58.27% ± 6.84%",
     "$2.94"
    ],
    [
     "6",
     "DeepSeek-V4.1-Flash-1017 (max, DSH)",
     "DeepSeek",
     "42.48% ± 6.32%",
     "$0.33"
    ],
    [
     "7",
     "Inkling 2 Small (0.99)",
     "Thinking Machines Lab",
     "31.88% ± 5.14%",
     "$0.60"
    ],
    [
     "8",
     "Muse Spark Open (xhigh)",
     "Meta AI",
     "27.02% ± 5.61%",
     "$1.03"
    ]
   ],
   "url": "https://benchmarks.site/internet/matharena/",
   "csv": "https://benchmarks.site/data/internet/matharena.csv"
  },
  {
   "benchmark": "putnambench",
   "title": "PutnamBench",
   "area": "Math",
   "section": "Published results",
   "source": {
    "name": "PutnamBench (Trishul lab, UT Austin)",
    "url": "https://trishullab.github.io/PutnamBench/results.json",
    "as_of": "2026-10-06",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Method",
    "Solved (of 672)",
    "Date added",
    "Compute budget"
   ],
   "rows": [
    [
     "1",
     "Aleph Prover (Logical Intelligence)   ",
     "672",
     "2026-08-26",
     "Avg $74, Max $1468 per problem"
    ],
    [
     "2",
     "Humanfia",
     "672",
     "2026-08-27",
     "Avg $44.5 total cost/problem"
    ],
    [
     "3",
     "NEAR AI (w/ DeepSeek V4)",
     "672",
     "2026-09-04",
     "Mean $0.17, median $0.04, max $11.18 per problem"
    ],
    [
     "4",
     "Forall (Astrio)",
     "672",
     "2026-09-24",
     "Mean $3.52 actor-side cost/problem; median $2.49"
    ],
    [
     "5",
     "Aleph Prover (Logical Intelligence)  ",
     "668",
     "2026-01-11",
     "Avg $68, Limit $1400 per problem"
    ],
    [
     "6",
     "Mistral Prover (DeepSeek V4 Pro + Leanstral)",
     "658",
     "2026-09-04",
     "Avg 5.38M tokens per solved problem"
    ],
    [
     "7",
     "Goedel-Architect ",
     "642",
     "2026-07-02",
     "Avg $5.8 total cost/problem"
    ],
    [
     "8",
     "Aleph Prover (Logical Intelligence) ",
     "637",
     "2025-12-26",
     "Avg $54, Limit $400 per problem"
    ],
    [
     "9",
     "Goedel-Architect",
     "597",
     "2026-06-06",
     "pass@4, Avg. $1.47 total cost / problem"
    ],
    [
     "10",
     "Leanstral 1.5",
     "588",
     "2026-07-19",
     "4m tokens per problem"
    ],
    [
     "11",
     "Seed-Prover 1.5 (ByteDance)",
     "581",
     "2025-12-27",
     "10 H20 days per problem"
    ],
    [
     "12",
     "Aleph Prover (Logical Intelligence)",
     "500",
     "2025-12-23",
     "Avg $23, Limit $100 per problem"
    ],
    [
     "13",
     "Hilbert",
     "462",
     "2025-10-06",
     "avg pass@1840"
    ],
    [
     "14",
     "AxProverBase (Axiomatic AI)",
     "365",
     "2026-04-14",
     "Avg $12.60, Opus 4.5 w/ 50 iterations"
    ],
    [
     "15",
     "Seed-Prover (ByteDance)",
     "329",
     "2025-12-23",
     "MEDIUM"
    ],
    [
     "16",
     "C2C-Goedel",
     "110",
     "2026-08-01",
     "pass@256"
    ],
    [
     "17",
     "Ax-Prover (Axiomatic AI)",
     "91",
     "2025-12-23",
     "pass@1, avg. 100 tool calls"
    ],
    [
     "18",
     "Goedel-Prover-V2",
     "86",
     "2025-07-22",
     "pass@184"
    ],
    [
     "19",
     "Ensemble Prover (Graviterra)",
     "65",
     "2026-09-04",
     "Not reported"
    ],
    [
     "20",
     "DeepSeek-Prover-V2",
     "47",
     "2025-06-03",
     "pass@1024"
    ],
    [
     "21",
     "GPT-5 (ReAct, 10 turns)",
     "28",
     "2025-11-20",
     "pass@1, 10 tool calls"
    ],
    [
     "22",
     "C2C-Kimina",
     "25",
     "2026-08-01",
     "pass@256"
    ],
    [
     "23",
     "DSP+",
     "23",
     "2025-06-28",
     "pass@128"
    ],
    [
     "24",
     "Bourbaki",
     "14",
     "2025-07-16",
     "pass@512"
    ],
    [
     "25",
     "Kimina-Prover-7B-Distill",
     "10",
     "2025-04-14",
     "pass@192"
    ]
   ],
   "url": "https://benchmarks.site/internet/putnambench/",
   "csv": "https://benchmarks.site/data/internet/putnambench.csv"
  },
  {
   "benchmark": "swe-bench-pro",
   "title": "SWE-Bench Pro",
   "area": "Coding & software engineering",
   "section": "Published results",
   "source": {
    "name": "Scale AI Labs",
    "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2",
    "as_of": "2026-09-22",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model (scaffold, effort)",
    "Score (%)"
   ],
   "rows": [
    [
     "1",
     "Opus 5 (Claude Code) xhigh",
     "98.00"
    ],
    [
     "2",
     "Fable 5.1 (Claude Code) high",
     "92.20"
    ],
    [
     "3",
     "Gemini 4 Argon (mini-swe-agent) high",
     "92.16"
    ],
    [
     "4",
     "GPT-6 Astra (Codex) high",
     "90.20"
    ],
    [
     "4",
     "Muse Wonder 1.0 (mini-swe-agent) xhigh",
     "90.20"
    ],
    [
     "6",
     "Kimi-K3 (mini-swe-agent) max",
     "88.20"
    ],
    [
     "6",
     "Sonnet 5 (Claude Code) xhigh",
     "88.20"
    ],
    [
     "8",
     "GPT-5.6 Terra (Codex) xhigh",
     "86.30"
    ],
    [
     "8",
     "Inkling 2 Small",
     "86.27"
    ],
    [
     "9",
     "Minimax M3p1 Flash (mini-swe-agent) xhigh",
     "84.31"
    ],
    [
     "10",
     "GLM-5.3 (mini-swe-agent) max",
     "84.30"
    ],
    [
     "11",
     "GPT-5.6 Sol (Codex) xhigh",
     "82.40"
    ],
    [
     "12",
     "Muse Spark Open (mini-swe-agent) xhigh",
     "82.35"
    ],
    [
     "13",
     "Sonnet 5.5 (mini-swe-agent) xhigh",
     "70.59"
    ],
    [
     "14",
     "Gemini 3.8 Flash (mini-swe-agent) high",
     "58.80"
    ],
    [
     "15",
     "Inkling (mini-swe-agent) xhigh",
     "56.90"
    ],
    [
     "16",
     "Haiku 5.5 (mini-swe-agent) high",
     "52.94"
    ],
    [
     "17",
     "Haiku 4.5 (Claude Code) xhigh",
     "25.50"
    ]
   ],
   "url": "https://benchmarks.site/internet/swe-bench-pro/",
   "csv": "https://benchmarks.site/data/internet/swe-bench-pro.csv"
  },
  {
   "benchmark": "swe-rebench",
   "title": "SWE-rebench",
   "area": "Coding & software engineering",
   "section": "Published results",
   "source": {
    "name": "SWE-rebench (Nebius)",
    "url": "https://swe-rebench.com/",
    "as_of": "2026-07-01",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model / Agent",
    "Resolved Rate (%)",
    "Pass@5 (%)",
    "Cost per Problem"
   ],
   "rows": [
    [
     "1",
     "Anthropic Fable 5 [high]",
     "64.5",
     "78.4",
     "$4.40"
    ],
    [
     "2",
     "Grok Grok 4.5 [high]",
     "63.8",
     "77.5",
     "$1.47"
    ],
    [
     "3",
     "Anthropic Opus 5 [high]",
     "63.4",
     "74.8",
     "$3.47"
    ],
    [
     "4",
     "Z.ai GLM-5.2 [high]",
     "62.9",
     "81.1",
     "$1.40"
    ],
    [
     "5",
     "OpenAI GPT-5.6 Sol [medium]",
     "62.3",
     "79.3",
     "$0.85"
    ],
    [
     "6",
     "Junie Junie",
     "61.8",
     "73.9",
     "$0.81"
    ],
    [
     "7",
     "Anthropic Claude Code",
     "60.4",
     "75.7",
     "$3.39"
    ],
    [
     "8",
     "OpenAI Codex",
     "58.0",
     "73.0",
     "$1.59"
    ],
    [
     "9",
     "Anthropic Sonnet 5 [high]",
     "56.8",
     "74.8",
     "$1.43"
    ],
    [
     "10",
     "Cursor Cursor",
     "51.7",
     "65.8",
     "$0.41"
    ],
    [
     "11",
     "Minimax MiniMax M3",
     "47.2",
     "69.4",
     "$0.95"
    ],
    [
     "12",
     "XiaomiMiMo MiMo V2.5 Pro",
     "46.5",
     "65.8",
     "$0.10"
    ],
    [
     "13",
     "OpenAI GPT-5.6 Luna [medium]",
     "43.6",
     "59.5",
     "$0.11"
    ],
    [
     "14",
     "DeepSeek DeepSeek-V4 Pro [high]",
     "40.2",
     "64.0",
     "$0.15"
    ],
    [
     "15",
     "Qwen Qwen3.6-27B",
     "31.2",
     "57.7",
     "$0.62"
    ],
    [
     "16",
     "Qwen Qwen3.6-35B-A3B",
     "24.7",
     "43.2",
     "$0.27"
    ],
    [
     "17",
     "Qwen Qwen3.5-35B-A3B",
     "17.1",
     "36.9",
     "$0.99"
    ]
   ],
   "url": "https://benchmarks.site/internet/swe-rebench/",
   "csv": "https://benchmarks.site/data/internet/swe-rebench.csv"
  },
  {
   "benchmark": "terminal-bench",
   "title": "Terminal-Bench",
   "area": "Coding & software engineering",
   "section": "Published results",
   "source": {
    "name": "Terminal-Bench (Stanford / Harbor / Laude Institute)",
    "url": "https://www.tbench.ai/leaderboard",
    "as_of": null,
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model",
    "Agent",
    "Resolution rate",
    "Cost"
   ],
   "rows": [
    [
     "1",
     "Opus 5.5 ( max )",
     "Claude Code",
     "65.8% ± 3.1%",
     "$4.7k"
    ],
    [
     "1",
     "Fable 5.2 ( max )",
     "Claude Code",
     "65.8% ± 2.8%",
     "$16.5k"
    ],
    [
     "3",
     "Gemini 4 Carbon ( high )",
     "Antigravity CLI",
     "64.5% ± 3.2%",
     "$1.4k"
    ],
    [
     "4",
     "Sonnet 5.5 ( max )",
     "Claude Code",
     "63.9% ± 3.0%",
     "$7.3k"
    ],
    [
     "5",
     "GPT-6.1 Sol ( max )",
     "Codex",
     "60.9% ± 3.0%",
     "$0.6k"
    ],
    [
     "6",
     "GPT-6 Astra ( max )",
     "Codex",
     "60.3% ± 2.8%",
     "$3.3k"
    ],
    [
     "7",
     "Kimi K3.1 Pro ( max )",
     "Claude Code",
     "59.7% ± 4.2%",
     "$1.1k"
    ],
    [
     "8",
     "Fable 5.1 ( xhigh )",
     "Claude Code",
     "57.6% ± 3.3%",
     "$4.9k"
    ],
    [
     "9",
     "Gemini 4 Hydrogen ( high )",
     "Antigravity CLI",
     "56.4% ± 3.0%",
     "$0.8k"
    ],
    [
     "10",
     "Gemini 4 Argon ( high )",
     "Antigravity CLI",
     "55.5% ± 2.8%",
     "$1.1k"
    ],
    [
     "11",
     "Opus 5 ( xhigh )",
     "Claude Code",
     "53.3% ± 3.1%",
     "$6.1k"
    ],
    [
     "11",
     "GPT-6.1 Terra ( max )",
     "Codex",
     "53.3% ± 3.1%",
     "$0.9k"
    ],
    [
     "13",
     "GPT-6 Sol ( max )",
     "Codex",
     "50.3% ± 3.2%",
     "$1.5k"
    ],
    [
     "14",
     "Muse Wonder ( xhigh )",
     "Muse Code",
     "47.0% ± 3.8%",
     "$2.5k"
    ],
    [
     "15",
     "Fable 5 ( max )",
     "Claude Code",
     "44.8% ± 3.8%",
     "$7.3k"
    ],
    [
     "16",
     "GLM-5.3 ( max )",
     "Claude Code",
     "41.2% ± 3.3%",
     "$2.7k"
    ],
    [
     "17",
     "Haiku 5.5 ( max )",
     "Claude Code",
     "40.0% ± 3.6%",
     "$1.0k"
    ],
    [
     "18",
     "Grok 4.7 ( xhigh )",
     "Grok Build",
     "38.8% ± 3.6%",
     "$3.7k"
    ],
    [
     "19",
     "GPT-5.6 Sol ( max )",
     "Codex",
     "38.5% ± 3.8%",
     "$2.5k"
    ],
    [
     "19",
     "DeepSeek V4.1 Flash 1017 ( max )",
     "Claude Code",
     "38.5% ± 3.8%",
     "$0.2k"
    ],
    [
     "21",
     "Hy4 ( max )",
     "Claude Code",
     "37.9% ± 3.5%",
     "$0.3k"
    ],
    [
     "22",
     "GLM-5.3-Flash ( max )",
     "Claude Code",
     "37.0% ± 3.6%",
     "$0.4k"
    ],
    [
     "23",
     "Muse Ember ( xhigh )",
     "Muse Code",
     "30.6% ± 3.8%",
     "$0.2k"
    ],
    [
     "24",
     "Mistral Large 4 ( max )",
     "Claude Code",
     "27.6% ± 3.9%",
     "$0.7k"
    ],
    [
     "25",
     "Qwen3.8-Max-0902 ( max )",
     "Claude Code",
     "27.0% ± 3.9%",
     "$4.6k"
    ]
   ],
   "url": "https://benchmarks.site/internet/terminal-bench/",
   "csv": "https://benchmarks.site/data/internet/terminal-bench.csv"
  },
  {
   "benchmark": "swe-atlas",
   "title": "SWE Atlas",
   "area": "Coding & software engineering",
   "section": "Published results",
   "source": {
    "name": "Scale AI Labs",
    "url": "https://labs.scale.com/leaderboard/swe-atlas",
    "as_of": "2026-11",
    "license": "not stated",
    "retrieved": "2026-11-17"
   },
   "columns": [
    "Rank",
    "Model",
    "Score (%)",
    "± margin"
   ],
   "rows": [
    [
     "1",
     "Fable 5.2",
     "80.55",
     "4.80"
    ],
    [
     "2",
     "Opus 5.5",
     "77.45",
     "5.05"
    ],
    [
     "3",
     "Fable 5.1",
     "76.45",
     "5.16"
    ],
    [
     "4",
     "Gemini 4 Carbon",
     "71.45",
     "5.46"
    ],
    [
     "5",
     "Gemini 4 Argon",
     "70.08",
     "5.48"
    ],
    [
     "6",
     "GPT-6 Astra",
     "69.42",
     "5.55"
    ],
    [
     "7",
     "Sonnet 5.5",
     "68.28",
     "5.64"
    ],
    [
     "8",
     "GPT-6.1 Sol",
     "65.88",
     "5.74"
    ],
    [
     "9",
     "Muse Wonder 1.0",
     "64.83",
     "5.77"
    ],
    [
     "10",
     "Qwen 3.8 Max",
     "64.40",
     "5.71"
    ],
    [
     "11",
     "Grok 4.7",
     "62.13",
     "5.79"
    ],
    [
     "12",
     "DeepSeek V4.1 Flash",
     "62.12",
     "5.85"
    ],
    [
     "13",
     "Inkling 2 Small",
     "61.89",
     "5.83"
    ],
    [
     "14",
     "Kimi K3.1 Pro",
     "61.11",
     "5.88"
    ],
    [
     "15",
     "Kimi K3",
     "60.98",
     "5.89"
    ],
    [
     "16",
     "Haiku 5.5",
     "59.43",
     "5.92"
    ],
    [
     "17",
     "GLM 5.3",
     "58.10",
     "5.90"
    ],
    [
     "18",
     "GPT-6.1 Terra",
     "57.69",
     "5.94"
    ],
    [
     "19",
     "Gemini 3.8 Flash",
     "57.33",
     "5.83"
    ],
    [
     "20",
     "Muse Ember 1.0",
     "55.99",
     "5.98"
    ],
    [
     "21",
     "GLM 5.3 Flash",
     "55.33",
     "5.94"
    ],
    [
     "22",
     "GPT-6 Luna",
     "50.80",
     "6.02"
    ],
    [
     "23",
     "Mistral Large 4",
     "45.19",
     "5.83"
    ],
    [
     "24",
     "Muse Spark Open",
     "42.28",
     "5.90"
    ],
    [
     "25",
     "MiniMax M3",
     "39.26",
     "5.75"
    ]
   ],
   "url": "https://benchmarks.site/internet/swe-atlas/",
   "csv": "https://benchmarks.site/data/internet/swe-atlas.csv"
  },
  {
   "benchmark": "osworld-osworld-verified",
   "title": "OSWorld (OSWorld-Verified)",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "OSWorld team (XLANG Lab, University of Hong Kong)",
    "url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx",
    "retrieved": "2026-11-17",
    "as_of": "2026-08-01",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model / agent",
    "Organization",
    "Approach type",
    "Max steps",
    "Success rate (%)",
    "Date"
   ],
   "rows": [
    [
     "1",
     "Intelligence-Indeed Agent",
     "Intelligence Indeed",
     "Agentic framework",
     "100",
     "90.2",
     "2026-07-25"
    ],
    [
     "2",
     "claude-fable-5[1m]",
     "Anthropic",
     "General model",
     "100",
     "86.0",
     "2026-08-01"
    ],
    [
     "3",
     "Pointer Agent w/ Opus 4.7",
     "Pointer",
     "Agentic framework",
     "100",
     "83.6",
     "2026-05-21"
    ],
    [
     "4",
     "claude-opus-5[1m]",
     "Anthropic",
     "General model",
     "100",
     "83.4",
     "2026-08-01"
    ],
    [
     "5",
     "Coasty CUA v1",
     "Coasty Team",
     "General model",
     "100",
     "82.8",
     "2026-07-01"
    ],
    [
     "6",
     "Pointer Agent w/ Sonnet 4.6",
     "Pointer",
     "Agentic framework",
     "100",
     "81.5",
     "2026-05-21"
    ],
    [
     "7",
     "Muse Spark 1.1",
     "Meta Superintelligence Labs",
     "General model",
     "100",
     "80.7",
     "2026-07-09"
    ],
    [
     "8",
     "Holo3-35B-A3B",
     "H Company",
     "Specialized model",
     "100",
     "80.4±2.2",
     "2026-04-20"
    ],
    [
     "9",
     "OpenAPA w/ gemini-3.1-pro",
     "Laiye",
     "Agentic framework",
     "100",
     "78.3",
     "2026-04-17"
    ],
    [
     "10",
     "VLAA-GUI w/ Opus 4.5",
     "UCSC VLAA Lab",
     "Agentic framework",
     "100",
     "76.3",
     "2026-04-20"
    ],
    [
     "11",
     "MiniMax M3",
     "MiniMax",
     "General model",
     "100",
     "75.2",
     "2026-06-08"
    ],
    [
     "12",
     "HIPPO Agent w/ Opus 4.5",
     "Lenovo",
     "Agentic framework",
     "100",
     "74.5",
     "2026-02-25"
    ],
    [
     "13",
     "Qwen 3.7 Plus",
     "Qwen Team, Alibaba Group",
     "General model",
     "100",
     "73.3",
     "2026-05-25"
    ],
    [
     "14",
     "Kimi K2.6",
     "Moonshot AI",
     "General model",
     "100",
     "73.1",
     "2026-04-20"
    ],
    [
     "15",
     "agent s3 w/ Opus 4.5 + GPT-5 bBoN (N=10)",
     "Simular",
     "Agentic framework",
     "100",
     "72.6",
     "2025-12-11"
    ],
    [
     "16",
     "claude-sonnet-4-6",
     "Anthropic",
     "General model",
     "100",
     "72.1",
     "2026-03-08"
    ],
    [
     "17",
     "agent s3 w/ GPT-5 bBoN (N=10)",
     "Simular",
     "Agentic framework",
     "100",
     "69.9",
     "2025-10-04"
    ],
    [
     "18",
     "UiPath Screen Agent w/ Opus 4.5",
     "UiPath",
     "Agentic framework",
     "100",
     "67.1",
     "2025-12-24"
    ],
    [
     "19",
     "agent s3 w/ Opus 4.5 bBoN (N=1)",
     "Simular",
     "Agentic framework",
     "100",
     "66.0±1.1",
     "2025-12-11"
    ],
    [
     "20",
     "OS-Symphony w/ GPT-5",
     "Shanghai AI Laboratory",
     "Agentic framework",
     "50",
     "65.8",
     "2026-01-04"
    ],
    [
     "21",
     "UiPath Screen Agent w/ Opus 4.5",
     "UiPath",
     "Agentic framework",
     "50",
     "64.4",
     "2025-12-24"
    ],
    [
     "22",
     "GBOX Agent",
     "GBOX.AI",
     "Agentic framework",
     "15",
     "64.2",
     "2025-11-25"
    ],
    [
     "23",
     "GTA1 w/ GPT-5",
     "Salesforce & The Australian National University & The University of Hong Kong",
     "Agentic framework",
     "100",
     "63.4",
     "2025-10-03"
    ],
    [
     "24",
     "Kimi K2.5",
     "Moonshot AI",
     "General model",
     "100",
     "63.3",
     "2026-01-31"
    ],
    [
     "25",
     "claude-sonnet-4-5-20250929",
     "Anthropic",
     "General model",
     "100",
     "62.9",
     "2025-10-31"
    ]
   ],
   "url": "https://benchmarks.site/internet/osworld-osworld-verified/",
   "csv": "https://benchmarks.site/data/internet/osworld-osworld-verified.csv"
  },
  {
   "benchmark": "osworld-2-0",
   "title": "OSWorld 2.0",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "OSWorld 2.0 team (XLANG Lab, University of Hong Kong)",
    "url": "https://osworld-v2.xlang.ai/static/data/leaderboard/official-results.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-14",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Reasoning effort",
    "Tool setting",
    "Task release",
    "Binary accuracy (%)",
    "Partial score (%)",
    "Official"
   ],
   "rows": [
    [
     "1",
     "Claude Opus 5",
     "Anthropic",
     "max",
     "batch tool",
     "v2.1",
     "44.33",
     "77.67",
     "yes"
    ],
    [
     "2",
     "MuseSpark 1.3",
     "MuseSpark",
     "max",
     "batch tool",
     "v2.1",
     "39.25",
     "69.56",
     "yes"
    ],
    [
     "3",
     "Claude Opus 5",
     "Anthropic",
     "high",
     "batch tool",
     "v2.1",
     "36.89",
     "68.92",
     "yes"
    ],
    [
     "4",
     "Claude Opus 5",
     "Anthropic",
     "xhigh",
     "batch tool",
     "v2.1",
     "33.33",
     "73.11",
     "yes"
    ],
    [
     "5",
     "Claude Opus 5",
     "Anthropic",
     "medium",
     "batch tool",
     "v2.1",
     "33.01",
     "64.85",
     "yes"
    ],
    [
     "6",
     "Claude Opus 5",
     "Anthropic",
     "max",
     "batch tool",
     "v2026.08.08",
     "31.43",
     "68.31",
     "yes"
    ],
    [
     "7",
     "Claude Opus 5",
     "Anthropic",
     "xhigh",
     "batch tool",
     "v2026.08.08",
     "30.21",
     "67.67",
     "yes"
    ],
    [
     "8",
     "Claude Opus 5",
     "Anthropic",
     "high",
     "batch tool",
     "v2026.08.08",
     "28.98",
     "63.79",
     "yes"
    ],
    [
     "9",
     "GPT-5.6 Sol",
     "OpenAI",
     "max",
     "batch tool",
     "v2026.08.08",
     "27.34",
     "62.72",
     "yes"
    ],
    [
     "10",
     "MiniMax M3.1 Flash",
     "MiniMax",
     "adaptive",
     "batch tool",
     "v2026.08.08",
     "26.85",
     "62.01",
     "no"
    ],
    [
     "11",
     "Claude Opus 5",
     "Anthropic",
     "medium",
     "batch tool",
     "v2026.08.08",
     "25.33",
     "60.49",
     "yes"
    ],
    [
     "12",
     "Claude Opus 5",
     "Anthropic",
     "low",
     "batch tool",
     "v2026.08.08",
     "22.32",
     "54.89",
     "yes"
    ],
    [
     "13",
     "Claude Opus 4.8",
     "Anthropic",
     "max",
     "batched tool",
     "v2026.06.24",
     "20.6",
     "54.8",
     "yes"
    ],
    [
     "14",
     "Claude Opus 5",
     "Anthropic",
     "low",
     "batch tool",
     "v2.1",
     "18.81",
     "59.09",
     "yes"
    ],
    [
     "15",
     "Claude Opus 4.8",
     "Anthropic",
     "max",
     "standard",
     "v2026.06.24",
     "18.52",
     "49.33",
     "yes"
    ],
    [
     "16",
     "Claude Opus 4.7",
     "Anthropic",
     "max",
     "batched tool",
     "v2026.06.24",
     "18.2",
     "48.91",
     "yes"
    ],
    [
     "17",
     "MuseSpark 1.2",
     "MuseSpark",
     "xhigh",
     "batch tool",
     "v2.1",
     "15.84",
     "44.08",
     "no"
    ],
    [
     "18",
     "Claude Opus 4.7",
     "Anthropic",
     "max",
     "standard",
     "v2026.06.24",
     "13.9",
     "49.1",
     "yes"
    ],
    [
     "19",
     "GPT-5.5",
     "OpenAI",
     "xhigh",
     "batch tool",
     "v2026.06.24",
     "13.0",
     "49.5",
     "yes"
    ],
    [
     "20",
     "Claude Sonnet 4.6",
     "Anthropic",
     "medium",
     "standard",
     "v2026.06.24",
     "9.3",
     "33.9",
     "yes"
    ],
    [
     "21",
     "Claude Sonnet 4.6",
     "Anthropic",
     "max",
     "standard",
     "v2026.06.24",
     "8.3",
     "41.5",
     "yes"
    ],
    [
     "22",
     "MiniMax M3",
     "MiniMax",
     "enabled",
     "standard",
     "v2026.06.24",
     "4.6",
     "22.3",
     "yes"
    ],
    [
     "23",
     "Kimi 2.6",
     "Moonshot AI",
     "enabled",
     "standard",
     "v2026.06.24",
     "4.6",
     "22.1",
     "yes"
    ],
    [
     "24",
     "Qwen 3.7-Plus",
     "Alibaba",
     "thinking",
     "standard",
     "v2026.06.24",
     "2.8",
     "21.5",
     "yes"
    ]
   ],
   "url": "https://benchmarks.site/internet/osworld-2-0/",
   "csv": "https://benchmarks.site/data/internet/osworld-2-0.csv"
  },
  {
   "benchmark": "mcpmark",
   "title": "MCPMark",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "MCPMark team",
    "url": "https://mcpmark.ai/leaderboard/verified",
    "retrieved": "2026-11-17",
    "as_of": "2026-07-20",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Pass@1 (%)"
   ],
   "rows": [
    [
     "1",
     "kimi-k3-max",
     "96.06"
    ],
    [
     "2",
     "gpt-5-5-xhigh",
     "92.91"
    ],
    [
     "3",
     "gpt-5-6-sol-max",
     "92.91"
    ],
    [
     "4",
     "claude-fable-5-max",
     "86.61"
    ],
    [
     "5",
     "kimi-k2-7-code",
     "81.89"
    ],
    [
     "6",
     "claude-opus-4-8-max",
     "76.38"
    ],
    [
     "7",
     "kimi-k2-6",
     "72.83"
    ],
    [
     "8",
     "deepseek-v4-pro-max",
     "71.65"
    ]
   ],
   "url": "https://benchmarks.site/internet/mcpmark/",
   "csv": "https://benchmarks.site/data/internet/mcpmark.csv"
  },
  {
   "benchmark": "toolathlon-tool-decathlon",
   "title": "Toolathlon (Tool Decathlon)",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "Toolathlon team",
    "url": "https://toolathlon.xyz/",
    "retrieved": "2026-11-17",
    "as_of": "2026-08-30",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Pass@1 (%)"
   ],
   "rows": [
    [
     "1",
     "GLM 5.3 Flash (max)",
     "78.4 ± 1.9"
    ],
    [
     "2",
     "Kimi K3 (max)",
     "76.5 ± 1.9"
    ],
    [
     "3",
     "Claude Opus 4.8 (max)",
     "76.2 ± 3.4"
    ],
    [
     "4",
     "Muse Spark 1.2 (xhigh)",
     "75.9 ± 1.3"
    ],
    [
     "5",
     "DeepSeek V4 Pro 0813 (max)",
     "74.4 ± 1.2"
    ],
    [
     "6",
     "GPT-5.5 (xhigh)",
     "73.5 ± 1.2"
    ],
    [
     "7",
     "Gemini 3.5 Flash (high)",
     "67.3 ± 1.2"
    ]
   ],
   "url": "https://benchmarks.site/internet/toolathlon-tool-decathlon/",
   "csv": "https://benchmarks.site/data/internet/toolathlon-tool-decathlon.csv"
  },
  {
   "benchmark": "vending-bench-2",
   "title": "Vending-Bench 2",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "Andon Labs",
    "url": "https://andonlabs.com/evals/vending-bench-2",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Final money balance"
   ],
   "rows": [
    [
     "1",
     "Gemini 4 Carbon",
     "$16,484.21 ± $1,096"
    ],
    [
     "2",
     "GPT-6 Astra",
     "$15,514.70 ± $1,074"
    ],
    [
     "3",
     "Gemini 4 Hydrogen",
     "$14,549.49 ± $495"
    ],
    [
     "4",
     "GPT-6 Sol",
     "$14,427.85 ± $1,051"
    ],
    [
     "5",
     "Gemini 4 Argon",
     "$13,718.16 ± $3,100"
    ],
    [
     "6",
     "GPT-6.1 Sol",
     "$12,860.61 ± $1,043"
    ],
    [
     "7",
     "Muse Wonder Full",
     "$12,291.56 ± $885"
    ],
    [
     "8",
     "Claude Opus 5",
     "$11,181.87 ± $2,094"
    ],
    [
     "9",
     "Claude Sonnet 5.5",
     "$11,165.28 ± $1,585"
    ],
    [
     "10",
     "Claude Opus 4.7",
     "$10,936.76 ± $1,181"
    ]
   ],
   "url": "https://benchmarks.site/internet/vending-bench-2/",
   "csv": "https://benchmarks.site/data/internet/vending-bench-2.csv"
  },
  {
   "benchmark": "metr-time-horizons",
   "title": "METR Time Horizons",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "METR",
    "url": "https://metr.org/assets/benchmark_results_1_1.yaml",
    "retrieved": "2026-11-17",
    "as_of": "2026-09-08",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "50% time horizon (min, CI)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "claude_mythos_preview_early_inspect",
     "1044.8 min (508.9-3304.3)",
     "2026-04-07"
    ],
    [
     "2",
     "claude_opus_4_6_inspect",
     "718.8 min (316.7-3633.8)",
     "2026-02-05"
    ],
    [
     "3",
     "gemini_3_1_pro",
     "384.1 min (233.5-694.8)",
     "2026-02-19"
    ],
    [
     "4",
     "gpt_5_2",
     "352.2 min (198.1-815.2)",
     "2025-12-11"
    ],
    [
     "5",
     "gpt_5_3_codex",
     "349.5 min (194.9-816.4)",
     "2026-02-05"
    ],
    [
     "6",
     "gpt_5_4",
     "341.7 min (186.6-768.8)",
     "2026-03-05"
    ],
    [
     "7",
     "claude_opus_4_5_inspect",
     "293.0 min (161.7-623.7)",
     "2025-11-24"
    ],
    [
     "8",
     "gemini_3_pro",
     "224.3 min (139.6-379.2)",
     "2025-11-18"
    ],
    [
     "9",
     "gpt_5_1_codex_max_inspect",
     "223.7 min (134.3-396.2)",
     "2025-11-19"
    ],
    [
     "10",
     "gpt_5_2025_08_07_inspect",
     "203.0 min (112.6-405.6)",
     "2025-08-07"
    ],
    [
     "11",
     "o3_inspect",
     "119.7 min (74.6-190.9)",
     "2025-04-16"
    ],
    [
     "12",
     "claude_4_1_opus_inspect",
     "100.5 min (59.3-159.5)",
     "2025-08-05"
    ],
    [
     "13",
     "claude_4_opus_inspect",
     "100.4 min (60.0-163.5)",
     "2025-05-22"
    ],
    [
     "14",
     "claude_3_7_sonnet_inspect",
     "60.4 min (33.0-104.2)",
     "2025-02-24"
    ],
    [
     "15",
     "o1_inspect",
     "38.8 min (21.2-65.0)",
     "2024-12-05"
    ],
    [
     "16",
     "claude_3_5_sonnet_20241022_inspect",
     "20.5 min (10.1-40.8)",
     "2024-10-22"
    ],
    [
     "17",
     "o1_preview",
     "20.3 min (11.7-33.4)",
     "2024-09-12"
    ],
    [
     "18",
     "claude_3_5_sonnet_20240620_inspect",
     "11.4 min (5.5-22.4)",
     "2024-06-20"
    ],
    [
     "19",
     "gpt_4o_inspect",
     "7.0 min (4.0-12.9)",
     "2024-05-13"
    ],
    [
     "20",
     "gpt_4_1106_inspect",
     "4.0 min (1.9-8.4)",
     "2023-11-06"
    ],
    [
     "21",
     "gpt_4",
     "4.0 min (1.9-8.0)",
     "2023-03-14"
    ],
    [
     "22",
     "claude_3_opus_inspect",
     "4.0 min (1.7-8.8)",
     "2024-03-04"
    ],
    [
     "23",
     "gpt_4_turbo_inspect",
     "3.7 min (2.0-6.7)",
     "2024-04-09"
    ],
    [
     "24",
     "gpt_3_5_turbo_instruct",
     "0.6 min (0.3-1.1)",
     "2022-03-15"
    ],
    [
     "25",
     "davinci_002",
     "0.1 min (0.1-0.2)",
     "2020-05-28"
    ]
   ],
   "url": "https://benchmarks.site/internet/metr-time-horizons/",
   "csv": "https://benchmarks.site/data/internet/metr-time-horizons.csv"
  },
  {
   "benchmark": "gdpval-aa",
   "title": "GDPval-AA",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "Artificial Analysis",
    "url": "https://artificialanalysis.ai/evaluations/gdpval-aa",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated (site has Terms of Use)"
   },
   "columns": [
    "Rank",
    "Model",
    "Elo",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "Claude Opus 5.5 (Max, Default Fallback)",
     "1866",
     "Sep 2026"
    ],
    [
     "2",
     "Kimi K3.1 Pro",
     "1861",
     "Oct 2026"
    ],
    [
     "3",
     "Muse Wonder Full (Max)",
     "1857",
     "Oct 2026"
    ],
    [
     "4",
     "Claude Sonnet 5.5 (Max, Default Fallback)",
     "1838",
     "Sep 2026"
    ],
    [
     "5",
     "Claude Opus 5.5 (Xhigh, Default Fallback)",
     "1836",
     "Sep 2026"
    ],
    [
     "6",
     "Claude Fable 5.2 (Max, Default Fallback)",
     "1831",
     "Nov 2026"
    ],
    [
     "7",
     "Qwen4 Max",
     "1809",
     "Nov 2026"
    ],
    [
     "8",
     "Muse Wonder (Max)",
     "1770",
     "Oct 2026"
    ],
    [
     "9",
     "Claude Fable 5.1 (Max, Default Fallback)",
     "1765",
     "Sep 2026"
    ],
    [
     "10",
     "Claude Fable 5.2 (Xhigh, Default Fallback)",
     "1751",
     "Nov 2026"
    ],
    [
     "11",
     "DeepSeek V4.1 Flash 1017 (Max)",
     "1746",
     "Oct 2026"
    ],
    [
     "12",
     "Claude Sonnet 5.5 (Xhigh, Default Fallback)",
     "1743",
     "Sep 2026"
    ],
    [
     "13",
     "Claude Fable 5.1 (Xhigh, Default Fallback)",
     "1741",
     "Sep 2026"
    ],
    [
     "14",
     "Gemini 4 Carbon (High)",
     "1733",
     "Oct 2026"
    ],
    [
     "15",
     "Claude Opus 5 (Max)",
     "1729",
     "Jul 2026"
    ],
    [
     "16",
     "Claude Fable 5.2 (High, Default Fallback)",
     "1718",
     "Nov 2026"
    ],
    [
     "17",
     "Grok 4.7 (High)",
     "1718",
     "Sep 2026"
    ],
    [
     "18",
     "Grok 4.7 (Xhigh)",
     "1717",
     "Sep 2026"
    ],
    [
     "19",
     "Claude Opus 5.5 (High, Default Fallback)",
     "1709",
     "Sep 2026"
    ],
    [
     "20",
     "Claude Opus 5 (Xhigh)",
     "1695",
     "Jul 2026"
    ],
    [
     "21",
     "MiMo-V2.6-Pro",
     "1692",
     "Sep 2026"
    ],
    [
     "22",
     "Muse Spark 1.3 (Max)",
     "1681",
     "Sep 2026"
    ],
    [
     "23",
     "Qwen3.8 Max (0902)",
     "1674",
     "Sep 2026"
    ],
    [
     "24",
     "Grok 4.6 (Xhigh)",
     "1658",
     "Aug 2026"
    ],
    [
     "25",
     "Ling 3.1 Flash 1019",
     "1655",
     "Oct 2026"
    ]
   ],
   "url": "https://benchmarks.site/internet/gdpval-aa/",
   "csv": "https://benchmarks.site/data/internet/gdpval-aa.csv"
  },
  {
   "benchmark": "apex-agents",
   "title": "APEX-Agents",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "Mercor",
    "url": "https://www.mercor.com/apex/",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Score"
   ],
   "rows": [
    [
     "1",
     "Gemini 4 Carbon (High)",
     "84.2% ±4.3%"
    ],
    [
     "2",
     "Gemini 4 Argon (High)",
     "82.4% ±4.3%"
    ],
    [
     "3",
     "Fable 5.2 (Max)",
     "80.1% ±4.7%"
    ],
    [
     "4",
     "Gemini 4 Hydrogen (xHigh)",
     "78.0% ±4.8%"
    ],
    [
     "5",
     "Hy4 (High)",
     "77.6% ±4.5%"
    ]
   ],
   "url": "https://benchmarks.site/internet/apex-agents/",
   "csv": "https://benchmarks.site/data/internet/apex-agents.csv"
  },
  {
   "benchmark": "remote-labor-index-rli",
   "title": "Remote Labor Index (RLI)",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "Scale AI / Center for AI Safety",
    "url": "https://labs.scale.com/leaderboard/rli",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Automation rate (%)"
   ],
   "rows": [
    [
     "1",
     "Fable 5.2 (NEW)",
     "41.25"
    ],
    [
     "2",
     "Opus 5.5",
     "21.25"
    ],
    [
     "3",
     "GPT 6 Astra",
     "20.83"
    ],
    [
     "4",
     "Fable 5.1",
     "17.92"
    ],
    [
     "5",
     "Gemini Argon 4",
     "17.08"
    ],
    [
     "6",
     "Fable 5",
     "15.83"
    ],
    [
     "7",
     "Deepseek V4.1 Flash (NEW)",
     "9.58"
    ],
    [
     "8",
     "Opus 4.8",
     "8.33"
    ],
    [
     "9",
     "Codex GPT 5.5",
     "6.25"
    ],
    [
     "10",
     "Gemini 3.8 Flash",
     "5.83"
    ],
    [
     "11",
     "Gemini 3.7 Flash",
     "5.00"
    ],
    [
     "12",
     "claude-opus-4-6 (CoWork)",
     "4.17"
    ],
    [
     "13",
     "claude-opus-4-5-20251101-thinking",
     "3.75"
    ],
    [
     "14",
     "Manus_1.6 (Max)",
     "2.92"
    ],
    [
     "15",
     "gpt-5.2-2025-12-11 (medium)",
     "2.50"
    ],
    [
     "15",
     "Manus 1.0",
     "2.50"
    ],
    [
     "15",
     "Manus 1.5",
     "2.50"
    ],
    [
     "18",
     "claude-4-5-Sonnet",
     "2.08"
    ],
    [
     "18",
     "gpt-5.2-2025-12-11 (default)",
     "2.08"
    ],
    [
     "20",
     "gpt-5-2025-08-07",
     "1.67"
    ],
    [
     "21",
     "ChatGPT agent",
     "1.25"
    ],
    [
     "21",
     "gemini-3-pro-preview",
     "1.25"
    ],
    [
     "23",
     "gemini-2.5-pro-preview-06-05",
     "0.83"
    ]
   ],
   "url": "https://benchmarks.site/internet/remote-labor-index-rli/",
   "csv": "https://benchmarks.site/data/internet/remote-labor-index-rli.csv"
  },
  {
   "benchmark": "screenspot-pro",
   "title": "ScreenSpot-Pro",
   "area": "Agents, tool use & computer use",
   "section": "Published results",
   "source": {
    "name": "ScreenSpot-Pro team (GUI-Agent grounding leaderboard)",
    "url": "https://gui-agent.github.io/grounding-leaderboard/results/screenspot_pro.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-10 (last repo commit)",
    "license": "Apache-2.0"
   },
   "columns": [
    "Rank",
    "Model / system",
    "Overall (%)",
    "Text targets (%)",
    "Icon targets (%)"
   ],
   "rows": [
    [
     "1",
     "GroundCascade (Indeed-UI-32B + KV-Ground-8B)",
     "85.14",
     "92.12",
     "73.84"
    ],
    [
     "2",
     "GroundCascade (Indeed-UI-32B + Indeed-UI-8B)",
     "85.01",
     "92.43",
     "73.01"
    ],
    [
     "3",
     "Indeed-UI-32B-zoomin",
     "82.73",
     "90.38",
     "70.36"
    ],
    [
     "4",
     "Indeed-UI-8B-zoom-in",
     "80.96",
     "89.87",
     "66.56"
    ],
    [
     "5",
     "KV-Ground-8B + Qwen3.5-27B Consistency Router",
     "80.90",
     "90.38",
     "65.56"
    ],
    [
     "6",
     "KV-Ground-GuiOwl1.5-0315-8B-ZoomIn",
     "80.52",
     "89.97",
     "65.23"
    ],
    [
     "7",
     "Holo2-235B-A22B (Agentic)",
     "78.49",
     "78.81",
     "77.98"
    ],
    [
     "8",
     "MAI-UI-32B (MVP)",
     "77.46",
     "88.25",
     "60.00"
    ],
    [
     "9",
     "AdaZoom-GUI-4B (cond. zoom-in, refine w/ Qwen3.5-397B)",
     "76.79",
     "90.07",
     "55.30"
    ],
    [
     "10",
     "KV-Ground-GuiOwl1.5-4B-0228-ZoomIn",
     "76.41",
     "85.57",
     "61.59"
    ],
    [
     "11",
     "Holo2-30B-A3B (Agentic)",
     "75.21",
     "76.46",
     "73.18"
    ],
    [
     "12",
     "MVP_Qwen3VL-32B",
     "74.15",
     "88.04",
     "51.66"
    ],
    [
     "13",
     "MAI-UI-32B (Zoom In)",
     "73.50",
     "85.36",
     "54.30"
    ],
    [
     "14",
     "Indeed-UI-8B",
     "73.37",
     "83.62",
     "56.79"
    ],
    [
     "15",
     "Indeed-UI-32B",
     "73.31",
     "85.26",
     "53.97"
    ],
    [
     "16",
     "KV-Ground-GuiOwl1.5-0315-8B",
     "73.18",
     "83.52",
     "56.46"
    ],
    [
     "17",
     "Duvo Eye-1 (Holo-3.1-35B-A3B + LoRA)",
     "72.93",
     "81.37",
     "59.27"
    ],
    [
     "18",
     "HuzzleWorld-3B (Zoom In)",
     "72.42",
     "79.22",
     "61.42"
    ],
    [
     "19",
     "MAI-UI-8B (Zoom In)",
     "71.92",
     "82.09",
     "55.46"
    ],
    [
     "20",
     "Holo2-8B (Agentic)",
     "71.41",
     "71.03",
     "72.02"
    ],
    [
     "21",
     "HuzzleWorld-2B (Zoom In)",
     "70.84",
     "80.35",
     "55.46"
    ],
    [
     "22",
     "AdaZoom-GUI-4B (conditional zoom-in)",
     "70.59",
     "85.36",
     "46.69"
    ],
    [
     "23",
     "Holo2-235B-A22B",
     "70.59",
     "70.11",
     "71.36"
    ],
    [
     "24",
     "KV-Ground-Qwen3VL-4B-ZoomIn",
     "70.27",
     "84.34",
     "47.52"
    ],
    [
     "25",
     "UI-Venus-1-5-30B-A3B",
     "69.64",
     "81.17",
     "50.99"
    ]
   ],
   "url": "https://benchmarks.site/internet/screenspot-pro/",
   "csv": "https://benchmarks.site/data/internet/screenspot-pro.csv"
  },
  {
   "benchmark": "context-arena-mrcr",
   "title": "Context Arena (MRCR)",
   "area": "Long context",
   "section": "Published results",
   "source": {
    "name": "Context Arena",
    "url": "https://contextarena.ai/ (data from https://contextarena.ai/api/leaderboard?needles=8)",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model [reasoning mode]",
    "Score at 64k-128k tokens (%)",
    "Overall score (%)"
   ],
   "rows": [
    [
     "1",
     "openai/gpt-5.6-sol [max]",
     "92.4",
     "85.5"
    ],
    [
     "2",
     "qwen/qwen3.8-max [xhigh]",
     "92.3",
     "73.7"
    ],
    [
     "3",
     "anthropic/claude-opus-5 [max]",
     "91.3",
     "77.3"
    ],
    [
     "4",
     "google/gemini-3.7-flash [high]",
     "88.6",
     "83.0"
    ],
    [
     "5",
     "qwen/qwen3.8-max [medium]",
     "87.3",
     "72.1"
    ],
    [
     "6",
     "openai/gpt-5.5 [xhigh]",
     "85.8",
     "79.8"
    ],
    [
     "7",
     "openai/gpt-5.5 [medium]",
     "83.0",
     "78.6"
    ],
    [
     "8",
     "google/gemini-3.7-flash [medium]",
     "82.1",
     "79.6"
    ],
    [
     "9",
     "qwen/qwen3.8-27b [low]",
     "81.8",
     "92.8"
    ],
    [
     "10",
     "qwen/qwen3.8-max [low]",
     "81.5",
     "71.6"
    ],
    [
     "11",
     "openai/gpt-5.6-terra [max]",
     "80.2",
     "76.6"
    ],
    [
     "12",
     "openai/gpt-5.5 [high]",
     "79.3",
     "79.0"
    ],
    [
     "13",
     "qwen/qwen3.8-27b [medium]",
     "78.9",
     "94.0"
    ],
    [
     "14",
     "google/gemini-3.7-flash [low]",
     "78.3",
     "74.5"
    ],
    [
     "15",
     "google/gemini-3.6-flash [high]",
     "77.3",
     "73.3"
    ],
    [
     "16",
     "anthropic/claude-opus-4.8 [max]",
     "75.2",
     "72.6"
    ],
    [
     "17",
     "google/gemini-3.6-flash [medium]",
     "74.5",
     "69.9"
    ],
    [
     "18",
     "qwen/qwen3.8-27b [xhigh]",
     "74.3",
     "90.2"
    ],
    [
     "19",
     "anthropic/claude-sonnet-4.6 [low]",
     "72.6",
     "70.4"
    ],
    [
     "20",
     "anthropic/claude-opus-4.6 [medium]",
     "72.5",
     "73.1"
    ],
    [
     "21",
     "anthropic/claude-sonnet-4.6 [high]",
     "72.4",
     "70.5"
    ],
    [
     "22",
     "anthropic/claude-sonnet-4.6 [medium]",
     "71.7",
     "69.6"
    ],
    [
     "23",
     "qwen/qwen3.6-max-preview [enabled]",
     "71.6",
     "88.1"
    ],
    [
     "24",
     "z-ai/glm-5.3 [high]",
     "69.3",
     "70.4"
    ],
    [
     "25",
     "anthropic/claude-opus-4.6 [low]",
     "68.8",
     "72.4"
    ]
   ],
   "url": "https://benchmarks.site/internet/context-arena-mrcr/",
   "csv": "https://benchmarks.site/data/internet/context-arena-mrcr.csv"
  },
  {
   "benchmark": "multichallenge",
   "title": "MultiChallenge",
   "area": "Instruction following",
   "section": "Published results",
   "source": {
    "name": "Scale AI (Scale Labs)",
    "url": "https://labs.scale.com/leaderboard/multichallenge",
    "retrieved": "2026-11-17",
    "as_of": "2026-11 (entries tagged NEW; judge update Feb 3, 2026)",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Score (%)",
    "Margin (±)"
   ],
   "rows": [
    [
     "1",
     "gpt 6.1 Sol",
     "81.95",
     "4.16"
    ],
    [
     "2",
     "Gemini 4 Argon",
     "78.70",
     "5.30"
    ],
    [
     "3",
     "Muse Spark",
     "75.52",
     "4.05"
    ],
    [
     "4",
     "Muse Spark 1.1",
     "75.30",
     "0.60"
    ],
    [
     "5",
     "Deepseek V4p1 Flash 1017",
     "74.90",
     "5.30"
    ],
    [
     "6",
     "GPT 6.1 Terra",
     "74.20",
     "5.40"
    ],
    [
     "6",
     "Muse Spark Open",
     "74.20",
     "5.60"
    ],
    [
     "8",
     "gemini-3.1-pro-preview",
     "71.37",
     "1.74"
    ],
    [
     "9",
     "gpt-5.4-pro-2026-03-05",
     "69.23",
     "3.05"
    ],
    [
     "10",
     "Claude Haiku 5.5",
     "67.67",
     "5.34"
    ],
    [
     "11",
     "gemini-3-pro-preview",
     "65.67",
     "2.20"
    ],
    [
     "12",
     "gpt-5.1-2025-11-13-thinking",
     "63.41",
     "2.11"
    ],
    [
     "13",
     "gpt-5-thinking",
     "63.19",
     "1.63"
    ],
    [
     "14",
     "o3-pro-2025-06-10-reasoning-high",
     "62.40",
     "2.49"
    ],
    [
     "15",
     "kimi-k2.5",
     "61.39",
     "0.46"
    ],
    [
     "16",
     "gemini-3.1-flash-lite-preview",
     "60.61",
     "0.38"
    ],
    [
     "17",
     "gpt-5-mini-thinking",
     "58.99",
     "4.07"
    ],
    [
     "18",
     "claude-opus-4-5-20251101-thinking",
     "58.97",
     "0.42"
    ],
    [
     "19",
     "claude-4-opus-thinking",
     "58.62",
     "0.77"
    ],
    [
     "20",
     "claude-opus-4-1-20250805-thinking",
     "57.20",
     "0.95"
    ],
    [
     "21",
     "claude-4-sonnet-thinking",
     "57.11",
     "0.81"
    ],
    [
     "22",
     "o3-2025-04-16-reasoning-high",
     "56.62",
     "3.32"
    ],
    [
     "23",
     "claude-opus-4-6 (Non-Thinking)",
     "56.02",
     "1.08"
    ],
    [
     "24",
     "kimi-k2-thinking",
     "55.42",
     "1.88"
    ],
    [
     "25",
     "claude-sonnet-4-5-20250929-thinking",
     "55.32",
     "1.47"
    ]
   ],
   "url": "https://benchmarks.site/internet/multichallenge/",
   "csv": "https://benchmarks.site/data/internet/multichallenge.csv"
  },
  {
   "benchmark": "vectara-hallucination-leaderboard",
   "title": "Vectara Hallucination Leaderboard",
   "area": "Hallucination & factuality",
   "section": "Published results",
   "source": {
    "name": "Vectara",
    "url": "https://github.com/vectara/hallucination-leaderboard/blob/main/README.md",
    "retrieved": "2026-11-17",
    "as_of": "2026-10-22",
    "license": "Apache-2.0 (repository license)"
   },
   "columns": [
    "Rank",
    "Model",
    "Hallucination Rate",
    "Factual Consistency Rate",
    "Answer Rate",
    "Average Summary Length (Words)"
   ],
   "rows": [
    [
     "1",
     "antgroup/finix_s1_32b",
     "1.8 %",
     "98.2 %",
     "99.5 %",
     "172.4"
    ],
    [
     "2",
     "openai/gpt-5.4-nano-2026-03-17",
     "3.1 %",
     "96.9 %",
     "100.0 %",
     "144.4"
    ],
    [
     "3",
     "google/gemini-2.5-flash-lite",
     "3.3 %",
     "96.7 %",
     "99.5 %",
     "95.7"
    ],
    [
     "4",
     "microsoft/Phi-4",
     "3.7 %",
     "96.3 %",
     "80.7 %",
     "120.9"
    ],
    [
     "5",
     "meta-llama/Llama-3.3-70B-Instruct-Turbo",
     "4.1 %",
     "95.9 %",
     "99.5 %",
     "64.6"
    ],
    [
     "6",
     "deepseek-ai/DeepSeek-V4.1-Flash",
     "4.2 %",
     "95.8 %",
     "99.9 %",
     "108.2"
    ],
    [
     "7",
     "snowflake/snowflake-arctic-instruct",
     "4.3 %",
     "95.7 %",
     "62.7 %",
     "81.4"
    ],
    [
     "8",
     "google/gemma-3-12b-it",
     "4.4 %",
     "95.6 %",
     "97.4 %",
     "89.7"
    ],
    [
     "9",
     "mistralai/mistral-large-2411",
     "4.5 %",
     "95.5 %",
     "99.9 %",
     "85.0"
    ],
    [
     "10",
     "qwen/qwen3-8b",
     "4.8 %",
     "95.2 %",
     "99.9 %",
     "83.6"
    ],
    [
     "11",
     "amazon/nova-pro-v1:0",
     "5.1 %",
     "94.9 %",
     "99.3 %",
     "66.2"
    ],
    [
     "12",
     "amazon/nova-2-lite-v1:0",
     "5.1 %",
     "94.9 %",
     "99.6 %",
     "94.1"
    ],
    [
     "13",
     "mistralai/mistral-small-2501",
     "5.1 %",
     "94.9 %",
     "97.9 %",
     "98.8"
    ],
    [
     "14",
     "ibm-granite/granite-4.0-h-small",
     "5.2 %",
     "94.8 %",
     "100.0 %",
     "107.4"
    ],
    [
     "15",
     "google/gemma-4-26b-a4b-it",
     "5.2 %",
     "94.8 %",
     "99.8 %",
     "67.1"
    ],
    [
     "16",
     "ai21labs/jamba-mini-2",
     "5.3 %",
     "94.7 %",
     "99.6 %",
     "109.4"
    ],
    [
     "17",
     "deepseek-ai/DeepSeek-V3.2-Exp",
     "5.3 %",
     "94.7 %",
     "96.6 %",
     "64.6"
    ],
    [
     "18",
     "qwen/qwen3-14b",
     "5.4 %",
     "94.6 %",
     "99.9 %",
     "111.1"
    ],
    [
     "19",
     "amazon/nova-micro-v1:0",
     "5.5 %",
     "94.5 %",
     "100.0 %",
     "100.0"
    ],
    [
     "20",
     "deepseek-ai/DeepSeek-V3.1",
     "5.5 %",
     "94.5 %",
     "94.5 %",
     "63.7"
    ],
    [
     "21",
     "openai/gpt-5.4-mini-2026-03-17",
     "5.5 %",
     "94.5 %",
     "100.0 %",
     "54.7"
    ],
    [
     "22",
     "openai/gpt-4.1-2025-04-14",
     "5.6 %",
     "94.4 %",
     "99.9 %",
     "91.7"
    ],
    [
     "23",
     "qwen/qwen3-4b",
     "5.7 %",
     "94.3 %",
     "99.9 %",
     "104.7"
    ],
    [
     "24",
     "xai-org/grok-3",
     "5.8 %",
     "94.2 %",
     "93.0 %",
     "95.9"
    ],
    [
     "25",
     "qwen/qwen3-32b",
     "5.9 %",
     "94.1 %",
     "99.9 %",
     "115.8"
    ]
   ],
   "url": "https://benchmarks.site/internet/vectara-hallucination-leaderboard/",
   "csv": "https://benchmarks.site/data/internet/vectara-hallucination-leaderboard.csv"
  },
  {
   "benchmark": "aa-omniscience",
   "title": "AA-Omniscience",
   "area": "Hallucination & factuality",
   "section": "Published results",
   "source": {
    "name": "Artificial Analysis",
    "url": "https://artificialanalysis.ai/evaluations/omniscience",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Omniscience Index",
    "Accuracy (%)"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2 (Max, Default Fallback)",
     "67",
     "79"
    ],
    [
     "2",
     "Claude Fable 5.2 (Xhigh, Default Fallback)",
     "65",
     "78"
    ],
    [
     "3",
     "Claude Fable 5.2 (High, Default Fallback)",
     "64",
     "77"
    ]
   ],
   "url": "https://benchmarks.site/internet/aa-omniscience/",
   "csv": "https://benchmarks.site/data/internet/aa-omniscience.csv"
  },
  {
   "benchmark": "arena-formerly-lmarena-chatbot-arena",
   "title": "Arena (formerly LMArena / Chatbot Arena)",
   "area": "Human-preference arenas",
   "section": "Published results",
   "source": {
    "name": "Arena Intelligence (arena.ai, formerly LMArena / LMSYS)",
    "url": "https://arena.ai/leaderboard/text",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-11",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Score",
    "±CI",
    "Votes",
    "License"
   ],
   "rows": [
    [
     "1",
     "Gemini 4 Carbon (High)",
     "Google",
     "1526",
     "±10",
     "4,273",
     "Proprietary"
    ],
    [
     "2",
     "Gemini 4 Argon (High)",
     "Google",
     "1522",
     "±5",
     "20,779",
     "Proprietary"
    ],
    [
     "3",
     "Muse Wonder (Max)",
     "Meta",
     "1511",
     "±6",
     "12,696",
     "Proprietary"
    ],
    [
     "4",
     "Claude Opus 5.5 (High)",
     "Anthropic",
     "1507",
     "±6",
     "15,793",
     "Proprietary"
    ],
    [
     "5",
     "Claude Fable 5 (High)",
     "Anthropic",
     "1504",
     "±4",
     "45,469",
     "Proprietary"
    ],
    [
     "6",
     "Claude Opus 4.6 (High)",
     "Anthropic",
     "1503",
     "±3",
     "84,375",
     "Proprietary"
    ],
    [
     "7",
     "Claude Fable 5.1 (Max)",
     "Anthropic",
     "1502",
     "±5",
     "20,511",
     "Proprietary"
    ],
    [
     "8",
     "Claude Opus 4.7 (High)",
     "Anthropic",
     "1501",
     "±4",
     "64,260",
     "Proprietary"
    ],
    [
     "9",
     "Gemini 3.8 Flash (High)",
     "Google",
     "1497",
     "±4",
     "62,675",
     "Proprietary"
    ],
    [
     "10",
     "Muse Spark 1.3 (Max)",
     "Meta",
     "1497",
     "±5",
     "26,806",
     "Proprietary"
    ],
    [
     "11",
     "Claude Opus 4.6",
     "Anthropic",
     "1496",
     "±3",
     "89,177",
     "Proprietary"
    ],
    [
     "12",
     "Claude Opus 4.7",
     "Anthropic",
     "1494",
     "±4",
     "65,309",
     "Proprietary"
    ],
    [
     "13",
     "Muse Spark 1.2 (xHigh)",
     "Meta",
     "1493",
     "±6",
     "15,563",
     "Proprietary"
    ],
    [
     "14",
     "Muse Spark 1.1",
     "Meta",
     "1491",
     "±4",
     "45,519",
     "Proprietary"
    ],
    [
     "15",
     "Gemini 4 Hydrogen (High)",
     "Google",
     "1490",
     "±9",
     "4,741",
     "Proprietary"
    ],
    [
     "16",
     "Claude Opus 5 (High)",
     "Anthropic",
     "1490",
     "±4",
     "73,632",
     "Proprietary"
    ],
    [
     "17",
     "Gemini 3.7 Flash (High)",
     "Google",
     "1489",
     "±5",
     "33,251",
     "Proprietary"
    ],
    [
     "18",
     "Muse Spark",
     "Meta",
     "1489",
     "±6",
     "13,985",
     "Proprietary"
    ],
    [
     "19",
     "Claude Opus 5 (Max)",
     "Anthropic",
     "1489",
     "±5",
     "35,670",
     "Proprietary"
    ],
    [
     "20",
     "Kimi K3 (Max)",
     "Moonshot",
     "1487",
     "±4",
     "41,402",
     "Kimi K3 license"
    ],
    [
     "21",
     "Gemini 3.1 Pro Preview",
     "Google",
     "1487",
     "±3",
     "132,181",
     "Proprietary"
    ],
    [
     "22",
     "Gemini 3 Pro",
     "Google",
     "1486",
     "±4",
     "41,803",
     "Proprietary"
    ],
    [
     "23",
     "GPT-5.6 Sol (xHigh)",
     "OpenAI",
     "1485",
     "±4",
     "43,925",
     "Proprietary"
    ],
    [
     "24",
     "Kimi K3.1 Pro (Max)",
     "Moonshot",
     "1485",
     "±10",
     "3,995",
     "Kimi K3.1 Pro License"
    ],
    [
     "25",
     "Qwen3.8-Max",
     "Alibaba",
     "1484",
     "±5",
     "36,936",
     "Proprietary"
    ]
   ],
   "url": "https://benchmarks.site/internet/arena-formerly-lmarena-chatbot-arena/",
   "csv": "https://benchmarks.site/data/internet/arena-formerly-lmarena-chatbot-arena.csv"
  },
  {
   "benchmark": "design-arena",
   "title": "Design Arena",
   "area": "Human-preference arenas",
   "section": "Published results",
   "source": {
    "name": "DesignArena",
    "url": "https://www.designarena.ai/leaderboard",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Elo",
    "Win Rate",
    "Battles"
   ],
   "rows": [
    [
     "1",
     "Claude Opus 5.5 (high)",
     "Anthropic",
     "1445",
     "69.1%",
     "2,670"
    ],
    [
     "2",
     "GPT-6 Astra (max)",
     "OpenAI",
     "1399",
     "67.2%",
     "2,526"
    ],
    [
     "3",
     "Claude Fable 5.2",
     "Anthropic",
     "1367",
     "69.0%",
     "1,738"
    ],
    [
     "4",
     "GPT-6.1 Sol (max)",
     "OpenAI",
     "1366",
     "51.3%",
     "2,069"
    ],
    [
     "5",
     "Qwen4-Max",
     "Alibaba",
     "1360",
     "62.5%",
     "3,474"
    ],
    [
     "6",
     "Claude Fable 5.1",
     "Anthropic",
     "1336",
     "59.5%",
     "5,155"
    ],
    [
     "7",
     "Inkling 2 Small",
     "Thinking Machines",
     "1311",
     "57.2%",
     "7,217"
    ],
    [
     "8",
     "Gemini 4 Carbon",
     "Google",
     "1306",
     "60.5%",
     "1,054"
    ],
    [
     "9",
     "Kimi K3.1 Pro",
     "Moonshot AI",
     "1297",
     "59.4%",
     "4,489"
    ],
    [
     "10",
     "Claude Sonnet 5.5",
     "Anthropic",
     "1290",
     "57.5%",
     "3,030"
    ],
    [
     "11",
     "DeepSeek-V4.1-Flash (1017)",
     "DeepSeek",
     "1288",
     "47.8%",
     "404"
    ],
    [
     "12",
     "Muse Spark 1.3 Max",
     "Meta",
     "1279",
     "56.9%",
     "13,091"
    ],
    [
     "13",
     "Grok 4.7",
     "xAI",
     "1273",
     "54.4%",
     "1,506"
    ],
    [
     "14",
     "Claude Fable 5",
     "Anthropic",
     "1272",
     "60.5%",
     "13,563"
    ],
    [
     "15",
     "Gemini 4 Hydrogen",
     "Google",
     "1268",
     "56.6%",
     "380"
    ]
   ],
   "url": "https://benchmarks.site/internet/design-arena/",
   "csv": "https://benchmarks.site/data/internet/design-arena.csv"
  },
  {
   "benchmark": "mmmu-mmmu-pro",
   "title": "MMMU / MMMU-Pro",
   "area": "Multimodal & vision",
   "section": "Published results",
   "source": {
    "name": "MMMU team (mmmu-benchmark.github.io)",
    "url": "https://mmmu-benchmark.github.io/leaderboard_data.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-10",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Type",
    "Date",
    "MMMU-Pro Overall",
    "MMMU Val Overall"
   ],
   "rows": [
    [
     "1",
     "Chance Vision 1.5",
     "proprietary",
     "2026-07-01",
     "86.9",
     "-"
    ],
    [
     "2",
     "GPT-5.4 Thinking w/ tools",
     "proprietary",
     "2026-03-05",
     "82.1",
     "-"
    ],
    [
     "3",
     "GPT-5.4 Thinking w/o tools",
     "proprietary",
     "2026-03-05",
     "81.2",
     "-"
    ],
    [
     "4",
     "Gemini 3.0 Pro",
     "proprietary",
     "2025-11-18",
     "81.0",
     "-"
    ],
    [
     "5",
     "Gemini 3.1 Pro Thinking (High)",
     "proprietary",
     "2026-02-19",
     "80.5",
     "-"
    ],
    [
     "6",
     "Muse Spark Thinking",
     "proprietary",
     "2026-04-08",
     "80.4",
     "-"
    ],
    [
     "7",
     "GPT-5.2 Thinking w/o Python",
     "proprietary",
     "2025-12-11",
     "80.4",
     "-"
    ],
    [
     "8",
     "GPT-5.2 Thinking w/o tools",
     "proprietary",
     "2025-12-11",
     "79.5",
     "-"
    ],
    [
     "9",
     "GPT-5.1 Thinking",
     "proprietary",
     "2025-11-13",
     "79.0",
     "-"
    ],
    [
     "10",
     "GPT-5 w/ thinking",
     "proprietary",
     "2025-08-07",
     "78.4",
     "84.2"
    ],
    [
     "11",
     "Claude Opus 4.6 w/ tools",
     "proprietary",
     "2026-02-05",
     "77.3",
     "-"
    ],
    [
     "12",
     "Gemma 4 31B",
     "open source",
     "2026-04-02",
     "76.9",
     "-"
    ],
    [
     "13",
     "o3",
     "proprietary",
     "2025-04-16",
     "76.4",
     "82.9"
    ],
    [
     "14",
     "GPT-5.1",
     "proprietary",
     "2025-11-13",
     "76.0",
     "85.4"
    ],
    [
     "15",
     "Claude Sonnet 4.6 w/ tools",
     "proprietary",
     "2026-02-17",
     "75.6",
     "-"
    ],
    [
     "16",
     "Claude Sonnet 4.6 w/o tools",
     "proprietary",
     "2026-02-17",
     "74.5",
     "-"
    ],
    [
     "17",
     "Claude Opus 4.6 w/o tools",
     "proprietary",
     "2026-02-05",
     "73.9",
     "-"
    ],
    [
     "18",
     "Claude Opus 4.5",
     "proprietary",
     "2025-11-24",
     "73.9",
     "80.7"
    ],
    [
     "19",
     "Gemma 4 26B A4B",
     "open source",
     "2026-04-02",
     "73.8",
     "-"
    ],
    [
     "20",
     "dots.vlm1",
     "open source",
     "2025-08-05",
     "70.1",
     "80.1"
    ],
    [
     "21",
     "Claude Sonnet 4.5",
     "proprietary",
     "2025-09-29",
     "68.9",
     "77.8"
    ],
    [
     "22",
     "Qwen3-VL 235B-A22B",
     "open source",
     "2025-09-22",
     "68.1",
     "78.7"
    ],
    [
     "23",
     "Gemini 2.5 Pro 05-06",
     "proprietary",
     "2025-05-06",
     "68.0",
     "79.6"
    ],
    [
     "24",
     "Seed 1.5-VL Thinking",
     "proprietary",
     "2025-05-13",
     "67.6",
     "77.9"
    ],
    [
     "25",
     "Seed 1.6-Thinking",
     "proprietary",
     "2025-06-25",
     "66.4",
     "74.8"
    ]
   ],
   "url": "https://benchmarks.site/internet/mmmu-mmmu-pro/",
   "csv": "https://benchmarks.site/data/internet/mmmu-mmmu-pro.csv"
  },
  {
   "benchmark": "omnidocbench",
   "title": "OmniDocBench",
   "area": "OCR & document understanding",
   "section": "Published results",
   "source": {
    "name": "OpenDataLab / Shanghai AI Lab (OmniDocBench repo)",
    "url": "https://github.com/opendatalab/OmniDocBench",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-12",
    "license": "not stated (dataset research-only, non-commercial per repo copyright statement)"
   },
   "columns": [
    "Rank",
    "Model",
    "Method type",
    "Size",
    "Overall",
    "Text edit distance (lower better)",
    "Formula CDM",
    "Table TEDS"
   ],
   "rows": [
    [
     "1",
     "OXR-1.0",
     "Specialized VLMs",
     "1.3B",
     "97.03",
     "0.0244",
     "96.93",
     "96.61"
    ],
    [
     "2",
     "TeleOCR",
     "Specialized VLMs",
     "1.2B",
     "96.91",
     "0.0267",
     "96.5895",
     "96.8183"
    ],
    [
     "3",
     "OvisOCR2",
     "Specialized VLMs",
     "0.8B",
     "96.47",
     "0.0265",
     "97.4854",
     "94.5842"
    ],
    [
     "4",
     "MiniCPM-OCR 1.0",
     "Specialized VLMs",
     "1.3B",
     "96.44",
     "0.0316",
     "98.0814",
     "94.3999"
    ],
    [
     "5",
     "PaddleOCR-VL-1.6",
     "Specialized VLMs",
     "0.9B",
     "96.34",
     "0.0326",
     "97.5304",
     "94.7619"
    ],
    [
     "6",
     "MinerU2.5-Pro",
     "Specialized VLMs",
     "1.2B",
     "95.75",
     "0.036",
     "97.45",
     "93.42"
    ],
    [
     "7",
     "GLM-OCR",
     "Specialized VLMs",
     "0.9B",
     "95.22",
     "0.044",
     "97.18",
     "92.83"
    ],
    [
     "8",
     "PaddleOCR-VL-1.5",
     "Specialized VLMs",
     "0.9B",
     "94.93",
     "0.038",
     "96.89",
     "91.67"
    ],
    [
     "9",
     "PaddleOCR-VL",
     "Specialized VLMs",
     "0.9B",
     "94.18",
     "0.040",
     "95.91",
     "90.65"
    ],
    [
     "10",
     "Unlimited-OCR",
     "Specialized VLMs",
     "3B",
     "94.00",
     "0.0394",
     "95.7234",
     "90.21"
    ],
    [
     "11",
     "Youtu-Parsing",
     "Specialized VLMs",
     "2.5B",
     "93.74",
     "0.044",
     "93.63",
     "92.02"
    ],
    [
     "12",
     "Qianfan-OCR",
     "Specialized VLMs",
     "4B",
     "93.90",
     "0.04",
     "95.08",
     "90.53"
    ],
    [
     "13",
     "Ovis2.6-30B-A3B",
     "General VLMs",
     "30B",
     "93.70",
     "0.035",
     "95.17",
     "89.44"
    ],
    [
     "14",
     "Logics-Parsing-v2",
     "Specialized VLMs",
     "4B",
     "93.33",
     "0.041",
     "95.65",
     "88.42"
    ],
    [
     "15",
     "ABot-OCR",
     "Specialized VLMs",
     "2B",
     "93.30",
     "0.037",
     "94.86",
     "88.69"
    ],
    [
     "16",
     "FireRed-OCR",
     "Specialized VLMs",
     "2B",
     "93.26",
     "0.037",
     "95.44",
     "88.04"
    ],
    [
     "17",
     "MinerU-2.5",
     "Specialized VLMs",
     "1.2B",
     "93.04",
     "0.045",
     "95.77",
     "87.88"
    ],
    [
     "18",
     "Gemini 3 Pro",
     "General VLMs",
     "-",
     "92.91",
     "0.064",
     "95.99",
     "89.15"
    ],
    [
     "19",
     "Gemini 3 Flash",
     "General VLMs",
     "-",
     "92.62",
     "0.066",
     "95.16",
     "89.29"
    ],
    [
     "20",
     "MonkeyOCR V2",
     "Specialized VLMs",
     "-",
     "91.80",
     "0.0636",
     "93.4985",
     "88.2628"
    ],
    [
     "21",
     "dots.ocr",
     "Specialized VLMs",
     "3B",
     "90.77",
     "0.048",
     "89.95",
     "87.18"
    ],
    [
     "22",
     "OpenDoc-0.1B",
     "Specialized VLMs",
     "0.1B",
     "90.67",
     "0.049",
     "93.02",
     "83.88"
    ],
    [
     "23",
     "DeepSeek-OCR 2",
     "Specialized VLMs",
     "3B",
     "90.25",
     "0.050",
     "91.84",
     "83.89"
    ],
    [
     "24",
     "HunyuanOCR",
     "Specialized VLMs",
     "1B",
     "89.95",
     "0.088",
     "87.68",
     "91.01"
    ],
    [
     "25",
     "Qwen3-VL-235B",
     "General VLMs",
     "235B",
     "89.78",
     "0.063",
     "92.55",
     "83.07"
    ]
   ],
   "url": "https://benchmarks.site/internet/omnidocbench/",
   "csv": "https://benchmarks.site/data/internet/omnidocbench.csv"
  },
  {
   "benchmark": "open-asr-leaderboard",
   "title": "Open ASR Leaderboard",
   "area": "Speech recognition (ASR)",
   "section": "Published results",
   "source": {
    "name": "Hugging Face Open ASR Leaderboard (hf-audio)",
    "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-13",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Average WER (%)",
    "RTFx (speed)"
   ],
   "rows": [
    [
     "1",
     "zoom/scribe_v2.1_pro",
     "4.87",
     ""
    ],
    [
     "2",
     "OtterAI/speech-v3.1",
     "5.21",
     ""
    ],
    [
     "3",
     "yoob/yoob-asr-pro-v1",
     "5.33",
     ""
    ],
    [
     "4",
     "microsoft/azure-speech-09-2026",
     "5.5",
     ""
    ],
    [
     "5",
     "Musci-research/Musci-ASR-qwen-1019-hf",
     "5.7",
     "774.69"
    ],
    [
     "6",
     "p99lab/audrey-0.7",
     "5.7",
     ""
    ],
    [
     "7",
     "elevenlabs/scribe_v2",
     "5.93",
     ""
    ],
    [
     "8",
     "reson8/rumble-2",
     "5.95",
     ""
    ],
    [
     "9",
     "modulate/multilingual",
     "5.96",
     ""
    ],
    [
     "10",
     "salesforce-sovereign-intelligence/SOV-Transcribe-1",
     "5.96",
     "600.92"
    ],
    [
     "11",
     "FluidVoice-Labs/fluid-speech-pro",
     "6.06",
     "3690.47"
    ],
    [
     "12",
     "sophea/asr-k1",
     "6.11",
     ""
    ],
    [
     "13",
     "nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B",
     "6.13",
     "472.62"
    ],
    [
     "14",
     "Qwen/Qwen3-ASR-1.7B-hf",
     "6.15",
     "835.62"
    ],
    [
     "15",
     "assemblyai/universal-3-5-pro",
     "6.21",
     ""
    ],
    [
     "16",
     "smallestai/pulse",
     "6.57",
     ""
    ],
    [
     "17",
     "ibm-granite/granite-speech-5.0-2b",
     "6.68",
     "548.43"
    ],
    [
     "18",
     "CohereLabs/cohere-transcribe-03-2026",
     "6.72",
     "923.71"
    ],
    [
     "19",
     "HojoAI/Hojo-ASR-V1",
     "6.74",
     "73.39"
    ],
    [
     "20",
     "TheStageAI/thewhisper-large-v3-turbo",
     "6.74",
     "1183.92"
    ],
    [
     "21",
     "Edge0/ARK-ASR-0.6B",
     "6.75",
     "664.56"
    ],
    [
     "22",
     "openai/gpt-transcribe",
     "6.8",
     ""
    ],
    [
     "23",
     "sprag/symphony",
     "6.85",
     ""
    ],
    [
     "24",
     "gladia/solaria-3",
     "6.86",
     ""
    ],
    [
     "25",
     "nvidia/parakeet-tdt-0.6b-v2 (fast-gpu-asr)",
     "6.91",
     "17254.19"
    ]
   ],
   "url": "https://benchmarks.site/internet/open-asr-leaderboard/",
   "csv": "https://benchmarks.site/data/internet/open-asr-leaderboard.csv"
  },
  {
   "benchmark": "tts-arena-v2",
   "title": "TTS Arena V2",
   "area": "Text-to-speech",
   "section": "Published results",
   "source": {
    "name": "TTS-AGI community (Hugging Face)",
    "url": "https://tts-agi-tts-arena-v2.hf.space/api/leaderboard",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-11",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Elo",
    "Win rate (%)",
    "Votes"
   ],
   "rows": [
    [
     "1",
     "Aurora",
     "1687",
     "73.4",
     "312"
    ],
    [
     "2",
     "star-june-2026",
     "1633",
     "64.7",
     "295"
    ],
    [
     "3",
     "Inworld TTS MAX",
     "1567",
     "63.8",
     "1193"
    ],
    [
     "4",
     "CastleFlow v1.0",
     "1558",
     "61.9",
     "1490"
    ],
    [
     "5",
     "Inworld TTS 1.5 MAX",
     "1565",
     "59.7",
     "390"
    ],
    [
     "6",
     "Hume Octave",
     "1551",
     "59.7",
     "949"
    ],
    [
     "7",
     "Inworld TTS",
     "1543",
     "60.0",
     "1712"
    ],
    [
     "8",
     "Papla P1",
     "1552",
     "61.2",
     "836"
    ],
    [
     "9",
     "Gradium TTS 2026.10",
     "1588",
     "62.7",
     "102"
    ],
    [
     "10",
     "Gradium TTS (pre-2026.08)",
     "1554",
     "53.7",
     "259"
    ],
    [
     "11",
     "MiniMax Speech 2.8 HD",
     "1545",
     "54.2",
     "253"
    ],
    [
     "12",
     "OpenAudio S2",
     "1539",
     "52.2",
     "322"
    ],
    [
     "13",
     "Lightning v3.1 Pro",
     "1544",
     "53.6",
     "295"
    ],
    [
     "14",
     "Deepdub eTTS 3.2",
     "1532",
     "52.1",
     "217"
    ],
    [
     "15",
     "Eleven Multilingual v2",
     "1507",
     "54.0",
     "968"
    ],
    [
     "16",
     "Eleven Turbo v2.5",
     "1505",
     "53.5",
     "951"
    ],
    [
     "17",
     "MiniMax Speech 2.6 Turbo",
     "1525",
     "50",
     "270"
    ],
    [
     "18",
     "Eleven Flash v2.5",
     "1503",
     "53.2",
     "964"
    ],
    [
     "19",
     "MiniMax Speech 02 HD",
     "1503",
     "53.7",
     "1028"
    ],
    [
     "20",
     "Eleven v3",
     "1521",
     "49.3",
     "288"
    ],
    [
     "21",
     "MiniMax Speech 2.6 HD",
     "1504",
     "47.5",
     "284"
    ],
    [
     "22",
     "MiniMax Speech 2.8 Turbo",
     "1502",
     "48.2",
     "282"
    ],
    [
     "23",
     "MiniMax Speech 02 Turbo",
     "1483",
     "50.9",
     "1037"
    ],
    [
     "24",
     "Parmesan",
     "1481",
     "52.4",
     "984"
    ],
    [
     "25",
     "Kokoro v1.0",
     "1470",
     "48.8",
     "903"
    ]
   ],
   "url": "https://benchmarks.site/internet/tts-arena-v2/",
   "csv": "https://benchmarks.site/data/internet/tts-arena-v2.csv"
  },
  {
   "benchmark": "f5-labs-casi-ars-leaderboard",
   "title": "F5 Labs CASI & ARS Leaderboard",
   "area": "Safety & security",
   "section": "Published results",
   "source": {
    "name": "F5 Labs",
    "url": "https://www.f5.com/labs/casi",
    "retrieved": "2026-11-17",
    "as_of": "2026-11",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Provider",
    "Model",
    "CASI",
    "Avg. Performance"
   ],
   "rows": [
    [
     "1",
     "Anthropic",
     "Claude Sonnet 5",
     "98.00",
     "23.20%"
    ],
    [
     "2",
     "Anthropic",
     "Claude Haiku 5.5",
     "94.61",
     "43.40%"
    ],
    [
     "3",
     "xAI",
     "Grok 4.7 High",
     "93.31",
     "46.40%"
    ],
    [
     "4",
     "OpenAI",
     "GPT-5.4",
     "93.21",
     "18.20%"
    ],
    [
     "5",
     "Anthropic",
     "Claude Opus 5",
     "93.05",
     "50.80%"
    ],
    [
     "6",
     "Qwen",
     "Qwen 3.8-Max",
     "92.31",
     "40.20%"
    ],
    [
     "7",
     "Anthropic",
     "Claude Fable 5.1 Max Effort",
     "92.30",
     "53.40%"
    ],
    [
     "8",
     "Anthropic",
     "Claude Opus 5.5 Reasoning",
     "92.00",
     "57.60%"
    ],
    [
     "9",
     "Anthropic",
     "Claude Sonnet 5.5",
     "91.78",
     "56.00%"
    ],
    [
     "10",
     "OpenAI",
     "GPT-6 Luna",
     "89.15",
     "38.10%"
    ]
   ],
   "url": "https://benchmarks.site/internet/f5-labs-casi-ars-leaderboard/",
   "csv": "https://benchmarks.site/data/internet/f5-labs-casi-ars-leaderboard.csv"
  },
  {
   "benchmark": "scale-fortress",
   "title": "Scale Fortress",
   "area": "Safety & security",
   "section": "Published results",
   "source": {
    "name": "Scale AI (SEAL)",
    "url": "https://labs.scale.com/leaderboard/fortress",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "ARS ± margin"
   ],
   "rows": [
    [
     "1",
     "claude-opus-5-5-max",
     "5.78±1.26"
    ],
    [
     "2",
     "muse-spark-open",
     "7.64±1.15"
    ],
    [
     "3",
     "gpt-oss-120b",
     "8.24±1.93"
    ],
    [
     "4",
     "Claude Haiku 5.5",
     "9.48±1.34"
    ],
    [
     "5",
     "claude-opus-4-5-20251101-thinking",
     "9.63±2.11"
    ],
    [
     "6",
     "gpt-6-luna",
     "10.78±1.22"
    ],
    [
     "7",
     "beam-1p0-501b-a23b",
     "11.87±1.48"
    ],
    [
     "8",
     "muse-ember-1.0",
     "11.92±1.58"
    ],
    [
     "9",
     "gpt-6-astra",
     "12.14±1.39"
    ],
    [
     "10",
     "Muse Spark 1.1",
     "12.40±1.48"
    ],
    [
     "11",
     "claude-sonnet-4-5-20250929-thinking",
     "12.80±2.36"
    ],
    [
     "12",
     "Claude 3.5 Sonnet",
     "12.96±2.34"
    ],
    [
     "13",
     "claude-opus-4-6-thinking-max",
     "13.00±2.41"
    ],
    [
     "14",
     "claude-opus-5-5",
     "13.18±1.77"
    ],
    [
     "15",
     "gpt-6-sol",
     "13.46±1.40"
    ],
    [
     "16",
     "claude-opus-4-5-20251101",
     "13.56±2.49"
    ],
    [
     "17",
     "Fable-5.1",
     "13.72±1.85"
    ],
    [
     "18",
     "claude-opus-4-1-20250805-thinking",
     "14.79±2.49"
    ],
    [
     "19",
     "gpt-5.4-pro-2026-03-05",
     "14.84±2.29"
    ],
    [
     "20",
     "gpt-5-pro-2025-10-06",
     "15.20±2.30"
    ],
    [
     "21",
     "claude-fable-5-2",
     "15.60±1.92"
    ],
    [
     "22",
     "o3",
     "16.01±2.68"
    ],
    [
     "23",
     "claude-opus-4-1-20250805",
     "16.15±2.58"
    ],
    [
     "24",
     "Inkling 2 Small",
     "16.23±2.28"
    ],
    [
     "25",
     "gpt-5.5-xhigh",
     "16.28±1.64"
    ]
   ],
   "url": "https://benchmarks.site/internet/scale-fortress/",
   "csv": "https://benchmarks.site/data/internet/scale-fortress.csv"
  },
  {
   "benchmark": "cybergym",
   "title": "CyberGym",
   "area": "Safety & security",
   "section": "Published results",
   "source": {
    "name": "CyberGym team (UC Berkeley)",
    "url": "https://www.cybergym.io/assets/data/cybergym.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-10-24",
    "license": "not stated (benchmark code Apache-2.0)"
   },
   "columns": [
    "Rank",
    "Model",
    "Agent",
    "Success Rate",
    "Date",
    "Source"
   ],
   "rows": [
    [
     "1",
     "Creation Model",
     "Claude Code",
     "90.6%",
     "2026-10-21",
     "Creation Lab (Creation Lab, Ant Group)"
    ],
    [
     "2",
     "XekRung-1.5-27B-Preview (finetuned from Qwen3.8-27B)",
     "XekRung Agent",
     "88.9%",
     "2026-09-13",
     "Alibaba Security"
    ],
    [
     "3",
     "Gemini 3.8 Flash Cyber",
     "Google Antigravity",
     "86.3%",
     "2026-09-10",
     "Google"
    ],
    [
     "4",
     "GPT-5.5-Cyber",
     "OpenAI Agent",
     "85.6%",
     "2026-06-22",
     "OpenAI"
    ],
    [
     "5",
     "GLM-5.3",
     "Claude Code",
     "84.5%",
     "2026-08-14",
     "Zhipu AI"
    ],
    [
     "6",
     "Feyospace-v1.1 (finetuned from Qwen3.8-27B)",
     "Claude Code",
     "84.4%",
     "2026-09-23",
     "Vera Praxis Lab"
    ],
    [
     "7",
     "DeepSeek-V4-Pro",
     "DeepSeek Agent",
     "83.3%",
     "2026-08-13",
     "DeepSeek"
    ],
    [
     "8",
     "Claude Mythos Preview",
     "Anthropic Agent",
     "83.1%",
     "2026-04-07",
     "Anthropic"
    ],
    [
     "9",
     "GPT-5.5",
     "OpenAI Agent",
     "81.8%",
     "2026-04-23",
     "OpenAI"
    ],
    [
     "10",
     "Basin (basin1-0912-pre-dpo)",
     "Codex CLI",
     "80.0%",
     "2026-09-23",
     "Submersion AI"
    ],
    [
     "11",
     "Grok 4.6",
     "Grok Build",
     "79.7%",
     "2026-08-12",
     "xAI"
    ],
    [
     "12",
     "Grok 4.5",
     "Grok Build",
     "79.0%",
     "2026-07-14",
     "xAI"
    ],
    [
     "13",
     "GPT-5.4",
     "OpenAI Agent",
     "79.0%",
     "2026-04-23",
     "OpenAI"
    ],
    [
     "14",
     "GLM-5.2",
     "Claude Code",
     "77.2%",
     "2026-07-21",
     "Zhipu AI"
    ],
    [
     "15",
     "DeepSeek-V4-Flash",
     "DeepSeek Agent",
     "76.7%",
     "2026-07-31",
     "DeepSeek"
    ],
    [
     "16",
     "GLM-5.1",
     "Claude Code",
     "68.7%",
     "2026-04-07",
     "Zhipu AI"
    ],
    [
     "17",
     "Claude Opus 4.6",
     "Anthropic Agent",
     "66.6%",
     "2026-02-05",
     "Anthropic"
    ],
    [
     "18",
     "GPT-5.4",
     "Codex CLI",
     "66.3%",
     "2026-04-07",
     "Zhipu AI"
    ],
    [
     "19",
     "Claude Sonnet 4.6",
     "Anthropic Agent",
     "65.2%",
     "2026-02-17",
     "Anthropic"
    ],
    [
     "20",
     "Muse Spark 1.1 (helpful-only)",
     "Meta Agent",
     "59.0%",
     "2026-07-09",
     "Meta AI"
    ],
    [
     "21",
     "DeepSeek-V4-Pro Preview",
     "Claude Code",
     "57.7%",
     "2026-07-13",
     "XDxAI"
    ],
    [
     "22",
     "Claude Opus 4.5",
     "Anthropic Agent",
     "50.6%",
     "2026-02-05",
     "Anthropic"
    ],
    [
     "23",
     "Muse Spark",
     "Meta Agent",
     "43.5%",
     "2026-04-14",
     "Meta AI"
    ],
    [
     "24",
     "GLM-5",
     "Claude Code",
     "43.2%",
     "2026-02-12",
     "Zhipu AI"
    ],
    [
     "25",
     "Kimi K2.5",
     "Kimi Agent",
     "41.3%",
     "2026-02-02",
     "Kimi"
    ]
   ],
   "url": "https://benchmarks.site/internet/cybergym/",
   "csv": "https://benchmarks.site/data/internet/cybergym.csv"
  },
  {
   "benchmark": "euroeval",
   "title": "EuroEval",
   "area": "Multilingual",
   "section": "Published results",
   "source": {
    "name": "EuroEval team",
    "url": "https://euroeval.com/leaderboards/Multilingual/european/",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-15",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Mean rank score",
    "Open weights"
   ],
   "rows": [
    [
     "1",
     "gpt-6-astra (zero-shot, val)",
     "1.17 ± 0.05",
     "No"
    ],
    [
     "1",
     "gpt-6.1-sol (zero-shot, val)",
     "1.18 ± 0.04",
     "No"
    ],
    [
     "2",
     "gemini/gemini-3.7-flash (zero-shot, val)",
     "1.24 ± 0.03",
     "No"
    ],
    [
     "3",
     "gemini/gemini-3-flash-preview#thinking (zero-shot, val)",
     "1.32 ± 0.04",
     "No"
    ],
    [
     "3",
     "gemini/gemini-3.6-flash (zero-shot, val)",
     "1.31 ± 0.04",
     "No"
    ],
    [
     "3",
     "zai-org/GLM-5.3-Flash (val)",
     "1.36 ± 0.04",
     "Yes"
    ],
    [
     "4",
     "gpt-5.6-sol (zero-shot, val)",
     "1.40 ± 0.04",
     "No"
    ],
    [
     "5",
     "gpt-6-sol (zero-shot, val)",
     "1.45 ± 0.05",
     "No"
    ],
    [
     "6",
     "zai-org/GLM-5.3-Flash (zero-shot, val)",
     "1.54 ± 0.08",
     "Yes"
    ],
    [
     "6",
     "deepseek/deepseek-flash#low (zero-shot, val)",
     "1.50 ± 0.05",
     "No"
    ],
    [
     "7",
     "Qwen/Qwen3.6-27B-FP8 (val)",
     "1.60 ± 0.07",
     "Yes"
    ],
    [
     "7",
     "gemini/gemini-3.1-flash-lite-preview (zero-shot, val)",
     "1.59 ± 0.08",
     "No"
    ],
    [
     "7",
     "gpt-5.6-terra (zero-shot, val)",
     "1.58 ± 0.07",
     "No"
    ],
    [
     "7",
     "openai/gpt-5.4-mini-2026-03-17#high (zero-shot, val)",
     "1.61 ± 0.05",
     "No"
    ],
    [
     "7",
     "claude-sonnet-4-5-20250929#thinking (zero-shot, val)",
     "1.56 ± 0.08",
     "No"
    ],
    [
     "7",
     "deepseek/deepseek-flash#no-thinking (val)",
     "1.57 ± 0.06",
     "No"
    ],
    [
     "8",
     "Qwen/Qwen3.8-27B (val)",
     "1.67 ± 0.09",
     "Yes"
    ],
    [
     "8",
     "Qwen/Qwen3.5-35B-A3B-FP8 (val)",
     "1.66 ± 0.09",
     "Yes"
    ],
    [
     "8",
     "google/gemma-4-31B-it (val)",
     "1.67 ± 0.10",
     "Yes"
    ],
    [
     "8",
     "claude-sonnet-4-6 (zero-shot, val)",
     "1.69 ± 0.08",
     "No"
    ],
    [
     "8",
     "xai/grok-4-1-fast-reasoning (zero-shot, val)",
     "1.69 ± 0.07",
     "No"
    ],
    [
     "8",
     "openai/gpt-5.4-mini-2026-03-17#medium (zero-shot, val)",
     "1.64 ± 0.06",
     "No"
    ],
    [
     "9",
     "gpt-5-mini-2025-08-07 (zero-shot, val)",
     "1.71 ± 0.07",
     "No"
    ],
    [
     "9",
     "openai/gpt-5.4-mini-2026-03-17#low (zero-shot, val)",
     "1.70 ± 0.06",
     "No"
    ],
    [
     "9",
     "openai/gpt-5.4-2026-03-05#none (zero-shot, val)",
     "1.76 ± 0.09",
     "No"
    ]
   ],
   "url": "https://benchmarks.site/internet/euroeval/",
   "csv": "https://benchmarks.site/data/internet/euroeval.csv"
  },
  {
   "benchmark": "cptu-bench",
   "title": "CPTU-Bench",
   "area": "Polish language",
   "section": "Published results",
   "source": {
    "name": "SpeakLeash",
    "url": "https://huggingface.co/spaces/speakleash/cptu_bench/resolve/main/data.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-08-11",
    "license": "MIT"
   },
   "columns": [
    "Rank",
    "Model",
    "Params",
    "Average",
    "Sentiment",
    "Language understanding",
    "Phraseology",
    "Tricky questions"
   ],
   "rows": [
    [
     "1",
     "meta-models/Muse-Glimmer-30B reasoning (API, FP8)",
     "30B",
     "4.42",
     "4.52",
     "4.39",
     "4.23",
     "4.52"
    ],
    [
     "2",
     "Qwen/Qwen3.6-27B reasoning (API, FP8)",
     "27B",
     "4.41",
     "4.58",
     "4.21",
     "4.21",
     "4.64"
    ],
    [
     "3",
     "Qwen/Qwen3.5-122B-A10B reasoning (API, FP8)",
     "122B",
     "4.36",
     "4.36",
     "4.18",
     "4.19",
     "4.72"
    ],
    [
     "4",
     "Qwen/Qwen3.5-27B thinking (API)",
     "27B",
     "4.34",
     "4.42",
     "4.21",
     "4.11",
     "4.61"
    ],
    [
     "5",
     "google/gemma-4-31B-it non-reasoning (API, FP8)",
     "31B",
     "4.32",
     "4.58",
     "4.46",
     "4.07",
     "4.19"
    ],
    [
     "6",
     "google/gemma-4-31B-it reasoning (API, FP8)",
     "31B",
     "4.31",
     "4.58",
     "4.38",
     "3.77",
     "4.52"
    ],
    [
     "7",
     "Qwen/Qwen3.6-27B non-reasoning (API, FP8)",
     "27B",
     "4.29",
     "4.33",
     "4.08",
     "4.40",
     "4.36"
    ],
    [
     "8",
     "gemini-2.0-flash-001",
     "",
     "4.29",
     "4.52",
     "4.32",
     "4.34",
     "3.99"
    ],
    [
     "9",
     "Qwen/Qwen3.5-27B non-thinking (API)",
     "27B",
     "4.27",
     "4.29",
     "4.17",
     "4.20",
     "4.43"
    ],
    [
     "10",
     "Qwen/Qwen3.5-122B-A10B non-reasoning (API, FP8)",
     "122B",
     "4.26",
     "4.33",
     "4.24",
     "3.94",
     "4.53"
    ],
    [
     "11",
     "google/gemma-4-26B-A4B-it non-reasoning (API, FP8)",
     "26B",
     "4.23",
     "4.39",
     "4.00",
     "4.21",
     "4.30"
    ],
    [
     "12",
     "Qwen/Qwen3.5-35B-A3B thinking (API)",
     "35B",
     "4.22",
     "4.10",
     "3.94",
     "4.15",
     "4.70"
    ],
    [
     "13",
     "google/gemma-4-26B-A4B-it reasoning (API, FP8)",
     "26B",
     "4.21",
     "4.39",
     "4.08",
     "3.91",
     "4.47"
    ],
    [
     "14",
     "Qwen/Qwen3.5-35B-A3B non-thinking (API)",
     "35B",
     "4.18",
     "4.23",
     "4.05",
     "4.23",
     "4.19"
    ],
    [
     "15",
     "deepseek-ai/DeepSeek-V3.2 (API)",
     "685B",
     "4.14",
     "4.46",
     "4.36",
     "3.54",
     "4.20"
    ],
    [
     "16",
     "deepseek-ai/DeepSeek-R1 (API)",
     "685B",
     "4.14",
     "4.49",
     "4.34",
     "3.60",
     "4.12"
    ],
    [
     "17",
     "gemini-2.0-flash-lite-001",
     "",
     "4.09",
     "4.23",
     "4.05",
     "4.24",
     "3.85"
    ],
    [
     "18",
     "deepseek-ai/DeepSeek-V3-0324 (API)",
     "685B",
     "4.03",
     "4.36",
     "4.20",
     "3.54",
     "4.02"
    ],
    [
     "19",
     "deepseek-ai/DeepSeek-V3.1 (API)",
     "685B",
     "4.03",
     "4.42",
     "4.33",
     "3.48",
     "3.87"
    ],
    [
     "20",
     "deepseek-ai/DeepSeek-V3 (API)",
     "685B",
     "4.02",
     "4.36",
     "4.22",
     "3.52",
     "3.99"
    ],
    [
     "21",
     "mistralai/Mistral-Large-Instruct-2411",
     "123B",
     "4.00",
     "4.33",
     "3.98",
     "3.99",
     "3.72"
    ],
    [
     "22",
     "moonshotai/Kimi-K2-Instruct-0905 (API)",
     "1000B",
     "3.98",
     "4.39",
     "4.18",
     "3.43",
     "3.93"
    ],
    [
     "23",
     "Qwen/Qwen2.5-72B-Instruct",
     "72.7B",
     "3.95",
     "4.08",
     "3.97",
     "3.93",
     "3.81"
    ],
    [
     "24",
     "mistralai/Mistral-Large-Instruct-2407",
     "123B",
     "3.93",
     "4.23",
     "4.00",
     "3.86",
     "3.65"
    ],
    [
     "25",
     "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 (API)",
     "402B",
     "3.93",
     "4.39",
     "4.11",
     "3.48",
     "3.76"
    ]
   ],
   "url": "https://benchmarks.site/internet/cptu-bench/",
   "csv": "https://benchmarks.site/data/internet/cptu-bench.csv"
  },
  {
   "benchmark": "polish-linguistic-and-cultural-competency-plcc",
   "title": "Polish Linguistic and Cultural Competency (PLCC)",
   "area": "Polish language",
   "section": "Published results",
   "source": {
    "name": "Sławomir Dadas et al. (OPI PIB)",
    "url": "https://huggingface.co/spaces/sdadas/plcc/resolve/main/data.json",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-16",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Provider",
    "Overall",
    "Art & entertainment",
    "Culture & tradition",
    "Geography",
    "Grammar",
    "History",
    "Vocabulary"
   ],
   "rows": [
    [
     "1",
     "Gemini-4-Carbon",
     "Google",
     "98.33",
     "97",
     "100",
     "100",
     "97",
     "98",
     "98"
    ],
    [
     "2",
     "Gemini-4-Argon",
     "Google",
     "97.33",
     "95",
     "98",
     "99",
     "96",
     "98",
     "98"
    ],
    [
     "3",
     "Claude-Fable-5.2",
     "Anthropic",
     "97.33",
     "96",
     "98",
     "98",
     "97",
     "99",
     "96"
    ],
    [
     "4",
     "Gemini-3.1-Pro-Preview",
     "Google",
     "97",
     "95",
     "100",
     "100",
     "93",
     "98",
     "96"
    ],
    [
     "5",
     "Gemini-3.8-Flash",
     "Google",
     "97",
     "95",
     "99",
     "100",
     "95",
     "98",
     "95"
    ],
    [
     "6",
     "GPT-6.1-Sol",
     "OpenAI",
     "96.83",
     "93",
     "98",
     "98",
     "98",
     "99",
     "95"
    ],
    [
     "7",
     "Gemini-4-Hydrogen",
     "Google",
     "96.83",
     "93",
     "97",
     "100",
     "97",
     "97",
     "97"
    ],
    [
     "8",
     "Gemini-3.7-Flash",
     "Google",
     "96.67",
     "95",
     "98",
     "100",
     "95",
     "97",
     "95"
    ],
    [
     "9",
     "Gemini-3.5-Flash",
     "Google",
     "96.33",
     "94",
     "100",
     "100",
     "97",
     "96",
     "91"
    ],
    [
     "10",
     "GPT-6-Astra",
     "OpenAI",
     "96.33",
     "94",
     "98",
     "98",
     "95",
     "99",
     "94"
    ],
    [
     "11",
     "Gemini-3.0-Pro-Preview",
     "Google",
     "95.83",
     "95",
     "99",
     "100",
     "91",
     "95",
     "95"
    ],
    [
     "12",
     "GPT-5.5 (high reasoning)",
     "OpenAI",
     "95.83",
     "93",
     "98",
     "98",
     "97",
     "95",
     "94"
    ],
    [
     "13",
     "Gemini-3.6-Flash",
     "Google",
     "95.5",
     "92",
     "98",
     "99",
     "94",
     "96",
     "94"
    ],
    [
     "14",
     "Claude-Fable-5.1",
     "Anthropic",
     "95.17",
     "91",
     "95",
     "99",
     "95",
     "97",
     "94"
    ],
    [
     "15",
     "Claude-Fable-5",
     "Anthropic",
     "95",
     "93",
     "96",
     "98",
     "94",
     "97",
     "92"
    ],
    [
     "16",
     "GPT-5.6-Sol (medium reasoning)",
     "OpenAI",
     "94.67",
     "93",
     "96",
     "99",
     "91",
     "96",
     "93"
    ],
    [
     "17",
     "GPT-5.5 (low reasoning)",
     "OpenAI",
     "94.5",
     "93",
     "96",
     "99",
     "91",
     "95",
     "93"
    ],
    [
     "18",
     "GPT-6.1-Terra",
     "OpenAI",
     "94",
     "90",
     "94",
     "97",
     "93",
     "98",
     "92"
    ],
    [
     "19",
     "GPT-6-Sol (medium reasoning)",
     "OpenAI",
     "93.83",
     "94",
     "93",
     "98",
     "93",
     "97",
     "88"
    ],
    [
     "20",
     "Muse-Wonder",
     "Meta",
     "93.83",
     "92",
     "92",
     "98",
     "92",
     "97",
     "92"
    ],
    [
     "21",
     "Claude-Opus-5.5",
     "Anthropic",
     "92.5",
     "86",
     "93",
     "94",
     "94",
     "97",
     "91"
    ],
    [
     "22",
     "Gemini-2.5-Pro-Preview-06-05",
     "Google",
     "92.17",
     "91",
     "96",
     "98",
     "86",
     "92",
     "90"
    ],
    [
     "23",
     "GPT-5.4-2026-03-05 (high reasoning)",
     "OpenAI",
     "92.17",
     "91",
     "93",
     "96",
     "90",
     "92",
     "91"
    ],
    [
     "24",
     "Claude-Opus-5",
     "Anthropic",
     "92.17",
     "86",
     "94",
     "97",
     "92",
     "96",
     "88"
    ],
    [
     "25",
     "Muse-Spark-1.3",
     "Meta",
     "91.83",
     "89",
     "94",
     "97",
     "89",
     "94",
     "88"
    ]
   ],
   "url": "https://benchmarks.site/internet/polish-linguistic-and-cultural-competency-plcc/",
   "csv": "https://benchmarks.site/data/internet/polish-linguistic-and-cultural-competency-plcc.csv"
  },
  {
   "benchmark": "pl-mteb-polish-mteb",
   "title": "PL-MTEB (Polish MTEB)",
   "area": "Polish language",
   "section": "Published results",
   "source": {
    "name": "MTEB team (Hugging Face MTEB leaderboard)",
    "url": "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB(pol, v1)/scores",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "MIT (MTEB project)"
   },
   "columns": [
    "Rank",
    "Model",
    "Mean (Task)",
    "Mean (Task type)",
    "Params (B)"
   ],
   "rows": [
    [
     "1",
     "codefuse-ai/F2LLM-v2-14B",
     "75.13",
     "71.96",
     "13.99"
    ],
    [
     "2",
     "codefuse-ai/F2LLM-v2-8B",
     "74.61",
     "71.48",
     "7.568"
    ],
    [
     "3",
     "Qwen/Qwen3-Embedding-8B",
     "73.54",
     "71.66",
     "7.567"
    ],
    [
     "5",
     "Qwen/Qwen3-Embedding-4B",
     "72.34",
     "70.53",
     "4.022"
    ],
    [
     "6",
     "codefuse-ai/F2LLM-v2-4B",
     "73.42",
     "70.28",
     "4.022"
    ],
    [
     "7",
     "Linq-AI-Research/Linq-Embed-Mistral",
     "71.10",
     "69.28",
     "7.111"
    ],
    [
     "8",
     "microsoft/harrier-oss-v1-0.6b",
     "70.47",
     "68.65",
     "0.596"
    ],
    [
     "9",
     "codefuse-ai/F2LLM-v2-1.7B",
     "72.03",
     "68.96",
     "1.721"
    ],
    [
     "10",
     "Salesforce/SFR-Embedding-Mistral",
     "69.31",
     "68.29",
     "7.111"
    ],
    [
     "11",
     "intfloat/multilingual-e5-large-instruct",
     "69.13",
     "68.35",
     "0.56"
    ],
    [
     "12",
     "intfloat/e5-mistral-7b-instruct",
     "69.94",
     "68.66",
     "7.111"
    ],
    [
     "13",
     "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "70.00",
     "67.98",
     "1.543"
    ],
    [
     "14",
     "Salesforce/SFR-Embedding-2_R",
     "68.88",
     "67.18",
     "7.111"
    ],
    [
     "16",
     "codefuse-ai/F2LLM-v2-0.6B",
     "69.63",
     "66.72",
     "0.596"
    ],
    [
     "17",
     "manu/bge-m3-custom-fr",
     "64.74",
     "63.88",
     "0.568"
    ],
    [
     "17",
     "microsoft/harrier-oss-v1-270m",
     "66.93",
     "66.04",
     "0.268"
    ],
    [
     "21",
     "OrdalieTech/Solon-embeddings-large-0.1",
     "63.49",
     "63.01",
     "0.56"
    ],
    [
     "22",
     "codefuse-ai/F2LLM-v2-330M",
     "66.53",
     "64.39",
     "0.334"
    ],
    [
     "23",
     "Qwen/Qwen3-Embedding-0.6B",
     "65.50",
     "64.59",
     "0.596"
    ],
    [
     "24",
     "omarelshehy/arabic-english-sts-matryoshka",
     "65.07",
     "64.30",
     "0.56"
    ],
    [
     "26",
     "sdadas/mmlw-e5-small",
     "63.41",
     "63.35",
     "0.118"
    ],
    [
     "28",
     "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1",
     "64.57",
     "63.70",
     "0.494"
    ],
    [
     "30",
     "hotchpotch/bekko-embedding-v1-a25m",
     "61.50",
     "61.66",
     "0.123"
    ],
    [
     "32",
     "Snowflake/snowflake-arctic-embed-l-v2.0",
     "61.31",
     "60.96",
     "0.568"
    ],
    [
     "35",
     "deepvk/USER-bge-m3",
     "62.46",
     "61.27",
     "0.359"
    ]
   ],
   "url": "https://benchmarks.site/internet/pl-mteb-polish-mteb/",
   "csv": "https://benchmarks.site/data/internet/pl-mteb-polish-mteb.csv"
  },
  {
   "benchmark": "openrouter-rankings",
   "title": "OpenRouter Rankings",
   "area": "Speed, price & providers",
   "section": "Published results",
   "source": {
    "name": "OpenRouter",
    "url": "https://openrouter.ai/rankings",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-16",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Tokens (B)",
    "Requests (M)"
   ],
   "rows": [
    [
     "1",
     "deepseek/deepseek-v4.1-flash-20261017",
     "61170.1",
     "978.5"
    ],
    [
     "2",
     "tencent/hy4-20261104",
     "16042.2",
     "115.5"
    ],
    [
     "3",
     "deepseek/deepseek-v4.1-flash-20260910",
     "15455.0",
     "495.9"
    ],
    [
     "4",
     "z-ai/glm-5.3-flash-20260826",
     "11806.2",
     "361.0"
    ],
    [
     "5",
     "openai/gpt-6-luna-20260922",
     "8081.3",
     "614.8"
    ],
    [
     "6",
     "anthropic/claude-opus-5.5-20260921",
     "7365.4",
     "63.3"
    ],
    [
     "7",
     "stepfun/step-5-preview-20261008",
     "6360.0",
     "24.8"
    ],
    [
     "8",
     "xiaomi/mimo-v2.6-flash-20260921",
     "4164.6",
     "59.1"
    ],
    [
     "9",
     "z-ai/glm-5.3-20260816",
     "3738.6",
     "81.6"
    ],
    [
     "10",
     "openai/gpt-6.1-sol-20260929",
     "3711.0",
     "78.7"
    ],
    [
     "11",
     "minimax/minimax-m3.1-flash-20261019",
     "3350.9",
     "46.9"
    ],
    [
     "12",
     "moonshotai/kimi-k3.1-pro-20261023",
     "3241.9",
     "39.8"
    ],
    [
     "13",
     "nvidia/nemotron-3-ultra-550b-a55b-20260604 (free)",
     "3198.3",
     "37.8"
    ],
    [
     "14",
     "deepseek/deepseek-v4-flash-20260731",
     "3196.4",
     "305.6"
    ],
    [
     "15",
     "nvidia/nemotron-3.5-super-vl-20261015 (free)",
     "2969.1",
     "27.0"
    ],
    [
     "16",
     "meta/muse-spark-1.3-contributor-20260902",
     "2964.9",
     "34.3"
    ],
    [
     "17",
     "typesafe/jev-1.13-20260917",
     "2868.8",
     "956.9"
    ],
    [
     "18",
     "anthropic/claude-haiku-5.5-20261007",
     "2779.1",
     "229.7"
    ],
    [
     "19",
     "anthropic/claude-sonnet-5.5-20260928",
     "2666.3",
     "63.9"
    ],
    [
     "20",
     "baidu/ernie-5.5-flash-preview-20261110 (free)",
     "2470.8",
     "19.1"
    ]
   ],
   "url": "https://benchmarks.site/internet/openrouter-rankings/",
   "csv": "https://benchmarks.site/data/internet/openrouter-rankings.csv"
  },
  {
   "benchmark": "epoch-ai-benchmarking-hub",
   "title": "Epoch AI Benchmarking Hub",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "Epoch AI",
    "url": "https://epoch.ai/data/benchmark_data.zip",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "Score (%)",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "claude-fable-5-2_max",
     "Anthropic",
     "95.1",
     "2026-11-09"
    ],
    [
     "2",
     "gpt-6.1-sol_max",
     "OpenAI",
     "93.7",
     "2026-09-29"
    ],
    [
     "3",
     "gpt-6-astra_max",
     "OpenAI",
     "93.7",
     "2026-09-03"
    ],
    [
     "4",
     "claude-opus-5-5_max",
     "Anthropic",
     "91.2",
     "2026-09-22"
    ],
    [
     "5",
     "gpt-6.1-terra_max",
     "OpenAI",
     "90.9",
     "2026-10-26"
    ],
    [
     "6",
     "gemini-4-carbon_high",
     "Google DeepMind",
     "90.5",
     "2026-11-10"
    ],
    [
     "7",
     "claude-fable-5-1_max",
     "Anthropic",
     "90.2",
     "2026-09-01"
    ],
    [
     "8",
     "gpt-6-sol_max",
     "OpenAI",
     "89.8",
     "2026-09-22"
    ],
    [
     "9",
     "gpt-5.6-sol_max",
     "OpenAI",
     "89.1",
     "2026-07-09"
    ],
    [
     "10",
     "claude-sonnet-5-5_max",
     "Anthropic",
     "88.8",
     "2026-09-28"
    ],
    [
     "11",
     "gpt-5.5-pro_xhigh",
     "OpenAI",
     "87.7",
     "2026-04-23"
    ],
    [
     "12",
     "claude-fable-5_max",
     "Anthropic",
     "87.0",
     "2026-06-09"
    ],
    [
     "13",
     "gemini-4-argon_high",
     "Google DeepMind",
     "86.0",
     "2026-10-13"
    ],
    [
     "14",
     "gpt-5.6-terra_max",
     "OpenAI",
     "86.0",
     "2026-07-09"
    ],
    [
     "15",
     "claude-opus-5_max",
     "Anthropic",
     "85.6",
     "2026-07-24"
    ],
    [
     "16",
     "gpt-5.5_xhigh",
     "OpenAI",
     "85.3",
     "2026-04-23"
    ],
    [
     "17",
     "gpt-5.4-pro-2026-03-05_xhigh",
     "OpenAI",
     "82.5",
     "2026-03-05"
    ],
    [
     "18",
     "muse-wonder-1.0_max",
     "Meta AI",
     "82.1",
     "2026-10-12"
    ],
    [
     "19",
     "gpt-5.6-luna_max",
     "OpenAI",
     "82.1",
     "2026-07-09"
    ],
    [
     "20",
     "claude-opus-4-8_max",
     "Anthropic",
     "80.0",
     "2026-05-28"
    ],
    [
     "21",
     "gpt-6-luna_max",
     "OpenAI",
     "78.9",
     "2026-09-22"
    ],
    [
     "22",
     "gpt-5.4-2026-03-05_xhigh",
     "OpenAI",
     "78.6",
     "2026-03-05"
    ],
    [
     "23",
     "kimi-k3.1-pro-together_max",
     "Moonshot",
     "78.2",
     "2026-10-23"
    ],
    [
     "24",
     "claude-haiku-5-5_max",
     "Anthropic",
     "75.1",
     "2026-10-07"
    ],
    [
     "25",
     "qwen3.8-max_xhigh",
     "Alibaba",
     "74.7",
     "2026-08-02"
    ]
   ],
   "url": "https://benchmarks.site/internet/epoch-ai-benchmarking-hub/",
   "csv": "https://benchmarks.site/data/internet/epoch-ai-benchmarking-hub.csv"
  },
  {
   "benchmark": "epoch-capabilities-index-eci",
   "title": "Epoch Capabilities Index (ECI)",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "Epoch AI",
    "url": "https://epoch.ai/data/eci_scores.csv",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-17",
    "license": "CC-BY-4.0 (Epoch AI data; check epoch.ai/data)"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "ECI",
    "Access",
    "Release date"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2",
     "Anthropic",
     "173.44",
     "API access",
     "2026-11-09"
    ],
    [
     "2",
     "Gemini 4 Carbon",
     "Google DeepMind",
     "169.76",
     "Unreleased",
     "2026-10-28"
    ],
    [
     "3",
     "GPT-6 Astra",
     "OpenAI",
     "169.0",
     "API access",
     "2026-09-03"
    ],
    [
     "4",
     "Claude Opus 5.5",
     "Anthropic",
     "168.54",
     "API access",
     "2026-09-22"
    ],
    [
     "5",
     "GPT-6.1 Sol",
     "OpenAI",
     "168.48",
     "API access",
     "2026-09-29"
    ],
    [
     "6",
     "GPT-6.1 Terra",
     "OpenAI",
     "167.67",
     "API access",
     "2026-10-26"
    ],
    [
     "7",
     "Claude Fable 5.1",
     "Anthropic",
     "166.99",
     "API access",
     "2026-09-01"
    ],
    [
     "8",
     "Claude Sonnet 5.5",
     "Anthropic",
     "166.56",
     "API access",
     "2026-09-28"
    ],
    [
     "9",
     "Gemini 4 Argon",
     "Google DeepMind",
     "166.48",
     "Limited access",
     "2026-09-30"
    ],
    [
     "10",
     "Claude Opus 5",
     "Anthropic",
     "164.68",
     "API access",
     "2026-07-24"
    ],
    [
     "11",
     "Claude Fable 5",
     "Anthropic",
     "164.13",
     "API access",
     "2026-06-09"
    ],
    [
     "12",
     "GPT-6 Sol",
     "OpenAI",
     "163.75",
     "API access",
     "2026-09-22"
    ],
    [
     "13",
     "GPT-5.6 Sol",
     "OpenAI",
     "162.99",
     "API access",
     "2026-07-09"
    ],
    [
     "14",
     "GPT-5.5 Pro",
     "OpenAI",
     "162.7",
     "API access",
     "2026-04-23"
    ],
    [
     "15",
     "Muse Wonder",
     "Meta AI",
     "161.68",
     "API access",
     "2026-10-12"
    ],
    [
     "16",
     "Kimi K3.1 Pro",
     "Moonshot",
     "161.37",
     "Open weights (restricted use)",
     "2026-10-23"
    ],
    [
     "17",
     "GPT-5.6 Terra",
     "OpenAI",
     "160.73",
     "API access",
     "2026-07-09"
    ],
    [
     "18",
     "Claude Haiku 5.5",
     "Anthropic",
     "160.43",
     "API access",
     "2026-10-07"
    ],
    [
     "19",
     "GPT-5.4 Pro",
     "OpenAI",
     "160.24",
     "API access",
     "2026-03-05"
    ],
    [
     "20",
     "GPT-5.5",
     "OpenAI",
     "160.1",
     "API access",
     "2026-04-23"
    ],
    [
     "21",
     "Claude Opus 4.8",
     "Anthropic",
     "159.94",
     "API access",
     "2026-05-28"
    ],
    [
     "22",
     "DeepSeek V4.1 Flash 1017",
     "DeepSeek",
     "159.41",
     "Open weights (unrestricted)",
     "2026-10-17"
    ],
    [
     "23",
     "Gemini 3.8 Flash",
     "Google DeepMind",
     "158.79",
     "API access",
     "2026-09-02"
    ],
    [
     "24",
     "Kimi K3",
     "Moonshot",
     "158.58",
     "Open weights (non-commercial)",
     "2026-07-16"
    ],
    [
     "25",
     "Gemini 3.7 Flash",
     "Google DeepMind",
     "158.36",
     "API access",
     "2026-08-13"
    ]
   ],
   "url": "https://benchmarks.site/internet/epoch-capabilities-index-eci/",
   "csv": "https://benchmarks.site/data/internet/epoch-capabilities-index-eci.csv"
  },
  {
   "benchmark": "scale-seal-leaderboards",
   "title": "Scale SEAL Leaderboards",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "Scale AI (SEAL)",
    "url": "https://labs.scale.com/leaderboard/hle-diamond",
    "retrieved": "2026-11-17",
    "as_of": "2026-09-23",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Company",
    "Score (%)",
    "Confidence interval (+/-)"
   ],
   "rows": [
    [
     "1",
     "claude-fable-5-2",
     "anthropic",
     "72.4",
     "2.8"
    ],
    [
     "2",
     "gpt-6-astra",
     "openai",
     "60.6",
     "3"
    ],
    [
     "3",
     "claude-opus-5-5",
     "anthropic",
     "55",
     "3.1"
    ],
    [
     "4",
     "gemini-4-argon",
     "google",
     "53.2",
     "3.07"
    ],
    [
     "5",
     "claude-fable-5-1",
     "anthropic",
     "51.3",
     "3.1"
    ],
    [
     "6",
     "claude-opus-5",
     "anthropic",
     "38.6",
     "3"
    ]
   ],
   "url": "https://benchmarks.site/internet/scale-seal-leaderboards/",
   "csv": "https://benchmarks.site/data/internet/scale-seal-leaderboards.csv"
  },
  {
   "benchmark": "vals-ai",
   "title": "Vals AI",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "Vals AI",
    "url": "https://www.vals.ai/benchmarks/vals_index",
    "retrieved": "2026-11-17",
    "as_of": "2026-11-15",
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Accuracy (%)",
    "Cost / test",
    "Duration / test"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2",
     "71.31",
     "$17.94",
     "43m42s"
    ],
    [
     "2",
     "Claude Sonnet 5.5",
     "70.85",
     "$16.18",
     "1h26m"
    ],
    [
     "3",
     "Gemini 4 Carbon",
     "70.45",
     "$10.51",
     "47m45s"
    ],
    [
     "4",
     "Claude Opus 5.5",
     "70.14",
     "$24.77",
     "1h25m"
    ],
    [
     "5",
     "Gemini 4 Argon",
     "69.39",
     "$13.37",
     "44m58s"
    ],
    [
     "6",
     "Gemini 4 Hydrogen",
     "68.49",
     "$9.84",
     "32m05s"
    ],
    [
     "7",
     "Wonder Full",
     "66.79",
     "$6.81",
     "59m19s"
    ],
    [
     "8",
     "GPT-6 Astra",
     "65.02",
     "$21.28",
     "33m39s"
    ],
    [
     "9",
     "Wonder",
     "64.88",
     "$10.65",
     "35m24s"
    ],
    [
     "10",
     "GPT-6.1 Sol",
     "63.94",
     "$3.61",
     "44m24s"
    ],
    [
     "11",
     "Qwen 4 Max",
     "63.20",
     "$9.33",
     "1h05m"
    ],
    [
     "12",
     "Kimi K3.1 Pro",
     "62.52",
     "$6.44",
     "1h04m"
    ],
    [
     "13",
     "DeepSeek V4.1 Flash 1017",
     "61.26",
     "$0.66",
     "39m55s"
    ],
    [
     "14",
     "GPT-6.1 Terra",
     "60.54",
     "$2.94",
     "39m43s"
    ],
    [
     "15",
     "Muse Ember",
     "60.48",
     "$0.15",
     "15m02s"
    ],
    [
     "16",
     "GLM 5.3",
     "59.85",
     "$8.52",
     "1h04m"
    ],
    [
     "17",
     "Hy4",
     "59.35",
     "$1.72",
     "51m31s"
    ],
    [
     "18",
     "Qwen 3.8 Max 0902",
     "59.05",
     "$8.73",
     "1h46m"
    ],
    [
     "19",
     "Claude Haiku 5.5",
     "58.73",
     "$3.17",
     "49m57s"
    ],
    [
     "20",
     "Grok 4.7",
     "58.62",
     "$11.63",
     "33m34s"
    ],
    [
     "21",
     "MiMo V2.6 Pro",
     "57.81",
     "$0.43",
     "1h13m"
    ],
    [
     "22",
     "Mistral Large 4",
     "56.53",
     "$3.72",
     "47m48s"
    ],
    [
     "23",
     "Inkling 2 Small",
     "56.08",
     "$3.51",
     "50m35s"
    ],
    [
     "24",
     "Gemini 3.8 Flash",
     "55.95",
     "$5.03",
     "35m58s"
    ],
    [
     "25",
     "MiMo V2.6 Flash",
     "54.83",
     "$0.22",
     "1h07m"
    ]
   ],
   "url": "https://benchmarks.site/internet/vals-ai/",
   "csv": "https://benchmarks.site/data/internet/vals-ai.csv"
  },
  {
   "benchmark": "llm-stats",
   "title": "LLM Stats",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "LLM Stats (llm-stats.com)",
    "url": "https://llm-stats.com/",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Organization",
    "LLM Stats score",
    "Reasoning",
    "Coding",
    "Agent"
   ],
   "rows": [
    [
     "1",
     "Claude Fable 5.2",
     "Anthropic",
     "60.8",
     "60.0",
     "49.8",
     "46.3"
    ],
    [
     "2",
     "Claude Mythos 5.2",
     "Anthropic",
     "59.3",
     "53.4",
     "44.5",
     "44.2"
    ],
    [
     "3",
     "Claude Opus 5.5",
     "Anthropic",
     "58.5",
     "57.4",
     "47.8",
     "43.6"
    ],
    [
     "4",
     "Gemini 4 Carbon",
     "Google",
     "57.4",
     "54.0",
     "45.7",
     "43.7"
    ],
    [
     "5",
     "Claude Sonnet 5.5",
     "Anthropic",
     "56.7",
     "55.4",
     "46.0",
     "41.9"
    ],
    [
     "6",
     "GPT-6 Astra",
     "OpenAI",
     "56.3",
     "55.0",
     "43.9",
     "42.3"
    ],
    [
     "7",
     "Kimi K3.1 Pro",
     "Moonshot AI",
     "54.4",
     "53.3",
     "40.5",
     "39.9"
    ],
    [
     "8",
     "Claude Mythos Preview",
     "Anthropic",
     "53.6",
     "54.6",
     "42.5",
     "35.1"
    ],
    [
     "9",
     "Gemini 4 Argon",
     "Google",
     "53.3",
     "51.2",
     "41.3",
     "39.6"
    ],
    [
     "10",
     "Muse Wonder Full",
     "Meta",
     "52.6",
     "44.8",
     "42.0",
     "40.4"
    ],
    [
     "11",
     "Claude Opus 5",
     "Anthropic",
     "52.5",
     "52.5",
     "37.3",
     "36.7"
    ],
    [
     "12",
     "GPT-6.1 Sol",
     "OpenAI",
     "52.2",
     "49.6",
     "37.6",
     "35.8"
    ],
    [
     "13",
     "Claude Fable 5.1",
     "Anthropic",
     "52.0",
     "49.8",
     "34.4",
     "37.1"
    ],
    [
     "14",
     "Muse Wonder Full",
     "Meta",
     "52.0",
     "40.1",
     "40.9",
     "40.0"
    ],
    [
     "15",
     "Claude Fable 5",
     "Anthropic",
     "51.9",
     "49.8",
     "41.6",
     "37.0"
    ]
   ],
   "url": "https://benchmarks.site/internet/llm-stats/",
   "csv": "https://benchmarks.site/data/internet/llm-stats.csv"
  },
  {
   "benchmark": "kagi-llm-benchmarking-project",
   "title": "Kagi LLM Benchmarking Project",
   "area": "Aggregated leaderboards",
   "section": "Published results",
   "source": {
    "name": "Kagi Search",
    "url": "https://help.kagi.com/kagi/ai/llm-benchmark.html",
    "retrieved": "2026-11-17",
    "as_of": null,
    "license": "not stated"
   },
   "columns": [
    "Rank",
    "Model",
    "Accuracy (%)",
    "Cost ($, whole suite)",
    "Provider"
   ],
   "rows": [
    [
     "1",
     "claude-5-fable-thinking",
     "91.4",
     "10.4",
     "kagi (ult)"
    ],
    [
     "2",
     "claude-5-fable",
     "88.8",
     "9.9",
     "kagi (ult)"
    ],
    [
     "3",
     "claude-4-8-opus-thinking",
     "88.8",
     "8.8",
     "kagi (ult)"
    ],
    [
     "4",
     "gpt-5-5",
     "88.8",
     "2.2",
     "kagi (ult)"
    ],
    [
     "5",
     "claude-4-6-opus-thinking",
     "83.6",
     "4.8",
     "kagi (ult)"
    ],
    [
     "6",
     "grok-4-5",
     "83.5",
     "1.0",
     "kagi (ult)"
    ],
    [
     "7",
     "claude-4-7-opus-thinking",
     "80.7",
     "2.8",
     "kagi (ult)"
    ],
    [
     "8",
     "claude-4-5-opus-thinking",
     "80.2",
     "4.5",
     "kagi (ult)"
    ],
    [
     "9",
     "gemini-3-pro",
     "80.1",
     "0.4",
     "kagi (ult)"
    ],
    [
     "10",
     "kimi-k2-5-thinking",
     "78.5",
     "0.5",
     "kagi"
    ],
    [
     "11",
     "gpt-5-pro",
     "76.8",
     "31.8",
     "kagi (ult)"
    ],
    [
     "12",
     "glm-5-thinking",
     "75.0",
     "2.0",
     "openrouter"
    ],
    [
     "13",
     "grok-4-20-thinking",
     "75.0",
     "1.3",
     "kagi (ult)"
    ],
    [
     "14",
     "chatgpt-4o (points to gpt 5.3)",
     "75.0",
     "0.8",
     "kagi (ult)"
    ],
    [
     "15",
     "claude-4-opus-thinking",
     "74.3",
     "22.4",
     "kagi (ult)"
    ],
    [
     "16",
     "qwen-3-5-397b",
     "73.7",
     "1.8",
     "together"
    ],
    [
     "17",
     "grok-4",
     "73.6",
     "1.0",
     "kagi (ult)"
    ],
    [
     "18",
     "gpt-5-2",
     "73.3",
     "1.4",
     "kagi (ult)"
    ],
    [
     "19",
     "claude-4-7-opus",
     "73.3",
     "2.2",
     "kagi (ult)"
    ],
    [
     "20",
     "claude-4-sonnet-thinking",
     "73.0",
     "5.4",
     "kagi (ult)"
    ],
    [
     "21",
     "gpt-5",
     "72.7",
     "7.1",
     "kagi (ult)"
    ],
    [
     "22",
     "qwen3-max",
     "72.5",
     "1.9",
     "openrouter"
    ],
    [
     "23",
     "claude-4-6-opus",
     "72.4",
     "1.6",
     "kagi (ult)"
    ],
    [
     "24",
     "o3-pro",
     "72.1",
     "34.2",
     "kagi (ult)"
    ],
    [
     "25",
     "claude-4-5-opus",
     "70.7",
     "1.4",
     "kagi (ult)"
    ]
   ],
   "url": "https://benchmarks.site/internet/kagi-llm-benchmarking-project/",
   "csv": "https://benchmarks.site/data/internet/kagi-llm-benchmarking-project.csv"
  }
 ]
}