{
 "checked": "2026-11-17",
 "categories": [
  {
   "slug": "reasoning",
   "title": "General knowledge & reasoning",
   "blurb": "Broad tests of expert knowledge, scientific reasoning and novel problem solving."
  },
  {
   "slug": "math",
   "title": "Math",
   "blurb": "Competition, research-level and formal mathematics, from final-answer problems to proofs."
  },
  {
   "slug": "coding",
   "title": "Coding & software engineering",
   "blurb": "Code generation, bug fixing in real repositories and terminal-based engineering work."
  },
  {
   "slug": "agents",
   "title": "Agents, tool use & computer use",
   "blurb": "Multi-step tasks where a model calls tools, browses the web or operates a computer."
  },
  {
   "slug": "long-context",
   "title": "Long context",
   "blurb": "How well models find, track and reason over information in very long inputs."
  },
  {
   "slug": "instruction-following",
   "title": "Instruction following",
   "blurb": "Whether models obey explicit constraints and keep track of instructions across turns."
  },
  {
   "slug": "factuality",
   "title": "Hallucination & factuality",
   "blurb": "How often models state false facts, invent details or fail to stay grounded in sources."
  },
  {
   "slug": "arenas",
   "title": "Human-preference arenas",
   "blurb": "Leaderboards built from blind side-by-side votes by real users."
  },
  {
   "slug": "multimodal",
   "title": "Multimodal & vision",
   "blurb": "Understanding and reasoning over images, charts, diagrams and video."
  },
  {
   "slug": "ocr",
   "title": "OCR & document understanding",
   "blurb": "Reading text, tables, formulas and layout from scanned and digital documents."
  },
  {
   "slug": "asr",
   "title": "Speech recognition (ASR)",
   "blurb": "Speech-to-text accuracy and speed, usually measured by word error rate."
  },
  {
   "slug": "tts",
   "title": "Text-to-speech",
   "blurb": "Quality and naturalness of synthetic speech, mostly judged by blind listening votes."
  },
  {
   "slug": "embeddings",
   "title": "Embeddings & retrieval",
   "blurb": "Quality of embedding and retrieval models for search, RAG and classification."
  },
  {
   "slug": "safety",
   "title": "Safety & security",
   "blurb": "Resistance to jailbreaks and prompt injection, hazardous knowledge and cyber capabilities."
  },
  {
   "slug": "multilingual",
   "title": "Multilingual",
   "blurb": "Knowledge and understanding across many languages, not just English."
  },
  {
   "slug": "polish",
   "title": "Polish language",
   "blurb": "Benchmarks and leaderboards built specifically for Polish."
  },
  {
   "slug": "performance",
   "title": "Speed, price & providers",
   "blurb": "Throughput, latency, cost and usage of models, API providers and inference hardware."
  },
  {
   "slug": "aggregators",
   "title": "Aggregated leaderboards",
   "blurb": "Sites that combine many benchmarks into one place or one index."
  }
 ],
 "items": [
  {
   "name": "Humanity's Last Exam (HLE)",
   "category": "reasoning",
   "url": "https://lastexam.ai/",
   "paper": "https://arxiv.org/abs/2501.14249",
   "maintainer": "Center for AI Safety and Scale AI",
   "measures": "About 2,500 expert-written questions across more than 100 subjects, some with images, scored by accuracy on exact-match and multiple-choice answers.",
   "type": "dataset",
   "status": "active",
   "notes": "Audits have reported wrong reference answers in part of the set; a revised HLE-Verified version was published in 2026 (arXiv 2602.13964).",
   "id": "humanitys-last-exam-hle",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Accuracy (%)",
     "Calibration error (%)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "GPT 6 Astra",
      "OpenAI",
      "54.8",
      "39.0",
      "2026-09-03"
     ],
     [
      "2",
      "Fable 5.1 (xhigh)",
      "Anthropic",
      "46.5",
      "20.0",
      "2026-09-01"
     ],
     [
      "3",
      "gemini-3.1-pro-preview (thinking high)",
      "Google DeepMind",
      "46.4",
      "51.0",
      "2026-02-19"
     ],
     [
      "4",
      "Sonnet 5.5 (xhigh)",
      "Anthropic",
      "46.1",
      "",
      "2026-09-28"
     ],
     [
      "5",
      "Gemini 3.8 Flash",
      "Google DeepMind",
      "44.5",
      "51.0",
      "2026-09-02"
     ],
     [
      "6",
      "gpt-5.4-pro-2026-03-05",
      "OpenAI",
      "44.3",
      "38.0",
      "2026-03-05"
     ],
     [
      "7",
      "Muse Spark",
      "Meta AI",
      "40.6",
      "50.0",
      "2026-04-08"
     ],
     [
      "8",
      "gemini-3-pro-preview",
      "Google DeepMind",
      "37.5",
      "57.0",
      "2025-11-18"
     ],
     [
      "9",
      "gpt-5.4-2026-03-05 (xhigh thinking)",
      "OpenAI",
      "36.2",
      "42.0",
      "2026-03-05"
     ],
     [
      "10",
      "claude-opus-4-7",
      "Anthropic",
      "36.2",
      "47.0",
      "2026-04-16"
     ],
     [
      "11",
      "claude-opus-4-6-thinking-max",
      "Anthropic",
      "34.4",
      "46.0",
      "2026-02-05"
     ],
     [
      "12",
      "gpt-5-pro-2025-10-06",
      "OpenAI",
      "31.6",
      "49.0",
      "2025-10-07"
     ],
     [
      "13",
      "Claude Haiku 5.5",
      "Anthropic",
      "30.1",
      "",
      "2026-10-07"
     ],
     [
      "14",
      "gpt-5.2-2025-12-11",
      "OpenAI",
      "27.8",
      "45.0",
      "2025-12-11"
     ],
     [
      "15",
      "gpt-5-2025-08-07",
      "OpenAI",
      "25.3",
      "50.0",
      "2025-08-07"
     ],
     [
      "16",
      "gpt-5-2025-08-07",
      "OpenAI",
      "25.3",
      "50.0",
      "2025-08-07"
     ],
     [
      "17",
      "claude-opus-4-5-20251101-thinking",
      "Anthropic",
      "25.2",
      "55.0",
      "2025-11-24"
     ],
     [
      "18",
      "kimi-k2.5",
      "Moonshot",
      "24.4",
      "67.0",
      "2026-01-27"
     ],
     [
      "19",
      "gpt-5.1-thinking",
      "OpenAI",
      "23.7",
      "55.0",
      "2025-11-13"
     ],
     [
      "20",
      "gemini-2.5-pro-preview-06-05",
      "Google DeepMind",
      "21.6",
      "72.0",
      "2025-06-05"
     ],
     [
      "21",
      "o3 (high) (April 2025)",
      "OpenAI",
      "20.3",
      "34.0",
      "2025-04-16"
     ],
     [
      "22",
      "gpt-5-mini-2025-08-07",
      "OpenAI",
      "19.4",
      "65.0",
      "2025-08-07"
     ],
     [
      "23",
      "o3 (medium) (April 2025)",
      "OpenAI",
      "19.2",
      "39.0",
      "2025-04-16"
     ],
     [
      "24",
      "claude-opus-4-6 (Non-Thinking)",
      "Anthropic",
      "19.0",
      "44.0",
      "2026-02-05"
     ],
     [
      "25",
      "Gemini 2.5 Pro Experimental (March 2025)",
      "Google DeepMind",
      "18.2",
      "71.0",
      "2025-03-25"
     ]
    ],
    "main_column": "Accuracy (%)",
    "higher_is_better": true,
    "metric": "Share of questions answered correctly (%)",
    "source": {
     "name": "Epoch AI (Benchmarking Hub)",
     "url": "https://epoch.ai/data/benchmark_data.zip",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0"
    },
    "notes": "Externally reported results mirrored by Epoch AI (hle_external.csv); accuracy converted to % (1 decimal), calibration error copied as stored. Audits have found wrong reference answers in part of the set."
   },
   "plain": "About 2,500 extremely hard questions written by experts in over 100 subjects, from classics to quantum physics, meant to sit at the edge of human knowledge. It is one of the hardest general-knowledge exams for AI."
  },
  {
   "name": "ARC-AGI (ARC Prize)",
   "category": "reasoning",
   "url": "https://arcprize.org/leaderboard",
   "paper": "https://arxiv.org/abs/2603.24621",
   "maintainer": "ARC Prize Foundation",
   "measures": "Novel abstraction puzzles that are easy for humans (grid transformations in ARC-AGI-1 and 2, interactive game environments in ARC-AGI-3), scored as the share of tasks solved and plotted against cost per task.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Three versions share one leaderboard; ARC-AGI-3 launched in 2026, so check which version a score refers to.",
   "id": "arc-agi-arc-prize",
   "results": {
    "columns": [
     "Rank",
     "System",
     "Organization",
     "ARC-AGI-3 score (%)",
     "Total cost (USD)"
    ],
    "rows": [
     [
      "1",
      "GPT-6 Astra - Provider Adapter (High)",
      "OpenAI",
      "99.9",
      "18816.63"
     ],
     [
      "2",
      "Gemini 4 Carbon - Provider Adapter (Medium)",
      "Google",
      "99.1",
      "4559.50"
     ],
     [
      "3",
      "GPT-6 Astra - Provider Adapter (Max)",
      "OpenAI",
      "98.6",
      "17331.95"
     ],
     [
      "4",
      "GPT-6 Astra - Provider Adapter (XHigh)",
      "OpenAI",
      "98.4",
      "18147.19"
     ],
     [
      "5",
      "GPT-6 Astra - Provider Adapter (Medium)",
      "OpenAI",
      "98.4",
      "19284.78"
     ],
     [
      "6",
      "GPT-6 Astra - Provider Adapter (Low)",
      "OpenAI",
      "98.0",
      "21297.92"
     ],
     [
      "7",
      "Claude Fable 5.2 - Provider Adapter (High)",
      "Anthropic",
      "97.7",
      "18624.91"
     ],
     [
      "8",
      "Gemini 4 Carbon - Provider Adapter (Low)",
      "Google",
      "97.7",
      "4350.87"
     ],
     [
      "9",
      "Gemini 4 Argon - Provider Adapter (Low)",
      "Google",
      "97.5",
      "4770.15"
     ],
     [
      "10",
      "Gemini 4 Argon - Provider Adapter (Medium)",
      "Google",
      "97.3",
      "4981.20"
     ],
     [
      "11",
      "Gemini 4 Argon - Provider Adapter (High)",
      "Google",
      "97.3",
      "4625.84"
     ],
     [
      "12",
      "Claude Fable 5.2 - Provider Adapter (XHigh)",
      "Anthropic",
      "97.3",
      "18251.68"
     ],
     [
      "13",
      "Gemini 4 Carbon - Provider Adapter (High)",
      "Google",
      "96.9",
      "4350.03"
     ],
     [
      "14",
      "Claude Fable 5.2 - Provider Adapter (Medium)",
      "Anthropic",
      "96.8",
      "21662.29"
     ],
     [
      "15",
      "GPT-6 Astra - Provider Adapter (None)",
      "OpenAI",
      "96.7",
      "23456.66"
     ],
     [
      "16",
      "GPT-6.1 Sol - Provider Adapter (XHigh)",
      "OpenAI",
      "96.4",
      "4360.21"
     ],
     [
      "17",
      "GPT-6.1 Sol - Provider Adapter (Max)",
      "OpenAI",
      "96.2",
      "3816.80"
     ],
     [
      "18",
      "GPT-6.1 Sol - Provider Adapter (High)",
      "OpenAI",
      "95.0",
      "5403.98"
     ],
     [
      "19",
      "Claude Fable 5.2 - Provider Adapter (Max)",
      "Anthropic",
      "94.9",
      "17875.84"
     ],
     [
      "20",
      "Claude Fable 5.2 - Provider Adapter (Low)",
      "Anthropic",
      "92.3",
      "27377.33"
     ],
     [
      "21",
      "GPT-6.1 Sol - Provider Adapter (Medium)",
      "OpenAI",
      "91.0",
      "5834.91"
     ],
     [
      "22",
      "GPT-6.1 Sol - Provider Adapter (Low)",
      "OpenAI",
      "82.8",
      "6622.20"
     ],
     [
      "23",
      "Claude Fable 5.2 (XHigh)",
      "Anthropic",
      "81.2",
      "21948.83"
     ],
     [
      "24",
      "Claude Fable 5.2 (High)",
      "Anthropic",
      "79.3",
      "25173.94"
     ],
     [
      "25",
      "Claude Fable 5.2 (Medium)",
      "Anthropic",
      "73.3",
      "29034.48"
     ]
    ],
    "main_column": "ARC-AGI-3 score (%)",
    "higher_is_better": true,
    "metric": "Share of ARC-AGI-3 game environments solved on the semi-private test set (%)",
    "source": {
     "name": "ARC Prize Foundation",
     "url": "https://arcprize.org/media/data/leaderboard/v3.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-16",
     "license": "not stated"
    },
    "notes": "ARC-AGI-3 semi-private set, from the leaderboard data file behind arcprize.org/leaderboard; raw fractions converted to % (1 decimal). Rows are system/effort configurations, several per model; 'Provider Adapter' entries use a provider-specific harness. Cost is as listed in the file (units per ARC Prize; shown to 2 decimals). ARC-AGI-1 and ARC-AGI-2 are near-saturated (top systems ~98.5% and ~95%) and are also in v1.json/v2.json."
   },
   "plain": "Puzzles that most people find easy but that require spotting a brand-new pattern rather than recalling facts; the newest version (ARC-AGI-3) drops the AI into small interactive games it has never seen. It is a popular test of whether AI can genuinely learn new skills on the fly."
  },
  {
   "name": "SimpleBench",
   "category": "reasoning",
   "url": "https://simple-bench.com/",
   "paper": null,
   "maintainer": "SimpleBench team (AI Explained)",
   "measures": "Multiple-choice trick questions on spatial and temporal reasoning, social situations and common sense that non-expert humans answer well, scored by accuracy against a human baseline.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Most questions are private to limit contamination.",
   "id": "simplebench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Score AVG@5 (%)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2 (unknown)",
      "Anthropic",
      "94.9",
      "2026-11-09"
     ],
     [
      "2",
      "Gemini 4 Carbon (unknown)",
      "Google DeepMind",
      "92.6",
      "2026-11-10"
     ],
     [
      "3",
      "Gemini 4 Argon (unknown)",
      "Google DeepMind",
      "91.7",
      "2026-10-13"
     ],
     [
      "4",
      "Claude Opus 5.5 (unknown)",
      "Anthropic",
      "88.4",
      "2026-09-22"
     ],
     [
      "5",
      "Claude Fable 5.1 (unknown)",
      "Anthropic",
      "86.6",
      "2026-09-01"
     ],
     [
      "6",
      "GPT-6 Astra (pro, max)",
      "OpenAI",
      "86.5",
      "2026-09-03"
     ],
     [
      "7",
      "GPT-6 Astra (unknown thinking)",
      "OpenAI",
      "83.6",
      "2026-09-03"
     ],
     [
      "8",
      "GPT-6.1 Sol (unknown thinking)",
      "OpenAI",
      "82.9",
      "2026-09-29"
     ],
     [
      "9",
      "Gemini 3.8 Flash (unknown)",
      "Google DeepMind",
      "82.4",
      "2026-09-02"
     ],
     [
      "10",
      "Claude Fable 5 (max)",
      "Anthropic",
      "81.9",
      "2026-06-09"
     ],
     [
      "11",
      "Muse Spark 1.3 (unknown)",
      "Meta AI",
      "81.8",
      "2026-09-02"
     ],
     [
      "12",
      "Claude Opus 5 (unknown)",
      "Anthropic",
      "80.6",
      "2026-07-24"
     ],
     [
      "13",
      "Gemini 3.1 Pro Preview",
      "Google DeepMind",
      "79.6",
      "2026-02-19"
     ],
     [
      "14",
      "GPT-5.5 Pro (unknown thinking)",
      "OpenAI",
      "76.9",
      "2026-04-23"
     ],
     [
      "15",
      "GPT-5.5 Pro (high)",
      "OpenAI",
      "76.9",
      "2026-04-23"
     ],
     [
      "16",
      "Gemini 3.5 Flash",
      "Google DeepMind",
      "76.7",
      "2026-05-19"
     ],
     [
      "17",
      "Gemini 3 Pro Preview",
      "Google DeepMind",
      "76.4",
      "2025-11-18"
     ],
     [
      "18",
      "Grok 4.6 (unknown)",
      "xAI",
      "75.9",
      "2026-08-12"
     ],
     [
      "19",
      "Claude Sonnet 5.5 (unknown)",
      "Anthropic",
      "75.9",
      "2026-09-28"
     ],
     [
      "20",
      "Muse Spark 1.2",
      "Meta AI",
      "74.5",
      "2026-08-05"
     ],
     [
      "21",
      "GPT-5.4 Pro",
      "OpenAI",
      "74.1",
      "2026-03-05"
     ],
     [
      "22",
      "GPT-6 Sol (unknown thinking)",
      "OpenAI",
      "73.1",
      "2026-09-22"
     ],
     [
      "23",
      "Muse Ember (unknown)",
      "Meta AI",
      "72.7",
      "2026-11-09"
     ],
     [
      "24",
      "GPT-6.1 Terra (unknown thinking)",
      "OpenAI",
      "72.2",
      "2026-10-26"
     ],
     [
      "25",
      "GPT-5.6 Sol (pro, xhigh)",
      "OpenAI",
      "71.7",
      "2026-07-09"
     ]
    ],
    "main_column": "Score AVG@5 (%)",
    "higher_is_better": true,
    "metric": "Share of questions answered correctly, averaged over 5 runs (%); the human baseline is about 84%",
    "source": {
     "name": "Epoch AI (Benchmarking Hub)",
     "url": "https://epoch.ai/data/benchmark_data.zip",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0"
    },
    "notes": "Scores published on simple-bench.com, as mirrored by Epoch AI (simplebench_external.csv), converted to % (1 decimal). '(unknown)' means the reasoning-effort setting was not stated. Human baseline figure is from simple-bench.com."
   },
   "plain": "Trick questions about everyday situations, space, time and people that ordinary adults usually get right but that can trip up AI. It tests common sense rather than book knowledge."
  },
  {
   "name": "LiveBench",
   "category": "reasoning",
   "url": "https://livebench.ai/",
   "paper": "https://arxiv.org/abs/2406.19314",
   "maintainer": "LiveBench team (Abacus.AI, NYU and others)",
   "measures": "Regularly refreshed questions in math, coding, reasoning, language, data analysis and instruction following, scored automatically against objective ground-truth answers.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Questions are rotated over time to limit training-data contamination.",
   "id": "livebench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Overall (computed)",
     "Long Agentic",
     "Reasoning",
     "Coding"
    ],
    "rows": [
     [
      "1",
      "claude-fable-5.2",
      "84.9",
      "82.9",
      "88.8",
      "79.2"
     ],
     [
      "2",
      "gpt-6-astra",
      "84.4",
      "75.6",
      "92.7",
      "87.5"
     ],
     [
      "3",
      "claude-fable-5.1",
      "84.0",
      "74.2",
      "91.7",
      "89.7"
     ],
     [
      "4",
      "gemini-4-carbon",
      "83.5",
      "72.7",
      "90.6",
      "91.0"
     ],
     [
      "5",
      "claude-opus-5.5",
      "83.4",
      "72.8",
      "92.2",
      "87.1"
     ],
     [
      "6",
      "gpt-6.1-sol",
      "81.4",
      "65.6",
      "92.6",
      "82.1"
     ],
     [
      "7",
      "gpt-6.1-terra",
      "78.8",
      "56.2",
      "88.7",
      "89.2"
     ],
     [
      "8",
      "gemini-4-argon",
      "77.7",
      "58.5",
      "95.1",
      "85.7"
     ],
     [
      "9",
      "gemini-4-hydrogen",
      "77.0",
      "61.8",
      "88.6",
      "78.5"
     ],
     [
      "10",
      "claude-sonnet-5.5",
      "77.0",
      "63.4",
      "91.6",
      "77.2"
     ],
     [
      "11",
      "qwen4-max",
      "73.3",
      "47.5",
      "92.2",
      "69.8"
     ],
     [
      "12",
      "muse-wonder-1.0",
      "70.7",
      "46.2",
      "90.6",
      "74.9"
     ],
     [
      "13",
      "deepseek-v4.1-flash-1017",
      "70.3",
      "38.8",
      "89.1",
      "80.4"
     ],
     [
      "14",
      "claude-haiku-5.5",
      "68.9",
      "47.9",
      "82.4",
      "77.6"
     ],
     [
      "15",
      "kimi-k3.1-pro",
      "68.7",
      "38.7",
      "91.1",
      "67.8"
     ],
     [
      "16",
      "muse-spark-1.3",
      "68.6",
      "47.9",
      "89.7",
      "84.4"
     ],
     [
      "17",
      "grok-4.7",
      "67.9",
      "45.3",
      "82.7",
      "73.1"
     ],
     [
      "18",
      "gpt-5.6-terra",
      "67.2",
      "25.9",
      "90.6",
      "84.2"
     ],
     [
      "19",
      "mimo-v2.6-pro",
      "66.1",
      "36.4",
      "79.6",
      "64.5"
     ],
     [
      "20",
      "kimi-k3",
      "65.0",
      "36.0",
      "90.7",
      "53.1"
     ],
     [
      "21",
      "minimax-m3.1-flash",
      "63.7",
      "25.7",
      "82.9",
      "68.8"
     ],
     [
      "22",
      "grok-4.6",
      "63.5",
      "18.7",
      "91.2",
      "76.7"
     ],
     [
      "23",
      "qwen3.8-max",
      "60.9",
      "26.6",
      "88.2",
      "60.9"
     ],
     [
      "24",
      "glm-5.3-flash",
      "60.7",
      "20.7",
      "77.6",
      "70.9"
     ],
     [
      "25",
      "gpt-6-luna",
      "60.4",
      "24.7",
      "81.8",
      "70.8"
     ]
    ],
    "main_column": "Overall (computed)",
    "higher_is_better": true,
    "metric": "Weighted overall score across LiveBench 2.0 categories (0-100)",
    "source": {
     "name": "LiveBench team (Abacus.AI and collaborators)",
     "url": "https://livebench.ai/data/20261106/table.csv",
     "as_of": "2026-11-06",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "Release 2026-11-06 of LiveBench 2.0. Site publishes only category scores in table.csv; Overall computed here as weighted mean using the site categories.json weights (Long Agentic 2, Coding 1.5, Design 1.5, others 1) and may differ slightly from the site display. Also has Browser, Data Analysis, Research, Design columns in source."
   },
   "plain": "A set of regularly refreshed tests (agent tasks, reasoning, coding, research, design, etc.) designed so AI models cannot have seen the questions beforehand."
  },
  {
   "name": "CritPt",
   "category": "reasoning",
   "url": "https://critpt.com/",
   "paper": "https://arxiv.org/abs/2509.26574",
   "maintainer": "CritPt team (academic physicists)",
   "measures": "Unpublished, research-level physics problems across fields such as condensed matter, quantum and astrophysics, scored by automatically checked final answers.",
   "type": "dataset",
   "status": "active",
   "notes": null,
   "id": "critpt",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Accuracy (%)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "Gemini 4 Carbon (High)",
      "Google DeepMind",
      "36.0",
      "2026-11-10"
     ],
     [
      "2",
      "Claude Fable 5.2 (Xhigh, Default Fallback)",
      "Anthropic",
      "35.7",
      "2026-11-09"
     ],
     [
      "3",
      "Claude Fable 5.2 (Max, Default Fallback)",
      "Anthropic",
      "35.4",
      "2026-11-09"
     ],
     [
      "4",
      "Claude Fable 5.2 (High, Default Fallback)",
      "Anthropic",
      "32.9",
      "2026-11-09"
     ],
     [
      "5",
      "GPT-5.6 Sol (Max)",
      "OpenAI",
      "32.3",
      "2026-07-09"
     ],
     [
      "6",
      "GPT-6 Astra (Max)",
      "OpenAI",
      "31.7",
      "2026-09-03"
     ],
     [
      "7",
      "Claude Opus 5.5 (Max, Default Fallback)",
      "Anthropic",
      "31.7",
      "2026-09-22"
     ],
     [
      "8",
      "Claude Opus 5.5 (Xhigh, Default Fallback)",
      "Anthropic",
      "31.7",
      "2026-09-22"
     ],
     [
      "9",
      "GPT-6.1 Sol (Max)",
      "OpenAI",
      "31.7",
      "2026-09-29"
     ],
     [
      "10",
      "GPT-6.1 Sol (Xhigh)",
      "OpenAI",
      "31.7",
      "2026-09-29"
     ],
     [
      "11",
      "GPT-6 Astra (Xhigh)",
      "OpenAI",
      "31.4",
      "2026-09-03"
     ],
     [
      "12",
      "Claude Sonnet 5.5 (Max, Default Fallback)",
      "Anthropic",
      "31.4",
      "2026-09-28"
     ],
     [
      "13",
      "Claude Fable 5.1 (Xhigh, Default Fallback)",
      "Anthropic",
      "31.1",
      "2026-09-01"
     ],
     [
      "14",
      "Claude Sonnet 5.5 (Xhigh, Default Fallback)",
      "Anthropic",
      "31.1",
      "2026-09-28"
     ],
     [
      "15",
      "GPT-6 Sol (Max)",
      "OpenAI",
      "30.9",
      "2026-09-22"
     ],
     [
      "16",
      "Claude Opus 5.5 (High, Default Fallback)",
      "Anthropic",
      "30.9",
      "2026-09-22"
     ],
     [
      "17",
      "Claude Fable 5.2 (Medium, Default Fallback)",
      "Anthropic",
      "30.9",
      "2026-11-09"
     ],
     [
      "18",
      "GPT-5.5 Pro (Xhigh)",
      "OpenAI",
      "30.6",
      "2026-04-23"
     ],
     [
      "19",
      "Claude Fable 5.1 (High, Default Fallback)",
      "Anthropic",
      "30.3",
      "2026-09-01"
     ],
     [
      "20",
      "GPT-5.4 Pro (Xhigh)",
      "OpenAI",
      "30.0",
      "2026-03-05"
     ],
     [
      "21",
      "GPT-5.6 Terra (Max)",
      "OpenAI",
      "30.0",
      "2026-07-09"
     ],
     [
      "22",
      "GPT-6.1 Sol (High)",
      "OpenAI",
      "30.0",
      "2026-09-29"
     ],
     [
      "23",
      "Claude Fable 5.1 (Max, Default Fallback)",
      "Anthropic",
      "29.7",
      "2026-09-01"
     ],
     [
      "24",
      "Claude Fable 5.2 (Low, Default Fallback)",
      "Anthropic",
      "29.4",
      "2026-11-09"
     ],
     [
      "25",
      "Claude Opus 5 (Adaptive Reasoning, Max Effort)",
      "Anthropic",
      "29.1",
      "2026-07-24"
     ]
    ],
    "main_column": "Accuracy (%)",
    "higher_is_better": true,
    "metric": "Share of problems with the correct final answer (%)",
    "source": {
     "name": "Epoch AI (Benchmarking Hub)",
     "url": "https://epoch.ai/data/benchmark_data.zip",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0"
    },
    "notes": "External results mirrored by Epoch AI (critpt_external.csv), converted to % (1 decimal). One row per model + reasoning-effort configuration."
   },
   "plain": "Unpublished research-level physics problems written by working physicists, in areas like quantum physics and astrophysics. It tests whether AI can do the kind of reasoning needed in real scientific research."
  },
  {
   "name": "EQ-Bench",
   "category": "reasoning",
   "url": "https://eqbench.com/",
   "paper": "https://arxiv.org/abs/2312.06281",
   "maintainer": "Sam Paech",
   "measures": "Emotional intelligence in multi-turn role-play scenarios, plus separate creative-writing tests, scored by an LLM judge.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Scores depend on an LLM judge rather than fixed answers.",
   "id": "eq-bench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Elo",
     "95% CI"
    ],
    "rows": [
     [
      "1",
      "claude-opus-5",
      "1364.9",
      "1349.5 - 1378.4"
     ],
     [
      "2",
      "claude-opus-5-5",
      "1339.5",
      "1329.4 - 1349.6"
     ],
     [
      "3",
      "muse-wonder",
      "1324.9",
      "1312.5 - 1339.0"
     ],
     [
      "4",
      "kimi-k3",
      "1319.2",
      "1308.7 - 1333.0"
     ],
     [
      "5",
      "claude-fable-5",
      "1312.5",
      "1302.8 - 1323.6"
     ],
     [
      "6",
      "gpt-5.5",
      "1295.4",
      "1285.0 - 1305.6"
     ],
     [
      "7",
      "claude-opus-4-7",
      "1293.2",
      "1283.6 - 1303.4"
     ],
     [
      "8",
      "gpt-6.1-sol",
      "1272.4",
      "1262.9 - 1282.7"
     ],
     [
      "9",
      "claude-opus-4-8",
      "1267.8",
      "1258.6 - 1277.0"
     ],
     [
      "10",
      "gpt-5.4",
      "1257.7",
      "1250.5 - 1265.3"
     ],
     [
      "11",
      "muse-spark-1.1",
      "1244.4",
      "1230.8 - 1256.2"
     ],
     [
      "12",
      "gemini-4-argon",
      "1236.3",
      "1225.0 - 1248.1"
     ],
     [
      "13",
      "gpt-5.6-sol",
      "1236.1",
      "1225.4 - 1247.7"
     ],
     [
      "14",
      "claude-sonnet-5",
      "1221.2",
      "1211.6 - 1233.5"
     ],
     [
      "15",
      "gpt-5.6-terra",
      "1220.1",
      "1209.9 - 1229.9"
     ],
     [
      "16",
      "Inkling",
      "1212.1",
      "1201.2 - 1222.0"
     ],
     [
      "17",
      "claude-opus-4-6",
      "1210.0",
      "1198.5 - 1219.6"
     ],
     [
      "18",
      "glm-5.2",
      "1208.1",
      "1197.1 - 1216.9"
     ],
     [
      "19",
      "mimo-v2.5-pro",
      "1197.7",
      "1185.0 - 1212.5"
     ],
     [
      "20",
      "claude-sonnet-4-6",
      "1196.8",
      "1187.9 - 1204.9"
     ],
     [
      "21",
      "kimi-k2.6",
      "1191.4",
      "1181.4 - 1200.8"
     ],
     [
      "22",
      "deepseek-v4-pro",
      "1156.3",
      "1144.7 - 1166.7"
     ],
     [
      "23",
      "gpt-5.6-luna",
      "1145.9",
      "1134.2 - 1156.6"
     ],
     [
      "24",
      "minimax-m3",
      "1140.1",
      "1128.8 - 1152.1"
     ],
     [
      "25",
      "gemini-3.1-pro-preview",
      "1132.3",
      "1118.9 - 1144.4"
     ]
    ],
    "main_column": "Elo",
    "higher_is_better": true,
    "metric": "EQ-Bench 4 Elo rating (higher means judges preferred the model more often)",
    "source": {
     "name": "EQ-Bench (Sam Paech)",
     "url": "https://eqbench.com/eqbench4/eqbench4_data.js",
     "as_of": "2026-10-24",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "EQ-Bench 4 (120 personas, judged by Gemini 3.1 Pro, GPT-5.5, Claude Opus 4.6); sorted by Elo from site data file. Date is data generation time."
   },
   "plain": "Tests how well AI models handle emotionally sensitive conversations with simulated people who have different personalities and needs."
  },
  {
   "name": "FrontierMath",
   "category": "math",
   "url": "https://epoch.ai/frontiermath",
   "paper": "https://arxiv.org/abs/2411.04872",
   "maintainer": "Epoch AI",
   "measures": "Unpublished, expert-written math problems in four difficulty tiers up to research level, with answers checked automatically, scored by accuracy.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Most problems are private; OpenAI funded the benchmark and has access to much of the problem set.",
   "id": "frontiermath",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Tiers 1-3 (%)",
     "Tier 4 (%)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2 (max)",
      "Anthropic",
      "95.1",
      "100.0",
      "2026-11-09"
     ],
     [
      "2",
      "GPT-6.1 Sol (max)",
      "OpenAI",
      "93.7",
      "100.0",
      "2026-09-29"
     ],
     [
      "3",
      "GPT-6 Astra (max)",
      "OpenAI",
      "93.7",
      "97.6",
      "2026-09-03"
     ],
     [
      "4",
      "Claude Opus 5.5 (max)",
      "Anthropic",
      "91.2",
      "95.0",
      "2026-09-22"
     ],
     [
      "5",
      "GPT-6.1 Terra (max)",
      "OpenAI",
      "90.9",
      "92.7",
      "2026-10-26"
     ],
     [
      "6",
      "Gemini 4 Carbon (high)",
      "Google DeepMind",
      "90.5",
      "95.1",
      "2026-11-10"
     ],
     [
      "7",
      "Claude Fable 5.1 (max)",
      "Anthropic",
      "90.2",
      "87.8",
      "2026-09-01"
     ],
     [
      "8",
      "GPT-6 Sol (max)",
      "OpenAI",
      "89.8",
      "90.0",
      "2026-09-22"
     ],
     [
      "9",
      "GPT-5.6 Sol (max)",
      "OpenAI",
      "89.1",
      "82.9",
      "2026-07-09"
     ],
     [
      "10",
      "Claude Sonnet 5.5 (max)",
      "Anthropic",
      "88.8",
      "80.5",
      "2026-09-28"
     ],
     [
      "11",
      "GPT-5.5 Pro (xhigh)",
      "OpenAI",
      "87.7",
      "78.0",
      "2026-04-23"
     ],
     [
      "12",
      "Claude Fable 5 (max)",
      "Anthropic",
      "87.0",
      "90.2",
      "2026-06-09"
     ],
     [
      "13",
      "Gemini 4 Argon (high)",
      "Google DeepMind",
      "86.0",
      "78.0",
      "2026-10-13"
     ],
     [
      "14",
      "GPT-5.6 Terra (max)",
      "OpenAI",
      "86.0",
      "70.7",
      "2026-07-09"
     ],
     [
      "15",
      "Claude Opus 5 (max)",
      "Anthropic",
      "85.6",
      "73.2",
      "2026-07-24"
     ],
     [
      "16",
      "GPT-5.5 (xhigh)",
      "OpenAI",
      "85.3",
      "72.5",
      "2026-04-23"
     ],
     [
      "17",
      "GPT-5.4 Pro (xhigh)",
      "OpenAI",
      "82.5",
      "58.5",
      "2026-03-05"
     ],
     [
      "18",
      "Muse Wonder (max)",
      "Meta AI",
      "82.1",
      "70.7",
      "2026-10-12"
     ],
     [
      "19",
      "GPT-5.6 Luna (max)",
      "OpenAI",
      "82.1",
      "61.0",
      "2026-07-09"
     ],
     [
      "20",
      "Claude Opus 4.8",
      "Anthropic",
      "80.0",
      "56.1",
      "2026-05-28"
     ],
     [
      "21",
      "GPT-6 Luna (max)",
      "OpenAI",
      "78.9",
      "56.1",
      "2026-09-22"
     ],
     [
      "22",
      "GPT-5.4 (xhigh)",
      "OpenAI",
      "78.6",
      "49.0",
      "2026-03-05"
     ],
     [
      "23",
      "Kimi K3.1 Pro (Together, max)",
      "Moonshot",
      "78.2",
      "65.9",
      "2026-10-23"
     ],
     [
      "24",
      "Claude Haiku 5.5 (max)",
      "Anthropic",
      "75.1",
      "46.3",
      "2026-10-07"
     ],
     [
      "25",
      "Qwen3.8 Max (xhigh)",
      "Alibaba",
      "74.7",
      "46.3",
      "2026-08-02"
     ]
    ],
    "main_column": "Tiers 1-3 (%)",
    "higher_is_better": true,
    "metric": "Share of Tiers 1-3 problems solved correctly (%)",
    "source": {
     "name": "Epoch AI (Benchmarking Hub)",
     "url": "https://epoch.ai/data/benchmark_data.zip",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0"
    },
    "notes": "Epoch AI runs on the private v2 problem sets (frontiermath_tiers_1_3_v2.csv and frontiermath_tier_4_v2.csv), converted to % (1 decimal). Tier 4 is the research-level tier; blank = not evaluated on Tier 4 v2. OpenAI funded the benchmark and has access to much of the problem set."
   },
   "plain": "Hundreds of brand-new, unpublished math problems written by professional mathematicians, ranging from hard competition level up to problems that take experts days. Because the problems are secret, it is a strong check of real mathematical ability rather than memorization."
  },
  {
   "name": "MathArena",
   "category": "math",
   "url": "https://matharena.ai/",
   "paper": "https://arxiv.org/abs/2505.23281",
   "maintainer": "SRI Lab, ETH Zurich",
   "measures": "Models are tested on new math competitions (such as AIME, HMMT and olympiads) shortly after they take place, scored by final-answer accuracy or human grading of proofs.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Using freshly released problems limits contamination.",
   "id": "matharena",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Provider",
     "Accuracy (± 95% CI)",
     "Cost"
    ],
    "rows": [
     [
      "1",
      "Gemini 4 Carbon (high)",
      "Google",
      "94.28% ± 3.64%",
      "$5.08"
     ],
     [
      "2",
      "GPT-6.1 Sol (max)",
      "OpenAI",
      "92.81% ± 3.10%",
      "$0.96"
     ],
     [
      "3",
      "Gemini 4 Argon (high)",
      "Google",
      "84.39% ± 4.65%",
      "$3.74"
     ],
     [
      "4",
      "Claude-Opus-5.5 (high)",
      "Anthropic",
      "82.61% ± 4.72%",
      "$1.93"
     ],
     [
      "5",
      "Kimi K3.1 Pro (max)",
      "Moonshot AI",
      "58.27% ± 6.84%",
      "$2.94"
     ],
     [
      "6",
      "DeepSeek-V4.1-Flash-1017 (max, DSH)",
      "DeepSeek",
      "42.48% ± 6.32%",
      "$0.33"
     ],
     [
      "7",
      "Inkling 2 Small (0.99)",
      "Thinking Machines Lab",
      "31.88% ± 5.14%",
      "$0.60"
     ],
     [
      "8",
      "Muse Spark Open (xhigh)",
      "Meta AI",
      "27.02% ± 5.61%",
      "$1.03"
     ]
    ],
    "main_column": "Accuracy (± 95% CI)",
    "higher_is_better": true,
    "metric": "Accuracy on ArXivMath (questions from recent research papers), % with 95% CI",
    "source": {
     "name": "MathArena (SRI Lab ETH Zurich / INSAIT)",
     "url": "https://matharena.ai/",
     "as_of": "2026-11-13",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "Table is the ArXivMath overall board (most recently updated). MathArena has many other boards (final-answer comps, proof comps, Project Euler, etc.)."
   },
   "plain": "Tests AI models on fresh, hard math problems taken from new competitions and research papers, so the answers cannot have been memorised."
  },
  {
   "name": "PutnamBench",
   "category": "math",
   "url": "https://trishullab.github.io/PutnamBench/",
   "paper": "https://arxiv.org/abs/2407.11214",
   "maintainer": "Trishul Lab (UT Austin)",
   "measures": "Putnam competition problems formalized in Lean and other proof assistants, scored by the number of problems for which a machine-checked proof is produced.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Tests formal theorem proving, not natural-language answers.",
   "id": "putnambench",
   "results": {
    "columns": [
     "Rank",
     "Method",
     "Solved (of 672)",
     "Date added",
     "Compute budget"
    ],
    "rows": [
     [
      "1",
      "Aleph Prover (Logical Intelligence)   ",
      "672",
      "2026-08-26",
      "Avg $74, Max $1468 per problem"
     ],
     [
      "2",
      "Humanfia",
      "672",
      "2026-08-27",
      "Avg $44.5 total cost/problem"
     ],
     [
      "3",
      "NEAR AI (w/ DeepSeek V4)",
      "672",
      "2026-09-04",
      "Mean $0.17, median $0.04, max $11.18 per problem"
     ],
     [
      "4",
      "Forall (Astrio)",
      "672",
      "2026-09-24",
      "Mean $3.52 actor-side cost/problem; median $2.49"
     ],
     [
      "5",
      "Aleph Prover (Logical Intelligence)  ",
      "668",
      "2026-01-11",
      "Avg $68, Limit $1400 per problem"
     ],
     [
      "6",
      "Mistral Prover (DeepSeek V4 Pro + Leanstral)",
      "658",
      "2026-09-04",
      "Avg 5.38M tokens per solved problem"
     ],
     [
      "7",
      "Goedel-Architect ",
      "642",
      "2026-07-02",
      "Avg $5.8 total cost/problem"
     ],
     [
      "8",
      "Aleph Prover (Logical Intelligence) ",
      "637",
      "2025-12-26",
      "Avg $54, Limit $400 per problem"
     ],
     [
      "9",
      "Goedel-Architect",
      "597",
      "2026-06-06",
      "pass@4, Avg. $1.47 total cost / problem"
     ],
     [
      "10",
      "Leanstral 1.5",
      "588",
      "2026-07-19",
      "4m tokens per problem"
     ],
     [
      "11",
      "Seed-Prover 1.5 (ByteDance)",
      "581",
      "2025-12-27",
      "10 H20 days per problem"
     ],
     [
      "12",
      "Aleph Prover (Logical Intelligence)",
      "500",
      "2025-12-23",
      "Avg $23, Limit $100 per problem"
     ],
     [
      "13",
      "Hilbert",
      "462",
      "2025-10-06",
      "avg pass@1840"
     ],
     [
      "14",
      "AxProverBase (Axiomatic AI)",
      "365",
      "2026-04-14",
      "Avg $12.60, Opus 4.5 w/ 50 iterations"
     ],
     [
      "15",
      "Seed-Prover (ByteDance)",
      "329",
      "2025-12-23",
      "MEDIUM"
     ],
     [
      "16",
      "C2C-Goedel",
      "110",
      "2026-08-01",
      "pass@256"
     ],
     [
      "17",
      "Ax-Prover (Axiomatic AI)",
      "91",
      "2025-12-23",
      "pass@1, avg. 100 tool calls"
     ],
     [
      "18",
      "Goedel-Prover-V2",
      "86",
      "2025-07-22",
      "pass@184"
     ],
     [
      "19",
      "Ensemble Prover (Graviterra)",
      "65",
      "2026-09-04",
      "Not reported"
     ],
     [
      "20",
      "DeepSeek-Prover-V2",
      "47",
      "2025-06-03",
      "pass@1024"
     ],
     [
      "21",
      "GPT-5 (ReAct, 10 turns)",
      "28",
      "2025-11-20",
      "pass@1, 10 tool calls"
     ],
     [
      "22",
      "C2C-Kimina",
      "25",
      "2026-08-01",
      "pass@256"
     ],
     [
      "23",
      "DSP+",
      "23",
      "2025-06-28",
      "pass@128"
     ],
     [
      "24",
      "Bourbaki",
      "14",
      "2025-07-16",
      "pass@512"
     ],
     [
      "25",
      "Kimina-Prover-7B-Distill",
      "10",
      "2025-04-14",
      "pass@192"
     ]
    ],
    "main_column": "Solved (of 672)",
    "higher_is_better": true,
    "metric": "Number of problems solved (out of 672) in Lean with answers",
    "source": {
     "name": "PutnamBench (Trishul lab, UT Austin)",
     "url": "https://trishullab.github.io/PutnamBench/results.json",
     "as_of": "2026-10-06",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "Lean (with solution/answer placeholders pre-filled) track only. Ties broken by earliest date added; the site ranks several tied 672/672 entries cheapest first. as_of is last repo commit. Isabelle/Rocq tracks not shown."
   },
   "plain": "Tests whether AI systems can write machine-checkable formal proofs for problems from the Putnam, a famously hard undergraduate math competition."
  },
  {
   "name": "SWE-Bench Pro",
   "category": "coding",
   "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2",
   "paper": "https://arxiv.org/abs/2509.16941",
   "maintainer": "Scale AI",
   "measures": "Long-horizon software engineering tasks from open-source repositories in several languages, scored by the share of tasks resolved under hidden tests.",
   "type": "leaderboard",
   "status": "active",
   "notes": "A revised V2 split (2026, arXiv 2610.24206) repaired or retired many tasks; an independent SWE-Bench Pro Verified variant also exists. Scores across versions are not comparable.",
   "id": "swe-bench-pro",
   "results": {
    "columns": [
     "Rank",
     "Model (scaffold, effort)",
     "Score (%)"
    ],
    "rows": [
     [
      "1",
      "Opus 5 (Claude Code) xhigh",
      "98.00"
     ],
     [
      "2",
      "Fable 5.1 (Claude Code) high",
      "92.20"
     ],
     [
      "3",
      "Gemini 4 Argon (mini-swe-agent) high",
      "92.16"
     ],
     [
      "4",
      "GPT-6 Astra (Codex) high",
      "90.20"
     ],
     [
      "4",
      "Muse Wonder 1.0 (mini-swe-agent) xhigh",
      "90.20"
     ],
     [
      "6",
      "Kimi-K3 (mini-swe-agent) max",
      "88.20"
     ],
     [
      "6",
      "Sonnet 5 (Claude Code) xhigh",
      "88.20"
     ],
     [
      "8",
      "GPT-5.6 Terra (Codex) xhigh",
      "86.30"
     ],
     [
      "8",
      "Inkling 2 Small",
      "86.27"
     ],
     [
      "9",
      "Minimax M3p1 Flash (mini-swe-agent) xhigh",
      "84.31"
     ],
     [
      "10",
      "GLM-5.3 (mini-swe-agent) max",
      "84.30"
     ],
     [
      "11",
      "GPT-5.6 Sol (Codex) xhigh",
      "82.40"
     ],
     [
      "12",
      "Muse Spark Open (mini-swe-agent) xhigh",
      "82.35"
     ],
     [
      "13",
      "Sonnet 5.5 (mini-swe-agent) xhigh",
      "70.59"
     ],
     [
      "14",
      "Gemini 3.8 Flash (mini-swe-agent) high",
      "58.80"
     ],
     [
      "15",
      "Inkling (mini-swe-agent) xhigh",
      "56.90"
     ],
     [
      "16",
      "Haiku 5.5 (mini-swe-agent) high",
      "52.94"
     ],
     [
      "17",
      "Haiku 4.5 (Claude Code) xhigh",
      "25.50"
     ]
    ],
    "main_column": "Score (%)",
    "higher_is_better": true,
    "metric": "Share of tasks solved (%)",
    "source": {
     "name": "Scale AI Labs",
     "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2",
     "as_of": "2026-09-22",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "SWE-Bench Pro V2 public split (642 tasks). Page has Full/HARD tabs; the table scraped is the default view and which tab it is was not identifiable from static HTML. Some entries gray-out/capped runs."
   },
   "plain": "Tests whether AI agents can solve long, realistic software tasks in real open-source projects, harder than older bug-fixing tests."
  },
  {
   "name": "SWE-rebench",
   "category": "coding",
   "url": "https://swe-rebench.com/",
   "paper": "https://arxiv.org/abs/2505.20411",
   "maintainer": "SWE-rebench team (Nebius)",
   "measures": "Fresh real-world GitHub tasks collected in rolling time windows, scored by resolved rate and pass@5, with cost and tokens per problem.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Results are flagged when a model may have seen the tasks during training.",
   "id": "swe-rebench",
   "results": {
    "columns": [
     "Rank",
     "Model / Agent",
     "Resolved Rate (%)",
     "Pass@5 (%)",
     "Cost per Problem"
    ],
    "rows": [
     [
      "1",
      "Anthropic Fable 5 [high]",
      "64.5",
      "78.4",
      "$4.40"
     ],
     [
      "2",
      "Grok Grok 4.5 [high]",
      "63.8",
      "77.5",
      "$1.47"
     ],
     [
      "3",
      "Anthropic Opus 5 [high]",
      "63.4",
      "74.8",
      "$3.47"
     ],
     [
      "4",
      "Z.ai GLM-5.2 [high]",
      "62.9",
      "81.1",
      "$1.40"
     ],
     [
      "5",
      "OpenAI GPT-5.6 Sol [medium]",
      "62.3",
      "79.3",
      "$0.85"
     ],
     [
      "6",
      "Junie Junie",
      "61.8",
      "73.9",
      "$0.81"
     ],
     [
      "7",
      "Anthropic Claude Code",
      "60.4",
      "75.7",
      "$3.39"
     ],
     [
      "8",
      "OpenAI Codex",
      "58.0",
      "73.0",
      "$1.59"
     ],
     [
      "9",
      "Anthropic Sonnet 5 [high]",
      "56.8",
      "74.8",
      "$1.43"
     ],
     [
      "10",
      "Cursor Cursor",
      "51.7",
      "65.8",
      "$0.41"
     ],
     [
      "11",
      "Minimax MiniMax M3",
      "47.2",
      "69.4",
      "$0.95"
     ],
     [
      "12",
      "XiaomiMiMo MiMo V2.5 Pro",
      "46.5",
      "65.8",
      "$0.10"
     ],
     [
      "13",
      "OpenAI GPT-5.6 Luna [medium]",
      "43.6",
      "59.5",
      "$0.11"
     ],
     [
      "14",
      "DeepSeek DeepSeek-V4 Pro [high]",
      "40.2",
      "64.0",
      "$0.15"
     ],
     [
      "15",
      "Qwen Qwen3.6-27B",
      "31.2",
      "57.7",
      "$0.62"
     ],
     [
      "16",
      "Qwen Qwen3.6-35B-A3B",
      "24.7",
      "43.2",
      "$0.27"
     ],
     [
      "17",
      "Qwen Qwen3.5-35B-A3B",
      "17.1",
      "36.9",
      "$0.99"
     ]
    ],
    "main_column": "Resolved Rate (%)",
    "higher_is_better": true,
    "metric": "Share of tasks resolved on the first attempt (%)",
    "source": {
     "name": "SWE-rebench (Nebius)",
     "url": "https://swe-rebench.com/",
     "as_of": "2026-07-01",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "SWE-rebench v2, default time window 15/05/2026-01/07/2026 (111 problems). Entries without data in this window omitted. as_of is window end."
   },
   "plain": "Tests AI coding tools on fresh real-world bug-fix tasks collected after models were trained, so they cannot have seen the answers."
  },
  {
   "name": "Terminal-Bench",
   "category": "coding",
   "url": "https://www.tbench.ai/",
   "paper": "https://arxiv.org/abs/2601.11868",
   "maintainer": "Stanford, Laude Institute and the Harbor project",
   "measures": "Hard, realistic tasks an agent must complete in a sandboxed terminal (building, configuring, debugging, data processing), each checked by tests, scored by task success rate per agent and model.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Versions 2.0 (Nov 2025), 2.1, 3.0 and 4.0 (Aug 2026) exist; scores are only comparable within one version and harness.",
   "id": "terminal-bench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Agent",
     "Resolution rate",
     "Cost"
    ],
    "rows": [
     [
      "1",
      "Opus 5.5 ( max )",
      "Claude Code",
      "65.8% ± 3.1%",
      "$4.7k"
     ],
     [
      "1",
      "Fable 5.2 ( max )",
      "Claude Code",
      "65.8% ± 2.8%",
      "$16.5k"
     ],
     [
      "3",
      "Gemini 4 Carbon ( high )",
      "Antigravity CLI",
      "64.5% ± 3.2%",
      "$1.4k"
     ],
     [
      "4",
      "Sonnet 5.5 ( max )",
      "Claude Code",
      "63.9% ± 3.0%",
      "$7.3k"
     ],
     [
      "5",
      "GPT-6.1 Sol ( max )",
      "Codex",
      "60.9% ± 3.0%",
      "$0.6k"
     ],
     [
      "6",
      "GPT-6 Astra ( max )",
      "Codex",
      "60.3% ± 2.8%",
      "$3.3k"
     ],
     [
      "7",
      "Kimi K3.1 Pro ( max )",
      "Claude Code",
      "59.7% ± 4.2%",
      "$1.1k"
     ],
     [
      "8",
      "Fable 5.1 ( xhigh )",
      "Claude Code",
      "57.6% ± 3.3%",
      "$4.9k"
     ],
     [
      "9",
      "Gemini 4 Hydrogen ( high )",
      "Antigravity CLI",
      "56.4% ± 3.0%",
      "$0.8k"
     ],
     [
      "10",
      "Gemini 4 Argon ( high )",
      "Antigravity CLI",
      "55.5% ± 2.8%",
      "$1.1k"
     ],
     [
      "11",
      "Opus 5 ( xhigh )",
      "Claude Code",
      "53.3% ± 3.1%",
      "$6.1k"
     ],
     [
      "11",
      "GPT-6.1 Terra ( max )",
      "Codex",
      "53.3% ± 3.1%",
      "$0.9k"
     ],
     [
      "13",
      "GPT-6 Sol ( max )",
      "Codex",
      "50.3% ± 3.2%",
      "$1.5k"
     ],
     [
      "14",
      "Muse Wonder ( xhigh )",
      "Muse Code",
      "47.0% ± 3.8%",
      "$2.5k"
     ],
     [
      "15",
      "Fable 5 ( max )",
      "Claude Code",
      "44.8% ± 3.8%",
      "$7.3k"
     ],
     [
      "16",
      "GLM-5.3 ( max )",
      "Claude Code",
      "41.2% ± 3.3%",
      "$2.7k"
     ],
     [
      "17",
      "Haiku 5.5 ( max )",
      "Claude Code",
      "40.0% ± 3.6%",
      "$1.0k"
     ],
     [
      "18",
      "Grok 4.7 ( xhigh )",
      "Grok Build",
      "38.8% ± 3.6%",
      "$3.7k"
     ],
     [
      "19",
      "GPT-5.6 Sol ( max )",
      "Codex",
      "38.5% ± 3.8%",
      "$2.5k"
     ],
     [
      "19",
      "DeepSeek V4.1 Flash 1017 ( max )",
      "Claude Code",
      "38.5% ± 3.8%",
      "$0.2k"
     ],
     [
      "21",
      "Hy4 ( max )",
      "Claude Code",
      "37.9% ± 3.5%",
      "$0.3k"
     ],
     [
      "22",
      "GLM-5.3-Flash ( max )",
      "Claude Code",
      "37.0% ± 3.6%",
      "$0.4k"
     ],
     [
      "23",
      "Muse Ember ( xhigh )",
      "Muse Code",
      "30.6% ± 3.8%",
      "$0.2k"
     ],
     [
      "24",
      "Mistral Large 4 ( max )",
      "Claude Code",
      "27.6% ± 3.9%",
      "$0.7k"
     ],
     [
      "25",
      "Qwen3.8-Max-0902 ( max )",
      "Claude Code",
      "27.0% ± 3.9%",
      "$4.6k"
     ]
    ],
    "main_column": "Resolution rate",
    "higher_is_better": true,
    "metric": "Share of Terminal-Bench 4.1 tasks completed (%) with 95% CI",
    "source": {
     "name": "Terminal-Bench (Stanford / Harbor / Laude Institute)",
     "url": "https://www.tbench.ai/leaderboard",
     "as_of": null,
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "Terminal-Bench 4.1 leaderboard; as-of date not shown on page (latest listed release Nov 12, 2026)."
   },
   "plain": "Tests whether AI agents can complete real tasks by typing commands in a computer terminal, like a developer or system administrator would."
  },
  {
   "name": "SWE Atlas",
   "category": "coding",
   "url": "https://labs.scale.com/leaderboard/swe-atlas",
   "paper": null,
   "maintainer": "Scale AI",
   "measures": "Coding agents on codebase comprehension, test writing and refactoring tasks.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "swe-atlas",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Score (%)",
     "± margin"
    ],
    "rows": [
     [
      "1",
      "Fable 5.2",
      "80.55",
      "4.80"
     ],
     [
      "2",
      "Opus 5.5",
      "77.45",
      "5.05"
     ],
     [
      "3",
      "Fable 5.1",
      "76.45",
      "5.16"
     ],
     [
      "4",
      "Gemini 4 Carbon",
      "71.45",
      "5.46"
     ],
     [
      "5",
      "Gemini 4 Argon",
      "70.08",
      "5.48"
     ],
     [
      "6",
      "GPT-6 Astra",
      "69.42",
      "5.55"
     ],
     [
      "7",
      "Sonnet 5.5",
      "68.28",
      "5.64"
     ],
     [
      "8",
      "GPT-6.1 Sol",
      "65.88",
      "5.74"
     ],
     [
      "9",
      "Muse Wonder 1.0",
      "64.83",
      "5.77"
     ],
     [
      "10",
      "Qwen 3.8 Max",
      "64.40",
      "5.71"
     ],
     [
      "11",
      "Grok 4.7",
      "62.13",
      "5.79"
     ],
     [
      "12",
      "DeepSeek V4.1 Flash",
      "62.12",
      "5.85"
     ],
     [
      "13",
      "Inkling 2 Small",
      "61.89",
      "5.83"
     ],
     [
      "14",
      "Kimi K3.1 Pro",
      "61.11",
      "5.88"
     ],
     [
      "15",
      "Kimi K3",
      "60.98",
      "5.89"
     ],
     [
      "16",
      "Haiku 5.5",
      "59.43",
      "5.92"
     ],
     [
      "17",
      "GLM 5.3",
      "58.10",
      "5.90"
     ],
     [
      "18",
      "GPT-6.1 Terra",
      "57.69",
      "5.94"
     ],
     [
      "19",
      "Gemini 3.8 Flash",
      "57.33",
      "5.83"
     ],
     [
      "20",
      "Muse Ember 1.0",
      "55.99",
      "5.98"
     ],
     [
      "21",
      "GLM 5.3 Flash",
      "55.33",
      "5.94"
     ],
     [
      "22",
      "GPT-6 Luna",
      "50.80",
      "6.02"
     ],
     [
      "23",
      "Mistral Large 4",
      "45.19",
      "5.83"
     ],
     [
      "24",
      "Muse Spark Open",
      "42.28",
      "5.90"
     ],
     [
      "25",
      "MiniMax M3",
      "39.26",
      "5.75"
     ]
    ],
    "main_column": "Score (%)",
    "higher_is_better": true,
    "metric": "Overall SWE Atlas v1.1 index score (%) with ± confidence margin",
    "source": {
     "name": "Scale AI Labs",
     "url": "https://labs.scale.com/leaderboard/swe-atlas",
     "as_of": "2026-11",
     "license": "not stated",
     "retrieved": "2026-11-17"
    },
    "notes": "SWE Atlas v1.1 (Nov 2026), 284 tasks; not comparable to v1.0. Judge: GPT-6.1 Sol majority of 3."
   },
   "plain": "Tests whether AI coding assistants can do everyday professional software work beyond fixing bugs: answering questions about a codebase, writing tests and refactoring code."
  },
  {
   "name": "OSWorld (OSWorld-Verified)",
   "category": "agents",
   "url": "http://osworld-v1.xlang.ai/",
   "paper": "https://arxiv.org/abs/2404.07972",
   "maintainer": "XLANG Lab, University of Hong Kong",
   "measures": "369 tasks in real desktop applications on a virtual machine, checked by execution-based scripts, scored by success rate.",
   "type": "leaderboard",
   "status": "active",
   "notes": "OSWorld-Verified (July 2025) fixed problematic tasks; the harder OSWorld 2.0 is listed separately.",
   "id": "osworld-osworld-verified",
   "results": {
    "columns": [
     "Rank",
     "Model / agent",
     "Organization",
     "Approach type",
     "Max steps",
     "Success rate (%)",
     "Date"
    ],
    "rows": [
     [
      "1",
      "Intelligence-Indeed Agent",
      "Intelligence Indeed",
      "Agentic framework",
      "100",
      "90.2",
      "2026-07-25"
     ],
     [
      "2",
      "claude-fable-5[1m]",
      "Anthropic",
      "General model",
      "100",
      "86.0",
      "2026-08-01"
     ],
     [
      "3",
      "Pointer Agent w/ Opus 4.7",
      "Pointer",
      "Agentic framework",
      "100",
      "83.6",
      "2026-05-21"
     ],
     [
      "4",
      "claude-opus-5[1m]",
      "Anthropic",
      "General model",
      "100",
      "83.4",
      "2026-08-01"
     ],
     [
      "5",
      "Coasty CUA v1",
      "Coasty Team",
      "General model",
      "100",
      "82.8",
      "2026-07-01"
     ],
     [
      "6",
      "Pointer Agent w/ Sonnet 4.6",
      "Pointer",
      "Agentic framework",
      "100",
      "81.5",
      "2026-05-21"
     ],
     [
      "7",
      "Muse Spark 1.1",
      "Meta Superintelligence Labs",
      "General model",
      "100",
      "80.7",
      "2026-07-09"
     ],
     [
      "8",
      "Holo3-35B-A3B",
      "H Company",
      "Specialized model",
      "100",
      "80.4±2.2",
      "2026-04-20"
     ],
     [
      "9",
      "OpenAPA w/ gemini-3.1-pro",
      "Laiye",
      "Agentic framework",
      "100",
      "78.3",
      "2026-04-17"
     ],
     [
      "10",
      "VLAA-GUI w/ Opus 4.5",
      "UCSC VLAA Lab",
      "Agentic framework",
      "100",
      "76.3",
      "2026-04-20"
     ],
     [
      "11",
      "MiniMax M3",
      "MiniMax",
      "General model",
      "100",
      "75.2",
      "2026-06-08"
     ],
     [
      "12",
      "HIPPO Agent w/ Opus 4.5",
      "Lenovo",
      "Agentic framework",
      "100",
      "74.5",
      "2026-02-25"
     ],
     [
      "13",
      "Qwen 3.7 Plus",
      "Qwen Team, Alibaba Group",
      "General model",
      "100",
      "73.3",
      "2026-05-25"
     ],
     [
      "14",
      "Kimi K2.6",
      "Moonshot AI",
      "General model",
      "100",
      "73.1",
      "2026-04-20"
     ],
     [
      "15",
      "agent s3 w/ Opus 4.5 + GPT-5 bBoN (N=10)",
      "Simular",
      "Agentic framework",
      "100",
      "72.6",
      "2025-12-11"
     ],
     [
      "16",
      "claude-sonnet-4-6",
      "Anthropic",
      "General model",
      "100",
      "72.1",
      "2026-03-08"
     ],
     [
      "17",
      "agent s3 w/ GPT-5 bBoN (N=10)",
      "Simular",
      "Agentic framework",
      "100",
      "69.9",
      "2025-10-04"
     ],
     [
      "18",
      "UiPath Screen Agent w/ Opus 4.5",
      "UiPath",
      "Agentic framework",
      "100",
      "67.1",
      "2025-12-24"
     ],
     [
      "19",
      "agent s3 w/ Opus 4.5 bBoN (N=1)",
      "Simular",
      "Agentic framework",
      "100",
      "66.0±1.1",
      "2025-12-11"
     ],
     [
      "20",
      "OS-Symphony w/ GPT-5",
      "Shanghai AI Laboratory",
      "Agentic framework",
      "50",
      "65.8",
      "2026-01-04"
     ],
     [
      "21",
      "UiPath Screen Agent w/ Opus 4.5",
      "UiPath",
      "Agentic framework",
      "50",
      "64.4",
      "2025-12-24"
     ],
     [
      "22",
      "GBOX Agent",
      "GBOX.AI",
      "Agentic framework",
      "15",
      "64.2",
      "2025-11-25"
     ],
     [
      "23",
      "GTA1 w/ GPT-5",
      "Salesforce & The Australian National University & The University of Hong Kong",
      "Agentic framework",
      "100",
      "63.4",
      "2025-10-03"
     ],
     [
      "24",
      "Kimi K2.5",
      "Moonshot AI",
      "General model",
      "100",
      "63.3",
      "2026-01-31"
     ],
     [
      "25",
      "claude-sonnet-4-5-20250929",
      "Anthropic",
      "General model",
      "100",
      "62.9",
      "2025-10-31"
     ]
    ],
    "main_column": "Success rate (%)",
    "higher_is_better": true,
    "metric": "Share of the 369 desktop tasks the agent completed successfully (%), averaged over runs where several were submitted",
    "source": {
     "name": "OSWorld team (XLANG Lab, University of Hong Kong)",
     "url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx",
     "retrieved": "2026-11-17",
     "as_of": "2026-08-01",
     "license": "not stated"
    },
    "notes": "OSWorld-Verified results file behind the official leaderboard. Rows are grouped by model + max-steps setting and averaged the same way the official page does (mean, with ± population std when several runs exist), sorted by score; all approach types (general models, specialized models, agentic frameworks) included. as_of = latest result date in the file."
   },
   "plain": "Gives an AI control of a real computer (mouse, keyboard, screen) and asks it to do everyday jobs in apps like a browser, spreadsheet or image editor. It shows how close AI is to operating a PC the way a person does."
  },
  {
   "name": "OSWorld 2.0",
   "category": "agents",
   "url": "https://osworld-v2.xlang.ai/",
   "paper": "https://arxiv.org/abs/2606.29537",
   "maintainer": "XLANG Lab, University of Hong Kong, with collaborators",
   "measures": "108 long-horizon computer-use workflows across seven professional domains, scored by binary completion within a 500-step limit plus a partial checkpoint score.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Released in 2026; tasks take humans a median of about 1.6 hours.",
   "id": "osworld-2-0",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Reasoning effort",
     "Tool setting",
     "Task release",
     "Binary accuracy (%)",
     "Partial score (%)",
     "Official"
    ],
    "rows": [
     [
      "1",
      "Claude Opus 5",
      "Anthropic",
      "max",
      "batch tool",
      "v2.1",
      "44.33",
      "77.67",
      "yes"
     ],
     [
      "2",
      "MuseSpark 1.3",
      "MuseSpark",
      "max",
      "batch tool",
      "v2.1",
      "39.25",
      "69.56",
      "yes"
     ],
     [
      "3",
      "Claude Opus 5",
      "Anthropic",
      "high",
      "batch tool",
      "v2.1",
      "36.89",
      "68.92",
      "yes"
     ],
     [
      "4",
      "Claude Opus 5",
      "Anthropic",
      "xhigh",
      "batch tool",
      "v2.1",
      "33.33",
      "73.11",
      "yes"
     ],
     [
      "5",
      "Claude Opus 5",
      "Anthropic",
      "medium",
      "batch tool",
      "v2.1",
      "33.01",
      "64.85",
      "yes"
     ],
     [
      "6",
      "Claude Opus 5",
      "Anthropic",
      "max",
      "batch tool",
      "v2026.08.08",
      "31.43",
      "68.31",
      "yes"
     ],
     [
      "7",
      "Claude Opus 5",
      "Anthropic",
      "xhigh",
      "batch tool",
      "v2026.08.08",
      "30.21",
      "67.67",
      "yes"
     ],
     [
      "8",
      "Claude Opus 5",
      "Anthropic",
      "high",
      "batch tool",
      "v2026.08.08",
      "28.98",
      "63.79",
      "yes"
     ],
     [
      "9",
      "GPT-5.6 Sol",
      "OpenAI",
      "max",
      "batch tool",
      "v2026.08.08",
      "27.34",
      "62.72",
      "yes"
     ],
     [
      "10",
      "MiniMax M3.1 Flash",
      "MiniMax",
      "adaptive",
      "batch tool",
      "v2026.08.08",
      "26.85",
      "62.01",
      "no"
     ],
     [
      "11",
      "Claude Opus 5",
      "Anthropic",
      "medium",
      "batch tool",
      "v2026.08.08",
      "25.33",
      "60.49",
      "yes"
     ],
     [
      "12",
      "Claude Opus 5",
      "Anthropic",
      "low",
      "batch tool",
      "v2026.08.08",
      "22.32",
      "54.89",
      "yes"
     ],
     [
      "13",
      "Claude Opus 4.8",
      "Anthropic",
      "max",
      "batched tool",
      "v2026.06.24",
      "20.6",
      "54.8",
      "yes"
     ],
     [
      "14",
      "Claude Opus 5",
      "Anthropic",
      "low",
      "batch tool",
      "v2.1",
      "18.81",
      "59.09",
      "yes"
     ],
     [
      "15",
      "Claude Opus 4.8",
      "Anthropic",
      "max",
      "standard",
      "v2026.06.24",
      "18.52",
      "49.33",
      "yes"
     ],
     [
      "16",
      "Claude Opus 4.7",
      "Anthropic",
      "max",
      "batched tool",
      "v2026.06.24",
      "18.2",
      "48.91",
      "yes"
     ],
     [
      "17",
      "MuseSpark 1.2",
      "MuseSpark",
      "xhigh",
      "batch tool",
      "v2.1",
      "15.84",
      "44.08",
      "no"
     ],
     [
      "18",
      "Claude Opus 4.7",
      "Anthropic",
      "max",
      "standard",
      "v2026.06.24",
      "13.9",
      "49.1",
      "yes"
     ],
     [
      "19",
      "GPT-5.5",
      "OpenAI",
      "xhigh",
      "batch tool",
      "v2026.06.24",
      "13.0",
      "49.5",
      "yes"
     ],
     [
      "20",
      "Claude Sonnet 4.6",
      "Anthropic",
      "medium",
      "standard",
      "v2026.06.24",
      "9.3",
      "33.9",
      "yes"
     ],
     [
      "21",
      "Claude Sonnet 4.6",
      "Anthropic",
      "max",
      "standard",
      "v2026.06.24",
      "8.3",
      "41.5",
      "yes"
     ],
     [
      "22",
      "MiniMax M3",
      "MiniMax",
      "enabled",
      "standard",
      "v2026.06.24",
      "4.6",
      "22.3",
      "yes"
     ],
     [
      "23",
      "Kimi 2.6",
      "Moonshot AI",
      "enabled",
      "standard",
      "v2026.06.24",
      "4.6",
      "22.1",
      "yes"
     ],
     [
      "24",
      "Qwen 3.7-Plus",
      "Alibaba",
      "thinking",
      "standard",
      "v2026.06.24",
      "2.8",
      "21.5",
      "yes"
     ]
    ],
    "main_column": "Binary accuracy (%)",
    "higher_is_better": true,
    "metric": "Share of the 108 long workflows completed fully and correctly within 500 steps (binary accuracy, %)",
    "source": {
     "name": "OSWorld 2.0 team (XLANG Lab, University of Hong Kong)",
     "url": "https://osworld-v2.xlang.ai/static/data/leaderboard/official-results.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-14",
     "license": "not stated"
    },
    "notes": "Default leaderboard view: full 108-task set, 500-step budget, foundation end-to-end GUI models. Results span several task releases (v2026.06.24, v2026.08.08, v2.1), which are not directly comparable; the site mixes them by default. Partial score = share of checkpoints passed. Organization mapped from model family as the site does."
   },
   "plain": "A much harder follow-up to OSWorld: the AI must carry out long, multi-step office and professional workflows on a real computer, jobs that take a person about an hour and a half. It shows whether AI can handle long, realistic computer work rather than quick clicks."
  },
  {
   "name": "MCPMark",
   "category": "agents",
   "url": "https://mcpmark.ai/",
   "paper": "https://arxiv.org/abs/2509.24002",
   "maintainer": null,
   "measures": "Realistic multi-step tasks carried out through MCP servers for common services, verified programmatically.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "mcpmark",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Pass@1 (%)"
    ],
    "rows": [
     [
      "1",
      "kimi-k3-max",
      "96.06"
     ],
     [
      "2",
      "gpt-5-5-xhigh",
      "92.91"
     ],
     [
      "3",
      "gpt-5-6-sol-max",
      "92.91"
     ],
     [
      "4",
      "claude-fable-5-max",
      "86.61"
     ],
     [
      "5",
      "kimi-k2-7-code",
      "81.89"
     ],
     [
      "6",
      "claude-opus-4-8-max",
      "76.38"
     ],
     [
      "7",
      "kimi-k2-6",
      "72.83"
     ],
     [
      "8",
      "deepseek-v4-pro-max",
      "71.65"
     ]
    ],
    "main_column": "Pass@1 (%)",
    "higher_is_better": true,
    "metric": "Pass@1: share of tasks solved on first try (%)",
    "source": {
     "name": "MCPMark team",
     "url": "https://mcpmark.ai/leaderboard/verified",
     "retrieved": "2026-11-17",
     "as_of": "2026-07-20",
     "license": "not stated"
    },
    "notes": "Verified leaderboard, 8 models; ranks 2 and 3 tie on score but are listed in page order."
   },
   "plain": "Tests AI agents on tasks that require using MCP tool servers such as Notion and GitHub."
  },
  {
   "name": "Toolathlon (Tool Decathlon)",
   "category": "agents",
   "url": "https://toolathlon.xyz/",
   "paper": "https://arxiv.org/abs/2510.25726",
   "maintainer": null,
   "measures": "Long-horizon tasks that span many real applications and tools, checked by execution-based verification.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "toolathlon-tool-decathlon",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Pass@1 (%)"
    ],
    "rows": [
     [
      "1",
      "GLM 5.3 Flash (max)",
      "78.4 ± 1.9"
     ],
     [
      "2",
      "Kimi K3 (max)",
      "76.5 ± 1.9"
     ],
     [
      "3",
      "Claude Opus 4.8 (max)",
      "76.2 ± 3.4"
     ],
     [
      "4",
      "Muse Spark 1.2 (xhigh)",
      "75.9 ± 1.3"
     ],
     [
      "5",
      "DeepSeek V4 Pro 0813 (max)",
      "74.4 ± 1.2"
     ],
     [
      "6",
      "GPT-5.5 (xhigh)",
      "73.5 ± 1.2"
     ],
     [
      "7",
      "Gemini 3.5 Flash (high)",
      "67.3 ± 1.2"
     ]
    ],
    "main_column": "Pass@1 (%)",
    "higher_is_better": true,
    "metric": "Pass@1 (%), with margin",
    "source": {
     "name": "Toolathlon team",
     "url": "https://toolathlon.xyz/",
     "retrieved": "2026-11-17",
     "as_of": "2026-08-30",
     "license": "not stated"
    },
    "notes": "as_of is the most recent entry date shown; page has no explicit update date."
   },
   "plain": "Tests AI agents on long, multi-step tasks across many everyday software tools."
  },
  {
   "name": "Vending-Bench 2",
   "category": "agents",
   "url": "https://andonlabs.com/evals/vending-bench-2",
   "paper": "https://arxiv.org/abs/2502.15840",
   "maintainer": "Andon Labs",
   "measures": "An agent runs a simulated vending-machine business for a simulated year (suppliers, prices, refunds, stock), scored by its money balance at the end.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Paper describes the original Vending-Bench; version 2 is documented on the site.",
   "id": "vending-bench-2",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Final money balance"
    ],
    "rows": [
     [
      "1",
      "Gemini 4 Carbon",
      "$16,484.21 ± $1,096"
     ],
     [
      "2",
      "GPT-6 Astra",
      "$15,514.70 ± $1,074"
     ],
     [
      "3",
      "Gemini 4 Hydrogen",
      "$14,549.49 ± $495"
     ],
     [
      "4",
      "GPT-6 Sol",
      "$14,427.85 ± $1,051"
     ],
     [
      "5",
      "Gemini 4 Argon",
      "$13,718.16 ± $3,100"
     ],
     [
      "6",
      "GPT-6.1 Sol",
      "$12,860.61 ± $1,043"
     ],
     [
      "7",
      "Muse Wonder Full",
      "$12,291.56 ± $885"
     ],
     [
      "8",
      "Claude Opus 5",
      "$11,181.87 ± $2,094"
     ],
     [
      "9",
      "Claude Sonnet 5.5",
      "$11,165.28 ± $1,585"
     ],
     [
      "10",
      "Claude Opus 4.7",
      "$10,936.76 ± $1,181"
     ]
    ],
    "main_column": "Final money balance",
    "higher_is_better": true,
    "metric": "Average final money balance ($)",
    "source": {
     "name": "Andon Labs",
     "url": "https://andonlabs.com/evals/vending-bench-2",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "Top 10 of 76 shown on page; no update date shown, but includes very recent models."
   },
   "plain": "An AI runs a simulated vending-machine business for a long time; tests staying sensible over many days."
  },
  {
   "name": "METR Time Horizons",
   "category": "agents",
   "url": "https://metr.org/time-horizons/",
   "paper": "https://arxiv.org/abs/2503.14499",
   "maintainer": "METR",
   "measures": "Estimates the length of software and research tasks, measured in human working time, that an AI agent can complete with 50% reliability.",
   "type": "leaderboard",
   "status": "active",
   "notes": "A summary metric with wide confidence intervals, not a single test set.",
   "id": "metr-time-horizons",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "50% time horizon (min, CI)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "claude_mythos_preview_early_inspect",
      "1044.8 min (508.9-3304.3)",
      "2026-04-07"
     ],
     [
      "2",
      "claude_opus_4_6_inspect",
      "718.8 min (316.7-3633.8)",
      "2026-02-05"
     ],
     [
      "3",
      "gemini_3_1_pro",
      "384.1 min (233.5-694.8)",
      "2026-02-19"
     ],
     [
      "4",
      "gpt_5_2",
      "352.2 min (198.1-815.2)",
      "2025-12-11"
     ],
     [
      "5",
      "gpt_5_3_codex",
      "349.5 min (194.9-816.4)",
      "2026-02-05"
     ],
     [
      "6",
      "gpt_5_4",
      "341.7 min (186.6-768.8)",
      "2026-03-05"
     ],
     [
      "7",
      "claude_opus_4_5_inspect",
      "293.0 min (161.7-623.7)",
      "2025-11-24"
     ],
     [
      "8",
      "gemini_3_pro",
      "224.3 min (139.6-379.2)",
      "2025-11-18"
     ],
     [
      "9",
      "gpt_5_1_codex_max_inspect",
      "223.7 min (134.3-396.2)",
      "2025-11-19"
     ],
     [
      "10",
      "gpt_5_2025_08_07_inspect",
      "203.0 min (112.6-405.6)",
      "2025-08-07"
     ],
     [
      "11",
      "o3_inspect",
      "119.7 min (74.6-190.9)",
      "2025-04-16"
     ],
     [
      "12",
      "claude_4_1_opus_inspect",
      "100.5 min (59.3-159.5)",
      "2025-08-05"
     ],
     [
      "13",
      "claude_4_opus_inspect",
      "100.4 min (60.0-163.5)",
      "2025-05-22"
     ],
     [
      "14",
      "claude_3_7_sonnet_inspect",
      "60.4 min (33.0-104.2)",
      "2025-02-24"
     ],
     [
      "15",
      "o1_inspect",
      "38.8 min (21.2-65.0)",
      "2024-12-05"
     ],
     [
      "16",
      "claude_3_5_sonnet_20241022_inspect",
      "20.5 min (10.1-40.8)",
      "2024-10-22"
     ],
     [
      "17",
      "o1_preview",
      "20.3 min (11.7-33.4)",
      "2024-09-12"
     ],
     [
      "18",
      "claude_3_5_sonnet_20240620_inspect",
      "11.4 min (5.5-22.4)",
      "2024-06-20"
     ],
     [
      "19",
      "gpt_4o_inspect",
      "7.0 min (4.0-12.9)",
      "2024-05-13"
     ],
     [
      "20",
      "gpt_4_1106_inspect",
      "4.0 min (1.9-8.4)",
      "2023-11-06"
     ],
     [
      "21",
      "gpt_4",
      "4.0 min (1.9-8.0)",
      "2023-03-14"
     ],
     [
      "22",
      "claude_3_opus_inspect",
      "4.0 min (1.7-8.8)",
      "2024-03-04"
     ],
     [
      "23",
      "gpt_4_turbo_inspect",
      "3.7 min (2.0-6.7)",
      "2024-04-09"
     ],
     [
      "24",
      "gpt_3_5_turbo_instruct",
      "0.6 min (0.3-1.1)",
      "2022-03-15"
     ],
     [
      "25",
      "davinci_002",
      "0.1 min (0.1-0.2)",
      "2020-05-28"
     ]
    ],
    "main_column": "50% time horizon (min, CI)",
    "higher_is_better": true,
    "metric": "50% time horizon, minutes (95% CI), as in METR data file (estimate field; unit assumed minutes)",
    "source": {
     "name": "METR",
     "url": "https://metr.org/assets/benchmark_results_1_1.yaml",
     "retrieved": "2026-11-17",
     "as_of": "2026-09-08",
     "license": "not stated"
    },
    "notes": "METR-Horizon-v1.1 data file, top 25 sorted by p50 estimate (rank is mine, not source order). Page says it is no longer actively updated; unit minutes inferred (GPT-5 ~203 min vs. page's ~2h17m)."
   },
   "plain": "Measures how long a task (in human working time) an AI can complete about half the time; shows how fast agents are improving."
  },
  {
   "name": "GDPval-AA",
   "category": "agents",
   "url": "https://artificialanalysis.ai/evaluations/gdpval-aa",
   "paper": "https://arxiv.org/abs/2510.04374",
   "maintainer": "Artificial Analysis (based on OpenAI's GDPval)",
   "measures": "Real work tasks from 44 occupations across the 9 largest US GDP sectors, where models produce file deliverables; Artificial Analysis runs an agentic version reported as an Elo-style rating.",
   "type": "leaderboard",
   "status": "active",
   "notes": "OpenAI's original GDPval is graded by industry experts; the AA version uses its own grading setup.",
   "id": "gdpval-aa",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Elo",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "Claude Opus 5.5 (Max, Default Fallback)",
      "1866",
      "Sep 2026"
     ],
     [
      "2",
      "Kimi K3.1 Pro",
      "1861",
      "Oct 2026"
     ],
     [
      "3",
      "Muse Wonder Full (Max)",
      "1857",
      "Oct 2026"
     ],
     [
      "4",
      "Claude Sonnet 5.5 (Max, Default Fallback)",
      "1838",
      "Sep 2026"
     ],
     [
      "5",
      "Claude Opus 5.5 (Xhigh, Default Fallback)",
      "1836",
      "Sep 2026"
     ],
     [
      "6",
      "Claude Fable 5.2 (Max, Default Fallback)",
      "1831",
      "Nov 2026"
     ],
     [
      "7",
      "Qwen4 Max",
      "1809",
      "Nov 2026"
     ],
     [
      "8",
      "Muse Wonder (Max)",
      "1770",
      "Oct 2026"
     ],
     [
      "9",
      "Claude Fable 5.1 (Max, Default Fallback)",
      "1765",
      "Sep 2026"
     ],
     [
      "10",
      "Claude Fable 5.2 (Xhigh, Default Fallback)",
      "1751",
      "Nov 2026"
     ],
     [
      "11",
      "DeepSeek V4.1 Flash 1017 (Max)",
      "1746",
      "Oct 2026"
     ],
     [
      "12",
      "Claude Sonnet 5.5 (Xhigh, Default Fallback)",
      "1743",
      "Sep 2026"
     ],
     [
      "13",
      "Claude Fable 5.1 (Xhigh, Default Fallback)",
      "1741",
      "Sep 2026"
     ],
     [
      "14",
      "Gemini 4 Carbon (High)",
      "1733",
      "Oct 2026"
     ],
     [
      "15",
      "Claude Opus 5 (Max)",
      "1729",
      "Jul 2026"
     ],
     [
      "16",
      "Claude Fable 5.2 (High, Default Fallback)",
      "1718",
      "Nov 2026"
     ],
     [
      "17",
      "Grok 4.7 (High)",
      "1718",
      "Sep 2026"
     ],
     [
      "18",
      "Grok 4.7 (Xhigh)",
      "1717",
      "Sep 2026"
     ],
     [
      "19",
      "Claude Opus 5.5 (High, Default Fallback)",
      "1709",
      "Sep 2026"
     ],
     [
      "20",
      "Claude Opus 5 (Xhigh)",
      "1695",
      "Jul 2026"
     ],
     [
      "21",
      "MiMo-V2.6-Pro",
      "1692",
      "Sep 2026"
     ],
     [
      "22",
      "Muse Spark 1.3 (Max)",
      "1681",
      "Sep 2026"
     ],
     [
      "23",
      "Qwen3.8 Max (0902)",
      "1674",
      "Sep 2026"
     ],
     [
      "24",
      "Grok 4.6 (Xhigh)",
      "1658",
      "Aug 2026"
     ],
     [
      "25",
      "Ling 3.1 Flash 1019",
      "1655",
      "Oct 2026"
     ]
    ],
    "main_column": "Elo",
    "higher_is_better": true,
    "metric": "Elo rating from pairwise comparisons (higher is better)",
    "source": {
     "name": "Artificial Analysis",
     "url": "https://artificialanalysis.ai/evaluations/gdpval-aa",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated (site has Terms of Use)"
    },
    "notes": "GDPval-AA v2.1, top 25 of 319 models. Site terms of use not reviewed in depth."
   },
   "plain": "Tests AI on real office-style professional tasks across many occupations, judged head-to-head."
  },
  {
   "name": "APEX-Agents",
   "category": "agents",
   "url": "https://www.mercor.com/apex/",
   "paper": "https://arxiv.org/abs/2601.14242",
   "maintainer": "Mercor",
   "measures": "480 long-horizon tasks written by investment bankers, consultants and lawyers, done in realistic file and tool environments, scored by Pass@1 against rubrics.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "apex-agents",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Score"
    ],
    "rows": [
     [
      "1",
      "Gemini 4 Carbon (High)",
      "84.2% ±4.3%"
     ],
     [
      "2",
      "Gemini 4 Argon (High)",
      "82.4% ±4.3%"
     ],
     [
      "3",
      "Fable 5.2 (Max)",
      "80.1% ±4.7%"
     ],
     [
      "4",
      "Gemini 4 Hydrogen (xHigh)",
      "78.0% ±4.8%"
     ],
     [
      "5",
      "Hy4 (High)",
      "77.6% ±4.5%"
     ]
    ],
    "main_column": "Score",
    "higher_is_better": true,
    "metric": "Score (%) with margin",
    "source": {
     "name": "Mercor",
     "url": "https://www.mercor.com/apex/",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "Only top 5 shown on landing page; full leaderboard behind a link. No update date shown."
   },
   "plain": "Tests AI agents on long professional tasks like those of bankers, consultants and lawyers."
  },
  {
   "name": "Remote Labor Index (RLI)",
   "category": "agents",
   "url": "https://labs.scale.com/leaderboard/rli",
   "paper": "https://arxiv.org/abs/2510.26787",
   "maintainer": "Scale AI and Center for AI Safety",
   "measures": "Real paid remote freelance projects, scored by the share of projects where the AI's deliverable would be accepted as the finished work.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "remote-labor-index-rli",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Automation rate (%)"
    ],
    "rows": [
     [
      "1",
      "Fable 5.2 (NEW)",
      "41.25"
     ],
     [
      "2",
      "Opus 5.5",
      "21.25"
     ],
     [
      "3",
      "GPT 6 Astra",
      "20.83"
     ],
     [
      "4",
      "Fable 5.1",
      "17.92"
     ],
     [
      "5",
      "Gemini Argon 4",
      "17.08"
     ],
     [
      "6",
      "Fable 5",
      "15.83"
     ],
     [
      "7",
      "Deepseek V4.1 Flash (NEW)",
      "9.58"
     ],
     [
      "8",
      "Opus 4.8",
      "8.33"
     ],
     [
      "9",
      "Codex GPT 5.5",
      "6.25"
     ],
     [
      "10",
      "Gemini 3.8 Flash",
      "5.83"
     ],
     [
      "11",
      "Gemini 3.7 Flash",
      "5.00"
     ],
     [
      "12",
      "claude-opus-4-6 (CoWork)",
      "4.17"
     ],
     [
      "13",
      "claude-opus-4-5-20251101-thinking",
      "3.75"
     ],
     [
      "14",
      "Manus_1.6 (Max)",
      "2.92"
     ],
     [
      "15",
      "gpt-5.2-2025-12-11 (medium)",
      "2.50"
     ],
     [
      "15",
      "Manus 1.0",
      "2.50"
     ],
     [
      "15",
      "Manus 1.5",
      "2.50"
     ],
     [
      "18",
      "claude-4-5-Sonnet",
      "2.08"
     ],
     [
      "18",
      "gpt-5.2-2025-12-11 (default)",
      "2.08"
     ],
     [
      "20",
      "gpt-5-2025-08-07",
      "1.67"
     ],
     [
      "21",
      "ChatGPT agent",
      "1.25"
     ],
     [
      "21",
      "gemini-3-pro-preview",
      "1.25"
     ],
     [
      "23",
      "gemini-2.5-pro-preview-06-05",
      "0.83"
     ]
    ],
    "main_column": "Automation rate (%)",
    "higher_is_better": true,
    "metric": "Automation rate: share of projects completed acceptably (%)",
    "source": {
     "name": "Scale AI / Center for AI Safety",
     "url": "https://labs.scale.com/leaderboard/rli",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "No update date shown; includes entries tagged NEW. Tied ranks as on page."
   },
   "plain": "Tests whether AI can complete real paid freelance projects well enough that a client would accept the work."
  },
  {
   "name": "ScreenSpot-Pro",
   "category": "agents",
   "url": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding",
   "paper": "https://arxiv.org/abs/2504.07981",
   "maintainer": "ScreenSpot-Pro authors",
   "measures": "Locating target interface elements in high-resolution screenshots of professional software, scored by click accuracy.",
   "type": "dataset",
   "status": "active",
   "notes": "Measures GUI grounding only, not full task completion.",
   "id": "screenspot-pro",
   "results": {
    "columns": [
     "Rank",
     "Model / system",
     "Overall (%)",
     "Text targets (%)",
     "Icon targets (%)"
    ],
    "rows": [
     [
      "1",
      "GroundCascade (Indeed-UI-32B + KV-Ground-8B)",
      "85.14",
      "92.12",
      "73.84"
     ],
     [
      "2",
      "GroundCascade (Indeed-UI-32B + Indeed-UI-8B)",
      "85.01",
      "92.43",
      "73.01"
     ],
     [
      "3",
      "Indeed-UI-32B-zoomin",
      "82.73",
      "90.38",
      "70.36"
     ],
     [
      "4",
      "Indeed-UI-8B-zoom-in",
      "80.96",
      "89.87",
      "66.56"
     ],
     [
      "5",
      "KV-Ground-8B + Qwen3.5-27B Consistency Router",
      "80.90",
      "90.38",
      "65.56"
     ],
     [
      "6",
      "KV-Ground-GuiOwl1.5-0315-8B-ZoomIn",
      "80.52",
      "89.97",
      "65.23"
     ],
     [
      "7",
      "Holo2-235B-A22B (Agentic)",
      "78.49",
      "78.81",
      "77.98"
     ],
     [
      "8",
      "MAI-UI-32B (MVP)",
      "77.46",
      "88.25",
      "60.00"
     ],
     [
      "9",
      "AdaZoom-GUI-4B (cond. zoom-in, refine w/ Qwen3.5-397B)",
      "76.79",
      "90.07",
      "55.30"
     ],
     [
      "10",
      "KV-Ground-GuiOwl1.5-4B-0228-ZoomIn",
      "76.41",
      "85.57",
      "61.59"
     ],
     [
      "11",
      "Holo2-30B-A3B (Agentic)",
      "75.21",
      "76.46",
      "73.18"
     ],
     [
      "12",
      "MVP_Qwen3VL-32B",
      "74.15",
      "88.04",
      "51.66"
     ],
     [
      "13",
      "MAI-UI-32B (Zoom In)",
      "73.50",
      "85.36",
      "54.30"
     ],
     [
      "14",
      "Indeed-UI-8B",
      "73.37",
      "83.62",
      "56.79"
     ],
     [
      "15",
      "Indeed-UI-32B",
      "73.31",
      "85.26",
      "53.97"
     ],
     [
      "16",
      "KV-Ground-GuiOwl1.5-0315-8B",
      "73.18",
      "83.52",
      "56.46"
     ],
     [
      "17",
      "Duvo Eye-1 (Holo-3.1-35B-A3B + LoRA)",
      "72.93",
      "81.37",
      "59.27"
     ],
     [
      "18",
      "HuzzleWorld-3B (Zoom In)",
      "72.42",
      "79.22",
      "61.42"
     ],
     [
      "19",
      "MAI-UI-8B (Zoom In)",
      "71.92",
      "82.09",
      "55.46"
     ],
     [
      "20",
      "Holo2-8B (Agentic)",
      "71.41",
      "71.03",
      "72.02"
     ],
     [
      "21",
      "HuzzleWorld-2B (Zoom In)",
      "70.84",
      "80.35",
      "55.46"
     ],
     [
      "22",
      "AdaZoom-GUI-4B (conditional zoom-in)",
      "70.59",
      "85.36",
      "46.69"
     ],
     [
      "23",
      "Holo2-235B-A22B",
      "70.59",
      "70.11",
      "71.36"
     ],
     [
      "24",
      "KV-Ground-Qwen3VL-4B-ZoomIn",
      "70.27",
      "84.34",
      "47.52"
     ],
     [
      "25",
      "UI-Venus-1-5-30B-A3B",
      "69.64",
      "81.17",
      "50.99"
     ]
    ],
    "main_column": "Overall (%)",
    "higher_is_better": true,
    "metric": "Share of clicks that landed on the correct on-screen element (%), micro-average",
    "source": {
     "name": "ScreenSpot-Pro team (GUI-Agent grounding leaderboard)",
     "url": "https://gui-agent.github.io/grounding-leaderboard/results/screenspot_pro.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-10 (last repo commit)",
     "license": "Apache-2.0"
    },
    "notes": "Ordered by overall average (page default sort). Entries are mostly self-submitted, including multi-model pipelines and zoom-in methods; values converted from fractions to %. Measures GUI grounding only, not full task completion."
   },
   "plain": "Shows an AI a screenshot of professional software (like CAD or video editors) and asks it to point to the right button or text. It tests whether an AI can \"see\" where to click, a basic step for computer-using agents."
  },
  {
   "name": "Context Arena (MRCR)",
   "category": "long-context",
   "url": "https://contextarena.ai/",
   "paper": null,
   "maintainer": "Context Arena (independent)",
   "measures": "Leaderboard for OpenAI's MRCR test, where the model must reproduce a specific one of several similar requests buried in a long conversation, scored at multiple context lengths.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Uses its own standardized settings, so numbers differ from MRCR scores reported by labs.",
   "id": "context-arena-mrcr",
   "results": {
    "columns": [
     "Rank",
     "Model [reasoning mode]",
     "Score at 64k-128k tokens (%)",
     "Overall score (%)"
    ],
    "rows": [
     [
      "1",
      "openai/gpt-5.6-sol [max]",
      "92.4",
      "85.5"
     ],
     [
      "2",
      "qwen/qwen3.8-max [xhigh]",
      "92.3",
      "73.7"
     ],
     [
      "3",
      "anthropic/claude-opus-5 [max]",
      "91.3",
      "77.3"
     ],
     [
      "4",
      "google/gemini-3.7-flash [high]",
      "88.6",
      "83.0"
     ],
     [
      "5",
      "qwen/qwen3.8-max [medium]",
      "87.3",
      "72.1"
     ],
     [
      "6",
      "openai/gpt-5.5 [xhigh]",
      "85.8",
      "79.8"
     ],
     [
      "7",
      "openai/gpt-5.5 [medium]",
      "83.0",
      "78.6"
     ],
     [
      "8",
      "google/gemini-3.7-flash [medium]",
      "82.1",
      "79.6"
     ],
     [
      "9",
      "qwen/qwen3.8-27b [low]",
      "81.8",
      "92.8"
     ],
     [
      "10",
      "qwen/qwen3.8-max [low]",
      "81.5",
      "71.6"
     ],
     [
      "11",
      "openai/gpt-5.6-terra [max]",
      "80.2",
      "76.6"
     ],
     [
      "12",
      "openai/gpt-5.5 [high]",
      "79.3",
      "79.0"
     ],
     [
      "13",
      "qwen/qwen3.8-27b [medium]",
      "78.9",
      "94.0"
     ],
     [
      "14",
      "google/gemini-3.7-flash [low]",
      "78.3",
      "74.5"
     ],
     [
      "15",
      "google/gemini-3.6-flash [high]",
      "77.3",
      "73.3"
     ],
     [
      "16",
      "anthropic/claude-opus-4.8 [max]",
      "75.2",
      "72.6"
     ],
     [
      "17",
      "google/gemini-3.6-flash [medium]",
      "74.5",
      "69.9"
     ],
     [
      "18",
      "qwen/qwen3.8-27b [xhigh]",
      "74.3",
      "90.2"
     ],
     [
      "19",
      "anthropic/claude-sonnet-4.6 [low]",
      "72.6",
      "70.4"
     ],
     [
      "20",
      "anthropic/claude-opus-4.6 [medium]",
      "72.5",
      "73.1"
     ],
     [
      "21",
      "anthropic/claude-sonnet-4.6 [high]",
      "72.4",
      "70.5"
     ],
     [
      "22",
      "anthropic/claude-sonnet-4.6 [medium]",
      "71.7",
      "69.6"
     ],
     [
      "23",
      "qwen/qwen3.6-max-preview [enabled]",
      "71.6",
      "88.1"
     ],
     [
      "24",
      "z-ai/glm-5.3 [high]",
      "69.3",
      "70.4"
     ],
     [
      "25",
      "anthropic/claude-opus-4.6 [low]",
      "68.8",
      "72.4"
     ]
    ],
    "main_column": "Score at 64k-128k tokens (%)",
    "higher_is_better": true,
    "metric": "Average retrieval score (%) on the 8-needle task for conversations of 64k-128k tokens",
    "source": {
     "name": "Context Arena",
     "url": "https://contextarena.ai/ (data from https://contextarena.ai/api/leaderboard?needles=8)",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "8-needle, full mode, from the site API (184 entries). Ranked here by the 64k-128k token bin because every model has it; the site API gives no ordering and no update date. Overall score averages over whichever lengths each model was tested at, so is not directly comparable. The 2-needle view returned empty at retrieval."
   },
   "plain": "Hides several near-identical items in a very long conversation and asks the AI to retrieve a specific one, checking whether it can find the right detail in a huge pile of text."
  },
  {
   "name": "MultiChallenge",
   "category": "instruction-following",
   "url": "https://labs.scale.com/leaderboard/multichallenge",
   "paper": "https://arxiv.org/abs/2501.17399",
   "maintainer": "Scale AI",
   "measures": "Multi-turn conversations that test retaining instructions, using earlier context, handling edits and staying self-consistent, graded by an LLM judge with rubrics.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "multichallenge",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Score (%)",
     "Margin (±)"
    ],
    "rows": [
     [
      "1",
      "gpt 6.1 Sol",
      "81.95",
      "4.16"
     ],
     [
      "2",
      "Gemini 4 Argon",
      "78.70",
      "5.30"
     ],
     [
      "3",
      "Muse Spark",
      "75.52",
      "4.05"
     ],
     [
      "4",
      "Muse Spark 1.1",
      "75.30",
      "0.60"
     ],
     [
      "5",
      "Deepseek V4p1 Flash 1017",
      "74.90",
      "5.30"
     ],
     [
      "6",
      "GPT 6.1 Terra",
      "74.20",
      "5.40"
     ],
     [
      "6",
      "Muse Spark Open",
      "74.20",
      "5.60"
     ],
     [
      "8",
      "gemini-3.1-pro-preview",
      "71.37",
      "1.74"
     ],
     [
      "9",
      "gpt-5.4-pro-2026-03-05",
      "69.23",
      "3.05"
     ],
     [
      "10",
      "Claude Haiku 5.5",
      "67.67",
      "5.34"
     ],
     [
      "11",
      "gemini-3-pro-preview",
      "65.67",
      "2.20"
     ],
     [
      "12",
      "gpt-5.1-2025-11-13-thinking",
      "63.41",
      "2.11"
     ],
     [
      "13",
      "gpt-5-thinking",
      "63.19",
      "1.63"
     ],
     [
      "14",
      "o3-pro-2025-06-10-reasoning-high",
      "62.40",
      "2.49"
     ],
     [
      "15",
      "kimi-k2.5",
      "61.39",
      "0.46"
     ],
     [
      "16",
      "gemini-3.1-flash-lite-preview",
      "60.61",
      "0.38"
     ],
     [
      "17",
      "gpt-5-mini-thinking",
      "58.99",
      "4.07"
     ],
     [
      "18",
      "claude-opus-4-5-20251101-thinking",
      "58.97",
      "0.42"
     ],
     [
      "19",
      "claude-4-opus-thinking",
      "58.62",
      "0.77"
     ],
     [
      "20",
      "claude-opus-4-1-20250805-thinking",
      "57.20",
      "0.95"
     ],
     [
      "21",
      "claude-4-sonnet-thinking",
      "57.11",
      "0.81"
     ],
     [
      "22",
      "o3-2025-04-16-reasoning-high",
      "56.62",
      "3.32"
     ],
     [
      "23",
      "claude-opus-4-6 (Non-Thinking)",
      "56.02",
      "1.08"
     ],
     [
      "24",
      "kimi-k2-thinking",
      "55.42",
      "1.88"
     ],
     [
      "25",
      "claude-sonnet-4-5-20250929-thinking",
      "55.32",
      "1.47"
     ]
    ],
    "main_column": "Score (%)",
    "higher_is_better": true,
    "metric": "Share of multi-turn conversation test cases answered correctly (%), with a 95% confidence margin",
    "source": {
     "name": "Scale AI (Scale Labs)",
     "url": "https://labs.scale.com/leaderboard/multichallenge",
     "retrieved": "2026-11-17",
     "as_of": "2026-11 (entries tagged NEW; judge update Feb 3, 2026)",
     "license": "not stated"
    },
    "notes": "Rank is rank-by-upper-bound (ties possible). Scored by an LLM judge (Gemini 2.5 Pro since Feb 2026). Model names as published; terms of reuse not stated on page."
   },
   "plain": "Tests whether an AI can hold a realistic multi-turn conversation: remembering what you said earlier, following your instructions across turns, and reasoning over the whole chat."
  },
  {
   "name": "Vectara Hallucination Leaderboard",
   "category": "factuality",
   "url": "https://github.com/vectara/hallucination-leaderboard",
   "paper": null,
   "maintainer": "Vectara",
   "measures": "Hallucination rate when models summarize short documents, judged by Vectara's HHEM classifier.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Covers summarization faithfulness only, judged by one vendor's model.",
   "id": "vectara-hallucination-leaderboard",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Hallucination Rate",
     "Factual Consistency Rate",
     "Answer Rate",
     "Average Summary Length (Words)"
    ],
    "rows": [
     [
      "1",
      "antgroup/finix_s1_32b",
      "1.8 %",
      "98.2 %",
      "99.5 %",
      "172.4"
     ],
     [
      "2",
      "openai/gpt-5.4-nano-2026-03-17",
      "3.1 %",
      "96.9 %",
      "100.0 %",
      "144.4"
     ],
     [
      "3",
      "google/gemini-2.5-flash-lite",
      "3.3 %",
      "96.7 %",
      "99.5 %",
      "95.7"
     ],
     [
      "4",
      "microsoft/Phi-4",
      "3.7 %",
      "96.3 %",
      "80.7 %",
      "120.9"
     ],
     [
      "5",
      "meta-llama/Llama-3.3-70B-Instruct-Turbo",
      "4.1 %",
      "95.9 %",
      "99.5 %",
      "64.6"
     ],
     [
      "6",
      "deepseek-ai/DeepSeek-V4.1-Flash",
      "4.2 %",
      "95.8 %",
      "99.9 %",
      "108.2"
     ],
     [
      "7",
      "snowflake/snowflake-arctic-instruct",
      "4.3 %",
      "95.7 %",
      "62.7 %",
      "81.4"
     ],
     [
      "8",
      "google/gemma-3-12b-it",
      "4.4 %",
      "95.6 %",
      "97.4 %",
      "89.7"
     ],
     [
      "9",
      "mistralai/mistral-large-2411",
      "4.5 %",
      "95.5 %",
      "99.9 %",
      "85.0"
     ],
     [
      "10",
      "qwen/qwen3-8b",
      "4.8 %",
      "95.2 %",
      "99.9 %",
      "83.6"
     ],
     [
      "11",
      "amazon/nova-pro-v1:0",
      "5.1 %",
      "94.9 %",
      "99.3 %",
      "66.2"
     ],
     [
      "12",
      "amazon/nova-2-lite-v1:0",
      "5.1 %",
      "94.9 %",
      "99.6 %",
      "94.1"
     ],
     [
      "13",
      "mistralai/mistral-small-2501",
      "5.1 %",
      "94.9 %",
      "97.9 %",
      "98.8"
     ],
     [
      "14",
      "ibm-granite/granite-4.0-h-small",
      "5.2 %",
      "94.8 %",
      "100.0 %",
      "107.4"
     ],
     [
      "15",
      "google/gemma-4-26b-a4b-it",
      "5.2 %",
      "94.8 %",
      "99.8 %",
      "67.1"
     ],
     [
      "16",
      "ai21labs/jamba-mini-2",
      "5.3 %",
      "94.7 %",
      "99.6 %",
      "109.4"
     ],
     [
      "17",
      "deepseek-ai/DeepSeek-V3.2-Exp",
      "5.3 %",
      "94.7 %",
      "96.6 %",
      "64.6"
     ],
     [
      "18",
      "qwen/qwen3-14b",
      "5.4 %",
      "94.6 %",
      "99.9 %",
      "111.1"
     ],
     [
      "19",
      "amazon/nova-micro-v1:0",
      "5.5 %",
      "94.5 %",
      "100.0 %",
      "100.0"
     ],
     [
      "20",
      "deepseek-ai/DeepSeek-V3.1",
      "5.5 %",
      "94.5 %",
      "94.5 %",
      "63.7"
     ],
     [
      "21",
      "openai/gpt-5.4-mini-2026-03-17",
      "5.5 %",
      "94.5 %",
      "100.0 %",
      "54.7"
     ],
     [
      "22",
      "openai/gpt-4.1-2025-04-14",
      "5.6 %",
      "94.4 %",
      "99.9 %",
      "91.7"
     ],
     [
      "23",
      "qwen/qwen3-4b",
      "5.7 %",
      "94.3 %",
      "99.9 %",
      "104.7"
     ],
     [
      "24",
      "xai-org/grok-3",
      "5.8 %",
      "94.2 %",
      "93.0 %",
      "95.9"
     ],
     [
      "25",
      "qwen/qwen3-32b",
      "5.9 %",
      "94.1 %",
      "99.9 %",
      "115.8"
     ]
    ],
    "main_column": "Hallucination Rate",
    "higher_is_better": false,
    "metric": "Share of summaries containing information not supported by the source document (hallucination rate, %)",
    "source": {
     "name": "Vectara",
     "url": "https://github.com/vectara/hallucination-leaderboard/blob/main/README.md",
     "retrieved": "2026-11-17",
     "as_of": "2026-10-22",
     "license": "Apache-2.0 (repository license)"
    },
    "notes": "Top 25 by lowest hallucination rate, in README order. Judged by Vectara's own HHEM classifier on its current (new) document set; covers summarization faithfulness only. Answer rate = share of documents the model agreed to summarize."
   },
   "plain": "Asks AI models to summarize short documents and uses an automated checker to count how often the summary adds things that are not in the original text. It shows how likely a model is to make things up when it should just be faithful to a source."
  },
  {
   "name": "AA-Omniscience",
   "category": "factuality",
   "url": "https://artificialanalysis.ai/evaluations/omniscience",
   "paper": "https://arxiv.org/abs/2511.13029",
   "maintainer": "Artificial Analysis",
   "measures": "6,000 knowledge questions across 42 topics, scored with an index from -100 to 100 that penalizes wrong answers and rewards abstaining when unsure.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "aa-omniscience",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Omniscience Index",
     "Accuracy (%)"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2 (Max, Default Fallback)",
      "67",
      "79"
     ],
     [
      "2",
      "Claude Fable 5.2 (Xhigh, Default Fallback)",
      "65",
      "78"
     ],
     [
      "3",
      "Claude Fable 5.2 (High, Default Fallback)",
      "64",
      "77"
     ]
    ],
    "main_column": "Omniscience Index",
    "higher_is_better": true,
    "metric": "Omniscience Index (-100 to 100): rewards correct answers, penalizes wrong answers, neutral for abstaining; higher is better",
    "source": {
     "name": "Artificial Analysis",
     "url": "https://artificialanalysis.ai/evaluations/omniscience",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "Only the top 3 are shown in the page's rendered text; the full table is client-side and the embedded data could not be reliably matched to model names, so rows are limited to these 3 (values rounded as displayed). Benchmark v1.1 (Oct 2026)."
   },
   "plain": "Tests how much factual knowledge an AI really has across 42 topics, and punishes it for confidently making things up instead of saying 'I don't know'. A score of 0 means as many wrong as right answers."
  },
  {
   "name": "Arena (formerly LMArena / Chatbot Arena)",
   "category": "arenas",
   "url": "https://arena.ai/leaderboard",
   "paper": "https://arxiv.org/abs/2403.04132",
   "maintainer": "Arena Intelligence (from LMSYS, UC Berkeley)",
   "measures": "Anonymous side-by-side battles on real user prompts; votes are fitted into Bradley-Terry ratings, with separate boards for text, vision, documents, search, WebDev, agents, image and video.",
   "type": "arena",
   "status": "active",
   "notes": "Ratings reflect user preference, which can reward style and length; labs can test private variants before release.",
   "id": "arena-formerly-lmarena-chatbot-arena",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Score",
     "±CI",
     "Votes",
     "License"
    ],
    "rows": [
     [
      "1",
      "Gemini 4 Carbon (High)",
      "Google",
      "1526",
      "±10",
      "4,273",
      "Proprietary"
     ],
     [
      "2",
      "Gemini 4 Argon (High)",
      "Google",
      "1522",
      "±5",
      "20,779",
      "Proprietary"
     ],
     [
      "3",
      "Muse Wonder (Max)",
      "Meta",
      "1511",
      "±6",
      "12,696",
      "Proprietary"
     ],
     [
      "4",
      "Claude Opus 5.5 (High)",
      "Anthropic",
      "1507",
      "±6",
      "15,793",
      "Proprietary"
     ],
     [
      "5",
      "Claude Fable 5 (High)",
      "Anthropic",
      "1504",
      "±4",
      "45,469",
      "Proprietary"
     ],
     [
      "6",
      "Claude Opus 4.6 (High)",
      "Anthropic",
      "1503",
      "±3",
      "84,375",
      "Proprietary"
     ],
     [
      "7",
      "Claude Fable 5.1 (Max)",
      "Anthropic",
      "1502",
      "±5",
      "20,511",
      "Proprietary"
     ],
     [
      "8",
      "Claude Opus 4.7 (High)",
      "Anthropic",
      "1501",
      "±4",
      "64,260",
      "Proprietary"
     ],
     [
      "9",
      "Gemini 3.8 Flash (High)",
      "Google",
      "1497",
      "±4",
      "62,675",
      "Proprietary"
     ],
     [
      "10",
      "Muse Spark 1.3 (Max)",
      "Meta",
      "1497",
      "±5",
      "26,806",
      "Proprietary"
     ],
     [
      "11",
      "Claude Opus 4.6",
      "Anthropic",
      "1496",
      "±3",
      "89,177",
      "Proprietary"
     ],
     [
      "12",
      "Claude Opus 4.7",
      "Anthropic",
      "1494",
      "±4",
      "65,309",
      "Proprietary"
     ],
     [
      "13",
      "Muse Spark 1.2 (xHigh)",
      "Meta",
      "1493",
      "±6",
      "15,563",
      "Proprietary"
     ],
     [
      "14",
      "Muse Spark 1.1",
      "Meta",
      "1491",
      "±4",
      "45,519",
      "Proprietary"
     ],
     [
      "15",
      "Gemini 4 Hydrogen (High)",
      "Google",
      "1490",
      "±9",
      "4,741",
      "Proprietary"
     ],
     [
      "16",
      "Claude Opus 5 (High)",
      "Anthropic",
      "1490",
      "±4",
      "73,632",
      "Proprietary"
     ],
     [
      "17",
      "Gemini 3.7 Flash (High)",
      "Google",
      "1489",
      "±5",
      "33,251",
      "Proprietary"
     ],
     [
      "18",
      "Muse Spark",
      "Meta",
      "1489",
      "±6",
      "13,985",
      "Proprietary"
     ],
     [
      "19",
      "Claude Opus 5 (Max)",
      "Anthropic",
      "1489",
      "±5",
      "35,670",
      "Proprietary"
     ],
     [
      "20",
      "Kimi K3 (Max)",
      "Moonshot",
      "1487",
      "±4",
      "41,402",
      "Kimi K3 license"
     ],
     [
      "21",
      "Gemini 3.1 Pro Preview",
      "Google",
      "1487",
      "±3",
      "132,181",
      "Proprietary"
     ],
     [
      "22",
      "Gemini 3 Pro",
      "Google",
      "1486",
      "±4",
      "41,803",
      "Proprietary"
     ],
     [
      "23",
      "GPT-5.6 Sol (xHigh)",
      "OpenAI",
      "1485",
      "±4",
      "43,925",
      "Proprietary"
     ],
     [
      "24",
      "Kimi K3.1 Pro (Max)",
      "Moonshot",
      "1485",
      "±10",
      "3,995",
      "Kimi K3.1 Pro License"
     ],
     [
      "25",
      "Qwen3.8-Max",
      "Alibaba",
      "1484",
      "±5",
      "36,936",
      "Proprietary"
     ]
    ],
    "main_column": "Score",
    "higher_is_better": true,
    "metric": "Arena score: a rating computed from head-to-head user votes (higher means preferred more often)",
    "source": {
     "name": "Arena Intelligence (arena.ai, formerly LMArena / LMSYS)",
     "url": "https://arena.ai/leaderboard/text",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-11",
     "license": "not stated"
    },
    "notes": "Text leaderboard only (Arena also has vision, WebDev, search, image and video boards). Rank is position in the published list; models ranked 1, 2, 9, 15 and 17 were marked 'Preliminary' (few votes). The page shows Nov 11, 2026 next to the vote totals; no explicit 'last updated' label. The CI level is not stated on the page. Read via a page-to-text fetch because direct download was blocked (HTTP 403)."
   },
   "plain": "Real people type a question, get two anonymous answers from different AI chatbots, and vote for the better one. Millions of these votes are turned into a ranking of which chatbots people prefer."
  },
  {
   "name": "Artificial Analysis Image & Video Arenas",
   "category": "arenas",
   "url": "https://artificialanalysis.ai/text-to-image/arena",
   "paper": null,
   "maintainer": "Artificial Analysis",
   "measures": "Blind pairwise human votes between outputs of image and video generation models for the same prompt, converted to Elo ratings.",
   "type": "arena",
   "status": "active",
   "notes": null,
   "id": "artificial-analysis-image-video-arenas",
   "results": null,
   "plain": "People are shown two AI-made pictures (or videos) for the same request without knowing which model made them, and pick the better one. The votes become a ranking of image and video generators."
  },
  {
   "name": "Design Arena",
   "category": "arenas",
   "url": "https://www.designarena.ai/",
   "paper": null,
   "maintainer": null,
   "measures": "Crowdsourced votes comparing AI-generated designs, including websites, games, slides, images, video and audio, with separate leaderboards per output type.",
   "type": "arena",
   "status": "active",
   "notes": null,
   "id": "design-arena",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Elo",
     "Win Rate",
     "Battles"
    ],
    "rows": [
     [
      "1",
      "Claude Opus 5.5 (high)",
      "Anthropic",
      "1445",
      "69.1%",
      "2,670"
     ],
     [
      "2",
      "GPT-6 Astra (max)",
      "OpenAI",
      "1399",
      "67.2%",
      "2,526"
     ],
     [
      "3",
      "Claude Fable 5.2",
      "Anthropic",
      "1367",
      "69.0%",
      "1,738"
     ],
     [
      "4",
      "GPT-6.1 Sol (max)",
      "OpenAI",
      "1366",
      "51.3%",
      "2,069"
     ],
     [
      "5",
      "Qwen4-Max",
      "Alibaba",
      "1360",
      "62.5%",
      "3,474"
     ],
     [
      "6",
      "Claude Fable 5.1",
      "Anthropic",
      "1336",
      "59.5%",
      "5,155"
     ],
     [
      "7",
      "Inkling 2 Small",
      "Thinking Machines",
      "1311",
      "57.2%",
      "7,217"
     ],
     [
      "8",
      "Gemini 4 Carbon",
      "Google",
      "1306",
      "60.5%",
      "1,054"
     ],
     [
      "9",
      "Kimi K3.1 Pro",
      "Moonshot AI",
      "1297",
      "59.4%",
      "4,489"
     ],
     [
      "10",
      "Claude Sonnet 5.5",
      "Anthropic",
      "1290",
      "57.5%",
      "3,030"
     ],
     [
      "11",
      "DeepSeek-V4.1-Flash (1017)",
      "DeepSeek",
      "1288",
      "47.8%",
      "404"
     ],
     [
      "12",
      "Muse Spark 1.3 Max",
      "Meta",
      "1279",
      "56.9%",
      "13,091"
     ],
     [
      "13",
      "Grok 4.7",
      "xAI",
      "1273",
      "54.4%",
      "1,506"
     ],
     [
      "14",
      "Claude Fable 5",
      "Anthropic",
      "1272",
      "60.5%",
      "13,563"
     ],
     [
      "15",
      "Gemini 4 Hydrogen",
      "Google",
      "1268",
      "56.6%",
      "380"
     ]
    ],
    "main_column": "Elo",
    "higher_is_better": true,
    "metric": "Elo rating from head-to-head user votes (higher means the model's output was preferred more often)",
    "source": {
     "name": "DesignArena",
     "url": "https://www.designarena.ai/leaderboard",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "not stated"
    },
    "notes": "'Overall Agentic' board (combines web apps, full-stack, game dev, React Native, Android, slides and spreadsheets). Only the first 15 rows are shown on the page without expanding; separate boards exist per output type. Page says 'Updated Nov 17, 2026, 10:31 AM UTC'."
   },
   "plain": "People ask several AI models to build the same thing, such as a website, game or slide deck, then vote on which result looks and works better. The votes become a ranking of which models are best at making designs and apps."
  },
  {
   "name": "MMMU / MMMU-Pro",
   "category": "multimodal",
   "url": "https://mmmu-benchmark.github.io/",
   "paper": "https://arxiv.org/abs/2311.16502",
   "maintainer": "MMMU team",
   "measures": "About 11,500 college-level questions with images across 30 subjects; MMMU-Pro (arXiv 2409.02813) adds ten-option answers and vision-only inputs. Scored by accuracy.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "mmmu-mmmu-pro",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Type",
     "Date",
     "MMMU-Pro Overall",
     "MMMU Val Overall"
    ],
    "rows": [
     [
      "1",
      "Chance Vision 1.5",
      "proprietary",
      "2026-07-01",
      "86.9",
      "-"
     ],
     [
      "2",
      "GPT-5.4 Thinking w/ tools",
      "proprietary",
      "2026-03-05",
      "82.1",
      "-"
     ],
     [
      "3",
      "GPT-5.4 Thinking w/o tools",
      "proprietary",
      "2026-03-05",
      "81.2",
      "-"
     ],
     [
      "4",
      "Gemini 3.0 Pro",
      "proprietary",
      "2025-11-18",
      "81.0",
      "-"
     ],
     [
      "5",
      "Gemini 3.1 Pro Thinking (High)",
      "proprietary",
      "2026-02-19",
      "80.5",
      "-"
     ],
     [
      "6",
      "Muse Spark Thinking",
      "proprietary",
      "2026-04-08",
      "80.4",
      "-"
     ],
     [
      "7",
      "GPT-5.2 Thinking w/o Python",
      "proprietary",
      "2025-12-11",
      "80.4",
      "-"
     ],
     [
      "8",
      "GPT-5.2 Thinking w/o tools",
      "proprietary",
      "2025-12-11",
      "79.5",
      "-"
     ],
     [
      "9",
      "GPT-5.1 Thinking",
      "proprietary",
      "2025-11-13",
      "79.0",
      "-"
     ],
     [
      "10",
      "GPT-5 w/ thinking",
      "proprietary",
      "2025-08-07",
      "78.4",
      "84.2"
     ],
     [
      "11",
      "Claude Opus 4.6 w/ tools",
      "proprietary",
      "2026-02-05",
      "77.3",
      "-"
     ],
     [
      "12",
      "Gemma 4 31B",
      "open source",
      "2026-04-02",
      "76.9",
      "-"
     ],
     [
      "13",
      "o3",
      "proprietary",
      "2025-04-16",
      "76.4",
      "82.9"
     ],
     [
      "14",
      "GPT-5.1",
      "proprietary",
      "2025-11-13",
      "76.0",
      "85.4"
     ],
     [
      "15",
      "Claude Sonnet 4.6 w/ tools",
      "proprietary",
      "2026-02-17",
      "75.6",
      "-"
     ],
     [
      "16",
      "Claude Sonnet 4.6 w/o tools",
      "proprietary",
      "2026-02-17",
      "74.5",
      "-"
     ],
     [
      "17",
      "Claude Opus 4.6 w/o tools",
      "proprietary",
      "2026-02-05",
      "73.9",
      "-"
     ],
     [
      "18",
      "Claude Opus 4.5",
      "proprietary",
      "2025-11-24",
      "73.9",
      "80.7"
     ],
     [
      "19",
      "Gemma 4 26B A4B",
      "open source",
      "2026-04-02",
      "73.8",
      "-"
     ],
     [
      "20",
      "dots.vlm1",
      "open source",
      "2025-08-05",
      "70.1",
      "80.1"
     ],
     [
      "21",
      "Claude Sonnet 4.5",
      "proprietary",
      "2025-09-29",
      "68.9",
      "77.8"
     ],
     [
      "22",
      "Qwen3-VL 235B-A22B",
      "open source",
      "2025-09-22",
      "68.1",
      "78.7"
     ],
     [
      "23",
      "Gemini 2.5 Pro 05-06",
      "proprietary",
      "2025-05-06",
      "68.0",
      "79.6"
     ],
     [
      "24",
      "Seed 1.5-VL Thinking",
      "proprietary",
      "2025-05-13",
      "67.6",
      "77.9"
     ],
     [
      "25",
      "Seed 1.6-Thinking",
      "proprietary",
      "2025-06-25",
      "66.4",
      "74.8"
     ]
    ],
    "main_column": "MMMU-Pro Overall",
    "higher_is_better": true,
    "metric": "MMMU-Pro overall accuracy: share of questions answered correctly (%), on the harder version with ten answer options and questions embedded in images",
    "source": {
     "name": "MMMU team (mmmu-benchmark.github.io)",
     "url": "https://mmmu-benchmark.github.io/leaderboard_data.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-10",
     "license": "not stated"
    },
    "notes": "Rows are models with an MMMU-Pro score, sorted by MMMU-Pro overall (the page itself lists entries unsorted by default); rank is computed from that order. Human-expert and random baselines excluded (human expert high: 85.4 on MMMU-Pro). Nearly all scores are reported by model authors ('source: author'), not independently verified. '-' means no MMMU validation score listed. as_of is the latest model date in the data file, which the page uses as its 'Last updated' value."
   },
   "plain": "College exam-style questions that come with pictures, such as charts, diagrams, maps and sheet music, across 30 subjects. It checks whether an AI can understand images and reason about them like a university student."
  },
  {
   "name": "OmniDocBench",
   "category": "ocr",
   "url": "https://github.com/opendatalab/OmniDocBench",
   "paper": "https://arxiv.org/abs/2412.07626",
   "maintainer": "OpenDataLab (Shanghai AI Laboratory)",
   "measures": "PDF page parsing across diverse document types, scored on text, tables, formulas, layout and reading order with edit-distance and structure metrics.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Several versions exist (v1.5, v1.6); compare scores within one version.",
   "id": "omnidocbench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Method type",
     "Size",
     "Overall",
     "Text edit distance (lower better)",
     "Formula CDM",
     "Table TEDS"
    ],
    "rows": [
     [
      "1",
      "OXR-1.0",
      "Specialized VLMs",
      "1.3B",
      "97.03",
      "0.0244",
      "96.93",
      "96.61"
     ],
     [
      "2",
      "TeleOCR",
      "Specialized VLMs",
      "1.2B",
      "96.91",
      "0.0267",
      "96.5895",
      "96.8183"
     ],
     [
      "3",
      "OvisOCR2",
      "Specialized VLMs",
      "0.8B",
      "96.47",
      "0.0265",
      "97.4854",
      "94.5842"
     ],
     [
      "4",
      "MiniCPM-OCR 1.0",
      "Specialized VLMs",
      "1.3B",
      "96.44",
      "0.0316",
      "98.0814",
      "94.3999"
     ],
     [
      "5",
      "PaddleOCR-VL-1.6",
      "Specialized VLMs",
      "0.9B",
      "96.34",
      "0.0326",
      "97.5304",
      "94.7619"
     ],
     [
      "6",
      "MinerU2.5-Pro",
      "Specialized VLMs",
      "1.2B",
      "95.75",
      "0.036",
      "97.45",
      "93.42"
     ],
     [
      "7",
      "GLM-OCR",
      "Specialized VLMs",
      "0.9B",
      "95.22",
      "0.044",
      "97.18",
      "92.83"
     ],
     [
      "8",
      "PaddleOCR-VL-1.5",
      "Specialized VLMs",
      "0.9B",
      "94.93",
      "0.038",
      "96.89",
      "91.67"
     ],
     [
      "9",
      "PaddleOCR-VL",
      "Specialized VLMs",
      "0.9B",
      "94.18",
      "0.040",
      "95.91",
      "90.65"
     ],
     [
      "10",
      "Unlimited-OCR",
      "Specialized VLMs",
      "3B",
      "94.00",
      "0.0394",
      "95.7234",
      "90.21"
     ],
     [
      "11",
      "Youtu-Parsing",
      "Specialized VLMs",
      "2.5B",
      "93.74",
      "0.044",
      "93.63",
      "92.02"
     ],
     [
      "12",
      "Qianfan-OCR",
      "Specialized VLMs",
      "4B",
      "93.90",
      "0.04",
      "95.08",
      "90.53"
     ],
     [
      "13",
      "Ovis2.6-30B-A3B",
      "General VLMs",
      "30B",
      "93.70",
      "0.035",
      "95.17",
      "89.44"
     ],
     [
      "14",
      "Logics-Parsing-v2",
      "Specialized VLMs",
      "4B",
      "93.33",
      "0.041",
      "95.65",
      "88.42"
     ],
     [
      "15",
      "ABot-OCR",
      "Specialized VLMs",
      "2B",
      "93.30",
      "0.037",
      "94.86",
      "88.69"
     ],
     [
      "16",
      "FireRed-OCR",
      "Specialized VLMs",
      "2B",
      "93.26",
      "0.037",
      "95.44",
      "88.04"
     ],
     [
      "17",
      "MinerU-2.5",
      "Specialized VLMs",
      "1.2B",
      "93.04",
      "0.045",
      "95.77",
      "87.88"
     ],
     [
      "18",
      "Gemini 3 Pro",
      "General VLMs",
      "-",
      "92.91",
      "0.064",
      "95.99",
      "89.15"
     ],
     [
      "19",
      "Gemini 3 Flash",
      "General VLMs",
      "-",
      "92.62",
      "0.066",
      "95.16",
      "89.29"
     ],
     [
      "20",
      "MonkeyOCR V2",
      "Specialized VLMs",
      "-",
      "91.80",
      "0.0636",
      "93.4985",
      "88.2628"
     ],
     [
      "21",
      "dots.ocr",
      "Specialized VLMs",
      "3B",
      "90.77",
      "0.048",
      "89.95",
      "87.18"
     ],
     [
      "22",
      "OpenDoc-0.1B",
      "Specialized VLMs",
      "0.1B",
      "90.67",
      "0.049",
      "93.02",
      "83.88"
     ],
     [
      "23",
      "DeepSeek-OCR 2",
      "Specialized VLMs",
      "3B",
      "90.25",
      "0.050",
      "91.84",
      "83.89"
     ],
     [
      "24",
      "HunyuanOCR",
      "Specialized VLMs",
      "1B",
      "89.95",
      "0.088",
      "87.68",
      "91.01"
     ],
     [
      "25",
      "Qwen3-VL-235B",
      "General VLMs",
      "235B",
      "89.78",
      "0.063",
      "92.55",
      "83.07"
     ]
    ],
    "main_column": "Overall",
    "higher_is_better": true,
    "metric": "Overall document-parsing score (0-100, higher is better), averaging text accuracy, table accuracy and formula accuracy",
    "source": {
     "name": "OpenDataLab / Shanghai AI Lab (OmniDocBench repo)",
     "url": "https://github.com/opendatalab/OmniDocBench",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-12",
     "license": "not stated (dataset research-only, non-commercial per repo copyright statement)"
    },
    "notes": "End-to-end table, benchmark version v1.6_full, from repo README in the order listed (best first), top 25. Scores are mostly self-reported by model authors or the benchmark team."
   },
   "plain": "Tests how well AI can read a PDF page and turn it into clean text, including tables and math formulas. It matters for turning scanned or complex documents into usable data."
  },
  {
   "name": "Open ASR Leaderboard",
   "category": "asr",
   "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard",
   "paper": "https://arxiv.org/abs/2510.06961",
   "maintainer": "Hugging Face (with academic and industry partners)",
   "measures": "Word error rate of open and proprietary speech recognition models across English, multilingual and long-form test sets, plus inverse real-time factor for speed.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "open-asr-leaderboard",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Average WER (%)",
     "RTFx (speed)"
    ],
    "rows": [
     [
      "1",
      "zoom/scribe_v2.1_pro",
      "4.87",
      ""
     ],
     [
      "2",
      "OtterAI/speech-v3.1",
      "5.21",
      ""
     ],
     [
      "3",
      "yoob/yoob-asr-pro-v1",
      "5.33",
      ""
     ],
     [
      "4",
      "microsoft/azure-speech-09-2026",
      "5.5",
      ""
     ],
     [
      "5",
      "Musci-research/Musci-ASR-qwen-1019-hf",
      "5.7",
      "774.69"
     ],
     [
      "6",
      "p99lab/audrey-0.7",
      "5.7",
      ""
     ],
     [
      "7",
      "elevenlabs/scribe_v2",
      "5.93",
      ""
     ],
     [
      "8",
      "reson8/rumble-2",
      "5.95",
      ""
     ],
     [
      "9",
      "modulate/multilingual",
      "5.96",
      ""
     ],
     [
      "10",
      "salesforce-sovereign-intelligence/SOV-Transcribe-1",
      "5.96",
      "600.92"
     ],
     [
      "11",
      "FluidVoice-Labs/fluid-speech-pro",
      "6.06",
      "3690.47"
     ],
     [
      "12",
      "sophea/asr-k1",
      "6.11",
      ""
     ],
     [
      "13",
      "nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B",
      "6.13",
      "472.62"
     ],
     [
      "14",
      "Qwen/Qwen3-ASR-1.7B-hf",
      "6.15",
      "835.62"
     ],
     [
      "15",
      "assemblyai/universal-3-5-pro",
      "6.21",
      ""
     ],
     [
      "16",
      "smallestai/pulse",
      "6.57",
      ""
     ],
     [
      "17",
      "ibm-granite/granite-speech-5.0-2b",
      "6.68",
      "548.43"
     ],
     [
      "18",
      "CohereLabs/cohere-transcribe-03-2026",
      "6.72",
      "923.71"
     ],
     [
      "19",
      "HojoAI/Hojo-ASR-V1",
      "6.74",
      "73.39"
     ],
     [
      "20",
      "TheStageAI/thewhisper-large-v3-turbo",
      "6.74",
      "1183.92"
     ],
     [
      "21",
      "Edge0/ARK-ASR-0.6B",
      "6.75",
      "664.56"
     ],
     [
      "22",
      "openai/gpt-transcribe",
      "6.8",
      ""
     ],
     [
      "23",
      "sprag/symphony",
      "6.85",
      ""
     ],
     [
      "24",
      "gladia/solaria-3",
      "6.86",
      ""
     ],
     [
      "25",
      "nvidia/parakeet-tdt-0.6b-v2 (fast-gpu-asr)",
      "6.91",
      "17254.19"
     ]
    ],
    "main_column": "Average WER (%)",
    "higher_is_better": false,
    "metric": "Average word error rate across English datasets (%, lower is better)",
    "source": {
     "name": "Hugging Face Open ASR Leaderboard (hf-audio)",
     "url": "https://huggingface.co/spaces/hf-audio/open_asr_leaderboard",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-13",
     "license": "not stated"
    },
    "notes": "English tab, top 25 of 69. Mix of open-weight and proprietary models; RTFx blank where not reported. Multilingual and long-form tabs exist."
   },
   "plain": "Tests how accurately speech-to-text systems write down what people say in English recordings (meetings, earnings calls, audiobooks and more). Lower error means fewer wrong words."
  },
  {
   "name": "Artificial Analysis Speech to Text",
   "category": "asr",
   "url": "https://artificialanalysis.ai/speech-to-text",
   "paper": null,
   "maintainer": "Artificial Analysis",
   "measures": "Compares hosted and open speech-to-text models on word error rate together with speed and price.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "artificial-analysis-speech-to-text",
   "results": null,
   "plain": "Compares services that turn spoken audio into written text: how many words they get wrong, how fast they are and what they cost."
  },
  {
   "name": "TTS Arena V2",
   "category": "tts",
   "url": "https://huggingface.co/spaces/TTS-AGI/TTS-Arena-V2",
   "paper": null,
   "maintainer": "TTS-AGI (community)",
   "measures": "Blind pairwise listening votes between text-to-speech models on the same text, converted to Elo ratings.",
   "type": "arena",
   "status": "active",
   "notes": null,
   "id": "tts-arena-v2",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Elo",
     "Win rate (%)",
     "Votes"
    ],
    "rows": [
     [
      "1",
      "Aurora",
      "1687",
      "73.4",
      "312"
     ],
     [
      "2",
      "star-june-2026",
      "1633",
      "64.7",
      "295"
     ],
     [
      "3",
      "Inworld TTS MAX",
      "1567",
      "63.8",
      "1193"
     ],
     [
      "4",
      "CastleFlow v1.0",
      "1558",
      "61.9",
      "1490"
     ],
     [
      "5",
      "Inworld TTS 1.5 MAX",
      "1565",
      "59.7",
      "390"
     ],
     [
      "6",
      "Hume Octave",
      "1551",
      "59.7",
      "949"
     ],
     [
      "7",
      "Inworld TTS",
      "1543",
      "60.0",
      "1712"
     ],
     [
      "8",
      "Papla P1",
      "1552",
      "61.2",
      "836"
     ],
     [
      "9",
      "Gradium TTS 2026.10",
      "1588",
      "62.7",
      "102"
     ],
     [
      "10",
      "Gradium TTS (pre-2026.08)",
      "1554",
      "53.7",
      "259"
     ],
     [
      "11",
      "MiniMax Speech 2.8 HD",
      "1545",
      "54.2",
      "253"
     ],
     [
      "12",
      "OpenAudio S2",
      "1539",
      "52.2",
      "322"
     ],
     [
      "13",
      "Lightning v3.1 Pro",
      "1544",
      "53.6",
      "295"
     ],
     [
      "14",
      "Deepdub eTTS 3.2",
      "1532",
      "52.1",
      "217"
     ],
     [
      "15",
      "Eleven Multilingual v2",
      "1507",
      "54.0",
      "968"
     ],
     [
      "16",
      "Eleven Turbo v2.5",
      "1505",
      "53.5",
      "951"
     ],
     [
      "17",
      "MiniMax Speech 2.6 Turbo",
      "1525",
      "50",
      "270"
     ],
     [
      "18",
      "Eleven Flash v2.5",
      "1503",
      "53.2",
      "964"
     ],
     [
      "19",
      "MiniMax Speech 02 HD",
      "1503",
      "53.7",
      "1028"
     ],
     [
      "20",
      "Eleven v3",
      "1521",
      "49.3",
      "288"
     ],
     [
      "21",
      "MiniMax Speech 2.6 HD",
      "1504",
      "47.5",
      "284"
     ],
     [
      "22",
      "MiniMax Speech 2.8 Turbo",
      "1502",
      "48.2",
      "282"
     ],
     [
      "23",
      "MiniMax Speech 02 Turbo",
      "1483",
      "50.9",
      "1037"
     ],
     [
      "24",
      "Parmesan",
      "1481",
      "52.4",
      "984"
     ],
     [
      "25",
      "Kokoro v1.0",
      "1470",
      "48.8",
      "903"
     ]
    ],
    "main_column": "Elo",
    "higher_is_better": true,
    "metric": "Elo rating from blind human votes (higher = preferred more often)",
    "source": {
     "name": "TTS-AGI community (Hugging Face)",
     "url": "https://tts-agi-tts-arena-v2.hf.space/api/leaderboard",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-11",
     "license": "not stated"
    },
    "notes": "Rank/order as served by the site API; as_of is the Space's last modified date. Includes some preliminary/stealth entries."
   },
   "plain": "People listen to two computer-generated voices reading the same text and pick the one that sounds better. The ranking shows which text-to-speech systems people actually prefer."
  },
  {
   "name": "Artificial Analysis Speech Arena",
   "category": "tts",
   "url": "https://artificialanalysis.ai/text-to-speech/arena",
   "paper": null,
   "maintainer": "Artificial Analysis",
   "measures": "Blind listening comparisons of text-to-speech models and voices, converted to Elo ratings, shown next to price and speed.",
   "type": "arena",
   "status": "active",
   "notes": null,
   "id": "artificial-analysis-speech-arena",
   "results": null,
   "plain": "Listeners hear the same sentence read by two AI voices without knowing which is which and pick the more natural one. The votes become a ranking of text-to-speech models."
  },
  {
   "name": "F5 Labs CASI & ARS Leaderboard",
   "category": "safety",
   "url": "https://www.f5.com/labs/casi",
   "paper": null,
   "maintainer": "F5 Labs",
   "measures": "CASI scores how vulnerable a model is to common prompt-injection and jailbreak attacks; ARS covers multi-step agent scenarios. Published as monthly rankings.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Vendor-run; methodology is the vendor's own.",
   "id": "f5-labs-casi-ars-leaderboard",
   "results": {
    "columns": [
     "Rank",
     "Provider",
     "Model",
     "CASI",
     "Avg. Performance"
    ],
    "rows": [
     [
      "1",
      "Anthropic",
      "Claude Sonnet 5",
      "98.00",
      "23.20%"
     ],
     [
      "2",
      "Anthropic",
      "Claude Haiku 5.5",
      "94.61",
      "43.40%"
     ],
     [
      "3",
      "xAI",
      "Grok 4.7 High",
      "93.31",
      "46.40%"
     ],
     [
      "4",
      "OpenAI",
      "GPT-5.4",
      "93.21",
      "18.20%"
     ],
     [
      "5",
      "Anthropic",
      "Claude Opus 5",
      "93.05",
      "50.80%"
     ],
     [
      "6",
      "Qwen",
      "Qwen 3.8-Max",
      "92.31",
      "40.20%"
     ],
     [
      "7",
      "Anthropic",
      "Claude Fable 5.1 Max Effort",
      "92.30",
      "53.40%"
     ],
     [
      "8",
      "Anthropic",
      "Claude Opus 5.5 Reasoning",
      "92.00",
      "57.60%"
     ],
     [
      "9",
      "Anthropic",
      "Claude Sonnet 5.5",
      "91.78",
      "56.00%"
     ],
     [
      "10",
      "OpenAI",
      "GPT-6 Luna",
      "89.15",
      "38.10%"
     ]
    ],
    "main_column": "CASI",
    "higher_is_better": true,
    "metric": "CASI: F5's Comprehensive AI Security Index (higher = more secure)",
    "source": {
     "name": "F5 Labs",
     "url": "https://www.f5.com/labs/casi",
     "retrieved": "2026-11-17",
     "as_of": "2026-11",
     "license": "not stated"
    },
    "notes": "Only top 10 shown in the page's initial table; latest month selector shows November 2026. ARS view not extracted."
   },
   "plain": "Tests how hard it is to trick AI models into harmful behavior using attack prompts. A higher score means the model is harder to break."
  },
  {
   "name": "Scale Fortress",
   "category": "safety",
   "url": "https://labs.scale.com/leaderboard/fortress",
   "paper": null,
   "maintainer": "Scale AI",
   "measures": "Frontier model risk on adversarial prompts related to national security and public safety.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "scale-fortress",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "ARS ± margin"
    ],
    "rows": [
     [
      "1",
      "claude-opus-5-5-max",
      "5.78±1.26"
     ],
     [
      "2",
      "muse-spark-open",
      "7.64±1.15"
     ],
     [
      "3",
      "gpt-oss-120b",
      "8.24±1.93"
     ],
     [
      "4",
      "Claude Haiku 5.5",
      "9.48±1.34"
     ],
     [
      "5",
      "claude-opus-4-5-20251101-thinking",
      "9.63±2.11"
     ],
     [
      "6",
      "gpt-6-luna",
      "10.78±1.22"
     ],
     [
      "7",
      "beam-1p0-501b-a23b",
      "11.87±1.48"
     ],
     [
      "8",
      "muse-ember-1.0",
      "11.92±1.58"
     ],
     [
      "9",
      "gpt-6-astra",
      "12.14±1.39"
     ],
     [
      "10",
      "Muse Spark 1.1",
      "12.40±1.48"
     ],
     [
      "11",
      "claude-sonnet-4-5-20250929-thinking",
      "12.80±2.36"
     ],
     [
      "12",
      "Claude 3.5 Sonnet",
      "12.96±2.34"
     ],
     [
      "13",
      "claude-opus-4-6-thinking-max",
      "13.00±2.41"
     ],
     [
      "14",
      "claude-opus-5-5",
      "13.18±1.77"
     ],
     [
      "15",
      "gpt-6-sol",
      "13.46±1.40"
     ],
     [
      "16",
      "claude-opus-4-5-20251101",
      "13.56±2.49"
     ],
     [
      "17",
      "Fable-5.1",
      "13.72±1.85"
     ],
     [
      "18",
      "claude-opus-4-1-20250805-thinking",
      "14.79±2.49"
     ],
     [
      "19",
      "gpt-5.4-pro-2026-03-05",
      "14.84±2.29"
     ],
     [
      "20",
      "gpt-5-pro-2025-10-06",
      "15.20±2.30"
     ],
     [
      "21",
      "claude-fable-5-2",
      "15.60±1.92"
     ],
     [
      "22",
      "o3",
      "16.01±2.68"
     ],
     [
      "23",
      "claude-opus-4-1-20250805",
      "16.15±2.58"
     ],
     [
      "24",
      "Inkling 2 Small",
      "16.23±2.28"
     ],
     [
      "25",
      "gpt-5.5-xhigh",
      "16.28±1.64"
     ]
    ],
    "main_column": "ARS",
    "higher_is_better": false,
    "metric": "Average Risk Score (ARS) with margin; lower = safer",
    "source": {
     "name": "Scale AI (SEAL)",
     "url": "https://labs.scale.com/leaderboard/fortress",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "From the page's 'Performance Comparison' list; the page does not label the score but states ranking is by ARS (lower is better). No update date shown; includes models released in 2026. Rank ties not shown."
   },
   "plain": "Tests whether AI models can be pushed into giving dangerous help (e.g. weapons or crime) by adversarial prompts. Lower is safer."
  },
  {
   "name": "CyberGym",
   "category": "safety",
   "url": "https://www.cybergym.io/",
   "paper": "https://arxiv.org/abs/2506.02548",
   "maintainer": "UC Berkeley",
   "measures": "Agents must reproduce real vulnerabilities in open-source projects by writing proof-of-concept inputs, scored by the share successfully triggered.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Measures offensive cyber capability.",
   "id": "cybergym",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Agent",
     "Success Rate",
     "Date",
     "Source"
    ],
    "rows": [
     [
      "1",
      "Creation Model",
      "Claude Code",
      "90.6%",
      "2026-10-21",
      "Creation Lab (Creation Lab, Ant Group)"
     ],
     [
      "2",
      "XekRung-1.5-27B-Preview (finetuned from Qwen3.8-27B)",
      "XekRung Agent",
      "88.9%",
      "2026-09-13",
      "Alibaba Security"
     ],
     [
      "3",
      "Gemini 3.8 Flash Cyber",
      "Google Antigravity",
      "86.3%",
      "2026-09-10",
      "Google"
     ],
     [
      "4",
      "GPT-5.5-Cyber",
      "OpenAI Agent",
      "85.6%",
      "2026-06-22",
      "OpenAI"
     ],
     [
      "5",
      "GLM-5.3",
      "Claude Code",
      "84.5%",
      "2026-08-14",
      "Zhipu AI"
     ],
     [
      "6",
      "Feyospace-v1.1 (finetuned from Qwen3.8-27B)",
      "Claude Code",
      "84.4%",
      "2026-09-23",
      "Vera Praxis Lab"
     ],
     [
      "7",
      "DeepSeek-V4-Pro",
      "DeepSeek Agent",
      "83.3%",
      "2026-08-13",
      "DeepSeek"
     ],
     [
      "8",
      "Claude Mythos Preview",
      "Anthropic Agent",
      "83.1%",
      "2026-04-07",
      "Anthropic"
     ],
     [
      "9",
      "GPT-5.5",
      "OpenAI Agent",
      "81.8%",
      "2026-04-23",
      "OpenAI"
     ],
     [
      "10",
      "Basin (basin1-0912-pre-dpo)",
      "Codex CLI",
      "80.0%",
      "2026-09-23",
      "Submersion AI"
     ],
     [
      "11",
      "Grok 4.6",
      "Grok Build",
      "79.7%",
      "2026-08-12",
      "xAI"
     ],
     [
      "12",
      "Grok 4.5",
      "Grok Build",
      "79.0%",
      "2026-07-14",
      "xAI"
     ],
     [
      "13",
      "GPT-5.4",
      "OpenAI Agent",
      "79.0%",
      "2026-04-23",
      "OpenAI"
     ],
     [
      "14",
      "GLM-5.2",
      "Claude Code",
      "77.2%",
      "2026-07-21",
      "Zhipu AI"
     ],
     [
      "15",
      "DeepSeek-V4-Flash",
      "DeepSeek Agent",
      "76.7%",
      "2026-07-31",
      "DeepSeek"
     ],
     [
      "16",
      "GLM-5.1",
      "Claude Code",
      "68.7%",
      "2026-04-07",
      "Zhipu AI"
     ],
     [
      "17",
      "Claude Opus 4.6",
      "Anthropic Agent",
      "66.6%",
      "2026-02-05",
      "Anthropic"
     ],
     [
      "18",
      "GPT-5.4",
      "Codex CLI",
      "66.3%",
      "2026-04-07",
      "Zhipu AI"
     ],
     [
      "19",
      "Claude Sonnet 4.6",
      "Anthropic Agent",
      "65.2%",
      "2026-02-17",
      "Anthropic"
     ],
     [
      "20",
      "Muse Spark 1.1 (helpful-only)",
      "Meta Agent",
      "59.0%",
      "2026-07-09",
      "Meta AI"
     ],
     [
      "21",
      "DeepSeek-V4-Pro Preview",
      "Claude Code",
      "57.7%",
      "2026-07-13",
      "XDxAI"
     ],
     [
      "22",
      "Claude Opus 4.5",
      "Anthropic Agent",
      "50.6%",
      "2026-02-05",
      "Anthropic"
     ],
     [
      "23",
      "Muse Spark",
      "Meta Agent",
      "43.5%",
      "2026-04-14",
      "Meta AI"
     ],
     [
      "24",
      "GLM-5",
      "Claude Code",
      "43.2%",
      "2026-02-12",
      "Zhipu AI"
     ],
     [
      "25",
      "Kimi K2.5",
      "Kimi Agent",
      "41.3%",
      "2026-02-02",
      "Kimi"
     ]
    ],
    "main_column": "Success Rate",
    "higher_is_better": true,
    "metric": "Share of 1,507 real vulnerabilities the agent successfully reproduced (%)",
    "source": {
     "name": "CyberGym team (UC Berkeley)",
     "url": "https://www.cybergym.io/assets/data/cybergym.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-10-24",
     "license": "not stated (benchmark code Apache-2.0)"
    },
    "notes": "Level 1 (vulnerability description + codebase), single-trial runs. Reflects the site's default 'Model-focused' view, which excludes agent-focused systems and runs using the 'dynamic' (cleaned vulnerable Docker image) or 'test-time memory' settings; in the 'All' view many agent systems score 92–99%. Scores are mostly submitter-reported (Source column). The site shows entries above 90% as unranked cards in random order. as_of = newest entry date in the data file."
   },
   "plain": "Gives AI agents a description of a real security bug in open-source software plus the code, and asks them to write an input that actually triggers the bug. It shows how good AI has become at the hands-on work of finding and reproducing vulnerabilities, which helps defenders and could help attackers."
  },
  {
   "name": "EuroEval",
   "category": "multilingual",
   "url": "https://euroeval.com/leaderboards/",
   "paper": "https://arxiv.org/abs/2304.00906",
   "maintainer": "EuroEval team (formerly ScandEval)",
   "measures": "Language models on tasks such as classification, NER, reading comprehension and knowledge across European languages, with per-language leaderboards.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Paper describes the original ScandEval.",
   "id": "euroeval",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Mean rank score",
     "Open weights"
    ],
    "rows": [
     [
      "1",
      "gpt-6-astra (zero-shot, val)",
      "1.17 ± 0.05",
      "No"
     ],
     [
      "1",
      "gpt-6.1-sol (zero-shot, val)",
      "1.18 ± 0.04",
      "No"
     ],
     [
      "2",
      "gemini/gemini-3.7-flash (zero-shot, val)",
      "1.24 ± 0.03",
      "No"
     ],
     [
      "3",
      "gemini/gemini-3-flash-preview#thinking (zero-shot, val)",
      "1.32 ± 0.04",
      "No"
     ],
     [
      "3",
      "gemini/gemini-3.6-flash (zero-shot, val)",
      "1.31 ± 0.04",
      "No"
     ],
     [
      "3",
      "zai-org/GLM-5.3-Flash (val)",
      "1.36 ± 0.04",
      "Yes"
     ],
     [
      "4",
      "gpt-5.6-sol (zero-shot, val)",
      "1.40 ± 0.04",
      "No"
     ],
     [
      "5",
      "gpt-6-sol (zero-shot, val)",
      "1.45 ± 0.05",
      "No"
     ],
     [
      "6",
      "zai-org/GLM-5.3-Flash (zero-shot, val)",
      "1.54 ± 0.08",
      "Yes"
     ],
     [
      "6",
      "deepseek/deepseek-flash#low (zero-shot, val)",
      "1.50 ± 0.05",
      "No"
     ],
     [
      "7",
      "Qwen/Qwen3.6-27B-FP8 (val)",
      "1.60 ± 0.07",
      "Yes"
     ],
     [
      "7",
      "gemini/gemini-3.1-flash-lite-preview (zero-shot, val)",
      "1.59 ± 0.08",
      "No"
     ],
     [
      "7",
      "gpt-5.6-terra (zero-shot, val)",
      "1.58 ± 0.07",
      "No"
     ],
     [
      "7",
      "openai/gpt-5.4-mini-2026-03-17#high (zero-shot, val)",
      "1.61 ± 0.05",
      "No"
     ],
     [
      "7",
      "claude-sonnet-4-5-20250929#thinking (zero-shot, val)",
      "1.56 ± 0.08",
      "No"
     ],
     [
      "7",
      "deepseek/deepseek-flash#no-thinking (val)",
      "1.57 ± 0.06",
      "No"
     ],
     [
      "8",
      "Qwen/Qwen3.8-27B (val)",
      "1.67 ± 0.09",
      "Yes"
     ],
     [
      "8",
      "Qwen/Qwen3.5-35B-A3B-FP8 (val)",
      "1.66 ± 0.09",
      "Yes"
     ],
     [
      "8",
      "google/gemma-4-31B-it (val)",
      "1.67 ± 0.10",
      "Yes"
     ],
     [
      "8",
      "claude-sonnet-4-6 (zero-shot, val)",
      "1.69 ± 0.08",
      "No"
     ],
     [
      "8",
      "xai/grok-4-1-fast-reasoning (zero-shot, val)",
      "1.69 ± 0.07",
      "No"
     ],
     [
      "8",
      "openai/gpt-5.4-mini-2026-03-17#medium (zero-shot, val)",
      "1.64 ± 0.06",
      "No"
     ],
     [
      "9",
      "gpt-5-mini-2025-08-07 (zero-shot, val)",
      "1.71 ± 0.07",
      "No"
     ],
     [
      "9",
      "openai/gpt-5.4-mini-2026-03-17#low (zero-shot, val)",
      "1.70 ± 0.06",
      "No"
     ],
     [
      "9",
      "openai/gpt-5.4-2026-03-05#none (zero-shot, val)",
      "1.76 ± 0.09",
      "No"
     ]
    ],
    "main_column": "Mean rank score",
    "higher_is_better": false,
    "metric": "Average rank score across languages and tasks (lower is better; 1.00 would mean best on everything)",
    "source": {
     "name": "EuroEval team",
     "url": "https://euroeval.com/leaderboards/Multilingual/european/",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-15",
     "license": "not stated"
    },
    "notes": "European (all languages) leaderboard, All Models view, simplified CSV embedded in the site bundle. Rank score: lower is better. Ties share ranks."
   },
   "plain": "Tests how well AI language models understand and write in more than 30 European languages, across many kinds of tasks. It matters because most AI tests are English-only."
  },
  {
   "name": "CPTU-Bench",
   "category": "polish",
   "url": "https://huggingface.co/spaces/speakleash/cptu_bench",
   "paper": null,
   "maintainer": "SpeakLeash",
   "measures": "Understanding of Polish text: implicatures (sentiment, implied meaning, phraseology) and tricky questions (logic puzzles, ambiguity, absurdity), scored on a 0–5 scale.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "cptu-bench",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Params",
     "Average",
     "Sentiment",
     "Language understanding",
     "Phraseology",
     "Tricky questions"
    ],
    "rows": [
     [
      "1",
      "meta-models/Muse-Glimmer-30B reasoning (API, FP8)",
      "30B",
      "4.42",
      "4.52",
      "4.39",
      "4.23",
      "4.52"
     ],
     [
      "2",
      "Qwen/Qwen3.6-27B reasoning (API, FP8)",
      "27B",
      "4.41",
      "4.58",
      "4.21",
      "4.21",
      "4.64"
     ],
     [
      "3",
      "Qwen/Qwen3.5-122B-A10B reasoning (API, FP8)",
      "122B",
      "4.36",
      "4.36",
      "4.18",
      "4.19",
      "4.72"
     ],
     [
      "4",
      "Qwen/Qwen3.5-27B thinking (API)",
      "27B",
      "4.34",
      "4.42",
      "4.21",
      "4.11",
      "4.61"
     ],
     [
      "5",
      "google/gemma-4-31B-it non-reasoning (API, FP8)",
      "31B",
      "4.32",
      "4.58",
      "4.46",
      "4.07",
      "4.19"
     ],
     [
      "6",
      "google/gemma-4-31B-it reasoning (API, FP8)",
      "31B",
      "4.31",
      "4.58",
      "4.38",
      "3.77",
      "4.52"
     ],
     [
      "7",
      "Qwen/Qwen3.6-27B non-reasoning (API, FP8)",
      "27B",
      "4.29",
      "4.33",
      "4.08",
      "4.40",
      "4.36"
     ],
     [
      "8",
      "gemini-2.0-flash-001",
      "",
      "4.29",
      "4.52",
      "4.32",
      "4.34",
      "3.99"
     ],
     [
      "9",
      "Qwen/Qwen3.5-27B non-thinking (API)",
      "27B",
      "4.27",
      "4.29",
      "4.17",
      "4.20",
      "4.43"
     ],
     [
      "10",
      "Qwen/Qwen3.5-122B-A10B non-reasoning (API, FP8)",
      "122B",
      "4.26",
      "4.33",
      "4.24",
      "3.94",
      "4.53"
     ],
     [
      "11",
      "google/gemma-4-26B-A4B-it non-reasoning (API, FP8)",
      "26B",
      "4.23",
      "4.39",
      "4.00",
      "4.21",
      "4.30"
     ],
     [
      "12",
      "Qwen/Qwen3.5-35B-A3B thinking (API)",
      "35B",
      "4.22",
      "4.10",
      "3.94",
      "4.15",
      "4.70"
     ],
     [
      "13",
      "google/gemma-4-26B-A4B-it reasoning (API, FP8)",
      "26B",
      "4.21",
      "4.39",
      "4.08",
      "3.91",
      "4.47"
     ],
     [
      "14",
      "Qwen/Qwen3.5-35B-A3B non-thinking (API)",
      "35B",
      "4.18",
      "4.23",
      "4.05",
      "4.23",
      "4.19"
     ],
     [
      "15",
      "deepseek-ai/DeepSeek-V3.2 (API)",
      "685B",
      "4.14",
      "4.46",
      "4.36",
      "3.54",
      "4.20"
     ],
     [
      "16",
      "deepseek-ai/DeepSeek-R1 (API)",
      "685B",
      "4.14",
      "4.49",
      "4.34",
      "3.60",
      "4.12"
     ],
     [
      "17",
      "gemini-2.0-flash-lite-001",
      "",
      "4.09",
      "4.23",
      "4.05",
      "4.24",
      "3.85"
     ],
     [
      "18",
      "deepseek-ai/DeepSeek-V3-0324 (API)",
      "685B",
      "4.03",
      "4.36",
      "4.20",
      "3.54",
      "4.02"
     ],
     [
      "19",
      "deepseek-ai/DeepSeek-V3.1 (API)",
      "685B",
      "4.03",
      "4.42",
      "4.33",
      "3.48",
      "3.87"
     ],
     [
      "20",
      "deepseek-ai/DeepSeek-V3 (API)",
      "685B",
      "4.02",
      "4.36",
      "4.22",
      "3.52",
      "3.99"
     ],
     [
      "21",
      "mistralai/Mistral-Large-Instruct-2411",
      "123B",
      "4.00",
      "4.33",
      "3.98",
      "3.99",
      "3.72"
     ],
     [
      "22",
      "moonshotai/Kimi-K2-Instruct-0905 (API)",
      "1000B",
      "3.98",
      "4.39",
      "4.18",
      "3.43",
      "3.93"
     ],
     [
      "23",
      "Qwen/Qwen2.5-72B-Instruct",
      "72.7B",
      "3.95",
      "4.08",
      "3.97",
      "3.93",
      "3.81"
     ],
     [
      "24",
      "mistralai/Mistral-Large-Instruct-2407",
      "123B",
      "3.93",
      "4.23",
      "4.00",
      "3.86",
      "3.65"
     ],
     [
      "25",
      "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 (API)",
      "402B",
      "3.93",
      "4.39",
      "4.11",
      "3.48",
      "3.76"
     ]
    ],
    "main_column": "Average",
    "higher_is_better": true,
    "metric": "Average score on a 0–5 scale across four parts (sentiment, language understanding, phraseology, tricky questions)",
    "source": {
     "name": "SpeakLeash",
     "url": "https://huggingface.co/spaces/speakleash/cptu_bench/resolve/main/data.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-08-11",
     "license": "MIT"
    },
    "notes": "Per-category scores copied from the Space's data.json; 'Average' is the plain mean of the four categories, computed exactly as the leaderboard app does (app.py) and shown to 2 decimals like the leaderboard. as_of is the Space's last commit date. License is the Space repository's declared license (MIT). 102 models in total; top 25 shown."
   },
   "plain": "Checks how well AI models really understand Polish: reading between the lines (sarcasm, hidden meaning, idioms) and answering tricky questions with logic traps, ambiguity or absurd premises. It shows whether a model grasps Polish beyond the literal words."
  },
  {
   "name": "Polish Linguistic and Cultural Competency (PLCC)",
   "category": "polish",
   "url": "https://huggingface.co/spaces/sdadas/plcc",
   "paper": "https://arxiv.org/abs/2503.00995",
   "maintainer": "Sławomir Dadas et al. (National Information Processing Institute, OPI PIB)",
   "measures": "600 hand-written questions in six categories (history, geography, culture and tradition, art and entertainment, grammar, vocabulary), scored by accuracy.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "polish-linguistic-and-cultural-competency-plcc",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Provider",
     "Overall",
     "Art & entertainment",
     "Culture & tradition",
     "Geography",
     "Grammar",
     "History",
     "Vocabulary"
    ],
    "rows": [
     [
      "1",
      "Gemini-4-Carbon",
      "Google",
      "98.33",
      "97",
      "100",
      "100",
      "97",
      "98",
      "98"
     ],
     [
      "2",
      "Gemini-4-Argon",
      "Google",
      "97.33",
      "95",
      "98",
      "99",
      "96",
      "98",
      "98"
     ],
     [
      "3",
      "Claude-Fable-5.2",
      "Anthropic",
      "97.33",
      "96",
      "98",
      "98",
      "97",
      "99",
      "96"
     ],
     [
      "4",
      "Gemini-3.1-Pro-Preview",
      "Google",
      "97",
      "95",
      "100",
      "100",
      "93",
      "98",
      "96"
     ],
     [
      "5",
      "Gemini-3.8-Flash",
      "Google",
      "97",
      "95",
      "99",
      "100",
      "95",
      "98",
      "95"
     ],
     [
      "6",
      "GPT-6.1-Sol",
      "OpenAI",
      "96.83",
      "93",
      "98",
      "98",
      "98",
      "99",
      "95"
     ],
     [
      "7",
      "Gemini-4-Hydrogen",
      "Google",
      "96.83",
      "93",
      "97",
      "100",
      "97",
      "97",
      "97"
     ],
     [
      "8",
      "Gemini-3.7-Flash",
      "Google",
      "96.67",
      "95",
      "98",
      "100",
      "95",
      "97",
      "95"
     ],
     [
      "9",
      "Gemini-3.5-Flash",
      "Google",
      "96.33",
      "94",
      "100",
      "100",
      "97",
      "96",
      "91"
     ],
     [
      "10",
      "GPT-6-Astra",
      "OpenAI",
      "96.33",
      "94",
      "98",
      "98",
      "95",
      "99",
      "94"
     ],
     [
      "11",
      "Gemini-3.0-Pro-Preview",
      "Google",
      "95.83",
      "95",
      "99",
      "100",
      "91",
      "95",
      "95"
     ],
     [
      "12",
      "GPT-5.5 (high reasoning)",
      "OpenAI",
      "95.83",
      "93",
      "98",
      "98",
      "97",
      "95",
      "94"
     ],
     [
      "13",
      "Gemini-3.6-Flash",
      "Google",
      "95.5",
      "92",
      "98",
      "99",
      "94",
      "96",
      "94"
     ],
     [
      "14",
      "Claude-Fable-5.1",
      "Anthropic",
      "95.17",
      "91",
      "95",
      "99",
      "95",
      "97",
      "94"
     ],
     [
      "15",
      "Claude-Fable-5",
      "Anthropic",
      "95",
      "93",
      "96",
      "98",
      "94",
      "97",
      "92"
     ],
     [
      "16",
      "GPT-5.6-Sol (medium reasoning)",
      "OpenAI",
      "94.67",
      "93",
      "96",
      "99",
      "91",
      "96",
      "93"
     ],
     [
      "17",
      "GPT-5.5 (low reasoning)",
      "OpenAI",
      "94.5",
      "93",
      "96",
      "99",
      "91",
      "95",
      "93"
     ],
     [
      "18",
      "GPT-6.1-Terra",
      "OpenAI",
      "94",
      "90",
      "94",
      "97",
      "93",
      "98",
      "92"
     ],
     [
      "19",
      "GPT-6-Sol (medium reasoning)",
      "OpenAI",
      "93.83",
      "94",
      "93",
      "98",
      "93",
      "97",
      "88"
     ],
     [
      "20",
      "Muse-Wonder",
      "Meta",
      "93.83",
      "92",
      "92",
      "98",
      "92",
      "97",
      "92"
     ],
     [
      "21",
      "Claude-Opus-5.5",
      "Anthropic",
      "92.5",
      "86",
      "93",
      "94",
      "94",
      "97",
      "91"
     ],
     [
      "22",
      "Gemini-2.5-Pro-Preview-06-05",
      "Google",
      "92.17",
      "91",
      "96",
      "98",
      "86",
      "92",
      "90"
     ],
     [
      "23",
      "GPT-5.4-2026-03-05 (high reasoning)",
      "OpenAI",
      "92.17",
      "91",
      "93",
      "96",
      "90",
      "92",
      "91"
     ],
     [
      "24",
      "Claude-Opus-5",
      "Anthropic",
      "92.17",
      "86",
      "94",
      "97",
      "92",
      "96",
      "88"
     ],
     [
      "25",
      "Muse-Spark-1.3",
      "Meta",
      "91.83",
      "89",
      "94",
      "97",
      "89",
      "94",
      "88"
     ]
    ],
    "main_column": "Overall",
    "higher_is_better": true,
    "metric": "Share of the 600 questions answered fully correctly (%), averaged over the six categories",
    "source": {
     "name": "Sławomir Dadas et al. (OPI PIB)",
     "url": "https://huggingface.co/spaces/sdadas/plcc/resolve/main/data.json",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-16",
     "license": "CC-BY-4.0"
    },
    "notes": "Per-category accuracies copied from the Space's data.json (253 models). The leaderboard file does not include an overall column; 'Overall' here is the plain mean of the six category scores (each category has 100 questions, so this equals the overall percentage of correct answers), and rows are ranked by it. as_of is the Space's last commit date."
   },
   "plain": "600 hand-written questions about Polish history, geography, culture and traditions, art and entertainment, grammar and vocabulary. It shows how well an AI model knows Poland and the Polish language, not just how fluent it sounds."
  },
  {
   "name": "PL-MTEB (Polish MTEB)",
   "category": "polish",
   "url": "https://huggingface.co/spaces/mteb/leaderboard?benchmark_name=MTEB%28pol%2C+v1%29",
   "paper": "https://arxiv.org/abs/2405.10138",
   "maintainer": "Rafał Poświata, Sławomir Dadas et al. (OPI PIB); hosted in MTEB",
   "measures": "Polish text embedding models on retrieval, classification, clustering, pair classification and semantic similarity tasks.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "pl-mteb-polish-mteb",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Mean (Task)",
     "Mean (Task type)",
     "Params (B)"
    ],
    "rows": [
     [
      "1",
      "codefuse-ai/F2LLM-v2-14B",
      "75.13",
      "71.96",
      "13.99"
     ],
     [
      "2",
      "codefuse-ai/F2LLM-v2-8B",
      "74.61",
      "71.48",
      "7.568"
     ],
     [
      "3",
      "Qwen/Qwen3-Embedding-8B",
      "73.54",
      "71.66",
      "7.567"
     ],
     [
      "5",
      "Qwen/Qwen3-Embedding-4B",
      "72.34",
      "70.53",
      "4.022"
     ],
     [
      "6",
      "codefuse-ai/F2LLM-v2-4B",
      "73.42",
      "70.28",
      "4.022"
     ],
     [
      "7",
      "Linq-AI-Research/Linq-Embed-Mistral",
      "71.10",
      "69.28",
      "7.111"
     ],
     [
      "8",
      "microsoft/harrier-oss-v1-0.6b",
      "70.47",
      "68.65",
      "0.596"
     ],
     [
      "9",
      "codefuse-ai/F2LLM-v2-1.7B",
      "72.03",
      "68.96",
      "1.721"
     ],
     [
      "10",
      "Salesforce/SFR-Embedding-Mistral",
      "69.31",
      "68.29",
      "7.111"
     ],
     [
      "11",
      "intfloat/multilingual-e5-large-instruct",
      "69.13",
      "68.35",
      "0.56"
     ],
     [
      "12",
      "intfloat/e5-mistral-7b-instruct",
      "69.94",
      "68.66",
      "7.111"
     ],
     [
      "13",
      "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
      "70.00",
      "67.98",
      "1.543"
     ],
     [
      "14",
      "Salesforce/SFR-Embedding-2_R",
      "68.88",
      "67.18",
      "7.111"
     ],
     [
      "16",
      "codefuse-ai/F2LLM-v2-0.6B",
      "69.63",
      "66.72",
      "0.596"
     ],
     [
      "17",
      "manu/bge-m3-custom-fr",
      "64.74",
      "63.88",
      "0.568"
     ],
     [
      "17",
      "microsoft/harrier-oss-v1-270m",
      "66.93",
      "66.04",
      "0.268"
     ],
     [
      "21",
      "OrdalieTech/Solon-embeddings-large-0.1",
      "63.49",
      "63.01",
      "0.56"
     ],
     [
      "22",
      "codefuse-ai/F2LLM-v2-330M",
      "66.53",
      "64.39",
      "0.334"
     ],
     [
      "23",
      "Qwen/Qwen3-Embedding-0.6B",
      "65.50",
      "64.59",
      "0.596"
     ],
     [
      "24",
      "omarelshehy/arabic-english-sts-matryoshka",
      "65.07",
      "64.30",
      "0.56"
     ],
     [
      "26",
      "sdadas/mmlw-e5-small",
      "63.41",
      "63.35",
      "0.118"
     ],
     [
      "28",
      "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1",
      "64.57",
      "63.70",
      "0.494"
     ],
     [
      "30",
      "hotchpotch/bekko-embedding-v1-a25m",
      "61.50",
      "61.66",
      "0.123"
     ],
     [
      "32",
      "Snowflake/snowflake-arctic-embed-l-v2.0",
      "61.31",
      "60.96",
      "0.568"
     ],
     [
      "35",
      "deepvk/USER-bge-m3",
      "62.46",
      "61.27",
      "0.359"
     ]
    ],
    "main_column": "Mean (Task)",
    "higher_is_better": true,
    "metric": "Mean score over all Polish tasks (0-100, from 0-1 scale)",
    "source": {
     "name": "MTEB team (Hugging Face MTEB leaderboard)",
     "url": "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB(pol, v1)/scores",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "MIT (MTEB project)"
    },
    "notes": "Benchmark MTEB(pol, v1) from the MTEB leaderboard backend API; source scores are on 0-1 scale and shown here x100 to 2 decimals. Leaderboard Space last modified 2026-11-16. Models with no mean score (not run on all tasks) were skipped. Some models may have trained on part of the tasks (zero-shot share not shown)."
   },
   "plain": "Tests how well \"embedding\" models (the part of AI search that turns text into numbers so similar meanings can be found) work on Polish text, on tasks like finding similar sentences, sorting reviews and grouping documents."
  },
  {
   "name": "Artificial Analysis API Provider Benchmarks",
   "category": "performance",
   "url": "https://artificialanalysis.ai/leaderboards/providers",
   "paper": null,
   "maintainer": "Artificial Analysis",
   "measures": "Output speed, time to first token, latency and price for the same models served by different API providers, measured continuously.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "artificial-analysis-api-provider-benchmarks",
   "results": null,
   "plain": "Compares the companies that host AI models on speed, price and delay, so you can pick the fastest or cheapest provider for the same model."
  },
  {
   "name": "OpenRouter Rankings",
   "category": "performance",
   "url": "https://openrouter.ai/rankings",
   "paper": null,
   "maintainer": "OpenRouter",
   "measures": "Share of tokens processed per model, provider and app on OpenRouter over time.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Shows usage and popularity, not quality.",
   "id": "openrouter-rankings",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Tokens (B)",
     "Requests (M)"
    ],
    "rows": [
     [
      "1",
      "deepseek/deepseek-v4.1-flash-20261017",
      "61170.1",
      "978.5"
     ],
     [
      "2",
      "tencent/hy4-20261104",
      "16042.2",
      "115.5"
     ],
     [
      "3",
      "deepseek/deepseek-v4.1-flash-20260910",
      "15455.0",
      "495.9"
     ],
     [
      "4",
      "z-ai/glm-5.3-flash-20260826",
      "11806.2",
      "361.0"
     ],
     [
      "5",
      "openai/gpt-6-luna-20260922",
      "8081.3",
      "614.8"
     ],
     [
      "6",
      "anthropic/claude-opus-5.5-20260921",
      "7365.4",
      "63.3"
     ],
     [
      "7",
      "stepfun/step-5-preview-20261008",
      "6360.0",
      "24.8"
     ],
     [
      "8",
      "xiaomi/mimo-v2.6-flash-20260921",
      "4164.6",
      "59.1"
     ],
     [
      "9",
      "z-ai/glm-5.3-20260816",
      "3738.6",
      "81.6"
     ],
     [
      "10",
      "openai/gpt-6.1-sol-20260929",
      "3711.0",
      "78.7"
     ],
     [
      "11",
      "minimax/minimax-m3.1-flash-20261019",
      "3350.9",
      "46.9"
     ],
     [
      "12",
      "moonshotai/kimi-k3.1-pro-20261023",
      "3241.9",
      "39.8"
     ],
     [
      "13",
      "nvidia/nemotron-3-ultra-550b-a55b-20260604 (free)",
      "3198.3",
      "37.8"
     ],
     [
      "14",
      "deepseek/deepseek-v4-flash-20260731",
      "3196.4",
      "305.6"
     ],
     [
      "15",
      "nvidia/nemotron-3.5-super-vl-20261015 (free)",
      "2969.1",
      "27.0"
     ],
     [
      "16",
      "meta/muse-spark-1.3-contributor-20260902",
      "2964.9",
      "34.3"
     ],
     [
      "17",
      "typesafe/jev-1.13-20260917",
      "2868.8",
      "956.9"
     ],
     [
      "18",
      "anthropic/claude-haiku-5.5-20261007",
      "2779.1",
      "229.7"
     ],
     [
      "19",
      "anthropic/claude-sonnet-5.5-20260928",
      "2666.3",
      "63.9"
     ],
     [
      "20",
      "baidu/ernie-5.5-flash-preview-20261110 (free)",
      "2470.8",
      "19.1"
     ]
    ],
    "main_column": "Tokens (B)",
    "higher_is_better": true,
    "metric": "Tokens (pieces of text, in billions) processed by the model in the latest weekly period",
    "source": {
     "name": "OpenRouter",
     "url": "https://openrouter.ai/rankings",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-16",
     "license": "not stated"
    },
    "notes": "Weekly view (bucket dated 2026-11-16), data embedded in the rankings page. Tokens = prompt + completion tokens, ranked by that sum (sorted by this sum). Only top 20 models in the page payload. Popularity/usage measure, not quality."
   },
   "plain": "Shows which AI models people actually use most on OpenRouter, a service that lets developers switch between many AI models. It reflects real-world popularity, not test scores."
  },
  {
   "name": "Artificial Analysis Intelligence Index",
   "category": "aggregators",
   "url": "https://artificialanalysis.ai/leaderboards/models",
   "paper": null,
   "maintainer": "Artificial Analysis",
   "measures": "Composite score from about ten independently run evaluations across agents, general knowledge, coding and scientific reasoning, shown next to price, speed and context window.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Component evaluations and weights change between index versions; methodology at artificialanalysis.ai/methodology/intelligence-benchmarking.",
   "id": "artificial-analysis-intelligence-index",
   "results": null,
   "plain": "Artificial Analysis combines results from a bundle of tough tests into one 'intelligence' score per AI model, alongside price and speed, to help people choose a model."
  },
  {
   "name": "Epoch AI Benchmarking Hub",
   "category": "aggregators",
   "url": "https://epoch.ai/benchmarks",
   "paper": null,
   "maintainer": "Epoch AI",
   "measures": "Independently run and collected results on many benchmarks (such as FrontierMath, GPQA Diamond, SWE-bench) with trends over time.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "epoch-ai-benchmarking-hub",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "Score (%)",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "claude-fable-5-2_max",
      "Anthropic",
      "95.1",
      "2026-11-09"
     ],
     [
      "2",
      "gpt-6.1-sol_max",
      "OpenAI",
      "93.7",
      "2026-09-29"
     ],
     [
      "3",
      "gpt-6-astra_max",
      "OpenAI",
      "93.7",
      "2026-09-03"
     ],
     [
      "4",
      "claude-opus-5-5_max",
      "Anthropic",
      "91.2",
      "2026-09-22"
     ],
     [
      "5",
      "gpt-6.1-terra_max",
      "OpenAI",
      "90.9",
      "2026-10-26"
     ],
     [
      "6",
      "gemini-4-carbon_high",
      "Google DeepMind",
      "90.5",
      "2026-11-10"
     ],
     [
      "7",
      "claude-fable-5-1_max",
      "Anthropic",
      "90.2",
      "2026-09-01"
     ],
     [
      "8",
      "gpt-6-sol_max",
      "OpenAI",
      "89.8",
      "2026-09-22"
     ],
     [
      "9",
      "gpt-5.6-sol_max",
      "OpenAI",
      "89.1",
      "2026-07-09"
     ],
     [
      "10",
      "claude-sonnet-5-5_max",
      "Anthropic",
      "88.8",
      "2026-09-28"
     ],
     [
      "11",
      "gpt-5.5-pro_xhigh",
      "OpenAI",
      "87.7",
      "2026-04-23"
     ],
     [
      "12",
      "claude-fable-5_max",
      "Anthropic",
      "87.0",
      "2026-06-09"
     ],
     [
      "13",
      "gemini-4-argon_high",
      "Google DeepMind",
      "86.0",
      "2026-10-13"
     ],
     [
      "14",
      "gpt-5.6-terra_max",
      "OpenAI",
      "86.0",
      "2026-07-09"
     ],
     [
      "15",
      "claude-opus-5_max",
      "Anthropic",
      "85.6",
      "2026-07-24"
     ],
     [
      "16",
      "gpt-5.5_xhigh",
      "OpenAI",
      "85.3",
      "2026-04-23"
     ],
     [
      "17",
      "gpt-5.4-pro-2026-03-05_xhigh",
      "OpenAI",
      "82.5",
      "2026-03-05"
     ],
     [
      "18",
      "muse-wonder-1.0_max",
      "Meta AI",
      "82.1",
      "2026-10-12"
     ],
     [
      "19",
      "gpt-5.6-luna_max",
      "OpenAI",
      "82.1",
      "2026-07-09"
     ],
     [
      "20",
      "claude-opus-4-8_max",
      "Anthropic",
      "80.0",
      "2026-05-28"
     ],
     [
      "21",
      "gpt-6-luna_max",
      "OpenAI",
      "78.9",
      "2026-09-22"
     ],
     [
      "22",
      "gpt-5.4-2026-03-05_xhigh",
      "OpenAI",
      "78.6",
      "2026-03-05"
     ],
     [
      "23",
      "kimi-k3.1-pro-together_max",
      "Moonshot",
      "78.2",
      "2026-10-23"
     ],
     [
      "24",
      "claude-haiku-5-5_max",
      "Anthropic",
      "75.1",
      "2026-10-07"
     ],
     [
      "25",
      "qwen3.8-max_xhigh",
      "Alibaba",
      "74.7",
      "2026-08-02"
     ]
    ],
    "main_column": "Score (%)",
    "higher_is_better": true,
    "metric": "Share of FrontierMath Tiers 1-3 (v2, private set) maths problems solved (%)",
    "source": {
     "name": "Epoch AI",
     "url": "https://epoch.ai/data/benchmark_data.zip",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0"
    },
    "notes": "The Hub covers dozens of benchmarks; only one representative table is shown here (FrontierMath Tiers 1-3 v2, Epoch's 'best score across scorers', converted from 0-1 to %). Sorted by score; the file itself is not sorted. Data zip generated 2026-11-17."
   },
   "plain": "Epoch AI runs AI models itself on a set of hard tests (maths, science, coding) and publishes the results in one place, so you can compare models on tests the labs did not grade themselves."
  },
  {
   "name": "Epoch Capabilities Index (ECI)",
   "category": "aggregators",
   "url": "https://epoch.ai/eci",
   "paper": null,
   "maintainer": "Epoch AI",
   "measures": "Combines scores from many benchmarks into one capability scale so models evaluated on different benchmarks can be compared.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "epoch-capabilities-index-eci",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "ECI",
     "Access",
     "Release date"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2",
      "Anthropic",
      "173.44",
      "API access",
      "2026-11-09"
     ],
     [
      "2",
      "Gemini 4 Carbon",
      "Google DeepMind",
      "169.76",
      "Unreleased",
      "2026-10-28"
     ],
     [
      "3",
      "GPT-6 Astra",
      "OpenAI",
      "169.0",
      "API access",
      "2026-09-03"
     ],
     [
      "4",
      "Claude Opus 5.5",
      "Anthropic",
      "168.54",
      "API access",
      "2026-09-22"
     ],
     [
      "5",
      "GPT-6.1 Sol",
      "OpenAI",
      "168.48",
      "API access",
      "2026-09-29"
     ],
     [
      "6",
      "GPT-6.1 Terra",
      "OpenAI",
      "167.67",
      "API access",
      "2026-10-26"
     ],
     [
      "7",
      "Claude Fable 5.1",
      "Anthropic",
      "166.99",
      "API access",
      "2026-09-01"
     ],
     [
      "8",
      "Claude Sonnet 5.5",
      "Anthropic",
      "166.56",
      "API access",
      "2026-09-28"
     ],
     [
      "9",
      "Gemini 4 Argon",
      "Google DeepMind",
      "166.48",
      "Limited access",
      "2026-09-30"
     ],
     [
      "10",
      "Claude Opus 5",
      "Anthropic",
      "164.68",
      "API access",
      "2026-07-24"
     ],
     [
      "11",
      "Claude Fable 5",
      "Anthropic",
      "164.13",
      "API access",
      "2026-06-09"
     ],
     [
      "12",
      "GPT-6 Sol",
      "OpenAI",
      "163.75",
      "API access",
      "2026-09-22"
     ],
     [
      "13",
      "GPT-5.6 Sol",
      "OpenAI",
      "162.99",
      "API access",
      "2026-07-09"
     ],
     [
      "14",
      "GPT-5.5 Pro",
      "OpenAI",
      "162.7",
      "API access",
      "2026-04-23"
     ],
     [
      "15",
      "Muse Wonder",
      "Meta AI",
      "161.68",
      "API access",
      "2026-10-12"
     ],
     [
      "16",
      "Kimi K3.1 Pro",
      "Moonshot",
      "161.37",
      "Open weights (restricted use)",
      "2026-10-23"
     ],
     [
      "17",
      "GPT-5.6 Terra",
      "OpenAI",
      "160.73",
      "API access",
      "2026-07-09"
     ],
     [
      "18",
      "Claude Haiku 5.5",
      "Anthropic",
      "160.43",
      "API access",
      "2026-10-07"
     ],
     [
      "19",
      "GPT-5.4 Pro",
      "OpenAI",
      "160.24",
      "API access",
      "2026-03-05"
     ],
     [
      "20",
      "GPT-5.5",
      "OpenAI",
      "160.1",
      "API access",
      "2026-04-23"
     ],
     [
      "21",
      "Claude Opus 4.8",
      "Anthropic",
      "159.94",
      "API access",
      "2026-05-28"
     ],
     [
      "22",
      "DeepSeek V4.1 Flash 1017",
      "DeepSeek",
      "159.41",
      "Open weights (unrestricted)",
      "2026-10-17"
     ],
     [
      "23",
      "Gemini 3.8 Flash",
      "Google DeepMind",
      "158.79",
      "API access",
      "2026-09-02"
     ],
     [
      "24",
      "Kimi K3",
      "Moonshot",
      "158.58",
      "Open weights (non-commercial)",
      "2026-07-16"
     ],
     [
      "25",
      "Gemini 3.7 Flash",
      "Google DeepMind",
      "158.36",
      "API access",
      "2026-08-13"
     ]
    ],
    "main_column": "ECI",
    "higher_is_better": true,
    "metric": "ECI score (higher = more capable; arbitrary scale anchored by reference models)",
    "source": {
     "name": "Epoch AI",
     "url": "https://epoch.ai/data/eci_scores.csv",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-17",
     "license": "CC-BY-4.0 (Epoch AI data; check epoch.ai/data)"
    },
    "notes": "Top 25 of ~256 models by ECI, in file order. Page shows 'Updated Nov. 17, 2026'. Licence taken from Epoch's general data policy, not re-verified on the CSV itself."
   },
   "plain": "One combined score for how capable an AI model is overall, built by stitching together results from many different tests so models from different years can be compared on one scale."
  },
  {
   "name": "Scale SEAL Leaderboards",
   "category": "aggregators",
   "url": "https://labs.scale.com/leaderboard",
   "paper": null,
   "maintainer": "Scale AI",
   "measures": "Collection of expert-built, mostly private evaluations (such as SWE-Bench Pro, MCP Atlas, MultiChallenge, Remote Labor Index, HLE) run by Scale.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Private test sets limit contamination but cannot be inspected.",
   "id": "scale-seal-leaderboards",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Company",
     "Score (%)",
     "Confidence interval (+/-)"
    ],
    "rows": [
     [
      "1",
      "claude-fable-5-2",
      "anthropic",
      "72.4",
      "2.8"
     ],
     [
      "2",
      "gpt-6-astra",
      "openai",
      "60.6",
      "3"
     ],
     [
      "3",
      "claude-opus-5-5",
      "anthropic",
      "55",
      "3.1"
     ],
     [
      "4",
      "gemini-4-argon",
      "google",
      "53.2",
      "3.07"
     ],
     [
      "5",
      "claude-fable-5-1",
      "anthropic",
      "51.3",
      "3.1"
     ],
     [
      "6",
      "claude-opus-5",
      "anthropic",
      "38.6",
      "3"
     ]
    ],
    "main_column": "Score (%)",
    "higher_is_better": true,
    "metric": "Share of expert-level questions answered correctly in Humanity's Last Exam (Diamond) (%)",
    "source": {
     "name": "Scale AI (SEAL)",
     "url": "https://labs.scale.com/leaderboard/hle-diamond",
     "retrieved": "2026-11-17",
     "as_of": "2026-09-23",
     "license": "not stated"
    },
    "notes": "SEAL hosts ~30 leaderboards; only HLE (Diamond) shown, non-deprecated entries. as_of is the board's updatedAt field; newest entries dated Nov 2026. Some entries carry a contamination warning on the site. Data read from page-embedded JSON."
   },
   "plain": "Scale AI tests AI models on private, expert-written questions that models cannot have memorised, across areas like coding, reasoning and safety. One representative board (Humanity's Last Exam, Diamond) is shown."
  },
  {
   "name": "Vals AI",
   "category": "aggregators",
   "url": "https://www.vals.ai/benchmarks",
   "paper": null,
   "maintainer": "Vals AI",
   "measures": "Independent evaluations on industry tasks such as legal, finance, tax, medical and coding work, many with private test sets.",
   "type": "leaderboard",
   "status": "active",
   "notes": null,
   "id": "vals-ai",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Accuracy (%)",
     "Cost / test",
     "Duration / test"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2",
      "71.31",
      "$17.94",
      "43m42s"
     ],
     [
      "2",
      "Claude Sonnet 5.5",
      "70.85",
      "$16.18",
      "1h26m"
     ],
     [
      "3",
      "Gemini 4 Carbon",
      "70.45",
      "$10.51",
      "47m45s"
     ],
     [
      "4",
      "Claude Opus 5.5",
      "70.14",
      "$24.77",
      "1h25m"
     ],
     [
      "5",
      "Gemini 4 Argon",
      "69.39",
      "$13.37",
      "44m58s"
     ],
     [
      "6",
      "Gemini 4 Hydrogen",
      "68.49",
      "$9.84",
      "32m05s"
     ],
     [
      "7",
      "Wonder Full",
      "66.79",
      "$6.81",
      "59m19s"
     ],
     [
      "8",
      "GPT-6 Astra",
      "65.02",
      "$21.28",
      "33m39s"
     ],
     [
      "9",
      "Wonder",
      "64.88",
      "$10.65",
      "35m24s"
     ],
     [
      "10",
      "GPT-6.1 Sol",
      "63.94",
      "$3.61",
      "44m24s"
     ],
     [
      "11",
      "Qwen 4 Max",
      "63.20",
      "$9.33",
      "1h05m"
     ],
     [
      "12",
      "Kimi K3.1 Pro",
      "62.52",
      "$6.44",
      "1h04m"
     ],
     [
      "13",
      "DeepSeek V4.1 Flash 1017",
      "61.26",
      "$0.66",
      "39m55s"
     ],
     [
      "14",
      "GPT-6.1 Terra",
      "60.54",
      "$2.94",
      "39m43s"
     ],
     [
      "15",
      "Muse Ember",
      "60.48",
      "$0.15",
      "15m02s"
     ],
     [
      "16",
      "GLM 5.3",
      "59.85",
      "$8.52",
      "1h04m"
     ],
     [
      "17",
      "Hy4",
      "59.35",
      "$1.72",
      "51m31s"
     ],
     [
      "18",
      "Qwen 3.8 Max 0902",
      "59.05",
      "$8.73",
      "1h46m"
     ],
     [
      "19",
      "Claude Haiku 5.5",
      "58.73",
      "$3.17",
      "49m57s"
     ],
     [
      "20",
      "Grok 4.7",
      "58.62",
      "$11.63",
      "33m34s"
     ],
     [
      "21",
      "MiMo V2.6 Pro",
      "57.81",
      "$0.43",
      "1h13m"
     ],
     [
      "22",
      "Mistral Large 4",
      "56.53",
      "$3.72",
      "47m48s"
     ],
     [
      "23",
      "Inkling 2 Small",
      "56.08",
      "$3.51",
      "50m35s"
     ],
     [
      "24",
      "Gemini 3.8 Flash",
      "55.95",
      "$5.03",
      "35m58s"
     ],
     [
      "25",
      "MiMo V2.6 Flash",
      "54.83",
      "$0.22",
      "1h07m"
     ]
    ],
    "main_column": "Accuracy (%)",
    "higher_is_better": true,
    "metric": "Vals Index accuracy: average performance on finance, coding, legal and tax tasks, weighted by sector (%)",
    "source": {
     "name": "Vals AI",
     "url": "https://www.vals.ai/benchmarks/vals_index",
     "retrieved": "2026-11-17",
     "as_of": "2026-11-15",
     "license": "not stated"
    },
    "notes": "Vals Index v2.2 only; Vals hosts many other benchmarks. Read from the rendered page table."
   },
   "plain": "Vals AI tests AI models on realistic professional work in finance, law, tax and coding and combines the results into one index, aiming to show how useful models are for real jobs."
  },
  {
   "name": "LLM Stats",
   "category": "aggregators",
   "url": "https://llm-stats.com/",
   "paper": null,
   "maintainer": "LLM Stats",
   "measures": "Collects benchmark scores, prices and context sizes for many models into comparable tables.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Mostly aggregates scores reported by model developers rather than running its own tests.",
   "id": "llm-stats",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Organization",
     "LLM Stats score",
     "Reasoning",
     "Coding",
     "Agent"
    ],
    "rows": [
     [
      "1",
      "Claude Fable 5.2",
      "Anthropic",
      "60.8",
      "60.0",
      "49.8",
      "46.3"
     ],
     [
      "2",
      "Claude Mythos 5.2",
      "Anthropic",
      "59.3",
      "53.4",
      "44.5",
      "44.2"
     ],
     [
      "3",
      "Claude Opus 5.5",
      "Anthropic",
      "58.5",
      "57.4",
      "47.8",
      "43.6"
     ],
     [
      "4",
      "Gemini 4 Carbon",
      "Google",
      "57.4",
      "54.0",
      "45.7",
      "43.7"
     ],
     [
      "5",
      "Claude Sonnet 5.5",
      "Anthropic",
      "56.7",
      "55.4",
      "46.0",
      "41.9"
     ],
     [
      "6",
      "GPT-6 Astra",
      "OpenAI",
      "56.3",
      "55.0",
      "43.9",
      "42.3"
     ],
     [
      "7",
      "Kimi K3.1 Pro",
      "Moonshot AI",
      "54.4",
      "53.3",
      "40.5",
      "39.9"
     ],
     [
      "8",
      "Claude Mythos Preview",
      "Anthropic",
      "53.6",
      "54.6",
      "42.5",
      "35.1"
     ],
     [
      "9",
      "Gemini 4 Argon",
      "Google",
      "53.3",
      "51.2",
      "41.3",
      "39.6"
     ],
     [
      "10",
      "Muse Wonder Full",
      "Meta",
      "52.6",
      "44.8",
      "42.0",
      "40.4"
     ],
     [
      "11",
      "Claude Opus 5",
      "Anthropic",
      "52.5",
      "52.5",
      "37.3",
      "36.7"
     ],
     [
      "12",
      "GPT-6.1 Sol",
      "OpenAI",
      "52.2",
      "49.6",
      "37.6",
      "35.8"
     ],
     [
      "13",
      "Claude Fable 5.1",
      "Anthropic",
      "52.0",
      "49.8",
      "34.4",
      "37.1"
     ],
     [
      "14",
      "Muse Wonder Full",
      "Meta",
      "52.0",
      "40.1",
      "40.9",
      "40.0"
     ],
     [
      "15",
      "Claude Fable 5",
      "Anthropic",
      "51.9",
      "49.8",
      "41.6",
      "37.0"
     ]
    ],
    "main_column": "LLM Stats score",
    "higher_is_better": true,
    "metric": "LLM Stats composite score (higher is better) from public benchmark results",
    "source": {
     "name": "LLM Stats (llm-stats.com)",
     "url": "https://llm-stats.com/",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "Homepage leaderboard table (page says 'updated continuously'); includes some unreleased/preview models. Scores are an aggregate of other benchmarks' public results."
   },
   "plain": "A website that gathers public test results, prices and speeds for hundreds of AI models and ranks them with a single combined score."
  },
  {
   "name": "Kagi LLM Benchmarking Project",
   "category": "aggregators",
   "url": "https://help.kagi.com/kagi/ai/llm-benchmark.html",
   "paper": null,
   "maintainer": "Kagi",
   "measures": "Private, frequently changed set of reasoning, coding and instruction-following questions, reporting accuracy along with cost and speed.",
   "type": "leaderboard",
   "status": "active",
   "notes": "Test questions are not public.",
   "id": "kagi-llm-benchmarking-project",
   "results": {
    "columns": [
     "Rank",
     "Model",
     "Accuracy (%)",
     "Cost ($, whole suite)",
     "Provider"
    ],
    "rows": [
     [
      "1",
      "claude-5-fable-thinking",
      "91.4",
      "10.4",
      "kagi (ult)"
     ],
     [
      "2",
      "claude-5-fable",
      "88.8",
      "9.9",
      "kagi (ult)"
     ],
     [
      "3",
      "claude-4-8-opus-thinking",
      "88.8",
      "8.8",
      "kagi (ult)"
     ],
     [
      "4",
      "gpt-5-5",
      "88.8",
      "2.2",
      "kagi (ult)"
     ],
     [
      "5",
      "claude-4-6-opus-thinking",
      "83.6",
      "4.8",
      "kagi (ult)"
     ],
     [
      "6",
      "grok-4-5",
      "83.5",
      "1.0",
      "kagi (ult)"
     ],
     [
      "7",
      "claude-4-7-opus-thinking",
      "80.7",
      "2.8",
      "kagi (ult)"
     ],
     [
      "8",
      "claude-4-5-opus-thinking",
      "80.2",
      "4.5",
      "kagi (ult)"
     ],
     [
      "9",
      "gemini-3-pro",
      "80.1",
      "0.4",
      "kagi (ult)"
     ],
     [
      "10",
      "kimi-k2-5-thinking",
      "78.5",
      "0.5",
      "kagi"
     ],
     [
      "11",
      "gpt-5-pro",
      "76.8",
      "31.8",
      "kagi (ult)"
     ],
     [
      "12",
      "glm-5-thinking",
      "75.0",
      "2.0",
      "openrouter"
     ],
     [
      "13",
      "grok-4-20-thinking",
      "75.0",
      "1.3",
      "kagi (ult)"
     ],
     [
      "14",
      "chatgpt-4o (points to gpt 5.3)",
      "75.0",
      "0.8",
      "kagi (ult)"
     ],
     [
      "15",
      "claude-4-opus-thinking",
      "74.3",
      "22.4",
      "kagi (ult)"
     ],
     [
      "16",
      "qwen-3-5-397b",
      "73.7",
      "1.8",
      "together"
     ],
     [
      "17",
      "grok-4",
      "73.6",
      "1.0",
      "kagi (ult)"
     ],
     [
      "18",
      "gpt-5-2",
      "73.3",
      "1.4",
      "kagi (ult)"
     ],
     [
      "19",
      "claude-4-7-opus",
      "73.3",
      "2.2",
      "kagi (ult)"
     ],
     [
      "20",
      "claude-4-sonnet-thinking",
      "73.0",
      "5.4",
      "kagi (ult)"
     ],
     [
      "21",
      "gpt-5",
      "72.7",
      "7.1",
      "kagi (ult)"
     ],
     [
      "22",
      "qwen3-max",
      "72.5",
      "1.9",
      "openrouter"
     ],
     [
      "23",
      "claude-4-6-opus",
      "72.4",
      "1.6",
      "kagi (ult)"
     ],
     [
      "24",
      "o3-pro",
      "72.1",
      "34.2",
      "kagi (ult)"
     ],
     [
      "25",
      "claude-4-5-opus",
      "70.7",
      "1.4",
      "kagi (ult)"
     ]
    ],
    "main_column": "Accuracy (%)",
    "higher_is_better": true,
    "metric": "Share of benchmark tasks answered correctly (%)",
    "source": {
     "name": "Kagi Search",
     "url": "https://help.kagi.com/kagi/ai/llm-benchmark.html",
     "retrieved": "2026-11-17",
     "as_of": null,
     "license": "not stated"
    },
    "notes": "Reasoning benchmark table, 116 tasks, last task list revision Feb 13, 2026; page has no explicit update date but lists models released after that. Order as published (by accuracy)."
   },
   "plain": "Kagi, a search company, regularly quizzes AI models on a private set of tricky reasoning, coding and language questions that change over time, to see which ones actually think well."
  }
 ]
}