{
  "id": "benchmark-saturation",
  "code": "PTL-0186",
  "term": "Benchmark Saturation",
  "aliases": [
    "saturated benchmark"
  ],
  "category": "benchmarks",
  "definition": "Benchmark saturation is when the best models score near a benchmark's maximum, so that it no longer distinguishes between them and its remaining errors mostly reflect flawed or ambiguous questions.",
  "description": "Saturation has followed every widely used benchmark, often within a few years, and is a main reason new, harder benchmarks keep appearing. MMLU-Pro and Humanity's Last Exam were both created because earlier benchmarks had saturated. Contamination can make a benchmark look saturated before models have truly mastered it.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "data-contamination",
    "mmlu",
    "humanitys-last-exam"
  ],
  "introduced": null,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Humanity's Last Exam",
      "authors": "Phan et al.",
      "year": 2025,
      "url": "https://arxiv.org/abs/2501.14249"
    },
    {
      "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
      "authors": "Wang et al.",
      "year": 2024,
      "url": "https://arxiv.org/abs/2406.01574"
    }
  ],
  "url": "https://protologue.com/t/benchmark-saturation/",
  "citation": "Protologue. (2026). Benchmark Saturation. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0186). https://protologue.com/t/benchmark-saturation/"
}