{
  "id": "benchmark",
  "code": "PTL-0185",
  "term": "Benchmark",
  "aliases": [
    "LLM benchmark",
    "evaluation benchmark"
  ],
  "category": "benchmarks",
  "definition": "A benchmark is a fixed, public set of test tasks with a standard scoring method, used to measure and compare the capabilities of language models.",
  "description": "Reported scores depend on the prompting setup, such as the number of examples, whether chain-of-thought or extended thinking is used, and how many attempts are sampled, so scores are only comparable under the same conditions. Public benchmarks also risk data contamination and eventually saturate. They are useful for choosing models, but an application's own evals are the better guide to how a model will perform on its tasks.",
  "example": null,
  "broader": [],
  "narrower": [
    "benchmark-saturation",
    "mmlu",
    "mmlu-pro",
    "gpqa",
    "humanitys-last-exam",
    "big-bench-hard",
    "arc-agi",
    "simpleqa",
    "truthfulqa",
    "hellaswag",
    "mmmu",
    "gsm8k",
    "math-benchmark",
    "aime",
    "humaneval",
    "livecodebench",
    "swe-bench",
    "terminal-bench",
    "tau-bench",
    "osworld",
    "gaia",
    "browsecomp",
    "ifeval",
    "mt-bench",
    "chatbot-arena",
    "ruler"
  ],
  "related": [
    "evals",
    "data-contamination",
    "pass-at-k"
  ],
  "introduced": null,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Holistic Evaluation of Language Models",
      "authors": "Liang et al.",
      "year": 2022,
      "url": "https://arxiv.org/abs/2211.09110"
    },
    {
      "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "authors": "Srivastava et al.",
      "year": 2022,
      "url": "https://arxiv.org/abs/2206.04615"
    }
  ],
  "url": "https://protologue.com/t/benchmark/",
  "citation": "Protologue. (2026). Benchmark. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0185). https://protologue.com/t/benchmark/"
}