{
  "id": "humanitys-last-exam",
  "code": "PTL-0190",
  "term": "Humanity's Last Exam",
  "aliases": [
    "HLE"
  ],
  "category": "benchmarks",
  "definition": "Humanity's Last Exam (HLE) is a benchmark of 2,500 expert-written questions across dozens of academic subjects, some with images, designed to remain hard for frontier models after earlier academic benchmarks saturated.",
  "description": "Questions were contributed by subject experts and kept only if leading models failed them at the time, and answers are short and automatically checkable. Its creators also measure calibration, how well a model's stated confidence matches its accuracy, because models often answer the questions they get wrong with high confidence.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "benchmark-saturation",
    "gpqa",
    "confidence-calibration"
  ],
  "introduced": 2025,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Humanity's Last Exam",
      "authors": "Phan et al.",
      "year": 2025,
      "url": "https://arxiv.org/abs/2501.14249"
    }
  ],
  "url": "https://protologue.com/t/humanitys-last-exam/",
  "citation": "Protologue. (2026). Humanity's Last Exam. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0190). https://protologue.com/t/humanitys-last-exam/"
}