{
  "id": "humaneval",
  "code": "PTL-0200",
  "term": "HumanEval",
  "aliases": [],
  "category": "benchmarks",
  "definition": "HumanEval is a benchmark of 164 hand-written Python programming problems, each with a function signature, a description, and hidden unit tests, used to measure code generation with the pass@k metric.",
  "description": "It established functional correctness, whether generated code passes tests, as the standard for evaluating code models, rather than similarity to a reference solution. It is now saturated and small, so harder and contamination-resistant benchmarks such as LiveCodeBench and SWE-bench have taken its place.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "pass-at-k",
    "livecodebench",
    "swe-bench"
  ],
  "introduced": 2021,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Evaluating Large Language Models Trained on Code",
      "authors": "Chen et al.",
      "year": 2021,
      "url": "https://arxiv.org/abs/2107.03374"
    }
  ],
  "url": "https://protologue.com/t/humaneval/",
  "citation": "Protologue. (2026). HumanEval. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0200). https://protologue.com/t/humaneval/"
}