{
  "id": "pass-at-k",
  "code": "PTL-0180",
  "term": "pass@k",
  "aliases": [
    "pass@1",
    "pass at k"
  ],
  "category": "failure-modes",
  "definition": "pass@k is an evaluation metric for code generation, and other tasks with automatic checks, defined as the probability that at least one of k sampled solutions to a problem passes its tests.",
  "description": "It was popularized with an unbiased estimator computed from n ≥ k samples per problem. pass@1 measures single-attempt reliability, while larger k measures whether the model can solve the problem at all, which is relevant when best-of-n sampling or verification is available.",
  "example": null,
  "broader": [
    "evals"
  ],
  "narrower": [],
  "related": [
    "best-of-n-sampling",
    "coding-agent",
    "rlvr"
  ],
  "introduced": 2021,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Evaluating Large Language Models Trained on Code",
      "authors": "Chen et al.",
      "year": 2021,
      "url": "https://arxiv.org/abs/2107.03374"
    }
  ],
  "url": "https://protologue.com/t/pass-at-k/",
  "citation": "Protologue. (2026). pass@k. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0180). https://protologue.com/t/pass-at-k/"
}