{
  "id": "evals",
  "code": "PTL-0179",
  "term": "Evals",
  "aliases": [
    "evaluations",
    "LLM evaluation",
    "eval suite",
    "test set"
  ],
  "category": "failure-modes",
  "definition": "Evals are systematic tests of a model or LLM application, consisting of a set of representative inputs and a grading method, used to measure performance against defined success criteria and to compare prompts, models, and changes.",
  "description": "Grading can be exact-match or code-based, done by an LLM judge with a rubric, or done by people. Code-based grading is fastest and most reliable where it applies. A good eval set mirrors real usage, including edge cases, and turns prompt engineering from guesswork into measurement. Without one, changes that fix one case can silently break others.",
  "example": null,
  "broader": [],
  "narrower": [
    "pass-at-k",
    "rag-evaluation"
  ],
  "related": [
    "llm-as-a-judge",
    "prompt-optimization",
    "data-contamination"
  ],
  "introduced": null,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Define success criteria and build evaluations",
      "authors": "Anthropic",
      "year": 2024,
      "url": "https://platform.claude.com/docs/en/test-and-evaluate/develop-tests"
    },
    {
      "title": "Holistic Evaluation of Language Models",
      "authors": "Liang et al.",
      "year": 2022,
      "url": "https://arxiv.org/abs/2211.09110"
    }
  ],
  "url": "https://protologue.com/t/evals/",
  "citation": "Protologue. (2026). Evals. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0179). https://protologue.com/t/evals/"
}