{
  "id": "simpleqa",
  "code": "PTL-0193",
  "term": "SimpleQA",
  "aliases": [],
  "category": "benchmarks",
  "definition": "SimpleQA is a benchmark of short, fact-seeking questions with a single indisputable answer, collected adversarially against strong models, that measures whether a model answers factual questions correctly and declines when it does not know.",
  "description": "Answers are graded as correct, incorrect, or not attempted, so it rewards knowing what you don't know as well as knowing facts. It is used to measure hallucination in closed-book settings and the effect of adding search.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "hallucination",
    "confidence-calibration",
    "web-search-tool",
    "truthfulqa"
  ],
  "introduced": 2024,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Measuring short-form factuality in large language models",
      "authors": "Wei et al.",
      "year": 2024,
      "url": "https://arxiv.org/abs/2411.04368"
    }
  ],
  "url": "https://protologue.com/t/simpleqa/",
  "citation": "Protologue. (2026). SimpleQA. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0193). https://protologue.com/t/simpleqa/"
}