{
  "id": "big-bench-hard",
  "code": "PTL-0191",
  "term": "BIG-Bench Hard",
  "aliases": [
    "BBH",
    "BIG-bench"
  ],
  "category": "benchmarks",
  "definition": "BIG-Bench Hard (BBH) is a set of 23 challenging tasks from the BIG-bench collection on which earlier language models performed below average human raters, used to measure multi-step reasoning.",
  "description": "Its creators showed that chain-of-thought prompting greatly improved results on these tasks, which made BBH a standard test of reasoning-elicitation techniques. BIG-bench itself is a collaborative collection of over 200 tasks.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "chain-of-thought",
    "few-shot-prompting"
  ],
  "introduced": 2022,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them",
      "authors": "Suzgun et al.",
      "year": 2022,
      "url": "https://arxiv.org/abs/2210.09261"
    },
    {
      "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "authors": "Srivastava et al.",
      "year": 2022,
      "url": "https://arxiv.org/abs/2206.04615"
    }
  ],
  "url": "https://protologue.com/t/big-bench-hard/",
  "citation": "Protologue. (2026). BIG-Bench Hard. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0191). https://protologue.com/t/big-bench-hard/"
}