{
  "id": "swe-bench",
  "code": "PTL-0202",
  "term": "SWE-bench",
  "aliases": [
    "SWE-bench Verified"
  ],
  "category": "benchmarks",
  "definition": "SWE-bench is a benchmark of real GitHub issues from popular Python repositories, in which a model or agent must edit the codebase so that tests written for the actual fix pass.",
  "description": "It measures practical software engineering in large, unfamiliar codebases rather than isolated functions, and is the standard benchmark for coding agents. Because some original tasks were underspecified or had faulty tests, a 500-task subset screened by human developers, SWE-bench Verified, is the version most often reported.",
  "example": null,
  "broader": [
    "benchmark"
  ],
  "narrower": [],
  "related": [
    "coding-agent",
    "humaneval",
    "terminal-bench"
  ],
  "introduced": 2023,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
      "authors": "Jimenez et al.",
      "year": 2023,
      "url": "https://arxiv.org/abs/2310.06770"
    },
    {
      "title": "Introducing SWE-bench Verified",
      "authors": "OpenAI",
      "year": 2024,
      "url": "https://openai.com/index/introducing-swe-bench-verified/"
    }
  ],
  "url": "https://protologue.com/t/swe-bench/",
  "citation": "Protologue. (2026). SWE-bench. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0202). https://protologue.com/t/swe-bench/"
}