{
  "id": "llm-judge-bias",
  "code": "PTL-0177",
  "term": "LLM Judge Bias",
  "aliases": [
    "position bias",
    "verbosity bias",
    "self-enhancement bias",
    "self-preference bias"
  ],
  "category": "failure-modes",
  "definition": "LLM judge bias is the set of systematic errors that language models make when grading outputs, notably preferring the answer shown first (position bias), longer answers (verbosity bias), and their own outputs (self-enhancement bias).",
  "description": "These biases can make automated evaluations favor the wrong prompt or model. Common mitigations are evaluating each pair in both orders and counting only consistent verdicts, instructing the judge to ignore length, using clear rubrics and reference answers, and checking judge agreement against a sample of human ratings.",
  "example": null,
  "broader": [],
  "narrower": [],
  "related": [
    "llm-as-a-judge",
    "evals",
    "sycophancy"
  ],
  "introduced": 2023,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "authors": "Zheng et al.",
      "year": 2023,
      "url": "https://arxiv.org/abs/2306.05685"
    }
  ],
  "url": "https://protologue.com/t/llm-judge-bias/",
  "citation": "Protologue. (2026). LLM Judge Bias. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0177). https://protologue.com/t/llm-judge-bias/"
}