{
  "id": "knowledge-distillation",
  "code": "PTL-0183",
  "term": "Knowledge Distillation",
  "aliases": [
    "distillation",
    "model distillation",
    "teacher-student training"
  ],
  "category": "foundations",
  "definition": "Knowledge distillation is training a smaller student model to imitate a larger teacher model, using the teacher's output probabilities or generated responses as training targets, so that the student keeps much of the teacher's ability at lower cost.",
  "description": "The original method trains on the teacher's full probability distribution, whose soft targets carry more information than single correct labels. With LLMs, distillation is commonly done by fine-tuning a small model on responses, including reasoning traces, generated by a large one, which is how several small reasoning models have been built. Some model providers' terms restrict using their outputs to train competing models.",
  "example": null,
  "broader": [
    "fine-tuning"
  ],
  "narrower": [],
  "related": [
    "reasoning-model",
    "model-cascade"
  ],
  "introduced": 2015,
  "added": "2026-10-10",
  "sources": [
    {
      "title": "Distilling the Knowledge in a Neural Network",
      "authors": "Hinton et al.",
      "year": 2015,
      "url": "https://arxiv.org/abs/1503.02531"
    },
    {
      "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
      "authors": "DeepSeek-AI et al.",
      "year": 2025,
      "url": "https://arxiv.org/abs/2501.12948"
    }
  ],
  "url": "https://protologue.com/t/knowledge-distillation/",
  "citation": "Protologue. (2026). Knowledge Distillation. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0183). https://protologue.com/t/knowledge-distillation/"
}