{
  "id": "mixture-of-experts",
  "code": "PTL-0229",
  "term": "Mixture of Experts",
  "aliases": [
    "MoE",
    "sparse mixture of experts",
    "sparsely-gated mixture of experts"
  ],
  "category": "foundations",
  "definition": "Mixture of experts (MoE) is a neural network architecture in which a layer contains many parallel sub-networks, called experts, and a learned router sends each token to only a few of them, so a model's total parameter count can grow much faster than the computation it uses per token.",
  "description": "The sparsely gated mixture-of-experts layer placed thousands of experts between the layers of a language model and activated only a small number per input. The Switch Transformer simplified this by routing each token to a single expert, which reduced communication costs and made training at a trillion parameters practical. MoE language models are therefore described by two numbers, total parameters and active parameters per token. Training needs a balancing mechanism so the router does not send most tokens to a few experts, and all experts must still be held in memory at inference time. The name is unrelated to mixture-of-agents, which combines the outputs of separate models.",
  "example": null,
  "broader": [],
  "narrower": [],
  "related": [
    "large-language-model",
    "foundation-model",
    "routing",
    "mixture-of-agents"
  ],
  "introduced": 2017,
  "added": "2026-10-27",
  "sources": [
    {
      "title": "Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer",
      "authors": "Shazeer et al.",
      "year": 2017,
      "url": "https://arxiv.org/abs/1701.06538"
    },
    {
      "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity",
      "authors": "Fedus, Zoph and Shazeer",
      "year": 2021,
      "url": "https://arxiv.org/abs/2101.03961"
    }
  ],
  "url": "https://protologue.com/t/mixture-of-experts/",
  "citation": "Protologue. (2026). Mixture of Experts. In Protologue: A Taxonomy of Prompting and LLM Techniques (v1.4.0, PTL-0229). https://protologue.com/t/mixture-of-experts/"
}