[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"glossary-model-serving::en":3,"gloss-cluster-model-serving::en":26,"gloss-next-model-serving::en":9},{"slug":4,"category":5,"name":6,"definition":7,"meta_desc":8,"faq":9,"schema_markup":9,"related":10},"model-serving","mlops","Model Serving","Model serving is the infrastructure layer that puts a trained model behind an API so applications can request predictions in real time. It handles loading the weights into memory, batching incoming requests, autoscaling GPUs, and exposing versioned endpoints. If training is a one-time event, serving is the part that runs forever — and it is where latency, cost, and reliability are actually won or lost. When you call a closed model like Claude or GPT, the vendor handles serving for you. The topic becomes hands-on the moment you self-host an open-weight model, where you reach for engines like vLLM, Text Generation Inference, or NVIDIA Triton, or managed platforms such as Modal, Replicate, and Baseten. Techniques like request batching and quantization dramatically cut the per-request GPU bill. Practical note: measure tail latency and cost per thousand requests, not just averages — one slow percentile can wreck a real-time feature, and idle GPUs quietly drain a startup's runway.","Model serving is the layer that puts a trained model behind an API — loading weights, batching requests, autoscaling GPUs, and exposing versioned endpoints.",null,[11,14,17,20,23],{"slug":12,"name":13},"gpu","GPU (Graphics Processing Unit)",{"slug":15,"name":16},"inference","Inference",{"slug":18,"name":19},"latency","Latency",{"slug":21,"name":22},"model-weights","Model Weights",{"slug":24,"name":25},"quantization","Quantization",[27,31,35,38,41,45,48,51,54,57,60,63],{"slug":28,"category":5,"name":29,"updated_at":30},"annotation-guidelines","Annotation Guidelines","2026-08-24T03:30:02+00:00",{"slug":32,"category":5,"name":33,"updated_at":34},"baseline-model","Baseline Model","2026-08-24T02:46:38+00:00",{"slug":36,"category":5,"name":37,"updated_at":34},"batch-inference","Batch Inference",{"slug":39,"category":5,"name":40,"updated_at":34},"canary-prompt","Canary Prompt",{"slug":42,"category":5,"name":43,"updated_at":44},"champion-challenger","Champion-Challenger (A\u002FB Model Testing)","2026-08-24T02:46:37+00:00",{"slug":46,"category":5,"name":47,"updated_at":34},"class-imbalance","Class Imbalance",{"slug":49,"category":5,"name":50,"updated_at":34},"continuous-batching","Continuous Batching",{"slug":52,"category":5,"name":53,"updated_at":34},"cross-validation","Cross-Validation",{"slug":55,"category":5,"name":56,"updated_at":34},"data-labeling","Data Labeling",{"slug":58,"category":5,"name":59,"updated_at":44},"drift-detection","Drift Detection",{"slug":61,"category":5,"name":62,"updated_at":44},"eval-harness","Eval Harness",{"slug":64,"category":5,"name":65,"updated_at":44},"experiment-tracking","Experiment Tracking"]