[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"glossary-eval-set-leakage::en":3,"gloss-cluster-eval-set-leakage::en":23,"gloss-next-eval-set-leakage::en":9},{"slug":4,"category":5,"name":6,"definition":7,"meta_desc":8,"faq":9,"schema_markup":9,"related":10},"eval-set-leakage","mlops","Eval Set Leakage","Eval set leakage is your own evaluation cases finding their way inside the system you are evaluating. It is the local cousin of benchmark contamination, which concerns public test material ending up in a model's training data. This version is entirely self-inflicted and considerably easier to cause. The routes are ordinary. A few tricky cases from the set are pasted into a prompt as examples, because they are exactly the cases someone was trying to fix. The document collection the system retrieves from is refreshed with an export that happens to include the labelled examples. A fine-tuning dataset is assembled from the same source as the evaluation material without anyone checking the overlap. A prompt is tuned by iterating against the set until the score is satisfactory, which is a slower and subtler version of the same thing: the set has been fitted to rather than measured against. The symptom is a score that improves in a step, without a matching improvement anyone can see in production, and often with the gain concentrated in the specific cases someone had been working on. That pattern deserves suspicion rather than a celebration email. Prevention is mostly procedural. Keep the evaluation material somewhere the system cannot read from, and keep a separate pool of examples for prompt material so that nothing has to be borrowed. Check overlap between training or retrieval sources and the set mechanically rather than by memory. And accept that a set which has been iterated against many times is partly spent whatever the precautions were: reserve a portion that is looked at rarely, and refresh the working part on a schedule, so there is always some material the system has never been tuned towards. One more habit helps: record where each case came from when it enters the set. Provenance is what lets you answer, six months later, whether a group of examples has also been used somewhere else, and it turns an unanswerable suspicion into a query.","Eval set leakage is your own test cases getting inside the system under test. The ordinary routes, the step-change symptom, and how to prevent it.",null,[11,14,17,20],{"slug":12,"name":13},"benchmark-contamination","Benchmark Contamination",{"slug":15,"name":16},"eval-harness","Eval Harness",{"slug":18,"name":19},"golden-dataset","Golden Dataset",{"slug":21,"name":22},"train-test-split","Train\u002FTest Split",[24,28,31,35,38,41,44,47,50,53,56,59],{"slug":25,"category":5,"name":26,"updated_at":27},"acceptance-threshold","Acceptance Threshold","2026-09-10T03:30:02+00:00",{"slug":29,"category":5,"name":30,"updated_at":27},"adjudication","Adjudication",{"slug":32,"category":5,"name":33,"updated_at":34},"annotation-guidelines","Annotation Guidelines","2026-09-08T21:32:36+00:00",{"slug":36,"category":5,"name":37,"updated_at":34},"baseline-model","Baseline Model",{"slug":39,"category":5,"name":40,"updated_at":34},"batch-inference","Batch Inference",{"slug":42,"category":5,"name":43,"updated_at":27},"blind-evaluation","Blind Evaluation",{"slug":45,"category":5,"name":46,"updated_at":34},"canary-prompt","Canary Prompt",{"slug":48,"category":5,"name":49,"updated_at":27},"ceiling-effect","Ceiling Effect",{"slug":51,"category":5,"name":52,"updated_at":34},"champion-challenger","Champion-Challenger (A\u002FB Model Testing)",{"slug":54,"category":5,"name":55,"updated_at":34},"class-imbalance","Class Imbalance",{"slug":57,"category":5,"name":58,"updated_at":34},"continuous-batching","Continuous Batching",{"slug":60,"category":5,"name":61,"updated_at":27},"coverage-gap","Coverage Gap"]