[{"data":1,"prerenderedAt":222},["ShallowReactive",2],{"navigation_docs":3,"-replay-limits":134,"-replay-limits-surround":217},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":100,"body":136,"description":210,"extension":211,"links":212,"meta":213,"navigation":214,"path":101,"seo":215,"stem":102,"__hash__":216},"docs\u002F4.replay\u002F5.limits.md",{"type":137,"value":138,"toc":200},"minimark",[139,144,148,156,160,163,167,170,174,182,186,189,193],[140,141,143],"h2",{"id":142},"capacity-to-plan-a-fleet-by","Capacity to plan a fleet by",[145,146,147],"p",{},"The replay sends the requests of its sample at steady rates: each goes out on schedule, whether or not the ones before it have finished. Real users send their next message after they have read the last answer, so under load they slow down with the engine; and a sample of 300 requests is a few seconds to a few minutes of your traffic.",[145,149,150,151,155],{},"On the H100 test, the replay found 22.5 requests a second where the same traffic, sent live, was comfortable at about 22: one test, on one traffic mix. Use a replay's rates ",[152,153,154],"strong",{},"to compare configs with each other",", and before you plan a fleet by them, for example to run fewer replicas, confirm the rate on live traffic.",[140,157,159],{"id":158},"answer-quality","Answer quality",[145,161,162],{},"The replay sends token ids, not text, and does not look at the answers. A config that changes precision, such as FP8 weights or an FP8 KV cache, is measured but never recommended. Check its answers on your own evaluation before you adopt it.",[140,164,166],{"id":165},"speculative-decoding","Speculative decoding",[145,168,169],{},"How much speculative decoding speeds up generation depends on the content of the text. Synthetic prompts have none, so the replay cannot measure it.",[140,171,173],{"id":172},"downtime-on-a-single-replica","Downtime on a single replica",[145,175,176,177,181],{},"The tested replica takes no traffic for the length of the run. With one replica per model, that model is unavailable meanwhile: ",[178,179,180],"code",{},"--single-replica"," asks you to accept that, so run it when traffic is low.",[140,183,185],{"id":184},"gpus-you-do-not-have","GPUs you do not have",[145,187,188],{},"A replay runs on the replica's own GPUs. To compare another GPU type, you need a machine with it, set up like the others.",[140,190,192],{"id":191},"kubernetes","Kubernetes",[145,194,195,196,199],{},"There is no launcher for Kubernetes yet. See ",[197,198,90],"a",{"href":91},".",{"title":201,"searchDepth":202,"depth":202,"links":203},"",2,[204,205,206,207,208,209],{"id":142,"depth":202,"text":143},{"id":158,"depth":202,"text":159},{"id":165,"depth":202,"text":166},{"id":172,"depth":202,"text":173},{"id":184,"depth":202,"text":185},{"id":191,"depth":202,"text":192},"Where the replay's numbers stop, so you can use them safely.","md",null,{},{"icon":103},{"title":100,"description":210},"8RXMXNWBXRJGNQosCA_ftqXncpHORb7v0YdhBce0aLI",[218,220],{"title":95,"path":96,"stem":97,"description":219,"icon":98,"children":-1},"List the engine configs a replay tries.",{"title":110,"path":111,"stem":112,"description":221,"icon":113,"children":-1},"Every key of the YAML config file, its default, and which part of ico reads it.",1792165824444]