[{"data":1,"prerenderedAt":464},["ShallowReactive",2],{"navigation_docs":3,"-replay-sweep-file":134,"-replay-sweep-file-surround":459},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":95,"body":136,"description":452,"extension":453,"links":454,"meta":455,"navigation":456,"path":96,"seo":457,"stem":97,"__hash__":458},"docs\u002F4.replay\u002F4.sweep-file.md",{"type":137,"value":138,"toc":447},"minimark",[139,143,286,291,294,360,363,367,390,394,397,440,443],[140,141,142],"p",{},"A sweep file lists the configs to try. Each config is the replica's flags in production with some of them changed. The current config is always measured too, first.",[144,145,151],"pre",{"className":146,"code":147,"filename":148,"language":149,"meta":150,"style":150},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# Configs listed one by one.\nconfigs:\n  - max-num-batched-tokens: 16384\n  - max-num-batched-tokens: 2048\n  - enable-prefix-caching: false\n  - kv-cache-dtype: fp8\n  - quantization: fp8\n\n# Every combination of these values, on top of the list above.\ngrid:\n  max-num-seqs: [128, 256]\n","sweep.yaml","yaml","",[152,153,154,163,174,190,202,216,230,242,249,255,263],"code",{"__ignoreMap":150},[155,156,159],"span",{"class":157,"line":158},"line",1,[155,160,162],{"class":161},"sHwdD","# Configs listed one by one.\n",[155,164,166,170],{"class":157,"line":165},2,[155,167,169],{"class":168},"swJcz","configs",[155,171,173],{"class":172},"sMK4o",":\n",[155,175,177,180,183,186],{"class":157,"line":176},3,[155,178,179],{"class":172},"  -",[155,181,182],{"class":168}," max-num-batched-tokens",[155,184,185],{"class":172},":",[155,187,189],{"class":188},"sbssI"," 16384\n",[155,191,193,195,197,199],{"class":157,"line":192},4,[155,194,179],{"class":172},[155,196,182],{"class":168},[155,198,185],{"class":172},[155,200,201],{"class":188}," 2048\n",[155,203,205,207,210,212],{"class":157,"line":204},5,[155,206,179],{"class":172},[155,208,209],{"class":168}," enable-prefix-caching",[155,211,185],{"class":172},[155,213,215],{"class":214},"sfNiH"," false\n",[155,217,219,221,224,226],{"class":157,"line":218},6,[155,220,179],{"class":172},[155,222,223],{"class":168}," kv-cache-dtype",[155,225,185],{"class":172},[155,227,229],{"class":228},"sfazB"," fp8\n",[155,231,233,235,238,240],{"class":157,"line":232},7,[155,234,179],{"class":172},[155,236,237],{"class":168}," quantization",[155,239,185],{"class":172},[155,241,229],{"class":228},[155,243,245],{"class":157,"line":244},8,[155,246,248],{"emptyLinePlaceholder":247},true,"\n",[155,250,252],{"class":157,"line":251},9,[155,253,254],{"class":161},"# Every combination of these values, on top of the list above.\n",[155,256,258,261],{"class":157,"line":257},10,[155,259,260],{"class":168},"grid",[155,262,173],{"class":172},[155,264,266,269,271,274,277,280,283],{"class":157,"line":265},11,[155,267,268],{"class":168},"  max-num-seqs",[155,270,185],{"class":172},[155,272,273],{"class":172}," [",[155,275,276],{"class":188},"128",[155,278,279],{"class":172},",",[155,281,282],{"class":188}," 256",[155,284,285],{"class":172},"]\n",[287,288,290],"h2",{"id":289},"how-flags-are-written","How flags are written",[140,292,293],{},"Flags are written as vLLM takes them, without the leading dashes:",[295,296,297,310],"table",{},[298,299,300],"thead",{},[301,302,303,307],"tr",{},[304,305,306],"th",{},"In the sweep file",[304,308,309],{},"On vLLM's command line",[311,312,313,326,338,350],"tbody",{},[301,314,315,321],{},[316,317,318],"td",{},[152,319,320],{},"max-num-seqs: 128",[316,322,323],{},[152,324,325],{},"--max-num-seqs 128",[301,327,328,333],{},[316,329,330],{},[152,331,332],{},"enable-prefix-caching: true",[316,334,335],{},[152,336,337],{},"--enable-prefix-caching",[301,339,340,345],{},[316,341,342],{},[152,343,344],{},"enable-prefix-caching: false",[316,346,347],{},[152,348,349],{},"--no-enable-prefix-caching",[301,351,352,357],{},[316,353,354],{},[152,355,356],{},"max-model-len: null",[316,358,359],{},"the flag is removed",[140,361,362],{},"A config that comes out the same as one already listed, or as the current config, is tried once.",[287,364,366],{"id":365},"configs-that-change-precision","Configs that change precision",[140,368,369,370,373,374,373,377,380,381,384,385,389],{},"A config that changes ",[152,371,372],{},"quantization",", ",[152,375,376],{},"dtype",[152,378,379],{},"kv-cache-dtype"," or ",[152,382,383],{},"model"," is measured, but it cannot be named the cheapest: it may answer worse, and the replay does not check answer quality yet. It shows in the results as ",[386,387,388],"strong",{},"quality unverified",". Check its answers yourself before you adopt it.",[287,391,393],{"id":392},"what-is-worth-trying","What is worth trying",[140,395,396],{},"Start with the flags that trade latency for throughput on your traffic:",[398,399,400,409,417,425,433],"ul",{},[401,402,403,408],"li",{},[386,404,405],{},[152,406,407],{},"max-num-seqs",": how many requests run at once. Fewer shortens the wait for the first token; more serves more at a time, as long as the KV cache holds them.",[401,410,411,416],{},[386,412,413],{},[152,414,415],{},"max-num-batched-tokens",": how many tokens one engine step processes. Larger chunks prefill long prompts in fewer steps but stall the decoding of others for longer.",[401,418,419,424],{},[386,420,421],{},[152,422,423],{},"gpu-memory-utilization",": how much GPU memory vLLM takes, most of it for the KV cache.",[401,426,427,432],{},[386,428,429],{},[152,430,431],{},"kv-cache-dtype: fp8",": twice the KV cache in the same memory. Changes precision.",[401,434,435,439],{},[386,436,437],{},[152,438,344],{},": to see what the prefix cache is worth on your traffic.",[140,441,442],{},"Configs that cannot fit the model in GPU memory on paper are ruled out before any restart, so a generous grid costs nothing for those.",[444,445,446],"style",{},"html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html pre.shiki code .sfNiH, html code.shiki .sfNiH{--shiki-light:#FF5370;--shiki-default:#FF9CAC;--shiki-dark:#FF9CAC}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":150,"searchDepth":165,"depth":165,"links":448},[449,450,451],{"id":289,"depth":165,"text":290},{"id":365,"depth":165,"text":366},{"id":392,"depth":165,"text":393},"List the engine configs a replay tries.","md",null,{},{"icon":98},{"title":95,"description":452},"VOBYl5tev04kbIKu5vFr72L0f2KITzeu9HWnVtdujr8",[460,462],{"title":90,"path":91,"stem":92,"description":461,"icon":93,"children":-1},"How the replay restarts your engine under other flags, set per replica.",{"title":100,"path":101,"stem":102,"description":463,"icon":103,"children":-1},"Where the replay's numbers stop, so you can use them safely.",1792165824344]