[{"data":1,"prerenderedAt":454},["ShallowReactive",2],{"navigation_docs":3,"-replay-launchers":134,"-replay-launchers-surround":449},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":90,"body":136,"description":442,"extension":443,"links":444,"meta":445,"navigation":446,"path":91,"seo":447,"stem":92,"__hash__":448},"docs\u002F4.replay\u002F3.launchers.md",{"type":137,"value":138,"toc":440},"minimark",[139,148,432,436],[140,141,142,143,147],"p",{},"A launcher stops a replica's engine and starts it again with other flags. The replay does nothing else to your engine. Health checks, timeouts and restoring the original are the same whichever launcher you use. Set it per replica under ",[144,145,146],"code",{},"launcher"," in the config the replay reads.",[149,150,151,221,385],"tabs",{},[152,153,156,163,212,215],"tabs-item",{"icon":154,"label":155},"i-simple-icons-docker","docker",[140,157,158,162],{},[159,160,161],"strong",{},"For vLLM running in a Docker container."," The simplest one.",[164,165,170],"pre",{"className":166,"code":167,"language":168,"meta":169,"style":169},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","launcher:\n  kind: docker\n  container: vllm      # the name of the engine's container\n","yaml","",[144,171,172,184,197],{"__ignoreMap":169},[173,174,177,180],"span",{"class":175,"line":176},"line",1,[173,178,146],{"class":179},"swJcz",[173,181,183],{"class":182},"sMK4o",":\n",[173,185,187,190,193],{"class":175,"line":186},2,[173,188,189],{"class":179},"  kind",[173,191,192],{"class":182},":",[173,194,196],{"class":195},"sfazB"," docker\n",[173,198,200,203,205,208],{"class":175,"line":199},3,[173,201,202],{"class":179},"  container",[173,204,192],{"class":182},[173,206,207],{"class":195}," vllm",[173,209,211],{"class":210},"sHwdD","      # the name of the engine's container\n",[140,213,214],{},"Each config runs in a copy of the engine's container: the same image, GPUs, volumes, environment, ports and networks, with only the engine's flags changed. The flags in production are read from the running container itself. The original container is only stopped, never changed or removed, so restoring it is starting it again: it comes back with whatever it was started with.",[140,216,217,218,220],{},"The replay needs access to the Docker socket, so run it as a user in the ",[144,219,155],{}," group or as root.",[152,222,225,231,345,358],{"icon":223,"label":224},"i-lucide-terminal","command",[140,226,227,230],{},[159,228,229],{},"For vLLM run any other way:"," systemd, a process manager, a script of yours.",[164,232,234],{"className":166,"code":233,"language":168,"meta":169,"style":169},"launcher:\n  kind: command\n  start: \u002Fopt\u002Fllm\u002Fvllm.sh start {flags}\n  stop: \u002Fopt\u002Fllm\u002Fvllm.sh stop\n  env:\n    VLLM_EXTRA: \"{flags}\"        # {flags} works in env values too\n  cwd: \u002Fopt\u002Fllm\n  flags:                         # the flags in production today\n    gpu-memory-utilization: 0.9\n    max-model-len: 32768\n",[144,235,236,242,251,261,272,280,300,311,322,334],{"__ignoreMap":169},[173,237,238,240],{"class":175,"line":176},[173,239,146],{"class":179},[173,241,183],{"class":182},[173,243,244,246,248],{"class":175,"line":186},[173,245,189],{"class":179},[173,247,192],{"class":182},[173,249,250],{"class":195}," command\n",[173,252,253,256,258],{"class":175,"line":199},[173,254,255],{"class":179},"  start",[173,257,192],{"class":182},[173,259,260],{"class":195}," \u002Fopt\u002Fllm\u002Fvllm.sh start {flags}\n",[173,262,264,267,269],{"class":175,"line":263},4,[173,265,266],{"class":179},"  stop",[173,268,192],{"class":182},[173,270,271],{"class":195}," \u002Fopt\u002Fllm\u002Fvllm.sh stop\n",[173,273,275,278],{"class":175,"line":274},5,[173,276,277],{"class":179},"  env",[173,279,183],{"class":182},[173,281,283,286,288,291,294,297],{"class":175,"line":282},6,[173,284,285],{"class":179},"    VLLM_EXTRA",[173,287,192],{"class":182},[173,289,290],{"class":182}," \"",[173,292,293],{"class":195},"{flags}",[173,295,296],{"class":182},"\"",[173,298,299],{"class":210},"        # {flags} works in env values too\n",[173,301,303,306,308],{"class":175,"line":302},7,[173,304,305],{"class":179},"  cwd",[173,307,192],{"class":182},[173,309,310],{"class":195}," \u002Fopt\u002Fllm\n",[173,312,314,317,319],{"class":175,"line":313},8,[173,315,316],{"class":179},"  flags",[173,318,192],{"class":182},[173,320,321],{"class":210},"                         # the flags in production today\n",[173,323,325,328,330],{"class":175,"line":324},9,[173,326,327],{"class":179},"    gpu-memory-utilization",[173,329,192],{"class":182},[173,331,333],{"class":332},"sbssI"," 0.9\n",[173,335,337,340,342],{"class":175,"line":336},10,[173,338,339],{"class":179},"    max-model-len",[173,341,192],{"class":182},[173,343,344],{"class":332}," 32768\n",[140,346,347,349,350,353,354,357],{},[144,348,293],{}," is replaced by the config's flags as they are typed on vLLM's command line, such as ",[144,351,352],{},"--gpu-memory-utilization 0.9 --max-model-len 32768",". ",[144,355,356],{},"flags"," must list the flags in production: every config is these with some changed, and they are what gets restored.",[359,360,361,374,382],"ul",{},[362,363,364,369,370,373],"li",{},[159,365,366],{},[144,367,368],{},"start"," must return once the engine is starting; it should not wait for it to be ready. The replay waits for ",[144,371,372],{},"\u002Fhealth"," itself.",[362,375,376,381],{},[159,377,378],{},[144,379,380],{},"stop"," must return once the engine is gone and its GPU memory is free. The next config will need it.",[362,383,384],{},"Each must exit with 0 when it worked. Either may take up to 10 minutes.",[152,386,389,395,429],{"icon":387,"label":388},"i-lucide-hand","manual",[140,390,391,394],{},[159,392,393],{},"When restarts must go through a person,"," or to try the replay before automating anything.",[164,396,398],{"className":166,"code":397,"language":168,"meta":169,"style":169},"launcher:\n  kind: manual\n  flags:\n    gpu-memory-utilization: 0.9\n",[144,399,400,406,415,421],{"__ignoreMap":169},[173,401,402,404],{"class":175,"line":176},[173,403,146],{"class":179},[173,405,183],{"class":182},[173,407,408,410,412],{"class":175,"line":186},[173,409,189],{"class":179},[173,411,192],{"class":182},[173,413,414],{"class":195}," manual\n",[173,416,417,419],{"class":175,"line":199},[173,418,316],{"class":179},[173,420,183],{"class":182},[173,422,423,425,427],{"class":175,"line":263},[173,424,327],{"class":179},[173,426,192],{"class":182},[173,428,333],{"class":332},[140,430,431],{},"The replay prints the flags of each config and waits for you to restart the engine with them and press Enter. A watchdog cannot restore the original through a person, so if the replay is killed, restore it yourself.",[433,434,435],"note",{},"Kubernetes is not supported yet: there is no launcher that restarts a Deployment under other flags.",[437,438,439],"style",{},"html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}",{"title":169,"searchDepth":186,"depth":186,"links":441},[],"How the replay restarts your engine under other flags, set per replica.","md",null,{},{"icon":93},{"title":90,"description":442},"42sMXRIX_mQeJSqL1MNpkgnFtPxIm1gxnSwVfU3lXiM",[450,452],{"title":85,"path":86,"stem":87,"description":451,"icon":88,"children":-1},"Set up a GPU machine for replays, start one, and read its plan.",{"title":95,"path":96,"stem":97,"description":453,"icon":98,"children":-1},"List the engine configs a replay tries.",1792165824122]