[{"data":1,"prerenderedAt":786},["ShallowReactive",2],{"navigation_docs":3,"-replay-run-a-replay":134,"-replay-run-a-replay-surround":781},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":85,"body":136,"description":774,"extension":775,"links":776,"meta":777,"navigation":778,"path":86,"seo":779,"stem":87,"__hash__":780},"docs\u002F4.replay\u002F2.run-a-replay.md",{"type":137,"value":138,"toc":764},"minimark",[139,143,148,606,610,686,724,731,735,738,746,753,757,760],[140,141,142],"p",{},"A replay runs on the GPU machine of the replica it tests: it restarts that machine's engine and sends load to it directly, so latency is measured without a network in between.",[144,145,147],"h2",{"id":146},"prepare-the-gpu-machine","Prepare the GPU machine",[149,150,152,157,173,217,224,228,231,528,531,535,540],"steps",{"level":151},"3",[153,154,156],"h3",{"id":155},"install-the-cli","Install the CLI",[140,158,159,160,164,165,172],{},"The ",[161,162,163],"code",{},"ico"," CLI is a Python package in the repository. With ",[166,167,171],"a",{"href":168,"rel":169},"https:\u002F\u002Fdocs.astral.sh\u002Fuv\u002F",[170],"nofollow","uv",":",[174,175,181],"pre",{"className":176,"code":177,"filename":178,"language":179,"meta":180,"style":180},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","cd inference-cost-optimizer\u002Fpy\nuv sync --no-dev\nsource .venv\u002Fbin\u002Factivate\n","gpu-1","bash","",[161,182,183,196,208],{"__ignoreMap":180},[184,185,188,192],"span",{"class":186,"line":187},"line",1,[184,189,191],{"class":190},"s2Zo4","cd",[184,193,195],{"class":194},"sfazB"," inference-cost-optimizer\u002Fpy\n",[184,197,199,202,205],{"class":186,"line":198},2,[184,200,171],{"class":201},"sBMFI",[184,203,204],{"class":194}," sync",[184,206,207],{"class":194}," --no-dev\n",[184,209,211,214],{"class":186,"line":210},3,[184,212,213],{"class":190},"source",[184,215,216],{"class":194}," .venv\u002Fbin\u002Factivate\n",[140,218,219,220,223],{},"Or with pip, in a virtual environment: ",[161,221,222],{},"pip install .\u002Finference-cost-optimizer\u002Fpy",".",[153,225,227],{"id":226},"extend-the-machines-config","Extend the machine's config",[140,229,230],{},"The replay reads the same file as the gateway on this machine. Add what it needs: how to restart the engine, the latency limits, prices, the model's size, and where ClickHouse is.",[174,232,237],{"className":233,"code":234,"filename":235,"language":236,"meta":180,"style":180},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","replicas:\n  - name: gpu-1-r1\n    url: http:\u002F\u002F10.0.0.11:8000\n    host: gpu-1\n    gpu_model: H100-SXM\n    gpu_count: 1\n    launcher:                       # how to restart the engine; see Launchers\n      kind: docker\n      container: vllm\n\n# ... headers and gateway, as before ...\n\nslo:                                # the limits a config must hold\n  ttft_p95_ms: 1000\n  itl_p95_ms: 50\n\ngpu_price_per_hour:\n  H100-SXM: 2.99\n\nmodels:\n  Qwen\u002FQwen3-8B:\n    params: 8_190_735_360\n    weight_bytes_per_param: 2\n    kv_bytes_per_token: 147_456\n\nclickhouse:\n  host: 10.0.0.5                    # the ico host\n  port: 8123\n  username: ico\n  database: ico\n","gateway.yaml","yaml",[161,238,239,249,262,272,283,294,306,318,329,340,347,353,358,369,380,391,396,404,415,420,428,436,447,458,469,474,482,496,507,518],{"__ignoreMap":180},[184,240,241,245],{"class":186,"line":187},[184,242,244],{"class":243},"swJcz","replicas",[184,246,248],{"class":247},"sMK4o",":\n",[184,250,251,254,257,259],{"class":186,"line":198},[184,252,253],{"class":247},"  -",[184,255,256],{"class":243}," name",[184,258,172],{"class":247},[184,260,261],{"class":194}," gpu-1-r1\n",[184,263,264,267,269],{"class":186,"line":210},[184,265,266],{"class":243},"    url",[184,268,172],{"class":247},[184,270,271],{"class":194}," http:\u002F\u002F10.0.0.11:8000\n",[184,273,275,278,280],{"class":186,"line":274},4,[184,276,277],{"class":243},"    host",[184,279,172],{"class":247},[184,281,282],{"class":194}," gpu-1\n",[184,284,286,289,291],{"class":186,"line":285},5,[184,287,288],{"class":243},"    gpu_model",[184,290,172],{"class":247},[184,292,293],{"class":194}," H100-SXM\n",[184,295,297,300,302],{"class":186,"line":296},6,[184,298,299],{"class":243},"    gpu_count",[184,301,172],{"class":247},[184,303,305],{"class":304},"sbssI"," 1\n",[184,307,309,312,314],{"class":186,"line":308},7,[184,310,311],{"class":243},"    launcher",[184,313,172],{"class":247},[184,315,317],{"class":316},"sHwdD","                       # how to restart the engine; see Launchers\n",[184,319,321,324,326],{"class":186,"line":320},8,[184,322,323],{"class":243},"      kind",[184,325,172],{"class":247},[184,327,328],{"class":194}," docker\n",[184,330,332,335,337],{"class":186,"line":331},9,[184,333,334],{"class":243},"      container",[184,336,172],{"class":247},[184,338,339],{"class":194}," vllm\n",[184,341,343],{"class":186,"line":342},10,[184,344,346],{"emptyLinePlaceholder":345},true,"\n",[184,348,350],{"class":186,"line":349},11,[184,351,352],{"class":316},"# ... headers and gateway, as before ...\n",[184,354,356],{"class":186,"line":355},12,[184,357,346],{"emptyLinePlaceholder":345},[184,359,361,364,366],{"class":186,"line":360},13,[184,362,363],{"class":243},"slo",[184,365,172],{"class":247},[184,367,368],{"class":316},"                                # the limits a config must hold\n",[184,370,372,375,377],{"class":186,"line":371},14,[184,373,374],{"class":243},"  ttft_p95_ms",[184,376,172],{"class":247},[184,378,379],{"class":304}," 1000\n",[184,381,383,386,388],{"class":186,"line":382},15,[184,384,385],{"class":243},"  itl_p95_ms",[184,387,172],{"class":247},[184,389,390],{"class":304}," 50\n",[184,392,394],{"class":186,"line":393},16,[184,395,346],{"emptyLinePlaceholder":345},[184,397,399,402],{"class":186,"line":398},17,[184,400,401],{"class":243},"gpu_price_per_hour",[184,403,248],{"class":247},[184,405,407,410,412],{"class":186,"line":406},18,[184,408,409],{"class":243},"  H100-SXM",[184,411,172],{"class":247},[184,413,414],{"class":304}," 2.99\n",[184,416,418],{"class":186,"line":417},19,[184,419,346],{"emptyLinePlaceholder":345},[184,421,423,426],{"class":186,"line":422},20,[184,424,425],{"class":243},"models",[184,427,248],{"class":247},[184,429,431,434],{"class":186,"line":430},21,[184,432,433],{"class":243},"  Qwen\u002FQwen3-8B",[184,435,248],{"class":247},[184,437,439,442,444],{"class":186,"line":438},22,[184,440,441],{"class":243},"    params",[184,443,172],{"class":247},[184,445,446],{"class":304}," 8_190_735_360\n",[184,448,450,453,455],{"class":186,"line":449},23,[184,451,452],{"class":243},"    weight_bytes_per_param",[184,454,172],{"class":247},[184,456,457],{"class":304}," 2\n",[184,459,461,464,466],{"class":186,"line":460},24,[184,462,463],{"class":243},"    kv_bytes_per_token",[184,465,172],{"class":247},[184,467,468],{"class":304}," 147_456\n",[184,470,472],{"class":186,"line":471},25,[184,473,346],{"emptyLinePlaceholder":345},[184,475,477,480],{"class":186,"line":476},26,[184,478,479],{"class":243},"clickhouse",[184,481,248],{"class":247},[184,483,485,488,490,493],{"class":186,"line":484},27,[184,486,487],{"class":243},"  host",[184,489,172],{"class":247},[184,491,492],{"class":304}," 10.0.0.5",[184,494,495],{"class":316},"                    # the ico host\n",[184,497,499,502,504],{"class":186,"line":498},28,[184,500,501],{"class":243},"  port",[184,503,172],{"class":247},[184,505,506],{"class":304}," 8123\n",[184,508,510,513,515],{"class":186,"line":509},29,[184,511,512],{"class":243},"  username",[184,514,172],{"class":247},[184,516,517],{"class":194}," ico\n",[184,519,521,524,526],{"class":186,"line":520},30,[184,522,523],{"class":243},"  database",[184,525,172],{"class":247},[184,527,517],{"class":194},[140,529,530],{},"The gateway ignores the keys it does not use.",[153,532,534],{"id":533},"write-a-sweep-file","Write a sweep file",[140,536,537,538,223],{},"The configs to try, as changes to the flags in production. See ",[166,539,95],{"href":96},[174,541,544],{"className":233,"code":542,"filename":543,"language":236,"meta":180,"style":180},"configs:\n  - max-num-batched-tokens: 16384\n  - kv-cache-dtype: fp8\ngrid:\n  max-num-seqs: [128, 256]\n","sweep.yaml",[161,545,546,553,565,577,584],{"__ignoreMap":180},[184,547,548,551],{"class":186,"line":187},[184,549,550],{"class":243},"configs",[184,552,248],{"class":247},[184,554,555,557,560,562],{"class":186,"line":198},[184,556,253],{"class":247},[184,558,559],{"class":243}," max-num-batched-tokens",[184,561,172],{"class":247},[184,563,564],{"class":304}," 16384\n",[184,566,567,569,572,574],{"class":186,"line":210},[184,568,253],{"class":247},[184,570,571],{"class":243}," kv-cache-dtype",[184,573,172],{"class":247},[184,575,576],{"class":194}," fp8\n",[184,578,579,582],{"class":186,"line":274},[184,580,581],{"class":243},"grid",[184,583,248],{"class":247},[184,585,586,589,591,594,597,600,603],{"class":186,"line":285},[184,587,588],{"class":243},"  max-num-seqs",[184,590,172],{"class":247},[184,592,593],{"class":247}," [",[184,595,596],{"class":304},"128",[184,598,599],{"class":247},",",[184,601,602],{"class":304}," 256",[184,604,605],{"class":247},"]\n",[144,607,609],{"id":608},"start-it","Start it",[174,611,613],{"className":176,"code":612,"filename":178,"language":179,"meta":180,"style":180},"export ICO_ADMIN_TOKEN=...           # the gateway's admin token\nexport ICO_CLICKHOUSE_PASSWORD=...   # the ico user's password\nico replay --config gateway.yaml --replica gpu-1-r1 --configs sweep.yaml --window 24h --budget 45m\n",[161,614,615,634,649],{"__ignoreMap":180},[184,616,617,621,625,628,631],{"class":186,"line":187},[184,618,620],{"class":619},"spNyl","export",[184,622,624],{"class":623},"sTEyZ"," ICO_ADMIN_TOKEN",[184,626,627],{"class":247},"=",[184,629,630],{"class":623},"...           ",[184,632,633],{"class":316},"# the gateway's admin token\n",[184,635,636,638,641,643,646],{"class":186,"line":198},[184,637,620],{"class":619},[184,639,640],{"class":623}," ICO_CLICKHOUSE_PASSWORD",[184,642,627],{"class":247},[184,644,645],{"class":623},"...   ",[184,647,648],{"class":316},"# the ico user's password\n",[184,650,651,653,656,659,662,665,668,671,674,677,680,683],{"class":186,"line":210},[184,652,163],{"class":201},[184,654,655],{"class":194}," replay",[184,657,658],{"class":194}," --config",[184,660,661],{"class":194}," gateway.yaml",[184,663,664],{"class":194}," --replica",[184,666,667],{"class":194}," gpu-1-r1",[184,669,670],{"class":194}," --configs",[184,672,673],{"class":194}," sweep.yaml",[184,675,676],{"class":194}," --window",[184,678,679],{"class":194}," 24h",[184,681,682],{"class":194}," --budget",[184,684,685],{"class":194}," 45m\n",[687,688,689,700,706,712,718],"ul",{},[690,691,692,695,696,699],"li",{},[161,693,694],{},"--window"," and ",[161,697,698],{},"--until"," choose the recorded traffic the replay is built from.",[690,701,702,705],{},[161,703,704],{},"--budget"," is the most the whole run may take, restore included.",[690,707,708,711],{},[161,709,710],{},"--slo ttft_p95=1000ms,itl_p95=50ms"," overrides the limits in the config.",[690,713,714,717],{},[161,715,716],{},"--min-saving 10%"," is how much cheaper than the current config a config must measure to be recommended.",[690,719,720,723],{},[161,721,722],{},"--single-replica"," is required when this replica is the only one serving its model: it accepts that the model is unavailable for the length of the run.",[140,725,726,727,223],{},"All options are in the ",[166,728,730],{"href":729},"\u002Freference\u002Fcli#ico-replay","CLI reference",[144,732,734],{"id":733},"read-its-plan-and-say-yes","Read its plan, and say yes",[140,736,737],{},"The replay first measures the current config. Then it knows how long loading a config and testing one take, and prints its plan:",[174,739,744],{"className":740,"code":742,"language":743,"meta":180},[741],"language-text","current: holds 22.50 requests\u002Fs, 46,747 tokens\u002Fs, $0.0237 per million tokens\nLoading a config takes about 40s and a full test about 131s. Within the budget that leaves time for 5 of the 7 configs:\n  test  max-num-batched-tokens=16384\n  test  max-num-batched-tokens=2048\n  test  enable-prefix-caching=False\n  test  kv-cache-dtype=fp8\n  test  quantization=fp8\n  skip  max-num-seqs=128\n  skip  max-num-seqs=256\n","text",[161,745,742],{"__ignoreMap":180},[140,747,748,749,752],{},"It waits for a yes; ",[161,750,751],{},"--yes"," skips the question. When a full test of every config does not fit the budget but successive halving does, every config gets a short test, the better half a longer one, and the last two a full test. Only a full test can win.",[144,754,756],{"id":755},"pick-latency-limits-with-room","Pick latency limits with room",[140,758,759],{},"The replay prints the time to first token your traffic saw in the window. Set the limits above it, with room. When the limit sits right at what the traffic sees today, whether a rate holds comes down to a few requests in each step, and the rate found swings from run to run. If your traffic is already over the limit, the replay says so before it starts.",[761,762,763],"style",{},"html pre.shiki code .s2Zo4, html code.shiki .s2Zo4{--shiki-light:#6182B8;--shiki-default:#82AAFF;--shiki-dark:#82AAFF}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .spNyl, html code.shiki .spNyl{--shiki-light:#9C3EDA;--shiki-default:#C792EA;--shiki-dark:#C792EA}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}",{"title":180,"searchDepth":198,"depth":198,"links":765},[766,771,772,773],{"id":146,"depth":198,"text":147,"children":767},[768,769,770],{"id":155,"depth":210,"text":156},{"id":226,"depth":210,"text":227},{"id":533,"depth":210,"text":534},{"id":608,"depth":198,"text":609},{"id":733,"depth":198,"text":734},{"id":755,"depth":198,"text":756},"Set up a GPU machine for replays, start one, and read its plan.","md",null,{},{"icon":88},{"title":85,"description":774},"ZnGVNWK1zD6j3pgpvVbGyopyM4r5fuQm0bndc3GZaKs",[782,784],{"title":31,"path":81,"stem":82,"description":783,"icon":83,"children":-1},"What the replay lab tests, how, and what it leaves behind.",{"title":90,"path":91,"stem":92,"description":785,"icon":93,"children":-1},"How the replay restarts your engine under other flags, set per replica.",1792165823925]