[{"data":1,"prerenderedAt":669},["ShallowReactive",2],{"navigation_docs":3,"-install-overview":134,"-install-overview-surround":664},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":31,"body":136,"description":657,"extension":658,"links":659,"meta":660,"navigation":661,"path":32,"seo":662,"stem":33,"__hash__":663},"docs\u002F2.install\u002F1.overview.md",{"type":137,"value":138,"toc":645},"minimark",[139,143,148,199,202,215,219,222,268,272,276,391,394,398,402,405,505,514,525,591,599,603,641],[140,141,142],"p",{},"This section installs ico on GPU machines you rent, from RunPod, Lambda, CoreWeave or any other provider, or run in your own racks, with vLLM already serving on them. It assumes Docker on every machine; where a provider gives you a container without Docker, the same programs run as plain processes.",[144,145,147],"h2",{"id":146},"what-runs-where","What runs where",[149,150,151,167],"table",{},[152,153,154],"thead",{},[155,156,157,161,164],"tr",{},[158,159,160],"th",{},"Machine",[158,162,163],{},"What runs there",[158,165,166],{},"Why there",[168,169,170,186],"tbody",{},[155,171,172,180,183],{},[173,174,175,179],"td",{},[176,177,178],"strong",{},"The ico host",": one small machine without a GPU",[173,181,182],{},"The collector and ClickHouse",[173,184,185],{},"They need CPU, memory and disk, not a GPU. Keeping them off the GPU machines means they never take anything from your engines.",[155,187,188,193,196],{},[173,189,190],{},[176,191,192],{},"Every GPU machine",[173,194,195],{},"Your vLLM replicas (already there), the ico gateway, a GPU exporter",[173,197,198],{},"The exporter reads the local GPUs. The gateway times every request, so it sits as close to the engine as possible.",[140,200,201],{},"For the ico host, 2 vCPUs, 8 GB of memory and 50 GB of disk are a good start for a fleet of a few dozen GPUs. It can be any machine your GPU machines can reach: a small VM at the same provider, ideally in the same region.",[203,204,205,206,210,211,214],"note",{},"The replay lab (",[207,208,209],"code",{},"ico replay",") runs on the GPU machine of the replica it tests, and only while it runs. See ",[212,213,76],"a",{"href":81},".",[144,216,218],{"id":217},"where-the-gateway-goes","Where the gateway goes",[140,220,221],{},"Your clients or your router have to send requests to an ico gateway instead of to vLLM. There are two ways to place it.",[223,224,225,249],"tabs",{},[226,227,230,236,246],"tabs-item",{"icon":228,"label":229},"i-lucide-layers","One per GPU machine",[140,231,232,235],{},[176,233,234],{},"Use this when you already have a router or load balancer"," in front of your replicas.",[237,238,244],"pre",{"className":239,"code":241,"language":242,"meta":243},[240],"language-text","clients ──▶ your router ──▶ gateway on gpu-1 ──▶ vLLM on gpu-1\n                        └──▶ gateway on gpu-2 ──▶ vLLM on gpu-2\n","text","",[207,245,241],{"__ignoreMap":243},[140,247,248],{},"Your router keeps doing what it does; each of its upstreams moves from vLLM's port to the gateway's port on the same machine. Every gateway knows only the replicas on its own machine.",[226,250,253,259,265],{"icon":251,"label":252},"i-lucide-merge","One in front of all",[140,254,255,258],{},[176,256,257],{},"Use this when clients talk to vLLM directly"," and you have no router yet.",[237,260,263],{"className":261,"code":262,"language":242,"meta":243},[240],"clients ──▶ gateway ──┬──▶ vLLM on gpu-1\n                      └──▶ vLLM on gpu-2\n",[207,264,262],{"__ignoreMap":243},[140,266,267],{},"The gateway becomes your load balancer: round robin across the replicas of each model, skipping drained ones and any that refuse the connection. Run it on one of the GPU machines or on the ico host.",[269,270,271],"warning",{},"Do not put the gateway in front of an existing router. It would see only the router, so it could not tell which replica served a request, and costs could no longer be split by replica.",[144,273,275],{"id":274},"what-has-to-reach-what","What has to reach what",[149,277,278,294],{},[152,279,280],{},[155,281,282,285,288,291],{},[158,283,284],{},"From",[158,286,287],{},"To",[158,289,290],{},"Port",[158,292,293],{},"What crosses",[168,295,296,309,322,335,346,362,375],{},[155,297,298,301,303,306],{},[173,299,300],{},"Clients or your router",[173,302,120],{},[173,304,305],{},"8080",[173,307,308],{},"Your users' requests",[155,310,311,313,316,319],{},[173,312,120],{},[173,314,315],{},"vLLM",[173,317,318],{},"8000",[173,320,321],{},"The same requests, forwarded",[155,323,324,326,329,332],{},[173,325,120],{},[173,327,328],{},"Collector on the ico host",[173,330,331],{},"4317",[173,333,334],{},"One record per request (gRPC)",[155,336,337,339,341,343],{},[173,338,315],{},[173,340,328],{},[173,342,331],{},[173,344,345],{},"One trace span per request (OpenTelemetry over gRPC)",[155,347,348,351,354,356],{},[173,349,350],{},"Collector",[173,352,353],{},"vLLM on every GPU machine",[173,355,318],{},[173,357,358,361],{},[207,359,360],{},"\u002Fmetrics",", every 5 seconds",[155,363,364,366,369,372],{},[173,365,350],{},[173,367,368],{},"GPU exporter on every GPU machine",[173,370,371],{},"9400 (dcgm-exporter) or 9835 (nvidia_gpu_exporter)",[173,373,374],{},"GPU counters, every 5 seconds",[155,376,377,382,385,388],{},[173,378,379,381],{},[207,380,209],{}," on a GPU machine",[173,383,384],{},"ClickHouse on the ico host",[173,386,387],{},"8123",[173,389,390],{},"Recorded traffic in, results out",[140,392,393],{},"Keep all of these on a private network. The gateway's admin API (port 8081) needs only to be reached from the machine it runs on.",[395,396,397],"caution",{},"The collector accepts writes without authentication. Publish port 4317 only on an interface your GPU machines reach privately, or behind a firewall that admits only them.",[144,399,401],{"id":400},"machines-without-a-private-network","Machines without a private network",[140,403,404],{},"Some providers give each machine a public IP and nothing else; a RunPod pod, for example, is a container that is reached over SSH. There, one SSH tunnel from the ico host to each GPU machine carries everything, in both directions:",[237,406,411],{"className":407,"code":408,"filename":409,"language":410,"meta":243,"style":243},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","ssh -N -p \u003Cssh port> root@\u003Cgpu machine> \\\n  -L 127.0.0.1:18000:127.0.0.1:8000 \\\n  -L 127.0.0.1:19400:127.0.0.1:9400 \\\n  -R 127.0.0.1:4317:127.0.0.1:4317 \\\n  -R 127.0.0.1:8123:127.0.0.1:8123\n","On the ico host","bash",[207,412,413,465,476,486,497],{"__ignoreMap":243},[414,415,418,422,426,429,433,435,438,442,445,448,451,454,457,460,462],"span",{"class":416,"line":417},"line",1,[414,419,421],{"class":420},"sBMFI","ssh",[414,423,425],{"class":424},"sfazB"," -N",[414,427,428],{"class":424}," -p",[414,430,432],{"class":431},"sMK4o"," \u003C",[414,434,421],{"class":424},[414,436,437],{"class":424}," por",[414,439,441],{"class":440},"sTEyZ","t",[414,443,444],{"class":431},">",[414,446,447],{"class":424}," root@",[414,449,450],{"class":431},"\u003C",[414,452,453],{"class":424},"gpu",[414,455,456],{"class":424}," machin",[414,458,459],{"class":440},"e",[414,461,444],{"class":431},[414,463,464],{"class":440}," \\\n",[414,466,468,471,474],{"class":416,"line":467},2,[414,469,470],{"class":424},"  -L",[414,472,473],{"class":424}," 127.0.0.1:18000:127.0.0.1:8000",[414,475,464],{"class":440},[414,477,479,481,484],{"class":416,"line":478},3,[414,480,470],{"class":424},[414,482,483],{"class":424}," 127.0.0.1:19400:127.0.0.1:9400",[414,485,464],{"class":440},[414,487,489,492,495],{"class":416,"line":488},4,[414,490,491],{"class":424},"  -R",[414,493,494],{"class":424}," 127.0.0.1:4317:127.0.0.1:4317",[414,496,464],{"class":440},[414,498,500,502],{"class":416,"line":499},5,[414,501,491],{"class":424},[414,503,504],{"class":424}," 127.0.0.1:8123:127.0.0.1:8123\n",[140,506,507,508,510,511,513],{},"The gateway, vLLM and ",[207,509,209],{}," on the GPU machine then find the collector and ClickHouse at 127.0.0.1, and everything on the GPU machine can listen on 127.0.0.1 only. On the ico host, keep Compose's default ports, published on 127.0.0.1: skip the override file of ",[212,512,36],{"href":37},", since the tunnel is the only way in. The collector scrapes vLLM and the exporter at the tunnel's local ends; give each GPU machine's tunnel its own pair of local ports (18000 and 19400, 18001 and 19401, and so on).",[140,515,516,517,520,521,524],{},"The collector runs in a container, so it reaches the tunnel's local ends at ",[207,518,519],{},"host.docker.internal",", for example ",[207,522,523],{},"http:\u002F\u002Fhost.docker.internal:18000",":",[526,527,528,532],"ul",{},[529,530,531],"li",{},"With Docker Desktop that works as it is.",[529,533,534,535,538,539,542,543,524,546],{},"On a Linux host, bind the two ",[207,536,537],{},"-L"," forwards to Docker's bridge address instead of 127.0.0.1 (",[207,540,541],{},"-L 172.17.0.1:18000:127.0.0.1:8000","), and map the name for the collector in ",[207,544,545],{},"docker-compose.override.yml",[237,547,552],{"className":548,"code":549,"filename":550,"language":551,"meta":243,"style":243},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","services:\n  collector:\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n","deploy\u002Fdocker-compose.override.yml","yaml",[207,553,554,563,570,577],{"__ignoreMap":243},[414,555,556,560],{"class":416,"line":417},[414,557,559],{"class":558},"swJcz","services",[414,561,562],{"class":431},":\n",[414,564,565,568],{"class":416,"line":467},[414,566,567],{"class":558},"  collector",[414,569,562],{"class":431},[414,571,572,575],{"class":416,"line":478},[414,573,574],{"class":558},"    extra_hosts",[414,576,562],{"class":431},[414,578,579,582,585,588],{"class":416,"line":488},[414,580,581],{"class":431},"      -",[414,583,584],{"class":431}," \"",[414,586,587],{"class":424},"host.docker.internal:host-gateway",[414,589,590],{"class":431},"\"\n",[592,593,594,595,598],"tip",{},"Each request in flight through a tunnel holds an open file of the GPU machine's SSH server. Raise that server's limit before heavy traffic, for example with ",[207,596,597],{},"prlimit --pid $(cat \u002Frun\u002Fsshd.pid) --nofile=65536:65536",", then connect. With the default of 1,024, a tunnel refuses new connections at about a thousand requests in flight.",[144,600,602],{"id":601},"the-steps","The steps",[604,605,607,611,616,620,625,628,633,637],"steps",{"level":606},"3",[608,609,36],"h3",{"id":610},"set-up-the-ico-host",[140,612,613,614],{},"ClickHouse and the collector, with Docker Compose. ",[212,615,36],{"href":37},[608,617,619],{"id":618},"set-up-every-gpu-machine","Set up every GPU machine",[140,621,622,623],{},"Tracing in vLLM, a GPU exporter, and the gateway. ",[212,624,41],{"href":42},[608,626,46],{"id":627},"send-traffic-through-the-gateway",[140,629,630,631],{},"Point your router or clients at it and tag requests with a tenant. ",[212,632,46],{"href":47},[608,634,636],{"id":635},"check-that-everything-arrives","Check that everything arrives",[140,638,639],{},[212,640,51],{"href":52},[642,643,644],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}",{"title":243,"searchDepth":467,"depth":467,"links":646},[647,648,649,650,651],{"id":146,"depth":467,"text":147},{"id":217,"depth":467,"text":218},{"id":274,"depth":467,"text":275},{"id":400,"depth":467,"text":401},{"id":601,"depth":467,"text":602,"children":652},[653,654,655,656],{"id":610,"depth":478,"text":36},{"id":618,"depth":478,"text":619},{"id":627,"depth":478,"text":46},{"id":635,"depth":478,"text":636},"What runs where on a fleet of rented GPU machines, and what has to reach what.","md",null,{},{"icon":34},{"title":31,"description":657},"KwS4LEL2QdhQAGR5mMJk9rvYKul9cxDbMjBaMkE5PMQ",[665,667],{"title":21,"path":22,"stem":23,"description":666,"icon":24,"children":-1},"A first report in ten minutes on your laptop, with two fake engines and no GPU.",{"title":36,"path":37,"stem":38,"description":668,"icon":39,"children":-1},"ClickHouse and the collector on one small machine, with Docker Compose.",1792165819409]