[{"data":1,"prerenderedAt":370},["ShallowReactive",2],{"navigation_docs":3,"-getting-started-how-it-works":134,"-getting-started-how-it-works-surround":365},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":16,"body":136,"description":358,"extension":359,"links":360,"meta":361,"navigation":362,"path":17,"seo":363,"stem":18,"__hash__":364},"docs\u002F1.getting-started\u002F2.how-it-works.md",{"type":137,"value":138,"toc":349},"minimark",[139,143,154,159,203,210,214,217,233,236,240,247,250,254,257,261,268,289,293,296,346],[140,141,142],"p",{},"ico has four parts. Only the gateway sits in the path of your users' requests. The rest reads what the gateway, the engines and the GPUs report, and stays out of the way.",[144,145,151],"pre",{"className":146,"code":148,"language":149,"meta":150},[147],"language-text"," your apps or router\n         │  OpenAI-compatible requests\n         ▼\n ┌──────────────┐  one record      ┌──────────────┐        ┌────────────┐\n │ ico gateway  │ ───────────────▶ │  collector   │ ─────▶ │ ClickHouse │\n └──────┬───────┘  per request     └──────────────┘        └─────┬──────┘\n        │                            ▲   ▲   ▲                    │\n        ▼                   traces   │   │   │ \u002Fmetrics           ├──▶ ico report\n ┌──────────────┐  ──────────────────┘   │   │ every 5 s          │\n │ vLLM replica │  ──────────────────────┘   │                    └──▶ ico replay\n └──────────────┘                            │                          │\n ┌──────────────┐   GPU counters, every 5 s  │      restarts one replica│\n │ GPU exporter │  ──────────────────────────┘      under other flags ◀─┘\n └──────────────┘\n","text","",[152,153,148],"code",{"__ignoreMap":150},[155,156,158],"h2",{"id":157},"where-each-part-runs","Where each part runs",[160,161,162,175],"table",{},[163,164,165],"thead",{},[166,167,168,172],"tr",{},[169,170,171],"th",{},"Machine",[169,173,174],{},"What runs there",[176,177,178,191],"tbody",{},[166,179,180,184],{},[181,182,183],"td",{},"Every GPU machine",[181,185,186,187,190],{},"Your vLLM replicas, a GPU exporter, and the ico gateway next to them. ",[152,188,189],{},"ico replay"," too, while it tests one of that machine's replicas.",[166,192,193,196],{},[181,194,195],{},"The ico host: one small machine without a GPU",[181,197,198,199,202],{},"The collector and ClickHouse. ",[152,200,201],{},"ico report"," runs here.",[140,204,205,206,209],{},"Only what has to be next to a GPU runs on a GPU machine. The collector and ClickHouse need CPU, memory and disk, so they run elsewhere and never take anything from your engines. The gateway stays next to the engine because it times every request, and the replay because it measures latency. See ",[207,208,26],"a",{"href":32},".",[155,211,213],{"id":212},"the-gateway","The gateway",[140,215,216],{},"An OpenAI-compatible pass-through, written in Go, in front of your engine replicas. Your clients or your router send requests to it instead of to the engine. It forwards them unchanged, streams included, and writes one record per request:",[218,219,220,224,227,230],"ul",{},[221,222,223],"li",{},"who it was for: the tenant and the feature, from two request headers;",[221,225,226],{},"what it was: the model, the endpoint, prompt, completion and cached token counts;",[221,228,229],{},"how it went: time to first token, end-to-end time, the gaps between streamed chunks, the status;",[221,231,232],{},"which earlier requests it shares a prefix with, as salted hashes of 256-character blocks.",[140,234,235],{},"It never stores prompt or completion text. It adds about 0.2 ms per request at the median. When the collector is unreachable it keeps serving and queues up to 16,384 records; while the queue is full, new records are dropped and counted. It drops telemetry, never requests.",[155,237,239],{"id":238},"the-collector","The collector",[140,241,242,243,246],{},"A Python service that receives the gateway's records and the engine's OpenTelemetry traces on one gRPC port (4317), scrapes each engine's ",[152,244,245],{},"\u002Fmetrics"," page and each GPU exporter every 5 seconds, and writes everything to ClickHouse under its own names and units. A report reads the same whichever vLLM version or exporter produced the numbers.",[140,248,249],{},"Engine traces carry the time each request spent queued, in prefill and in decode. Joined to the gateway's record by trace id, they let the report split a GPU's cost across tenants by the work each one caused, not just by token count.",[155,251,253],{"id":252},"clickhouse","ClickHouse",[140,255,256],{},"Holds everything for 30 days: requests, engine spans, engine metrics and GPU metrics. Replay results are kept without a time limit. The report's numbers come from views you can query yourself.",[155,258,260],{"id":259},"the-report-and-the-replay-lab","The report and the replay lab",[140,262,263,264,267],{},"Two commands of the ",[152,265,266],{},"ico"," CLI:",[218,269,270,280],{},[221,271,272,277,278,209],{},[273,274,275],"strong",{},[152,276,201],{}," turns a time window into one static HTML file: what the GPUs cost, how much of it was idle, and how it splits across models, replicas and tenants. The feature of each request is recorded too, for your own queries; the report does not break costs down by feature yet. See ",[207,279,66],{"href":67},[221,281,282,286,287,209],{},[273,283,284],{},[152,285,189],{}," drains one replica at the gateway, restarts its engine under each candidate config, replays the shape of your recorded traffic at rising rates until a latency limit breaks, and names the cheapest config that held. Then it restores the original. See ",[207,288,76],{"href":81},[155,290,292],{"id":291},"how-much-it-can-see","How much it can see",[140,294,295],{},"Idle time always comes from the gateway's requests: time with none in flight. What else the report can say depends on what your GPUs report about themselves. Each host gets a telemetry level:",[160,297,298,311],{},[163,299,300],{},[166,301,302,305,308],{},[169,303,304],{},"Level",[169,306,307],{},"Source",[169,309,310],{},"What it adds",[176,312,313,324,335],{},[166,314,315,318,321],{},[181,316,317],{},"L3",[181,319,320],{},"DCGM profiling counters",[181,322,323],{},"How busy the compute cores, tensor cores and memory bandwidth were. On a host that runs one replica, also what was paid for cores that had nothing to run while requests were being served.",[166,325,326,329,332],{},[181,327,328],{},"L2",[181,330,331],{},"Basic GPU counters",[181,333,334],{},"Utilization, memory and power.",[166,336,337,340,343],{},[181,338,339],{},"L1",[181,341,342],{},"Memory and power only",[181,344,345],{},"Memory and power.",[140,347,348],{},"Datacenter cards such as the H100, A100 or L40S reach L3 with dcgm-exporter. GeForce cards reach L2 with nvidia_gpu_exporter.",{"title":150,"searchDepth":350,"depth":350,"links":351},2,[352,353,354,355,356,357],{"id":157,"depth":350,"text":158},{"id":212,"depth":350,"text":213},{"id":238,"depth":350,"text":239},{"id":252,"depth":350,"text":253},{"id":259,"depth":350,"text":260},{"id":291,"depth":350,"text":292},"The four parts of ico, what each one does, and what flows between them.","md",null,{},{"icon":19},{"title":16,"description":358},"GtpzsQEZXoBEEorwIm0ddiOFgVIYVZGH8jYjGAQZTsk",[366,368],{"title":11,"path":12,"stem":13,"description":367,"icon":14,"children":-1},"What ico measures, who it is for, and what it works with today.",{"title":21,"path":22,"stem":23,"description":369,"icon":24,"children":-1},"A first report in ten minutes on your laptop, with two fake engines and no GPU.",1792165822474]