[{"data":1,"prerenderedAt":286},["ShallowReactive",2],{"navigation_docs":3,"-getting-started-introduction":134,"-getting-started-introduction-surround":283},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":11,"body":136,"description":276,"extension":277,"links":278,"meta":279,"navigation":280,"path":12,"seo":281,"stem":13,"__hash__":282},"docs\u002F1.getting-started\u002F1.introduction.md",{"type":137,"value":138,"toc":268},"minimark",[139,147,152,155,168,171,175,187,191,246,249,253],[140,141,142,146],"p",{},[143,144,145],"strong",{},"ico",", the inference cost optimizer, finds out what your self-hosted LLMs cost per million tokens and per customer, how much of that cost pays for GPUs with nothing to do, and which engine config serves the same traffic for less. It runs on your machines, next to your engines, and reads your real traffic.",[148,149,151],"h2",{"id":150},"who-it-is-for","Who it is for",[140,153,154],{},"Teams that serve open-weight or fine-tuned models from their own or rented GPUs, usually to many customers from the same replicas, and that want to know:",[156,157,158,162,165],"ul",{},[159,160,161],"li",{},"what each customer costs to serve, to price them or to see the margin on them;",[159,163,164],{},"how much of the GPU bill pays for idle time;",[159,166,167],{},"whether their engine flags are leaving capacity on the table.",[140,169,170],{},"ico is built first for GPU machines you rent or own, from RunPod, Lambda, CoreWeave or your own racks, running vLLM in Docker or as plain processes. Kubernetes comes later.",[148,172,174],{"id":173},"what-you-get","What you get",[176,177,178,183],"card-group",{},[179,180,182],"card",{"icon":69,"title":181,"to":67},"A cost and waste report","Cost per million tokens, idle share, and cost per model, replica and tenant, as one static HTML file.",[179,184,186],{"icon":83,"title":185,"to":81},"A replay lab","Engine configs tested on the shape of your recorded traffic, and the cheapest one that holds your latency limits.",[148,188,190],{"id":189},"what-it-works-with-today","What it works with today",[192,193,194,209],"table",{},[195,196,197],"thead",{},[198,199,200,203,206],"tr",{},[201,202],"th",{},[201,204,205],{},"Supported",[201,207,208],{},"Run on real hardware",[210,211,212,224,235],"tbody",{},[198,213,214,218,221],{},[215,216,217],"td",{},"Engine",[215,219,220],{},"vLLM",[215,222,223],{},"0.31.0 on an H100 SXM; 0.10.1.1 on a GeForce RTX 2060",[198,225,226,229,232],{},[215,227,228],{},"GPU telemetry",[215,230,231],{},"NVIDIA dcgm-exporter (datacenter GPUs), nvidia_gpu_exporter (GeForce)",[215,233,234],{},"dcgm-exporter 4.5.2-4.8.1 on the H100, with profiling counters; nvidia_gpu_exporter 1.15.1 on the RTX 2060",[198,236,237,240,243],{},[215,238,239],{},"Deployment",[215,241,242],{},"Docker Compose, or plain processes",[215,244,245],{},"RunPod (H100), a Windows desktop with Docker Desktop (RTX 2060)",[140,247,248],{},"Not built yet: SGLang and AMD support, Kubernetes and a Helm chart, replays that mirror live requests, and a check of answer quality. Until that check exists, configs that change precision, such as FP8, are measured but cannot be recommended.",[148,250,252],{"id":251},"where-to-go-next","Where to go next",[176,254,255,258,261,265],{},[179,256,257],{"icon":19,"title":16,"to":17},"The four parts and what flows between them.",[179,259,260],{"icon":24,"title":21,"to":22},"A first report in ten minutes on your laptop, with fake engines and no GPU.",[179,262,264],{"icon":263,"title":26,"to":32},"i-lucide-server","The real install, on machines from RunPod, Lambda, CoreWeave or your own racks.",[179,266,267],{"icon":59,"title":56,"to":57},"We set it up with you if you would rather not do it alone.",{"title":269,"searchDepth":270,"depth":270,"links":271},"",2,[272,273,274,275],{"id":150,"depth":270,"text":151},{"id":173,"depth":270,"text":174},{"id":189,"depth":270,"text":190},{"id":251,"depth":270,"text":252},"What ico measures, who it is for, and what it works with today.","md",null,{},{"icon":14},{"title":11,"description":276},"-4hvljtQ9l0TluKMFf4lpr89s5SjEjpNghNdx-4Y-uo",[278,284],{"title":16,"path":17,"stem":18,"description":285,"icon":19,"children":-1},"The four parts of ico, what each one does, and what flows between them.",1792165819409]