[{"data":1,"prerenderedAt":713},["ShallowReactive",2],{"navigation_docs":3,"-reference-data":134,"-reference-data-surround":708},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":125,"body":136,"description":701,"extension":702,"links":703,"meta":704,"navigation":705,"path":126,"seo":706,"stem":127,"__hash__":707},"docs\u002F5.reference\u002F4.data.md",{"type":137,"value":138,"toc":691},"minimark",[139,148,153,277,281,359,362,378,382,385,390,575,579,684,687],[140,141,142,143,147],"p",{},"Everything lives in one ClickHouse database, ",[144,145,146],"code",{},"ico"," by default. Raw rows expire after 30 days, in whole days; replay results are kept until you delete them.",[149,150,152],"h2",{"id":151},"tables","Tables",[154,155,156,175],"table",{},[157,158,159],"thead",{},[160,161,162,166,169,172],"tr",{},[163,164,165],"th",{},"Table",[163,167,168],{},"Written from",[163,170,171],{},"One row per",[163,173,174],{},"Kept",[176,177,178,195,210,228,243,261],"tbody",{},[160,179,180,186,189,192],{},[181,182,183],"td",{},[144,184,185],{},"requests",[181,187,188],{},"the gateways",[181,190,191],{},"request the gateway proxied",[181,193,194],{},"30 days",[160,196,197,202,205,208],{},[181,198,199],{},[144,200,201],{},"engine_spans",[181,203,204],{},"the engines' OpenTelemetry traces",[181,206,207],{},"engine span: queue, prefill, decode and end-to-end time, token counts",[181,209,194],{},[160,211,212,217,223,226],{},[181,213,214],{},[144,215,216],{},"engine_metrics",[181,218,219,220],{},"each engine's ",[144,221,222],{},"\u002Fmetrics",[181,224,225],{},"scraped sample",[181,227,194],{},[160,229,230,235,238,241],{},[181,231,232],{},[144,233,234],{},"gpu_metrics",[181,236,237],{},"the GPU exporters",[181,239,240],{},"scraped sample, with its telemetry level",[181,242,194],{},[160,244,245,250,255,258],{},[181,246,247],{},[144,248,249],{},"replay_runs",[181,251,252],{},[144,253,254],{},"ico replay",[181,256,257],{},"replay run: replica, window, limits, budget, margin, status",[181,259,260],{},"always",[160,262,263,268,272,275],{},[181,264,265],{},[144,266,267],{},"replay_results",[181,269,270],{},[144,271,254],{},[181,273,274],{},"config tested in a run, at each stage",[181,276,260],{},[149,278,280],{"id":279},"views","Views",[154,282,283,293],{},[157,284,285],{},[160,286,287,290],{},[163,288,289],{},"View",[163,291,292],{},"Gives",[176,294,295,309,319,329,339,349],{},[160,296,297,302],{},[181,298,299],{},[144,300,301],{},"request_spans",[181,303,304,305,308],{},"Every request with its engine span, joined on trace id; ",[144,306,307],{},"has_span"," is false where the engine sent none.",[160,310,311,316],{},[181,312,313],{},[144,314,315],{},"usage_by_tenant(since, until)",[181,317,318],{},"Requests, errors and tokens per replica, model and tenant.",[160,320,321,326],{},[181,322,323],{},[144,324,325],{},"replica_busy(since, until)",[181,327,328],{},"Per replica, the seconds with at least one request in flight, and the span of time its requests cover.",[160,330,331,336],{},[181,332,333],{},[144,334,335],{},"replica_phases(since, until)",[181,337,338],{},"Per replica, engine time in prefill and decode, with the tokens of the same requests.",[160,340,341,346],{},[181,342,343],{},[144,344,345],{},"replica_engine(since, until)",[181,347,348],{},"Per replica, when its engine was first and last heard from, and the model it reports.",[160,350,351,356],{},[181,352,353],{},[144,354,355],{},"host_gpus(since, until)",[181,357,358],{},"Per host, its GPUs, telemetry level and the window's average of each counter.",[140,360,361],{},"The parameterized views are queried like functions:",[363,364,369],"pre",{"className":365,"code":366,"language":367,"meta":368,"style":368},"language-sql shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","SELECT * FROM host_gpus(since = '2026-10-14 00:00:00', until = '2026-10-15 00:00:00')\n","sql","",[144,370,371],{"__ignoreMap":368},[372,373,376],"span",{"class":374,"line":375},"line",1,[372,377,366],{},[149,379,381],{"id":380},"metric-names","Metric names",[140,383,384],{},"Engine and GPU metrics are stored under ico's own names and units, so queries and reports read the same across vLLM versions and exporters.",[386,387,389],"h3",{"id":388},"gpu-metrics","GPU metrics",[154,391,392,411],{},[157,393,394],{},[160,395,396,399,402,405,408],{},[163,397,398],{},"Name",[163,400,401],{},"Unit",[163,403,404],{},"From dcgm-exporter",[163,406,407],{},"From nvidia_gpu_exporter",[163,409,410],{},"Level",[176,412,413,433,451,469,487,509,532,553],{},[160,414,415,420,423,428,430],{},[181,416,417],{},[144,418,419],{},"sm_active_ratio",[181,421,422],{},"0 to 1",[181,424,425],{},[144,426,427],{},"DCGM_FI_PROF_SM_ACTIVE",[181,429],{},[181,431,432],{},"L3",[160,434,435,440,442,447,449],{},[181,436,437],{},[144,438,439],{},"sm_occupancy_ratio",[181,441,422],{},[181,443,444],{},[144,445,446],{},"DCGM_FI_PROF_SM_OCCUPANCY",[181,448],{},[181,450,432],{},[160,452,453,458,460,465,467],{},[181,454,455],{},[144,456,457],{},"tensor_active_ratio",[181,459,422],{},[181,461,462],{},[144,463,464],{},"DCGM_FI_PROF_PIPE_TENSOR_ACTIVE",[181,466],{},[181,468,432],{},[160,470,471,476,478,483,485],{},[181,472,473],{},[144,474,475],{},"dram_active_ratio",[181,477,422],{},[181,479,480],{},[144,481,482],{},"DCGM_FI_PROF_DRAM_ACTIVE",[181,484],{},[181,486,432],{},[160,488,489,494,496,501,506],{},[181,490,491],{},[144,492,493],{},"util_ratio",[181,495,422],{},[181,497,498],{},[144,499,500],{},"DCGM_FI_DEV_GPU_UTIL",[181,502,503],{},[144,504,505],{},"nvidia_smi_utilization_gpu_ratio",[181,507,508],{},"L2",[160,510,511,516,519,524,529],{},[181,512,513],{},[144,514,515],{},"memory_used_bytes",[181,517,518],{},"bytes",[181,520,521],{},[144,522,523],{},"DCGM_FI_DEV_FB_USED",[181,525,526],{},[144,527,528],{},"nvidia_smi_memory_used_bytes",[181,530,531],{},"L1",[160,533,534,539,541,546,551],{},[181,535,536],{},[144,537,538],{},"memory_total_bytes",[181,540,518],{},[181,542,543],{},[144,544,545],{},"DCGM_FI_DEV_FB_TOTAL",[181,547,548],{},[144,549,550],{},"nvidia_smi_memory_total_bytes",[181,552,531],{},[160,554,555,560,563,568,573],{},[181,556,557],{},[144,558,559],{},"power_watts",[181,561,562],{},"watts",[181,564,565],{},[144,566,567],{},"DCGM_FI_DEV_POWER_USAGE",[181,569,570],{},[144,571,572],{},"nvidia_smi_power_draw_watts",[181,574,531],{},[386,576,578],{"id":577},"engine-metrics","Engine metrics",[154,580,581,590],{},[157,582,583],{},[160,584,585,587],{},[163,586,398],{},[163,588,589],{},"What it counts",[176,591,592,606,616,629,642,652],{},[160,593,594,603],{},[181,595,596,599,600],{},[144,597,598],{},"requests_running",", ",[144,601,602],{},"requests_waiting",[181,604,605],{},"Requests in the engine's batch, and queued for it",[160,607,608,613],{},[181,609,610],{},[144,611,612],{},"kv_cache_usage",[181,614,615],{},"Share of the KV cache in use",[160,617,618,626],{},[181,619,620,599,623],{},[144,621,622],{},"prompt_tokens_total",[144,624,625],{},"generation_tokens_total",[181,627,628],{},"Tokens processed and generated, cumulative",[160,630,631,639],{},[181,632,633,599,636],{},[144,634,635],{},"prefix_cache_queries_total",[144,637,638],{},"prefix_cache_hits_total",[181,640,641],{},"Prefix cache lookups and hits, cumulative",[160,643,644,649],{},[181,645,646],{},[144,647,648],{},"preemptions_total",[181,650,651],{},"Requests evicted to free KV cache, cumulative",[160,653,654,671],{},[181,655,656,599,659,599,662,599,665,599,668],{},[144,657,658],{},"ttft_seconds",[144,660,661],{},"inter_token_latency_seconds",[144,663,664],{},"queue_time_seconds",[144,666,667],{},"prefill_time_seconds",[144,669,670],{},"decode_time_seconds",[181,672,673,674,599,677,680,681],{},"Histograms, stored as ",[144,675,676],{},"_bucket",[144,678,679],{},"_sum"," and ",[144,682,683],{},"_count",[140,685,686],{},"Each maps every name vLLM has used for it, from 0.10 to 0.31.",[688,689,690],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":368,"searchDepth":692,"depth":692,"links":693},2,[694,695,696],{"id":151,"depth":692,"text":152},{"id":279,"depth":692,"text":280},{"id":380,"depth":692,"text":381,"children":697},[698,700],{"id":388,"depth":699,"text":389},3,{"id":577,"depth":699,"text":578},"The ClickHouse tables and views ico writes and reads, and the names it stores metrics under.","md",null,{},{"icon":128},{"title":125,"description":701},"jI-1fn4Iv8_fMVS04AkflmnCZrmuUab2hiIuaTNNLuE",[709,711],{"title":120,"path":121,"stem":122,"description":710,"icon":123,"children":-1},"What the gateway proxies, records and exposes.",{"title":130,"path":131,"stem":132,"description":712,"icon":133,"children":-1},"What ico stores, what it never stores, and what crosses your network.",1792165822351]