[{"data":1,"prerenderedAt":849},["ShallowReactive",2],{"navigation_docs":3,"-reference-configuration":134,"-reference-configuration-surround":844},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":110,"body":136,"description":837,"extension":838,"links":839,"meta":840,"navigation":841,"path":111,"seo":842,"stem":112,"__hash__":843},"docs\u002F5.reference\u002F1.configuration.md",{"type":137,"value":138,"toc":826},"minimark",[139,143,227,232,235,367,371,377,422,426,429,450,454,470,474,481,497,501,585,589,647,651,696,702,706,822],[140,141,142],"p",{},"The gateway, the collector and the CLI read the same YAML format. Each reads the keys it needs and ignores the rest, so one file can serve all three on a machine, and each machine can have its own. Secrets never go in the file: they come from the environment.",[144,145,146,159],"table",{},[147,148,149],"thead",{},[150,151,152,156],"tr",{},[153,154,155],"th",{},"Section",[153,157,158],{},"Read by",[160,161,162,174,192,217],"tbody",{},[150,163,164,171],{},[165,166,167],"td",{},[168,169,170],"code",{},"replicas",[165,172,173],{},"all",[150,175,176,185],{},[165,177,178,181,182],{},[168,179,180],{},"headers",", ",[168,183,184],{},"gateway",[165,186,187,188,191],{},"the gateway (and the CLI, for ",[168,189,190],{},"gateway.admin_url",")",[150,193,194,208],{},[165,195,196,181,199,181,202,181,205],{},[168,197,198],{},"gpu_price_per_hour",[168,200,201],{},"models",[168,203,204],{},"slo",[168,206,207],{},"clickhouse",[165,209,210,211,181,214],{},"the collector, ",[168,212,213],{},"ico report",[168,215,216],{},"ico replay",[150,218,219,224],{},[165,220,221],{},[168,222,223],{},"collector",[165,225,226],{},"the collector",[228,229,230],"h2",{"id":170},[168,231,170],{},[140,233,234],{},"The engine replicas. At least one; names must be unique.",[236,237,238,249,272,279,285,294,298],"field-group",{},[239,240,244,245,248],"field",{"name":241,"type":242,":required":243},"name","string","true","The replica's name. vLLM's ",[168,246,247],{},"OTEL_SERVICE_NAME"," must be the same, and every file that lists the replica must use the same name.",[239,250,252,253,256,257,260,261,264,265,267,268,271],{"name":251,"type":242,":required":243},"url","How the program reading this file reaches the engine, ",[168,254,255],{},"http:\u002F\u002F"," or ",[168,258,259],{},"https:\u002F\u002F",". The gateway sends requests there, the collector scrapes ",[168,262,263],{},"\u003Curl>\u002Fmetrics",", and ",[168,266,216],{}," sends its load there unless ",[168,269,270],{},"--replica-url"," says otherwise.",[239,273,275,276,278],{"name":274,"type":242},"host","The machine the replica runs on. It ties the replica to the GPU exporter with the same ",[168,277,274],{},".",[239,280,282,283,278],{"name":281,"type":242},"gpu_model","The card, as a key of ",[168,284,198],{},[239,286,289,290,293],{"name":287,"type":288},"gpu_count","integer","Default ",[168,291,292],{},"1",". How many GPUs the replica uses: its tensor-parallel size.",[239,295,297],{"name":201,"type":296},"string[]","Gateway only. Send only these models here. Empty: any model.",[239,299,302,311],{"name":300,"type":301},"launcher","object",[140,303,304,305,307,308,278],{},"How ",[168,306,216],{}," restarts this engine. See ",[309,310,90],"a",{"href":91},[312,313,314,331,340,356],"ul",{},[315,316,317,320,321,324,325,256,328,278],"li",{},[168,318,319],{},"kind",": ",[168,322,323],{},"manual"," (default), ",[168,326,327],{},"command",[168,329,330],{},"docker",[315,332,333,336,337,339],{},[168,334,335],{},"container",": for ",[168,338,330],{},", the engine's container.",[315,341,342,181,345,181,348,181,351,336,354,278],{},[168,343,344],{},"start",[168,346,347],{},"stop",[168,349,350],{},"env",[168,352,353],{},"cwd",[168,355,327],{},[315,357,358,336,361,363,364,366],{},[168,359,360],{},"flags",[168,362,327],{}," and ",[168,365,323],{},", the engine's flags in production, without the leading dashes.",[228,368,369],{"id":198},[168,370,198],{},[140,372,373,374,376],{},"USD per GPU-hour, by ",[168,375,281],{},". Put in what you pay. Idle hours are billed like busy ones.",[378,379,384],"pre",{"className":380,"code":381,"language":382,"meta":383,"style":383},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","gpu_price_per_hour:\n  H100-SXM: 2.99\n  L40S: 0.86\n","yaml","",[168,385,386,398,411],{"__ignoreMap":383},[387,388,391,394],"span",{"class":389,"line":390},"line",1,[387,392,198],{"class":393},"swJcz",[387,395,397],{"class":396},"sMK4o",":\n",[387,399,401,404,407],{"class":389,"line":400},2,[387,402,403],{"class":393},"  H100-SXM",[387,405,406],{"class":396},":",[387,408,410],{"class":409},"sbssI"," 2.99\n",[387,412,414,417,419],{"class":389,"line":413},3,[387,415,416],{"class":393},"  L40S",[387,418,406],{"class":396},[387,420,421],{"class":409}," 0.86\n",[228,423,424],{"id":201},[168,425,201],{},[140,427,428],{},"What each served model weighs, for the replay's check of whether a config fits in GPU memory. Every key is required for a model that is listed.",[236,430,431,435,446],{},[239,432,434],{"name":433,"type":288},"params","Number of parameters.",[239,436,439,442,443,445],{"name":437,"type":438},"weight_bytes_per_param","number",[168,440,441],{},"2"," for BF16 or FP16, ",[168,444,292],{}," for FP8.",[239,447,449],{"name":448,"type":288},"kv_bytes_per_token","KV cache per token: 2 × layers × KV heads × head size × bytes per value.",[228,451,452],{"id":180},[168,453,180],{},[236,455,456,463],{},[239,457,289,459,462],{"name":458,"type":242},"tenant",[168,460,461],{},"X-Tenant-Id",". The request header that names the customer.",[239,464,289,466,469],{"name":465,"type":242},"feature",[168,467,468],{},"X-Feature",". The request header that names the part of your product.",[228,471,472],{"id":204},[168,473,204],{},[140,475,476,477,480],{},"The latency limits a config must hold in a replay, at the 95th percentile. ",[168,478,479],{},"--slo"," overrides them.",[236,482,483,490],{},[239,484,289,486,489],{"name":485,"type":438},"ttft_p95_ms",[168,487,488],{},"500",". Time to first token, in milliseconds.",[239,491,289,493,496],{"name":492,"type":438},"itl_p95_ms",[168,494,495],{},"50",". Gap between tokens, in milliseconds.",[228,498,499],{"id":184},[168,500,184],{},[236,502,503,510,521,532,539,543,554,558,566],{},[239,504,289,506,509],{"name":505,"type":242},"listen",[168,507,508],{},":8080",". Where the gateway takes requests.",[239,511,289,513,516,517,520],{"name":512,"type":242},"admin_listen",[168,514,515],{},":8081",". The admin API. Keep it on ",[168,518,519],{},"127.0.0.1"," or a private address.",[239,522,289,524,527,528,531],{"name":523,"type":242},"admin_url",[168,525,526],{},"http:\u002F\u002Flocalhost:8081",". Where the CLI reaches the admin API. ",[168,529,530],{},"ICO_GATEWAY_ADMIN"," overrides it.",[239,533,534,535,538],{"name":223,"type":242},"The collector's gRPC address, ",[168,536,537],{},"host:port",". Empty: records are not exported.",[239,540,542],{"name":541,"type":242},"id","Default: the machine's hostname. Names this gateway to the collector.",[239,544,289,546,549,550,553],{"name":545,"type":242},"state_dir",[168,547,548],{},"\u002Fvar\u002Flib\u002Fico",". Where the gateway keeps the salt of its hashes, unless ",[168,551,552],{},"ICO_HASH_SALT"," is set.",[239,555,557],{"name":556,"type":288},"max_body_bytes","Default 32 MiB. The largest request body accepted.",[239,559,289,562,565],{"name":560,"type":561},"shutdown_timeout","duration",[168,563,564],{},"30s",". How long requests in flight get to finish when the gateway stops.",[239,567,569,570,573,574,577,578,573,581,584],{"name":568,"type":301},"prefix_hash","How request text is hashed to find shared prefixes: ",[168,571,572],{},"block_chars"," (default ",[168,575,576],{},"256",") characters per block, up to ",[168,579,580],{},"max_blocks",[168,582,583],{},"64",") blocks.",[228,586,587],{"id":223},[168,588,223],{},[236,590,591,598,605,634,640],{},[239,592,289,594,597],{"name":593,"type":242},"grpc_listen",[168,595,596],{},"0.0.0.0:4317",". Where gateway records and engine traces arrive.",[239,599,289,601,604],{"name":600,"type":438},"scrape_interval_s",[168,602,603],{},"5",". How often engines and GPU exporters are scraped, in seconds.",[239,606,609,610,612,613,616,617,620,621,623,624,627,628,630,631,633],{"name":607,"type":608},"gpu_exporters","object[]","The GPU exporters to scrape, each with ",[168,611,319],{}," (",[168,614,615],{},"dcgm"," for dcgm-exporter, ",[168,618,619],{},"nvidia_smi"," for nvidia_gpu_exporter), ",[168,622,251],{}," (its ",[168,625,626],{},"\u002Fmetrics"," page) and ",[168,629,274],{}," (matching the replicas' ",[168,632,274],{},").",[239,635,289,637,639],{"name":636,"type":438},"flush_interval_s",[168,638,292],{},". How often rows are written to ClickHouse.",[239,641,289,643,646],{"name":642,"type":288},"max_buffered_rows",[168,644,645],{},"500000",". Rows held while ClickHouse is unreachable. Past this, the oldest are dropped.",[228,648,649],{"id":207},[168,650,207],{},[236,652,653,662,672,682,688],{},[239,654,289,655,658,659,531],{"name":274,"type":242},[168,656,657],{},"localhost",". ",[168,660,661],{},"ICO_CLICKHOUSE_HOST",[239,663,289,665,668,669,531],{"name":664,"type":288},"port",[168,666,667],{},"8123",", the HTTP port. ",[168,670,671],{},"ICO_CLICKHOUSE_PORT",[239,673,289,675,678,679,278],{"name":674,"type":242},"username",[168,676,677],{},"default",". The Compose deployment uses ",[168,680,681],{},"ico",[239,683,289,685,687],{"name":684,"type":242},"database",[168,686,681],{},". Created by the collector if it does not exist.",[239,689,289,692,695],{"name":690,"type":691},"secure","boolean",[168,693,694],{},"false",". HTTPS to ClickHouse.",[140,697,698,699,278],{},"The password comes only from ",[168,700,701],{},"ICO_CLICKHOUSE_PASSWORD",[228,703,705],{"id":704},"environment-variables","Environment variables",[144,707,708,720],{},[147,709,710],{},[150,711,712,715,717],{},[153,713,714],{},"Variable",[153,716,158],{},[153,718,719],{},"Meaning",[160,721,722,737,752,763,775,794,807],{},[150,723,724,729,731],{},[165,725,726],{},[168,727,728],{},"ICO_CONFIG",[165,730,173],{},[165,732,733,734,278],{},"Path of the config file. Default ",[168,735,736],{},"\u002Fetc\u002Fico\u002Fconfig.yaml",[150,738,739,744,749],{},[165,740,741],{},[168,742,743],{},"ICO_ADMIN_TOKEN",[165,745,746,747],{},"gateway, ",[168,748,216],{},[165,750,751],{},"Bearer token of the admin API. Unset: the API refuses every call.",[150,753,754,758,760],{},[165,755,756],{},[168,757,552],{},[165,759,184],{},[165,761,762],{},"Salt of the prefix hashes. Use the same on every gateway.",[150,764,765,769,772],{},[165,766,767],{},[168,768,701],{},[165,770,771],{},"collector, CLI",[165,773,774],{},"ClickHouse password.",[150,776,777,783,785],{},[165,778,779,181,781],{},[168,780,661],{},[168,782,671],{},[165,784,771],{},[165,786,787,788,363,791,278],{},"Override ",[168,789,790],{},"clickhouse.host",[168,792,793],{},"clickhouse.port",[150,795,796,800,802],{},[165,797,798],{},[168,799,530],{},[165,801,115],{},[165,803,804,805,278],{},"Overrides ",[168,806,190],{},[150,808,809,814,816],{},[165,810,811],{},[168,812,813],{},"ICO_SCHEMA_DIR",[165,815,223],{},[165,817,818,819,278],{},"Where the ClickHouse migrations are. Default ",[168,820,821],{},"\u002Fopt\u002Fico\u002Fschema",[823,824,825],"style",{},"html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":383,"searchDepth":400,"depth":400,"links":827},[828,829,830,831,832,833,834,835,836],{"id":170,"depth":400,"text":170},{"id":198,"depth":400,"text":198},{"id":201,"depth":400,"text":201},{"id":180,"depth":400,"text":180},{"id":204,"depth":400,"text":204},{"id":184,"depth":400,"text":184},{"id":223,"depth":400,"text":223},{"id":207,"depth":400,"text":207},{"id":704,"depth":400,"text":705},"Every key of the YAML config file, its default, and which part of ico reads it.","md",null,{},{"icon":113},{"title":110,"description":837},"crs8_q-TKquAB4NKbGn1uUB6QgATkp_tC9q_f04R6dc",[845,847],{"title":100,"path":101,"stem":102,"description":846,"icon":103,"children":-1},"Where the replay's numbers stop, so you can use them safely.",{"title":115,"path":116,"stem":117,"description":848,"icon":118,"children":-1},"The ico command and its options.",1792165824610]