[{"data":1,"prerenderedAt":723},["ShallowReactive",2],{"navigation_docs":3,"-reference-gateway":134,"-reference-gateway-surround":718},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":120,"body":136,"description":711,"extension":712,"links":713,"meta":714,"navigation":715,"path":121,"seo":716,"stem":122,"__hash__":717},"docs\u002F5.reference\u002F3.gateway.md",{"type":137,"value":138,"toc":703},"minimark",[139,144,229,241,245,386,392,396,403,407,418,521,619,623,629,633,649,699],[140,141,143],"h2",{"id":142},"proxied-endpoints","Proxied endpoints",[145,146,147,160],"table",{},[148,149,150],"thead",{},[151,152,153,157],"tr",{},[154,155,156],"th",{},"Endpoint",[154,158,159],{},"What the gateway does",[161,162,163,175,185,194,204,218],"tbody",{},[151,164,165,172],{},[166,167,168],"td",{},[169,170,171],"code",{},"POST \u002Fv1\u002Fchat\u002Fcompletions",[166,173,174],{},"Forwards to a replica and records the request.",[151,176,177,182],{},[166,178,179],{},[169,180,181],{},"POST \u002Fv1\u002Fcompletions",[166,183,184],{},"Forwards and records.",[151,186,187,192],{},[166,188,189],{},[169,190,191],{},"POST \u002Fv1\u002Fembeddings",[166,193,184],{},[151,195,196,201],{},[166,197,198],{},[169,199,200],{},"GET \u002Fv1\u002Fmodels",[166,202,203],{},"Answers from the next replica; no record.",[151,205,206,211],{},[166,207,208],{},[169,209,210],{},"GET \u002Fhealthz",[166,212,213,214,217],{},"Answers ",[169,215,216],{},"ok"," itself.",[151,219,220,223],{},[166,221,222],{},"anything else",[166,224,225,228],{},[169,226,227],{},"404",", with an OpenAI-style error.",[230,231,232,233,236,237,240],"p",{},"Replicas are chosen round robin among those that serve the request's model and are not drained. A replica that refuses the connection is skipped for that request. When every replica of a model is drained, the gateway answers ",[169,234,235],{},"503","; when none can be reached, ",[169,238,239],{},"502",".",[140,242,244],{"id":243},"what-a-request-record-holds","What a request record holds",[145,246,247,257],{},[148,248,249],{},[151,250,251,254],{},[154,252,253],{},"Field",[154,255,256],{},"Meaning",[161,258,259,280,299,315,331,344,357,376],{},[151,260,261,270],{},[166,262,263,266,267],{},[169,264,265],{},"request_id",", ",[169,268,269],{},"trace_id",[166,271,272,273,276,277],{},"The gateway's id for the request, sent upstream as ",[169,274,275],{},"X-Request-Id",", and the W3C trace id it starts, sent as ",[169,278,279],{},"traceparent",[151,281,282,296],{},[166,283,284,266,287,266,290,266,293],{},[169,285,286],{},"ts",[169,288,289],{},"replica",[169,291,292],{},"model",[169,294,295],{},"endpoint",[166,297,298],{},"When, where and what",[151,300,301,309],{},[166,302,303,266,306],{},[169,304,305],{},"tenant",[169,307,308],{},"feature",[166,310,311,312],{},"From the request headers named in ",[169,313,314],{},"headers",[151,316,317,328],{},[166,318,319,266,322,266,325],{},[169,320,321],{},"prompt_tokens",[169,323,324],{},"completion_tokens",[169,326,327],{},"cached_tokens",[166,329,330],{},"From the engine's usage figures",[151,332,333,341],{},[166,334,335,266,338],{},[169,336,337],{},"ttft_ms",[169,339,340],{},"e2e_ms",[166,342,343],{},"Time to the first streamed token, and to the end of the response",[151,345,346,354],{},[166,347,348,266,351],{},[169,349,350],{},"gap_p50_ms",[169,352,353],{},"gap_p95_ms",[166,355,356],{},"Median and 95th percentile of the gaps between streamed chunks",[151,358,359,373],{},[166,360,361,266,364,266,367,266,370],{},[169,362,363],{},"status",[169,365,366],{},"stream",[169,368,369],{},"max_tokens",[169,371,372],{},"prompt_chars",[166,374,375],{},"The response status, whether it streamed, the requested limit, and the request text's length",[151,377,378,383],{},[166,379,380],{},[169,381,382],{},"prefix_hashes",[166,384,385],{},"Salted hashes of the request text, block by block",[230,387,388,389,240],{},"No prompt or completion text is kept. See ",[390,391,130],"a",{"href":131},[140,393,395],{"id":394},"streams","Streams",[230,397,398,399,402],{},"Streams are relayed event by event. The gateway turns on ",[169,400,401],{},"stream_options.include_usage"," upstream, so the engine reports token counts at the end of a stream, and withholds that extra chunk from clients that did not ask for it.",[140,404,406],{"id":405},"admin-api","Admin API",[230,408,409,410,413,414,417],{},"On ",[169,411,412],{},"gateway.admin_listen",", with the bearer token from ",[169,415,416],{},"ICO_ADMIN_TOKEN",". Without a token set, it refuses every call.",[419,420,425],"pre",{"className":421,"code":422,"language":423,"meta":424,"style":424},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","curl -H \"Authorization: Bearer $ICO_ADMIN_TOKEN\" 127.0.0.1:8081\u002Fadmin\u002Freplicas\ncurl -H \"Authorization: Bearer $ICO_ADMIN_TOKEN\" -X POST 127.0.0.1:8081\u002Fadmin\u002Freplicas\u002Fgpu-1-r1\u002Fdrain\ncurl -H \"Authorization: Bearer $ICO_ADMIN_TOKEN\" -X POST 127.0.0.1:8081\u002Fadmin\u002Freplicas\u002Fgpu-1-r1\u002Fundrain\ncurl -H \"Authorization: Bearer $ICO_ADMIN_TOKEN\" 127.0.0.1:8081\u002Fadmin\u002Fstats\n","bash","",[169,426,427,457,481,503],{"__ignoreMap":424},[428,429,432,436,440,444,447,451,454],"span",{"class":430,"line":431},"line",1,[428,433,435],{"class":434},"sBMFI","curl",[428,437,439],{"class":438},"sfazB"," -H",[428,441,443],{"class":442},"sMK4o"," \"",[428,445,446],{"class":438},"Authorization: Bearer ",[428,448,450],{"class":449},"sTEyZ","$ICO_ADMIN_TOKEN",[428,452,453],{"class":442},"\"",[428,455,456],{"class":438}," 127.0.0.1:8081\u002Fadmin\u002Freplicas\n",[428,458,460,462,464,466,468,470,472,475,478],{"class":430,"line":459},2,[428,461,435],{"class":434},[428,463,439],{"class":438},[428,465,443],{"class":442},[428,467,446],{"class":438},[428,469,450],{"class":449},[428,471,453],{"class":442},[428,473,474],{"class":438}," -X",[428,476,477],{"class":438}," POST",[428,479,480],{"class":438}," 127.0.0.1:8081\u002Fadmin\u002Freplicas\u002Fgpu-1-r1\u002Fdrain\n",[428,482,484,486,488,490,492,494,496,498,500],{"class":430,"line":483},3,[428,485,435],{"class":434},[428,487,439],{"class":438},[428,489,443],{"class":442},[428,491,446],{"class":438},[428,493,450],{"class":449},[428,495,453],{"class":442},[428,497,474],{"class":438},[428,499,477],{"class":438},[428,501,502],{"class":438}," 127.0.0.1:8081\u002Fadmin\u002Freplicas\u002Fgpu-1-r1\u002Fundrain\n",[428,504,506,508,510,512,514,516,518],{"class":430,"line":505},4,[428,507,435],{"class":434},[428,509,439],{"class":438},[428,511,443],{"class":442},[428,513,446],{"class":438},[428,515,450],{"class":449},[428,517,453],{"class":442},[428,519,520],{"class":438}," 127.0.0.1:8081\u002Fadmin\u002Fstats\n",[145,522,523,533],{},[148,524,525],{},[151,526,527,530],{},[154,528,529],{},"Call",[154,531,532],{},"Answer",[161,534,535,568,585,596],{},[151,536,537,542],{},[166,538,539],{},[169,540,541],{},"GET \u002Fadmin\u002Freplicas",[166,543,544,545,266,548,266,551,266,554,266,557,266,560,563,564,567],{},"Each replica: ",[169,546,547],{},"name",[169,549,550],{},"url",[169,552,553],{},"host",[169,555,556],{},"gpu_model",[169,558,559],{},"gpu_count",[169,561,562],{},"drained",", and ",[169,565,566],{},"inflight",", its requests in flight.",[151,569,570,575],{},[166,571,572],{},[169,573,574],{},"POST \u002Fadmin\u002Freplicas\u002F{name}\u002Fdrain",[166,576,577,578,581,582,584],{},"The replica, and ",[169,579,580],{},"was_drained",". A drained replica gets no new requests; it is idle once ",[169,583,566],{}," reads 0.",[151,586,587,592],{},[166,588,589],{},[169,590,591],{},"POST \u002Fadmin\u002Freplicas\u002F{name}\u002Fundrain",[166,593,577,594,240],{},[169,595,580],{},[151,597,598,603],{},[166,599,600],{},[169,601,602],{},"GET \u002Fadmin\u002Fstats",[166,604,605,606,266,609,266,612,615,616,240],{},"The export: ",[169,607,608],{},"exported",[169,610,611],{},"dropped",[169,613,614],{},"queued"," and ",[169,617,618],{},"failed_calls",[140,620,622],{"id":621},"when-the-collector-is-down","When the collector is down",[230,624,625,626,628],{},"Records leave the request path at once, into a queue of 16,384 that is sent in batches of up to 512, with retries. When the queue is full, new records are dropped and counted in ",[169,627,611],{},". Requests never wait for the collector.",[140,630,632],{"id":631},"running-it","Running it",[419,634,636],{"className":421,"code":635,"language":423,"meta":424,"style":424},"gateway -config \u002Fetc\u002Fico\u002Fconfig.yaml\n",[169,637,638],{"__ignoreMap":424},[428,639,640,643,646],{"class":430,"line":431},[428,641,642],{"class":434},"gateway",[428,644,645],{"class":438}," -config",[428,647,648],{"class":438}," \u002Fetc\u002Fico\u002Fconfig.yaml\n",[145,650,651,660],{},[148,652,653],{},[151,654,655,658],{},[154,656,657],{},"Variable",[154,659,256],{},[161,661,662,676,685],{},[151,663,664,669],{},[166,665,666],{},[169,667,668],{},"ICO_CONFIG",[166,670,671,672,675],{},"The config file, if ",[169,673,674],{},"-config"," is not given.",[151,677,678,682],{},[166,679,680],{},[169,681,416],{},[166,683,684],{},"Turns on the admin API.",[151,686,687,692],{},[166,688,689],{},[169,690,691],{},"ICO_HASH_SALT",[166,693,694,695,698],{},"The salt of the prefix hashes. Without it, the gateway makes one in ",[169,696,697],{},"state_dir"," on first start.",[700,701,702],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":424,"searchDepth":459,"depth":459,"links":704},[705,706,707,708,709,710],{"id":142,"depth":459,"text":143},{"id":243,"depth":459,"text":244},{"id":394,"depth":459,"text":395},{"id":405,"depth":459,"text":406},{"id":621,"depth":459,"text":622},{"id":631,"depth":459,"text":632},"What the gateway proxies, records and exposes.","md",null,{},{"icon":123},{"title":120,"description":711},"XoXHyi83NgwrWGghDW-_puoyJ7Soh8qDihsfmZ-loMU",[719,721],{"title":115,"path":116,"stem":117,"description":720,"icon":118,"children":-1},"The ico command and its options.",{"title":125,"path":126,"stem":127,"description":722,"icon":128,"children":-1},"The ClickHouse tables and views ico writes and reads, and the names it stores metrics under.",1792165819410]