[{"data":1,"prerenderedAt":1131},["ShallowReactive",2],{"navigation_docs":3,"-install-gpu-machines":134,"-install-gpu-machines-surround":1126},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":41,"body":136,"description":1119,"extension":1120,"links":1121,"meta":1122,"navigation":1123,"path":42,"seo":1124,"stem":43,"__hash__":1125},"docs\u002F2.install\u002F3.gpu-machines.md",{"type":137,"value":138,"toc":1113},"minimark",[139,165,168,199,204,207,356,374,377,419,423,427,430,591,594,628,631,635,638,841,844,1078,1101,1105,1109],[140,141,142,143,147,148,152,153,156,157,160,161,164],"p",{},"Do this on every GPU machine. The examples use the addresses of the ",[144,145,146],"a",{"href":37},"ico host page",": the ico host at ",[149,150,151],"code",{},"10.0.0.5",", and this machine, ",[149,154,155],{},"gpu-1",", at ",[149,158,159],{},"10.0.0.11"," with one vLLM replica named ",[149,162,163],{},"gpu-1-r1",".",[140,166,167],{},"Start by getting the code; the steps below use files from it:",[169,170,175],"pre",{"className":171,"code":172,"filename":155,"language":173,"meta":174,"style":174},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","git clone --branch v0.1.1 https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer.git\n","bash","",[149,176,177],{"__ignoreMap":174},[178,179,182,186,190,193,196],"span",{"class":180,"line":181},"line",1,[178,183,185],{"class":184},"sBMFI","git",[178,187,189],{"class":188},"sfazB"," clone",[178,191,192],{"class":188}," --branch",[178,194,195],{"class":188}," v0.1.1",[178,197,198],{"class":188}," https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer.git\n",[200,201,203],"h2",{"id":202},"_1-turn-on-tracing-in-vllm","1. Turn on tracing in vLLM",[140,205,206],{},"vLLM sends one OpenTelemetry span per request, with the time it spent queued, in prefill and in decode. Add one flag and two environment variables to the way you start it:",[208,209,210,302],"code-group",{},[169,211,214],{"className":171,"code":212,"filename":213,"language":173,"meta":174,"style":174},"docker run -d --name vllm --gpus all -p 10.0.0.11:8000:8000 \\\n  -e OTEL_SERVICE_NAME=gpu-1-r1 \\\n  -e OTEL_EXPORTER_OTLP_TRACES_INSECURE=true \\\n  vllm\u002Fvllm-openai:v0.31.0 \\\n  --model Qwen\u002FQwen3-8B \\\n  --otlp-traces-endpoint grpc:\u002F\u002F10.0.0.5:4317\n","docker run",[149,215,216,249,260,274,282,293],{"__ignoreMap":174},[178,217,218,221,224,227,230,233,236,239,242,245],{"class":180,"line":181},[178,219,220],{"class":184},"docker",[178,222,223],{"class":188}," run",[178,225,226],{"class":188}," -d",[178,228,229],{"class":188}," --name",[178,231,232],{"class":188}," vllm",[178,234,235],{"class":188}," --gpus",[178,237,238],{"class":188}," all",[178,240,241],{"class":188}," -p",[178,243,244],{"class":188}," 10.0.0.11:8000:8000",[178,246,248],{"class":247},"sTEyZ"," \\\n",[178,250,252,255,258],{"class":180,"line":251},2,[178,253,254],{"class":188},"  -e",[178,256,257],{"class":188}," OTEL_SERVICE_NAME=gpu-1-r1",[178,259,248],{"class":247},[178,261,263,265,268,272],{"class":180,"line":262},3,[178,264,254],{"class":188},[178,266,267],{"class":188}," OTEL_EXPORTER_OTLP_TRACES_INSECURE=",[178,269,271],{"class":270},"sMK4o","true",[178,273,248],{"class":247},[178,275,277,280],{"class":180,"line":276},4,[178,278,279],{"class":188},"  vllm\u002Fvllm-openai:v0.31.0",[178,281,248],{"class":247},[178,283,285,288,291],{"class":180,"line":284},5,[178,286,287],{"class":188},"  --model",[178,289,290],{"class":188}," Qwen\u002FQwen3-8B",[178,292,248],{"class":247},[178,294,296,299],{"class":180,"line":295},6,[178,297,298],{"class":188},"  --otlp-traces-endpoint",[178,300,301],{"class":188}," grpc:\u002F\u002F10.0.0.5:4317\n",[169,303,306],{"className":171,"code":304,"filename":305,"language":173,"meta":174,"style":174},"OTEL_SERVICE_NAME=gpu-1-r1 \\\nOTEL_EXPORTER_OTLP_TRACES_INSECURE=true \\\nvllm serve Qwen\u002FQwen3-8B --host 0.0.0.0 --port 8000 \\\n  --otlp-traces-endpoint grpc:\u002F\u002F10.0.0.5:4317\n","vllm serve",[149,307,308,320,325,350],{"__ignoreMap":174},[178,309,310,313,316,318],{"class":180,"line":181},[178,311,312],{"class":247},"OTEL_SERVICE_NAME",[178,314,315],{"class":270},"=",[178,317,163],{"class":188},[178,319,248],{"class":184},[178,321,322],{"class":180,"line":251},[178,323,324],{"class":247},"OTEL_EXPORTER_OTLP_TRACES_INSECURE=true \\\n",[178,326,327,330,333,335,338,342,345,348],{"class":180,"line":262},[178,328,329],{"class":247},"vllm ",[178,331,332],{"class":188},"serve",[178,334,290],{"class":188},[178,336,337],{"class":188}," --host",[178,339,341],{"class":340},"sbssI"," 0.0.0.0",[178,343,344],{"class":188}," --port",[178,346,347],{"class":340}," 8000",[178,349,248],{"class":247},[178,351,352,354],{"class":180,"line":276},[178,353,298],{"class":188},[178,355,301],{"class":188},[357,358,359,365,371],"ul",{},[360,361,362,364],"li",{},[149,363,312],{}," must be the replica's name in the config. The collector files each span under it.",[360,366,367,370],{},[149,368,369],{},"OTEL_EXPORTER_OTLP_TRACES_INSECURE=true",": the collector's port is plain gRPC, on your private network.",[360,372,373],{},"Keep your other flags as they are.",[140,375,376],{},"If vLLM exits saying that OpenTelemetry is not available, its environment lacks the packages. Add them:",[208,378,379,397],{},[169,380,385],{"className":381,"code":382,"filename":383,"language":384,"meta":174,"style":174},"language-dockerfile shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","FROM vllm\u002Fvllm-openai:v0.31.0\nRUN pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp\n","Dockerfile","dockerfile",[149,386,387,392],{"__ignoreMap":174},[178,388,389],{"class":180,"line":181},[178,390,391],{},"FROM vllm\u002Fvllm-openai:v0.31.0\n",[178,393,394],{"class":180,"line":251},[178,395,396],{},"RUN pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp\n",[169,398,401],{"className":171,"code":399,"filename":400,"language":173,"meta":174,"style":174},"pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp\n","pip",[149,402,403],{"__ignoreMap":174},[178,404,405,407,410,413,416],{"class":180,"line":181},[178,406,400],{"class":184},[178,408,409],{"class":188}," install",[178,411,412],{"class":188}," opentelemetry-sdk",[178,414,415],{"class":188}," opentelemetry-api",[178,417,418],{"class":188}," opentelemetry-exporter-otlp\n",[420,421,422],"note",{},"vLLM 0.31 puts prefill and decode time in its spans. Older versions, such as 0.10, leave them out; the report then splits costs across tenants by token counts instead.",[200,424,426],{"id":425},"_2-run-a-gpu-exporter","2. Run a GPU exporter",[140,428,429],{},"The collector reads the GPUs through NVIDIA's own exporters; ico's containers never touch the GPU driver.",[431,432,433,532],"tabs",{},[434,435,438,441,526],"tabs-item",{"icon":436,"label":437},"i-lucide-server","Datacenter GPUs",[140,439,440],{},"For H100, H200, A100, L40S, L4 and other datacenter cards, run NVIDIA's dcgm-exporter with the counters file from ico's repository:",[169,442,444],{"className":171,"code":443,"filename":155,"language":173,"meta":174,"style":174},"docker run -d --name dcgm-exporter --restart unless-stopped \\\n  --gpus all --cap-add SYS_ADMIN \\\n  -p 10.0.0.11:9400:9400 \\\n  -v \"$PWD\u002Finference-cost-optimizer\u002Fdeploy\u002Fdcgm-counters.csv:\u002Fetc\u002Fdcgm-exporter\u002Fico-counters.csv:ro\" \\\n  nvcr.io\u002Fnvidia\u002Fk8s\u002Fdcgm-exporter:4.5.2-4.8.1-ubuntu22.04 \\\n  -f \u002Fetc\u002Fdcgm-exporter\u002Fico-counters.csv\n",[149,445,446,467,482,492,511,518],{"__ignoreMap":174},[178,447,448,450,452,454,456,459,462,465],{"class":180,"line":181},[178,449,220],{"class":184},[178,451,223],{"class":188},[178,453,226],{"class":188},[178,455,229],{"class":188},[178,457,458],{"class":188}," dcgm-exporter",[178,460,461],{"class":188}," --restart",[178,463,464],{"class":188}," unless-stopped",[178,466,248],{"class":247},[178,468,469,472,474,477,480],{"class":180,"line":251},[178,470,471],{"class":188},"  --gpus",[178,473,238],{"class":188},[178,475,476],{"class":188}," --cap-add",[178,478,479],{"class":188}," SYS_ADMIN",[178,481,248],{"class":247},[178,483,484,487,490],{"class":180,"line":262},[178,485,486],{"class":188},"  -p",[178,488,489],{"class":188}," 10.0.0.11:9400:9400",[178,491,248],{"class":247},[178,493,494,497,500,503,506,509],{"class":180,"line":276},[178,495,496],{"class":188},"  -v",[178,498,499],{"class":270}," \"",[178,501,502],{"class":247},"$PWD",[178,504,505],{"class":188},"\u002Finference-cost-optimizer\u002Fdeploy\u002Fdcgm-counters.csv:\u002Fetc\u002Fdcgm-exporter\u002Fico-counters.csv:ro",[178,507,508],{"class":270},"\"",[178,510,248],{"class":247},[178,512,513,516],{"class":180,"line":284},[178,514,515],{"class":188},"  nvcr.io\u002Fnvidia\u002Fk8s\u002Fdcgm-exporter:4.5.2-4.8.1-ubuntu22.04",[178,517,248],{"class":247},[178,519,520,523],{"class":180,"line":295},[178,521,522],{"class":188},"  -f",[178,524,525],{"class":188}," \u002Fetc\u002Fdcgm-exporter\u002Fico-counters.csv\n",[140,527,528,531],{},[149,529,530],{},"SYS_ADMIN"," lets it read the profiling counters, which tell busy cores from idle ones (telemetry level L3). Some hosts allow them without it.",[434,533,536,539,581],{"icon":534,"label":535},"i-lucide-gamepad-2","GeForce GPUs",[140,537,538],{},"GeForce cards have no DCGM support. Run nvidia_gpu_exporter:",[169,540,542],{"className":171,"code":541,"filename":155,"language":173,"meta":174,"style":174},"docker run -d --name nvidia-gpu-exporter --restart unless-stopped \\\n  --gpus all -p 10.0.0.11:9835:9835 \\\n  utkuozdemir\u002Fnvidia_gpu_exporter:1.15.1\n",[149,543,544,563,576],{"__ignoreMap":174},[178,545,546,548,550,552,554,557,559,561],{"class":180,"line":181},[178,547,220],{"class":184},[178,549,223],{"class":188},[178,551,226],{"class":188},[178,553,229],{"class":188},[178,555,556],{"class":188}," nvidia-gpu-exporter",[178,558,461],{"class":188},[178,560,464],{"class":188},[178,562,248],{"class":247},[178,564,565,567,569,571,574],{"class":180,"line":251},[178,566,471],{"class":188},[178,568,238],{"class":188},[178,570,241],{"class":188},[178,572,573],{"class":188}," 10.0.0.11:9835:9835",[178,575,248],{"class":247},[178,577,578],{"class":180,"line":262},[178,579,580],{"class":188},"  utkuozdemir\u002Fnvidia_gpu_exporter:1.15.1\n",[140,582,583,584,587,588,164],{},"In the ico host's config, list it with ",[149,585,586],{},"kind: nvidia_smi"," and ",[149,589,590],{},"url: http:\u002F\u002F10.0.0.11:9835\u002Fmetrics",[140,592,593],{},"Check that the profiling counters are there, about 30 seconds after the start:",[169,595,597],{"className":171,"code":596,"filename":155,"language":173,"meta":174,"style":174},"curl -s 10.0.0.11:9400\u002Fmetrics | grep -E '^DCGM_FI_PROF_(SM_ACTIVE|PIPE_TENSOR_ACTIVE|DRAM_ACTIVE)'\n",[149,598,599],{"__ignoreMap":174},[178,600,601,604,607,610,613,616,619,622,625],{"class":180,"line":181},[178,602,603],{"class":184},"curl",[178,605,606],{"class":188}," -s",[178,608,609],{"class":188}," 10.0.0.11:9400\u002Fmetrics",[178,611,612],{"class":270}," |",[178,614,615],{"class":184}," grep",[178,617,618],{"class":188}," -E",[178,620,621],{"class":270}," '",[178,623,624],{"class":188},"^DCGM_FI_PROF_(SM_ACTIVE|PIPE_TENSOR_ACTIVE|DRAM_ACTIVE)",[178,626,627],{"class":270},"'\n",[140,629,630],{},"Three lines mean telemetry level L3. None means the host keeps them closed; ico still works, at level L2.",[200,632,634],{"id":633},"_3-run-the-gateway","3. Run the gateway",[140,636,637],{},"The gateway has its own config file on each machine, listing only the replicas it sends requests to:",[169,639,644],{"className":640,"code":641,"filename":642,"language":643,"meta":174,"style":174},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","replicas:\n  - name: gpu-1-r1                # the same name as in the ico host's config\n    url: http:\u002F\u002F10.0.0.11:8000    # how the gateway reaches the engine\n    host: gpu-1\n    gpu_model: H100-SXM\n    gpu_count: 1\n\nheaders:\n  tenant: X-Tenant-Id             # see Send traffic through the gateway\n  feature: X-Feature\n\ngateway:\n  listen: \"10.0.0.11:8080\"        # where your router or clients reach it\n  admin_listen: \"127.0.0.1:8081\"  # drain and undrain; keep it local\n  admin_url: http:\u002F\u002F127.0.0.1:8081 # where ico replay on this machine finds it\n  collector: 10.0.0.5:4317\n  state_dir: \u002Fvar\u002Flib\u002Fico\n","gateway.yaml","yaml",[149,645,646,655,673,686,696,706,716,723,731,745,756,761,769,787,805,819,830],{"__ignoreMap":174},[178,647,648,652],{"class":180,"line":181},[178,649,651],{"class":650},"swJcz","replicas",[178,653,654],{"class":270},":\n",[178,656,657,660,663,666,669],{"class":180,"line":251},[178,658,659],{"class":270},"  -",[178,661,662],{"class":650}," name",[178,664,665],{"class":270},":",[178,667,668],{"class":188}," gpu-1-r1",[178,670,672],{"class":671},"sHwdD","                # the same name as in the ico host's config\n",[178,674,675,678,680,683],{"class":180,"line":262},[178,676,677],{"class":650},"    url",[178,679,665],{"class":270},[178,681,682],{"class":188}," http:\u002F\u002F10.0.0.11:8000",[178,684,685],{"class":671},"    # how the gateway reaches the engine\n",[178,687,688,691,693],{"class":180,"line":276},[178,689,690],{"class":650},"    host",[178,692,665],{"class":270},[178,694,695],{"class":188}," gpu-1\n",[178,697,698,701,703],{"class":180,"line":284},[178,699,700],{"class":650},"    gpu_model",[178,702,665],{"class":270},[178,704,705],{"class":188}," H100-SXM\n",[178,707,708,711,713],{"class":180,"line":295},[178,709,710],{"class":650},"    gpu_count",[178,712,665],{"class":270},[178,714,715],{"class":340}," 1\n",[178,717,719],{"class":180,"line":718},7,[178,720,722],{"emptyLinePlaceholder":721},true,"\n",[178,724,726,729],{"class":180,"line":725},8,[178,727,728],{"class":650},"headers",[178,730,654],{"class":270},[178,732,734,737,739,742],{"class":180,"line":733},9,[178,735,736],{"class":650},"  tenant",[178,738,665],{"class":270},[178,740,741],{"class":188}," X-Tenant-Id",[178,743,744],{"class":671},"             # see Send traffic through the gateway\n",[178,746,748,751,753],{"class":180,"line":747},10,[178,749,750],{"class":650},"  feature",[178,752,665],{"class":270},[178,754,755],{"class":188}," X-Feature\n",[178,757,759],{"class":180,"line":758},11,[178,760,722],{"emptyLinePlaceholder":721},[178,762,764,767],{"class":180,"line":763},12,[178,765,766],{"class":650},"gateway",[178,768,654],{"class":270},[178,770,772,775,777,779,782,784],{"class":180,"line":771},13,[178,773,774],{"class":650},"  listen",[178,776,665],{"class":270},[178,778,499],{"class":270},[178,780,781],{"class":188},"10.0.0.11:8080",[178,783,508],{"class":270},[178,785,786],{"class":671},"        # where your router or clients reach it\n",[178,788,790,793,795,797,800,802],{"class":180,"line":789},14,[178,791,792],{"class":650},"  admin_listen",[178,794,665],{"class":270},[178,796,499],{"class":270},[178,798,799],{"class":188},"127.0.0.1:8081",[178,801,508],{"class":270},[178,803,804],{"class":671},"  # drain and undrain; keep it local\n",[178,806,808,811,813,816],{"class":180,"line":807},15,[178,809,810],{"class":650},"  admin_url",[178,812,665],{"class":270},[178,814,815],{"class":188}," http:\u002F\u002F127.0.0.1:8081",[178,817,818],{"class":671}," # where ico replay on this machine finds it\n",[178,820,822,825,827],{"class":180,"line":821},16,[178,823,824],{"class":650},"  collector",[178,826,665],{"class":270},[178,828,829],{"class":188}," 10.0.0.5:4317\n",[178,831,833,836,838],{"class":180,"line":832},17,[178,834,835],{"class":650},"  state_dir",[178,837,665],{"class":270},[178,839,840],{"class":188}," \u002Fvar\u002Flib\u002Fico\n",[140,842,843],{},"Start it next to vLLM:",[431,845,846,941],{},[434,847,850],{"icon":848,"label":849},"i-lucide-container","Docker",[169,851,853],{"className":171,"code":852,"filename":155,"language":173,"meta":174,"style":174},"docker run -d --name ico-gateway --restart unless-stopped --network host \\\n  -e ICO_ADMIN_TOKEN=\"\u003Cthe token from the ico host's .env>\" \\\n  -e ICO_HASH_SALT=\"\u003Cthe same salt on every gateway>\" \\\n  -v \"$PWD\u002Fgateway.yaml:\u002Fetc\u002Fico\u002Fconfig.yaml:ro\" \\\n  -v ico-gateway-state:\u002Fvar\u002Flib\u002Fico \\\n  ghcr.io\u002Fgudjon01\u002Fico-gateway:0.1.1\n",[149,854,855,880,896,912,927,936],{"__ignoreMap":174},[178,856,857,859,861,863,865,868,870,872,875,878],{"class":180,"line":181},[178,858,220],{"class":184},[178,860,223],{"class":188},[178,862,226],{"class":188},[178,864,229],{"class":188},[178,866,867],{"class":188}," ico-gateway",[178,869,461],{"class":188},[178,871,464],{"class":188},[178,873,874],{"class":188}," --network",[178,876,877],{"class":188}," host",[178,879,248],{"class":247},[178,881,882,884,887,889,892,894],{"class":180,"line":251},[178,883,254],{"class":188},[178,885,886],{"class":188}," ICO_ADMIN_TOKEN=",[178,888,508],{"class":270},[178,890,891],{"class":188},"\u003Cthe token from the ico host's .env>",[178,893,508],{"class":270},[178,895,248],{"class":247},[178,897,898,900,903,905,908,910],{"class":180,"line":262},[178,899,254],{"class":188},[178,901,902],{"class":188}," ICO_HASH_SALT=",[178,904,508],{"class":270},[178,906,907],{"class":188},"\u003Cthe same salt on every gateway>",[178,909,508],{"class":270},[178,911,248],{"class":247},[178,913,914,916,918,920,923,925],{"class":180,"line":276},[178,915,496],{"class":188},[178,917,499],{"class":270},[178,919,502],{"class":247},[178,921,922],{"class":188},"\u002Fgateway.yaml:\u002Fetc\u002Fico\u002Fconfig.yaml:ro",[178,924,508],{"class":270},[178,926,248],{"class":247},[178,928,929,931,934],{"class":180,"line":284},[178,930,496],{"class":188},[178,932,933],{"class":188}," ico-gateway-state:\u002Fvar\u002Flib\u002Fico",[178,935,248],{"class":247},[178,937,938],{"class":180,"line":295},[178,939,940],{"class":188},"  ghcr.io\u002Fgudjon01\u002Fico-gateway:0.1.1\n",[434,942,945,956,1069],{"icon":943,"label":944},"i-lucide-terminal","No Docker, such as a RunPod pod",[140,946,947,948,951,952,955],{},"The gateway is one static binary. Take it from the release, for ",[149,949,950],{},"amd64"," or ",[149,953,954],{},"arm64",", and check it against the release's checksums:",[169,957,959],{"className":171,"code":958,"filename":155,"language":173,"meta":174,"style":174},"curl -fLO https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer\u002Freleases\u002Fdownload\u002Fv0.1.1\u002Fico-gateway-linux-amd64\ncurl -fLO https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer\u002Freleases\u002Fdownload\u002Fv0.1.1\u002FSHA256SUMS\nsha256sum --check --ignore-missing SHA256SUMS\ninstall -m 755 ico-gateway-linux-amd64 gateway\n\nICO_ADMIN_TOKEN=\"\u003Cthe token from the ico host's .env>\" \\\nICO_HASH_SALT=\"\u003Cthe same salt on every gateway>\" \\\nnohup .\u002Fgateway -config gateway.yaml > gateway.log 2>&1 &\n",[149,960,961,971,980,994,1011,1015,1030,1043],{"__ignoreMap":174},[178,962,963,965,968],{"class":180,"line":181},[178,964,603],{"class":184},[178,966,967],{"class":188}," -fLO",[178,969,970],{"class":188}," https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer\u002Freleases\u002Fdownload\u002Fv0.1.1\u002Fico-gateway-linux-amd64\n",[178,972,973,975,977],{"class":180,"line":251},[178,974,603],{"class":184},[178,976,967],{"class":188},[178,978,979],{"class":188}," https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer\u002Freleases\u002Fdownload\u002Fv0.1.1\u002FSHA256SUMS\n",[178,981,982,985,988,991],{"class":180,"line":262},[178,983,984],{"class":184},"sha256sum",[178,986,987],{"class":188}," --check",[178,989,990],{"class":188}," --ignore-missing",[178,992,993],{"class":188}," SHA256SUMS\n",[178,995,996,999,1002,1005,1008],{"class":180,"line":276},[178,997,998],{"class":184},"install",[178,1000,1001],{"class":188}," -m",[178,1003,1004],{"class":340}," 755",[178,1006,1007],{"class":188}," ico-gateway-linux-amd64",[178,1009,1010],{"class":188}," gateway\n",[178,1012,1013],{"class":180,"line":284},[178,1014,722],{"emptyLinePlaceholder":721},[178,1016,1017,1020,1022,1024,1026,1028],{"class":180,"line":295},[178,1018,1019],{"class":247},"ICO_ADMIN_TOKEN",[178,1021,315],{"class":270},[178,1023,508],{"class":270},[178,1025,891],{"class":188},[178,1027,508],{"class":270},[178,1029,248],{"class":184},[178,1031,1032,1035,1037,1039,1041],{"class":180,"line":718},[178,1033,1034],{"class":247},"ICO_HASH_SALT=",[178,1036,508],{"class":270},[178,1038,907],{"class":188},[178,1040,508],{"class":270},[178,1042,248],{"class":247},[178,1044,1045,1048,1051,1054,1057,1060,1063,1066],{"class":180,"line":725},[178,1046,1047],{"class":247},"nohup ",[178,1049,1050],{"class":188},".\u002Fgateway",[178,1052,1053],{"class":188}," -config",[178,1055,1056],{"class":188}," gateway.yaml",[178,1058,1059],{"class":270}," >",[178,1061,1062],{"class":188}," gateway.log",[178,1064,1065],{"class":270}," 2>&1",[178,1067,1068],{"class":270}," &\n",[140,1070,1071,1074,1075,1077],{},[149,1072,1073],{},"state_dir"," in ",[149,1076,642],{}," must be a directory the gateway can write to. Nothing restarts it for you: start it again the same way after the machine restarts.",[357,1079,1080,1089],{},[360,1081,1082,1084,1085,1088],{},[149,1083,1019],{}," turns on the admin API, which ",[149,1086,1087],{},"ico replay"," uses to drain this replica. Without it the API refuses every call.",[360,1090,1091,1094,1095,1098,1099,164],{},[149,1092,1093],{},"ICO_HASH_SALT"," keys the hashes of request text. Give every gateway the same one, made once with ",[149,1096,1097],{},"openssl rand -hex 32",", so that a prompt prefix seen on two machines hashes the same. Treat it as a secret. Without it, each gateway makes its own salt in ",[149,1100,1073],{},[200,1102,1104],{"id":1103},"next","Next",[140,1106,1107,164],{},[144,1108,46],{"href":47},[1110,1111,1112],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}",{"title":174,"searchDepth":251,"depth":251,"links":1114},[1115,1116,1117,1118],{"id":202,"depth":251,"text":203},{"id":425,"depth":251,"text":426},{"id":633,"depth":251,"text":634},{"id":1103,"depth":251,"text":1104},"On every GPU machine, turn on tracing in vLLM, run a GPU exporter, and run the gateway.","md",null,{},{"icon":44},{"title":41,"description":1119},"IebyoGdtJiwMVqrxu1hARh5oZ3aYs73BmAKbbIkt-HE",[1127,1129],{"title":36,"path":37,"stem":38,"description":1128,"icon":39,"children":-1},"ClickHouse and the collector on one small machine, with Docker Compose.",{"title":46,"path":47,"stem":48,"description":1130,"icon":49,"children":-1},"Point your router or clients at the gateway, and tell it who each request is for.",1792165823135]