[{"data":1,"prerenderedAt":540},["ShallowReactive",2],{"navigation_docs":3,"-install-check":134,"-install-check-surround":535},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":51,"body":136,"description":528,"extension":529,"links":530,"meta":531,"navigation":532,"path":52,"seo":533,"stem":53,"__hash__":534},"docs\u002F2.install\u002F5.check.md",{"type":137,"value":138,"toc":521},"minimark",[139,148,153,196,267,282,286,324,327,335,341,345,387,394,398,446,457,461,517],[140,141,142,143,147],"p",{},"Send some traffic through the gateway, then check each source in turn. The commands run on the ico host from ",[144,145,146],"code",{},"deploy\u002F",", unless they say otherwise.",[149,150,152],"h2",{"id":151},"the-gateway-exports-its-records","The gateway exports its records",[154,155,161],"pre",{"className":156,"code":157,"filename":158,"language":159,"meta":160,"style":160},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","curl -s -H \"Authorization: Bearer $ICO_ADMIN_TOKEN\" 127.0.0.1:8081\u002Fadmin\u002Fstats\n","GPU machine","bash","",[144,162,163],{"__ignoreMap":160},[164,165,168,172,176,179,183,186,190,193],"span",{"class":166,"line":167},"line",1,[164,169,171],{"class":170},"sBMFI","curl",[164,173,175],{"class":174},"sfazB"," -s",[164,177,178],{"class":174}," -H",[164,180,182],{"class":181},"sMK4o"," \"",[164,184,185],{"class":174},"Authorization: Bearer ",[164,187,189],{"class":188},"sTEyZ","$ICO_ADMIN_TOKEN",[164,191,192],{"class":181},"\"",[164,194,195],{"class":174}," 127.0.0.1:8081\u002Fadmin\u002Fstats\n",[154,197,201],{"className":198,"code":199,"language":200,"meta":160,"style":160},"language-json shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","{\"exported\": 1520, \"dropped\": 0, \"queued\": 0, \"failed_calls\": 0}\n","json",[144,202,203],{"__ignoreMap":160},[164,204,205,208,210,214,216,219,223,226,228,231,233,235,238,240,242,245,247,249,251,253,255,258,260,262,264],{"class":166,"line":167},[164,206,207],{"class":181},"{",[164,209,192],{"class":181},[164,211,213],{"class":212},"spNyl","exported",[164,215,192],{"class":181},[164,217,218],{"class":181},":",[164,220,222],{"class":221},"sbssI"," 1520",[164,224,225],{"class":181},",",[164,227,182],{"class":181},[164,229,230],{"class":212},"dropped",[164,232,192],{"class":181},[164,234,218],{"class":181},[164,236,237],{"class":221}," 0",[164,239,225],{"class":181},[164,241,182],{"class":181},[164,243,244],{"class":212},"queued",[164,246,192],{"class":181},[164,248,218],{"class":181},[164,250,237],{"class":221},[164,252,225],{"class":181},[164,254,182],{"class":181},[164,256,257],{"class":212},"failed_calls",[164,259,192],{"class":181},[164,261,218],{"class":181},[164,263,237],{"class":221},[164,265,266],{"class":181},"}\n",[140,268,269,271,272,274,275,277,278,281],{},[144,270,213],{}," grows with traffic. ",[144,273,230],{}," or ",[144,276,257],{}," above zero mean the gateway cannot reach the collector at the address in ",[144,279,280],{},"gateway.collector",".",[149,283,285],{"id":284},"the-collector-scrapes-the-engines-and-the-gpus","The collector scrapes the engines and the GPUs",[154,287,290],{"className":156,"code":288,"filename":289,"language":159,"meta":160,"style":160},"docker compose logs collector | grep -E 'telemetry level|scraping again|scrape failed'\n","ico host",[144,291,292],{"__ignoreMap":160},[164,293,294,297,300,303,306,309,312,315,318,321],{"class":166,"line":167},[164,295,296],{"class":170},"docker",[164,298,299],{"class":174}," compose",[164,301,302],{"class":174}," logs",[164,304,305],{"class":174}," collector",[164,307,308],{"class":181}," |",[164,310,311],{"class":170}," grep",[164,313,314],{"class":174}," -E",[164,316,317],{"class":181}," '",[164,319,320],{"class":174},"telemetry level|scraping again|scrape failed",[164,322,323],{"class":181},"'\n",[140,325,326],{},"For each exporter, look for a line like:",[154,328,333],{"className":329,"code":331,"language":332,"meta":160},[330],"language-text","dcgm:gpu-1: reports dram_active_ratio, memory_total_bytes, ..., sm_active_ratio, tensor_active_ratio, util_ratio at telemetry level L3\n","text",[144,334,331],{"__ignoreMap":160},[140,336,337,340],{},[144,338,339],{},"scrape failed"," that does not go away means the ico host cannot reach that URL: check the address in the config, the port the engine or exporter publishes, and the firewall between them.",[149,342,344],{"id":343},"requests-and-their-engine-traces-arrive","Requests and their engine traces arrive",[154,346,348],{"className":156,"code":347,"filename":289,"language":159,"meta":160,"style":160},"docker compose exec clickhouse sh -c 'clickhouse-client --user \"$CLICKHOUSE_USER\" --password \"$CLICKHOUSE_PASSWORD\" -d ico -q \"\n  SELECT replica, count() AS requests, round(avg(has_span) * 100) AS traced_percent\n  FROM request_spans WHERE ts > now() - INTERVAL 1 HOUR GROUP BY replica\"'\n",[144,349,350,373,379],{"__ignoreMap":160},[164,351,352,354,356,359,362,365,368,370],{"class":166,"line":167},[164,353,296],{"class":170},[164,355,299],{"class":174},[164,357,358],{"class":174}," exec",[164,360,361],{"class":174}," clickhouse",[164,363,364],{"class":174}," sh",[164,366,367],{"class":174}," -c",[164,369,317],{"class":181},[164,371,372],{"class":174},"clickhouse-client --user \"$CLICKHOUSE_USER\" --password \"$CLICKHOUSE_PASSWORD\" -d ico -q \"\n",[164,374,376],{"class":166,"line":375},2,[164,377,378],{"class":174},"  SELECT replica, count() AS requests, round(avg(has_span) * 100) AS traced_percent\n",[164,380,382,385],{"class":166,"line":381},3,[164,383,384],{"class":174},"  FROM request_spans WHERE ts > now() - INTERVAL 1 HOUR GROUP BY replica\"",[164,386,323],{"class":181},[140,388,389,390,393],{},"Every replica should show requests, and ",[144,391,392],{},"traced_percent"," near 100.",[149,395,397],{"id":396},"write-a-first-report","Write a first report",[154,399,401],{"className":156,"code":400,"filename":289,"language":159,"meta":160,"style":160},"docker compose exec collector ico report --window 1h --out \u002Ftmp\u002Freport.html\ndocker compose cp collector:\u002Ftmp\u002Freport.html report.html\n",[144,402,403,431],{"__ignoreMap":160},[164,404,405,407,409,411,413,416,419,422,425,428],{"class":166,"line":167},[164,406,296],{"class":170},[164,408,299],{"class":174},[164,410,358],{"class":174},[164,412,305],{"class":174},[164,414,415],{"class":174}," ico",[164,417,418],{"class":174}," report",[164,420,421],{"class":174}," --window",[164,423,424],{"class":174}," 1h",[164,426,427],{"class":174}," --out",[164,429,430],{"class":174}," \u002Ftmp\u002Freport.html\n",[164,432,433,435,437,440,443],{"class":166,"line":375},[164,434,296],{"class":170},[164,436,299],{"class":174},[164,438,439],{"class":174}," cp",[164,441,442],{"class":174}," collector:\u002Ftmp\u002Freport.html",[164,444,445],{"class":174}," report.html\n",[140,447,448,449,452,453,456],{},"See ",[450,451,66],"a",{"href":67},". After a day of traffic, the same command with ",[144,454,455],{},"--window 24h"," gives the first report worth acting on.",[149,458,460],{"id":459},"when-something-is-missing","When something is missing",[462,463,464,474,490,498,513],"accordion",{},[465,466,469,470,473],"accordion-item",{"icon":467,"label":468},"i-lucide-circle-help","No requests recorded at all","Traffic does not go through the gateway yet, or the gateway cannot reach the collector. Check that requests arrive at the gateway's port, then ",[144,471,472],{},"\u002Fadmin\u002Fstats"," above.",[465,475,477,478,481,482,485,486,489],{"icon":467,"label":476},"Requests recorded, but traced_percent is 0","vLLM sends no spans, or files them under another name. Check that vLLM was restarted with ",[144,479,480],{},"--otlp-traces-endpoint",", that ",[144,483,484],{},"OTEL_SERVICE_NAME"," is exactly the replica's name in the config, and that ",[144,487,488],{},"OTEL_EXPORTER_OTLP_TRACES_INSECURE=true"," is set. vLLM's own log names the problem at startup.",[465,491,493,494,497],{"icon":467,"label":492},"Telemetry level L2 on a datacenter GPU","The profiling counters are closed. Give dcgm-exporter the ",[144,495,496],{},"SYS_ADMIN"," capability; on some providers they stay closed regardless. ico still works: idle time comes from the gateway's requests either way, and only the figure for cores unused while serving is missing.",[465,499,501,502,274,505,508,509,512],{"icon":467,"label":500},"The gateway will not start: address already in use","Something else holds the port. Some provider images run their own services, for example nginx on 8081 on RunPod. Move ",[144,503,504],{},"gateway.listen",[144,506,507],{},"gateway.admin_listen"," to a free port, and ",[144,510,511],{},"gateway.admin_url"," with it.",[465,514,516],{"icon":467,"label":515},"dropped grows on the gateway","The collector is unreachable, or was for a while. Requests are unaffected; records made while the gateway's queue of 16,384 was full are lost, and the report undercounts that stretch.",[518,519,520],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .spNyl, html code.shiki .spNyl{--shiki-light:#9C3EDA;--shiki-default:#C792EA;--shiki-dark:#C792EA}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}",{"title":160,"searchDepth":375,"depth":375,"links":522},[523,524,525,526,527],{"id":151,"depth":375,"text":152},{"id":284,"depth":375,"text":285},{"id":343,"depth":375,"text":344},{"id":396,"depth":375,"text":397},{"id":459,"depth":375,"text":460},"Make sure every part reports, and what to do when one does not.","md",null,{},{"icon":54},{"title":51,"description":528},"KHxsRbyLJVXBXuBk15EbXlS4l7vZG_zLZi6kSSdL0ow",[536,538],{"title":46,"path":47,"stem":48,"description":537,"icon":49,"children":-1},"Point your router or clients at the gateway, and tell it who each request is for.",{"title":56,"path":57,"stem":58,"description":539,"icon":59,"children":-1},"We install ico with you, on your machines, if you would rather not do it alone.",1792165823544]