[{"data":1,"prerenderedAt":679},["ShallowReactive",2],{"navigation_docs":3,"-getting-started-quickstart":134,"-getting-started-quickstart-surround":674},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":21,"body":136,"description":667,"extension":668,"links":669,"meta":670,"navigation":671,"path":22,"seo":672,"stem":23,"__hash__":673},"docs\u002F1.getting-started\u002F3.quickstart.md",{"type":137,"value":138,"toc":657},"minimark",[139,143,146,163,575,580,583,613,617,638,644,648,653],[140,141,142],"p",{},"This runs the whole of ico on one machine with Docker, with two fake engines standing in for vLLM. It shows what the gateway records and what the report looks like before you install anything next to a real GPU.",[140,144,145],{},"You need Docker with Compose and git.",[147,148,149,150,154,155,158,159,162],"warning",{},"Run it on a machine without a real ico install. The quickstart uses the same Compose project, ",[151,152,153],"code",{},"ico",", so its last step would delete a real install's data. To be safe anyway, add ",[151,156,157],{},"-p ico-quickstart"," to every ",[151,160,161],{},"docker compose"," command below.",[164,165,167,172,214,218,234,256,260,296,306,310,313,493,496,500,551,562],"steps",{"level":166},"3",[168,169,171],"h3",{"id":170},"get-the-code","Get the code",[173,174,180],"pre",{"className":175,"code":176,"filename":177,"language":178,"meta":179,"style":179},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","git clone --branch v0.1.1 https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer.git\ncd inference-cost-optimizer\u002Fdeploy\n","Terminal","bash","",[151,181,182,204],{"__ignoreMap":179},[183,184,187,191,195,198,201],"span",{"class":185,"line":186},"line",1,[183,188,190],{"class":189},"sBMFI","git",[183,192,194],{"class":193},"sfazB"," clone",[183,196,197],{"class":193}," --branch",[183,199,200],{"class":193}," v0.1.1",[183,202,203],{"class":193}," https:\u002F\u002Fgithub.com\u002FGudjon01\u002Finference-cost-optimizer.git\n",[183,205,207,211],{"class":185,"line":206},2,[183,208,210],{"class":209},"s2Zo4","cd",[183,212,213],{"class":193}," inference-cost-optimizer\u002Fdeploy\n",[168,215,217],{"id":216},"set-the-two-secrets","Set the two secrets",[173,219,221],{"className":175,"code":220,"filename":177,"language":178,"meta":179,"style":179},"cp .env.example .env\n",[151,222,223],{"__ignoreMap":179},[183,224,225,228,231],{"class":185,"line":186},[183,226,227],{"class":189},"cp",[183,229,230],{"class":193}," .env.example",[183,232,233],{"class":193}," .env\n",[140,235,236,237,240,241,244,245,248,249,252,253,255],{},"Set ",[151,238,239],{},"ICO_ADMIN_TOKEN"," and ",[151,242,243],{},"ICO_CLICKHOUSE_PASSWORD"," in ",[151,246,247],{},".env"," to long random strings, for example the output of ",[151,250,251],{},"openssl rand -hex 24",". The first unlocks the gateway's admin API, the second is the password of ClickHouse's ",[151,254,153],{}," user.",[168,257,259],{"id":258},"start-the-stack-with-fake-engines","Start the stack with fake engines",[173,261,263],{"className":175,"code":262,"filename":177,"language":178,"meta":179,"style":179},"ICO_CONFIG_FILE=.\u002Fconfig.mock.yaml docker compose --profile mock up -d\n",[151,264,265],{"__ignoreMap":179},[183,266,267,271,275,278,281,284,287,290,293],{"class":185,"line":186},[183,268,270],{"class":269},"sTEyZ","ICO_CONFIG_FILE",[183,272,274],{"class":273},"sMK4o","=",[183,276,277],{"class":193},".\u002Fconfig.mock.yaml",[183,279,280],{"class":189}," docker",[183,282,283],{"class":193}," compose",[183,285,286],{"class":193}," --profile",[183,288,289],{"class":193}," mock",[183,291,292],{"class":193}," up",[183,294,295],{"class":193}," -d\n",[140,297,298,299,240,302,305],{},"This starts the gateway on port 8080, the collector, ClickHouse, and two fake engines, ",[151,300,301],{},"mock-1",[151,303,304],{},"mock-2",". The first start downloads the images, which takes a few minutes.",[168,307,309],{"id":308},"send-some-traffic","Send some traffic",[140,311,312],{},"Send requests through the gateway, tagged with a tenant and a feature:",[173,314,316],{"className":175,"code":315,"filename":177,"language":178,"meta":179,"style":179},"for i in $(seq 1 200); do\n  tenant=$([ $((i % 3)) -eq 0 ] && echo globex || echo acme)\n  curl -s http:\u002F\u002Flocalhost:8080\u002Fv1\u002Fchat\u002Fcompletions \\\n    -H \"Content-Type: application\u002Fjson\" \\\n    -H \"X-Tenant-Id: $tenant\" -H \"X-Feature: chat\" \\\n    -d '{\"model\": \"mock-model\", \"max_tokens\": 16,\n         \"messages\": [{\"role\": \"user\", \"content\": \"Summarize our refund policy.\"}]}' > \u002Fdev\u002Fnull\ndone\n",[151,317,318,349,401,416,433,460,472,487],{"__ignoreMap":179},[183,319,320,324,327,330,333,336,340,343,346],{"class":185,"line":186},[183,321,323],{"class":322},"s7zQu","for",[183,325,326],{"class":269}," i ",[183,328,329],{"class":322},"in",[183,331,332],{"class":273}," $(",[183,334,335],{"class":189},"seq",[183,337,339],{"class":338},"sbssI"," 1",[183,341,342],{"class":338}," 200",[183,344,345],{"class":273},");",[183,347,348],{"class":322}," do\n",[183,350,351,354,357,360,363,366,369,372,375,378,381,384,387,390,393,395,398],{"class":185,"line":206},[183,352,353],{"class":269},"  tenant",[183,355,356],{"class":273},"=$([",[183,358,359],{"class":273}," $((",[183,361,362],{"class":189},"i",[183,364,365],{"class":193}," %",[183,367,368],{"class":338}," 3",[183,370,371],{"class":273},"))",[183,373,374],{"class":273}," -eq",[183,376,377],{"class":338}," 0",[183,379,380],{"class":273}," ]",[183,382,383],{"class":273}," &&",[183,385,386],{"class":209}," echo",[183,388,389],{"class":193}," globex",[183,391,392],{"class":273}," ||",[183,394,386],{"class":209},[183,396,397],{"class":193}," acme",[183,399,400],{"class":273},")\n",[183,402,404,407,410,413],{"class":185,"line":403},3,[183,405,406],{"class":189},"  curl",[183,408,409],{"class":193}," -s",[183,411,412],{"class":193}," http:\u002F\u002Flocalhost:8080\u002Fv1\u002Fchat\u002Fcompletions",[183,414,415],{"class":269}," \\\n",[183,417,419,422,425,428,431],{"class":185,"line":418},4,[183,420,421],{"class":193},"    -H",[183,423,424],{"class":273}," \"",[183,426,427],{"class":193},"Content-Type: application\u002Fjson",[183,429,430],{"class":273},"\"",[183,432,415],{"class":269},[183,434,436,438,440,443,446,448,451,453,456,458],{"class":185,"line":435},5,[183,437,421],{"class":193},[183,439,424],{"class":273},[183,441,442],{"class":193},"X-Tenant-Id: ",[183,444,445],{"class":269},"$tenant",[183,447,430],{"class":273},[183,449,450],{"class":193}," -H",[183,452,424],{"class":273},[183,454,455],{"class":193},"X-Feature: chat",[183,457,430],{"class":273},[183,459,415],{"class":269},[183,461,463,466,469],{"class":185,"line":462},6,[183,464,465],{"class":193},"    -d",[183,467,468],{"class":273}," '",[183,470,471],{"class":193},"{\"model\": \"mock-model\", \"max_tokens\": 16,\n",[183,473,475,478,481,484],{"class":185,"line":474},7,[183,476,477],{"class":193},"         \"messages\": [{\"role\": \"user\", \"content\": \"Summarize our refund policy.\"}]}",[183,479,480],{"class":273},"'",[183,482,483],{"class":273}," >",[183,485,486],{"class":193}," \u002Fdev\u002Fnull\n",[183,488,490],{"class":185,"line":489},8,[183,491,492],{"class":322},"done\n",[140,494,495],{},"The fake engines take about a third of a second per request, so this runs for a little over a minute. The gateway spreads the requests over both engines.",[168,497,499],{"id":498},"write-the-report","Write the report",[173,501,503],{"className":175,"code":502,"filename":177,"language":178,"meta":179,"style":179},"docker compose exec collector ico report --window 1h --out \u002Ftmp\u002Freport.html\ndocker compose cp collector:\u002Ftmp\u002Freport.html report.html\n",[151,504,505,536],{"__ignoreMap":179},[183,506,507,510,512,515,518,521,524,527,530,533],{"class":185,"line":186},[183,508,509],{"class":189},"docker",[183,511,283],{"class":193},[183,513,514],{"class":193}," exec",[183,516,517],{"class":193}," collector",[183,519,520],{"class":193}," ico",[183,522,523],{"class":193}," report",[183,525,526],{"class":193}," --window",[183,528,529],{"class":193}," 1h",[183,531,532],{"class":193}," --out",[183,534,535],{"class":193}," \u002Ftmp\u002Freport.html\n",[183,537,538,540,542,545,548],{"class":185,"line":206},[183,539,509],{"class":189},[183,541,283],{"class":193},[183,543,544],{"class":193}," cp",[183,546,547],{"class":193}," collector:\u002Ftmp\u002Freport.html",[183,549,550],{"class":193}," report.html\n",[140,552,553,554,557,558,561],{},"On Windows, in Git Bash, put ",[151,555,556],{},"MSYS_NO_PATHCONV=1"," before both commands, so ",[151,559,560],{},"\u002Ftmp\u002Freport.html"," is not rewritten as a Windows path.",[140,563,564,565,568,569,574],{},"Open ",[151,566,567],{},"report.html"," in a browser. It is one file with no scripts and nothing loaded from elsewhere. The fake engines report no GPU counters, so it measures idle time from request timing; see ",[570,571,573],"a",{"href":572},"\u002Fgetting-started\u002Fhow-it-works#how-much-it-can-see","How much it can see",".",[576,577,579],"h2",{"id":578},"look-at-the-raw-records","Look at the raw records",[140,581,582],{},"Every request the gateway proxied is one row in ClickHouse:",[173,584,586],{"className":175,"code":585,"filename":177,"language":178,"meta":179,"style":179},"docker compose exec clickhouse sh -c 'clickhouse-client --user \"$CLICKHOUSE_USER\" --password \"$CLICKHOUSE_PASSWORD\" -d ico -q \"SELECT ts, replica, tenant, prompt_tokens, completion_tokens, ttft_ms, e2e_ms FROM requests ORDER BY ts DESC LIMIT 10\"'\n",[151,587,588],{"__ignoreMap":179},[183,589,590,592,594,596,599,602,605,607,610],{"class":185,"line":186},[183,591,509],{"class":189},[183,593,283],{"class":193},[183,595,514],{"class":193},[183,597,598],{"class":193}," clickhouse",[183,600,601],{"class":193}," sh",[183,603,604],{"class":193}," -c",[183,606,468],{"class":273},[183,608,609],{"class":193},"clickhouse-client --user \"$CLICKHOUSE_USER\" --password \"$CLICKHOUSE_PASSWORD\" -d ico -q \"SELECT ts, replica, tenant, prompt_tokens, completion_tokens, ttft_ms, e2e_ms FROM requests ORDER BY ts DESC LIMIT 10\"",[183,611,612],{"class":273},"'\n",[576,614,616],{"id":615},"clean-up","Clean up",[173,618,620],{"className":175,"code":619,"filename":177,"language":178,"meta":179,"style":179},"docker compose --profile mock down -v\n",[151,621,622],{"__ignoreMap":179},[183,623,624,626,628,630,632,635],{"class":185,"line":186},[183,625,509],{"class":189},[183,627,283],{"class":193},[183,629,286],{"class":193},[183,631,289],{"class":193},[183,633,634],{"class":193}," down",[183,636,637],{"class":193}," -v\n",[140,639,640,643],{},[151,641,642],{},"-v"," also deletes the ClickHouse volume and everything recorded in it.",[576,645,647],{"id":646},"next","Next",[140,649,650,651,574],{},"When you are ready for real GPUs, start with ",[570,652,26],{"href":32},[654,655,656],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .s2Zo4, html code.shiki .s2Zo4{--shiki-light:#6182B8;--shiki-default:#82AAFF;--shiki-dark:#82AAFF}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .s7zQu, html code.shiki .s7zQu{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#89DDFF;--shiki-default-font-style:italic;--shiki-dark:#89DDFF;--shiki-dark-font-style:italic}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}",{"title":179,"searchDepth":206,"depth":206,"links":658},[659,660,661,662,663,664,665,666],{"id":170,"depth":403,"text":171},{"id":216,"depth":403,"text":217},{"id":258,"depth":403,"text":259},{"id":308,"depth":403,"text":309},{"id":498,"depth":403,"text":499},{"id":578,"depth":206,"text":579},{"id":615,"depth":206,"text":616},{"id":646,"depth":206,"text":647},"A first report in ten minutes on your laptop, with two fake engines and no GPU.","md",null,{},{"icon":24},{"title":21,"description":667},"9oa25FGXriaP-JP0Eap1pOVvp7tgNdFIp8HzV0WGOHo",[675,677],{"title":16,"path":17,"stem":18,"description":676,"icon":19,"children":-1},"The four parts of ico, what each one does, and what flows between them.",{"title":31,"path":32,"stem":33,"description":678,"icon":34,"children":-1},"What runs where on a fleet of rented GPU machines, and what has to reach what.",1792165822789]