[{"data":1,"prerenderedAt":601},["ShallowReactive",2],{"navigation_docs":3,"-install-send-traffic":134,"-install-send-traffic-surround":596},[4,25,60,75,104],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":16,"path":17,"stem":18,"icon":19},"How it works","\u002Fgetting-started\u002Fhow-it-works","1.getting-started\u002F2.how-it-works","i-lucide-workflow",{"title":21,"path":22,"stem":23,"icon":24},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F3.quickstart","i-lucide-rocket",{"title":26,"icon":6,"path":27,"stem":28,"children":29,"page":6},"Install on rented GPUs","\u002Finstall","2.install",[30,35,40,45,50,55],{"title":31,"path":32,"stem":33,"icon":34},"Overview","\u002Finstall\u002Foverview","2.install\u002F1.overview","i-lucide-map",{"title":36,"path":37,"stem":38,"icon":39},"Set up the ico host","\u002Finstall\u002Fico-host","2.install\u002F2.ico-host","i-lucide-database",{"title":41,"path":42,"stem":43,"icon":44},"Set up the GPU machines","\u002Finstall\u002Fgpu-machines","2.install\u002F3.gpu-machines","i-lucide-cpu",{"title":46,"path":47,"stem":48,"icon":49},"Send traffic through the gateway","\u002Finstall\u002Fsend-traffic","2.install\u002F4.send-traffic","i-lucide-route",{"title":51,"path":52,"stem":53,"icon":54},"Check the install","\u002Finstall\u002Fcheck","2.install\u002F5.check","i-lucide-circle-check",{"title":56,"path":57,"stem":58,"icon":59},"Install help","\u002Finstall\u002Finstall-help","2.install\u002F6.install-help","i-lucide-life-buoy",{"title":61,"icon":6,"path":62,"stem":63,"children":64,"page":6},"The report","\u002Freport","3.report",[65,70],{"title":66,"path":67,"stem":68,"icon":69},"Read the report","\u002Freport\u002Fread-the-report","3.report\u002F1.read-the-report","i-lucide-file-chart-column",{"title":71,"path":72,"stem":73,"icon":74},"How costs are counted","\u002Freport\u002Fhow-costs-are-counted","3.report\u002F2.how-costs-are-counted","i-lucide-calculator",{"title":76,"icon":6,"path":77,"stem":78,"children":79,"page":6},"The replay lab","\u002Freplay","4.replay",[80,84,89,94,99],{"title":31,"path":81,"stem":82,"icon":83},"\u002Freplay\u002Foverview","4.replay\u002F1.overview","i-lucide-flask-conical",{"title":85,"path":86,"stem":87,"icon":88},"Run a replay","\u002Freplay\u002Frun-a-replay","4.replay\u002F2.run-a-replay","i-lucide-play",{"title":90,"path":91,"stem":92,"icon":93},"Launchers","\u002Freplay\u002Flaunchers","4.replay\u002F3.launchers","i-lucide-power",{"title":95,"path":96,"stem":97,"icon":98},"Sweep files","\u002Freplay\u002Fsweep-file","4.replay\u002F4.sweep-file","i-lucide-list-checks",{"title":100,"path":101,"stem":102,"icon":103},"What a replay cannot tell you yet","\u002Freplay\u002Flimits","4.replay\u002F5.limits","i-lucide-triangle-alert",{"title":105,"icon":6,"path":106,"stem":107,"children":108,"page":6},"Reference","\u002Freference","5.reference",[109,114,119,124,129],{"title":110,"path":111,"stem":112,"icon":113},"Configuration","\u002Freference\u002Fconfiguration","5.reference\u002F1.configuration","i-lucide-settings",{"title":115,"path":116,"stem":117,"icon":118},"CLI","\u002Freference\u002Fcli","5.reference\u002F2.cli","i-lucide-square-terminal",{"title":120,"path":121,"stem":122,"icon":123},"Gateway","\u002Freference\u002Fgateway","5.reference\u002F3.gateway","i-lucide-split",{"title":125,"path":126,"stem":127,"icon":128},"Stored data","\u002Freference\u002Fdata","5.reference\u002F4.data","i-lucide-table",{"title":130,"path":131,"stem":132,"icon":133},"Privacy and security","\u002Freference\u002Fsecurity","5.reference\u002F5.security","i-lucide-shield-check",{"id":135,"title":46,"body":136,"description":589,"extension":590,"links":591,"meta":592,"navigation":593,"path":47,"seo":594,"stem":48,"__hash__":595},"docs\u002F2.install\u002F4.send-traffic.md",{"type":137,"value":138,"toc":584},"minimark",[139,143,148,235,238,296,307,310,314,317,371,374,377,516,519,557,565,572,576,580],[140,141,142],"p",{},"This is the one change in your request path. The gateway speaks the same OpenAI-compatible API as vLLM, so moving traffic to it is a change of address.",[144,145,147],"h2",{"id":146},"point-traffic-at-the-gateway","Point traffic at the gateway",[149,150,151,204],"tabs",{},[152,153,156,168],"tabs-item",{"icon":154,"label":155},"i-lucide-layers","Through your router",[140,157,158,159,163,164,167],{},"Change each upstream of your router from vLLM's port to the gateway's port on the same machine, for example from ",[160,161,162],"code",{},"10.0.0.11:8000"," to ",[160,165,166],{},"10.0.0.11:8080",". With nginx:",[169,170,176],"pre",{"className":171,"code":172,"filename":173,"language":174,"meta":175,"style":175},"language-nginx shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","upstream llm {\n  server 10.0.0.11:8080;   # was 10.0.0.11:8000\n  server 10.0.0.12:8080;   # was 10.0.0.12:8000\n}\n","nginx.conf","nginx","",[160,177,178,186,192,198],{"__ignoreMap":175},[179,180,183],"span",{"class":181,"line":182},"line",1,[179,184,185],{},"upstream llm {\n",[179,187,189],{"class":181,"line":188},2,[179,190,191],{},"  server 10.0.0.11:8080;   # was 10.0.0.11:8000\n",[179,193,195],{"class":181,"line":194},3,[179,196,197],{},"  server 10.0.0.12:8080;   # was 10.0.0.12:8000\n",[179,199,201],{"class":181,"line":200},4,[179,202,203],{},"}\n",[152,205,208,211],{"icon":206,"label":207},"i-lucide-code","Clients directly",[140,209,210],{},"Change the base URL your clients use:",[169,212,217],{"className":213,"code":214,"filename":215,"language":216,"meta":175,"style":175},"language-python shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","from openai import OpenAI\n\nclient = OpenAI(base_url=\"http:\u002F\u002F10.0.0.11:8080\u002Fv1\", api_key=\"unused\")\n","client.py","python",[160,218,219,224,230],{"__ignoreMap":175},[179,220,221],{"class":181,"line":182},[179,222,223],{},"from openai import OpenAI\n",[179,225,226],{"class":181,"line":188},[179,227,229],{"emptyLinePlaceholder":228},true,"\n",[179,231,232],{"class":181,"line":194},[179,233,234],{},"client = OpenAI(base_url=\"http:\u002F\u002F10.0.0.11:8080\u002Fv1\", api_key=\"unused\")\n",[140,236,237],{},"The gateway proxies these endpoints and answers everything else with 404:",[239,240,241,254],"table",{},[242,243,244],"thead",{},[245,246,247,251],"tr",{},[248,249,250],"th",{},"Endpoint",[248,252,253],{},"Recorded",[255,256,257,268,277,286],"tbody",{},[245,258,259,265],{},[260,261,262],"td",{},[160,263,264],{},"POST \u002Fv1\u002Fchat\u002Fcompletions",[260,266,267],{},"Yes",[245,269,270,275],{},[260,271,272],{},[160,273,274],{},"POST \u002Fv1\u002Fcompletions",[260,276,267],{},[245,278,279,284],{},[260,280,281],{},[160,282,283],{},"POST \u002Fv1\u002Fembeddings",[260,285,267],{},[245,287,288,293],{},[260,289,290],{},[160,291,292],{},"GET \u002Fv1\u002Fmodels",[260,294,295],{},"No, answered from the next replica",[140,297,298,299,302,303,306],{},"Health checks of your load balancer can use the gateway's own ",[160,300,301],{},"GET \u002Fhealthz",", which answers ",[160,304,305],{},"ok"," without touching an engine.",[140,308,309],{},"Streams pass through event by event. The gateway asks vLLM for token counts at the end of each stream and keeps that extra chunk from clients that did not ask for it, so nothing changes for them.",[144,311,313],{"id":312},"say-who-each-request-is-for","Say who each request is for",[140,315,316],{},"Two request headers tell the gateway who a request was for and what for:",[239,318,319,332],{},[242,320,321],{},[245,322,323,326,329],{},[248,324,325],{},"Header",[248,327,328],{},"Meaning",[248,330,331],{},"Example",[255,333,334,349],{},[245,335,336,341,344],{},[260,337,338],{},[160,339,340],{},"X-Tenant-Id",[260,342,343],{},"Your customer: whom the cost belongs to",[260,345,346],{},[160,347,348],{},"acme-corp",[245,350,351,356,359],{},[260,352,353],{},[160,354,355],{},"X-Feature",[260,357,358],{},"The part of your product that sent it",[260,360,361,364,365,364,368],{},[160,362,363],{},"chat",", ",[160,366,367],{},"search",[160,369,370],{},"agent",[140,372,373],{},"The report splits each replica's cost across tenants. The feature is recorded for your own queries.",[140,375,376],{},"Set them where you know the answer. Usually that is the service that authenticates your customers:",[378,379,380,412,441],"code-group",{},[169,381,384],{"className":213,"code":382,"filename":383,"language":216,"meta":175,"style":175},"client = OpenAI(\n    base_url=\"http:\u002F\u002F10.0.0.11:8080\u002Fv1\",\n    api_key=\"unused\",\n    default_headers={\"X-Tenant-Id\": customer.id, \"X-Feature\": \"chat\"},\n)\n","OpenAI SDK",[160,385,386,391,396,401,406],{"__ignoreMap":175},[179,387,388],{"class":181,"line":182},[179,389,390],{},"client = OpenAI(\n",[179,392,393],{"class":181,"line":188},[179,394,395],{},"    base_url=\"http:\u002F\u002F10.0.0.11:8080\u002Fv1\",\n",[179,397,398],{"class":181,"line":194},[179,399,400],{},"    api_key=\"unused\",\n",[179,402,403],{"class":181,"line":200},[179,404,405],{},"    default_headers={\"X-Tenant-Id\": customer.id, \"X-Feature\": \"chat\"},\n",[179,407,409],{"class":181,"line":408},5,[179,410,411],{},")\n",[169,413,415],{"className":171,"code":414,"filename":174,"language":174,"meta":175,"style":175},"location \u002Fv1\u002F {\n  proxy_set_header X-Tenant-Id $customer_id;   # from your auth layer\n  proxy_set_header X-Feature   chat;\n  proxy_pass http:\u002F\u002Fllm;\n}\n",[160,416,417,422,427,432,437],{"__ignoreMap":175},[179,418,419],{"class":181,"line":182},[179,420,421],{},"location \u002Fv1\u002F {\n",[179,423,424],{"class":181,"line":188},[179,425,426],{},"  proxy_set_header X-Tenant-Id $customer_id;   # from your auth layer\n",[179,428,429],{"class":181,"line":194},[179,430,431],{},"  proxy_set_header X-Feature   chat;\n",[179,433,434],{"class":181,"line":200},[179,435,436],{},"  proxy_pass http:\u002F\u002Fllm;\n",[179,438,439],{"class":181,"line":408},[179,440,203],{},[169,442,447],{"className":443,"code":444,"filename":445,"language":446,"meta":175,"style":175},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","curl http:\u002F\u002F10.0.0.11:8080\u002Fv1\u002Fchat\u002Fcompletions \\\n  -H \"Content-Type: application\u002Fjson\" \\\n  -H \"X-Tenant-Id: acme-corp\" -H \"X-Feature: chat\" \\\n  -d '{\"model\": \"Qwen\u002FQwen3-8B\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'\n","curl","bash",[160,448,449,462,479,502],{"__ignoreMap":175},[179,450,451,454,458],{"class":181,"line":182},[179,452,445],{"class":453},"sBMFI",[179,455,457],{"class":456},"sfazB"," http:\u002F\u002F10.0.0.11:8080\u002Fv1\u002Fchat\u002Fcompletions",[179,459,461],{"class":460},"sTEyZ"," \\\n",[179,463,464,467,471,474,477],{"class":181,"line":188},[179,465,466],{"class":456},"  -H",[179,468,470],{"class":469},"sMK4o"," \"",[179,472,473],{"class":456},"Content-Type: application\u002Fjson",[179,475,476],{"class":469},"\"",[179,478,461],{"class":460},[179,480,481,483,485,488,490,493,495,498,500],{"class":181,"line":194},[179,482,466],{"class":456},[179,484,470],{"class":469},[179,486,487],{"class":456},"X-Tenant-Id: acme-corp",[179,489,476],{"class":469},[179,491,492],{"class":456}," -H",[179,494,470],{"class":469},[179,496,497],{"class":456},"X-Feature: chat",[179,499,476],{"class":469},[179,501,461],{"class":460},[179,503,504,507,510,513],{"class":181,"line":200},[179,505,506],{"class":456},"  -d",[179,508,509],{"class":469}," '",[179,511,512],{"class":456},"{\"model\": \"Qwen\u002FQwen3-8B\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}",[179,514,515],{"class":469},"'\n",[140,517,518],{},"If your stack already carries the customer in a header of another name, tell the gateway its name instead of adding one:",[169,520,525],{"className":521,"code":522,"filename":523,"language":524,"meta":175,"style":175},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","headers:\n  tenant: X-Customer\n  feature: X-Product-Area\n","gateway.yaml","yaml",[160,526,527,536,547],{"__ignoreMap":175},[179,528,529,533],{"class":181,"line":182},[179,530,532],{"class":531},"swJcz","headers",[179,534,535],{"class":469},":\n",[179,537,538,541,544],{"class":181,"line":188},[179,539,540],{"class":531},"  tenant",[179,542,543],{"class":469},":",[179,545,546],{"class":456}," X-Customer\n",[179,548,549,552,554],{"class":181,"line":194},[179,550,551],{"class":531},"  feature",[179,553,543],{"class":469},[179,555,556],{"class":456}," X-Product-Area\n",[140,558,559,560,564],{},"Requests without the header are still recorded and still cost money; the report lists them as ",[561,562,563],"strong",{},"(no tenant header)",".",[566,567,568,569,564],"note",{},"The gateway never stores request or response text. It hashes the text of each request in memory, in blocks of 256 characters with a per-install salt, and keeps only the hashes, which show which requests share a prefix. See ",[570,571,130],"a",{"href":131},[144,573,575],{"id":574},"next","Next",[140,577,578,564],{},[570,579,51],{"href":52},[581,582,583],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}",{"title":175,"searchDepth":188,"depth":188,"links":585},[586,587,588],{"id":146,"depth":188,"text":147},{"id":312,"depth":188,"text":313},{"id":574,"depth":188,"text":575},"Point your router or clients at the gateway, and tell it who each request is for.","md",null,{},{"icon":49},{"title":46,"description":589},"k3zxiqzBBUUUjM0yLzHqqnde4kJFXzf1onn2nATvBs8",[597,599],{"title":41,"path":42,"stem":43,"description":598,"icon":44,"children":-1},"On every GPU machine, turn on tracing in vLLM, run a GPU exporter, and run the gateway.",{"title":51,"path":52,"stem":53,"description":600,"icon":54,"children":-1},"Make sure every part reports, and what to do when one does not.",1792165823238]