{"generated_at": "2026-10-08T00:56:43.268523+00:00", "source": "Cookbook recipe benchmarks via analytics.public.base_autoinference__flash_recipe_benchmark", "aa_intelligence_index_source": "https://artificialanalysis.ai/leaderboards/models", "workloads": [{"pattern": "real_time_generation", "label": "Real-time generation", "cached_input_tokens": 0, "uncached_input_tokens": 3000, "output_tokens": 100, "recipes": [{"recipe_id": "rec-9156c8980428", "recipe_name": "hy3-gptq-int4-vllm-b200x4", "model": "AngelSlim/Hy3-GPTQ-Int4", "model_family": "hy_v3", "engine": "vLLM", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-a33ca49dfc", "benchmarked_at": "2026-09-15T23:53:12.333750+00:00", "aa_intelligence_index": 25.3, "aa_slug": "hy3", "points": [{"request_rate": 1.28, "ttft_p50_ms": 239.00789600003236, "ttlt_p50_ms": 764.8893544999851, "itl_p50_ms": 14.610049000054914, "output_tokens_per_s": 128.2344077064499, "total_tokens_per_min": 238515.9983339968}, {"request_rate": 1.92, "ttft_p50_ms": 107.40893249993633, "ttlt_p50_ms": 1244.713284500051, "itl_p50_ms": 30.3365219999705, "output_tokens_per_s": 221.47504938922935, "total_tokens_per_min": 411943.5918639666}, {"request_rate": 2.88, "ttft_p50_ms": 102.20579399992857, "ttlt_p50_ms": 1277.5744504999693, "itl_p50_ms": 30.576890000020285, "output_tokens_per_s": 322.2337922849936, "total_tokens_per_min": 599354.8536500881}, {"request_rate": 4.32, "ttft_p50_ms": 121.91674149994469, "ttlt_p50_ms": 1536.3613099999611, "itl_p50_ms": 39.915947000054075, "output_tokens_per_s": 458.5053548649178, "total_tokens_per_min": 852819.9600487471}, {"request_rate": 6.48, "ttft_p50_ms": 153.88693399995645, "ttlt_p50_ms": 2036.0690549999845, "itl_p50_ms": 48.45902349984499, "output_tokens_per_s": 615.6807738800699, "total_tokens_per_min": 1145166.23941693}, {"request_rate": 9.72, "ttft_p50_ms": 930.5134199998975, "ttlt_p50_ms": 2997.3354790000712, "itl_p50_ms": 50.29205899995759, "output_tokens_per_s": 694.3866880631481, "total_tokens_per_min": 1291559.2397974553}, {"request_rate": 14.58, "ttft_p50_ms": 1883.6266084999806, "ttlt_p50_ms": 4096.864747999916, "itl_p50_ms": 50.33729299998413, "output_tokens_per_s": 696.2305666995491, "total_tokens_per_min": 1294988.854061161}]}, {"recipe_id": "rec-384b32610abf", "recipe_name": "deepseek-v4-flash-sglang-b200x4", "model": "deepseek-ai/DeepSeek-V4-Flash", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-74113c0aa0", "benchmarked_at": "2026-08-25T04:30:27.187389+00:00", "aa_intelligence_index": 34.3, "aa_slug": "deepseek-v4-flash", "points": [{"request_rate": 2.42, "ttft_p50_ms": 93.4620640000503, "ttlt_p50_ms": 345.66510000001927, "itl_p50_ms": 2.3372614999933226, "output_tokens_per_s": 241.51856873225404, "total_tokens_per_min": 449224.5378419925}, {"request_rate": 3.62, "ttft_p50_ms": 112.32497949993103, "ttlt_p50_ms": 597.4836809999715, "itl_p50_ms": 2.9012456666729727, "output_tokens_per_s": 382.00294633249916, "total_tokens_per_min": 710525.4801784485}, {"request_rate": 5.43, "ttft_p50_ms": 158.5451914999112, "ttlt_p50_ms": 751.7788124998788, "itl_p50_ms": 3.5214542500057178, "output_tokens_per_s": 563.4988308321169, "total_tokens_per_min": 1048107.8253477374}, {"request_rate": 8.14, "ttft_p50_ms": 161.26006650006275, "ttlt_p50_ms": 819.1362544999947, "itl_p50_ms": 3.8353336666053415, "output_tokens_per_s": 823.0913007971697, "total_tokens_per_min": 1530949.8194827354}, {"request_rate": 12.21, "ttft_p50_ms": 162.37040249984602, "ttlt_p50_ms": 1210.4900259998885, "itl_p50_ms": 4.207969000011265, "output_tokens_per_s": 1185.6376102256825, "total_tokens_per_min": 2205285.9550197697}, {"request_rate": 18.32, "ttft_p50_ms": 175.38023849999718, "ttlt_p50_ms": 2245.5452215000378, "itl_p50_ms": 6.6885518333871605, "output_tokens_per_s": 1620.3773092516617, "total_tokens_per_min": 3013901.7952080905}, {"request_rate": 27.48, "ttft_p50_ms": 189.26131100010934, "ttlt_p50_ms": 1922.329937499967, "itl_p50_ms": 5.759781999965223, "output_tokens_per_s": 2091.2980631102087, "total_tokens_per_min": 3889814.3973849886}]}, {"recipe_id": "rec-4c0e6a8145be", "recipe_name": "deepseek-v4-flash-0731-sglang-b200x2", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 2, "benchmark_id": "bench-4e712d21b6", "benchmarked_at": "2026-08-25T04:46:46.963068+00:00", "aa_intelligence_index": 34.3, "aa_slug": "deepseek-v4-flash", "points": [{"request_rate": 3.38, "ttft_p50_ms": 88.91956950003532, "ttlt_p50_ms": 235.47355299996298, "itl_p50_ms": 1.3959377500043502, "output_tokens_per_s": 338.4659340242426, "total_tokens_per_min": 629546.6372850912}, {"request_rate": 5.08, "ttft_p50_ms": 157.50635550000425, "ttlt_p50_ms": 561.6104305000249, "itl_p50_ms": 2.179645833336963, "output_tokens_per_s": 534.3684867615084, "total_tokens_per_min": 993925.3853764057}, {"request_rate": 7.62, "ttft_p50_ms": 2247.923984500005, "ttlt_p50_ms": 5683.872138999959, "itl_p50_ms": 4.42757216666223, "output_tokens_per_s": 642.4122582868299, "total_tokens_per_min": 1194886.8004135038}, {"request_rate": 11.43, "ttft_p50_ms": 167.80809150003506, "ttlt_p50_ms": 738.9565095000421, "itl_p50_ms": 3.0923780000004326, "output_tokens_per_s": 1146.1991591372625, "total_tokens_per_min": 2131930.4359953087}, {"request_rate": 17.14, "ttft_p50_ms": 206.25036799998497, "ttlt_p50_ms": 2313.6646694999286, "itl_p50_ms": 4.239211500002208, "output_tokens_per_s": 1362.659105173626, "total_tokens_per_min": 2534545.935622944}, {"request_rate": 25.71, "ttft_p50_ms": 200.22518899997976, "ttlt_p50_ms": 1774.5180489999939, "itl_p50_ms": 4.61953769999468, "output_tokens_per_s": 2149.570615108008, "total_tokens_per_min": 3998201.344100895}, {"request_rate": 38.56, "ttft_p50_ms": 211.58136399998284, "ttlt_p50_ms": 1563.5580309999568, "itl_p50_ms": 4.485976333323076, "output_tokens_per_s": 2677.8447258438223, "total_tokens_per_min": 4980791.190069511}]}, {"recipe_id": "rec-9c8e58e6bf05", "recipe_name": "deepseek-v4-pro-sglang-b200x8", "model": "deepseek-ai/DeepSeek-V4-Pro", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-db10027e2b", "benchmarked_at": "2026-08-25T03:45:57.475766+00:00", "aa_intelligence_index": 36.0, "aa_slug": "deepseek-v4-pro", "points": [{"request_rate": 1.31, "ttft_p50_ms": 235.53880899999058, "ttlt_p50_ms": 621.8832299999804, "itl_p50_ms": 3.518943666601141, "output_tokens_per_s": 130.54873246537667, "total_tokens_per_min": 242820.6423856006}, {"request_rate": 1.96, "ttft_p50_ms": 426.88091599995914, "ttlt_p50_ms": 1591.315882999993, "itl_p50_ms": 5.101484166705935, "output_tokens_per_s": 206.7250633154266, "total_tokens_per_min": 384508.61776669347}, {"request_rate": 2.94, "ttft_p50_ms": 427.6912679999896, "ttlt_p50_ms": 1635.6507959999362, "itl_p50_ms": 5.965452249995451, "output_tokens_per_s": 304.877059353346, "total_tokens_per_min": 567071.3303972236}, {"request_rate": 4.41, "ttft_p50_ms": 420.05247100019005, "ttlt_p50_ms": 1767.9621289998977, "itl_p50_ms": 6.839150999894628, "output_tokens_per_s": 444.4537614916051, "total_tokens_per_min": 826683.9963743854}, {"request_rate": 6.62, "ttft_p50_ms": 443.0508144998839, "ttlt_p50_ms": 5379.321757499838, "itl_p50_ms": 11.880574666671842, "output_tokens_per_s": 623.8158634681811, "total_tokens_per_min": 1160297.5060508167}, {"request_rate": 9.93, "ttft_p50_ms": 5564.665834999914, "ttlt_p50_ms": 8295.707700000321, "itl_p50_ms": 11.114951875015322, "output_tokens_per_s": 575.8959727545204, "total_tokens_per_min": 1071166.5093234077}]}, {"recipe_id": "rec-f4c9de85ddf2", "recipe_name": "deepseek-v4-pro-0813-sglang-b200x8", "model": "deepseek-ai/DeepSeek-V4-Pro-0813", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-ed78afa4b5", "benchmarked_at": "2026-08-25T09:31:47.769072+00:00", "aa_intelligence_index": 36.0, "aa_slug": "deepseek-v4-pro", "points": [{"request_rate": 1.71, "ttft_p50_ms": 224.13896650004972, "ttlt_p50_ms": 432.8346429999783, "itl_p50_ms": 1.9967851667009504, "output_tokens_per_s": 170.8591728674991, "total_tokens_per_min": 317798.06153354835}, {"request_rate": 2.56, "ttft_p50_ms": 3110.633464999978, "ttlt_p50_ms": 14826.291887000025, "itl_p50_ms": 7.831265333341738, "output_tokens_per_s": 235.04315819575427, "total_tokens_per_min": 437180.274244103}, {"request_rate": 3.84, "ttft_p50_ms": 449.946395999973, "ttlt_p50_ms": 1387.2049054999707, "itl_p50_ms": 3.6273207499561977, "output_tokens_per_s": 322.347587319492, "total_tokens_per_min": 599566.5124142551}, {"request_rate": 5.76, "ttft_p50_ms": 1281.6523599997254, "ttlt_p50_ms": 3162.0180779998464, "itl_p50_ms": 5.911018000006152, "output_tokens_per_s": 580.2824334448572, "total_tokens_per_min": 1079325.3262074345}, {"request_rate": 8.64, "ttft_p50_ms": 569.0120890001253, "ttlt_p50_ms": 11941.963686499776, "itl_p50_ms": 6.406529499978812, "output_tokens_per_s": 394.08707809616794, "total_tokens_per_min": 733001.9652588725}]}, {"recipe_id": "rec-c56a58d275c9", "recipe_name": "deepseek-v4-1-flash-sglang-b200x4", "model": "deepseek-ai/DeepSeek-V4.1-Flash", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-9413d410e9", "benchmarked_at": "2026-09-12T21:31:34.154288+00:00", "aa_intelligence_index": 39.5, "aa_slug": "deepseek-v4-1-flash", "points": [{"request_rate": 2.38, "ttft_p50_ms": 205.1200585000288, "ttlt_p50_ms": 400.19956300000104, "itl_p50_ms": 1.7001864999883765, "output_tokens_per_s": 237.50590326396292, "total_tokens_per_min": 441760.980070971}, {"request_rate": 3.56, "ttft_p50_ms": 450.6165845000396, "ttlt_p50_ms": 1565.0446575000387, "itl_p50_ms": 4.01171499999009, "output_tokens_per_s": 372.8599034040887, "total_tokens_per_min": 693519.420331605}, {"request_rate": 5.34, "ttft_p50_ms": 393.2429095000316, "ttlt_p50_ms": 1472.2911089999116, "itl_p50_ms": 4.283282500011865, "output_tokens_per_s": 547.2730273170333, "total_tokens_per_min": 1017927.8308096821}, {"request_rate": 8.01, "ttft_p50_ms": 461.6745619999847, "ttlt_p50_ms": 5049.050876000024, "itl_p50_ms": 9.5809679166526, "output_tokens_per_s": 740.5053185013822, "total_tokens_per_min": 1377339.892412571}, {"request_rate": 12.02, "ttft_p50_ms": 441.83542049995594, "ttlt_p50_ms": 4067.8554584999347, "itl_p50_ms": 9.999739333352409, "output_tokens_per_s": 943.7855233359329, "total_tokens_per_min": 1755441.073404835}, {"request_rate": 18.03, "ttft_p50_ms": 476.8023204999281, "ttlt_p50_ms": 3512.939986499987, "itl_p50_ms": 9.92598724999046, "output_tokens_per_s": 1232.8558841627341, "total_tokens_per_min": 2293111.944542685}, {"request_rate": 27.05, "ttft_p50_ms": 1250.0494264999134, "ttlt_p50_ms": 2937.520546000087, "itl_p50_ms": 9.957763749999534, "output_tokens_per_s": 1523.6761328569553, "total_tokens_per_min": 2834037.607113937}]}, {"recipe_id": "rec-09682113ae4f", "recipe_name": "gemma-3-1b-it-sglang-l40sx1", "model": "google/gemma-3-1b-it", "model_family": "gemma3", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "L40S", "gpu_count": 1, "benchmark_id": "bench-a7df543695", "benchmarked_at": "2026-08-31T03:31:28.137072+00:00", "aa_intelligence_index": 4.8, "aa_slug": "gemma-3-1b", "points": [{"request_rate": 2.12, "ttft_p50_ms": 47.42711099999042, "ttlt_p50_ms": 470.995778499983, "itl_p50_ms": 4.271884499999601, "output_tokens_per_s": 211.970766479922, "total_tokens_per_min": 394138.443192767}, {"request_rate": 3.18, "ttft_p50_ms": 52.84818900000232, "ttlt_p50_ms": 570.251996500005, "itl_p50_ms": 4.470998000002169, "output_tokens_per_s": 334.49219434542243, "total_tokens_per_min": 621954.7861658785}, {"request_rate": 4.77, "ttft_p50_ms": 57.09377750001465, "ttlt_p50_ms": 630.9390815000171, "itl_p50_ms": 4.5418539999957375, "output_tokens_per_s": 493.8356977546613, "total_tokens_per_min": 918238.0964050172}, {"request_rate": 7.15, "ttft_p50_ms": 64.36420749997751, "ttlt_p50_ms": 721.5271755000003, "itl_p50_ms": 4.684465999986287, "output_tokens_per_s": 723.9761258420071, "total_tokens_per_min": 1346161.2083906282}, {"request_rate": 10.73, "ttft_p50_ms": 71.8128809999996, "ttlt_p50_ms": 756.4776709999705, "itl_p50_ms": 4.8327380000046105, "output_tokens_per_s": 1048.9469038228058, "total_tokens_per_min": 1950411.8729681252}, {"request_rate": 16.09, "ttft_p50_ms": 80.25966850001964, "ttlt_p50_ms": 1144.4035485000086, "itl_p50_ms": 5.356506000026684, "output_tokens_per_s": 1496.7866589928774, "total_tokens_per_min": 2783125.1137313563}, {"request_rate": 24.13, "ttft_p50_ms": 89.38403549998952, "ttlt_p50_ms": 3824.1662835000056, "itl_p50_ms": 31.667539000011402, "output_tokens_per_s": 1187.1200946508354, "total_tokens_per_min": 2207331.103993763}]}, {"recipe_id": "rec-feb41616ea34", "recipe_name": "gemma-4-26b-a4b-it-sglang-b200x1", "model": "google/gemma-4-26B-A4B-it", "model_family": "gemma4", "engine": "SGLang", "speculative_draft_model": "z-lab/gemma-4-26B-A4B-it-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-a6d6920d6d", "benchmarked_at": "2026-08-25T09:37:26.946309+00:00", "aa_intelligence_index": 16.7, "aa_slug": "gemma-4-26b-a4b", "points": [{"request_rate": 3.82, "ttft_p50_ms": 68.83887349998474, "ttlt_p50_ms": 204.08260350001228, "itl_p50_ms": 1.1458297272730018, "output_tokens_per_s": 381.60709625184325, "total_tokens_per_min": 709789.1990284284}, {"request_rate": 5.72, "ttft_p50_ms": 67.66000100000724, "ttlt_p50_ms": 449.3427820000164, "itl_p50_ms": 1.9626648571439935, "output_tokens_per_s": 605.9481357746467, "total_tokens_per_min": 1127063.5325408431}, {"request_rate": 8.58, "ttft_p50_ms": 52.85593549999135, "ttlt_p50_ms": 257.6817795000039, "itl_p50_ms": 1.5197608000022456, "output_tokens_per_s": 898.7875193318554, "total_tokens_per_min": 1671744.7859572507}, {"request_rate": 12.87, "ttft_p50_ms": 78.9042534999993, "ttlt_p50_ms": 398.3834835000124, "itl_p50_ms": 2.051692500000968, "output_tokens_per_s": 1317.2649680652587, "total_tokens_per_min": 2450112.8406013814}, {"request_rate": 19.3, "ttft_p50_ms": 74.63337499999056, "ttlt_p50_ms": 492.08971400000223, "itl_p50_ms": 2.9310906458312287, "output_tokens_per_s": 1856.824444908117, "total_tokens_per_min": 3453693.4675290976}, {"request_rate": 28.95, "ttft_p50_ms": 96.82228100001566, "ttlt_p50_ms": 2765.8495990000065, "itl_p50_ms": 5.24377200000572, "output_tokens_per_s": 1284.1730103460657, "total_tokens_per_min": 2388561.7992436816}]}, {"recipe_id": "rec-aeeadd998e5e", "recipe_name": "gemma-4-31b-it-sglang-b200x1", "model": "google/gemma-4-31B-it", "model_family": "gemma4", "engine": "SGLang", "speculative_draft_model": "z-lab/gemma-4-31B-it-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-b8cfce3494", "benchmarked_at": "2026-09-18T01:17:03.525943+00:00", "aa_intelligence_index": 14.7, "aa_slug": "gemma-4-31b", "points": [{"request_rate": 2.02, "ttft_p50_ms": 256.26491299999543, "ttlt_p50_ms": 427.2032850000045, "itl_p50_ms": 1.5370105624992192, "output_tokens_per_s": 202.0245390432135, "total_tokens_per_min": 375765.6426203772}, {"request_rate": 3.03, "ttft_p50_ms": 2147.4663230000033, "ttlt_p50_ms": 7020.999615999983, "itl_p50_ms": 10.27636762499995, "output_tokens_per_s": 295.1267409257548, "total_tokens_per_min": 548935.7381219039}, {"request_rate": 4.54, "ttft_p50_ms": 2275.142126999981, "ttlt_p50_ms": 5500.7067425000005, "itl_p50_ms": 9.044010599999789, "output_tokens_per_s": 367.8296717392923, "total_tokens_per_min": 684163.1894350837}, {"request_rate": 6.81, "ttft_p50_ms": 3467.410316999974, "ttlt_p50_ms": 7765.933166500019, "itl_p50_ms": 8.504666687500162, "output_tokens_per_s": 380.5269977718687, "total_tokens_per_min": 707780.2158556759}]}, {"recipe_id": "rec-e423ff267417", "recipe_name": "gemma-4-e2b-it-sglang-a100x1", "model": "google/gemma-4-E2B-it", "model_family": "gemma4", "engine": "SGLang", "speculative_draft_model": "google/gemma-4-E2B-it-assistant", "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-8a5cf85f22", "benchmarked_at": "2026-09-15T22:44:45.319759+00:00", "aa_intelligence_index": 7.8, "aa_slug": "gemma-4-e2b", "points": [{"request_rate": 0.95, "ttft_p50_ms": 213.47398500000736, "ttlt_p50_ms": 1021.3468334999902, "itl_p50_ms": 7.76781950000327, "output_tokens_per_s": 94.66874588512884, "total_tokens_per_min": 176083.86734633966}, {"request_rate": 1.42, "ttft_p50_ms": 59.52189900003191, "ttlt_p50_ms": 977.3232430000007, "itl_p50_ms": 8.1217327499985, "output_tokens_per_s": 163.7107954629651, "total_tokens_per_min": 304502.07956111507}, {"request_rate": 2.13, "ttft_p50_ms": 66.10052900001051, "ttlt_p50_ms": 1058.84447050002, "itl_p50_ms": 8.532946249999895, "output_tokens_per_s": 242.3652314203204, "total_tokens_per_min": 450799.33044179593}, {"request_rate": 3.19, "ttft_p50_ms": 74.50230850000139, "ttlt_p50_ms": 1321.5658989999781, "itl_p50_ms": 10.55220512500199, "output_tokens_per_s": 355.5342284003241, "total_tokens_per_min": 661293.6648246027}, {"request_rate": 4.79, "ttft_p50_ms": 81.72756549998894, "ttlt_p50_ms": 1824.3624585000475, "itl_p50_ms": 12.745353500008605, "output_tokens_per_s": 505.0344449790666, "total_tokens_per_min": 939364.067661064}, {"request_rate": 7.19, "ttft_p50_ms": 69.63535499994578, "ttlt_p50_ms": 2222.288317499988, "itl_p50_ms": 18.265433999999914, "output_tokens_per_s": 694.8871481794976, "total_tokens_per_min": 1292490.0956138654}, {"request_rate": 10.79, "ttft_p50_ms": 124.76137300001255, "ttlt_p50_ms": 10011.80574, "itl_p50_ms": 39.6518990000061, "output_tokens_per_s": 453.36303782691584, "total_tokens_per_min": 843255.2503580634}]}, {"recipe_id": "rec-067bcca0a741", "recipe_name": "gemma-4-e4b-it-sglang-a100x1", "model": "google/gemma-4-E4B-it", "model_family": "gemma4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-f7b6e314f4", "benchmarked_at": "2026-09-01T21:04:06.789120+00:00", "aa_intelligence_index": 8.9, "aa_slug": "gemma-4-e4b", "points": [{"request_rate": 0.7, "ttft_p50_ms": 305.43077900000526, "ttlt_p50_ms": 1434.843475000008, "itl_p50_ms": 11.389997000009089, "output_tokens_per_s": 69.63643663841235, "total_tokens_per_min": 129523.77214744694}, {"request_rate": 1.04, "ttft_p50_ms": 325.56387099998574, "ttlt_p50_ms": 2010.7608300000095, "itl_p50_ms": 11.722662000011042, "output_tokens_per_s": 119.3158119851927, "total_tokens_per_min": 221927.41029245843}, {"request_rate": 1.56, "ttft_p50_ms": 550.0474849999932, "ttlt_p50_ms": 3224.6154929999875, "itl_p50_ms": 12.59092699999087, "output_tokens_per_s": 175.84245302883778, "total_tokens_per_min": 327066.9626336383}, {"request_rate": 2.34, "ttft_p50_ms": 1014.5416139999952, "ttlt_p50_ms": 5051.611535000006, "itl_p50_ms": 13.738985500026502, "output_tokens_per_s": 244.6607473979873, "total_tokens_per_min": 455068.99016025645}, {"request_rate": 3.51, "ttft_p50_ms": 2557.3919244999956, "ttlt_p50_ms": 17366.15375750003, "itl_p50_ms": 45.40341400002035, "output_tokens_per_s": 259.298668036543, "total_tokens_per_min": 482295.52254797}]}, {"recipe_id": "rec-1317cf6c1911", "recipe_name": "k2-horizon-0-9b-sglang-h100x1", "model": "IFM/K2-Horizon-0.9B", "model_family": "k2_horizon", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-25f40f6f23", "benchmarked_at": "2026-09-16T00:27:57.975623+00:00", "aa_intelligence_index": 3.0, "aa_slug": "k2-horizon-0-9b", "points": [{"request_rate": 4.94, "ttft_p50_ms": 25.574325999997427, "ttlt_p50_ms": 201.8102135000035, "itl_p50_ms": 1.7617645000029825, "output_tokens_per_s": 493.5313601501728, "total_tokens_per_min": 917968.3298793215}, {"request_rate": 7.4, "ttft_p50_ms": 25.28294800001163, "ttlt_p50_ms": 244.0194579999968, "itl_p50_ms": 1.9428965000116705, "output_tokens_per_s": 776.1683486028012, "total_tokens_per_min": 1443673.1284012103}, {"request_rate": 11.1, "ttft_p50_ms": 27.865741999988813, "ttlt_p50_ms": 271.9527470000003, "itl_p50_ms": 2.037570500007746, "output_tokens_per_s": 1143.569794315239, "total_tokens_per_min": 2127039.817426345}, {"request_rate": 16.65, "ttft_p50_ms": 29.841937500009408, "ttlt_p50_ms": 312.20614199999375, "itl_p50_ms": 2.1684440000058203, "output_tokens_per_s": 1670.3353381354088, "total_tokens_per_min": 3106823.7289318605}, {"request_rate": 24.97, "ttft_p50_ms": 111.13009799998963, "ttlt_p50_ms": 376.70976999999084, "itl_p50_ms": 2.211731499997427, "output_tokens_per_s": 2367.44246571314, "total_tokens_per_min": 4403442.98622644}, {"request_rate": 37.45, "ttft_p50_ms": 350.3231054999958, "ttlt_p50_ms": 613.1365749999986, "itl_p50_ms": 2.209985000007464, "output_tokens_per_s": 2765.365120768731, "total_tokens_per_min": 5143579.12462984}, {"request_rate": 56.18, "ttft_p50_ms": 540.1750369999547, "ttlt_p50_ms": 793.9925994999726, "itl_p50_ms": 2.2271679999903427, "output_tokens_per_s": 2844.3744639834554, "total_tokens_per_min": 5290536.503009227}]}, {"recipe_id": "rec-67f3d4398b35", "recipe_name": "muse-glimmer-30b-sglang-h200x1", "model": "meta-models/Muse-Glimmer-30B", "model_family": "muse_glimmer", "engine": "SGLang", "speculative_draft_model": "meta-models/Muse-Glimmer-30B-assistant", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-f0dbf1c9eb", "benchmarked_at": "2026-09-08T21:04:35.246809+00:00", "aa_intelligence_index": 17.5, "aa_slug": "muse-glimmer", "points": [{"request_rate": 1.31, "ttft_p50_ms": 254.75271999999904, "ttlt_p50_ms": 663.9803155000124, "itl_p50_ms": 3.618523800003004, "output_tokens_per_s": 131.45480508280468, "total_tokens_per_min": 244427.06457096702}, {"request_rate": 1.97, "ttft_p50_ms": 61.818888999994215, "ttlt_p50_ms": 505.23352100000807, "itl_p50_ms": 3.7427810000053796, "output_tokens_per_s": 209.50687564235983, "total_tokens_per_min": 389557.08456940384}, {"request_rate": 2.96, "ttft_p50_ms": 60.35200899998472, "ttlt_p50_ms": 508.29621549999615, "itl_p50_ms": 3.89141059999929, "output_tokens_per_s": 311.6341980742265, "total_tokens_per_min": 579452.6278992168}, {"request_rate": 4.44, "ttft_p50_ms": 67.8344180000181, "ttlt_p50_ms": 571.8122119999691, "itl_p50_ms": 4.220657799999117, "output_tokens_per_s": 448.8336379302955, "total_tokens_per_min": 834561.2663675915}, {"request_rate": 6.66, "ttft_p50_ms": 67.63240200001519, "ttlt_p50_ms": 633.422915500006, "itl_p50_ms": 4.545912500003624, "output_tokens_per_s": 635.9736700918463, "total_tokens_per_min": 1182529.442168779}, {"request_rate": 9.99, "ttft_p50_ms": 69.31229099996017, "ttlt_p50_ms": 723.9311989999919, "itl_p50_ms": 4.905539750012622, "output_tokens_per_s": 888.4010339491682, "total_tokens_per_min": 1651892.8825250834}, {"request_rate": 14.98, "ttft_p50_ms": 69.82415299995637, "ttlt_p50_ms": 756.2821030000464, "itl_p50_ms": 5.449620500002084, "output_tokens_per_s": 1195.809062477943, "total_tokens_per_min": 2223487.3707714872}]}, {"recipe_id": "rec-572a09aadee2", "recipe_name": "glm-5-3-nvfp4-sglang-b300x4", "model": "modal-labs/GLM-5.3-NVFP4", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": "modal-labs/GLM-5.3-NVFP4-DFlash", "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-95f90525e9", "benchmarked_at": "2026-09-20T23:51:59.924832+00:00", "aa_intelligence_index": 44.8, "aa_slug": "glm-5-3", "points": [{"request_rate": 2.87, "ttft_p50_ms": 187.56180700000868, "ttlt_p50_ms": 335.09958600001255, "itl_p50_ms": 1.4066460000066172, "output_tokens_per_s": 287.2465409495622, "total_tokens_per_min": 534278.5661661855}, {"request_rate": 4.31, "ttft_p50_ms": 135.82316800000171, "ttlt_p50_ms": 683.627371, "itl_p50_ms": 2.6607158750096005, "output_tokens_per_s": 453.37842261465164, "total_tokens_per_min": 843283.866063252}, {"request_rate": 6.46, "ttft_p50_ms": 126.38481400000501, "ttlt_p50_ms": 495.9962529999302, "itl_p50_ms": 2.5863572499957854, "output_tokens_per_s": 668.0607168978349, "total_tokens_per_min": 1242592.933429973}, {"request_rate": 9.69, "ttft_p50_ms": 136.1034314999756, "ttlt_p50_ms": 988.624434500025, "itl_p50_ms": 3.335119124997732, "output_tokens_per_s": 962.2496676986854, "total_tokens_per_min": 1789784.381919555}, {"request_rate": 14.54, "ttft_p50_ms": 162.21592650003913, "ttlt_p50_ms": 1344.6132969999667, "itl_p50_ms": 4.557640000001584, "output_tokens_per_s": 1353.4298792880502, "total_tokens_per_min": 2517379.5754757733}, {"request_rate": 21.81, "ttft_p50_ms": 173.44624599996905, "ttlt_p50_ms": 1161.4860820000672, "itl_p50_ms": 4.776735000007193, "output_tokens_per_s": 1749.4774266506163, "total_tokens_per_min": 3254028.013570146}, {"request_rate": 32.71, "ttft_p50_ms": 288.16984150000735, "ttlt_p50_ms": 1164.1095955000083, "itl_p50_ms": 3.9704568750096314, "output_tokens_per_s": 2263.9791847999145, "total_tokens_per_min": 4211001.2837278405}]}, {"recipe_id": "rec-bc81f4947654", "recipe_name": "qwen38-a95b-nvfp4-dflash-sglang-b300x8", "model": "modal-labs/Qwen3.8-2.4T-A95B-NVFP4", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.8-2.4T-A95B-DFlash", "gpu_type": "B300", "gpu_count": 8, "benchmark_id": "bench-d40c15f501", "benchmarked_at": "2026-08-18T13:04:14.892824+00:00", "aa_intelligence_index": 39.9, "aa_slug": "qwen3-8-2-4t-a95b", "points": [{"request_rate": 1.25, "ttft_p50_ms": 229.88669050005228, "ttlt_p50_ms": 707.9957125001783, "itl_p50_ms": 5.011551440675556, "output_tokens_per_s": 125.06711509329703, "total_tokens_per_min": 232624.8340735325}, {"request_rate": 1.88, "ttft_p50_ms": 236.65538199998082, "ttlt_p50_ms": 1052.0614875001684, "itl_p50_ms": 7.22460330303195, "output_tokens_per_s": 197.94359256403098, "total_tokens_per_min": 368175.0821690976}, {"request_rate": 2.82, "ttft_p50_ms": 277.46297950011467, "ttlt_p50_ms": 1109.4183970001268, "itl_p50_ms": 7.377410824994968, "output_tokens_per_s": 291.5707122619615, "total_tokens_per_min": 542321.5248072484}, {"request_rate": 4.23, "ttft_p50_ms": 310.01217649986756, "ttlt_p50_ms": 2042.0335769997564, "itl_p50_ms": 12.949462949995905, "output_tokens_per_s": 431.4341427705039, "total_tokens_per_min": 802467.5055531373}, {"request_rate": 6.35, "ttft_p50_ms": 307.9827454998849, "ttlt_p50_ms": 1463.3605520000401, "itl_p50_ms": 10.230144434343755, "output_tokens_per_s": 626.4803422753914, "total_tokens_per_min": 1165253.4366322279}, {"request_rate": 9.52, "ttft_p50_ms": 410.6298085000617, "ttlt_p50_ms": 8154.866373499999, "itl_p50_ms": 64.34435742424496, "output_tokens_per_s": 495.8606591160083, "total_tokens_per_min": 922300.8259557754}]}, {"recipe_id": "rec-3d66cbf014bf", "recipe_name": "kimi-k3-sglang-b300x8", "model": "moonshotai/Kimi-K3", "model_family": "kimi_k3", "engine": "SGLang", "speculative_draft_model": "modal-labs/Kimi-K3-DFlash", "gpu_type": "B300", "gpu_count": 8, "benchmark_id": "bench-c7f9def524", "benchmarked_at": "2026-08-18T20:06:44.673685+00:00", "aa_intelligence_index": 43.6, "aa_slug": "kimi-k3", "points": [{"request_rate": 2.7, "ttft_p50_ms": 187.40621100005228, "ttlt_p50_ms": 353.5771095000655, "itl_p50_ms": 1.5968560000061416, "output_tokens_per_s": 269.8556711168333, "total_tokens_per_min": 501931.54827731}, {"request_rate": 4.05, "ttft_p50_ms": 361.91698800007543, "ttlt_p50_ms": 1241.0528519999389, "itl_p50_ms": 3.4717325000030996, "output_tokens_per_s": 417.5165428187709, "total_tokens_per_min": 776580.7696429138}, {"request_rate": 6.07, "ttft_p50_ms": 718.7250869999389, "ttlt_p50_ms": 2570.7566730000053, "itl_p50_ms": 5.306713999999602, "output_tokens_per_s": 595.8352758537073, "total_tokens_per_min": 1108253.6130878956}, {"request_rate": 9.11, "ttft_p50_ms": 1600.4628849999563, "ttlt_p50_ms": 3407.012905999977, "itl_p50_ms": 5.8073619999845505, "output_tokens_per_s": 711.9901343683212, "total_tokens_per_min": 1324301.6499250773}, {"request_rate": 13.66, "ttft_p50_ms": 1834.1301664999037, "ttlt_p50_ms": 3335.2407415001153, "itl_p50_ms": 5.837886999984221, "output_tokens_per_s": 807.3637035617089, "total_tokens_per_min": 1501696.4886247788}, {"request_rate": 20.49, "ttft_p50_ms": 1556.9535785000426, "ttlt_p50_ms": 3569.9029544998666, "itl_p50_ms": 5.820610749992738, "output_tokens_per_s": 913.5119687347036, "total_tokens_per_min": 1699132.2618465486}, {"request_rate": 30.73, "ttft_p50_ms": 1591.7558729998973, "ttlt_p50_ms": 3442.6056235000715, "itl_p50_ms": 5.864391437484073, "output_tokens_per_s": 1043.9162022686037, "total_tokens_per_min": 1941684.1362196028}]}, {"recipe_id": "rec-c16e58aaaa92", "recipe_name": "glm-5-2-nvfp4-sglang-b200x4-brace-patch", "model": "nvidia/GLM-5.2-NVFP4", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-b023195c4e", "benchmarked_at": "2026-08-31T19:36:22.680330+00:00", "aa_intelligence_index": 33.7, "aa_slug": "glm-5-2", "points": [{"request_rate": 2.88, "ttft_p50_ms": 160.27853899998945, "ttlt_p50_ms": 340.8230570000228, "itl_p50_ms": 1.7728318333449047, "output_tokens_per_s": 287.67294038370966, "total_tokens_per_min": 535071.6691136999}, {"request_rate": 4.32, "ttft_p50_ms": 548.0654219999792, "ttlt_p50_ms": 5996.842378500049, "itl_p50_ms": 55.61428366667087, "output_tokens_per_s": 385.69176219867654, "total_tokens_per_min": 717386.6776895383}, {"request_rate": 6.48, "ttft_p50_ms": 458.18068950001134, "ttlt_p50_ms": 5695.392570999957, "itl_p50_ms": 30.451221166667136, "output_tokens_per_s": 530.1910855262802, "total_tokens_per_min": 986155.4190788812}, {"request_rate": 9.72, "ttft_p50_ms": 472.85672549998026, "ttlt_p50_ms": 5012.756386499973, "itl_p50_ms": 28.70573883332857, "output_tokens_per_s": 643.9731603610497, "total_tokens_per_min": 1197790.0782715525}, {"request_rate": 14.58, "ttft_p50_ms": 470.5429025000285, "ttlt_p50_ms": 4788.991874500027, "itl_p50_ms": 28.702298499998353, "output_tokens_per_s": 758.6978831505501, "total_tokens_per_min": 1411178.062660023}, {"request_rate": 21.87, "ttft_p50_ms": 456.4712069999928, "ttlt_p50_ms": 4272.958537000022, "itl_p50_ms": 28.311283333332916, "output_tokens_per_s": 941.9601588610548, "total_tokens_per_min": 1752045.8954815618}, {"request_rate": 32.8, "ttft_p50_ms": 541.9540180000126, "ttlt_p50_ms": 4077.3278355, "itl_p50_ms": 28.23428849999724, "output_tokens_per_s": 970.4640178398638, "total_tokens_per_min": 1805063.0731821465}]}, {"recipe_id": "rec-e062c16beb19", "recipe_name": "kimi-k2-6-nvfp4-sglang-b200x4", "model": "nvidia/Kimi-K2.6-NVFP4", "model_family": "kimi_k25", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-734742f0ce", "benchmarked_at": "2026-08-31T23:19:39.581485+00:00", "aa_intelligence_index": 27.0, "aa_slug": "kimi-k2-6", "points": [{"request_rate": 1.57, "ttft_p50_ms": 118.88835449997259, "ttlt_p50_ms": 635.167261499987, "itl_p50_ms": 5.1701869999760675, "output_tokens_per_s": 157.28295624521093, "total_tokens_per_min": 292546.2986160923}, {"request_rate": 2.36, "ttft_p50_ms": 85.7878759999835, "ttlt_p50_ms": 951.8615039999645, "itl_p50_ms": 6.707283500020367, "output_tokens_per_s": 247.48409841669513, "total_tokens_per_min": 460320.42305505293}, {"request_rate": 3.54, "ttft_p50_ms": 90.85388800002647, "ttlt_p50_ms": 1210.7611580000253, "itl_p50_ms": 8.37051949997658, "output_tokens_per_s": 364.8738476159974, "total_tokens_per_min": 678665.3565657553}, {"request_rate": 5.31, "ttft_p50_ms": 99.3049424999981, "ttlt_p50_ms": 1424.035594500026, "itl_p50_ms": 9.423217500000192, "output_tokens_per_s": 528.6802619030277, "total_tokens_per_min": 983345.2871396316}, {"request_rate": 7.96, "ttft_p50_ms": 127.80698850002636, "ttlt_p50_ms": 4656.557129500015, "itl_p50_ms": 15.098860000023251, "output_tokens_per_s": 652.6672498296763, "total_tokens_per_min": 1213961.0846831978}, {"request_rate": 11.94, "ttft_p50_ms": 174.2011069999876, "ttlt_p50_ms": 9861.387855500026, "itl_p50_ms": 97.63527000001204, "output_tokens_per_s": 484.63949353939734, "total_tokens_per_min": 901429.457983279}]}, {"recipe_id": "rec-78b54f7ee105", "recipe_name": "nvidia-nemotron-3-super-120b-a12b-nvfp4-sglang-b200x1", "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "model_family": "nemotron_h", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-51289a9f8c", "benchmarked_at": "2026-08-25T05:23:51.538417+00:00", "aa_intelligence_index": 12.8, "aa_slug": "nvidia-nemotron-3-super-120b-a12b", "points": [{"request_rate": 1.62, "ttft_p50_ms": 131.62344500003087, "ttlt_p50_ms": 605.4506705000051, "itl_p50_ms": 4.639551000025222, "output_tokens_per_s": 161.61619495154244, "total_tokens_per_min": 300606.12260986894}, {"request_rate": 2.42, "ttft_p50_ms": 58.708317499991836, "ttlt_p50_ms": 824.4082484999922, "itl_p50_ms": 6.178564999970604, "output_tokens_per_s": 255.53291386031373, "total_tokens_per_min": 475291.2197801835}, {"request_rate": 3.63, "ttft_p50_ms": 60.90786449996699, "ttlt_p50_ms": 973.3436694999966, "itl_p50_ms": 7.380756000031852, "output_tokens_per_s": 375.86135816369296, "total_tokens_per_min": 699102.1261844689}, {"request_rate": 5.45, "ttft_p50_ms": 63.286728500003164, "ttlt_p50_ms": 1170.3818025000317, "itl_p50_ms": 7.977143000005071, "output_tokens_per_s": 547.9492255408478, "total_tokens_per_min": 1019185.559505977}, {"request_rate": 8.18, "ttft_p50_ms": 85.08792549997679, "ttlt_p50_ms": 2128.3849915000133, "itl_p50_ms": 11.957369000015206, "output_tokens_per_s": 765.3548084475041, "total_tokens_per_min": 1423559.9437123577}, {"request_rate": 12.27, "ttft_p50_ms": 93.2216035000124, "ttlt_p50_ms": 2640.5437779999943, "itl_p50_ms": 14.923678999991807, "output_tokens_per_s": 998.9542553516702, "total_tokens_per_min": 1858054.9149541066}, {"request_rate": 18.41, "ttft_p50_ms": 106.03788099999178, "ttlt_p50_ms": 4439.61257899997, "itl_p50_ms": 16.004362999979094, "output_tokens_per_s": 880.3783175462091, "total_tokens_per_min": 1637503.670635949}]}, {"recipe_id": "rec-1753b0aaa744", "recipe_name": "nvidia-nemotron-3-ultra-550b-a55b-nvfp4-sglang-b200x4", "model": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "model_family": "nemotron_h", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-d269737f28", "benchmarked_at": "2026-09-18T01:11:55.282969+00:00", "aa_intelligence_index": 22.9, "aa_slug": "nvidia-nemotron-3-ultra-550b-a55b", "points": [{"request_rate": 3.28, "ttft_p50_ms": 125.70471300000463, "ttlt_p50_ms": 289.87498949999235, "itl_p50_ms": 1.5973092500018993, "output_tokens_per_s": 328.1484051112892, "total_tokens_per_min": 610356.0335069979}, {"request_rate": 4.92, "ttft_p50_ms": 310.90278649998027, "ttlt_p50_ms": 1173.475524999958, "itl_p50_ms": 3.4068601666679874, "output_tokens_per_s": 514.4594459820322, "total_tokens_per_min": 956894.5695265799}, {"request_rate": 7.38, "ttft_p50_ms": 238.64304349996246, "ttlt_p50_ms": 926.2291129999767, "itl_p50_ms": 3.3647361666642914, "output_tokens_per_s": 753.0386124024303, "total_tokens_per_min": 1400651.8190685203}, {"request_rate": 11.07, "ttft_p50_ms": 251.36705199997778, "ttlt_p50_ms": 1326.5747784999462, "itl_p50_ms": 3.831276666649804, "output_tokens_per_s": 1079.9810253611251, "total_tokens_per_min": 2008764.707171693}, {"request_rate": 16.61, "ttft_p50_ms": 362.24344700002575, "ttlt_p50_ms": 1686.9490960000348, "itl_p50_ms": 5.177609000005153, "output_tokens_per_s": 1525.68446541998, "total_tokens_per_min": 2837773.1056811623}, {"request_rate": 24.91, "ttft_p50_ms": 298.9611935000198, "ttlt_p50_ms": 1359.5968700000753, "itl_p50_ms": 5.165993500005091, "output_tokens_per_s": 2122.142060844371, "total_tokens_per_min": 3947184.23317053}, {"request_rate": 37.37, "ttft_p50_ms": 313.3421449999787, "ttlt_p50_ms": 1231.4414795000062, "itl_p50_ms": 5.1436072499958145, "output_tokens_per_s": 2623.6906649552916, "total_tokens_per_min": 4880064.6368168425}]}, {"recipe_id": "rec-492335383e6a", "recipe_name": "nvidia-nemotron-3-5-lightning-30b-a3b-nvfp4-sglang-b200x1", "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "model_family": "nemotron35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-082b1797af", "benchmarked_at": "2026-09-18T01:06:50.379004+00:00", "aa_intelligence_index": 12.9, "aa_slug": "nemotron-3-5-lightning", "points": [{"request_rate": 3.1, "ttft_p50_ms": 83.55663400001845, "ttlt_p50_ms": 312.0317430000057, "itl_p50_ms": 2.26466200001596, "output_tokens_per_s": 310.0905643670614, "total_tokens_per_min": 576768.4497227343}, {"request_rate": 4.65, "ttft_p50_ms": 47.639445999976715, "ttlt_p50_ms": 449.38410800000383, "itl_p50_ms": 3.2025719999921876, "output_tokens_per_s": 490.4396907118856, "total_tokens_per_min": 912217.8247241073}, {"request_rate": 6.98, "ttft_p50_ms": 52.93059749999429, "ttlt_p50_ms": 518.712014499954, "itl_p50_ms": 3.3472045000166872, "output_tokens_per_s": 721.086356815136, "total_tokens_per_min": 1341220.623676153}, {"request_rate": 10.47, "ttft_p50_ms": 76.25041250000208, "ttlt_p50_ms": 638.5349294999969, "itl_p50_ms": 4.09845900000505, "output_tokens_per_s": 1047.6264711126914, "total_tokens_per_min": 1948585.2362696063}, {"request_rate": 15.71, "ttft_p50_ms": 79.49681200000214, "ttlt_p50_ms": 1073.072539500032, "itl_p50_ms": 5.467468500029327, "output_tokens_per_s": 1486.8256926455224, "total_tokens_per_min": 2765495.7883206718}, {"request_rate": 23.57, "ttft_p50_ms": 80.78993499998433, "ttlt_p50_ms": 1651.6366159999905, "itl_p50_ms": 7.684422000011182, "output_tokens_per_s": 1960.899741872526, "total_tokens_per_min": 3647273.5198828983}, {"request_rate": 35.36, "ttft_p50_ms": 83.25421850000225, "ttlt_p50_ms": 1701.6699630000005, "itl_p50_ms": 8.79647399997907, "output_tokens_per_s": 2347.207041141095, "total_tokens_per_min": 4365805.096522437}]}, {"recipe_id": "rec-323026e40cb1", "recipe_name": "gpt-oss-120b-sglang-b200x1", "model": "openai/gpt-oss-120b", "model_family": "gptoss", "engine": "SGLang", "speculative_draft_model": "z-lab/gpt-oss-120b-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-bc5024e829", "benchmarked_at": "2026-09-18T01:04:56.180893+00:00", "aa_intelligence_index": 11.6, "aa_slug": "gpt-oss-120b", "points": [{"request_rate": 3.71, "ttft_p50_ms": 42.03738950002389, "ttlt_p50_ms": 262.42658750001624, "itl_p50_ms": 1.7695223333286474, "output_tokens_per_s": 370.5776099368884, "total_tokens_per_min": 689274.3544826123}, {"request_rate": 5.56, "ttft_p50_ms": 28.537132499991458, "ttlt_p50_ms": 378.64194549999297, "itl_p50_ms": 2.503344000009141, "output_tokens_per_s": 588.2975805024815, "total_tokens_per_min": 1094233.4997346154}, {"request_rate": 8.34, "ttft_p50_ms": 30.06463949998306, "ttlt_p50_ms": 446.0642629999825, "itl_p50_ms": 2.847422333331906, "output_tokens_per_s": 865.3004765355494, "total_tokens_per_min": 1609458.8863561219}, {"request_rate": 12.51, "ttft_p50_ms": 34.329174500015824, "ttlt_p50_ms": 518.9735085000109, "itl_p50_ms": 3.4172266666606292, "output_tokens_per_s": 1260.422687782168, "total_tokens_per_min": 2344386.1992748324}, {"request_rate": 18.77, "ttft_p50_ms": 40.76353499999641, "ttlt_p50_ms": 777.9012960000102, "itl_p50_ms": 5.003109499995162, "output_tokens_per_s": 1754.3333094251545, "total_tokens_per_min": 3263059.955530788}, {"request_rate": 28.16, "ttft_p50_ms": 43.82042849999834, "ttlt_p50_ms": 1200.5602254999985, "itl_p50_ms": 6.262120249999725, "output_tokens_per_s": 2249.141263105751, "total_tokens_per_min": 4183402.7493766965}, {"request_rate": 42.24, "ttft_p50_ms": 45.18705850000515, "ttlt_p50_ms": 1119.922439000021, "itl_p50_ms": 6.843993666658814, "output_tokens_per_s": 2746.681146116747, "total_tokens_per_min": 5108826.93177715}]}, {"recipe_id": "rec-8a6954502613", "recipe_name": "phonellm-alpha-1-vllm-b200x1", "model": "pipecat-ai/phonellm-alpha-1", "model_family": "Nemotron-H", "engine": "vLLM", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-b7e581ba09", "benchmarked_at": "2026-08-31T19:37:10.132034+00:00", "aa_intelligence_index": null, "aa_slug": null, "points": [{"request_rate": 2.61, "ttft_p50_ms": 122.67436549984723, "ttlt_p50_ms": 378.4358110000312, "itl_p50_ms": 2.561295500072447, "output_tokens_per_s": 261.01845698635276, "total_tokens_per_min": 485494.3299946161}, {"request_rate": 3.92, "ttft_p50_ms": 95.007601999896, "ttlt_p50_ms": 536.8214900001931, "itl_p50_ms": 3.4888860000137356, "output_tokens_per_s": 450.3920075543859, "total_tokens_per_min": 837729.1340511579}, {"request_rate": 5.88, "ttft_p50_ms": 130.86820650005393, "ttlt_p50_ms": 785.4722995000429, "itl_p50_ms": 4.417179000029137, "output_tokens_per_s": 653.2659527065975, "total_tokens_per_min": 1215074.6720342713}, {"request_rate": 8.82, "ttft_p50_ms": 133.96660650005288, "ttlt_p50_ms": 1084.1610310000078, "itl_p50_ms": 5.834656999923027, "output_tokens_per_s": 931.714331559077, "total_tokens_per_min": 1732988.6566998833}, {"request_rate": 13.23, "ttft_p50_ms": 153.24315600003047, "ttlt_p50_ms": 1724.5867630000475, "itl_p50_ms": 7.2907809999378514, "output_tokens_per_s": 1292.4292554446301, "total_tokens_per_min": 2403918.4151270115}, {"request_rate": 19.84, "ttft_p50_ms": 167.79971600010413, "ttlt_p50_ms": 2052.5195449999956, "itl_p50_ms": 8.861389999992753, "output_tokens_per_s": 1740.0691396678903, "total_tokens_per_min": 3236528.5997822755}, {"request_rate": 29.76, "ttft_p50_ms": 183.71164849997967, "ttlt_p50_ms": 1829.808082999989, "itl_p50_ms": 9.667710000030638, "output_tokens_per_s": 2232.299730075525, "total_tokens_per_min": 4152077.497940476}]}, {"recipe_id": "rec-d08310e9a9bb", "recipe_name": "qwen3-5-0-8b-sglang-a100x1", "model": "Qwen/Qwen3.5-0.8B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-43cfa47177", "benchmarked_at": "2026-09-10T04:54:13.004804+00:00", "aa_intelligence_index": 6.1, "aa_slug": "qwen3-5-0-8b", "points": [{"request_rate": 3.91, "ttft_p50_ms": 56.98890200000051, "ttlt_p50_ms": 246.28605350001465, "itl_p50_ms": 1.876096625000656, "output_tokens_per_s": 390.753307988984, "total_tokens_per_min": 726801.1528595103}, {"request_rate": 5.86, "ttft_p50_ms": 54.06575749999831, "ttlt_p50_ms": 321.9929480000161, "itl_p50_ms": 1.9537926249988402, "output_tokens_per_s": 614.470661176577, "total_tokens_per_min": 1142915.4297884332}, {"request_rate": 8.79, "ttft_p50_ms": 55.02212099997905, "ttlt_p50_ms": 321.65243199997917, "itl_p50_ms": 1.9409662500038394, "output_tokens_per_s": 908.9293893689044, "total_tokens_per_min": 1690608.664226162}, {"request_rate": 13.18, "ttft_p50_ms": 69.21442900002717, "ttlt_p50_ms": 403.55889300002445, "itl_p50_ms": 2.1001282500066054, "output_tokens_per_s": 1331.2381821475367, "total_tokens_per_min": 2476103.0187944183}, {"request_rate": 19.77, "ttft_p50_ms": 67.19410150000726, "ttlt_p50_ms": 417.1959070000071, "itl_p50_ms": 2.336492749989816, "output_tokens_per_s": 1930.4763484293815, "total_tokens_per_min": 3590686.0080786496}, {"request_rate": 29.66, "ttft_p50_ms": 78.40799650000463, "ttlt_p50_ms": 2806.10875749997, "itl_p50_ms": 4.0150037500126246, "output_tokens_per_s": 1496.406274997633, "total_tokens_per_min": 2783315.671495598}]}, {"recipe_id": "rec-73244be0ea37", "recipe_name": "qwen3-5-122b-a10b-fp8-sglang-b200x2", "model": "Qwen/Qwen3.5-122B-A10B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-122B-A10B-DFlash", "gpu_type": "B200", "gpu_count": 2, "benchmark_id": "bench-08002ec56c", "benchmarked_at": "2026-09-17T22:12:07.645483+00:00", "aa_intelligence_index": 15.6, "aa_slug": "qwen3-5-122b-a10b", "points": [{"request_rate": 4.36, "ttft_p50_ms": 105.32018250000874, "ttlt_p50_ms": 180.98315649999108, "itl_p50_ms": 0.6078518125001153, "output_tokens_per_s": 436.42678356991655, "total_tokens_per_min": 811753.8174400447}, {"request_rate": 6.55, "ttft_p50_ms": 224.52539500000057, "ttlt_p50_ms": 683.2242544999758, "itl_p50_ms": 1.4874056250029355, "output_tokens_per_s": 661.1230038680363, "total_tokens_per_min": 1229688.7871945475}, {"request_rate": 9.82, "ttft_p50_ms": 6707.861712499976, "ttlt_p50_ms": 8219.876746500006, "itl_p50_ms": 4.900478062499047, "output_tokens_per_s": 461.1800844258973, "total_tokens_per_min": 857794.957032169}]}, {"recipe_id": "rec-eb08cfab30d4", "recipe_name": "qwen3-5-27b-fp8-sglang-h100x1", "model": "Qwen/Qwen3.5-27B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-27B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-a29fc23960", "benchmarked_at": "2026-09-17T21:36:32.592166+00:00", "aa_intelligence_index": 22.9, "aa_slug": "qwen3-5-27b", "points": [{"request_rate": 1.42, "ttft_p50_ms": 299.5870274999959, "ttlt_p50_ms": 692.8959599999871, "itl_p50_ms": 1.7254984285719794, "output_tokens_per_s": 141.97495688628456, "total_tokens_per_min": 264073.4198084892}, {"request_rate": 2.13, "ttft_p50_ms": 1417.1808874999954, "ttlt_p50_ms": 2404.6020690000205, "itl_p50_ms": 2.443781750002927, "output_tokens_per_s": 215.98738295447296, "total_tokens_per_min": 401736.5322953197}, {"request_rate": 3.19, "ttft_p50_ms": 4795.3014424999765, "ttlt_p50_ms": 5954.443318499954, "itl_p50_ms": 2.3720181666675444, "output_tokens_per_s": 244.83803242074492, "total_tokens_per_min": 455398.74030258553}, {"request_rate": 4.79, "ttft_p50_ms": 7327.091854499997, "ttlt_p50_ms": 8453.578980499999, "itl_p50_ms": 2.357017636360194, "output_tokens_per_s": 248.06565346360364, "total_tokens_per_min": 461402.11544230283}]}, {"recipe_id": "rec-18de64e3b9a7", "recipe_name": "qwen3-5-2b-sglang-a100x1", "model": "Qwen/Qwen3.5-2B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-5947e98a78", "benchmarked_at": "2026-09-10T04:58:18.058887+00:00", "aa_intelligence_index": 6.9, "aa_slug": "qwen3-5-2b", "points": [{"request_rate": 3.13, "ttft_p50_ms": 80.30345100000602, "ttlt_p50_ms": 304.28119949999655, "itl_p50_ms": 2.191446000001207, "output_tokens_per_s": 312.7156912504756, "total_tokens_per_min": 581651.1857258846}, {"request_rate": 4.69, "ttft_p50_ms": 55.12478700001111, "ttlt_p50_ms": 353.3775474999885, "itl_p50_ms": 2.3874752499892793, "output_tokens_per_s": 492.80292163928556, "total_tokens_per_min": 916613.4342490712}, {"request_rate": 7.04, "ttft_p50_ms": 54.579100500006916, "ttlt_p50_ms": 380.93943550001086, "itl_p50_ms": 2.4838644999931603, "output_tokens_per_s": 730.4484927894707, "total_tokens_per_min": 1358634.1965884157}, {"request_rate": 10.56, "ttft_p50_ms": 72.7110214999982, "ttlt_p50_ms": 469.99632699998983, "itl_p50_ms": 2.7634504999980436, "output_tokens_per_s": 1068.9900557986791, "total_tokens_per_min": 1988321.503785543}, {"request_rate": 15.84, "ttft_p50_ms": 71.46804650000149, "ttlt_p50_ms": 519.9136779999947, "itl_p50_ms": 3.049449000002369, "output_tokens_per_s": 1549.7503818149753, "total_tokens_per_min": 2882535.7101758537}, {"request_rate": 23.76, "ttft_p50_ms": 75.35553349998736, "ttlt_p50_ms": 816.3598584999932, "itl_p50_ms": 4.135431625002184, "output_tokens_per_s": 2199.1203174430366, "total_tokens_per_min": 4090363.7904440477}, {"request_rate": 35.64, "ttft_p50_ms": 248.43908849999252, "ttlt_p50_ms": 4583.463887000022, "itl_p50_ms": 11.67406675000393, "output_tokens_per_s": 1077.3651052450755, "total_tokens_per_min": 2003899.0957558402}]}, {"recipe_id": "rec-e07d10272f16", "recipe_name": "qwen3-5-35b-a3b-fp8-sglang-h200x1", "model": "Qwen/Qwen3.5-35B-A3B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-35B-A3B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-903b206d52", "benchmarked_at": "2026-09-17T21:36:20.742086+00:00", "aa_intelligence_index": 19.3, "aa_slug": "qwen3-5-35b-a3b", "points": [{"request_rate": 4.36, "ttft_p50_ms": 70.69183950000024, "ttlt_p50_ms": 221.85714700000858, "itl_p50_ms": 0.7274189166679434, "output_tokens_per_s": 436.28995182962393, "total_tokens_per_min": 811499.3104031004}, {"request_rate": 6.54, "ttft_p50_ms": 124.51605700002233, "ttlt_p50_ms": 529.5799149999993, "itl_p50_ms": 1.5727045000005546, "output_tokens_per_s": 675.4663869938902, "total_tokens_per_min": 1256367.4798086355}, {"request_rate": 9.81, "ttft_p50_ms": 150.27563799998234, "ttlt_p50_ms": 654.6624549999933, "itl_p50_ms": 2.497202999999596, "output_tokens_per_s": 970.1139770116484, "total_tokens_per_min": 1804411.997241666}, {"request_rate": 14.71, "ttft_p50_ms": 304.7562699999844, "ttlt_p50_ms": 1153.346685999992, "itl_p50_ms": 3.591548599996486, "output_tokens_per_s": 1306.249688629112, "total_tokens_per_min": 2429624.420850148}, {"request_rate": 22.07, "ttft_p50_ms": 938.1617015000074, "ttlt_p50_ms": 2014.7424940000178, "itl_p50_ms": 4.185010749992557, "output_tokens_per_s": 1343.3318891563963, "total_tokens_per_min": 2498597.3138308977}]}, {"recipe_id": "rec-f85730b36ec2", "recipe_name": "qwen3-5-397b-a17b-fp8-sglang-b200x4", "model": "Qwen/Qwen3.5-397B-A17B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-397B-A17B-DFlash", "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-daa39e9623", "benchmarked_at": "2026-09-17T22:23:37.854389+00:00", "aa_intelligence_index": 18.4, "aa_slug": "qwen3-5-397b-a17b", "points": [{"request_rate": 3.79, "ttft_p50_ms": 109.78491449998273, "ttlt_p50_ms": 202.08486149999771, "itl_p50_ms": 0.7230304374985508, "output_tokens_per_s": 379.33260640510207, "total_tokens_per_min": 705558.6479134898}, {"request_rate": 5.69, "ttft_p50_ms": 620.6835015000252, "ttlt_p50_ms": 2597.0817949999514, "itl_p50_ms": 9.311526749996801, "output_tokens_per_s": 528.6441407279847, "total_tokens_per_min": 983278.1017540514}, {"request_rate": 8.54, "ttft_p50_ms": 1331.2338205000174, "ttlt_p50_ms": 3439.6203625000226, "itl_p50_ms": 9.301536599999357, "output_tokens_per_s": 627.7966337090729, "total_tokens_per_min": 1167701.7386988755}, {"request_rate": 12.81, "ttft_p50_ms": 1967.9426574999752, "ttlt_p50_ms": 5222.523435499966, "itl_p50_ms": 10.778387999998529, "output_tokens_per_s": 630.8761639393125, "total_tokens_per_min": 1173429.6649271213}]}, {"recipe_id": "rec-8339457cb66a", "recipe_name": "qwen3-5-4b-sglang-h100x1", "model": "Qwen/Qwen3.5-4B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.5-4B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-ec75cfeaae", "benchmarked_at": "2026-09-17T18:58:38.089472+00:00", "aa_intelligence_index": 13.1, "aa_slug": "qwen3-5-4b", "points": [{"request_rate": 6.24, "ttft_p50_ms": 64.98709650000478, "ttlt_p50_ms": 117.14366699999346, "itl_p50_ms": 0.4674763124992154, "output_tokens_per_s": 623.8602056375086, "total_tokens_per_min": 1160379.982485766}, {"request_rate": 9.36, "ttft_p50_ms": 227.4120285000123, "ttlt_p50_ms": 991.7526084999935, "itl_p50_ms": 1.7623235625006117, "output_tokens_per_s": 916.191139379524, "total_tokens_per_min": 1704115.5192459144}, {"request_rate": 14.04, "ttft_p50_ms": 263.8463200000132, "ttlt_p50_ms": 588.5911384999929, "itl_p50_ms": 1.3291426875010615, "output_tokens_per_s": 1377.4440607128747, "total_tokens_per_min": 2562045.952925947}, {"request_rate": 21.06, "ttft_p50_ms": 708.7640654999916, "ttlt_p50_ms": 1188.8940529999843, "itl_p50_ms": 1.8683939999988297, "output_tokens_per_s": 1561.7257279541204, "total_tokens_per_min": 2904809.8539946643}, {"request_rate": 31.59, "ttft_p50_ms": 944.1642544999809, "ttlt_p50_ms": 1592.326348499995, "itl_p50_ms": 1.4186932499988814, "output_tokens_per_s": 1603.1358680204808, "total_tokens_per_min": 2981832.7145180944}]}, {"recipe_id": "rec-9f1ca6f0b1bb", "recipe_name": "qwen3-5-9b-sglang-h100x1", "model": "Qwen/Qwen3.5-9B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-9B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-31cd99c27d", "benchmarked_at": "2026-09-17T21:30:02.818489+00:00", "aa_intelligence_index": 11.2, "aa_slug": "qwen3-5-9b", "points": [{"request_rate": 4.55, "ttft_p50_ms": 92.06680299999448, "ttlt_p50_ms": 167.82329849999655, "itl_p50_ms": 0.6836448125007166, "output_tokens_per_s": 454.67673773878937, "total_tokens_per_min": 845698.7321941482}, {"request_rate": 6.82, "ttft_p50_ms": 255.5875670000063, "ttlt_p50_ms": 731.9348019999978, "itl_p50_ms": 1.187599249998783, "output_tokens_per_s": 700.746176953097, "total_tokens_per_min": 1303387.8891327602}, {"request_rate": 10.23, "ttft_p50_ms": 572.9995899999949, "ttlt_p50_ms": 1195.260862000012, "itl_p50_ms": 1.479423250000167, "output_tokens_per_s": 953.58478647971, "total_tokens_per_min": 1773667.7028522606}, {"request_rate": 15.35, "ttft_p50_ms": 1301.1961864999932, "ttlt_p50_ms": 2167.9975270000114, "itl_p50_ms": 1.6925303636360862, "output_tokens_per_s": 994.3912624031775, "total_tokens_per_min": 1849567.7480699103}]}, {"recipe_id": "rec-ee25477ce1e0", "recipe_name": "qwen3-6-27b-sglang-h200x1", "model": "Qwen/Qwen3.6-27B", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.6-27B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-94dea9e3c4", "benchmarked_at": "2026-09-17T21:36:45.493239+00:00", "aa_intelligence_index": 21.4, "aa_slug": "qwen3-6-27b", "points": [{"request_rate": 1.31, "ttft_p50_ms": 276.67381200001273, "ttlt_p50_ms": 723.9968325000206, "itl_p50_ms": 3.3273133333343217, "output_tokens_per_s": 130.9109088846513, "total_tokens_per_min": 243494.29052545142}, {"request_rate": 1.96, "ttft_p50_ms": 899.2599989999803, "ttlt_p50_ms": 2234.854812999998, "itl_p50_ms": 4.266793800002233, "output_tokens_per_s": 204.64664977821548, "total_tokens_per_min": 380642.76858748076}, {"request_rate": 2.94, "ttft_p50_ms": 2234.373389000012, "ttlt_p50_ms": 4021.955818999999, "itl_p50_ms": 5.033792249989233, "output_tokens_per_s": 273.8136555771129, "total_tokens_per_min": 509293.39937343006}, {"request_rate": 4.41, "ttft_p50_ms": 5386.640843500032, "ttlt_p50_ms": 7210.464890999987, "itl_p50_ms": 5.534928166667896, "output_tokens_per_s": 279.9342642002146, "total_tokens_per_min": 520677.7314123992}]}, {"recipe_id": "rec-c4ac5ba2ffd8", "recipe_name": "qwen3-6-27b-fp8-sglang-h100x1", "model": "Qwen/Qwen3.6-27B-FP8", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.6-27B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-426c7d9693", "benchmarked_at": "2026-09-17T21:37:36.497822+00:00", "aa_intelligence_index": 21.4, "aa_slug": "qwen3-6-27b", "points": [{"request_rate": 1.15, "ttft_p50_ms": 300.26896799999747, "ttlt_p50_ms": 831.6330060000041, "itl_p50_ms": 3.894853500000295, "output_tokens_per_s": 115.4994582513521, "total_tokens_per_min": 214828.99234751493}, {"request_rate": 1.73, "ttft_p50_ms": 925.3211224999802, "ttlt_p50_ms": 2017.9464039999857, "itl_p50_ms": 4.655395499999789, "output_tokens_per_s": 178.36392780115312, "total_tokens_per_min": 331756.9057101448}, {"request_rate": 2.59, "ttft_p50_ms": 4455.844512999988, "ttlt_p50_ms": 5727.901056000036, "itl_p50_ms": 4.648032666665358, "output_tokens_per_s": 217.7063793951564, "total_tokens_per_min": 404933.8656749909}, {"request_rate": 3.88, "ttft_p50_ms": 7474.846528499995, "ttlt_p50_ms": 8667.634062500014, "itl_p50_ms": 5.383398399999351, "output_tokens_per_s": 220.43764991607014, "total_tokens_per_min": 410014.0288438904}]}, {"recipe_id": "rec-08a3e7455f15", "recipe_name": "qwen3-6-35b-a3b-sglang-b200x1", "model": "Qwen/Qwen3.6-35B-A3B", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.6-35B-A3B-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-679ed3fa00", "benchmarked_at": "2026-09-08T02:05:45.378239+00:00", "aa_intelligence_index": 18.2, "aa_slug": "qwen3-6-35b-a3b", "points": [{"request_rate": 3.17, "ttft_p50_ms": 70.50280499998962, "ttlt_p50_ms": 308.2152160000078, "itl_p50_ms": 1.3654129910725439, "output_tokens_per_s": 317.4549526920718, "total_tokens_per_min": 590466.2120072534}, {"request_rate": 4.76, "ttft_p50_ms": 134.80271449998327, "ttlt_p50_ms": 905.4243324999902, "itl_p50_ms": 3.801235666666495, "output_tokens_per_s": 499.24061524098215, "total_tokens_per_min": 928587.5443482269}, {"request_rate": 7.14, "ttft_p50_ms": 86.47356099999115, "ttlt_p50_ms": 523.7618295000175, "itl_p50_ms": 2.3405026666675135, "output_tokens_per_s": 732.7466224868111, "total_tokens_per_min": 1362908.7178254686}, {"request_rate": 10.71, "ttft_p50_ms": 101.56278600004498, "ttlt_p50_ms": 735.8589460000644, "itl_p50_ms": 3.3246515999962867, "output_tokens_per_s": 1062.103744536977, "total_tokens_per_min": 1975512.9648387772}, {"request_rate": 16.07, "ttft_p50_ms": 173.54793350000364, "ttlt_p50_ms": 1494.9369279999587, "itl_p50_ms": 5.473334333335818, "output_tokens_per_s": 1354.945493327862, "total_tokens_per_min": 2520198.617589823}, {"request_rate": 24.11, "ttft_p50_ms": 625.2124295000385, "ttlt_p50_ms": 2082.110491000037, "itl_p50_ms": 6.1856688124990455, "output_tokens_per_s": 1423.1153618333572, "total_tokens_per_min": 2646994.573010044}]}, {"recipe_id": "rec-2919c029201f", "recipe_name": "qwen3-6-35b-a3b-fp8-sglang-h200x1", "model": "Qwen/Qwen3.6-35B-A3B-FP8", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.6-35B-A3B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-2314812d9d", "benchmarked_at": "2026-09-17T21:36:50.856388+00:00", "aa_intelligence_index": 18.2, "aa_slug": "qwen3-6-35b-a3b", "points": [{"request_rate": 3.65, "ttft_p50_ms": 68.2869665000112, "ttlt_p50_ms": 275.6840220000072, "itl_p50_ms": 1.1098764999957211, "output_tokens_per_s": 365.47049452764116, "total_tokens_per_min": 679775.1198214125}, {"request_rate": 5.48, "ttft_p50_ms": 117.85925850000467, "ttlt_p50_ms": 590.7621360000235, "itl_p50_ms": 2.0310979999891288, "output_tokens_per_s": 572.0097071931358, "total_tokens_per_min": 1063938.0553792324}, {"request_rate": 8.22, "ttft_p50_ms": 151.76252099996645, "ttlt_p50_ms": 770.582832499997, "itl_p50_ms": 2.7480020000098193, "output_tokens_per_s": 828.2838896601362, "total_tokens_per_min": 1540608.0347678533}, {"request_rate": 12.33, "ttft_p50_ms": 297.92008000001147, "ttlt_p50_ms": 1631.6338280000195, "itl_p50_ms": 4.499098250001055, "output_tokens_per_s": 1164.5768468893657, "total_tokens_per_min": 2166112.93521422}, {"request_rate": 18.5, "ttft_p50_ms": 862.1177615000022, "ttlt_p50_ms": 2109.7124064999944, "itl_p50_ms": 6.368575156251666, "output_tokens_per_s": 1230.0273553817167, "total_tokens_per_min": 2287850.881009993}]}, {"recipe_id": "rec-e50ea0dc7fcb", "recipe_name": "qwen3-8-27b-sglang-h200x1", "model": "Qwen/Qwen3.8-27B", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": "incoai/Qwen3.8-27B-DFlash2", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-1291c879d6", "benchmarked_at": "2026-09-18T14:04:36.325948+00:00", "aa_intelligence_index": 33.7, "aa_slug": "qwen3-8-27b", "points": [{"request_rate": 1.36, "ttft_p50_ms": 274.5610679999828, "ttlt_p50_ms": 756.2176605000133, "itl_p50_ms": 2.7626852857147406, "output_tokens_per_s": 136.08237055053868, "total_tokens_per_min": 253113.20922400197}, {"request_rate": 2.04, "ttft_p50_ms": 887.9230535000033, "ttlt_p50_ms": 2204.361601500011, "itl_p50_ms": 3.8769236666666984, "output_tokens_per_s": 211.8456752403098, "total_tokens_per_min": 394032.9559469762}, {"request_rate": 3.06, "ttft_p50_ms": 1792.2859285000072, "ttlt_p50_ms": 4026.761232999974, "itl_p50_ms": 4.581305100003874, "output_tokens_per_s": 303.11117594806495, "total_tokens_per_min": 563786.7872634009}, {"request_rate": 4.59, "ttft_p50_ms": 4158.083806999997, "ttlt_p50_ms": 7418.217092000021, "itl_p50_ms": 5.150171599996156, "output_tokens_per_s": 317.2340824622196, "total_tokens_per_min": 590055.3933797285}]}, {"recipe_id": "rec-d049efcf1bc0", "recipe_name": "qwen3-8-flash-next-fp8-sglang-b200x4", "model": "Qwen/Qwen3.8-Flash-Next-FP8", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-3201f4b041", "benchmarked_at": "2026-08-26T14:31:35.420827+00:00", "aa_intelligence_index": 39.8, "aa_slug": "qwen3-8-flash-next", "points": [{"request_rate": 2.75, "ttft_p50_ms": 148.08348650001335, "ttlt_p50_ms": 349.9313404999782, "itl_p50_ms": 1.6345725000093125, "output_tokens_per_s": 275.35906886902734, "total_tokens_per_min": 512167.86809639086}, {"request_rate": 4.13, "ttft_p50_ms": 341.54462399999375, "ttlt_p50_ms": 1747.4960044999932, "itl_p50_ms": 3.624435750012367, "output_tokens_per_s": 433.7791932002499, "total_tokens_per_min": 806829.2993524647}, {"request_rate": 6.2, "ttft_p50_ms": 279.3977485000312, "ttlt_p50_ms": 1102.0977164999977, "itl_p50_ms": 3.1729508333360927, "output_tokens_per_s": 638.165244481632, "total_tokens_per_min": 1186987.3547358357}, {"request_rate": 9.3, "ttft_p50_ms": 303.3303079999996, "ttlt_p50_ms": 1792.7679565000005, "itl_p50_ms": 3.774595750002163, "output_tokens_per_s": 897.5047448414908, "total_tokens_per_min": 1669358.825405173}, {"request_rate": 13.95, "ttft_p50_ms": 356.03103149998105, "ttlt_p50_ms": 2716.6537620000213, "itl_p50_ms": 5.098620875003235, "output_tokens_per_s": 1242.355672473504, "total_tokens_per_min": 2310781.5508007174}, {"request_rate": 20.92, "ttft_p50_ms": 419.7140659999832, "ttlt_p50_ms": 1829.6446755000488, "itl_p50_ms": 4.550229666657894, "output_tokens_per_s": 1629.738029466609, "total_tokens_per_min": 3031312.7348078927}, {"request_rate": 31.38, "ttft_p50_ms": 380.78336600000284, "ttlt_p50_ms": 1575.2626335000173, "itl_p50_ms": 4.993161999976792, "output_tokens_per_s": 2040.1276019612471, "total_tokens_per_min": 3794637.3396479194}]}, {"recipe_id": "rec-13a575625f13", "recipe_name": "muse-glimmer-nvfp4-sglang-b200x1", "model": "RadixArk/Muse-Glimmer-NVFP4", "model_family": "muse_glimmer", "engine": "SGLang", "speculative_draft_model": "meta-models/Muse-Glimmer-30B-assistant", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-0317428f10", "benchmarked_at": "2026-09-08T21:17:37.068122+00:00", "aa_intelligence_index": 17.5, "aa_slug": "muse-glimmer", "points": [{"request_rate": 2.68, "ttft_p50_ms": 85.04405549999206, "ttlt_p50_ms": 318.70205850000843, "itl_p50_ms": 2.1465295999973932, "output_tokens_per_s": 268.1550040673237, "total_tokens_per_min": 498607.41456278163}, {"request_rate": 4.02, "ttft_p50_ms": 74.91561099999444, "ttlt_p50_ms": 425.65202000000113, "itl_p50_ms": 2.3605990000078236, "output_tokens_per_s": 423.75256104113214, "total_tokens_per_min": 787925.5119998811}, {"request_rate": 6.03, "ttft_p50_ms": 74.08532500005549, "ttlt_p50_ms": 460.54046999995535, "itl_p50_ms": 2.573809749989664, "output_tokens_per_s": 614.2817364947365, "total_tokens_per_min": 1142195.460838313}, {"request_rate": 9.04, "ttft_p50_ms": 73.88161250003122, "ttlt_p50_ms": 517.0335070000078, "itl_p50_ms": 2.7411512500066237, "output_tokens_per_s": 879.5771938617378, "total_tokens_per_min": 1635485.8342665152}, {"request_rate": 13.56, "ttft_p50_ms": 83.28460699999596, "ttlt_p50_ms": 699.2975754999975, "itl_p50_ms": 3.1065432499985945, "output_tokens_per_s": 1230.310626095204, "total_tokens_per_min": 2287639.578161422}, {"request_rate": 20.34, "ttft_p50_ms": 81.93424050000431, "ttlt_p50_ms": 731.3914084999737, "itl_p50_ms": 3.7764507499957745, "output_tokens_per_s": 1637.6418358683477, "total_tokens_per_min": 3045031.2296136054}, {"request_rate": 30.51, "ttft_p50_ms": 83.17172099998515, "ttlt_p50_ms": 1119.4442969999727, "itl_p50_ms": 5.443988749988193, "output_tokens_per_s": 2184.911205184783, "total_tokens_per_min": 4062623.8949205847}]}, {"recipe_id": "rec-aa8953352486", "recipe_name": "inkling-nvfp4-sglang-b300x4-dflash-mrr32", "model": "thinkingmachines/Inkling-NVFP4", "model_family": "inkling", "engine": "SGLang", "speculative_draft_model": "modal-labs/Inkling-NVFP4-DFlash", "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-05032484f3", "benchmarked_at": "2026-07-21T09:21:57.422307+00:00", "aa_intelligence_index": 25.0, "aa_slug": "inkling", "points": [{"request_rate": 1.77, "ttft_p50_ms": 153.81600649971006, "ttlt_p50_ms": 502.90458599988597, "itl_p50_ms": 3.0368655714352957, "output_tokens_per_s": 177.21046695440538, "total_tokens_per_min": 329611.468535194}, {"request_rate": 2.66, "ttft_p50_ms": 1036.2604580000152, "ttlt_p50_ms": 2837.397841499751, "itl_p50_ms": 6.321983500015449, "output_tokens_per_s": 281.9594301701135, "total_tokens_per_min": 524444.540116411}, {"request_rate": 3.99, "ttft_p50_ms": 98.99511250000614, "ttlt_p50_ms": 609.1037869998672, "itl_p50_ms": 4.241149166697748, "output_tokens_per_s": 417.92535042948396, "total_tokens_per_min": 777341.1517988403}, {"request_rate": 5.99, "ttft_p50_ms": 117.13075199986633, "ttlt_p50_ms": 811.8845240001065, "itl_p50_ms": 5.333738833314783, "output_tokens_per_s": 608.988478433844, "total_tokens_per_min": 1132718.5698869498}, {"request_rate": 8.98, "ttft_p50_ms": 130.08321450001858, "ttlt_p50_ms": 1030.4330505000507, "itl_p50_ms": 6.886525250024533, "output_tokens_per_s": 823.4820340579893, "total_tokens_per_min": 1531676.58334786}, {"request_rate": 13.47, "ttft_p50_ms": 146.9277844998942, "ttlt_p50_ms": 1460.4811330000302, "itl_p50_ms": 9.056587333361676, "output_tokens_per_s": 1087.4882000393713, "total_tokens_per_min": 2022728.0520732307}, {"request_rate": 20.21, "ttft_p50_ms": 360.3664079998907, "ttlt_p50_ms": 1636.6489805000128, "itl_p50_ms": 8.200011857167867, "output_tokens_per_s": 1464.6262583866048, "total_tokens_per_min": 2724204.840599085}]}, {"recipe_id": "rec-ff26f055e1c8", "recipe_name": "glm-4-7-sglang-b200x8", "model": "zai-org/GLM-4.7", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-c0a20fe6fb", "benchmarked_at": "2026-09-18T01:55:52.781120+00:00", "aa_intelligence_index": 22.2, "aa_slug": "glm-4-7", "points": [{"request_rate": 2.43, "ttft_p50_ms": 101.8189010000583, "ttlt_p50_ms": 399.032977499985, "itl_p50_ms": 2.945283749994587, "output_tokens_per_s": 243.16742862557342, "total_tokens_per_min": 452291.4172435666}, {"request_rate": 3.65, "ttft_p50_ms": 98.52549700002555, "ttlt_p50_ms": 766.1459275000198, "itl_p50_ms": 4.944988000005424, "output_tokens_per_s": 382.90584788749146, "total_tokens_per_min": 712204.8770707342}, {"request_rate": 5.47, "ttft_p50_ms": 131.2837544999752, "ttlt_p50_ms": 931.4407540000502, "itl_p50_ms": 5.496580249996441, "output_tokens_per_s": 559.2620305832131, "total_tokens_per_min": 1040227.3768847764}, {"request_rate": 8.21, "ttft_p50_ms": 134.11646550002843, "ttlt_p50_ms": 1174.6659675000046, "itl_p50_ms": 6.76721499999644, "output_tokens_per_s": 809.5847729980733, "total_tokens_per_min": 1505827.6777764163}, {"request_rate": 12.32, "ttft_p50_ms": 157.11887150007442, "ttlt_p50_ms": 2540.2294980000306, "itl_p50_ms": 8.442118749997007, "output_tokens_per_s": 1065.7093795398782, "total_tokens_per_min": 1982219.4459441733}, {"request_rate": 18.48, "ttft_p50_ms": 184.50059500003135, "ttlt_p50_ms": 2994.097599500037, "itl_p50_ms": 15.473965250009769, "output_tokens_per_s": 1185.7380526795814, "total_tokens_per_min": 2205472.777984021}, {"request_rate": 27.72, "ttft_p50_ms": 169.2672985000172, "ttlt_p50_ms": 2229.686610500039, "itl_p50_ms": 15.016024250030569, "output_tokens_per_s": 1592.9077919043973, "total_tokens_per_min": 2962808.492942179}]}, {"recipe_id": "rec-085d873cbeb4", "recipe_name": "glm-5-2-fp8-sglang-b200x8", "model": "zai-org/GLM-5.2-FP8", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-6c21c5ddc5", "benchmarked_at": "2026-08-25T04:09:45.994165+00:00", "aa_intelligence_index": 33.7, "aa_slug": "glm-5-2", "points": [{"request_rate": 2.36, "ttft_p50_ms": 226.16381849979916, "ttlt_p50_ms": 413.41018900004656, "itl_p50_ms": 1.780104999982411, "output_tokens_per_s": 235.64547741661173, "total_tokens_per_min": 438300.5879948978}, {"request_rate": 3.53, "ttft_p50_ms": 1234.5815129999664, "ttlt_p50_ms": 8257.361077000041, "itl_p50_ms": 59.69851728572654, "output_tokens_per_s": 274.827086576022, "total_tokens_per_min": 511178.3810314009}, {"request_rate": 5.29, "ttft_p50_ms": 618.910966499925, "ttlt_p50_ms": 6584.0022559999625, "itl_p50_ms": 31.422718000028127, "output_tokens_per_s": 468.33982957802743, "total_tokens_per_min": 871112.0830151311}, {"request_rate": 7.94, "ttft_p50_ms": 573.3827160000828, "ttlt_p50_ms": 6504.257230000007, "itl_p50_ms": 32.457115857141616, "output_tokens_per_s": 447.2368767118327, "total_tokens_per_min": 831860.5906840089}]}, {"recipe_id": "rec-ec02d67366b4", "recipe_name": "glm-5-3-flash-sglang-b300x4", "model": "zai-org/GLM-5.3-Flash", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-9ea80edb0f", "benchmarked_at": "2026-09-18T20:20:13.882588+00:00", "aa_intelligence_index": 41.8, "aa_slug": "glm-5-3-flash", "points": [{"request_rate": 1.9, "ttft_p50_ms": 273.61767699994743, "ttlt_p50_ms": 512.1546109999144, "itl_p50_ms": 2.0993759166761565, "output_tokens_per_s": 190.2530591517093, "total_tokens_per_min": 353870.6900221793}, {"request_rate": 2.85, "ttft_p50_ms": 49588.01787899983, "ttlt_p50_ms": 54801.04700349989, "itl_p50_ms": 35.12795283328766, "output_tokens_per_s": 92.69408673836293, "total_tokens_per_min": 172411.00133335506}]}]}, {"pattern": "online_agentic", "label": "Online agentic", "cached_input_tokens": 45000, "uncached_input_tokens": 5000, "output_tokens": 200, "recipes": [{"recipe_id": "rec-384b32610abf", "recipe_name": "deepseek-v4-flash-sglang-b200x4", "model": "deepseek-ai/DeepSeek-V4-Flash", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-33dd7618fd", "benchmarked_at": "2026-08-25T02:36:40.667989+00:00", "aa_intelligence_index": 34.3, "aa_slug": "deepseek-v4-flash", "points": [{"request_rate": 0.93, "ttft_p50_ms": 460.0557974999333, "ttlt_p50_ms": 1038.354171499975, "itl_p50_ms": 2.411484333341226, "output_tokens_per_s": 186.72005070249486, "total_tokens_per_min": 2812059.9795947834}, {"request_rate": 1.4, "ttft_p50_ms": 173.2110519999992, "ttlt_p50_ms": 941.4970260000359, "itl_p50_ms": 2.6843455000289396, "output_tokens_per_s": 297.0309507093694, "total_tokens_per_min": 4473375.226968315}, {"request_rate": 2.1, "ttft_p50_ms": 185.54295550006827, "ttlt_p50_ms": 997.0970495000415, "itl_p50_ms": 2.8743266666803415, "output_tokens_per_s": 439.729563824158, "total_tokens_per_min": 6622459.150060967}, {"request_rate": 3.15, "ttft_p50_ms": 213.09911450003938, "ttlt_p50_ms": 1206.3364530000058, "itl_p50_ms": 3.0424363333546958, "output_tokens_per_s": 646.3439088348897, "total_tokens_per_min": 9734133.17022609}, {"request_rate": 4.72, "ttft_p50_ms": 275.45967200001087, "ttlt_p50_ms": 1507.0320389999665, "itl_p50_ms": 3.2470806666727485, "output_tokens_per_s": 945.0537356003584, "total_tokens_per_min": 14232792.774262076}, {"request_rate": 7.08, "ttft_p50_ms": 726.3186559999895, "ttlt_p50_ms": 2931.8221074998974, "itl_p50_ms": 3.538437333361344, "output_tokens_per_s": 1307.4052628833358, "total_tokens_per_min": 19689915.480601903}, {"request_rate": 10.62, "ttft_p50_ms": 2420.630226999947, "ttlt_p50_ms": 4680.557068999974, "itl_p50_ms": 4.68199266663305, "output_tokens_per_s": 1703.8048030042683, "total_tokens_per_min": 25659811.47468518}]}, {"recipe_id": "rec-4c0e6a8145be", "recipe_name": "deepseek-v4-flash-0731-sglang-b200x2", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 2, "benchmark_id": "bench-72d291e330", "benchmarked_at": "2026-08-25T02:34:23.911128+00:00", "aa_intelligence_index": 34.3, "aa_slug": "deepseek-v4-flash", "points": [{"request_rate": 1.48, "ttft_p50_ms": 265.8765995000181, "ttlt_p50_ms": 647.415975499996, "itl_p50_ms": 1.3354559999925186, "output_tokens_per_s": 296.03172770459616, "total_tokens_per_min": 4458326.62874953}, {"request_rate": 2.22, "ttft_p50_ms": 178.14277299999048, "ttlt_p50_ms": 774.6093310000219, "itl_p50_ms": 1.5224266666677977, "output_tokens_per_s": 469.4303211537173, "total_tokens_per_min": 7069761.465671328}, {"request_rate": 3.33, "ttft_p50_ms": 216.46239599999717, "ttlt_p50_ms": 950.9649964999767, "itl_p50_ms": 1.5761241666609749, "output_tokens_per_s": 696.0561858806725, "total_tokens_per_min": 10482814.976218693}, {"request_rate": 5, "ttft_p50_ms": 323.58157850001135, "ttlt_p50_ms": 1231.0380655000017, "itl_p50_ms": 1.8843196666769775, "output_tokens_per_s": 1016.443657506781, "total_tokens_per_min": 15307946.415149376}, {"request_rate": 7.5, "ttft_p50_ms": 631.7747479999412, "ttlt_p50_ms": 1790.857831500034, "itl_p50_ms": 1.8817281666656527, "output_tokens_per_s": 1402.4484895012035, "total_tokens_per_min": 21121294.986434974}, {"request_rate": 11.25, "ttft_p50_ms": 2381.043123999973, "ttlt_p50_ms": 4297.507921999966, "itl_p50_ms": 3.1310068333330796, "output_tokens_per_s": 1773.2605895143656, "total_tokens_per_min": 26705836.4562632}, {"request_rate": 16.88, "ttft_p50_ms": 3630.132293000031, "ttlt_p50_ms": 4576.416651499983, "itl_p50_ms": 0.8343157500121379, "output_tokens_per_s": 1905.692193361571, "total_tokens_per_min": 28700296.13968327}]}, {"recipe_id": "rec-9c8e58e6bf05", "recipe_name": "deepseek-v4-pro-sglang-b200x8", "model": "deepseek-ai/DeepSeek-V4-Pro", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-18ffb3e09b", "benchmarked_at": "2026-08-25T06:12:15.469585+00:00", "aa_intelligence_index": 36.0, "aa_slug": "deepseek-v4-pro", "points": [{"request_rate": 0.69, "ttft_p50_ms": 537.5854415000276, "ttlt_p50_ms": 1398.0643305001195, "itl_p50_ms": 3.3096024999963447, "output_tokens_per_s": 137.74485030956, "total_tokens_per_min": 2074478.7691170664}, {"request_rate": 1.03, "ttft_p50_ms": 344.35137599984955, "ttlt_p50_ms": 1697.9841039999428, "itl_p50_ms": 3.7424382500148567, "output_tokens_per_s": 218.1158060180735, "total_tokens_per_min": 3284889.4733739924}, {"request_rate": 1.54, "ttft_p50_ms": 495.59144650015696, "ttlt_p50_ms": 2089.220634999947, "itl_p50_ms": 4.095808999977635, "output_tokens_per_s": 320.57264862758086, "total_tokens_per_min": 4827920.260125956}, {"request_rate": 2.31, "ttft_p50_ms": 526.3682925001376, "ttlt_p50_ms": 3160.262934000002, "itl_p50_ms": 5.074623500036068, "output_tokens_per_s": 472.3857412088381, "total_tokens_per_min": 7114270.978327465}, {"request_rate": 3.46, "ttft_p50_ms": 582.0652560000781, "ttlt_p50_ms": 3787.31249249995, "itl_p50_ms": 5.648998333375251, "output_tokens_per_s": 679.8760136416645, "total_tokens_per_min": 10239136.728247557}, {"request_rate": 5.19, "ttft_p50_ms": 743.6097004999738, "ttlt_p50_ms": 7999.062749999894, "itl_p50_ms": 9.409541500019714, "output_tokens_per_s": 898.3049276280656, "total_tokens_per_min": 13528741.701556956}, {"request_rate": 7.79, "ttft_p50_ms": 868.1311244999961, "ttlt_p50_ms": 7263.144383999816, "itl_p50_ms": 9.143833499933862, "output_tokens_per_s": 1145.2804009591623, "total_tokens_per_min": 17248266.422565274}]}, {"recipe_id": "rec-f4c9de85ddf2", "recipe_name": "deepseek-v4-pro-0813-sglang-b200x8", "model": "deepseek-ai/DeepSeek-V4-Pro-0813", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-17ee26418a", "benchmarked_at": "2026-08-25T10:17:04.660714+00:00", "aa_intelligence_index": 36.0, "aa_slug": "deepseek-v4-pro", "points": [{"request_rate": 0.85, "ttft_p50_ms": 548.5783965000337, "ttlt_p50_ms": 1088.665020999997, "itl_p50_ms": 2.055375333346395, "output_tokens_per_s": 170.79398081991917, "total_tokens_per_min": 2572208.589342229}, {"request_rate": 1.28, "ttft_p50_ms": 477.3388774999603, "ttlt_p50_ms": 1408.7390404997677, "itl_p50_ms": 2.2803954999896328, "output_tokens_per_s": 271.3916391373417, "total_tokens_per_min": 4087239.5029001073}, {"request_rate": 1.92, "ttft_p50_ms": 536.1700529997506, "ttlt_p50_ms": 1847.8591884997968, "itl_p50_ms": 2.650516499973795, "output_tokens_per_s": 398.9486307011455, "total_tokens_per_min": 6008286.062948461}, {"request_rate": 2.88, "ttft_p50_ms": 584.0843944997687, "ttlt_p50_ms": 2279.5843060000607, "itl_p50_ms": 3.112823833286408, "output_tokens_per_s": 583.3953394880099, "total_tokens_per_min": 8786108.831291275}, {"request_rate": 4.32, "ttft_p50_ms": 688.3485809999002, "ttlt_p50_ms": 3482.0444144997964, "itl_p50_ms": 3.8428292499853947, "output_tokens_per_s": 833.5723672646229, "total_tokens_per_min": 12553849.922715401}, {"request_rate": 6.48, "ttft_p50_ms": 1211.0746610001115, "ttlt_p50_ms": 19735.05605399987, "itl_p50_ms": 7.075632499966862, "output_tokens_per_s": 524.8241387304741, "total_tokens_per_min": 7904008.97652256}]}, {"recipe_id": "rec-c56a58d275c9", "recipe_name": "deepseek-v4-1-flash-sglang-b200x4", "model": "deepseek-ai/DeepSeek-V4.1-Flash", "model_family": "deepseek_v4", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-b52c31c492", "benchmarked_at": "2026-09-12T21:49:31.345015+00:00", "aa_intelligence_index": 39.5, "aa_slug": "deepseek-v4-1-flash", "points": [{"request_rate": 0.44, "ttft_p50_ms": 549.539191000008, "ttlt_p50_ms": 2331.832601999963, "itl_p50_ms": 9.005199499938499, "output_tokens_per_s": 88.7617847160949, "total_tokens_per_min": 1336779.1063598038}, {"request_rate": 0.67, "ttft_p50_ms": 336.1866074999398, "ttlt_p50_ms": 2677.9105209998306, "itl_p50_ms": 9.68143600005078, "output_tokens_per_s": 143.62008979155274, "total_tokens_per_min": 2162961.6382877217}, {"request_rate": 1.01, "ttft_p50_ms": 335.1693299996441, "ttlt_p50_ms": 3053.5784874998626, "itl_p50_ms": 10.342623000042295, "output_tokens_per_s": 210.4111306664623, "total_tokens_per_min": 3168854.751176122}, {"request_rate": 1.52, "ttft_p50_ms": 417.44532799998524, "ttlt_p50_ms": 3993.8219360001312, "itl_p50_ms": 12.509990500120693, "output_tokens_per_s": 305.8022346883144, "total_tokens_per_min": 4605473.395076421}, {"request_rate": 2.28, "ttft_p50_ms": 640.6708770000478, "ttlt_p50_ms": 9504.074404499988, "itl_p50_ms": 19.176659999857293, "output_tokens_per_s": 433.704763290292, "total_tokens_per_min": 6531723.846580786}, {"request_rate": 3.42, "ttft_p50_ms": 598.5301974997128, "ttlt_p50_ms": 12016.602378000016, "itl_p50_ms": 32.28621099992779, "output_tokens_per_s": 549.0489282670527, "total_tokens_per_min": 8268841.574380294}, {"request_rate": 5.13, "ttft_p50_ms": 597.8543630001241, "ttlt_p50_ms": 12533.130058000097, "itl_p50_ms": 45.35392000002503, "output_tokens_per_s": 649.8437274106619, "total_tokens_per_min": 9786841.487922793}]}, {"recipe_id": "rec-feb41616ea34", "recipe_name": "gemma-4-26b-a4b-it-sglang-b200x1", "model": "google/gemma-4-26B-A4B-it", "model_family": "gemma4", "engine": "SGLang", "speculative_draft_model": "z-lab/gemma-4-26B-A4B-it-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-7d6374fd34", "benchmarked_at": "2026-08-25T00:11:36.123403+00:00", "aa_intelligence_index": 16.7, "aa_slug": "gemma-4-26b-a4b", "points": [{"request_rate": 0.24, "ttft_p50_ms": 1995.2876600000025, "ttlt_p50_ms": 4078.7744625000073, "itl_p50_ms": 8.413040400000682, "output_tokens_per_s": 48.041796943926336, "total_tokens_per_min": 723523.8745146139}, {"request_rate": 0.36, "ttft_p50_ms": 153.14586849999046, "ttlt_p50_ms": 2448.6039689999757, "itl_p50_ms": 8.449770699996861, "output_tokens_per_s": 77.18836942675547, "total_tokens_per_min": 1162480.0000777654}, {"request_rate": 0.54, "ttft_p50_ms": 151.85884050004006, "ttlt_p50_ms": 2434.4670269999824, "itl_p50_ms": 8.444648399995458, "output_tokens_per_s": 115.23266281719557, "total_tokens_per_min": 1735438.4718258106}, {"request_rate": 0.81, "ttft_p50_ms": 158.56351600001517, "ttlt_p50_ms": 2454.1837940000164, "itl_p50_ms": 8.460534000005282, "output_tokens_per_s": 171.5047287197045, "total_tokens_per_min": 2582912.6659373655}, {"request_rate": 1.22, "ttft_p50_ms": 159.99000249996698, "ttlt_p50_ms": 2676.278088500055, "itl_p50_ms": 8.556041399992864, "output_tokens_per_s": 249.80898392149626, "total_tokens_per_min": 3762198.2405529097}, {"request_rate": 1.83, "ttft_p50_ms": 210.8130164999693, "ttlt_p50_ms": 3566.518953999946, "itl_p50_ms": 13.384608166669901, "output_tokens_per_s": 359.8996768077869, "total_tokens_per_min": 5420197.102628313}, {"request_rate": 2.75, "ttft_p50_ms": 219.71436249998533, "ttlt_p50_ms": 4991.319578999992, "itl_p50_ms": 19.479865666672442, "output_tokens_per_s": 481.2181297804327, "total_tokens_per_min": 7247289.399932251}]}, {"recipe_id": "rec-1317cf6c1911", "recipe_name": "k2-horizon-0-9b-sglang-h100x1", "model": "IFM/K2-Horizon-0.9B", "model_family": "k2_horizon", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-78ca98351a", "benchmarked_at": "2026-09-16T00:27:58.044107+00:00", "aa_intelligence_index": 3.0, "aa_slug": "k2-horizon-0-9b", "points": [{"request_rate": 1.26, "ttft_p50_ms": 247.23165300000005, "ttlt_p50_ms": 791.445773999996, "itl_p50_ms": 2.721809499988126, "output_tokens_per_s": 252.6273481389983, "total_tokens_per_min": 3804643.651177756}, {"request_rate": 1.89, "ttft_p50_ms": 135.0355419999829, "ttlt_p50_ms": 879.2982495000103, "itl_p50_ms": 3.08681600000682, "output_tokens_per_s": 399.3046587659703, "total_tokens_per_min": 6013647.952413143}, {"request_rate": 2.83, "ttft_p50_ms": 208.9039730000195, "ttlt_p50_ms": 1118.4909424999887, "itl_p50_ms": 3.166192000009005, "output_tokens_per_s": 579.618612569521, "total_tokens_per_min": 8729230.190880757}, {"request_rate": 4.25, "ttft_p50_ms": 442.5532945000157, "ttlt_p50_ms": 1699.3840160000104, "itl_p50_ms": 2.7710765000108495, "output_tokens_per_s": 837.7157124078708, "total_tokens_per_min": 12616249.943576256}, {"request_rate": 6.38, "ttft_p50_ms": 1645.5985309999903, "ttlt_p50_ms": 2971.8866950000233, "itl_p50_ms": 0.9842669999784448, "output_tokens_per_s": 1043.1437721702268, "total_tokens_per_min": 15710058.152015267}, {"request_rate": 9.57, "ttft_p50_ms": 4261.262107499988, "ttlt_p50_ms": 5250.020067999998, "itl_p50_ms": 0.7440425000311279, "output_tokens_per_s": 1052.2676550345348, "total_tokens_per_min": 15847466.565116605}]}, {"recipe_id": "rec-67f3d4398b35", "recipe_name": "muse-glimmer-30b-sglang-h200x1", "model": "meta-models/Muse-Glimmer-30B", "model_family": "muse_glimmer", "engine": "SGLang", "speculative_draft_model": "meta-models/Muse-Glimmer-30B-assistant", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-3770c92b79", "benchmarked_at": "2026-09-08T21:04:38.192470+00:00", "aa_intelligence_index": 17.5, "aa_slug": "muse-glimmer", "points": [{"request_rate": 0.54, "ttft_p50_ms": 1040.48358, "ttlt_p50_ms": 1850.4756730000054, "itl_p50_ms": 3.673314399998162, "output_tokens_per_s": 108.5850383749496, "total_tokens_per_min": 1635355.828949766}, {"request_rate": 0.81, "ttft_p50_ms": 178.06407699998772, "ttlt_p50_ms": 1016.1031210000147, "itl_p50_ms": 3.7083568000070954, "output_tokens_per_s": 172.7735545283517, "total_tokens_per_min": 2602073.3953296933}, {"request_rate": 1.22, "ttft_p50_ms": 161.59962899996572, "ttlt_p50_ms": 1014.9325270000418, "itl_p50_ms": 3.7229604000117433, "output_tokens_per_s": 258.30623500522177, "total_tokens_per_min": 3890246.8829196426}, {"request_rate": 1.83, "ttft_p50_ms": 177.69816200001287, "ttlt_p50_ms": 1061.8901834999974, "itl_p50_ms": 3.7769982000099844, "output_tokens_per_s": 383.7515291010268, "total_tokens_per_min": 5779528.279178924}, {"request_rate": 2.75, "ttft_p50_ms": 240.82481799996458, "ttlt_p50_ms": 1182.985828000028, "itl_p50_ms": 3.7034965999964697, "output_tokens_per_s": 565.9225047674445, "total_tokens_per_min": 8523132.475300575}, {"request_rate": 4.12, "ttft_p50_ms": 279.7406335000687, "ttlt_p50_ms": 1240.5436134999945, "itl_p50_ms": 3.6705748000031235, "output_tokens_per_s": 830.0978317234146, "total_tokens_per_min": 12501771.40445366}, {"request_rate": 6.18, "ttft_p50_ms": 451.58641549994627, "ttlt_p50_ms": 1432.6746950000029, "itl_p50_ms": 0.5071297999847957, "output_tokens_per_s": 1178.5486939001714, "total_tokens_per_min": 17749650.45935292}]}, {"recipe_id": "rec-572a09aadee2", "recipe_name": "glm-5-3-nvfp4-sglang-b300x4", "model": "modal-labs/GLM-5.3-NVFP4", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": "modal-labs/GLM-5.3-NVFP4-DFlash", "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-216eb18444", "benchmarked_at": "2026-09-20T23:52:00.799037+00:00", "aa_intelligence_index": 44.8, "aa_slug": "glm-5-3", "points": [{"request_rate": 1.47, "ttft_p50_ms": 324.1482799999744, "ttlt_p50_ms": 683.3477899999707, "itl_p50_ms": 1.3087701250000805, "output_tokens_per_s": 293.4792537293694, "total_tokens_per_min": 4419885.604940423}, {"request_rate": 2.2, "ttft_p50_ms": 190.77371049996827, "ttlt_p50_ms": 720.9719874999792, "itl_p50_ms": 1.4992305000021133, "output_tokens_per_s": 465.5156650714081, "total_tokens_per_min": 7010805.570674928}, {"request_rate": 3.3, "ttft_p50_ms": 221.74983900004008, "ttlt_p50_ms": 795.3565584999751, "itl_p50_ms": 1.5421479999986332, "output_tokens_per_s": 689.8995830201231, "total_tokens_per_min": 10390094.690157961}, {"request_rate": 4.95, "ttft_p50_ms": 371.15925200004085, "ttlt_p50_ms": 1251.1662320000028, "itl_p50_ms": 1.8737693124961652, "output_tokens_per_s": 1005.4280468917991, "total_tokens_per_min": 15142048.014604561}, {"request_rate": 7.43, "ttft_p50_ms": 510.3180355000063, "ttlt_p50_ms": 1370.7481474999668, "itl_p50_ms": 1.523227250004311, "output_tokens_per_s": 1437.2884032364138, "total_tokens_per_min": 21645994.539261363}, {"request_rate": 11.14, "ttft_p50_ms": 2488.740155000187, "ttlt_p50_ms": 3670.1022680000506, "itl_p50_ms": 0.26722299998027665, "output_tokens_per_s": 1913.4964122211127, "total_tokens_per_min": 28817830.016973622}, {"request_rate": 16.71, "ttft_p50_ms": 3776.5835674999835, "ttlt_p50_ms": 4372.962455999868, "itl_p50_ms": 0.2292788749969077, "output_tokens_per_s": 1974.5029768110676, "total_tokens_per_min": 29736607.181667723}]}, {"recipe_id": "rec-bc81f4947654", "recipe_name": "qwen38-a95b-nvfp4-dflash-sglang-b300x8", "model": "modal-labs/Qwen3.8-2.4T-A95B-NVFP4", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.8-2.4T-A95B-DFlash", "gpu_type": "B300", "gpu_count": 8, "benchmark_id": "bench-dd0e6643f7", "benchmarked_at": "2026-08-18T13:04:14.892824+00:00", "aa_intelligence_index": 39.9, "aa_slug": "qwen3-8-2-4t-a95b", "points": [{"request_rate": 0.98, "ttft_p50_ms": 341.35572900004263, "ttlt_p50_ms": 1007.3258495000346, "itl_p50_ms": 3.074454804019941, "output_tokens_per_s": 195.33569801873432, "total_tokens_per_min": 2941814.212871544}, {"request_rate": 1.47, "ttft_p50_ms": 202.22825099995134, "ttlt_p50_ms": 1019.4706534999796, "itl_p50_ms": 3.860645201004774, "output_tokens_per_s": 308.9586218091051, "total_tokens_per_min": 4653009.532031666}, {"request_rate": 2.21, "ttft_p50_ms": 193.9160519999632, "ttlt_p50_ms": 1021.8209444999502, "itl_p50_ms": 3.666532145729635, "output_tokens_per_s": 462.4434530377732, "total_tokens_per_min": 6964537.135784776}, {"request_rate": 3.31, "ttft_p50_ms": 220.6154329999208, "ttlt_p50_ms": 1258.5569849998137, "itl_p50_ms": 4.466114432160058, "output_tokens_per_s": 676.2126369837254, "total_tokens_per_min": 10183965.176765999}, {"request_rate": 4.96, "ttft_p50_ms": 252.05792349993317, "ttlt_p50_ms": 1547.162506999939, "itl_p50_ms": 5.692015814070397, "output_tokens_per_s": 979.3111619835801, "total_tokens_per_min": 14748719.892821312}, {"request_rate": 7.44, "ttft_p50_ms": 258.8651475000461, "ttlt_p50_ms": 2456.9465419999688, "itl_p50_ms": 8.606592162010589, "output_tokens_per_s": 1373.3411060191281, "total_tokens_per_min": 20682929.058979873}, {"request_rate": 11.16, "ttft_p50_ms": 308.69231099995886, "ttlt_p50_ms": 9977.261576000046, "itl_p50_ms": 36.13875391620088, "output_tokens_per_s": 879.0954364929031, "total_tokens_per_min": 13239441.002214069}]}, {"recipe_id": "rec-3d66cbf014bf", "recipe_name": "kimi-k3-sglang-b300x8", "model": "moonshotai/Kimi-K3", "model_family": "kimi_k3", "engine": "SGLang", "speculative_draft_model": "modal-labs/Kimi-K3-DFlash", "gpu_type": "B300", "gpu_count": 8, "benchmark_id": "bench-efa7c19af0", "benchmarked_at": "2026-08-18T20:27:34.652061+00:00", "aa_intelligence_index": 43.6, "aa_slug": "kimi-k3", "points": [{"request_rate": 1.29, "ttft_p50_ms": 293.90217700000676, "ttlt_p50_ms": 749.933652999971, "itl_p50_ms": 1.4962236249971284, "output_tokens_per_s": 258.48426245354494, "total_tokens_per_min": 3892850.537829123}, {"request_rate": 1.94, "ttft_p50_ms": 596.3880004999282, "ttlt_p50_ms": 2273.123618999989, "itl_p50_ms": 2.8408802500052843, "output_tokens_per_s": 402.3262206240984, "total_tokens_per_min": 6059153.58046511}, {"request_rate": 2.91, "ttft_p50_ms": 1268.231609499935, "ttlt_p50_ms": 4763.466631999904, "itl_p50_ms": 4.337515125001801, "output_tokens_per_s": 578.1130042010311, "total_tokens_per_min": 8706555.277168788}, {"request_rate": 4.37, "ttft_p50_ms": 3172.3561119999886, "ttlt_p50_ms": 9136.018658500006, "itl_p50_ms": 5.283481874982954, "output_tokens_per_s": 663.1210827107634, "total_tokens_per_min": 9986802.441948911}, {"request_rate": 6.55, "ttft_p50_ms": 4954.309098499948, "ttlt_p50_ms": 10088.02264750011, "itl_p50_ms": 5.13521206249834, "output_tokens_per_s": 694.6541979025458, "total_tokens_per_min": 10461700.61667171}]}, {"recipe_id": "rec-e062c16beb19", "recipe_name": "kimi-k2-6-nvfp4-sglang-b200x4", "model": "nvidia/Kimi-K2.6-NVFP4", "model_family": "kimi_k25", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-000c3afa84", "benchmarked_at": "2026-08-31T23:19:39.667119+00:00", "aa_intelligence_index": 27.0, "aa_slug": "kimi-k2-6", "points": [{"request_rate": 0.72, "ttft_p50_ms": 237.94375350001928, "ttlt_p50_ms": 1392.164325499948, "itl_p50_ms": 5.794465000008131, "output_tokens_per_s": 143.7118027398146, "total_tokens_per_min": 2164342.86280243}, {"request_rate": 1.08, "ttft_p50_ms": 98.05738949995657, "ttlt_p50_ms": 1678.7937540000257, "itl_p50_ms": 7.214586999964467, "output_tokens_per_s": 228.30047278890797, "total_tokens_per_min": 3438273.6103427904}, {"request_rate": 1.62, "ttft_p50_ms": 100.45575499998449, "ttlt_p50_ms": 2064.8639790000516, "itl_p50_ms": 8.873599999958515, "output_tokens_per_s": 336.57281779794175, "total_tokens_per_min": 5068887.607882342}, {"request_rate": 2.43, "ttft_p50_ms": 102.97392250004123, "ttlt_p50_ms": 2383.4073555000828, "itl_p50_ms": 9.351062499945328, "output_tokens_per_s": 490.98943987818467, "total_tokens_per_min": 7394448.261397425}, {"request_rate": 3.65, "ttft_p50_ms": 110.50997300003473, "ttlt_p50_ms": 3385.476142000016, "itl_p50_ms": 11.908118999997441, "output_tokens_per_s": 701.9014188041347, "total_tokens_per_min": 10570845.937615909}, {"request_rate": 5.47, "ttft_p50_ms": 158.02909550001232, "ttlt_p50_ms": 9924.182389999998, "itl_p50_ms": 49.653868999996575, "output_tokens_per_s": 731.5564397188314, "total_tokens_per_min": 11017459.449097516}]}, {"recipe_id": "rec-78b54f7ee105", "recipe_name": "nvidia-nemotron-3-super-120b-a12b-nvfp4-sglang-b200x1", "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "model_family": "nemotron_h", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-6f60941017", "benchmarked_at": "2026-08-25T02:18:19.986566+00:00", "aa_intelligence_index": 12.8, "aa_slug": "nvidia-nemotron-3-super-120b-a12b", "points": [{"request_rate": 0.72, "ttft_p50_ms": 381.0809080000013, "ttlt_p50_ms": 1361.8249619999858, "itl_p50_ms": 4.854374999979427, "output_tokens_per_s": 144.973002178301, "total_tokens_per_min": 2183336.9047058662}, {"request_rate": 1.09, "ttft_p50_ms": 113.80768399999397, "ttlt_p50_ms": 1399.1378824999856, "itl_p50_ms": 5.634270000001607, "output_tokens_per_s": 231.00034134938494, "total_tokens_per_min": 3478934.4408241417}, {"request_rate": 1.64, "ttft_p50_ms": 131.1738480000315, "ttlt_p50_ms": 1645.7815729999936, "itl_p50_ms": 6.1717119999684655, "output_tokens_per_s": 342.7050823845053, "total_tokens_per_min": 5161241.352235366}, {"request_rate": 2.46, "ttft_p50_ms": 149.6858704999795, "ttlt_p50_ms": 1870.982726999955, "itl_p50_ms": 7.245508499977404, "output_tokens_per_s": 500.94596283523805, "total_tokens_per_min": 7544396.484087536}, {"request_rate": 3.69, "ttft_p50_ms": 195.92596950002417, "ttlt_p50_ms": 2253.0041299999652, "itl_p50_ms": 7.5362450000397985, "output_tokens_per_s": 722.4429364096568, "total_tokens_per_min": 10880207.355210353}, {"request_rate": 5.54, "ttft_p50_ms": 313.1341335000002, "ttlt_p50_ms": 3813.41712349996, "itl_p50_ms": 8.794942500003344, "output_tokens_per_s": 995.1676438852592, "total_tokens_per_min": 14987523.267205168}, {"request_rate": 8.31, "ttft_p50_ms": 934.7060705000558, "ttlt_p50_ms": 4982.10816400001, "itl_p50_ms": 6.793564499957938, "output_tokens_per_s": 1182.0238605096622, "total_tokens_per_min": 17801633.946433663}]}, {"recipe_id": "rec-1753b0aaa744", "recipe_name": "nvidia-nemotron-3-ultra-550b-a55b-nvfp4-sglang-b200x4", "model": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "model_family": "nemotron_h", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-020c060d36", "benchmarked_at": "2026-09-18T01:17:59.086088+00:00", "aa_intelligence_index": 22.9, "aa_slug": "nvidia-nemotron-3-ultra-550b-a55b", "points": [{"request_rate": 0.99, "ttft_p50_ms": 525.8548395000275, "ttlt_p50_ms": 989.3851919999861, "itl_p50_ms": 1.623743000000862, "output_tokens_per_s": 198.89709875264188, "total_tokens_per_min": 2995449.976344412}, {"request_rate": 1.49, "ttft_p50_ms": 555.7792005000124, "ttlt_p50_ms": 1579.6882945000448, "itl_p50_ms": 2.311370666670124, "output_tokens_per_s": 315.0526998943243, "total_tokens_per_min": 4744788.176218492}, {"request_rate": 2.23, "ttft_p50_ms": 646.6814720000116, "ttlt_p50_ms": 2036.5575600000057, "itl_p50_ms": 3.145162999999229, "output_tokens_per_s": 462.0477823401782, "total_tokens_per_min": 6958578.2163777845}, {"request_rate": 3.34, "ttft_p50_ms": 713.0605670000705, "ttlt_p50_ms": 2478.419620000068, "itl_p50_ms": 3.436818333322359, "output_tokens_per_s": 652.2364522615896, "total_tokens_per_min": 9822876.64199522}, {"request_rate": 5.01, "ttft_p50_ms": 1304.3830285000695, "ttlt_p50_ms": 6611.753645000022, "itl_p50_ms": 4.94882283332269, "output_tokens_per_s": 939.6941524657645, "total_tokens_per_min": 14152075.844380153}, {"request_rate": 7.51, "ttft_p50_ms": 1925.548975500078, "ttlt_p50_ms": 5326.983307500086, "itl_p50_ms": 5.12504733330843, "output_tokens_per_s": 1285.308563047186, "total_tokens_per_min": 19357132.552059535}, {"request_rate": 11.27, "ttft_p50_ms": 3802.033359999996, "ttlt_p50_ms": 5430.2615154999785, "itl_p50_ms": 1.403472750041601, "output_tokens_per_s": 1353.849401027913, "total_tokens_per_min": 20389378.13430068}]}, {"recipe_id": "rec-492335383e6a", "recipe_name": "nvidia-nemotron-3-5-lightning-30b-a3b-nvfp4-sglang-b200x1", "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "model_family": "nemotron35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-19185f0ab6", "benchmarked_at": "2026-09-18T01:07:48.665973+00:00", "aa_intelligence_index": 12.9, "aa_slug": "nemotron-3-5-lightning", "points": [{"request_rate": 1.45, "ttft_p50_ms": 220.11395600000583, "ttlt_p50_ms": 678.6903610000081, "itl_p50_ms": 2.2710285000187014, "output_tokens_per_s": 289.45083955949696, "total_tokens_per_min": 4359216.479017893}, {"request_rate": 2.17, "ttft_p50_ms": 131.09335599997962, "ttlt_p50_ms": 727.386765499972, "itl_p50_ms": 2.4529459999769188, "output_tokens_per_s": 458.804693130267, "total_tokens_per_min": 6909736.319949761}, {"request_rate": 3.25, "ttft_p50_ms": 192.68701999999394, "ttlt_p50_ms": 848.8857275000044, "itl_p50_ms": 2.3974714999894786, "output_tokens_per_s": 677.8522231317813, "total_tokens_per_min": 10208657.836031565}, {"request_rate": 4.88, "ttft_p50_ms": 181.59139049998885, "ttlt_p50_ms": 955.8644644999958, "itl_p50_ms": 2.3720714999910797, "output_tokens_per_s": 991.2805542899937, "total_tokens_per_min": 14928982.531773591}, {"request_rate": 7.32, "ttft_p50_ms": 671.9440830000281, "ttlt_p50_ms": 1256.495846000007, "itl_p50_ms": 0.9048955000707792, "output_tokens_per_s": 1409.1601026800975, "total_tokens_per_min": 21222373.89439307}, {"request_rate": 10.98, "ttft_p50_ms": 3250.536226500003, "ttlt_p50_ms": 4187.442943000007, "itl_p50_ms": 0.811166999994839, "output_tokens_per_s": 1818.208270271197, "total_tokens_per_min": 27382762.012765307}, {"request_rate": 16.47, "ttft_p50_ms": 4966.474535999964, "ttlt_p50_ms": 5256.085213000006, "itl_p50_ms": 0.8102420000000166, "output_tokens_per_s": 1780.7037683799988, "total_tokens_per_min": 26817932.962933294}]}, {"recipe_id": "rec-323026e40cb1", "recipe_name": "gpt-oss-120b-sglang-b200x1", "model": "openai/gpt-oss-120b", "model_family": "gptoss", "engine": "SGLang", "speculative_draft_model": "z-lab/gpt-oss-120b-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-e98ebb833f", "benchmarked_at": "2026-09-18T01:11:58.640317+00:00", "aa_intelligence_index": 11.6, "aa_slug": "gpt-oss-120b", "points": [{"request_rate": 0.7, "ttft_p50_ms": 500.4253039999753, "ttlt_p50_ms": 1457.5390109999944, "itl_p50_ms": 5.118156999998291, "output_tokens_per_s": 140.73635448766416, "total_tokens_per_min": 2119531.719490568}, {"request_rate": 1.06, "ttft_p50_ms": 129.9552169999174, "ttlt_p50_ms": 1374.5732319999888, "itl_p50_ms": 5.604882499994801, "output_tokens_per_s": 226.59370064493504, "total_tokens_per_min": 3412569.109822915}, {"request_rate": 1.59, "ttft_p50_ms": 123.63022400001, "ttlt_p50_ms": 1459.555787499994, "itl_p50_ms": 6.027496999934101, "output_tokens_per_s": 332.5433768965746, "total_tokens_per_min": 5008203.019075483}, {"request_rate": 2.39, "ttft_p50_ms": 132.70038100000647, "ttlt_p50_ms": 1870.1387050000449, "itl_p50_ms": 7.640360250036338, "output_tokens_per_s": 486.66257834057507, "total_tokens_per_min": 7329284.428582562}, {"request_rate": 3.58, "ttft_p50_ms": 162.58310350002603, "ttlt_p50_ms": 3528.6707614999955, "itl_p50_ms": 11.64565100003756, "output_tokens_per_s": 682.9832329284325, "total_tokens_per_min": 10285932.382872071}, {"request_rate": 5.37, "ttft_p50_ms": 203.5640640000338, "ttlt_p50_ms": 5345.737678500029, "itl_p50_ms": 17.306607000023178, "output_tokens_per_s": 853.9516787761288, "total_tokens_per_min": 12860768.467872133}, {"request_rate": 8.05, "ttft_p50_ms": 385.56707550003466, "ttlt_p50_ms": 6595.924136999997, "itl_p50_ms": 23.330351500021607, "output_tokens_per_s": 960.0270375774669, "total_tokens_per_min": 14458295.194027925}]}, {"recipe_id": "rec-8a6954502613", "recipe_name": "phonellm-alpha-1-vllm-b200x1", "model": "pipecat-ai/phonellm-alpha-1", "model_family": "Nemotron-H", "engine": "vLLM", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-2557a1a847", "benchmarked_at": "2026-08-31T19:37:10.212954+00:00", "aa_intelligence_index": null, "aa_slug": null, "points": [{"request_rate": 1.61, "ttft_p50_ms": 151.07755550002366, "ttlt_p50_ms": 618.7639949999948, "itl_p50_ms": 2.6058569999349857, "output_tokens_per_s": 322.89033900957634, "total_tokens_per_min": 4862825.3725859225}, {"request_rate": 2.42, "ttft_p50_ms": 116.3678815000253, "ttlt_p50_ms": 847.4642979999771, "itl_p50_ms": 4.469953000011628, "output_tokens_per_s": 555.2386320438791, "total_tokens_per_min": 8362060.370170433}, {"request_rate": 3.63, "ttft_p50_ms": 131.38495100002956, "ttlt_p50_ms": 1010.9734669999852, "itl_p50_ms": 5.15069699997639, "output_tokens_per_s": 815.815106267787, "total_tokens_per_min": 12286420.24492475}, {"request_rate": 5.45, "ttft_p50_ms": 149.00338900002907, "ttlt_p50_ms": 1278.5202564999736, "itl_p50_ms": 5.765956499942604, "output_tokens_per_s": 1171.5595619498356, "total_tokens_per_min": 17644038.47083311}, {"request_rate": 8.18, "ttft_p50_ms": 174.50589649996573, "ttlt_p50_ms": 1746.3140515000646, "itl_p50_ms": 7.035120000068673, "output_tokens_per_s": 1630.8897297273147, "total_tokens_per_min": 24561688.59661228}, {"request_rate": 12.27, "ttft_p50_ms": 223.50877150006454, "ttlt_p50_ms": 2376.205137500051, "itl_p50_ms": 8.359664500005692, "output_tokens_per_s": 2148.5707510886996, "total_tokens_per_min": 32358120.082621146}, {"request_rate": 18.41, "ttft_p50_ms": 287.3146065000469, "ttlt_p50_ms": 3014.013473500029, "itl_p50_ms": 10.555255999975088, "output_tokens_per_s": 2560.131109434419, "total_tokens_per_min": 38556342.54741518}]}, {"recipe_id": "rec-d08310e9a9bb", "recipe_name": "qwen3-5-0-8b-sglang-a100x1", "model": "Qwen/Qwen3.5-0.8B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-cda301f21c", "benchmarked_at": "2026-09-10T04:54:15.561992+00:00", "aa_intelligence_index": 6.1, "aa_slug": "qwen3-5-0-8b", "points": [{"request_rate": 1.2, "ttft_p50_ms": 261.68969749998894, "ttlt_p50_ms": 710.3336830000018, "itl_p50_ms": 2.0468467500052157, "output_tokens_per_s": 239.6745643998939, "total_tokens_per_min": 3609570.842231722}, {"request_rate": 1.8, "ttft_p50_ms": 148.51809549998052, "ttlt_p50_ms": 705.8890644999849, "itl_p50_ms": 2.0476867499894524, "output_tokens_per_s": 382.1236921048063, "total_tokens_per_min": 5754897.4402060155}, {"request_rate": 2.7, "ttft_p50_ms": 190.90764149999018, "ttlt_p50_ms": 771.6820415000143, "itl_p50_ms": 2.098687750006434, "output_tokens_per_s": 567.6744749193923, "total_tokens_per_min": 8549347.894628523}, {"request_rate": 4.05, "ttft_p50_ms": 224.51905049999255, "ttlt_p50_ms": 939.433178999991, "itl_p50_ms": 1.9099035000067488, "output_tokens_per_s": 830.421773873056, "total_tokens_per_min": 12506401.041060386}, {"request_rate": 6.07, "ttft_p50_ms": 637.7658064999991, "ttlt_p50_ms": 1441.6013314999816, "itl_p50_ms": 0.45681349999426857, "output_tokens_per_s": 1179.2787850792972, "total_tokens_per_min": 17760292.28692974}, {"request_rate": 9.11, "ttft_p50_ms": 3233.932468999967, "ttlt_p50_ms": 3954.850782499989, "itl_p50_ms": 0.4958770000058621, "output_tokens_per_s": 1537.0948652944849, "total_tokens_per_min": 23149109.799794525}, {"request_rate": 13.66, "ttft_p50_ms": 4229.015196999967, "ttlt_p50_ms": 4895.1651595000385, "itl_p50_ms": 0.5992794999940543, "output_tokens_per_s": 1660.0140219231694, "total_tokens_per_min": 25000309.174369507}]}, {"recipe_id": "rec-73244be0ea37", "recipe_name": "qwen3-5-122b-a10b-fp8-sglang-b200x2", "model": "Qwen/Qwen3.5-122B-A10B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-122B-A10B-DFlash", "gpu_type": "B200", "gpu_count": 2, "benchmark_id": "bench-c4d5a2e074", "benchmarked_at": "2026-09-17T22:12:07.879068+00:00", "aa_intelligence_index": 15.6, "aa_slug": "qwen3-5-122b-a10b", "points": [{"request_rate": 2.21, "ttft_p50_ms": 289.946365999981, "ttlt_p50_ms": 440.4291009999781, "itl_p50_ms": 0.5077979374981112, "output_tokens_per_s": 442.7352480478378, "total_tokens_per_min": 6667725.656174852}, {"request_rate": 3.32, "ttft_p50_ms": 1138.1560629999967, "ttlt_p50_ms": 15576.532813499995, "itl_p50_ms": 7.3694488125006785, "output_tokens_per_s": 420.0999546605286, "total_tokens_per_min": 6326831.347173959}]}, {"recipe_id": "rec-eb08cfab30d4", "recipe_name": "qwen3-5-27b-fp8-sglang-h100x1", "model": "Qwen/Qwen3.5-27B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-27B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-72b8034e1f", "benchmarked_at": "2026-09-17T21:36:32.745390+00:00", "aa_intelligence_index": 22.9, "aa_slug": "qwen3-5-27b", "points": [{"request_rate": 0.65, "ttft_p50_ms": 691.8542985000187, "ttlt_p50_ms": 1477.4507684999776, "itl_p50_ms": 2.8103396666665503, "output_tokens_per_s": 129.35919786576926, "total_tokens_per_min": 1948188.3276178448}, {"request_rate": 0.97, "ttft_p50_ms": 3136.104573500006, "ttlt_p50_ms": 5529.431485499998, "itl_p50_ms": 3.2174672499962753, "output_tokens_per_s": 197.45521707751473, "total_tokens_per_min": 2973734.8057524953}, {"request_rate": 1.46, "ttft_p50_ms": 9682.002504000026, "ttlt_p50_ms": 12301.393044500002, "itl_p50_ms": 3.2186411999987286, "output_tokens_per_s": 228.9792095496648, "total_tokens_per_min": 3448495.5895808167}, {"request_rate": 2.19, "ttft_p50_ms": 16274.86465000004, "ttlt_p50_ms": 18872.45078799998, "itl_p50_ms": 3.4284817777713923, "output_tokens_per_s": 226.19768859508076, "total_tokens_per_min": 3406605.0495484946}]}, {"recipe_id": "rec-18de64e3b9a7", "recipe_name": "qwen3-5-2b-sglang-a100x1", "model": "Qwen/Qwen3.5-2B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "A100", "gpu_count": 1, "benchmark_id": "bench-929b65190c", "benchmarked_at": "2026-09-10T04:58:21.037049+00:00", "aa_intelligence_index": 6.9, "aa_slug": "qwen3-5-2b", "points": [{"request_rate": 0.99, "ttft_p50_ms": 368.593837499958, "ttlt_p50_ms": 994.0206534999732, "itl_p50_ms": 2.797297749992822, "output_tokens_per_s": 197.38554672830392, "total_tokens_per_min": 2972685.5493922755}, {"request_rate": 1.48, "ttft_p50_ms": 252.49201950001066, "ttlt_p50_ms": 985.104505999999, "itl_p50_ms": 2.866027500004975, "output_tokens_per_s": 313.3274687305538, "total_tokens_per_min": 4718805.677322759}, {"request_rate": 2.22, "ttft_p50_ms": 289.37385600002585, "ttlt_p50_ms": 1011.9726515000025, "itl_p50_ms": 2.7514962499992635, "output_tokens_per_s": 464.3264260406752, "total_tokens_per_min": 6992895.27410038}, {"request_rate": 3.33, "ttft_p50_ms": 405.82959500000015, "ttlt_p50_ms": 1224.0282120000074, "itl_p50_ms": 2.475599000007378, "output_tokens_per_s": 680.7562691710913, "total_tokens_per_min": 10252393.640597386}, {"request_rate": 5, "ttft_p50_ms": 1179.668940000056, "ttlt_p50_ms": 1799.2760149999754, "itl_p50_ms": 0.38738250000847074, "output_tokens_per_s": 967.8432629415661, "total_tokens_per_min": 14576009.892878868}, {"request_rate": 7.5, "ttft_p50_ms": 5477.7372240000095, "ttlt_p50_ms": 6538.378582999996, "itl_p50_ms": 0.42238000000338616, "output_tokens_per_s": 1240.9402245604874, "total_tokens_per_min": 18688932.063948315}, {"request_rate": 11.25, "ttft_p50_ms": 6833.693690999951, "ttlt_p50_ms": 7455.8864059999905, "itl_p50_ms": 0.4200015000037638, "output_tokens_per_s": 1265.0027920361006, "total_tokens_per_min": 19051321.548901286}]}, {"recipe_id": "rec-e07d10272f16", "recipe_name": "qwen3-5-35b-a3b-fp8-sglang-h200x1", "model": "Qwen/Qwen3.5-35B-A3B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-35B-A3B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-568cd57156", "benchmarked_at": "2026-09-17T21:36:20.894647+00:00", "aa_intelligence_index": 19.3, "aa_slug": "qwen3-5-35b-a3b", "points": [{"request_rate": 1.91, "ttft_p50_ms": 229.62129249998497, "ttlt_p50_ms": 496.40670399998044, "itl_p50_ms": 0.8895508888902744, "output_tokens_per_s": 382.5847678037877, "total_tokens_per_min": 5761841.3785553835}, {"request_rate": 2.87, "ttft_p50_ms": 366.79895450001254, "ttlt_p50_ms": 1161.3571620000016, "itl_p50_ms": 1.1717058750022602, "output_tokens_per_s": 601.0352127953566, "total_tokens_per_min": 9051770.615261912}, {"request_rate": 4.3, "ttft_p50_ms": 623.3387099999845, "ttlt_p50_ms": 2160.6967644999886, "itl_p50_ms": 2.4969467142860986, "output_tokens_per_s": 843.2194131290838, "total_tokens_per_min": 12699137.327547941}, {"request_rate": 6.45, "ttft_p50_ms": 1991.299981499992, "ttlt_p50_ms": 4766.302416000002, "itl_p50_ms": 3.546670800000129, "output_tokens_per_s": 977.1352243823568, "total_tokens_per_min": 14715949.619765608}, {"request_rate": 9.68, "ttft_p50_ms": 3179.0095935000127, "ttlt_p50_ms": 6400.401724000005, "itl_p50_ms": 2.7944303333337452, "output_tokens_per_s": 1019.9842640174567, "total_tokens_per_min": 15361269.011382103}]}, {"recipe_id": "rec-f85730b36ec2", "recipe_name": "qwen3-5-397b-a17b-fp8-sglang-b200x4", "model": "Qwen/Qwen3.5-397B-A17B-FP8", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-397B-A17B-DFlash", "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-3b1f5b60b9", "benchmarked_at": "2026-09-17T22:23:38.117555+00:00", "aa_intelligence_index": 18.4, "aa_slug": "qwen3-5-397b-a17b", "points": [{"request_rate": 1.52, "ttft_p50_ms": 261.7395334999628, "ttlt_p50_ms": 579.402603500057, "itl_p50_ms": 1.2029253750114322, "output_tokens_per_s": 304.68502503316466, "total_tokens_per_min": 4588647.882506969}, {"request_rate": 2.29, "ttft_p50_ms": 440.79338900002085, "ttlt_p50_ms": 1754.8000594999849, "itl_p50_ms": 1.5965944444486821, "output_tokens_per_s": 477.63977179769506, "total_tokens_per_min": 7193398.255204827}, {"request_rate": 3.44, "ttft_p50_ms": 1159.231418499985, "ttlt_p50_ms": 8069.1342089999125, "itl_p50_ms": 10.766601000000264, "output_tokens_per_s": 537.0996251514545, "total_tokens_per_min": 8088881.484668449}, {"request_rate": 5.16, "ttft_p50_ms": 2872.4932715000477, "ttlt_p50_ms": 9880.17968600002, "itl_p50_ms": 12.694369999962873, "output_tokens_per_s": 581.2082802890008, "total_tokens_per_min": 8753171.063636437}, {"request_rate": 7.74, "ttft_p50_ms": 5727.466035500015, "ttlt_p50_ms": 13076.111802000014, "itl_p50_ms": 15.089441187498664, "output_tokens_per_s": 545.5502841979091, "total_tokens_per_min": 8216150.945105771}]}, {"recipe_id": "rec-8339457cb66a", "recipe_name": "qwen3-5-4b-sglang-h100x1", "model": "Qwen/Qwen3.5-4B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.5-4B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-868b5cf00d", "benchmarked_at": "2026-09-17T19:07:14.356193+00:00", "aa_intelligence_index": 13.1, "aa_slug": "qwen3-5-4b", "points": [{"request_rate": 1.48, "ttft_p50_ms": 228.69005700000855, "ttlt_p50_ms": 532.7523925000008, "itl_p50_ms": 1.0956178571421202, "output_tokens_per_s": 296.84963179539426, "total_tokens_per_min": 4470644.509728176}, {"request_rate": 2.23, "ttft_p50_ms": 311.93719250001095, "ttlt_p50_ms": 828.4154035000029, "itl_p50_ms": 1.0756756363641464, "output_tokens_per_s": 472.2388974870304, "total_tokens_per_min": 7112059.467823923}, {"request_rate": 3.34, "ttft_p50_ms": 449.91566599998123, "ttlt_p50_ms": 1778.0366670000092, "itl_p50_ms": 1.6933958749980604, "output_tokens_per_s": 655.7789557324188, "total_tokens_per_min": 9876227.807016946}, {"request_rate": 5.01, "ttft_p50_ms": 1068.155847000014, "ttlt_p50_ms": 3426.5710105000267, "itl_p50_ms": 2.516908000025999, "output_tokens_per_s": 856.8056809851129, "total_tokens_per_min": 12903750.597340096}, {"request_rate": 7.51, "ttft_p50_ms": 2622.448415000008, "ttlt_p50_ms": 5264.264867000009, "itl_p50_ms": 3.014091800002916, "output_tokens_per_s": 877.9130644759382, "total_tokens_per_min": 13221634.124926971}]}, {"recipe_id": "rec-9f1ca6f0b1bb", "recipe_name": "qwen3-5-9b-sglang-h100x1", "model": "Qwen/Qwen3.5-9B", "model_family": "qwen35", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.5-9B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-e67cb307eb", "benchmarked_at": "2026-09-17T21:30:03.055630+00:00", "aa_intelligence_index": 11.2, "aa_slug": "qwen3-5-9b", "points": [{"request_rate": 1.78, "ttft_p50_ms": 306.8709144999957, "ttlt_p50_ms": 552.9835969999937, "itl_p50_ms": 0.7251511875008276, "output_tokens_per_s": 356.34404955579845, "total_tokens_per_min": 5366648.289525191}, {"request_rate": 2.67, "ttft_p50_ms": 1142.8988385000025, "ttlt_p50_ms": 4488.086912, "itl_p50_ms": 1.5213814999981423, "output_tokens_per_s": 556.9879320237559, "total_tokens_per_min": 8388405.35265737}, {"request_rate": 4, "ttft_p50_ms": 1024.0963454999985, "ttlt_p50_ms": 2430.4449420000083, "itl_p50_ms": 1.5128015624981117, "output_tokens_per_s": 798.3681011715807, "total_tokens_per_min": 12023663.114074359}, {"request_rate": 6, "ttft_p50_ms": 3083.731691499992, "ttlt_p50_ms": 5144.647385000013, "itl_p50_ms": 0.250715124998635, "output_tokens_per_s": 855.1600381824254, "total_tokens_per_min": 12878966.723038781}]}, {"recipe_id": "rec-ee25477ce1e0", "recipe_name": "qwen3-6-27b-sglang-h200x1", "model": "Qwen/Qwen3.6-27B", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.6-27B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-6afc141dd2", "benchmarked_at": "2026-09-17T21:36:45.644150+00:00", "aa_intelligence_index": 21.4, "aa_slug": "qwen3-6-27b", "points": [{"request_rate": 0.51, "ttft_p50_ms": 661.2824529999841, "ttlt_p50_ms": 2113.3504130000065, "itl_p50_ms": 5.2345459999969535, "output_tokens_per_s": 101.06508282693926, "total_tokens_per_min": 1522070.4668985533}, {"request_rate": 0.76, "ttft_p50_ms": 1409.5409265, "ttlt_p50_ms": 5428.134665999977, "itl_p50_ms": 7.483700999993441, "output_tokens_per_s": 157.37024829025134, "total_tokens_per_min": 2370043.1503256727}, {"request_rate": 1.14, "ttft_p50_ms": 4165.208227999983, "ttlt_p50_ms": 9809.43005399999, "itl_p50_ms": 8.054903000015656, "output_tokens_per_s": 219.69815626062763, "total_tokens_per_min": 3308720.1427319306}, {"request_rate": 1.71, "ttft_p50_ms": 11787.636401499982, "ttlt_p50_ms": 17189.322001999983, "itl_p50_ms": 9.29679775001091, "output_tokens_per_s": 226.8462741929446, "total_tokens_per_min": 3416372.943228004}]}, {"recipe_id": "rec-c4ac5ba2ffd8", "recipe_name": "qwen3-6-27b-fp8-sglang-h100x1", "model": "Qwen/Qwen3.6-27B-FP8", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "z-lab/Qwen3.6-27B-DFlash", "gpu_type": "H100", "gpu_count": 1, "benchmark_id": "bench-1b45a93871", "benchmarked_at": "2026-09-17T21:37:36.652334+00:00", "aa_intelligence_index": 21.4, "aa_slug": "qwen3-6-27b", "points": [{"request_rate": 0.46, "ttft_p50_ms": 687.1059240000079, "ttlt_p50_ms": 2535.448481000003, "itl_p50_ms": 5.041061100007482, "output_tokens_per_s": 91.22308299338728, "total_tokens_per_min": 1373846.9968053102}, {"request_rate": 0.68, "ttft_p50_ms": 1763.2775780000145, "ttlt_p50_ms": 5461.574451999979, "itl_p50_ms": 6.079360799998312, "output_tokens_per_s": 139.07347681993332, "total_tokens_per_min": 2094488.2829512414}, {"request_rate": 1.02, "ttft_p50_ms": 6939.151494499982, "ttlt_p50_ms": 9499.868322499935, "itl_p50_ms": 5.066294166662526, "output_tokens_per_s": 184.961984010212, "total_tokens_per_min": 2785582.9677889957}, {"request_rate": 1.53, "ttft_p50_ms": 16988.581173499937, "ttlt_p50_ms": 19446.716229000005, "itl_p50_ms": 7.543095500011532, "output_tokens_per_s": 179.44490442915884, "total_tokens_per_min": 2702494.094174461}]}, {"recipe_id": "rec-08a3e7455f15", "recipe_name": "qwen3-6-35b-a3b-sglang-b200x1", "model": "Qwen/Qwen3.6-35B-A3B", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.6-35B-A3B-DFlash", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-a0b89d0767", "benchmarked_at": "2026-09-08T02:05:46.298749+00:00", "aa_intelligence_index": 18.2, "aa_slug": "qwen3-6-35b-a3b", "points": [{"request_rate": 0.66, "ttft_p50_ms": 504.57170449999467, "ttlt_p50_ms": 1460.4207335000156, "itl_p50_ms": 3.1801222499998616, "output_tokens_per_s": 132.26529229395834, "total_tokens_per_min": 1991954.9815347004}, {"request_rate": 0.99, "ttft_p50_ms": 241.5587309999978, "ttlt_p50_ms": 1283.0760429999941, "itl_p50_ms": 3.1679284374988015, "output_tokens_per_s": 207.5117120483289, "total_tokens_per_min": 3125188.636961447}, {"request_rate": 1.48, "ttft_p50_ms": 1096.9851344999881, "ttlt_p50_ms": 4177.6523680000255, "itl_p50_ms": 5.828607562502697, "output_tokens_per_s": 308.0201971913876, "total_tokens_per_min": 4638876.575761455}, {"request_rate": 2.22, "ttft_p50_ms": 1080.9149929999933, "ttlt_p50_ms": 6067.064711000001, "itl_p50_ms": 7.892232999999749, "output_tokens_per_s": 407.18508497898813, "total_tokens_per_min": 6132329.535309056}, {"request_rate": 3.33, "ttft_p50_ms": 8131.1104089999735, "ttlt_p50_ms": 16248.37470199998, "itl_p50_ms": 11.804494062495507, "output_tokens_per_s": 382.8239605018823, "total_tokens_per_min": 5765443.692346497}]}, {"recipe_id": "rec-2919c029201f", "recipe_name": "qwen3-6-35b-a3b-fp8-sglang-h200x1", "model": "Qwen/Qwen3.6-35B-A3B-FP8", "model_family": "qwen36", "engine": "SGLang", "speculative_draft_model": "modal-labs/Qwen3.6-35B-A3B-DFlash", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-9a5e26391a", "benchmarked_at": "2026-09-17T21:36:51.014990+00:00", "aa_intelligence_index": 18.2, "aa_slug": "qwen3-6-35b-a3b", "points": [{"request_rate": 2.41, "ttft_p50_ms": 256.913024499994, "ttlt_p50_ms": 396.9673089999901, "itl_p50_ms": 0.5001172187490965, "output_tokens_per_s": 482.3592009805871, "total_tokens_per_min": 7264474.274527936}, {"request_rate": 3.62, "ttft_p50_ms": 547.0530179999855, "ttlt_p50_ms": 1493.5245635000172, "itl_p50_ms": 0.8415132499983713, "output_tokens_per_s": 751.4726584148619, "total_tokens_per_min": 11317403.677525343}, {"request_rate": 5.43, "ttft_p50_ms": 1008.2989459999965, "ttlt_p50_ms": 2465.0604674999668, "itl_p50_ms": 1.1320071875005056, "output_tokens_per_s": 1077.1242831306936, "total_tokens_per_min": 16221814.841233185}, {"request_rate": 8.14, "ttft_p50_ms": 2809.991889999992, "ttlt_p50_ms": 5721.449491499981, "itl_p50_ms": 1.575616499998489, "output_tokens_per_s": 1155.7096618465855, "total_tokens_per_min": 17405334.22030813}]}, {"recipe_id": "rec-e50ea0dc7fcb", "recipe_name": "qwen3-8-27b-sglang-h200x1", "model": "Qwen/Qwen3.8-27B", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": "incoai/Qwen3.8-27B-DFlash2", "gpu_type": "H200", "gpu_count": 1, "benchmark_id": "bench-ae49e8d1b3", "benchmarked_at": "2026-09-18T14:06:03.668541+00:00", "aa_intelligence_index": 33.7, "aa_slug": "qwen3-8-27b", "points": [{"request_rate": 0.78, "ttft_p50_ms": 652.7440229999968, "ttlt_p50_ms": 1260.6014314999925, "itl_p50_ms": 2.4656394375011104, "output_tokens_per_s": 156.74260556740785, "total_tokens_per_min": 2360590.6626268323}, {"request_rate": 1.18, "ttft_p50_ms": 2241.618520999992, "ttlt_p50_ms": 5755.398824499991, "itl_p50_ms": 4.2049244999944335, "output_tokens_per_s": 242.1060920375677, "total_tokens_per_min": 3646190.3779133805}, {"request_rate": 1.77, "ttft_p50_ms": 6411.557998499973, "ttlt_p50_ms": 11847.785832499965, "itl_p50_ms": 5.01325112499984, "output_tokens_per_s": 297.10809875712533, "total_tokens_per_min": 4474537.099711935}, {"request_rate": 2.66, "ttft_p50_ms": 11408.403069999991, "ttlt_p50_ms": 15950.940717499976, "itl_p50_ms": 5.0493793749950555, "output_tokens_per_s": 304.46704355933014, "total_tokens_per_min": 4585365.016116579}]}, {"recipe_id": "rec-d049efcf1bc0", "recipe_name": "qwen3-8-flash-next-fp8-sglang-b200x4", "model": "Qwen/Qwen3.8-Flash-Next-FP8", "model_family": "qwen38", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 4, "benchmark_id": "bench-4c9ec9f2cb", "benchmarked_at": "2026-08-26T14:21:36.163475+00:00", "aa_intelligence_index": 39.8, "aa_slug": "qwen3-8-flash-next", "points": [{"request_rate": 1.42, "ttft_p50_ms": 302.52993650003646, "ttlt_p50_ms": 684.7064465000017, "itl_p50_ms": 1.6201228750034602, "output_tokens_per_s": 284.67501167519566, "total_tokens_per_min": 4287291.078331949}, {"request_rate": 2.14, "ttft_p50_ms": 443.95609049996665, "ttlt_p50_ms": 1493.050956500042, "itl_p50_ms": 2.062241749996474, "output_tokens_per_s": 451.04899821713474, "total_tokens_per_min": 6792933.227849514}, {"request_rate": 3.21, "ttft_p50_ms": 483.38387899997315, "ttlt_p50_ms": 1807.7475225000512, "itl_p50_ms": 2.413383000013406, "output_tokens_per_s": 663.5026782228839, "total_tokens_per_min": 9992549.3848401}, {"request_rate": 4.81, "ttft_p50_ms": 551.2810114999525, "ttlt_p50_ms": 1994.3321234999871, "itl_p50_ms": 2.8686795000112397, "output_tokens_per_s": 945.5830873209968, "total_tokens_per_min": 14240764.969980408}, {"request_rate": 7.21, "ttft_p50_ms": 707.7472480000324, "ttlt_p50_ms": 5774.551110000004, "itl_p50_ms": 4.798192249978683, "output_tokens_per_s": 1264.7923722730852, "total_tokens_per_min": 19048152.564144347}]}, {"recipe_id": "rec-13a575625f13", "recipe_name": "muse-glimmer-nvfp4-sglang-b200x1", "model": "RadixArk/Muse-Glimmer-NVFP4", "model_family": "muse_glimmer", "engine": "SGLang", "speculative_draft_model": "meta-models/Muse-Glimmer-30B-assistant", "gpu_type": "B200", "gpu_count": 1, "benchmark_id": "bench-cc914fe398", "benchmarked_at": "2026-09-08T21:17:41.683996+00:00", "aa_intelligence_index": 17.5, "aa_slug": "muse-glimmer", "points": [{"request_rate": 0.59, "ttft_p50_ms": 1115.749997000023, "ttlt_p50_ms": 1689.0747945000157, "itl_p50_ms": 2.334448000010525, "output_tokens_per_s": 118.323450480248, "total_tokens_per_min": 1782022.158302823}, {"request_rate": 0.89, "ttft_p50_ms": 267.8950305000285, "ttlt_p50_ms": 966.5948389999812, "itl_p50_ms": 2.4110306000011406, "output_tokens_per_s": 189.9502746504505, "total_tokens_per_min": 2860765.1064005746}, {"request_rate": 1.33, "ttft_p50_ms": 265.93899350001493, "ttlt_p50_ms": 985.4362604999665, "itl_p50_ms": 2.438123600018116, "output_tokens_per_s": 281.7800953640172, "total_tokens_per_min": 4243777.304239318}, {"request_rate": 2, "ttft_p50_ms": 287.70579349998116, "ttlt_p50_ms": 1153.5286909999627, "itl_p50_ms": 2.600270300001739, "output_tokens_per_s": 419.14502955752914, "total_tokens_per_min": 6312575.632154124}, {"request_rate": 3, "ttft_p50_ms": 352.1363750001001, "ttlt_p50_ms": 1429.9460940000017, "itl_p50_ms": 2.751055599992469, "output_tokens_per_s": 612.0378466680979, "total_tokens_per_min": 9217657.193529556}, {"request_rate": 4.5, "ttft_p50_ms": 430.1755939999339, "ttlt_p50_ms": 1936.3389105000124, "itl_p50_ms": 3.053195800021058, "output_tokens_per_s": 898.459054616196, "total_tokens_per_min": 13531332.43795268}, {"request_rate": 6.75, "ttft_p50_ms": 584.4870304999858, "ttlt_p50_ms": 3579.5328059999747, "itl_p50_ms": 3.3373023999956786, "output_tokens_per_s": 1269.9299158628176, "total_tokens_per_min": 19125906.490843546}]}, {"recipe_id": "rec-aa8953352486", "recipe_name": "inkling-nvfp4-sglang-b300x4-dflash-mrr32", "model": "thinkingmachines/Inkling-NVFP4", "model_family": "inkling", "engine": "SGLang", "speculative_draft_model": "modal-labs/Inkling-NVFP4-DFlash", "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-c089821fad", "benchmarked_at": "2026-07-21T09:46:37.075028+00:00", "aa_intelligence_index": 25.0, "aa_slug": "inkling", "points": [{"request_rate": 0.75, "ttft_p50_ms": 287.97876549992907, "ttlt_p50_ms": 1263.692752499992, "itl_p50_ms": 3.8336954999635964, "output_tokens_per_s": 150.6127666532556, "total_tokens_per_min": 2268273.4496280253}, {"request_rate": 1.13, "ttft_p50_ms": 174.00576099998943, "ttlt_p50_ms": 1444.5016990000568, "itl_p50_ms": 4.542727999989893, "output_tokens_per_s": 239.4097041971726, "total_tokens_per_min": 3605581.9681206783}, {"request_rate": 1.69, "ttft_p50_ms": 185.8360984999763, "ttlt_p50_ms": 1599.0737380000155, "itl_p50_ms": 4.977783000015279, "output_tokens_per_s": 352.64088454842164, "total_tokens_per_min": 5310877.513564594}, {"request_rate": 2.54, "ttft_p50_ms": 208.896123000045, "ttlt_p50_ms": 1782.6254330001348, "itl_p50_ms": 5.168046249991676, "output_tokens_per_s": 519.8829966079527, "total_tokens_per_min": 7829593.893814749}, {"request_rate": 3.81, "ttft_p50_ms": 276.9846484999334, "ttlt_p50_ms": 2193.377134000002, "itl_p50_ms": 5.3504465000173695, "output_tokens_per_s": 755.147956582482, "total_tokens_per_min": 11372754.770519154}, {"request_rate": 5.71, "ttft_p50_ms": 360.55571449992385, "ttlt_p50_ms": 2803.1089680000605, "itl_p50_ms": 6.28631087499798, "output_tokens_per_s": 1017.9672172159412, "total_tokens_per_min": 15330891.681437239}, {"request_rate": 8.56, "ttft_p50_ms": 1160.9485634999146, "ttlt_p50_ms": 3782.164502500109, "itl_p50_ms": 1.8637281875015788, "output_tokens_per_s": 1326.1546189822523, "total_tokens_per_min": 19972286.408258412}]}, {"recipe_id": "rec-ff26f055e1c8", "recipe_name": "glm-4-7-sglang-b200x8", "model": "zai-org/GLM-4.7", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-151a4083a3", "benchmarked_at": "2026-09-18T01:40:58.160049+00:00", "aa_intelligence_index": 22.2, "aa_slug": "glm-4-7", "points": [{"request_rate": 0.66, "ttft_p50_ms": 718.4081204999302, "ttlt_p50_ms": 1520.5589159999704, "itl_p50_ms": 3.389323249990639, "output_tokens_per_s": 132.1246504904763, "total_tokens_per_min": 1989836.8737817204}, {"request_rate": 0.99, "ttft_p50_ms": 306.57792850001897, "ttlt_p50_ms": 1760.0631469999826, "itl_p50_ms": 5.09565875000817, "output_tokens_per_s": 210.2042833183674, "total_tokens_per_min": 3165739.5680596083}, {"request_rate": 1.48, "ttft_p50_ms": 356.33870950005075, "ttlt_p50_ms": 2325.3460214999677, "itl_p50_ms": 6.702994250005645, "output_tokens_per_s": 307.77348445070635, "total_tokens_per_min": 4635161.007872974}, {"request_rate": 2.22, "ttft_p50_ms": 418.90894950000757, "ttlt_p50_ms": 3439.3266299999823, "itl_p50_ms": 8.833264499997995, "output_tokens_per_s": 447.51983149345614, "total_tokens_per_min": 6739782.918240897}, {"request_rate": 3.33, "ttft_p50_ms": 571.3765415000012, "ttlt_p50_ms": 11018.058710499987, "itl_p50_ms": 22.003006874996345, "output_tokens_per_s": 544.5331602312335, "total_tokens_per_min": 8200832.7530304445}, {"request_rate": 5, "ttft_p50_ms": 891.3209869999719, "ttlt_p50_ms": 12028.179673000011, "itl_p50_ms": 34.79734300000814, "output_tokens_per_s": 626.506422333683, "total_tokens_per_min": 9435374.672271965}, {"request_rate": 7.5, "ttft_p50_ms": 1811.8200220000062, "ttlt_p50_ms": 12116.890834000003, "itl_p50_ms": 34.8015005000093, "output_tokens_per_s": 689.5068171462519, "total_tokens_per_min": 10384179.518267697}]}, {"recipe_id": "rec-085d873cbeb4", "recipe_name": "glm-5-2-fp8-sglang-b200x8", "model": "zai-org/GLM-5.2-FP8", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B200", "gpu_count": 8, "benchmark_id": "bench-6d3b90cbb0", "benchmarked_at": "2026-08-25T01:53:04.559932+00:00", "aa_intelligence_index": 33.7, "aa_slug": "glm-5-2", "points": [{"request_rate": 1.15, "ttft_p50_ms": 361.1388634999457, "ttlt_p50_ms": 850.5077809999193, "itl_p50_ms": 1.7872500000066793, "output_tokens_per_s": 230.35682146910935, "total_tokens_per_min": 3469242.8383712275}, {"request_rate": 1.73, "ttft_p50_ms": 862.916918499991, "ttlt_p50_ms": 7013.4752564998735, "itl_p50_ms": 3.481561000009086, "output_tokens_per_s": 361.85455946536445, "total_tokens_per_min": 5449638.2219162285}, {"request_rate": 2.59, "ttft_p50_ms": 549.9201464999715, "ttlt_p50_ms": 2294.9952690000828, "itl_p50_ms": 2.4035252857044855, "output_tokens_per_s": 536.2092085792259, "total_tokens_per_min": 8075471.543965716}, {"request_rate": 3.88, "ttft_p50_ms": 616.2546354998995, "ttlt_p50_ms": 2442.3206864998974, "itl_p50_ms": 3.02742528573877, "output_tokens_per_s": 736.6988751470572, "total_tokens_per_min": 11094906.069377227}, {"request_rate": 5.82, "ttft_p50_ms": 1120.385553999995, "ttlt_p50_ms": 12712.705209500087, "itl_p50_ms": 31.826600285707432, "output_tokens_per_s": 611.6702528031022, "total_tokens_per_min": 9211937.50829056}]}, {"recipe_id": "rec-ec02d67366b4", "recipe_name": "glm-5-3-flash-sglang-b300x4", "model": "zai-org/GLM-5.3-Flash", "model_family": "glm", "engine": "SGLang", "speculative_draft_model": null, "gpu_type": "B300", "gpu_count": 4, "benchmark_id": "bench-dc7e65a5b3", "benchmarked_at": "2026-09-18T20:19:53.572622+00:00", "aa_intelligence_index": 41.8, "aa_slug": "glm-5-3-flash", "points": [{"request_rate": 0.86, "ttft_p50_ms": 345.9867649999069, "ttlt_p50_ms": 951.8511155001761, "itl_p50_ms": 3.066371500040077, "output_tokens_per_s": 171.26664075336976, "total_tokens_per_min": 2579326.9897379745}, {"request_rate": 1.28, "ttft_p50_ms": 401.8383125003311, "ttlt_p50_ms": 1444.837550000102, "itl_p50_ms": 2.4363201666801615, "output_tokens_per_s": 272.1049468661612, "total_tokens_per_min": 4097982.131288447}, {"request_rate": 1.92, "ttft_p50_ms": 497.18913850006174, "ttlt_p50_ms": 1707.3874344998785, "itl_p50_ms": 2.8614551666426755, "output_tokens_per_s": 393.4603525565008, "total_tokens_per_min": 5925630.947606669}, {"request_rate": 2.88, "ttft_p50_ms": 536.5134950000083, "ttlt_p50_ms": 2010.522519999995, "itl_p50_ms": 3.0299848333470436, "output_tokens_per_s": 575.6232025142901, "total_tokens_per_min": 8669058.116825964}, {"request_rate": 4.32, "ttft_p50_ms": 649.8962305001896, "ttlt_p50_ms": 3075.0705140001173, "itl_p50_ms": 3.760276666677479, "output_tokens_per_s": 835.1095068112577, "total_tokens_per_min": 12576999.705429584}, {"request_rate": 6.48, "ttft_p50_ms": 1535.2319569999509, "ttlt_p50_ms": 45358.09196600007, "itl_p50_ms": 31.266119333319388, "output_tokens_per_s": 236.02287486107502, "total_tokens_per_min": 3554575.302270248}]}]}]}