{
  "schema_version": 1,
  "retrieved_utc": "2026-11-11T20:11:07+00:00",
  "method": "One run per model: across GB200 and GB300 NVL72, the AgentX run with the highest output tokens/s per physical GPU above 50 tokens/s/user (reciprocal P90 ITL). Fixed-rate whole-group extrapolation; no quality equivalence or uncached-prefill calibration.",
  "api_contract_url": "https://inferencex.semianalysis.com/api",
  "query_manifest": {
    "DeepSeek-V4-Pro": {
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4-Pro",
      "response_rows": 935,
      "response_sha256": "c59dbd3c1cc7a10632b7eb1207f4619ed31527ac8e106490e1a367cf08026d40"
    },
    "DeepSeek-V4.1-Flash": {
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4.1-Flash",
      "response_rows": 212,
      "response_sha256": "5f2f9938a034b33360037711bba0c21b3e1cf0826fd89491421b756f6ffc2328"
    },
    "GLM-5.2": {
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=GLM-5.2",
      "response_rows": 145,
      "response_sha256": "07d42a3ea071bc6352c9f2bbd5807f437ee9c927aefc97b4e75ec6b991c6fac5"
    },
    "Kimi-K3": {
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=Kimi-K3",
      "response_rows": 114,
      "response_sha256": "855ffc967496da61ba888931e10d6797b8e79bff9f7d329253b9a10c37abc31a"
    },
    "Qwen3.8-Flash-Next": {
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=Qwen3.8-Flash-Next",
      "response_rows": 72,
      "response_sha256": "3c0b61ea6318475f81fafca0a3c842b8ddfdbb09df7140b3ce0b3252c29c20a1"
    }
  },
  "unavailable": [
    {
      "model": "GLM-5.3",
      "hardware": "GB200/GB300 NVL72",
      "reason": "Retrieved runs cover VR200, B300 and MI355X only; GLM-5.2 is shown instead."
    }
  ],
  "points": [
    {
      "result_id": "444884",
      "model": "DeepSeek-V4-Pro",
      "raw_model_key": "dsv4",
      "hardware": "gb300",
      "hardware_label": "GB300 NVL72",
      "gpus_per_rack": 72,
      "rack_basis": "Physical NVL72 GPU count",
      "total_allocated_gpus": 28,
      "input_tps": 4353045.24809,
      "output_tps": 32715.40006,
      "input_tps_per_allocated_gpu": 155465.9017175,
      "output_tps_per_allocated_gpu": 1168.407145,
      "interactivity_p90_tok_s_user": 56.21135469364812,
      "framework": "dynamo-vllm",
      "precision": "fp4",
      "concurrency": 1152,
      "disaggregated": true,
      "date_api": "2026-10-14",
      "run_started_at": "2026-10-14 18:26:38+00",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38198902881/attempts/1",
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4-Pro",
      "mean_input_tokens": 130617.79744,
      "mean_output_tokens_actual": 981.66071,
      "gpu_cache_hit_rate": 0.08123162126006744,
      "external_cache_hit_rate": 0.8856,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "physical_count_note": "Job topology: 2P (TP8/EP8/DPA) x 1D (TP12/EP12/DPA) = 28 GPUs.",
      "architecture_note": "61 layers; 6 of 384 routed experts + 1 shared (≈28B active FFN parameters of ≈1.55T). KV is compressed 4× or 128× along the sequence and each token attends to its top 1,024 entries plus a 128-token window, so decode cost barely grows with context.",
      "raw_row": {
        "id": "444884",
        "hardware": "gb300",
        "framework": "dynamo-vllm",
        "model": "dsv4",
        "precision": "fp4",
        "spec_method": "draft_model",
        "disagg": true,
        "is_multinode": true,
        "prefill_tp": 8,
        "prefill_ep": 8,
        "prefill_dp_attention": true,
        "prefill_num_workers": 2,
        "decode_tp": 12,
        "decode_ep": 12,
        "decode_dp_attention": true,
        "decode_num_workers": 1,
        "num_prefill_gpu": 16,
        "num_decode_gpu": 12,
        "benchmark_type": "agentic_traces",
        "offload_mode": "on",
        "isl": null,
        "osl": null,
        "conc": 1152,
        "image": "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-20ebb28",
        "recipe_fingerprint": "f514a90655abcc85c7ed22f22af584389ec1c14f674e290f98144c5a798b5ee4",
        "metrics": {
          "p75_itl": 0.01703,
          "p75_qps": 39,
          "p90_itl": 0.01779,
          "p90_qps": 44,
          "p95_itl": 0.01843,
          "p95_qps": 47,
          "std_itl": 0.00304,
          "std_qps": 9.04526,
          "mean_itl": 0.01613,
          "mean_qps": 33.33398,
          "p75_e2el": 23.77724,
          "p75_tpot": 0.01702,
          "p75_ttft": 6.70189,
          "p90_e2el": 46.66404,
          "p90_tpot": 0.01776,
          "p90_ttft": 10.80822,
          "p95_e2el": 71.79025,
          "p95_tpot": 0.01838,
          "p95_ttft": 15.58248,
          "std_e2el": 32.93191,
          "std_tpot": 0.00302,
          "std_ttft": 9.49373,
          "decode_pp": 1,
          "mean_e2el": 22.76407,
          "mean_tpot": 0.01611,
          "mean_ttft": 6.84275,
          "median_itl": 0.01634,
          "median_qps": 33,
          "p75_intvty": 58.71990604815032,
          "p90_intvty": 56.21135469364812,
          "p95_intvty": 54.25935973955508,
          "prefill_pp": 1,
          "std_intvty": 46.35055,
          "mean_intvty": 61.996280223186616,
          "median_e2el": 13.35179,
          "median_tpot": 0.01633,
          "median_ttft": 4.92496,
          "power_valid": 0,
          "router_name": "dynamo-router",
          "offload_mode": "on",
          "tput_per_gpu": 156634.30886,
          "kv_offloading": "dram",
          "median_intvty": 61.19951040391677,
          "input_tput_tps": 4353045.24809,
          "router_version": "1.6.0.dev20261010",
          "spec_algorithm": "dspark",
          "total_tput_tps": 4385760.64815,
          "decode_dcp_size": 1,
          "decode_pcp_size": 1,
          "kv_p2p_transfer": "nixl",
          "output_tput_tps": 32715.40006,
          "duration_seconds": 3629.95478,
          "p75_input_tokens": 162000.5,
          "p90_input_tokens": 278600.8,
          "p95_input_tokens": 418724.2,
          "prefill_dcp_size": 1,
          "prefill_pcp_size": 1,
          "spec_draft_model": "native",
          "std_input_tokens": 132606.94306,
          "mean_input_tokens": 130617.79744,
          "input_tput_per_gpu": 272065.32801,
          "kv_offload_backend": "mooncake",
          "median_input_tokens": 91606,
          "output_tput_per_gpu": 2726.28335,
          "total_prompt_tokens": 15862146481,
          "allocated_cpu_dram_gb": 536,
          "p75_full_response_itl": 0.01703,
          "p90_full_response_itl": 0.01779,
          "p95_full_response_itl": 0.01843,
          "std_full_response_itl": 0.00304,
          "gpu_kv_cache_usage_pct": 0.4709,
          "mean_full_response_itl": 0.01613,
          "total_generation_tokens": 119988143,
          "input_tput_per_total_gpu": 155465.90172,
          "median_full_response_itl": 0.01634,
          "p75_full_response_intvty": 58.70714,
          "p75_output_tokens_actual": 995,
          "p90_full_response_intvty": 56.21064,
          "p90_output_tokens_actual": 2265.4,
          "p95_full_response_intvty": 54.27373,
          "p95_output_tokens_actual": 3735.35,
          "std_full_response_intvty": 46.35055,
          "std_output_tokens_actual": 1894.5699,
          "total_requests_completed": 120974,
          "mean_full_response_intvty": 61.98691,
          "mean_output_tokens_actual": 981.66071,
          "output_tput_per_total_gpu": 1168.40715,
          "server_cpu_cache_hit_rate": 0.8855945922061779,
          "server_gpu_cache_hit_rate": 0.08123162126006744,
          "kv_offload_backend_version": "0.3.11.post1",
          "p75_output_tokens_expected": 1349,
          "p90_output_tokens_expected": 3342,
          "p95_output_tokens_expected": 5509.9,
          "std_output_tokens_expected": 2537.82535,
          "theoretical_cache_hit_rate": 0.96867,
          "mean_output_tokens_expected": 1337.34007,
          "median_full_response_intvty": 61.19968,
          "median_output_tokens_actual": 423,
          "power_metric_schema_version": 2,
          "median_output_tokens_expected": 503,
          "server_external_cache_hit_rate": 0.8856
        },
        "workers": null,
        "power_invalid_reasons": null,
        "power_audit": null,
        "date": "2026-10-14",
        "workflow_run_id": "2640",
        "run_started_at": "2026-10-14 18:26:38+00",
        "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38198902881/attempts/1",
        "curve_date": "2026-10-14",
        "curve_workflow_run_id": "2640",
        "curve_run_started_at": "2026-10-14 18:26:38+00"
      }
    },
    {
      "result_id": "447464",
      "model": "DeepSeek-V4.1-Flash",
      "raw_model_key": "dsv41flash",
      "hardware": "gb300",
      "hardware_label": "GB300 NVL72",
      "gpus_per_rack": 72,
      "rack_basis": "Physical NVL72 GPU count",
      "total_allocated_gpus": 34,
      "input_tps": 37579748.63976,
      "output_tps": 275191.22437,
      "input_tps_per_allocated_gpu": 1105286.7246988236,
      "output_tps_per_allocated_gpu": 8093.859540294118,
      "interactivity_p90_tok_s_user": 98.71668311944718,
      "framework": "dynamo-sglang",
      "precision": "fp4",
      "concurrency": 7480,
      "disaggregated": true,
      "date_api": "2026-11-06",
      "run_started_at": "2026-11-06 01:38:55+00",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3",
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4.1-Flash",
      "mean_input_tokens": 139276.78165,
      "mean_output_tokens_actual": 1019.90432,
      "gpu_cache_hit_rate": 0.8807192504469962,
      "external_cache_hit_rate": 0.0819,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "physical_count_note": "Job topology: 9P (TP2/EP2/DPA) x 1D (TP16/EP16/DPA) = 34 GPUs.",
      "architecture_note": "40 layers; 6 of 384 experts + 1 shared (≈10B active FFN parameters of ≈0.54T) with the same compressed, sparse attention as V4-Pro: the cheapest model per token here.",
      "raw_row": {
        "id": "447464",
        "hardware": "gb300",
        "framework": "dynamo-sglang",
        "model": "dsv41flash",
        "precision": "fp4",
        "spec_method": "mtp",
        "disagg": true,
        "is_multinode": true,
        "prefill_tp": 2,
        "prefill_ep": 2,
        "prefill_dp_attention": true,
        "prefill_num_workers": 9,
        "decode_tp": 16,
        "decode_ep": 16,
        "decode_dp_attention": true,
        "decode_num_workers": 1,
        "num_prefill_gpu": 18,
        "num_decode_gpu": 16,
        "benchmark_type": "agentic_traces",
        "offload_mode": "on",
        "isl": null,
        "osl": null,
        "conc": 7480,
        "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579",
        "recipe_fingerprint": "9c5e8fba76df5de86873ecc15d76ce96c5aad8b4ef6558cce0330380fa7c9ed2",
        "metrics": {
          "p75_itl": 0.00964,
          "p75_qps": 301,
          "p90_itl": 0.01013,
          "p90_qps": 323,
          "p95_itl": 0.01108,
          "p95_qps": 337,
          "std_itl": 0.01366,
          "std_qps": 55.17865,
          "mean_itl": 0.00971,
          "mean_qps": 270.18372,
          "p75_e2el": 13.78541,
          "p75_tpot": 0.00962,
          "p75_ttft": 4.16718,
          "p90_e2el": 26.50657,
          "p90_tpot": 0.01009,
          "p90_ttft": 7.56368,
          "p95_e2el": 40.84442,
          "p95_tpot": 0.011,
          "p95_ttft": 10.35366,
          "std_e2el": 18.39726,
          "std_tpot": 0.01344,
          "std_ttft": 4.2459,
          "decode_pp": 1,
          "mean_e2el": 12.69317,
          "mean_tpot": 0.00968,
          "mean_ttft": 3.55429,
          "median_itl": 0.00919,
          "median_qps": 276,
          "p75_intvty": 103.7344398340249,
          "p90_intvty": 98.71668311944718,
          "p95_intvty": 90.25270758122744,
          "prefill_pp": 1,
          "std_intvty": 59.26525,
          "mean_intvty": 102.98661174047373,
          "median_e2el": 7.46566,
          "median_tpot": 0.00918,
          "median_ttft": 2.28285,
          "router_name": "dynamo-router",
          "offload_mode": "on",
          "tput_per_gpu": 1113380.58424,
          "kv_offloading": "dram",
          "median_intvty": 108.8139281828074,
          "input_tput_tps": 37579748.63976,
          "router_version": "1.7.0.dev20261029",
          "total_tput_tps": 37854939.86412,
          "decode_dcp_size": 1,
          "decode_pcp_size": 1,
          "kv_p2p_transfer": "mooncake",
          "output_tput_tps": 275191.22437,
          "duration_seconds": 3629.94486,
          "p75_input_tokens": 176466,
          "p90_input_tokens": 298241,
          "p95_input_tokens": 428970.35,
          "prefill_dcp_size": 1,
          "prefill_pcp_size": 1,
          "std_input_tokens": 137876.29026,
          "mean_input_tokens": 139276.78165,
          "input_tput_per_gpu": 2087763.813322222,
          "kv_offload_backend": "hicache",
          "median_input_tokens": 98298,
          "output_tput_per_gpu": 17199.4515225,
          "p75_e2e_norm_intvty": 42.74434,
          "p90_e2e_norm_intvty": 22.91834,
          "p95_e2e_norm_intvty": 14.30635,
          "std_e2e_norm_intvty": 34.36372,
          "total_prompt_tokens": 272955750606,
          "mean_e2e_norm_intvty": 32.26492,
          "allocated_cpu_dram_gb": 385,
          "p75_full_response_itl": 0.00964,
          "p90_full_response_itl": 0.01013,
          "p95_full_response_itl": 0.01108,
          "std_full_response_itl": 0.01366,
          "gpu_kv_cache_usage_pct": 0.54,
          "mean_full_response_itl": 0.00971,
          "median_e2e_norm_intvty": 68.93088,
          "total_generation_tokens": 1003291240,
          "input_tput_per_total_gpu": 1105286.7247,
          "median_full_response_itl": 0.00919,
          "p75_full_response_intvty": 103.7823,
          "p75_output_tokens_actual": 1020,
          "p90_full_response_intvty": 98.74977,
          "p90_output_tokens_actual": 2368,
          "p95_full_response_intvty": 90.24392,
          "p95_output_tokens_actual": 3903.05,
          "std_full_response_intvty": 59.26525,
          "std_output_tokens_actual": 1985.30979,
          "total_requests_completed": 979434,
          "mean_full_response_intvty": 102.96522,
          "mean_output_tokens_actual": 1019.90432,
          "output_tput_per_total_gpu": 8093.85954,
          "server_cpu_cache_hit_rate": 0.07462904924320077,
          "server_gpu_cache_hit_rate": 0.8807192504469962,
          "p75_output_tokens_expected": 1384,
          "p90_output_tokens_expected": 3408,
          "p95_output_tokens_expected": 5592,
          "std_output_tokens_expected": 2531.98181,
          "theoretical_cache_hit_rate": 0.97075,
          "mean_output_tokens_expected": 1359.8324,
          "median_full_response_intvty": 108.79495,
          "median_output_tokens_actual": 430,
          "decode_allocated_cpu_dram_gb": 771,
          "median_output_tokens_expected": 519,
          "server_external_cache_hit_rate": 0.0819
        },
        "workers": null,
        "power_invalid_reasons": null,
        "power_audit": null,
        "date": "2026-11-06",
        "workflow_run_id": "2913",
        "run_started_at": "2026-11-06 01:38:55+00",
        "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3",
        "curve_date": "2026-11-06",
        "curve_workflow_run_id": "2913",
        "curve_run_started_at": "2026-11-06 01:38:55+00"
      }
    },
    {
      "result_id": "445405",
      "model": "GLM-5.2",
      "raw_model_key": "glm5.2",
      "hardware": "gb300",
      "hardware_label": "GB300 NVL72",
      "gpus_per_rack": 72,
      "rack_basis": "Physical NVL72 GPU count",
      "total_allocated_gpus": 20,
      "input_tps": 1263781.1753,
      "output_tps": 9163.02736,
      "input_tps_per_allocated_gpu": 63189.058765,
      "output_tps_per_allocated_gpu": 458.151368,
      "interactivity_p90_tok_s_user": 89.44543828264759,
      "framework": "dynamo-sglang",
      "precision": "fp4",
      "concurrency": 264,
      "disaggregated": true,
      "date_api": "2026-10-20",
      "run_started_at": "2026-10-20 08:20:56+00",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39187209967/attempts/2",
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=GLM-5.2",
      "mean_input_tokens": 138651.70525,
      "mean_output_tokens_actual": 1005.29221,
      "gpu_cache_hit_rate": 0.643723915504642,
      "external_cache_hit_rate": 0.23865,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "physical_count_note": "Job topology: 3P (TP1/PP4) x 1D (TP8/EP8/DPA) = 20 GPUs.",
      "architecture_note": "78 layers; 8 of 256 experts + 1 shared (≈25B active of ≈0.72T). Sparse attention reads the top 2,048 positions, but full MLA KV is stored for every layer, so long contexts use more memory per sequence than DeepSeek.",
      "raw_row": {
        "id": "445405",
        "hardware": "gb300",
        "framework": "dynamo-sglang",
        "model": "glm5.2",
        "precision": "fp4",
        "spec_method": "mtp",
        "disagg": true,
        "is_multinode": true,
        "prefill_tp": 1,
        "prefill_ep": 1,
        "prefill_dp_attention": false,
        "prefill_num_workers": 3,
        "decode_tp": 8,
        "decode_ep": 8,
        "decode_dp_attention": true,
        "decode_num_workers": 1,
        "num_prefill_gpu": 12,
        "num_decode_gpu": 8,
        "benchmark_type": "agentic_traces",
        "offload_mode": "on",
        "isl": null,
        "osl": null,
        "conc": 264,
        "image": "lmsysorg/sglang:nightly-dev-20261016-a93f16f0@sha256:0b3e39704480c423d8a00ca15ebee63154defc62944df2d4b11c594563bab457",
        "recipe_fingerprint": "d7ae180cac162cdf40e0aa2ba6af9b64a236363ed4713ae3707ddae140b3fb22",
        "metrics": {
          "p75_itl": 0.01064,
          "p75_qps": 12,
          "p90_itl": 0.01118,
          "p90_qps": 15,
          "p95_itl": 0.01196,
          "p95_qps": 17,
          "std_itl": 0.02489,
          "std_qps": 4.30169,
          "mean_itl": 0.0111,
          "mean_qps": 9.27762,
          "p75_e2el": 16.02719,
          "p75_tpot": 0.01062,
          "p75_ttft": 4.10567,
          "p90_e2el": 32.98868,
          "p90_tpot": 0.01115,
          "p90_ttft": 9.43216,
          "p95_e2el": 49.01831,
          "p95_tpot": 0.01187,
          "p95_ttft": 16.23526,
          "std_e2el": 22.37354,
          "std_tpot": 0.02489,
          "std_ttft": 7.13663,
          "decode_pp": 1,
          "mean_e2el": 14.59325,
          "mean_tpot": 0.01107,
          "mean_ttft": 4.33651,
          "median_itl": 0.01035,
          "median_qps": 9,
          "p75_intvty": 93.98496240601504,
          "p90_intvty": 89.44543828264759,
          "p95_intvty": 83.61204013377926,
          "prefill_pp": 4,
          "std_intvty": 57.19499,
          "mean_intvty": 90.09009009009009,
          "median_e2el": 7.9005,
          "median_tpot": 0.01034,
          "median_ttft": 2.07217,
          "router_name": "dynamo-router",
          "offload_mode": "on",
          "tput_per_gpu": 63647.21013,
          "kv_offloading": "dram",
          "median_intvty": 96.61835748792271,
          "input_tput_tps": 1263781.1753,
          "router_version": "1.6.0.dev20261016",
          "spec_algorithm": "mtp",
          "total_tput_tps": 1272944.20266,
          "decode_dcp_size": 1,
          "decode_pcp_size": 1,
          "kv_p2p_transfer": "nixl",
          "output_tput_tps": 9163.02736,
          "duration_seconds": 3629.92455,
          "p75_input_tokens": 173767.5,
          "p90_input_tokens": 290707,
          "p95_input_tokens": 436688,
          "prefill_dcp_size": 1,
          "prefill_pcp_size": 1,
          "spec_draft_model": "native",
          "std_input_tokens": 135587.36594,
          "mean_input_tokens": 138651.70525,
          "input_tput_per_gpu": 105315.09795000001,
          "kv_offload_backend": "hicache",
          "median_input_tokens": 98666,
          "output_tput_per_gpu": 1145.3784249999999,
          "total_prompt_tokens": 9722715378,
          "allocated_cpu_dram_gb": 819,
          "p75_full_response_itl": 0.01064,
          "p90_full_response_itl": 0.01118,
          "p95_full_response_itl": 0.01196,
          "std_full_response_itl": 0.02489,
          "gpu_kv_cache_usage_pct": 1,
          "mean_full_response_itl": 0.0111,
          "total_generation_tokens": 33394801,
          "input_tput_per_total_gpu": 63189.05877,
          "median_full_response_itl": 0.01035,
          "p75_full_response_intvty": 93.95655,
          "p75_output_tokens_actual": 998,
          "p90_full_response_intvty": 89.42647,
          "p90_output_tokens_actual": 2348.5,
          "p95_full_response_intvty": 83.61606,
          "p95_output_tokens_actual": 3826.75,
          "std_full_response_intvty": 57.19499,
          "std_output_tokens_actual": 2073.07151,
          "total_requests_completed": 33086,
          "mean_full_response_intvty": 90.0875,
          "mean_output_tokens_actual": 1005.29221,
          "output_tput_per_total_gpu": 458.15137,
          "server_cpu_cache_hit_rate": 0.30822627131055375,
          "server_gpu_cache_hit_rate": 0.643723915504642,
          "p75_output_tokens_expected": 1360,
          "p90_output_tokens_expected": 3321,
          "p95_output_tokens_expected": 5279.75,
          "std_output_tokens_expected": 2276.70364,
          "theoretical_cache_hit_rate": 0.97034,
          "mean_output_tokens_expected": 1292.73078,
          "median_full_response_intvty": 96.60439,
          "median_output_tokens_actual": 416,
          "decode_allocated_cpu_dram_gb": 819,
          "median_output_tokens_expected": 523,
          "server_external_cache_hit_rate": 0.23865
        },
        "workers": null,
        "power_invalid_reasons": null,
        "power_audit": null,
        "date": "2026-10-20",
        "workflow_run_id": "2690",
        "run_started_at": "2026-10-20 08:20:56+00",
        "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39187209967/attempts/2",
        "curve_date": "2026-10-20",
        "curve_workflow_run_id": "2690",
        "curve_run_started_at": "2026-10-20 08:20:56+00"
      }
    },
    {
      "result_id": "446753",
      "model": "Kimi K3",
      "raw_model_key": "kimik3",
      "hardware": "gb300",
      "hardware_label": "GB300 NVL72",
      "gpus_per_rack": 72,
      "rack_basis": "Physical NVL72 GPU count",
      "total_allocated_gpus": 40,
      "input_tps": 968392.38943,
      "output_tps": 7104.23182,
      "input_tps_per_allocated_gpu": 24209.80973575,
      "output_tps_per_allocated_gpu": 177.6057955,
      "interactivity_p90_tok_s_user": 58.30903790087464,
      "framework": "dynamo-vllm",
      "precision": "fp4",
      "concurrency": 240,
      "disaggregated": true,
      "date_api": "2026-10-27",
      "run_started_at": "2026-10-27 21:08:00+00",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1",
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=Kimi-K3",
      "mean_input_tokens": 135837.55081,
      "mean_output_tokens_actual": 996.51904,
      "gpu_cache_hit_rate": 0.06750062160384929,
      "external_cache_hit_rate": 0.90492,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "physical_count_note": "Job topology: 2P (TP8/DCP8/EP8) x 3D (TP8/DCP8) = 40 GPUs.",
      "architecture_note": "Why so many racks: ≈4.5× DeepSeek V4-Pro’s compute per token and far more KV traffic. 93 layers activate 16 of 896 experts (3,584 wide) + 2 shared, ≈128B active FFN parameters of ≈6.3T. Its hybrid “Kimi Linear” attention keeps 24 full-attention MLA layers, which read the whole ~136k-token context every decode step (≈1.9GB per sequence at fp8, vs ≈0.2GB for V4-Pro’s sparse attention). Fewer sequences fit per GPU (≈6 vs ≈41 at ~55 tok/s/user), and speed falls below 50 tok/s/user beyond 240 concurrent requests.",
      "raw_row": {
        "id": "446753",
        "hardware": "gb300",
        "framework": "dynamo-vllm",
        "model": "kimik3",
        "precision": "fp4",
        "spec_method": "mtp",
        "disagg": true,
        "is_multinode": true,
        "prefill_tp": 8,
        "prefill_ep": 8,
        "prefill_dp_attention": false,
        "prefill_num_workers": 2,
        "decode_tp": 8,
        "decode_ep": 1,
        "decode_dp_attention": false,
        "decode_num_workers": 3,
        "num_prefill_gpu": 16,
        "num_decode_gpu": 24,
        "benchmark_type": "agentic_traces",
        "offload_mode": "on",
        "isl": null,
        "osl": null,
        "conc": 240,
        "image": "vllm/vllm-openai:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d@sha256:6b90105076216e861cd30fae8828efef9263a0399137b894289f3b6a685bf4ff",
        "recipe_fingerprint": "46ff216adf37881b9df632b52d6e599b0c647e5238fd77e11f446cd1e12c584d",
        "metrics": {
          "p75_itl": 0.01647,
          "p75_qps": 9,
          "p90_itl": 0.01715,
          "p90_qps": 11,
          "p95_itl": 0.01757,
          "p95_qps": 12,
          "std_itl": 0.00259,
          "std_qps": 3.10981,
          "mean_itl": 0.01501,
          "mean_qps": 7.13065,
          "p75_e2el": 21.78874,
          "p75_tpot": 0.01646,
          "p75_ttft": 6.19623,
          "p90_e2el": 44.11667,
          "p90_tpot": 0.01714,
          "p90_ttft": 10.6979,
          "p95_e2el": 67.50228,
          "p95_tpot": 0.01756,
          "p95_ttft": 15.43109,
          "std_e2el": 32.41802,
          "std_tpot": 0.00233,
          "std_ttft": 6.18772,
          "decode_pp": 1,
          "mean_e2el": 20.7221,
          "mean_tpot": 0.01509,
          "mean_ttft": 5.36007,
          "median_itl": 0.01563,
          "median_qps": 7,
          "p75_intvty": 60.71645415907712,
          "p90_intvty": 58.30903790087464,
          "p95_intvty": 56.91519635742744,
          "prefill_pp": 1,
          "std_intvty": 343.72682,
          "avg_power_w": 1043.841,
          "mean_intvty": 66.62225183211193,
          "median_e2el": 11.65989,
          "median_tpot": 0.01563,
          "median_ttft": 3.59894,
          "p75_power_w": 1103.154,
          "p90_power_w": 1123.57,
          "power_valid": 1,
          "router_name": "dynamo-router",
          "offload_mode": "on",
          "tput_per_gpu": 24387.41553,
          "kv_offloading": "dram",
          "median_intvty": 63.97952655150351,
          "input_tput_tps": 968392.38943,
          "router_version": "a0de541ea8c861d5ac01d15953ca529bea68637b",
          "spec_algorithm": "dspark",
          "total_tput_tps": 975496.62125,
          "decode_dcp_size": 8,
          "decode_pcp_size": 1,
          "kv_p2p_transfer": "nixl",
          "output_tput_tps": 7104.23182,
          "duration_seconds": 3628.9559,
          "p75_input_tokens": 166256.5,
          "p90_input_tokens": 283308,
          "p95_input_tokens": 418481,
          "prefill_dcp_size": 8,
          "prefill_pcp_size": 1,
          "spec_draft_model": "Inferact/Kimi-K3-DSpark",
          "std_input_tokens": 133536.57167,
          "mean_input_tokens": 135837.55081,
          "decode_avg_power_w": 1035.686,
          "input_tput_per_gpu": 60524.52435,
          "kv_offload_backend": "mooncake",
          "total_gpu_energy_j": 151983422.768,
          "decode_gpu_energy_j": 90477625.24,
          "median_input_tokens": 97590,
          "output_tput_per_gpu": 296.00966666666665,
          "prefill_avg_power_w": 1056.074,
          "total_prompt_tokens": 3524111837,
          "prefill_gpu_energy_j": 61505797.528,
          "allocated_cpu_dram_gb": 171,
          "avg_total_gpu_power_w": 41753.65,
          "p75_full_response_itl": 0.01647,
          "p75_total_gpu_power_w": 44126.144,
          "p90_full_response_itl": 0.01715,
          "p90_total_gpu_power_w": 44942.818,
          "p95_full_response_itl": 0.01757,
          "std_full_response_itl": 0.00259,
          "gpu_kv_cache_usage_pct": 0.93348,
          "joules_per_input_token": 0.043248,
          "joules_per_total_token": 0.042933,
          "mean_full_response_itl": 0.01501,
          "joules_per_output_token": 5.895185,
          "total_generation_tokens": 26051241,
          "input_tput_per_total_gpu": 24209.80974,
          "median_full_response_itl": 0.01563,
          "p75_full_response_intvty": 60.72627,
          "p75_output_tokens_actual": 962,
          "p90_full_response_intvty": 58.30768,
          "p90_output_tokens_actual": 2301,
          "p95_full_response_intvty": 56.91252,
          "p95_output_tokens_actual": 3833,
          "std_full_response_intvty": 343.72682,
          "std_output_tokens_actual": 2038.96832,
          "total_requests_completed": 25871,
          "mean_full_response_intvty": 66.6087,
          "mean_output_tokens_actual": 996.51904,
          "output_tput_per_total_gpu": 177.6058,
          "server_cpu_cache_hit_rate": 0.9025163572825313,
          "server_gpu_cache_hit_rate": 0.06750062160384929,
          "kv_offload_backend_version": "0.3.13.post1",
          "p75_output_tokens_expected": 1374.25,
          "p90_output_tokens_expected": 3432,
          "p95_output_tokens_expected": 5498,
          "std_output_tokens_expected": 2371.97384,
          "theoretical_cache_hit_rate": 0.96946,
          "joules_per_successful_query": 5874.66363,
          "mean_output_tokens_expected": 1333.32808,
          "median_full_response_intvty": 63.96077,
          "median_output_tokens_actual": 409,
          "power_metric_schema_version": 2,
          "decode_allocated_cpu_dram_gb": 171,
          "median_output_tokens_expected": 528,
          "decode_joules_per_output_token": 3.509477,
          "prefill_joules_per_input_token": 0.017502,
          "server_external_cache_hit_rate": 0.90492
        },
        "workers": null,
        "power_invalid_reasons": null,
        "power_audit": {
          "source": "power_validation_kimik3_p2x8ep8_d3x8_conc240_kvdram-mooncake_fp4_dynamo-vllm_prefill-tp8-pp1-dcp8-pcp1-ep8-dpfalse-n-ae070f26395399d7d466_conc240.json",
          "window_end_unix": 1792988961.190611,
          "window_start_unix": 1792985321.18734
        },
        "date": "2026-10-27",
        "workflow_run_id": "2840",
        "run_started_at": "2026-10-27 21:08:00+00",
        "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1",
        "curve_date": "2026-10-27",
        "curve_workflow_run_id": "2840",
        "curve_run_started_at": "2026-10-27 21:08:00+00"
      }
    },
    {
      "result_id": "447843",
      "model": "Qwen3.8-Flash-Next",
      "raw_model_key": "qwen3.8next",
      "hardware": "gb300",
      "hardware_label": "GB300 NVL72",
      "gpus_per_rack": 72,
      "rack_basis": "Physical NVL72 GPU count",
      "total_allocated_gpus": 1,
      "input_tps": 237863.63951,
      "output_tps": 2506.14468,
      "input_tps_per_allocated_gpu": 237863.63951,
      "output_tps_per_allocated_gpu": 2506.14468,
      "interactivity_p90_tok_s_user": 77.8816199376947,
      "framework": "tokenspeed",
      "precision": "fp4",
      "concurrency": 56,
      "disaggregated": false,
      "date_api": "2026-11-10",
      "run_started_at": "2026-11-10 03:50:51+00",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42593696369/attempts/2",
      "api_url": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=Qwen3.8-Flash-Next",
      "mean_input_tokens": 88040.2941,
      "mean_output_tokens_actual": 927.59749,
      "gpu_cache_hit_rate": null,
      "external_cache_hit_rate": null,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "physical_count_note": "Job topology: one TP1 engine = 1 GPU; API workers and power audit both confirm one physical GPU.",
      "architecture_note": "48 layers; 10 of 512 small experts + 1 shared (≈2.6B active), linear attention in 3 of every 4 layers. A small model; its selected NVL72 run uses a single GPU, so this row reflects that run’s limits more than the architecture.",
      "raw_row": {
        "id": "447843",
        "hardware": "gb300",
        "framework": "tokenspeed",
        "model": "qwen3.8next",
        "precision": "fp4",
        "spec_method": "mtp",
        "disagg": false,
        "is_multinode": false,
        "prefill_tp": 1,
        "prefill_ep": 1,
        "prefill_dp_attention": false,
        "prefill_num_workers": 0,
        "decode_tp": 1,
        "decode_ep": 1,
        "decode_dp_attention": false,
        "decode_num_workers": 0,
        "num_prefill_gpu": 1,
        "num_decode_gpu": 1,
        "benchmark_type": "agentic_traces",
        "offload_mode": "on",
        "isl": null,
        "osl": null,
        "conc": 56,
        "image": "ghcr.io/lightseekorg/tokenspeed-runner:cu130-torch-2.14.0-flashinfer-0.7.3@sha256:e359ef0b3d761357f52bd3675fcc831896288448ee06b11807d09f237ec8b3bd",
        "recipe_fingerprint": "c8517d9acfb0889a98630d595013d0fdae585477f9608d103c9a6f3679127d9e",
        "metrics": {
          "pp": 1,
          "p75_itl": 0.01094,
          "p75_qps": 4,
          "p90_itl": 0.01284,
          "p90_qps": 5,
          "p95_itl": 0.01465,
          "p95_qps": 6,
          "std_itl": 0.00466,
          "std_qps": 1.92432,
          "dcp_size": 1,
          "mean_itl": 0.00957,
          "mean_qps": 2.70431,
          "p75_e2el": 10.38792,
          "p75_tpot": 0.01094,
          "p75_ttft": 0.97119,
          "p90_e2el": 20.35889,
          "p90_tpot": 0.01284,
          "p90_ttft": 2.98577,
          "p95_e2el": 33.82996,
          "p95_tpot": 0.01465,
          "p95_ttft": 4.07745,
          "pcp_size": 1,
          "std_e2el": 16.65666,
          "std_tpot": 0.00466,
          "std_ttft": 1.42057,
          "mean_e2el": 9.41313,
          "mean_tpot": 0.00957,
          "mean_ttft": 0.86459,
          "median_itl": 0.00894,
          "median_qps": 3,
          "p75_intvty": 91.40767824497257,
          "p90_intvty": 77.8816199376947,
          "p95_intvty": 68.25938566552901,
          "std_intvty": 37.87794,
          "avg_power_w": 904.352,
          "mean_intvty": 104.4932079414838,
          "median_e2el": 4.84188,
          "median_tpot": 0.00894,
          "median_ttft": 0.15461,
          "p75_power_w": 940.066,
          "p90_power_w": 1010.354,
          "power_valid": 1,
          "router_name": "smg",
          "offload_mode": "on",
          "tput_per_gpu": 240369.78419,
          "kv_offloading": "dram",
          "median_intvty": 111.85682326621924,
          "input_tput_tps": 237863.63951,
          "router_version": "1.13.0.post20261027",
          "spec_algorithm": "mtp",
          "total_tput_tps": 240369.78419,
          "cpu_power_valid": 1,
          "output_tput_tps": 2506.14468,
          "duration_seconds": 3625.78628,
          "p75_input_tokens": 126100,
          "p90_input_tokens": 190768.5,
          "p95_input_tokens": 224676.25,
          "spec_draft_model": "native",
          "std_input_tokens": 64742.21113,
          "mean_input_tokens": 88040.2941,
          "input_tput_per_gpu": 237863.63951,
          "kv_offload_backend": "kvstore",
          "total_cpu_energy_j": 673550.842,
          "total_gpu_energy_j": 3278997.166,
          "median_input_tokens": 72498,
          "output_tput_per_gpu": 2506.14468,
          "p75_e2e_norm_intvty": 72.38732,
          "p90_e2e_norm_intvty": 46.51172,
          "p95_e2e_norm_intvty": 32.7081,
          "std_e2e_norm_intvty": 42.54552,
          "total_prompt_tokens": 862442721,
          "mean_e2e_norm_intvty": 63.33017,
          "allocated_cpu_dram_gb": 241,
          "avg_total_cpu_power_w": 185.766,
          "avg_total_gpu_power_w": 904.352,
          "p75_full_response_itl": 0.01094,
          "p75_total_gpu_power_w": 940.066,
          "p90_full_response_itl": 0.01284,
          "p90_total_gpu_power_w": 1010.354,
          "p95_full_response_itl": 0.01465,
          "std_full_response_itl": 0.00466,
          "avg_cpu_socket_power_w": 92.883,
          "gpu_kv_cache_usage_pct": 0.74116,
          "joules_per_input_token": 0.003802,
          "joules_per_total_token": 0.003762,
          "mean_full_response_itl": 0.00957,
          "median_e2e_norm_intvty": 101.45532,
          "joules_per_output_token": 0.360855,
          "total_cpu_rail_energy_j": 320509.785,
          "total_generation_tokens": 9086745,
          "median_full_response_itl": 0.00894,
          "p75_full_response_intvty": 91.44567,
          "p75_output_tokens_actual": 1000,
          "p90_full_response_intvty": 77.86941,
          "p90_output_tokens_actual": 2100.5,
          "p95_full_response_intvty": 68.25954,
          "p95_output_tokens_actual": 3489,
          "std_full_response_intvty": 37.87794,
          "std_output_tokens_actual": 1800.20979,
          "total_cpu_sysio_energy_j": 43081.841,
          "total_requests_completed": 9796,
          "mean_full_response_intvty": 104.44611,
          "mean_output_tokens_actual": 927.59749,
          "avg_total_cpu_rail_power_w": 88.397,
          "p75_output_tokens_expected": 1259,
          "p90_output_tokens_expected": 3233.4,
          "p95_output_tokens_expected": 5127.8,
          "std_output_tokens_expected": 2134.58729,
          "theoretical_cache_hit_rate": 0.9546,
          "avg_total_cpu_sysio_power_w": 11.882,
          "joules_per_successful_query": 334.728171,
          "mean_output_tokens_expected": 1217.75348,
          "median_full_response_intvty": 111.89259,
          "median_output_tokens_actual": 418,
          "power_metric_schema_version": 2,
          "median_output_tokens_expected": 506
        },
        "workers": [
          {
            "role": "agg",
            "hosts": [
              "im-gb300-r03-c005"
            ],
            "num_gpus": 1,
            "worker_idx": 0,
            "avg_power_w": 904.352
          }
        ],
        "power_invalid_reasons": null,
        "power_audit": {
          "cpu": {
            "source": "acpi",
            "sensor_kind": "grace_socket",
            "reason_codes": [],
            "expected_sockets": 2,
            "observed_sockets": 2,
            "sample_row_count": 10168
          },
          "source": "power_validation_qwen3.8next_tp1_conc56_kvdram-kvstore_spec-mtp_fp4_tokenspeed_tp1-pp1-dcp1-pcp1-ep1-dpafalse_disagg-6370bdeacf02f4c67895.json",
          "producer_sha": "acfb4dcc3bd48fd6a1e7b615e849e7954c0c3852",
          "window_end_unix": 1794261599.339789,
          "max_sample_gap_s": 1.001521110534668,
          "observed_gpu_ids": [
            "im-gb300-r03-c005/GPU-67696646-a6a3-2414-7ad2-0e94ab9f2189"
          ],
          "window_start_unix": 1794257973.541706,
          "expected_gpu_count": 1,
          "observed_gpu_count": 1,
          "exporter_image_sha256": "b726092000b9dbbec030d6326c514bc33d049f08ef4db04ea2fe2eac95ae3ebf"
        },
        "date": "2026-11-10",
        "workflow_run_id": "2953",
        "run_started_at": "2026-11-10 03:50:51+00",
        "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42593696369/attempts/2",
        "curve_date": "2026-11-10",
        "curve_workflow_run_id": "2953",
        "curve_run_started_at": "2026-11-10 03:50:51+00"
      }
    }
  ]
}
