{
  "retrieved_utc": "2026-11-02T21:42:00.302387+00:00",
  "api_urls": {
    "DeepSeek-V4-Pro": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4-Pro",
    "DeepSeek-V4.1-Flash": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4.1-Flash"
  },
  "notes": [
    "AgentX input counts include reused prefixes: input TPS is not uncached prefill compute TPS.",
    "Aggregate Flash rows mirror 4 GPUs in both P/D columns; physical count is 4, not 8.",
    "Pro per-GPU input/output/total already divide by P+D.",
    "100 tok/s/user headlines are spline interpolation, not single measured points; P90 ITL reciprocal defines target.",
    "API date field and GitHub run metadata differ; preserve both rather than treating API date as measurement date."
  ],
  "points": [
    {
      "result_id": "444914",
      "model": "DeepSeek-V4-Pro",
      "hardware": "gb200",
      "date_api": "2026-10-14",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36425419107/attempts/3",
      "workflow_db_id": "2641",
      "recipe_fingerprint": "a8b5659f936d126934f8e3ea4719f963228a4d78d6cd78021226cdc9d2109d2c",
      "framework": "dynamo-vllm",
      "precision": "fp4",
      "spec_method": "draft_model",
      "concurrency": 128,
      "disaggregated": true,
      "total_allocated_gpus": 16,
      "prefill_gpus": 8,
      "decode_gpus": 8,
      "aggregate_engine_gpus": null,
      "per_gpu_denominator": "all allocated physical GPUs",
      "interactivity_p90_tok_s_user": 104.60251046025104,
      "interactivity_median_tok_s_user": 112.48593925759279,
      "total_tps": 618484.1601,
      "input_tps": 613533.27093,
      "output_tps": 4950.88918,
      "total_tps_per_allocated_gpu": 38655.26001,
      "input_tps_per_allocated_gpu": 38345.82943,
      "output_tps_per_allocated_gpu": 309.43057,
      "mean_input_tokens": 129515.21396,
      "median_input_tokens": 91623.5,
      "p90_input_tokens": 270987.6,
      "mean_output_tokens_actual": 1045.11931,
      "median_output_tokens_actual": 469,
      "p90_output_tokens_actual": 2353.1,
      "gpu_cache_hit_rate": 0.26306,
      "cpu_cache_hit_rate": 0.70983,
      "external_cache_hit_rate": 0.70983,
      "theoretical_cache_hit_rate": 0.96889,
      "offload_mode": "off",
      "kv_offloading": "none",
      "kv_offload_backend": null,
      "image": "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-20ebb28",
      "isolated_uncached_prefill_capacity_tps_per_gpu": null,
      "context_limit_tokens_engine_log": 1048576,
      "kv_precision_engine_log": "fp8",
      "derived_output_tps_per_decode_gpu": 618.8611475,
      "role_note": "API P/D TP fields encode serving topology; worker logs show TP1 with data/expert parallelism. Do not multiply TP by EP to infer physical count.",
      "cache_note": "CPU/external rate may denote external KV transfer. Offload_mode off and allocated_cpu_dram_gb 0; do not infer CPU offload or sum CPU and external rates.",
      "precision_note": "API fp4; inspected engine log quantization=deepseek_v4_fp8 and dtype=bfloat16; precision labels unresolved, not uniform FP4 arithmetic.",
      "hypothetical_same_group_rack_packing": {
        "rack_gpus": 72,
        "whole_groups": 4,
        "allocated_gpus": 64,
        "unused_gpus": 8,
        "output_tps": 19803.55672,
        "scope": "arithmetic extrapolation with identical replay and independent groups; not a measured full rack"
      }
    },
    {
      "result_id": "444911",
      "model": "DeepSeek-V4-Pro",
      "hardware": "gb200",
      "date_api": "2026-10-14",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36425419107/attempts/3",
      "workflow_db_id": "2641",
      "recipe_fingerprint": "2bae653797b283a2acc89cd5616fb9abb222a485035cbb33887229da08debbcc",
      "framework": "dynamo-vllm",
      "precision": "fp4",
      "spec_method": "draft_model",
      "concurrency": 360,
      "disaggregated": true,
      "total_allocated_gpus": 20,
      "prefill_gpus": 8,
      "decode_gpus": 12,
      "aggregate_engine_gpus": null,
      "per_gpu_denominator": "all allocated physical GPUs",
      "interactivity_p90_tok_s_user": 95.78544061302682,
      "interactivity_median_tok_s_user": 101.52284263959392,
      "total_tps": 1701318.64019,
      "input_tps": 1688667.64671,
      "output_tps": 12650.99349,
      "total_tps_per_allocated_gpu": 85065.93201,
      "input_tps_per_allocated_gpu": 84433.38234,
      "output_tps_per_allocated_gpu": 632.54967,
      "mean_input_tokens": 134298.4109,
      "median_input_tokens": 93292,
      "p90_input_tokens": 284512.1,
      "mean_output_tokens_actual": 1006.12357,
      "median_output_tokens_actual": 430,
      "p90_output_tokens_actual": 2328,
      "gpu_cache_hit_rate": 0.10242614337544448,
      "cpu_cache_hit_rate": 0.8657066256024453,
      "external_cache_hit_rate": 0.86567,
      "theoretical_cache_hit_rate": 0.96981,
      "offload_mode": "off",
      "kv_offloading": "none",
      "kv_offload_backend": null,
      "image": "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-20ebb28",
      "isolated_uncached_prefill_capacity_tps_per_gpu": null,
      "context_limit_tokens_engine_log": 1048576,
      "kv_precision_engine_log": "fp8",
      "derived_output_tps_per_decode_gpu": 1054.2494575,
      "role_note": "API P/D TP fields encode serving topology; worker logs show TP1 with data/expert parallelism. Do not multiply TP by EP to infer physical count.",
      "cache_note": "CPU/external rate may denote external KV transfer. Offload_mode off and allocated_cpu_dram_gb 0; do not infer CPU offload or sum CPU and external rates.",
      "precision_note": "API fp4; inspected engine log quantization=deepseek_v4_fp8 and dtype=bfloat16; precision labels unresolved, not uniform FP4 arithmetic.",
      "hypothetical_same_group_rack_packing": {
        "rack_gpus": 72,
        "whole_groups": 3,
        "allocated_gpus": 60,
        "unused_gpus": 12,
        "output_tps": 37952.98047,
        "scope": "arithmetic extrapolation with identical replay and independent groups; not a measured full rack"
      }
    },
    {
      "result_id": "444212",
      "model": "DeepSeek-V4.1-Flash",
      "hardware": "gb200",
      "date_api": "2026-10-07",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37173401305/attempts/4",
      "workflow_db_id": "2602",
      "recipe_fingerprint": "a0d6c3c06f5b5f83c37cd56dd7a870fa8a05dd82f5a00f90dde7013f750ea586",
      "framework": "vllm",
      "precision": "fp4",
      "spec_method": "mtp",
      "concurrency": 64,
      "disaggregated": false,
      "total_allocated_gpus": 4,
      "prefill_gpus": null,
      "decode_gpus": null,
      "aggregate_engine_gpus": 4,
      "per_gpu_denominator": "all allocated physical GPUs",
      "interactivity_p90_tok_s_user": 132.4503311258278,
      "interactivity_median_tok_s_user": 198.01980198019803,
      "total_tps": 466209.0557,
      "input_tps": 462963.95725,
      "output_tps": 3245.09845,
      "total_tps_per_allocated_gpu": 116552.26393,
      "input_tps_per_allocated_gpu": 115740.98931,
      "output_tps_per_allocated_gpu": 811.27461,
      "mean_input_tokens": 146421.23793,
      "median_input_tokens": 87876,
      "p90_input_tokens": 342432.2,
      "mean_output_tokens_actual": 1026.32467,
      "median_output_tokens_actual": 461.5,
      "p90_output_tokens_actual": 2350.1,
      "gpu_cache_hit_rate": 0.9596622140923428,
      "cpu_cache_hit_rate": 0,
      "external_cache_hit_rate": 0,
      "theoretical_cache_hit_rate": 0.97337,
      "offload_mode": "off",
      "kv_offloading": "none",
      "kv_offload_backend": null,
      "image": "vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7",
      "isolated_uncached_prefill_capacity_tps_per_gpu": null,
      "context_limit_tokens_engine_log": 1048576,
      "kv_precision_engine_log": "fp8",
      "cache_note": "P/D GPU fields mirror one aggregate engine; do not sum them.",
      "speculative_note": "vLLM DSpark five draft tokens, synthetic rejection, synthetic_acceptance_length 3.51; adaptive verification enabled (verified on row444210 engine log; same recipe fingerprint for row444212).",
      "hypothetical_same_group_rack_packing": {
        "rack_gpus": 72,
        "whole_groups": 18,
        "allocated_gpus": 72,
        "unused_gpus": 0,
        "output_tps": 58411.7721,
        "scope": "arithmetic extrapolation with identical replay and independent groups; not a measured full rack"
      }
    },
    {
      "result_id": "444210",
      "model": "DeepSeek-V4.1-Flash",
      "hardware": "gb200",
      "date_api": "2026-10-07",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37173401305/attempts/4",
      "workflow_db_id": "2602",
      "recipe_fingerprint": "a0d6c3c06f5b5f83c37cd56dd7a870fa8a05dd82f5a00f90dde7013f750ea586",
      "framework": "vllm",
      "precision": "fp4",
      "spec_method": "mtp",
      "concurrency": 128,
      "disaggregated": false,
      "total_allocated_gpus": 4,
      "prefill_gpus": null,
      "decode_gpus": null,
      "aggregate_engine_gpus": 4,
      "per_gpu_denominator": "all allocated physical GPUs",
      "interactivity_p90_tok_s_user": 85.6898029134533,
      "interactivity_median_tok_s_user": 134.77088948787062,
      "total_tps": 702473.31615,
      "input_tps": 696973.17057,
      "output_tps": 5500.14558,
      "total_tps_per_allocated_gpu": 175618.32904,
      "input_tps_per_allocated_gpu": 174243.29264,
      "output_tps_per_allocated_gpu": 1375.0364,
      "mean_input_tokens": 132906.47682,
      "median_input_tokens": 93105,
      "p90_input_tokens": 279171.2,
      "mean_output_tokens_actual": 1048.82799,
      "median_output_tokens_actual": 468,
      "p90_output_tokens_actual": 2363,
      "gpu_cache_hit_rate": 0.9523450330453175,
      "cpu_cache_hit_rate": 0,
      "external_cache_hit_rate": 0,
      "theoretical_cache_hit_rate": 0.96984,
      "offload_mode": "off",
      "kv_offloading": "none",
      "kv_offload_backend": null,
      "image": "vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7",
      "isolated_uncached_prefill_capacity_tps_per_gpu": null,
      "context_limit_tokens_engine_log": 1048576,
      "kv_precision_engine_log": "fp8",
      "cache_note": "P/D GPU fields mirror one aggregate engine; do not sum them.",
      "speculative_note": "vLLM DSpark five draft tokens, synthetic rejection, synthetic_acceptance_length 3.51; adaptive verification enabled (verified on row444210 engine log; same recipe fingerprint for row444212).",
      "hypothetical_same_group_rack_packing": {
        "rack_gpus": 72,
        "whole_groups": 18,
        "allocated_gpus": 72,
        "unused_gpus": 0,
        "output_tps": 99002.62044,
        "scope": "arithmetic extrapolation with identical replay and independent groups; not a measured full rack"
      }
    },
    {
      "result_id": "446626",
      "model": "DeepSeek-V4.1-Flash",
      "hardware": "gb200",
      "date_api": "2026-10-26",
      "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40261068322/attempts/1",
      "workflow_db_id": "2828",
      "recipe_fingerprint": "139106d97a964588c9227e6aaf24a870302ddb20c3ca2325f96d43b9e36325d2",
      "framework": "sglang",
      "precision": "fp4",
      "spec_method": "mtp",
      "concurrency": 256,
      "disaggregated": false,
      "total_allocated_gpus": 4,
      "prefill_gpus": null,
      "decode_gpus": null,
      "aggregate_engine_gpus": 4,
      "per_gpu_denominator": "all allocated physical GPUs",
      "interactivity_p90_tok_s_user": 112.73957158962797,
      "interactivity_median_tok_s_user": 121.95121951219511,
      "total_tps": 1446799.42836,
      "input_tps": 1436664.84734,
      "output_tps": 10134.58102,
      "total_tps_per_allocated_gpu": 361699.85709,
      "input_tps_per_allocated_gpu": 359166.21183,
      "output_tps_per_allocated_gpu": 2533.64526,
      "mean_input_tokens": 141693.42782,
      "median_input_tokens": 102206,
      "p90_input_tokens": 296117.8,
      "mean_output_tokens_actual": 999.53968,
      "median_output_tokens_actual": 415,
      "p90_output_tokens_actual": 2320.6,
      "gpu_cache_hit_rate": 0.9245942230518933,
      "cpu_cache_hit_rate": 0.024765823876327243,
      "external_cache_hit_rate": 0.02713,
      "theoretical_cache_hit_rate": 0.97187,
      "offload_mode": "on",
      "kv_offloading": "dram",
      "kv_offload_backend": "hicache",
      "image": "lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727",
      "isolated_uncached_prefill_capacity_tps_per_gpu": null,
      "context_limit_tokens_engine_log": null,
      "kv_precision_engine_log": null,
      "cache_note": "P/D GPU fields mirror one aggregate engine; do not sum them.",
      "hypothetical_same_group_rack_packing": {
        "rack_gpus": 72,
        "whole_groups": 18,
        "allocated_gpus": 72,
        "unused_gpus": 0,
        "output_tps": 182422.45836,
        "scope": "arithmetic extrapolation with identical replay and independent groups; not a measured full rack"
      }
    }
  ],
  "exact_run_api_urls": [
    "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4-Pro&runId=36425419107&exactRun=true&sequence=agentic-traces",
    "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4.1-Flash&runId=37173401305&exactRun=true&sequence=agentic-traces",
    "https://inferencex.semianalysis.com/api/v1/benchmarks?model=DeepSeek-V4.1-Flash&runId=40261068322&exactRun=true&sequence=agentic-traces"
  ],
  "headline_100": {
    "Pro": {
      "reported_total_tps_per_gpu": 37029.643,
      "interpolated": true,
      "bracket_result_ids": [
        "444911",
        "444914"
      ],
      "extra_disagg_factor_in_overview": true,
      "use_as_decode_tps": false
    },
    "Flash": {
      "reported_total_tps_per_gpu": 156095.303,
      "interpolated": true,
      "bracket_result_ids": [
        "444210",
        "444212"
      ],
      "extra_disagg_factor_in_overview": false,
      "use_as_decode_tps": false
    }
  },
  "validation": "Every selected raw total/input/output per-GPU number matches absolute TPS divided by all allocated physical GPUs within 6e-6 tok/s/GPU.",
  "hardware": {
    "name": "NVIDIA GB200 NVL72",
    "gpus_per_rack": 72,
    "cost_per_gpu_hour": 1.86,
    "all_in_power_kw_per_gpu": 1.87,
    "source_url": "https://inferencex.semianalysis.com/chips/gb200-nvl72",
    "retrieved": "2026-11-02"
  },
  "method": "Fixed replay rates: max(logical input demand / group input TPS, output demand / group output TPS) / usable capacity; ceil groups and whole racks. Cache composition is illustrative and does not calibrate fresh-prefill capacity. No aggregation of overlapping traffic disclosures.",
  "presets": [
    {
      "label": "Illustrative market \u00b7 500T/day (assumption)",
      "value": 500,
      "note": "500T/day is an editable scenario assumption, not a measured market total. These sources cannot establish the whole market without overlap."
    },
    {
      "label": "Fireworks \u00b7 40T/day (disclosed floor)",
      "value": 40,
      "note": "Fireworks disclosed more than 40T tokens/day in July 2026. This uses the 40T floor; it is neither current global demand nor evidence that Fireworks runs only DeepSeek.",
      "source_url": "https://fireworks.ai/blog/fireworks-series-d-funding"
    },
    {
      "label": "OpenAI API \u00b7 21.6T/day (derived floor)",
      "value": 21.6,
      "note": "OpenAI disclosed more than 15B API tokens/minute in March 2026; \u00d7 1,440 = more than 21.6T/day. API scope only.",
      "source_url": "https://openai.com/index/accelerating-the-next-phase-ai/"
    },
    {
      "label": "OpenRouter \u00b7 18.34T/day (weekly average)",
      "value": 18.342857142857145,
      "note": "128.4T in the week of September 14, 2026 \u00f7 7. Router traffic can overlap underlying providers; not additive with their disclosures.",
      "source_url": "https://openrouter.ai/data/newsletter/the-token-state-of-play"
    },
    {
      "label": "DeepSeek \u00b7 155T/day (one Flash cluster)",
      "value": 155,
      "note": "DeepInfer reports 155T logical tokens/day for one V4.1 Flash cluster. This is not a company-wide or global total; native tokenization and caching may differ.",
      "source_url": "https://github.com/deepseek-ai/DeepInfer/blob/774ce244b87cc0eb8d4b94ab9f5de2ab22049f01/doc/inference_deployment_qa.md"
    },
    {
      "label": "Custom assumption",
      "value": null,
      "note": "Custom assumed logical token demand, expressed in the benchmark token basis. No market total is inferred from the provider disclosures."
    }
  ]
}