{
  "hf_id": "Qwen/Qwen3.5-397B-A17B",
  "meta": {
    "title": "Qwen3.5-397B",
    "slug": "qwen3.5-397b",
    "provider": "Qwen",
    "description": "Multimodal MoE model with gated delta networks architecture, 397B total / 17B active parameters, up to 262K context",
    "date_added": "2026-02-16",
    "date_updated": "2026-11-11",
    "tasks": [
      "multimodal",
      "text"
    ],
    "performance_headline": "Verified on 8x H200, 8x MI300X/MI355X, a single MI350X/MI355X (MXFP4), GB200 nodes, and Ascend 950DT",
    "related_recipes": [
      "Qwen/Qwen3.6-35B-A3B"
    ],
    "hardware": {
      "h200": "verified",
      "gb200": "verified",
      "mi300x": "verified",
      "mi325x": "verified",
      "mi350x": "verified",
      "mi355x": "verified",
      "ascend_950dt": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "hub": "huggingface",
    "checkpoint_url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {
      "VLLM_DEEP_GEMM_WARMUP": "skip",
      "VLLM_USE_DEEP_GEMM": "0",
      "VLLM_USE_RUST_FRONTEND": "1"
    },
    "docker_image": "vllm/vllm-openai:latest",
    "command": "vllm serve Qwen/Qwen3.5-397B-A17B \\\n  --trust-remote-code \\\n  --tensor-parallel-size 8 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser qwen3 \\\n  --mm-encoder-tp-mode data",
    "argv": [
      "vllm",
      "serve",
      "Qwen/Qwen3.5-397B-A17B",
      "--trust-remote-code",
      "--tensor-parallel-size",
      "8",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "qwen3",
      "--mm-encoder-tp-mode",
      "data"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  -e VLLM_DEEP_GEMM_WARMUP=skip \\\n  -e VLLM_USE_DEEP_GEMM=0 \\\n  -e VLLM_USE_RUST_FRONTEND=1 \\\n  vllm/vllm-openai:latest Qwen/Qwen3.5-397B-A17B \\\n  --trust-remote-code \\\n  --tensor-parallel-size 8 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser qwen3 \\\n  --mm-encoder-tp-mode data",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "-e",
      "VLLM_DEEP_GEMM_WARMUP=skip",
      "-e",
      "VLLM_USE_DEEP_GEMM=0",
      "-e",
      "VLLM_USE_RUST_FRONTEND=1",
      "vllm/vllm-openai:latest",
      "Qwen/Qwen3.5-397B-A17B",
      "--trust-remote-code",
      "--tensor-parallel-size",
      "8",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "qwen3",
      "--mm-encoder-tp-mode",
      "data"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "single_node_tep": "/Qwen/Qwen3.5-397B-A17B/strategies/single_node_tep.json",
      "single_node_dep": "/Qwen/Qwen3.5-397B-A17B/strategies/single_node_dep.json",
      "multi_node_tp": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tp.json",
      "multi_node_tp_pp": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tp_pp.json",
      "multi_node_dep": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_dep.json",
      "multi_node_tep": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tep.json",
      "pd_cluster": "/Qwen/Qwen3.5-397B-A17B/strategies/pd_cluster.json",
      "kv_store_centralized_mooncake": "/Qwen/Qwen3.5-397B-A17B/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/Qwen/Qwen3.5-397B-A17B/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/Qwen/Qwen3.5-397B-A17B/hw/h200.json",
      "h100": "/Qwen/Qwen3.5-397B-A17B/hw/h100.json",
      "b200": "/Qwen/Qwen3.5-397B-A17B/hw/b200.json",
      "gb200": "/Qwen/Qwen3.5-397B-A17B/hw/gb200.json",
      "b300": "/Qwen/Qwen3.5-397B-A17B/hw/b300.json",
      "gb300": "/Qwen/Qwen3.5-397B-A17B/hw/gb300.json",
      "mi300x": "/Qwen/Qwen3.5-397B-A17B/hw/mi300x.json",
      "mi325x": "/Qwen/Qwen3.5-397B-A17B/hw/mi325x.json",
      "mi350x": "/Qwen/Qwen3.5-397B-A17B/hw/mi350x.json",
      "mi355x": "/Qwen/Qwen3.5-397B-A17B/hw/mi355x.json",
      "ascend_950dt": "/Qwen/Qwen3.5-397B-A17B/hw/ascend_950dt.json"
    }
  },
  "model": {
    "model_id": "Qwen/Qwen3.5-397B-A17B",
    "min_vllm_version": "0.17.0",
    "default_frontend": "rust",
    "architecture": "moe",
    "parameter_count": "397B",
    "active_parameters": "17B",
    "context_length": 262144,
    "base_args": [
      "--trust-remote-code"
    ],
    "base_env": {
      "VLLM_DEEP_GEMM_WARMUP": "skip",
      "VLLM_USE_DEEP_GEMM": "0"
    },
    "install": {
      "pip": {
        "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto"
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:latest"
      }
    }
  },
  "features": {
    "tool_calling": {
      "description": "Enable automatic tool choice with Qwen3 Coder parser",
      "args": [
        "--enable-auto-tool-choice",
        "--tool-call-parser",
        "qwen3_coder"
      ]
    },
    "reasoning": {
      "description": "Enable chain-of-thought reasoning with Qwen3 parser",
      "args": [
        "--reasoning-parser",
        "qwen3"
      ]
    },
    "spec_decoding": {
      "description": "Multiple options for multi-token prediction speculative decoding for lower latency. Also has DFlash option. ",
      "default_mode": "mtp",
      "modes": {
        "mtp": {
          "description": "Native MTP head, 3 draft tokens.",
          "variants": [
            "default",
            "nvfp4",
            "fp8",
            "gptq_int4",
            "amd_mxfp4"
          ],
          "args": [
            "--speculative-config",
            "{\"method\":\"mtp\",\"num_speculative_tokens\":3}"
          ]
        },
        "dflash": {
          "label": "DFlash",
          "description": "DFlash speculative decoding with an externally trained draft model; the served checkpoint is unchanged.",
          "variants": [
            "default"
          ],
          "args": [
            "--speculative-config",
            "{\"model\": \"RedHatAI/Qwen3.5-397B-A17B-speculator.dflash\", \"num_speculative_tokens\": 7, \"method\": \"dflash\"}"
          ]
        },
        "qwen3_5_mtp": {
          "label": "MTP (Ascend)",
          "description": "MTP mode documented for vLLM Ascend.",
          "variants": [
            "ascend_w8a8_mtp",
            "ascend_w4a4_mxfp4"
          ],
          "args": [
            "--speculative-config",
            "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":3}"
          ]
        }
      }
    },
    "text_only": {
      "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.",
      "args": [
        "--language-model-only"
      ]
    },
    "encoder_parallel": {
      "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.",
      "args": [
        "--mm-encoder-tp-mode",
        "data"
      ]
    },
    "long_context": {
      "description": "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0.",
      "args": [
        "--hf-overrides",
        "{\"text_config\": {\"rope_parameters\": {\"mrope_interleaved\": true, \"mrope_section\": [11, 11, 10], \"rope_type\": \"yarn\", \"rope_theta\": 10000000, \"partial_rotary_factor\": 0.25, \"factor\": 4.0, \"original_max_position_embeddings\": 262144}}}",
        "--max-model-len",
        "1010000"
      ],
      "env": {
        "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1"
      }
    }
  },
  "opt_in_features": [
    "spec_decoding",
    "text_only",
    "long_context"
  ],
  "variants": {
    "default": {
      "precision": "bf16",
      "vram_minimum_gb": 953,
      "description": "Full precision BF16 — requires 8x H200 or equivalent"
    },
    "nvfp4": {
      "model_id": "nvidia/Qwen3.5-397B-A17B-NVFP4",
      "precision": "nvfp4",
      "vram_minimum_gb": 238,
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/nvidia/Qwen3.5-397B-A17B-NVFP4.json"
    },
    "fp8": {
      "model_id": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
      "precision": "fp8",
      "vram_minimum_gb": 477,
      "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic.json"
    },
    "gptq_int4": {
      "model_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4",
      "precision": "int4",
      "vram_minimum_gb": 239,
      "description": "GPTQ Int4 checkpoint — halves VRAM vs FP8",
      "json": "/Qwen/Qwen3.5-397B-A17B-GPTQ-Int4.json"
    },
    "amd_mxfp4": {
      "model_id": "amd/Qwen3.5-397B-A17B-MXFP4",
      "precision": "mxfp4",
      "label": "MXFP4 (W4A4)",
      "vram_minimum_gb": 287,
      "description": "AMD Quark MXFP4 checkpoint for MI350X/MI355X (gfx950) — fits one 288 GB GPU",
      "supported_hardware": [
        "arch:gfx950"
      ],
      "tp": 1,
      "min_vllm_version": "0.31.0",
      "extra_env": {
        "VLLM_ROCM_USE_AITER": "1"
      },
      "extra_args": [
        "--attention-backend",
        "ROCM_AITER_FA",
        "--max-num-seqs",
        "64"
      ],
      "json": "/amd/Qwen3.5-397B-A17B-MXFP4.json"
    },
    "ascend_w8a8_mtp": {
      "model_id": "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp",
      "precision": "int8",
      "label": "W8A8 (Ascend)",
      "vram_minimum_gb": 477,
      "supported_hardware": [
        "ascend_950dt"
      ],
      "tp": 8,
      "min_vllm_version": "0.23.0",
      "default_modes": {
        "spec_decoding": "qwen3_5_mtp"
      },
      "extra_args": [
        "--distributed-executor-backend",
        "mp",
        "--data-parallel-size",
        "1",
        "--quantization",
        "ascend",
        "--served-model-name",
        "qwen3.5",
        "--max-num-seqs",
        "128",
        "--max-model-len",
        "133000",
        "--max-num-batched-tokens",
        "8192",
        "--gpu-memory-utilization",
        "0.95",
        "--enable-prefix-caching",
        "--async-scheduling",
        "--compilation-config",
        "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}",
        "--additional-config",
        "{\"enable_cpu_binding\":true}"
      ],
      "extra_env": {
        "VLLM_USE_MODELSCOPE": "True",
        "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
        "HCCL_OP_EXPANSION_MODE": "AIV",
        "HCCL_BUFFSIZE": "400",
        "OMP_PROC_BIND": "false",
        "OMP_NUM_THREADS": "100",
        "TASK_QUEUE_ENABLE": "1",
        "VLLM_ASCEND_ENABLE_PREFETCH_MLP": "1",
        "HCCL_INTRA_PCIE_ENABLE": "1",
        "HCCL_INTRA_ROCE_ENABLE": "0"
      },
      "hardware_overrides": {
        "ascend_950dt": {
          "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-950DT"
        }
      },
      "json": "/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp.json"
    },
    "ascend_w4a4_mxfp4": {
      "model_id": "Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4",
      "precision": "mxfp4",
      "label": "W4A4 (Ascend)",
      "vram_minimum_gb": 239,
      "supported_hardware": [
        "ascend_950dt"
      ],
      "tp": 8,
      "min_vllm_version": "0.23.0",
      "default_modes": {
        "spec_decoding": "qwen3_5_mtp"
      },
      "extra_args": [
        "--distributed-executor-backend",
        "mp",
        "--data-parallel-size",
        "1",
        "--quantization",
        "ascend",
        "--served-model-name",
        "qwen3.5",
        "--max-num-seqs",
        "128",
        "--max-model-len",
        "133000",
        "--max-num-batched-tokens",
        "8192",
        "--gpu-memory-utilization",
        "0.95",
        "--enable-prefix-caching",
        "--async-scheduling",
        "--compilation-config",
        "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}",
        "--additional-config",
        "{\"enable_cpu_binding\":true}"
      ],
      "extra_env": {
        "VLLM_USE_MODELSCOPE": "True",
        "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
        "HCCL_OP_EXPANSION_MODE": "AIV",
        "HCCL_BUFFSIZE": "400",
        "OMP_PROC_BIND": "false",
        "OMP_NUM_THREADS": "100",
        "TASK_QUEUE_ENABLE": "1",
        "VLLM_ASCEND_ENABLE_PREFETCH_MLP": "1",
        "HCCL_INTRA_PCIE_ENABLE": "1",
        "HCCL_INTRA_ROCE_ENABLE": "0"
      },
      "hardware_overrides": {
        "ascend_950dt": {
          "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-950DT"
        }
      },
      "json": "/Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4.json"
    }
  },
  "hardware_overrides": {
    "blackwell": {
      "extra_args": [
        "--moe-backend",
        "flashinfer_trtllm"
      ]
    }
  },
  "presets": {
    "b200_tp2_kv_offload": {
      "label": "TP2 + KV offload (multi-turn agentic)",
      "hardware": "b200",
      "strategy": "single_node_tp",
      "variants": [
        "nvfp4"
      ],
      "workload": "Multi-turn agentic coding, ~64k input, KV-reuse-bound (mean ITL under 20 ms)",
      "description": "One TP2 engine with expert parallelism on two of the node's eight B200 GPUs: MTP with three draft tokens, FP8 KV cache, float16 Mamba SSM cache, async scheduling, and a 160 GiB host KV tier so prefixes survive across turns; admission is capped at 48 sequences to hold mean ITL under 20 ms.",
      "source": {
        "label": "vLLM #64607",
        "href": "https://github.com/vllm-project/vllm/pull/64607"
      },
      "image": "vllm/vllm-openai:v0.32.0",
      "tp": 2,
      "select": {
        "frontend": "python",
        "features": {
          "spec_decoding": "mtp",
          "encoder_parallel": false
        }
      },
      "args": [
        "--enable-expert-parallel",
        "--async-scheduling",
        "--block-size",
        "64",
        "--mamba-ssm-cache-dtype",
        "float16",
        "--max-num-batched-tokens",
        "16384",
        "--max-num-seqs",
        "48",
        "--no-disable-hybrid-kv-cache-manager",
        "--kv-offloading-backend",
        "native",
        "--kv-offloading-size",
        "160"
      ],
      "runbook": "Measured by NVIDIA on vLLM v0.32.0 (`vllm/vllm-openai:v0.32.0`) on two B200 SXM 180 GB with a\n942-request multi-turn agentic coding trace (about 64k mean input tokens, most prefix tokens\nreused across turns, outputs capped at 12k tokens), replayed at fixed concurrency with real\nMTP acceptance (mean acceptance length 3.2 to 3.3). Host: 512 GiB RAM and 200 GiB /dev/shm per\nserver for the 160 GiB offload tier.\n\n| Concurrency | Output tok/s/GPU | Mean ITL | Mean TTFT |\n|---|---:|---:|---:|\n| 16 | 936 | 8.7 ms | 0.70 s |\n\nFurther concurrencies and the comparison against this recipe's baseline command on the same\nimage and trace are being collected and will follow in a separate update.\n\nThe 48-sequence cap trades time to first token for inter-token latency at 64 clients; raise\n`--max-num-seqs` if TTFT matters more than ITL for your traffic. The offload tier only helps\nworkloads that revisit long prefixes; drop the `--kv-offloading-*` flags for single-turn traffic.\n",
      "json": "/Qwen/Qwen3.5-397B-A17B/presets/b200_tp2_kv_offload.json"
    }
  },
  "notes": [
    {
      "title": "MTP on ROCm: use an FP8 KV cache",
      "when": {
        "hardware": [
          "amd"
        ],
        "features": [
          "spec_decoding"
        ]
      },
      "body": "MTP runs on ROCm with vLLM 0.31.0. Pair it with `--kv-cache-dtype fp8`:\nwith the default KV cache its speedup fades as the prompt grows.\n"
    }
  ],
  "troubleshooting": [
    {
      "title": "assert num_cache_lines >= batch",
      "when": {
        "hardware": [
          "nvidia",
          "amd"
        ]
      },
      "body": "The CUDA graph capture size is larger than the Mamba cache. Reduce\n`--max-cudagraph-capture-size` (default 512); see\n[vllm#34571](https://github.com/vllm-project/vllm/pull/34571).\n"
    },
    {
      "title": "Prefix caching is experimental",
      "body": "Prefix caching for the Mamba cache runs in the experimental \"align\" mode.\n"
    },
    {
      "title": "Out of memory with Encoder Parallel",
      "when": {
        "features": [
          "encoder_parallel"
        ]
      },
      "body": "`--mm-encoder-tp-mode data` places a copy of the vision encoder on every\nGPU, which takes extra memory. If the server runs out of memory, lower\n`--max-num-seqs` or `--max-num-batched-tokens`, or drop\n`--mm-encoder-tp-mode data`.\n"
    }
  ],
  "guide": "## Overview\n\n[Qwen3.5](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) is a multimodal mixture-of-experts model featuring a gated delta networks architecture with 397B total parameters and 17B active parameters. This guide covers how to efficiently deploy and serve the model across different hardware configurations and workload profiles using vLLM.\n\n## Verified configurations\n\nThese configurations were verified on 8x H200, 8x MI300X/MI355X and GB200. On\nH200 and MI300X/MI355X they used the official FP8 checkpoint\n[Qwen/Qwen3.5-397B-A17B-FP8](https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8).\n\n| Workload | Hardware | Checkpoint | Parallelism | Other settings |\n|---|---|---|---|---|\n| Throughput, text only | 8x H200, 8x MI300X/MI355X | FP8 | DP8 + EP | `--language-model-only`, `--enable-prefix-caching` |\n| Throughput, multimodal | 8x H200, 8x MI300X/MI355X | FP8 | DP8 + EP | `--mm-encoder-tp-mode data`, `--mm-processor-cache-type shm`, `--enable-prefix-caching` |\n| Latency | 8x H200, 8x MI300X/MI355X | FP8 | TP8 | MTP-1, prefix caching off |\n| Throughput, text only | GB200 (4 GPUs) | [NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4) | DP4 + EP | `--language-model-only`, `--enable-prefix-caching` |\n| Throughput, text only | MI355X | FP8 | TP2 + EP | `--language-model-only`, `--enable-prefix-caching` |\n\n`--language-model-only` skips loading the vision encoder and frees memory for KV\ncache. For multimodal workloads, `--mm-encoder-tp-mode data` runs the vision\nencoder data-parallel and `--mm-processor-cache-type shm` caches preprocessed\nmultimodal inputs in shared memory. MTP-1 lowers time per output token with a\nhigh acceptance rate, but speculative tokens take KV cache capacity, so\nthroughput drops under high concurrency; tune `num_speculative_tokens` (1-5) to\nthe workload.\n\n## Single MI350X / MI355X (MXFP4)\n\nAMD's MXFP4 checkpoint [amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4) (239 GB)\nfits one 288 GB MI350X or MI355X: pick the **MXFP4 (W4A4)** variant above. It sets three things vLLM does not\npick on its own: `VLLM_ROCM_USE_AITER=1` turns on the AITER kernels; `--attention-backend ROCM_AITER_FA` replaces\nthe auto-selected `ROCM_ATTN`, whose decode slows down as the context grows on this hybrid model; and\n`--max-num-seqs 64`, because the default (1024) is more than the ~296 Mamba cache blocks a single GPU leaves,\nand the server would refuse to start.\n\nSingle-stream decode in tokens/s at ~200 / ~2k / ~12k / ~30k prompt tokens (128 output tokens), one GPU,\n`vllm/vllm-openai-rocm:v0.31.0`:\n\n| | ~200 | ~2k | ~12k | ~30k |\n|---|---|---|---|---|\n| MXFP4, this variant (MI355X) | 119 | 119 | 119 | 117 |\n| + MTP (3 tokens) + `--kv-cache-dtype fp8` (MI355X) | 219 | 201 | 209 | 188 |\n| + MTP (3 tokens), default KV cache (MI350X) | 198 | 164 | 125 | 80 |\n| auto-selected `ROCM_ATTN` instead of `ROCM_AITER_FA` (MI355X) | 116 | 89 | 39 | 19 |\n| GPTQ-Int4, AITER + `ROCM_AITER_FA` (MI350X) | 75 | 74 | 74 | 74 |\n\nWith MTP, add `--kv-cache-dtype fp8`: on ROCm, MTP's gain otherwise fades as the prompt grows. The MXFP4\ncheckpoint quantizes activations as well as weights (W4A4): on the same wikitext-2 tokens vLLM scored PPL 3.43\nwith it and 3.39 with GPTQ-Int4 (W4A16), so GPTQ-Int4 is the higher-fidelity option at about two thirds of the speed.\n\n## Processing Ultra-Long Texts\n\nQwen3.5-397B-A17B natively supports `262,144` tokens. To serve longer inputs, turn on the\n**Long Context** feature above: it applies the model card's YaRN RoPE\nconfiguration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.\nStatic YaRN holds the scaling factor constant regardless of input length, so\nleave it off unless you actually serve long prompts; for a ~524K workload,\nhalve the card's `factor` to `2.0`.\n\nSee the [model card](https://huggingface.co/Qwen/Qwen3.5-397B-A17B#processing-ultra-long-texts) for the full parameter reference.\n\n## Client Usage\n\n```python\nimport time\nfrom openai import OpenAI\n\nclient = OpenAI(\n    api_key=\"EMPTY\",\n    base_url=\"http://localhost:8000/v1\",\n    timeout=3600\n)\n# The server's model name is the checkpoint it runs: the selected variant's.\nmodel = client.models.list().data[0].id\n\nmessages = [\n    {\n        \"role\": \"user\",\n        \"content\": [\n            {\n                \"type\": \"image_url\",\n                \"image_url\": {\n                    \"url\": \"https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png\"\n                }\n            },\n            {\n                \"type\": \"text\",\n                \"text\": \"Read all the text in the image.\"\n            }\n        ]\n    }\n]\n\nstart = time.time()\nresponse = client.chat.completions.create(\n    model=model,\n    messages=messages,\n    max_tokens=2048\n)\nprint(f\"Response costs: {time.time() - start:.2f}s\")\nprint(f\"Generated text: {response.choices[0].message.content}\")\n```\n\nQwen3.5 thinks before it answers. To turn thinking off for one request, pass\n`extra_body={\"chat_template_kwargs\": {\"enable_thinking\": False}}`; for every\nrequest, start the server with\n`--default-chat-template-kwargs '{\"enable_thinking\": false}'`.\n\n## References\n\n- Model card: https://huggingface.co/Qwen/Qwen3.5-397B-A17B\n- FP8 checkpoint: https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8\n- NVFP4 checkpoint: https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4\n- GPTQ-Int4 checkpoint: https://huggingface.co/Qwen/Qwen3.5-397B-A17B-GPTQ-Int4\n- AMD MXFP4 checkpoint: https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4\n- Ascend W8A8 checkpoint: https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp\n- Ascend W4A4 checkpoint: https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4\n- vLLM Ascend tutorial: https://docs.vllm.ai/projects/ascend/en/latest/tutorials/models/Qwen3.5-397B-A17B.html\n- vLLM documentation: https://docs.vllm.ai/\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "1bf414e97acf7563297726531f702cdbea6b2921",
    "path": "recipes/models/Qwen/Qwen3.5-397B-A17B.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3.5-397B-A17B.yaml",
    "overlays": [
      {
        "source": "vllm-ascend",
        "repo": "vllm-project/vllm-ascend",
        "ref": "main",
        "sha": "931bfd948e04a89b735a2f78520093aa9dbc49f0",
        "path": "recipes/overlays/Qwen/Qwen3.5-397B-A17B.yaml",
        "edit_url": "https://github.com/vllm-project/vllm-ascend/edit/main/recipes/overlays/Qwen/Qwen3.5-397B-A17B.yaml"
      }
    ]
  }
}