orthozany/vllm-jasl-dsv4 repository overview

# ~150 GB.  HF_HUB_ENABLE_HF_TRANSFER=1 uses the fast Rust downloader.
mkdir -p /path/to/models/deepseek-ai
HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download \
  deepseek-ai/DeepSeek-V4-Flash \
  --local-dir /path/to/models/deepseek-ai/DeepSeek-V4-Flash

services:
  vllm-dsv4-flash:
    image: orthozany/vllm-jasl-dsv4:pr41834-2026-05-13
    container_name: vllm-dsv4-flash
    init: true
    runtime: nvidia
    privileged: true
    ipc: host
    shm_size: "32g"
    restart: unless-stopped

    # The image's ENTRYPOINT is the venv's `vllm` binary, but the venv needs
    # to be `source activate`d so its env (PYTHONPATH for editable install,
    # LD_LIBRARY_PATH for cuda libs in /home/user/jasl-dsv4/.venv) is set.
    entrypoint:
      - "/bin/bash"
      - "-c"
      - |
        set -euo pipefail
        source /home/user/jasl-dsv4/.venv/bin/activate
        exec vllm serve "$$@"
      - "--"

    environment:
      # ----- GPU pinning -----------------------------------------------------
      # IMPORTANT: NVIDIA_VISIBLE_DEVICES alone does not always restrict on
      # every docker / nvidia-container-runtime combination. Also set
      # CUDA_VISIBLE_DEVICES to the in-container indices you want CUDA to use.
      # Do NOT additionally set deploy.resources.reservations.devices — it
      # can conflict and accidentally select a non-Blackwell GPU as cuda:0.
      - NVIDIA_VISIBLE_DEVICES=0,1
      - CUDA_VISIBLE_DEVICES=0,1
      - CUDA_DEVICE_ORDER=PCI_BUS_ID

      # ----- SM 12.0 workarounds (required) ----------------------------------
      # FlashInfer JIT mis-parses TORCH_CUDA_ARCH_LIST=12.0a; fall back to
      # the PyTorch sampler.
      - VLLM_USE_FLASHINFER_SAMPLER=0
      # Sparse-MLA warmup adds many seconds to startup with no benefit.
      - VLLM_ENABLE_DEEPSEEK_V4_SPARSE_MLA_WARMUP=0
      # Wide timeout: the v1 engine compiles many TileLang sparse-MLA kernels
      # on first start; 3600 s comfortably covers cold start.
      - VLLM_ENGINE_READY_TIMEOUT_S=3600

      # ----- NCCL: PCIe-only Blackwell, no NVLink ----------------------------
      # LL + Ring + 8 channels + 512 threads is the small-message tuning that
      # keeps TTFT low on Blackwell workstation cards. Safe because we
      # disable custom-allreduce (see --disable-custom-all-reduce in command:).
      - NCCL_DEBUG=WARN
      - NCCL_P2P_DISABLE=1
      - NCCL_SHM_DISABLE=0
      - NCCL_IB_DISABLE=1
      - NCCL_PROTO=LL
      - NCCL_ALGO=Ring
      - NCCL_MIN_NCHANNELS=8
      - NCCL_NTHREADS=512

      # ----- Standard --------------------------------------------------------
      - PYTHONUNBUFFERED=1
      - VLLM_WORKER_MULTIPROC_METHOD=spawn

    volumes:
      # Read-only mount for the downloaded weights.
      - /path/to/models/deepseek-ai/DeepSeek-V4-Flash:/model:ro
      # Persistent caches so torch.compile, TileLang, triton don't re-JIT on
      # every container restart.  Local subdirectories OK.
      - ./cache/huggingface:/root/.cache/huggingface
      - ./cache/vllm:/root/.cache/vllm
      - ./cache/torch:/root/.cache/torch
      - ./cache/triton:/root/.cache/triton
      - ./cache/torchinductor:/root/.cache/torchinductor
      - ./cache/flashinfer:/root/.cache/flashinfer

    ports:
      - "8000:8000"

    command:
      # Positional: the model directory inside the container.
      - "/model"

      # ----- Names the API responds to ---------------------------------------
      - "--served-model-name"
      - "deepseek-v4-flash"
      - "deepseek-v4-flash-mtp"
      - "deepseek-ai/DeepSeek-V4-Flash"

      # ----- Parallelism + quant ---------------------------------------------
      - "--tensor-parallel-size=2"
      - "--kv-cache-dtype=fp8"
      - "--block-size=256"

      # ----- Context window --------------------------------------------------
      # 131072 (128K) is a safe default with --gpu-memory-utilization=0.93.
      # For the model's native max (524288) bump to 0.95.
      - "--max-model-len=131072"
      - "--max-num-seqs=4"
      - "--max-num-batched-tokens=8192"
      - "--gpu-memory-utilization=0.93"

      # ----- DeepSeek-V4 tokenizer + parsers (required) ----------------------
      - "--tokenizer-mode=deepseek_v4"
      - "--tool-call-parser=deepseek_v4"
      - "--enable-auto-tool-choice"
      - "--reasoning-parser=deepseek_v4"
      - "--trust-remote-code"

      # ----- Custom all-reduce off (no NVLink on workstation Blackwell) ------
      - "--disable-custom-all-reduce"

      # ----- MTP self-speculation: ~+40% decode throughput -------------------
      # Set num_speculative_tokens=1 instead if you see acceptance below 25%.
      - "--speculative-config={\"method\":\"mtp\",\"num_speculative_tokens\":2}"

      - "--host=0.0.0.0"
      - "--port=8000"

    ulimits:
      memlock: -1
      stack: 67108864

docker compose up -d

# Cold start takes 2–4 minutes (weight load + warmup + TileLang JIT).
# Wait for "Application startup complete" in the logs.
docker compose logs -f vllm-dsv4-flash | grep -E "Application startup|ERROR"

# Smoke test
curl -sS -X POST http://localhost:8000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{"model":"deepseek-v4-flash","stream":false,"max_tokens":64,
       "temperature":0.0,"messages":[{"role":"user","content":"Say OK."}]}'

/home/user/jasl-dsv4/                  ← venv + vllm-jasl/ + DeepGEMM-jasl/
/home/user/.local/share/uv/python/     ← uv-managed cpython-3.13.13