orthozany/k26-v4-clean repository overview

huggingface-cli download moonshotai/Kimi-K2.6 \
  --local-dir /path/to/Kimi-K2.6 --local-dir-use-symlinks=False

export KIMI_K26_MODEL_PATH=/path/to/Kimi-K2.6
export CACHE_ROOT=$PWD/cache-v4 && mkdir -p $CACHE_ROOT
docker compose -f compose.v4-mtp3-fp8draft.yml up -d

# Wait ~5-8 min for model load + CUDA graph capture
until curl -fsS http://localhost:8402/v1/models >/dev/null; do sleep 10; done
echo "ready"

curl -sS http://localhost:8402/v1/chat/completions -H 'Content-Type: application/json' -d '{
  "model": "Kimi-K2.6", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 20
}'

services:
  vllm-k26-v4:
    image: orthozany/k26-v4-clean:v1
    container_name: vllm-k26-v4
    init: true
    ipc: host
    network_mode: host
    privileged: true
    runtime: nvidia
    shm_size: "32g"
    environment:
      - NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:-all}
      - CUDA_DEVICE_ORDER=PCI_BUS_ID
      - OMP_NUM_THREADS=16
      - CUTE_DSL_ARCH=sm_120a
      - CUDA_DEVICE_MAX_CONNECTIONS=32
      # NCCL via patched libnccl-pr2127 (baked into image)
      - NCCL_IB_DISABLE=1
      - NCCL_P2P_LEVEL=SYS
      - NCCL_PROTO=LL,LL128,Simple
      - USE_NCCL_XML=0
      - NCCL_GRAPH_FILE=
      - VLLM_NCCL_SO_PATH=/opt/libnccl-pr2127.so.2.30.3
      - LD_PRELOAD=/opt/libnccl-pr2127.so.2.30.3
      # PCIe AR (cpp backend with 56KB 1-stage NCCL cutoff)
      - VLLM_ENABLE_PCIE_ALLREDUCE=1
      - VLLM_PCIE_ALLREDUCE_BACKEND=cpp
      - VLLM_CPP_AR_1STAGE_NCCL_CUTOFF=56KB
      - VLLM_CPP_AR_IGNORE_CUTOFF_MAX_ROWS=8
      - VLLM_RTX6K_FUSED_ALLREDUCE_ADD=0
      - VLLM_RTX6K_FUSED_ALLREDUCE_ADD_END_BARRIER=0
      # Misc kernel flags
      - VLLM_USE_B12X_SPARSE_INDEXER=0
      - VLLM_DISABLE_SHARED_EXPERTS_STREAM=0
      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
      - VLLM_LOG_STATS_INTERVAL=1
      - VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel
      # Server (run-kimi26-vllm reads these envs)
      - PORT=8402
      - MODEL=/model
      - SERVED_MODEL_NAME=Kimi-K2.6
      - TP_SIZE=8
      - DCP_SIZE=8
      - GPU_MEMORY_UTILIZATION=0.90
      - MAX_MODEL_LEN=262144
      - MAX_NUM_BATCHED_TOKENS=8192
      - MAX_NUM_SEQS=128
      - MAX_CUDAGRAPH_CAPTURE_SIZE=512
      - KV_CACHE_DTYPE=fp8
      - ATTENTION_BACKEND=TRITON_MLA
      - KIMI_DISABLE_MTP=0
      - SPEC_CONFIG={"model":"festr2/kimi-k2.6-eagle3-mla-fp8","method":"eagle3","num_speculative_tokens":3,"draft_attention_backend":"TRITON_MLA","draft_kv_cache_dtype":"fp8"}
      # NOTE: this pushed image was built before flashinfer-cubin was pinned.
      # Bypass version check with the env var below. Drop it if you rebuild from Dockerfile (pinned).
      - FLASHINFER_DISABLE_VERSION_CHECK=1
      # JIT caches inside container
      - XDG_CACHE_HOME=/cache/jit
      - CUDA_CACHE_PATH=/cache/jit
      - VLLM_CACHE_DIR=/cache/jit/vllm
      - TVM_FFI_CACHE_DIR=/cache/jit/tvm-ffi
      - FLASHINFER_WORKSPACE_BASE=/cache/jit/flashinfer
      - HF_HOME=/root/.cache/huggingface
    command: ["--compilation-config", "{\"pass_config\":{\"fuse_rope_kvcache_cat_mla\":true},\"splitting_ops\":[]}"]
    volumes:
      - ${KIMI_K26_MODEL_PATH:?Set KIMI_K26_MODEL_PATH to absolute path of Kimi-K2.6 weights}:/model:ro
      - ${CACHE_ROOT:-./cache-v4}/jit:/cache/jit
      - ${CACHE_ROOT:-./cache-v4}/triton:/root/.cache/triton
      - ${CACHE_ROOT:-./cache-v4}/torchinductor:/root/.cache/torchinductor
      - ${CACHE_ROOT:-./cache-v4}/vllm:/root/.cache/vllm
      - ${CACHE_ROOT:-./cache-v4}/cutlass_dsl:/root/.cache/cutlass_dsl
      - ${CACHE_ROOT:-./cache-v4}/huggingface:/root/.cache/huggingface
    ulimits:
      memlock: -1
      stack: 67108864
    restart: "no"

- SPEC_CONFIG={"model":"festr2/kimi-k2.6-eagle3-mla-fp8","method":"eagle3","num_speculative_tokens":1,"draft_attention_backend":"TRITON_MLA","draft_kv_cache_dtype":"fp8"}

FROM nvidia/cuda:13.0.0-devel-ubuntu24.04

ARG CUTLASS_DSL_VERSION=4.5.0
ARG NCCL_VERSION=2.29.7
ARG TRITON_VERSION=3.6.0
ARG TORCH_VERSION=2.11.0+cu130
ARG TORCHVISION_VERSION=0.26.0+cu130
ARG CUDA_TILE_VERSION=1.3.0
ARG FLASHINFER_CUBIN_VERSION=0.6.11
ARG VLLM_FORK_COMMIT=68b3569f2bd2e8f6568f25f2144fb648b200d832
ARG FLASHINFER_COMMIT=1a60071db81239d4b0e5f28cd0563acceba2c878
ARG B12X_COMMIT=c929144c7689668b07ca65af10ceadf1c745165d

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
        python3.12 python3.12-dev python3.12-venv python3-pip \
        git build-essential ninja-build cmake ca-certificates curl wget \
    && rm -rf /var/lib/apt/lists/* \
    && ln -sf /usr/bin/python3.12 /usr/bin/python3 \
    && ln -sf /usr/bin/python3.12 /usr/bin/python

RUN python3 -m venv /opt/venv
ENV PATH=/opt/venv/bin:$PATH

RUN pip install --no-cache-dir --upgrade pip "setuptools>=77.0.3,<81.0.0" setuptools-scm wheel ninja

RUN pip install --no-cache-dir \
        torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION} \
        --index-url https://download.pytorch.org/whl/cu130

RUN pip install --no-cache-dir --no-deps \
        nvidia-cutlass-dsl==${CUTLASS_DSL_VERSION} \
        nvidia-cutlass-dsl-libs-base==${CUTLASS_DSL_VERSION} \
        nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION} \
        nvidia-nccl-cu13==${NCCL_VERSION} \
        triton==${TRITON_VERSION} \
        cuda-tile==${CUDA_TILE_VERSION} \
        apache-tvm-ffi==0.1.10

RUN git clone https://github.com/voipmonitor/vllm /opt/vllm \
    && cd /opt/vllm && git checkout ${VLLM_FORK_COMMIT}
WORKDIR /opt/vllm
RUN pip install --no-cache-dir -e . --no-build-isolation

RUN git clone https://github.com/flashinfer-ai/flashinfer.git /opt/flashinfer-src \
    && cd /opt/flashinfer-src && git checkout ${FLASHINFER_COMMIT} \
    && git submodule update --init --recursive
RUN cd /opt/flashinfer-src && pip install --no-cache-dir -e . --no-build-isolation
RUN pip install --no-cache-dir --force-reinstall flashinfer-cubin==${FLASHINFER_CUBIN_VERSION}

RUN git clone https://github.com/local-inference-lab/b12x.git /opt/b12x \
    && cd /opt/b12x && git checkout ${B12X_COMMIT}
RUN cd /opt/b12x && pip install --no-cache-dir -e . --no-build-isolation

# image-side helper script + patched libnccl (PR #2127) — extracted from upstream image
# Get them via:
#   docker create --name x voipmonitor/vllm:glm-kimi-canonical-rebase-layered-vllm68b3569f-b12xc929144-flashinfergit1a60071-cutedsl45-20260514
#   docker cp x:/opt/vllm/scripts/run-kimi26-vllm .
#   docker cp x:/opt/libnccl-pr2127.so.2.30.3 .
#   docker rm x
COPY run-kimi26-vllm /opt/vllm/scripts/run-kimi26-vllm
RUN chmod +x /opt/vllm/scripts/run-kimi26-vllm
COPY libnccl-pr2127.so.2.30.3 /opt/libnccl-pr2127.so.2.30.3

ENV CUDA_DEVICE_ORDER=PCI_BUS_ID \
    VLLM_NCCL_SO_PATH=/opt/libnccl-pr2127.so.2.30.3

WORKDIR /opt/vllm
ENTRYPOINT ["/bin/bash", "-lc", "rm -f /usr/local/cuda-13.0/targets/x86_64-linux/lib/libcudart.so.12 2>/dev/null; exec /opt/vllm/scripts/run-kimi26-vllm \"$@\"", "--"]