orthozany/k26-0424-clean repository overview

huggingface-cli download moonshotai/Kimi-K2.6 \
  --local-dir /path/to/Kimi-K2.6 --local-dir-use-symlinks=False

export KIMI_K26_MODEL_PATH=/path/to/Kimi-K2.6
export CACHE_ROOT=$PWD/cache-0424 && mkdir -p $CACHE_ROOT
docker compose -f compose.0424-mtp3.yml up -d

# Wait ~5-8 min for model load + CUDA graph capture
until curl -fsS http://localhost:8402/v1/models >/dev/null; do sleep 10; done
echo "ready"

curl -sS http://localhost:8402/v1/chat/completions -H 'Content-Type: application/json' -d '{
  "model": "Kimi-K2.6", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 20
}'

services:
  vllm-k26-0424:
    image: orthozany/k26-0424-clean:v12
    container_name: vllm-k26-0424
    ipc: host
    network_mode: host
    runtime: nvidia
    shm_size: "32g"
    environment:
      - NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:-all}
      - CUDA_DEVICE_ORDER=PCI_BUS_ID
      - OMP_NUM_THREADS=16
      - HF_HOME=/root/.cache/huggingface
      # Server flags (read by baked /opt/run-k26.sh)
      - PORT=8402
      - MODEL=/model
      - SERVED_MODEL_NAME=Kimi-K2.6
      - TP_SIZE=8
      - DCP_SIZE=8
      - GPU_MEMORY_UTILIZATION=0.97
      - MAX_MODEL_LEN=262144
      - MAX_NUM_BATCHED_TOKENS=8192
      - MAX_NUM_SEQS=128
      - KV_CACHE_DTYPE=fp8
      - ATTENTION_BACKEND=TRITON_MLA
      # default SPEC_CONFIG (baked into /opt/run-k26.sh) =
      #   MTP=3 fp8 draft + probabilistic rejection (vLLM 0.16.1 syntax)
    volumes:
      - ${KIMI_K26_MODEL_PATH:?Set KIMI_K26_MODEL_PATH to absolute path of Kimi-K2.6 weights}:/model:ro
      - ${CACHE_ROOT:-./cache-0424}/vllm:/root/.cache/vllm
      - ${CACHE_ROOT:-./cache-0424}/torch:/root/.cache/torch
      - ${CACHE_ROOT:-./cache-0424}/triton:/root/.cache/triton
      - ${CACHE_ROOT:-./cache-0424}/torchinductor:/root/.cache/torchinductor
      - ${CACHE_ROOT:-./cache-0424}/flashinfer:/root/.cache/flashinfer
      - ${CACHE_ROOT:-./cache-0424}/huggingface:/root/.cache/huggingface
    ulimits:
      memlock: -1
      stack: 67108864
    restart: "no"

- SPEC_CONFIG={"model":"festr2/kimi-k2.6-eagle3-mla-fp8","method":"eagle3","num_speculative_tokens":1,"draft_attention_backend":"TRITON_MLA","draft_kv_cache_dtype":"fp8","rejection_sample_method":"probabilistic"}

FROM nvidia/cuda:13.0.0-devel-ubuntu24.04

ARG CUTLASS_DSL_VERSION=4.4.2
ARG NCCL_VERSION=2.29.7
ARG TRITON_VERSION=3.6.0
ARG FLASHINFER_VERSION=0.6.10rc1
ARG TORCH_VERSION=2.11.0+cu130
ARG TORCHVISION_VERSION=0.26.0+cu130
ARG VLLM_FORK_COMMIT=79369d6baa20bf676e8a4e85e805b488e43fc078

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
        python3.12 python3.12-dev python3.12-venv python3-pip \
        git build-essential ninja-build cmake ca-certificates curl wget \
    && rm -rf /var/lib/apt/lists/* \
    && ln -sf /usr/bin/python3.12 /usr/bin/python3 \
    && ln -sf /usr/bin/python3.12 /usr/bin/python

RUN python3 -m venv /opt/venv
ENV PATH=/opt/venv/bin:$PATH

RUN pip install --no-cache-dir --upgrade pip "setuptools>=77.0.3,<81.0.0" setuptools-scm wheel ninja

RUN pip install --no-cache-dir \
        torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION} \
        --index-url https://download.pytorch.org/whl/cu130

RUN pip install --no-cache-dir --no-deps \
        nvidia-cutlass-dsl==${CUTLASS_DSL_VERSION} \
        nvidia-cutlass-dsl-libs-base==${CUTLASS_DSL_VERSION} \
        nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION} \
        nvidia-nccl-cu13==${NCCL_VERSION} \
        triton==${TRITON_VERSION}

RUN git clone https://github.com/voipmonitor/vllm /opt/vllm \
    && cd /opt/vllm \
    && git checkout ${VLLM_FORK_COMMIT}

WORKDIR /opt/vllm
RUN pip install -e . --no-build-isolation

RUN pip install --no-cache-dir --no-deps \
        "flashinfer-python==${FLASHINFER_VERSION}" \
        "flashinfer-cubin==${FLASHINFER_VERSION}" \
    && pip install --no-cache-dir --no-deps --index-url https://flashinfer.ai/whl/cu130 \
        "flashinfer-jit-cache==${FLASHINFER_VERSION}+cu130"

ENV NCCL_PROTO=LL,LL128,Simple \
    NCCL_P2P_LEVEL=SYS \
    NCCL_IB_DISABLE=1 \
    VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel \
    VLLM_LOG_STATS_INTERVAL=1 \
    VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 \
    VLLM_ENABLE_PCIE_ALLREDUCE=0 \
    CUDA_DEVICE_ORDER=PCI_BUS_ID

COPY run-k26.sh /opt/run-k26.sh
RUN chmod +x /opt/run-k26.sh

ENTRYPOINT ["/opt/run-k26.sh"]

#!/usr/bin/env bash
set -euo pipefail
exec vllm serve "${MODEL:-/model}" \
  --served-model-name="${SERVED_MODEL_NAME:-Kimi-K2.6}" \
  --trust-remote-code \
  --host="${HOST:-0.0.0.0}" \
  --port="${PORT:-8402}" \
  --tensor-parallel-size="${TP_SIZE:-8}" \
  --pipeline-parallel-size=1 \
  --decode-context-parallel-size="${DCP_SIZE:-8}" \
  --enable-chunked-prefill \
  --enable-prefix-caching \
  --load-format=fastsafetensors \
  --async-scheduling \
  --gpu-memory-utilization="${GPU_MEMORY_UTILIZATION:-0.97}" \
  --max-num-batched-tokens="${MAX_NUM_BATCHED_TOKENS:-8192}" \
  --max-num-seqs="${MAX_NUM_SEQS:-128}" \
  --max-model-len="${MAX_MODEL_LEN:-262144}" \
  --mm-processor-cache-gb=0 \
  --mm-encoder-tp-mode=weights \
  --attention-backend="${ATTENTION_BACKEND:-TRITON_MLA}" \
  --kv-cache-dtype="${KV_CACHE_DTYPE:-fp8}" \
  --tool-call-parser=kimi_k2 \
  --enable-auto-tool-choice \
  --reasoning-parser=kimi_k2 \
  --speculative-config="${SPEC_CONFIG:-{\"model\":\"festr2/kimi-k2.6-eagle3-mla-fp8\",\"method\":\"eagle3\",\"num_speculative_tokens\":3,\"draft_attention_backend\":\"TRITON_MLA\",\"draft_kv_cache_dtype\":\"fp8\",\"rejection_sample_method\":\"probabilistic\"}}" \
  "$@"