orthozany/sglang-blackwell repository overview

# Built from luke's SGLang fork with Blackwell support
# https://github.com/lukealonso/sglang @ 24994611c

FROM nvidia/cuda:13.0.1-cudnn-devel-ubuntu24.04

# See full Dockerfile at:
# https://github.com/lukealonso/sglang/blob/main/docker/Dockerfile

# Build command:
# cd <sglang-repo>
# docker build -t orthozany/sglang-blackwell:luke-ar \
#   -f docker/Dockerfile \
#   --build-arg CUDA_VERSION=13.0.1 \
#   --build-arg GRACE_BLACKWELL=1 \
#   --build-arg BUILD_AND_DOWNLOAD_PARALLEL=8 \
#   --build-arg USE_LATEST_SGLANG=0 \
#   --build-arg BRANCH_TYPE=local \
#   .

docker run --gpus all --ipc host --shm-size 32g \
  -v /path/to/models/moonshotai/Kimi-K2.5:/model:ro \
  -e NCCL_P2P_LEVEL=SYS -e NCCL_IB_DISABLE=1 -e NCCL_PROTO=LL \
  -p 8400:8000 \
  orthozany/sglang-blackwell:luke-ar \
  python3 -m sglang.launch_server \
    --model-path=/model \
    --served-model-name=Kimi-K2.5 \
    --tp=8 \
    --host=0.0.0.0 --port=8000 \
    --mem-fraction-static=0.94 \
    --attention-backend=flashinfer \
    --enable-pcie-oneshot-allreduce \
    --enable-pcie-oneshot-allreduce-fusion \
    --pcie-oneshot-allreduce-max-size=65536 \
    --enable-torch-compile \
    --enable-mixed-chunk \
    --trust-remote-code \
    --reasoning-parser=kimi_k2 \
    --tool-call-parser=kimi_k2