soldemeyer/koboldcpp-gpu repository overview

docker run --rm soldemeyer/koboldcpp-gpu:latest --help

# Example Compose file for soldemeyer/koboldcpp-gpu. Optional settings are commented
# out; uncomment an option together with its value.
services:
  koboldcpp:
    image: soldemeyer/koboldcpp-gpu:latest
    restart: unless-stopped

    # Web UI and APIs on http://localhost:5001. KOBOLDCPP_BIND_IP is read by Compose,
    # not the container: set it to 0.0.0.0 in a .env file next to this file to reach
    # KoboldCpp from your LAN (set KCPP_PASSWORD first). The container-side port must
    # match --port below.
    ports:
      - "${KOBOLDCPP_BIND_IP:-127.0.0.1}:5001:5001"

    # Every NVIDIA GPU. For specific ones, replace "count: all" with device_ids: ["0"].
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

    # Passwords: uncomment "environment:" and the lines you need.
    # environment:
    #   # API key required by the text endpoints (same as --password). Image endpoints
    #   # aren't protected. Default: none.
    #   KCPP_PASSWORD: "replace-with-a-key"
    #   # Password for the admin functions --admin enables (same as --adminpassword).
    #   # Default: none.
    #   KCPP_ADMINPASSWORD: "replace-with-a-password"

    volumes:
      # Models, and .kcpps settings files.
      - ./models:/models

    # Every value must be a quoted string.
    command:
      # Port inside the container. Default: 5001.
      - "--port"
      - "5001"
      # The model: a file in /models, or a https:// URL to download on start.
      - "--model"
      - "/models/model.gguf"
      # Where downloaded models are saved. Default: the working directory (/app),
      # which is lost when the container is recreated.
      - "--downloaddir"
      - "/models"
      # Use CUDA on every GPU; add a GPU number to use only that one ("--usecuda", "0").
      # Without it, KoboldCpp picks a backend itself. "--usecpu" runs on the CPU only.
      - "--usecuda"
      # Maximum context size in tokens. Default: 16384.
      - "--contextsize"
      - "16384"

      # --- GPU and performance ---------------------------------------------------
      # Layers to put on the GPU: -1 fits automatically (default), 0 puts none.
      # - "--gpulayers"
      # - "-1"
      # Split the model across GPUs by ratio, e.g. 7 3.
      # - "--tensor_split"
      # - "7"
      # - "3"
      # Main GPU in a multi-GPU setup. Default: chosen automatically.
      # - "--maingpu"
      # - "0"
      # CPU threads. Default: based on the number of CPU cores.
      # - "--threads"
      # - "8"
      # K/V cache type: f16 (default), bf16, q8_0, q5_1 or q4_0. Lower uses less memory.
      # - "--quantkv"
      # - "q8_0"

      # --- More models -----------------------------------------------------------
      # Vision: the multimodal projector for the model.
      # - "--mmproj"
      # - "/models/mmproj.gguf"
      # Speculative decoding: a small draft model with the same vocabulary.
      # - "--draftmodel"
      # - "/models/draft.gguf"
      # Image generation (.safetensors or .gguf).
      # - "--sdmodel"
      # - "/models/image-model.safetensors"
      # Speech-to-text (Whisper .bin).
      # - "--whispermodel"
      # - "/models/whisper.bin"
      # Text-to-speech (.gguf).
      # - "--ttsmodel"
      # - "/models/tts.gguf"
      # Embeddings.
      # - "--embeddingsmodel"
      # - "/models/embeddings.gguf"

      # --- Behaviour -------------------------------------------------------------
      # Use the model's Jinja chat template, including tool calls.
      # - "--jinja"
      # Load settings from a .kcpps file; arguments here override it.
      # - "--config"
      # - "/models/settings.kcpps"
      # Admin mode: switch between the .kcpps files in --admindir from the web UI.
      # - "--admin"
      # - "--admindir"
      # - "/models"
      # Leave prompts and generated text out of the log.
      # - "--quiet"
      # Serve over HTTPS with an unencrypted certificate and key (.pem). Then change
      # the healthcheck URL to https:// and add -k.
      # - "--ssl"
      # - "/models/cert.pem"
      # - "/models/key.pem"

    logging:
      driver: json-file
      options:
        max-size: 10m
        max-file: "3"

    # Only shows that the server responds. start_period allows for slow model loads.
    healthcheck:
      test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:5001/api/extra/version"]
      interval: 30s
      timeout: 5s
      start_period: 300s
      retries: 3