shilazi/fastchat repository overview

FROM nvidia/cuda:12.2.0-runtime-ubuntu20.04

RUN DEBIAN_FRONTEND=noninteractive apt-get update \
    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
        python3.9 python3.9-distutils curl \
    && rm -rf /var/lib/apt/lists/*
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
RUN python3.9 get-pip.py
RUN pip3 config set global.index-url https://mirror.nju.edu.cn/pypi/web/simple
RUN pip3 install fschat==0.2.31
RUN pip3 install fschat[model_worker,webui]==0.2.31 pydantic==1.10.13
RUN pip3 install bitsandbytes==0.41.1 scipy==1.11.3

version: "3.9"

services:
  fastchat-controller:
    image: shilazi/fastchat:v0.2.31
    ports:
    - "21001:21001"
    entrypoint: ["python3.9", "-m", "fastchat.serve.controller", "--host", "0.0.0.0", "--port", "21001"]
    restart: unless-stopped
  fastchat-model-worker:
    volumes:
    - ./models:/models
    image: shilazi/fastchat:v0.2.31
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            count: 1
            capabilities: [gpu]
    entrypoint: ["python3.9", "-m", "fastchat.serve.model_worker", "--model-names", "fastchat-t5-3b-v1.0", "--model-path", "/models/lmsys/fastchat-t5-3b-v1.0", "--worker-address", "http://fastchat-model-worker:21002", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "21002"]
    # https://github.com/lm-sys/FastChat#not-enough-memory
    #entrypoint: ["python3.9", "-m", "fastchat.serve.model_worker", "--model-names", "fastchat-t5-3b-v1.0", "--model-path", "/models/lmsys/fastchat-t5-3b-v1.0", "--worker-address", "http://fastchat-model-worker:21002", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "21002", "--load-8bit", "--cpu-offloading"]
    restart: unless-stopped
  fastchat-api-server:
    image: shilazi/fastchat:v0.2.31
    ports:
    - "8000:8000"
    entrypoint: ["python3.9", "-m", "fastchat.serve.openai_api_server", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "8000"]
    restart: unless-stopped
  fastchat-web-server:
    image: shilazi/fastchat:v0.2.31
    ports:
    - "7860:7860"
    entrypoint: ["python3.9", "-m", "fastchat.serve.gradio_web_server", "--controller-url", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "7860", "--concurrency-count", "10", "--model-list-mode", "reload"]
    restart: unless-stopped