zen3515/functionary-vllm repository overview

$ docker run --gpus all -it --ipc=host --name functionary-vllm \
  --workdir /workspace \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  zen3515/functionary-vllm:v2.5 --model "meetkai/functionary-small-v2.5" --host "0.0.0.0" --max-model-len "8192"

services:
  functionary_vllm:
    image: zen3515/functionary-vllm:v2.5
    container_name: functionary-vllm
    restart: unless-stopped
    shm_size: "8g"
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    command:
      - --model 
      - meetkai/functionary-small-v2.5
      - --host
      - "0.0.0.0"
      - --max-model-len
      - "8192"
    ports:
      - "8000:8000"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    environment:
      HUGGING_FACE_HUB_TOKEN: "${HUGGING_FACE_HUB_TOKEN}"

# This is specifically for functionary 2.5
FROM vllm/vllm-openai:v0.4.1
ARG COMMIT=858e341bd5bf82a1a49a8270be07160b2c200a5b

RUN /bin/sh -c "pip install --no-cache-dir jsonref"

WORKDIR /functionary

# https://github.com/MeetKai/functionary/commit/858e341bd5bf82a1a49a8270be07160b2c200a5b
RUN /bin/sh -c "git init \
   && git remote add origin https://github.com/MeetKai/functionary.git \
   && git fetch origin $COMMIT \
   && git reset --hard FETCH_HEAD"

ENTRYPOINT ["python3", "server_vllm.py"]

CMD ["--model", "meetkai/functionary-small-v2.5"]