ocpub/oc9-llama-cpp repository overview

docker build -t oc9-llama-cpp:b8981 .

# 下载模型（示例：Qwen2.5-0.5B）
wget https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf -O /tmp/model.gguf

# 运行推理
docker run --rm --gpus all \
  -v /tmp/model.gguf:/models/model.gguf \
  oc9-llama-cpp:b8981 \
  llama-cli -m /models/model.gguf -p "你好，请介绍一下你自己" -n 256

# 启动推理服务
docker run --rm --gpus all \
  -p 8080:8080 \
  -v /tmp/model.gguf:/models/model.gguf \
  oc9-llama-cpp:b8981 \
  llama-server -m /models/model.gguf --host 0.0.0.0 --port 8080

# 访问服务
curl http://localhost:8080/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "model",
    "messages": [{"role": "user", "content": "你好"}],
    "max_tokens": 128
  }'

docker run --rm --gpus all \
  -v /tmp/model.gguf:/models/model.gguf \
  oc9-llama-cpp:b8981 \
  python3 -c "
from llama_cpp import Llama
llm = Llama(model_path='/models/model.gguf', n_gpu_layers=-1)
print(llm.create_chat_completion(
    messages=[{'role': 'user', 'content': '你好'}],
    max_tokens=128
))
"

# 执行测试脚本（需要 GPU 环境）
bash test.sh oc9-llama-cpp:b8981