holtmann/llm-benchmark repository overview

docker pull holtmann/llm-benchmark:latest

# Run interactively
docker run --gpus all -it holtmann/llm-benchmark:latest

# Start vLLM server
docker run --gpus all -p 8000:8000 holtmann/llm-benchmark:latest \
  python -m vllm.entrypoints.openai.api_server \
  --model nvidia/Qwen3-8B-NVFP4 \
  --max-model-len 8192

# Start container with GPU access
docker run --gpus all -it \
  -e HF_TOKEN=your_token \
  -v $(pwd)/results:/workspace/results \
  holtmann/llm-benchmark:latest

# Inside container: start vLLM
python -m vllm.entrypoints.openai.api_server \
  --model nvidia/Qwen3-8B-NVFP4 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9 &

# Run AIPerf benchmark
aiperf \
  --endpoint-type chat \
  --base-url http://localhost:8000 \
  --model nvidia/Qwen3-8B-NVFP4 \
  --streaming \
  --concurrency 8 \
  --num-requests 100 \
  --synthetic-input-tokens-mean 8192 \
  --output-tokens-mean 512

# Run lm-eval benchmarks
lm_eval --model vllm \
  --model_args pretrained=nvidia/Qwen3-8B-NVFP4 \
  --tasks mmlu,gsm8k,hellaswag \
  --batch_size auto

# Start DCGM monitoring (inside container)
dcgmi dmon -e 155,156,203,204 -d 1000

# Fields: Power (W), Temperature (C), GPU Util (%), Memory Util (%)