semianalysiswork/inference-benchmark

By semianalysiswork

•Updated over 1 year ago

Image

209

Overview Tags

semianalysiswork/inference-benchmark repository overview

⁠Inference Benchmark

⁠NVIDIA Instructions

Docker image:

Build the Docker image:

docker build -t vllm-bench:nv-v0.8.2 -f Dockerfile.nv .

In the repo, add your Hugging Face token to a file named hf_token.txt.

echo "<YOUR_HF_TOKEN>" >> hf_token.txt

To reproduce experiments, first create the results folder:

mkdir results/

Clone the vLLM repo for benchmarking scripts:

git clone https://github.com/vllm-project/vllm/tree/main

To run experiments:

python3 nv_bmk.py -g <GPU_NAME> # [GPU_NAME: h100, h200]

To print experiment results:

python3 nv_bmk.py -g <GPU_NAME> -p

⁠AMD Instructions

Docker image:

In the repo, add your Hugging Face token to a file named hf_token.txt.

echo "<YOUR_HF_TOKEN>" >> hf_token.txt

To reproduce experiments, first create the results folder:

mkdir results/

Clone the vLLM repo for benchmarking scripts:

git clone https://github.com/ROCm/vllm

To run experiments:

python3 amd_bmk.py -g <GPU_NAME> # [GPU_NAME: mi300x, mi325x]

To print experiment results:

python3 amd_bmk.py -g <GPU_NAME> -p

⁠AMD

import json
import subprocess
import time
from argparse import ArgumentParser
from pathlib import Path


parser = ArgumentParser()
parser.add_argument('-g', '--gpu', type=str, choices=['mi300x', 'mi325x'], required=True)
parser.add_argument('-p', '--print-results', action='store_true', default=False)
args = parser.parse_args()

if args.print_results:
    print(f'config, tp, conc, mnbt, ttft, tpot, itl, e2el, total_tput')


def launch_bmk_llama(model_name, input_len, output_len, tp_size, max_concurrency, max_num_batched_tokens):
    if model_name == 'meta-llama/Llama-3.1-70B':
        model_code = '70b'
    elif model_name == 'amd/Llama-3.1-405B-Instruct-FP8-KV':
        model_code = '405b'
    else:
        raise ValueError(f'{model_name} not supported')

    result_filename = (
            f'{model_code}_tp{tp_size}_isl{input_len}_osl{output_len}_'
            f'c{max_concurrency}_mnbt{max_num_batched_tokens}'
    )
    result_file_path = Path(f'results/{result_filename}.json')

    if args.print_results:
        if not result_file_path.exists():
            return
        fields = ['median_ttft_ms', 'median_tpot_ms', 'median_itl_ms', 'median_e2el_ms', 'total_token_throughput']
        with open(result_file_path) as f:
            results = json.load(f)
        print(f'{result_filename}, {tp_size}, {max_concurrency}, {max_num_batched_tokens},', ', '.join(f'{results[f]:.3f}' for f in fields))
        return

    if result_file_path.exists():
        print(f'Skipping {result_filename}')
        return

    network_name = 'bmk-net'
    server_name = 'bmk-server'
    port = 8000
    image_name = 'rocm/vllm:rocm6.3.1_vllm_0.8.5_20250513'

    max_model_len = int((input_len + output_len) * 1.125)
    dist_backend = '--distributed-executor-backend mp' if tp_size > 1 else ''
    quant_flags = '--dtype bfloat16 --quantization fp8 --kv-cache-dtype fp8' if model_code == '405b' else ''

    if max_concurrency >= 64:
        aiter_flags = '-e VLLM_ROCM_USE_AITER=1'
        if model_code == '405b':
            aiter_flags += 'VLLM_ROCM_USE_AITER_PAGED_ATTN=1'
    else:
        aiter_flags = ''

    script = f'''#!/usr/bin/env bash
docker network create {network_name}

docker run --rm -d --network {network_name} --ipc host --name {server_name} \
    --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
    --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
    -e VLLM_USE_TRITON_FLASH_ATTN=0 {aiter_flags} \
    -v "$PWD/.hf_cache/":/root/.cache/huggingface/hub/ -v "$PWD/.vllm_cache/":/root/.cache/vllm/ -e HF_TOKEN="$(cat hf_token.txt)" \
    {image_name} \
    vllm serve {model_name} --port {port} \
        --tensor-parallel-size {tp_size} {dist_backend} {quant_flags} \
        --max-num-batched-tokens {max_num_batched_tokens} --max-num-seqs {max_concurrency} \
        --max-model-len {max_model_len} --max-seq-len-to-capture {max_model_len} \
        --swap-space 16 --num-scheduler-steps 10 \
        --disable-log-requests

printf "RESULT_FILENAME=%s\n" "{result_filename}"
while ! docker logs {server_name} 2>&1 | grep -q "Application startup complete."; do
    sleep 1
    if docker logs {server_name} 2>&1 | grep -q "ERROR"; then
        docker logs {server_name} >& "failed_runs/{result_filename}.log"
        docker stop {server_name}; docker network rm {network_name}
        exit 1
    fi
done

docker run --rm -t --network {network_name} --name bmk-client \
    --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
    --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
    -v $PWD:/workspace/ -w /workspace/vllm/benchmarks/ -e HF_TOKEN=$(cat hf_token.txt) \
    {image_name} \
        python benchmark_serving.py \
            --model {model_name} --backend vllm --base-url "http://{server_name}:{port}" \
            --dataset-name "random" --random-input-len {input_len} --random-output-len {output_len} --random-prefix-len 0 \
            --num-prompts $(( {max_concurrency} * 10 )) --max-concurrency {max_concurrency} --request-rate "inf" --ignore-eos \
            --save-result --result-dir "/workspace/results/" --result-filename "{result_filename}.json" --percentile-metrics "ttft,tpot,itl,e2el"

docker stop {server_name}; docker network rm {network_name}
sleep 60
'''
    subprocess.run(script, shell=True, check=True)


def launch_bmk_deepseek(input_len, output_len, tp_size, max_concurrency):
    result_filename = f'dsv3_tp{tp_size}_isl{input_len}_osl{output_len}_c{max_concurrency}'
    result_file_path = Path(f'results/{result_filename}.json')

    if args.print_results:
        if not result_file_path.exists():
            return
        fields = ['median_ttft_ms', 'median_tpot_ms', 'median_itl_ms', 'median_e2el_ms', 'total_token_throughput']
        with open(result_file_path) as f:
            results = json.load(f)
        print(f'{result_filename}, {tp_size}, {max_concurrency}, -1,', ', '.join(f'{results[f]:.3f}' for f in fields))
        return

    if result_file_path.exists():
        print(f'Skipping {result_filename}')
        return

    model_name = 'deepseek-ai/DeepSeek-V3'
    network_name = 'bmk-net'
    server_name = 'bmk-server'
    port = 8000
    image_name = 'rocm/sgl-dev:upstream_20250422'

    script = f'''#!/usr/bin/env bash
docker network create {network_name}

docker run --rm -d --network {network_name} --ipc host --name {server_name} \
    --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
    --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
    -v "$PWD/.hf_cache/":/root/hf_cache/ -v "$PWD/.inductor_cache/":/tmp/torchinductor_root/ \
    -e HF_HUB_CACHE=/root/hf_cache/ -e HF_TOKEN="$(cat hf_token.txt)" -e SGLANG_AITER_MOE=1 \
    {image_name} \
    python3 -m sglang.launch_server --model-path {model_name} --host 0.0.0.0 --port {port} --tp {tp_size} --trust-remote-code \
    --chunked-prefill-size 131072 --enable-torch-compile --torch-compile-max-bs 256

printf "RESULT_FILENAME=%s\n" "{result_filename}"
while ! docker logs {server_name} 2>&1 | grep -q "The server is fired up and ready to roll!"; do
    sleep 1
done

docker run --rm -t --network {network_name} --name bmk-client \
    --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
    --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
    -v $PWD:/workspace/ -w /workspace/vllm/benchmarks/ -e HF_TOKEN=$(cat hf_token.txt) \
    rocm/vllm:rocm6.3.1_instinct_vllm0.8.3_20250410 \
        python benchmark_serving.py \
            --model {model_name} --backend vllm --base-url "http://{server_name}:{port}" \
            --dataset-name "random" --random-input-len {input_len} --random-output-len {output_len} --random-prefix-len 0 \
            --num-prompts $(( {max_concurrency} * 10 )) --max-concurrency {max_concurrency} --request-rate "inf" --ignore-eos \
            --save-result --result-dir "/workspace/results/" --result-filename "{result_filename}.json" --percentile-metrics "ttft,tpot,itl,e2el"

docker stop {server_name}; docker network rm {network_name}
sleep 60
'''
    subprocess.run(script, shell=True, check=True)


if args.gpu == 'mi300x':
    max_num_batched_tokens = 65536

    for input_len, output_len in [(1024, 1024), (1024, 4096), (4096, 1024)]:
        t_s = time.time()
        # LLaMA 70B
        for tp_size in [1, 2, 4, 8]: 
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('meta-llama/Llama-3.1-70B', input_len, output_len, tp_size, max_concurrency, max_num_batched_tokens)
        # LLaMA 405B FP8
        for tp_size in [4, 8]:
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('amd/Llama-3.1-405B-Instruct-FP8-KV', input_len, output_len, tp_size, max_concurrency, max_num_batched_tokens)
        # DeepseekV3
        tp_size = 8
        for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
            launch_bmk_deepseek(input_len, output_len, tp_size, max_concurrency)
        t_e = time.time()
        if not args.print_results:
            print(f'ISL{input_len}/OSL{output_len} BENCHMARK TIME ELAPSED: {((t_e - t_s) / 60.0):.2f} minutes')
elif args.gpu == 'mi325x':
    max_num_batched_tokens = 65536

    for input_len, output_len in [(1024, 1024), (1024, 4096), (4096, 1024)]:
        t_s = time.time()
        # LLaMA 70B
        for tp_size in [1, 2, 4, 8]: 
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('meta-llama/Llama-3.1-70B', input_len, output_len, tp_size, max_concurrency, max_num_batched_tokens)
        # LLaMA 405B FP8
        for tp_size in [4, 8]:
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('amd/Llama-3.1-405B-Instruct-FP8-KV', input_len, output_len, tp_size, max_concurrency, max_num_batched_tokens)
        # DeepseekV3
        tp_size = 8
        for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
            launch_bmk_deepseek(input_len, output_len, tp_size, max_concurrency)
        t_e = time.time()
        if not args.print_results:
            print(f'ISL{input_len}/OSL{output_len} BENCHMARK TIME ELAPSED: {((t_e - t_s) / 60.0):.2f} minutes')
else:
    raise ValueError(f'Unknown GPU {args.gpu}')

⁠NVIDIA Docker

FROM vllm/vllm-openai:v0.8.2
RUN pip install datasets pandas

⁠NVIDIA BMK

import json
import subprocess
import time
from argparse import ArgumentParser
from pathlib import Path


parser = ArgumentParser()
parser.add_argument('-g', '--gpu', type=str, choices=['h100', 'h200'], required=True)
parser.add_argument('-p', '--print-results', action='store_true', default=False)
args = parser.parse_args()

if args.print_results:
    print(f'config, tp, conc, mnbt, ttft, tpot, itl, e2el, total_tput')


def launch_bmk_llama(model_name, input_len, output_len, tp_size, max_concurrency, max_num_seqs, max_num_batched_tokens):
    model_handle = '70b' if '70' in model_name else '405b'
    result_filename = (
            f'{model_handle}_tp{tp_size}_isl{input_len}_osl{output_len}_'
            f'c{max_concurrency}_s{max_num_seqs}_mnbt{max_num_batched_tokens}'
    )
    result_file_path = Path(f'results/{result_filename}.json')

    if args.print_results:
        if not result_file_path.exists():
            return
        fields = ['median_ttft_ms', 'median_tpot_ms', 'median_itl_ms', 'median_e2el_ms', 'total_token_throughput']
        with open(result_file_path) as f:
            results = json.load(f)
        print(f'{result_filename}, {tp_size}, {max_concurrency}, {max_num_batched_tokens},', ', '.join(f'{results[f]:.3f}' for f in fields))
        return

    if result_file_path.exists():
        return

    network_name = 'bmk-net'
    server_name = 'bmk-server'
    port = 8000
    image_name = 'vllm-bench:nv-v0.8.2'

    dist_backend = '--distributed-executor-backend mp' if tp_size > 1 else ''
    quant_flags = '--dtype bfloat16 --quantization fbgemm_fp8' if 'FP8' in model_name else ''
    max_model_len = int((input_len + output_len) * 1.125)

    script = f'''
docker network create {network_name}

docker run --rm -d --network {network_name} --name {server_name} \
    --runtime nvidia --gpus all --ipc host --privileged --ulimit memlock=-1 --ulimit stack=67108864 \
    -v "$PWD/.hf_cache/":/root/.cache/huggingface/hub/ -v "$PWD/.vllm_cache/":/root/.cache/vllm/ -e HF_TOKEN="$(cat hf_token.txt)" \
    {image_name} \
        --model {model_name} --port {port} \
        --tensor-parallel-size {tp_size} --distributed-executor-backend mp \
        --max-num-seqs {max_num_seqs} --enable-chunked-prefill false --gpu-memory-utilization 0.95 \
        --max-model-len {max_model_len} --max-seq-len-to-capture {max_model_len} \
        --disable-log-requests

printf 'RESULT_FILENAME: %s\n' "{result_filename}"
while ! docker logs {server_name} 2>&1 | grep -q "Application startup complete."; do
    sleep 1
done

docker run --rm -t --network {network_name} --name bmk-client \
    --runtime nvidia \
    -v $PWD:/workspace/ -w /workspace/vllm/benchmarks/ -e HF_TOKEN="$(cat hf_token.txt)" \
    --entrypoint "/usr/bin/python3" vllm-bench:nv-v0.8.2 \
    benchmark_serving.py \
        --model {model_name} --backend vllm --base-url "http://{server_name}:{port}" \
        --dataset-name "random" --random-input-len {input_len} --random-output-len {output_len} --random-prefix-len 0 \
        --num-prompts $(( {max_concurrency} * 10 )) --max-concurrency {max_concurrency} --request-rate "inf" --ignore-eos \
        --save-result --result-dir "/workspace/results/" --result-filename "{result_filename}.json" --percentile-metrics "ttft,tpot,itl,e2el"

docker stop {server_name}; docker network rm {network_name}
'''
    subprocess.run(script, shell=True, check=True)


def launch_bmk_deepseek(input_len, output_len, tp_size, max_concurrency, max_num_seqs):
    result_filename = f'dsv3_tp{tp_size}_isl{input_len}_osl{output_len}_c{max_concurrency}_s{max_num_seqs}'
    result_file_path = Path(f'results/{result_filename}.json')

    if args.print_results:
        if not result_file_path.exists():
            return
        fields = ['median_ttft_ms', 'median_tpot_ms', 'median_itl_ms', 'median_e2el_ms', 'total_token_throughput']
        with open(result_file_path) as f:
            results = json.load(f)
        print(f'{result_filename}, {tp_size}, {max_concurrency}, -1,', ', '.join(f'{results[f]:.3f}' for f in fields))
        return

    if result_file_path.exists():
        print(f'Skipping {result_filename}')
        return

    model_name = 'deepseek-ai/DeepSeek-V3'
    image_name = 'lmsysorg/sglang:v0.4.6.post4-cu124'
    network_name = 'bmk-net'
    server_name = 'bmk-server'
    port = 8000

    script = f'''#!/usr/bin/env bash
docker network create {network_name}

docker run --rm -d --network {network_name} --name {server_name} \
    --runtime nvidia --gpus all --ipc host --privileged --ulimit memlock=-1 --ulimit stack=67108864 \
    -v "$PWD/.hf_cache/":/root/.cache/huggingface/hub/ -v "$PWD/.inductor_cache/":/tmp/torchinductor_root/ -e HF_TOKEN="$(cat hf_token.txt)" \
    -v "$PWD/.dg_cache/":/root/.cache/deep_gemm/ -e SGL_ENABLE_JIT_DEEPGEMM=1 \
    {image_name} \
    python3 -m sglang.launch_server --model-path {model_name} --host 0.0.0.0 --port {port} --tp {tp_size} --trust-remote-code

printf 'RESULT_FILENAME: %s\n' "{result_filename}"
while ! docker logs {server_name} 2>&1 | grep -q "The server is fired up and ready to roll!"; do
    sleep 1
done

docker run --rm -t --network {network_name} --name bmk-client \
    --runtime nvidia \
    -v $PWD:/workspace/ -w /workspace/vllm/benchmarks/ -e HF_TOKEN="$(cat hf_token.txt)" \
    --entrypoint "/usr/bin/python3" vllm-bench:nv-v0.8.2 \
    benchmark_serving.py \
        --model {model_name} --backend sglang --base-url "http://{server_name}:{port}" \
        --dataset-name "random" --random-input-len {input_len} --random-output-len {output_len} --random-prefix-len 0 \
        --num-prompts $(( {max_concurrency} * 10 )) --max-concurrency {max_concurrency} --request-rate "inf" --ignore-eos \
        --save-result --result-dir "/workspace/results/" --result-filename "{result_filename}.json" --percentile-metrics "ttft,tpot,itl,e2el"

docker rename {server_name} {server_name}-old; docker stop {server_name}-old; docker network rm {network_name}
'''
    subprocess.run(script, shell=True, check=True)


if args.gpu == 'h100':
    max_num_batched_tokens = 8192  # V1 engine default
    for input_len, output_len in [(1024, 1024), (1024, 4096), (4096, 1024)]:
        t_s = time.time()

        # LLaMA 70B
        for tp_size in [4, 8]:
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('meta-llama/Llama-3.1-70B', input_len, output_len, tp_size, max_concurrency, max_concurrency, max_num_batched_tokens)
        # LLaMA 405B FP8
        tp_size = 8
        for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
            launch_bmk_llama('meta-llama/Llama-3.1-405B-FP8', input_len, output_len, tp_size, max_concurrency, max_concurrency, max_num_batched_tokens)

        t_e = time.time()
        print(f'ISL{input_len}/OSL{output_len} BENCHMARK TIME ELAPSED: {((t_e - t_s) / 60.0):.2f} minutes')

elif args.gpu == 'h200':
    max_num_batched_tokens = 8192  # V1 engine default

    for input_len, output_len in [(1024, 1024), (1024, 4096), (4096, 1024)]:
        t_s = time.time()

        # LLaMA 70B
        for tp_size in [2, 4, 8]:
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('meta-llama/Llama-3.1-70B', input_len, output_len, tp_size, max_concurrency, max_concurrency, max_num_batched_tokens)
        # LLaMA 405B FP8
        for tp_size in [4, 8]:
            for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
                launch_bmk_llama('meta-llama/Llama-3.1-405B-FP8', input_len, output_len, tp_size, max_concurrency, max_concurrency, max_num_batched_tokens)
        # DeepseekV3
        tp_size = 8
        for max_concurrency in [4, 8, 16, 32, 64, 128, 256]: 
            launch_bmk_deepseek(input_len, output_len, tp_size, max_concurrency, max_concurrency)

        t_e = time.time()
        print(f'ISL{input_len}/OSL{output_len} BENCHMARK TIME ELAPSED: {((t_e - t_s) / 60.0):.2f} minutes')

⁠TRT BMK

import json
import subprocess
from argparse import ArgumentParser
from pathlib import Path


parser = ArgumentParser()
parser.add_argument('-p', '--print-results', action='store_true', default=False)
args = parser.parse_args()

if args.print_results:
    print(f'config, tp, conc, mnbt, ttft, tpot, itl, e2el, total_tput')


def launch_bmk_trt(model_name, input_len, output_len, tp_size, max_concurrency, max_num_tokens):
    model_handle = model_name.split('/')[1].split('-')[2].lower()

    result_filename = f'trt_{model_handle}_tp{tp_size}_isl{input_len}_osl{output_len}_c{max_concurrency}_mnt{max_num_tokens}'
    result_file_path = Path(f'results/{result_filename}.json')

    if args.print_results:
        if not result_file_path.exists():
            return
        fields = ['median_ttft_ms', 'median_tpot_ms', 'median_itl_ms', 'median_e2el_ms', 'total_token_throughput']
        with open(result_file_path) as f:
            results = json.load(f)
        print(f'{result_filename}, {tp_size}, {max_concurrency}, -1,', ', '.join(f'{results[f]:.3f}' for f in fields))
        return

    if result_file_path.exists():
        return

    network_name = 'bmk-net'
    server_name = 'bmk-server'
    port = 8000
    image_name = '' # build trtllm-serve container

    script = f'''#!/usr/bin/env bash
docker network create {network_name}

docker run --rm -d --network {network_name} --name {server_name} \
    --runtime nvidia --gpus all --ipc host --privileged --ulimit memlock=-1 --ulimit stack=67108864 \
    -v "$PWD/.hf_cache/":/root/.cache/huggingface/hub/ -v "$PWD/configs_trtllm/":/root/config/ -e HF_TOKEN="$(cat hf_token.txt)" \
    {image_name} \
    trtllm-serve {model_name} --host 0.0.0.0 --port {port} --backend pytorch --tp_size {tp_size} --max_num_tokens {max_num_tokens} \
    --extra_llm_api_options /root/config/extra_llm_options.yml

printf 'RESULT_FILENAME: %s\n' "{result_filename}"
while ! docker logs {server_name} 2>&1 | grep -q "Application startup complete."; do
    sleep 1
done

docker run --rm -t --network {network_name} --name bmk-client \
    --runtime nvidia \
    -v $PWD:/workspace/ -w /workspace/vllm/benchmarks/ -e HF_TOKEN="$(cat hf_token.txt)" \
    --entrypoint "/usr/bin/python3" vllm-bench:nv-v0.8.2 \
    benchmark_serving.py \
        --model {model_name} --backend vllm --base-url "http://{server_name}:{port}" \
        --dataset-name "random" --random-input-len {input_len} --random-output-len {output_len} --random-prefix-len 0 \
        --num-prompts $(( {max_concurrency} * 10 )) --max-concurrency {max_concurrency} --request-rate "inf" --ignore-eos \
        --save-result --result-dir "/workspace/results/" --result-filename "{result_filename}.json" --percentile-metrics "ttft,tpot,itl,e2el"

docker stop {server_name}; docker network rm {network_name}
'''
    subprocess.run(script, shell=True, check=True)


for input_len, output_len, max_num_tokens in [(1024, 1024, 2500), (1024, 4096, 5500), (4096, 1024, 5500)]:
    for tp_size in [2, 4, 8]:
        for max_concurrency in [256, 128, 64, 32, 16, 8, 4]:
            launch_bmk_trt('meta-llama/Llama-3.1-70B', input_len, output_len, tp_size, max_concurrency, max_num_tokens)
    for tp_size in [4, 8]:
        for max_concurrency in [256, 128, 64, 32, 16, 8, 4]:
            launch_bmk_trt('nvidia/Llama-3.1-405B-Instruct-FP8', input_len, output_len, tp_size, max_concurrency, max_num_tokens)

⁠extra_llm_options.yml

pytorch_backend_config:
  enable_overlap_scheduler: true
  use_cuda_graph: true
  cuda_graph_max_batch_size: 256
  cuda_graph_padding_enabled: true

Tag summary

Recent tags

Content type

Image

Digest

sha256:e9ad643f1…

Size

8.5 GB

Last updated

over 1 year ago

docker pull semianalysiswork/inference-benchmark