It seems that the cluster cannot see the GPU when running HTcondor job with Universe = Docker.
Starting with a base image, trying to see if any of the compute nodes can see the GPU.
(Mostly with help from MS Copilot)
FROM nvidia/cuda:11.8.0-base-ubuntu20.04
# Set environment variables to automatically accept prompts
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
nvidia-utils-525 \
nvidia-driver-525 \
curl wget nano \
&& rm -rf /var/lib/apt/lists/*
# Add the NVIDIA package repositories
# Add the NVIDIA package repositories
RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub | gpg --dearmor -o /usr/share/keyrings/cuda-archive-keyring.gpg && \
echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \
apt-get update && apt-get install -y \
cuda-toolkit-11-8 \
&& rm -rf /var/lib/apt/lists/*
SHELL ["/bin/bash", "-c"]
Universe = docker
docker_image = jysgro/nvidia_base:1180_ub2004
executable = gpu_test.sh
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
output = out.$(Process)
error = err.$(Process)
log = log.$(Process)
request_gpus = 1
request_cpus = 1
request_memory = 10Gb
# machine 009 is the only one with 11.8
# requirements=(machine=="xxx-0009.xxx.xxx.xxx")
queue 1
gpu_test.sh file:
If can see the GPU will provide info.
#!/bin/bash
nvidia-smi
Content type
Image
Digest
sha256:9f2f43706…
Size
4.8 GB
Last updated
almost 2 years ago
docker pull jysgro/nvidia_base:1180_ub2004