longddrhz/nemo repository overview

main: DOCKER_BUILDKIT=1 docker build . --build-arg HTTP_PROXY=http://127.0.0.1:8086⁠ --build-arg HTTPS_PROXY=http://127.0.0.1:8086⁠ --network host -t nemo:main20240124

main20240124_deepspeed: we need to reinstall openssh-server, otherwise , some cuda init error will occur.

command: export NODE_LIST=xxx,xxx; pdsh -R ssh -w ${NODE_LIST} docker rm -f nemoimage0 && pdsh -R ssh -w ${NODE_LIST} docker run -d -t --privileged --cap-add SYS_ADMIN --security-opt seccomp:unconfined --device=/dev/infiniband:/dev/infiniband --gpus all --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --name nemoimage0 --network host --hostname nemoimage0 -v /cloudfs/ai2/db/workspace:/cloudfs/db -v /cloudfs/ai2/db/data:/cloudfs-data/db/ nemo:main20240124_deepspeed bash &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt auto-remove -y openssh-server &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt-get install -y openssh-server && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 service ssh restart

deepspeed_without_ssh: If with ssh, it will cause cuda error.

pdsh -R ssh -w ${NODE_LIST} docker rm -f nemoimage0 && pdsh -R ssh -w ${NODE_LIST} docker run -d -t --privileged --cap-add SYS_ADMIN --security-opt seccomp:unconfined --device=/dev/infiniband:/dev/infiniband --gpus all --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --name nemoimage0 --network host --hostname nemoimage0 -v /cloudfs/ai2/db/workspace:/cloudfs/db -v /cloudfs/ai2/db/data:/cloudfs-data/db/ nemo:main20240124_deepspeed_without_ssh bash &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt update && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt install -y openssh-server && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/# Port 22/Port 22333/g' /etc/ssh/ssh_config" && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/#Port 22/Port 22333/g' /etc/ssh/sshd_config" && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 service ssh restart && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config" &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 service ssh restart

export NODE_LIST=llm042,llm043; pdsh -R ssh -w ${NODE_LIST} docker rm -f nemoimage0 && pdsh -R ssh -w ${NODE_LIST} docker run -d -t --privileged --cap-add SYS_ADMIN --security-opt seccomp:unconfined --device=/dev/infiniband:/dev/infiniband --gpus all --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --name nemoimage0 --network host --hostname nemoimage0 -v /cloudfs/ai2/db/workspace:/cloudfs/db -v /cloudfs/ai2/db/data:/cloudfs-data/db/ nemo:main20240124 bash &&pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt update && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 apt install -y openssh-server && pdsh -R ssh -w ${NODE_LIST} docker cp /cloudfs/ai2/db/workspace/.ssh/ nemoimage0:/root/ && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/# Port 22/Port 22333/g' /etc/ssh/ssh_config" && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/#Port 22/Port 22333/g' /etc/ssh/sshd_config" && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 service ssh restart && pdsh -R ssh -w ${NODE_LIST} "docker exec nemoimage0 sed -i 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config" && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 pip install deepspeed==0.10.3 -i https://pypi.tuna.tsinghua.edu.cn/simple⁠ && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 pip install protobuf==3.20.* -i https://pypi.tuna.tsinghua.edu.cn/simple⁠ && pdsh -R ssh -w ${NODE_LIST} 'docker exec nemoimage0 bash -c "cd /cloudfs/db/pdsh && make install && ln /usr/local/bin/pdsh -s /usr/bin/pdsh"' && pdsh -R ssh -w ${NODE_LIST} docker exec nemoimage0 pip install tiktoken -i https://pypi.tuna.tsinghua.edu.cn/simple⁠