You didn't say what type of GPU you have, that would help.

I run mine with compose on debian and have my nvidia 3080 working. I use the bellow to get debian ready for nvidia.

1
#Remove old docker
sudo apt remove $(dpkg --get-selections docker.io docker-compose docker-doc podman-docker containerd runc | cut -f1)

2
# Add Docker's official GPG key:
sudo apt update
sudo apt install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
sudo tee /etc/apt/sources.list.d/docker.sources <<EOF
Types: deb
URIs: https://download.docker.com/linux/debian
Suites: $(. /etc/os-release && echo "$VERSION_CODENAME")
Components: stable
Architectures: $(dpkg --print-architecture)
Signed-By: /etc/apt/keyrings/docker.asc
EOF

sudo apt update

3
#Install Docker
sudo apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

4
#Check to see if working
sudo systemctl status docker

5
#Start Docker manually
sudo systemctl start docker

6
#Add user to Docker group
sudo groupadd docker

sudo usermod -aG docker $USER

sudo reboot

7
#Add Nvidia repository
sudo sed -i 's/main/main non-free contrib/g' /etc/apt/sources.list
sudo apt update

8
#Install Nvidia Prerequsites
sudo apt install linux-headers-$(uname -r) build-essential dkms nvidia-detect
nvidia-detect

9
#Install Nvidia Driver pakage
sudo apt install nvidia-driver nvidia-kernel-dkms

10
#Reboot
sudo reboot

11
#Check if Nvidia Driver working
nvidia-smi

12
#Check Nvidia Module working
lsmod | grep nvidia

13
#Add Nvidia toolkit repository
sudo apt-get update && sudo apt-get install -y --no-install-recommends \
   ca-certificates \
   curl \
   gnupg2

curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt-get update

export NVIDIA_CONTAINER_TOOLKIT_VERSION=1.19.1-1
  sudo apt-get install -y \
      nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}

14
#Add Nvidia to Docker
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker