diff --git a/deployments/systemd/nvidia-cdi-refresh.env b/deployments/systemd/nvidia-cdi-refresh.env
index 2d510ac6e..45315914b 100644
--- a/deployments/systemd/nvidia-cdi-refresh.env
+++ b/deployments/systemd/nvidia-cdi-refresh.env
@@ -18,3 +18,12 @@
 # For example, to change the for the generated CDI specification update and
 # uncomment the following line:
 # NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml
+
+# The service also runs
+#   nvidia-ctk system create-device-nodes --control-devices --load-kernel-modules
+# before generating the CDI specification. Its driver and device roots can be
+# overridden by uncommenting the following lines (nvidia-ctk cdi generate uses
+# the separate NVIDIA_CTK_DRIVER_ROOT and NVIDIA_CTK_DEV_ROOT variables):
+#
+# NVIDIA_DRIVER_ROOT=/
+# NVIDIA_DEV_ROOT=/
diff --git a/deployments/systemd/nvidia-cdi-refresh.service b/deployments/systemd/nvidia-cdi-refresh.service
index 7c483c81a..974c7b8f8 100644
--- a/deployments/systemd/nvidia-cdi-refresh.service
+++ b/deployments/systemd/nvidia-cdi-refresh.service
@@ -29,6 +29,13 @@ Environment=NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml
 EnvironmentFile=-/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env
 ExecCondition=/bin/sh -c '/usr/bin/grep -qE "/(nvidia|nvidia-current)[.]ko" /lib/modules/%v/modules.dep || [ -e /dev/dxg ]'
 ExecStart=/bin/sh -c '/usr/bin/nvidia-smi -L || /usr/sbin/nvidia-smi -L || /usr/lib/wsl/lib/nvidia-smi -L'
+# Create any missing NVIDIA control device nodes (loading the kernel modules
+# they require) before generating the CDI specification; nothing else is
+# guaranteed to create them. Best-effort, and skipped on WSL where /dev/dxg is
+# used instead, as well as on Tegra platforms using the nvgpu kernel module.
+# If module loading fails (for example while a driver upgrade has replaced the
+# on-disk modules) still create the nodes for the modules that are loaded.
+ExecStart=-/bin/sh -c '[ -e /dev/dxg ] || /usr/bin/grep -q "^nvgpu " /proc/modules || /usr/bin/nvidia-ctk system create-device-nodes --control-devices --load-kernel-modules || /usr/bin/nvidia-ctk system create-device-nodes --control-devices'
 ExecStart=/usr/bin/nvidia-ctk cdi generate
 CapabilityBoundingSet=CAP_SYS_MODULE CAP_SYS_ADMIN CAP_MKNOD
 # We set the service to restart on failure to ensure that a CDI spec is
diff --git a/deployments/udev/99-nvidia-cdi-refresh.rules b/deployments/udev/99-nvidia-cdi-refresh.rules
new file mode 100644
index 000000000..002b3d85d
--- /dev/null
+++ b/deployments/udev/99-nvidia-cdi-refresh.rules
@@ -0,0 +1,22 @@
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The NVIDIA control device nodes (/dev/nvidiactl, /dev/nvidia-modeset, and
+# /dev/nvidia-uvm*) are created on demand by userspace and are not provided by
+# devtmpfs, so loading the NVIDIA kernel module does not guarantee that they
+# exist. Trigger nvidia-cdi-refresh.service on module load to create the
+# missing nodes and refresh the CDI specification.
+# Debian packages the kernel module as nvidia-current, which appears in sysfs
+# as nvidia_current.
+ACTION=="add", SUBSYSTEM=="module", KERNEL=="nvidia|nvidia_current", TAG+="systemd", ENV{SYSTEMD_WANTS}+="nvidia-cdi-refresh.service"
diff --git a/docker/Dockerfile.debian b/docker/Dockerfile.debian
index 52138ac63..2f0905324 100644
--- a/docker/Dockerfile.debian
+++ b/docker/Dockerfile.debian
@@ -56,6 +56,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 COPY deployments/systemd/ .
+COPY deployments/udev/ .
 
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
diff --git a/docker/Dockerfile.opensuse-leap b/docker/Dockerfile.opensuse-leap
index 945155dc1..d14a77e1b 100644
--- a/docker/Dockerfile.opensuse-leap
+++ b/docker/Dockerfile.opensuse-leap
@@ -48,6 +48,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 COPY deployments/systemd/ ${DIST_DIR}/
+COPY deployments/udev/ ${DIST_DIR}/
 
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
diff --git a/docker/Dockerfile.rpm-yum b/docker/Dockerfile.rpm-yum
index 6a54355c6..cea9bd8d0 100644
--- a/docker/Dockerfile.rpm-yum
+++ b/docker/Dockerfile.rpm-yum
@@ -76,6 +76,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 COPY deployments/systemd/ ${DIST_DIR}/
+COPY deployments/udev/ ${DIST_DIR}/
 
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu
index 63e2e5cc7..c71e23a12 100644
--- a/docker/Dockerfile.ubuntu
+++ b/docker/Dockerfile.ubuntu
@@ -54,6 +54,7 @@ RUN make PREFIX=${DIST_DIR} cmds
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 COPY deployments/systemd/ .
+COPY deployments/udev/ .
 
 RUN dch --create --package="${PKG_NAME}" \
         --newversion "${REVISION}" \
diff --git a/packaging/debian/nvidia-container-toolkit-base.install b/packaging/debian/nvidia-container-toolkit-base.install
index 09a3eb8b4..d30c63be0 100644
--- a/packaging/debian/nvidia-container-toolkit-base.install
+++ b/packaging/debian/nvidia-container-toolkit-base.install
@@ -4,3 +4,4 @@ nvidia-cdi-hook /usr/bin
 nvidia-cdi-refresh.service /lib/systemd/system/
 nvidia-cdi-refresh.path /lib/systemd/system/
 nvidia-cdi-refresh.env /etc/nvidia-container-toolkit/
+99-nvidia-cdi-refresh.rules /lib/udev/rules.d/
diff --git a/packaging/debian/nvidia-container-toolkit-base.postinst b/packaging/debian/nvidia-container-toolkit-base.postinst
index 7ee72e46f..a237f171e 100644
--- a/packaging/debian/nvidia-container-toolkit-base.postinst
+++ b/packaging/debian/nvidia-container-toolkit-base.postinst
@@ -5,6 +5,12 @@ set -e
 case "$1" in
     configure)
         /usr/bin/nvidia-ctk --quiet config --config-file=/etc/nvidia-container-runtime/config.toml --in-place
+
+        # Reload udev rules so that the nvidia-cdi-refresh rules take effect
+        # without a reboot.
+        if command -v udevadm >/dev/null 2>&1; then
+            udevadm control --reload-rules >/dev/null 2>&1 || true
+        fi
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)
diff --git a/packaging/debian/rules b/packaging/debian/rules
index dd5e0966b..90b3afb39 100755
--- a/packaging/debian/rules
+++ b/packaging/debian/rules
@@ -16,3 +16,4 @@ override_dh_fixperms:
 	chmod 755 debian/$(shell dh_listpackages)/usr/bin/nvidia-cdi-hook || true
 	chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.service || true
 	chmod 644 debian/$(shell dh_listpackages)/lib/systemd/system/nvidia-cdi-refresh.path || true
+	chmod 644 debian/$(shell dh_listpackages)/lib/udev/rules.d/99-nvidia-cdi-refresh.rules || true
diff --git a/packaging/rpm/SPECS/nvidia-container-toolkit.spec b/packaging/rpm/SPECS/nvidia-container-toolkit.spec
index 606d45d57..daeb13246 100644
--- a/packaging/rpm/SPECS/nvidia-container-toolkit.spec
+++ b/packaging/rpm/SPECS/nvidia-container-toolkit.spec
@@ -21,6 +21,7 @@ Source7: nvidia-cdi-refresh.service
 Source8: nvidia-cdi-refresh.path
 Source9: nvidia-cdi-refresh.env
 Source10: 90-nvidia-container-toolkit.preset
+Source11: 99-nvidia-cdi-refresh.rules
 
 %if 0%{?rhel} == 7 || 0%{?amzn} == 2
 BuildRequires: systemd
@@ -28,6 +29,10 @@ BuildRequires: systemd
 BuildRequires: systemd-rpm-macros
 %endif
 
+# Distributions that don't define the udev rules directory macro fall back to
+# the default location.
+%{!?_udevrulesdir: %global _udevrulesdir %{_prefix}/lib/udev/rules.d}
+
 Obsoletes: nvidia-container-runtime <= 3.5.0-1, nvidia-container-runtime-hook <= 1.4.0-2
 Provides: nvidia-container-runtime
 Provides: nvidia-container-runtime-hook
@@ -38,12 +43,13 @@ Requires: nvidia-container-toolkit-base == %{version}-%{release}
 Provides tools and utilities to enable GPU support in containers.
 
 %prep
-cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} %{SOURCE9} %{SOURCE10} .
+cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} %{SOURCE9} %{SOURCE10} %{SOURCE11} .
 
 %install
 mkdir -p %{buildroot}%{_bindir}
 mkdir -p %{buildroot}%{_unitdir}
 mkdir -p %{buildroot}%{_presetdir}
+mkdir -p %{buildroot}%{_udevrulesdir}
 mkdir -p %{buildroot}%{_sysconfdir}/nvidia-container-toolkit
 
 install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime-hook
@@ -55,6 +61,7 @@ install -m 755 -t %{buildroot}%{_bindir} nvidia-cdi-hook
 install -m 644 -t %{buildroot}%{_unitdir} nvidia-cdi-refresh.service
 install -m 644 -t %{buildroot}%{_unitdir} nvidia-cdi-refresh.path
 install -m 644 -t %{buildroot}%{_presetdir} 90-nvidia-container-toolkit.preset
+install -m 644 -t %{buildroot}%{_udevrulesdir} 99-nvidia-cdi-refresh.rules
 install -m 644 -t %{buildroot}%{_sysconfdir}/nvidia-container-toolkit nvidia-cdi-refresh.env
 
 %post
@@ -129,6 +136,12 @@ for unit in nvidia-cdi-refresh.path nvidia-cdi-refresh.service; do
   fi
 done
 
+# Reload udev rules so that the nvidia-cdi-refresh rules take effect without a
+# reboot.
+if command -v udevadm >/dev/null 2>&1; then
+  udevadm control --reload-rules >/dev/null 2>&1 || :
+fi
+
 # Trigger CDI refresh on running systemd hosts without making install depend on
 # the current system state.
 if command -v systemctl >/dev/null 2>&1; then
@@ -148,6 +161,7 @@ fi
 %{_unitdir}/nvidia-cdi-refresh.service
 %{_unitdir}/nvidia-cdi-refresh.path
 %{_presetdir}/90-nvidia-container-toolkit.preset
+%{_udevrulesdir}/99-nvidia-cdi-refresh.rules
 %config(noreplace) %{_sysconfdir}/nvidia-container-toolkit/nvidia-cdi-refresh.env
 
 # The OPERATOR EXTENSIONS package consists of components that are required to enable GPU support in Kubernetes.
diff --git a/tests/e2e/nvidia-cdi-refresh_test.go b/tests/e2e/nvidia-cdi-refresh_test.go
index 21265dedf..13e894b28 100644
--- a/tests/e2e/nvidia-cdi-refresh_test.go
+++ b/tests/e2e/nvidia-cdi-refresh_test.go
@@ -93,6 +93,13 @@ EOF
 	fi
 	`
 
+	nvidiaCdiRefreshUdevRulesInstalledTemplate = `
+	if [ ! -f /lib/udev/rules.d/99-nvidia-cdi-refresh.rules ]; then
+		echo "99-nvidia-cdi-refresh.rules is not installed"
+		exit 1
+	fi
+	`
+
 	nvidiaCdiRefreshFileExistsTemplate = `
 	# is /var/run/cdi/nvidia.yaml exists? and exit with 0 if it does not exist
 	if [ ! -f /var/run/cdi/nvidia.yaml ]; then
@@ -209,6 +216,11 @@ var _ = Describe("nvidia-cdi-refresh", Ordered, ContinueOnFailure, Label("system
 			Expect(err).ToNot(HaveOccurred())
 		})
 
+		It("should install the nvidia-cdi-refresh udev rules", func(ctx context.Context) {
+			_, _, err := systemdRunner.Run(nvidiaCdiRefreshUdevRulesInstalledTemplate)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("should generate the nvidia.yaml file", func(ctx context.Context) {
 			_, _, err := systemdRunner.Run(nvidiaCdiRefreshFileExistsTemplate)
 			Expect(err).ToNot(HaveOccurred())
@@ -258,6 +270,11 @@ var _ = Describe("nvidia-cdi-refresh", Ordered, ContinueOnFailure, Label("system
 			Expect(err).ToNot(HaveOccurred())
 		})
 
+		It("should install the nvidia-cdi-refresh udev rules", func(ctx context.Context) {
+			_, _, err := systemdRunner.Run(nvidiaCdiRefreshUdevRulesInstalledTemplate)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("should generate the nvidia.yaml file", func(ctx context.Context) {
 			_, _, err := systemdRunner.Run(nvidiaCdiRefreshFileExistsTemplate)
 			Expect(err).ToNot(HaveOccurred())