From f379f384704b6a0394aefd28d128e1bd96ce1950 Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:52:16 -0400 Subject: [PATCH 1/7] Add docs for 26.7.0 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/release-notes.rst | 18 ------------------ gpu-operator/versions1.json | 4 ++++ repo.toml | 2 +- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 394f7e2f9..5c2dec9b1 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -346,24 +346,6 @@ Known Issues Refer to the MIG Controller issue `#329 `_ for more information. -* After you delete the default NVIDIADriver custom resource, any custom NVIDIADriver - custom resources that you created might not become active automatically. - The custom resources remain in a pending state because the Operator controller - does not re-evaluate them after the conflicting default custom resource is removed. - - To work around this issue, restart the GPU Operator controller by deleting - the controller pod: - - .. code-block:: console - - $ kubectl delete pod -n gpu-operator -l app=gpu-operator - - Restarting the controller pod does not disrupt running GPU workloads or - driver pods on nodes. - - Refer to issue `#2259 `_ - for more information. - Removals and Deprecations ------------------------- diff --git a/gpu-operator/versions1.json b/gpu-operator/versions1.json index 27150f84d..508fbb389 100644 --- a/gpu-operator/versions1.json +++ b/gpu-operator/versions1.json @@ -1,6 +1,10 @@ [ { "preferred": "true", + "url": "../26.7", + "version": "26.7" + }, + { "url": "../26.3/", "version": "26.3" }, diff --git a/repo.toml b/repo.toml index 374e1d164..4395991d1 100644 --- a/repo.toml +++ b/repo.toml @@ -225,7 +225,7 @@ output_format = "linkcheck" docs_root = "${root}/openshift" project = "gpu-operator-openshift" name = "NVIDIA GPU Operator on Red Hat OpenShift Container Platform" -version = "26.3" # Check that this value matches maj.min version of repo_docs.projects.gpu-operator.version +version = "26.7" # Check that this value matches maj.min version of repo_docs.projects.gpu-operator.version copyright_start = 2020 sphinx_exclude_patterns = [ "get-entitlement.rst", From 8248def83c76945e17f0579c512aac68726449db Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:21:57 -0400 Subject: [PATCH 2/7] add docs for 26.7.0 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/release-notes.rst | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 5c2dec9b1..aa1569fa0 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -33,6 +33,41 @@ Refer to the :ref:`GPU Operator Component Matrix` for a list of software compone ---- +.. _v26.7.0: + +26.7.0 +======= + +.. This 26.7.0 entry is in progress. Only changes confirmed against merged code are listed. + Pending items (NVIDIADriver CR migration, CDI/NRI default change, image hardening, + Ubuntu 26.04, new hardware, component versions) are added as they are verified on the + release branch. + +New Features +------------ + +* Added an ``upgradePolicy`` field to the NVIDIA driver custom resource definition (CRD). + You can now define a driver upgrade policy per NVIDIADriver custom resource. + When the field is unset, the driver-upgrade controller falls back to the default upgrade policy that is defined in the Helm chart values. + (`PR #2582 `__) + +* Added the ``hostPaths.kubeletRootDir`` Helm value to configure a custom kubelet root directory. + When left empty, the GPU Operator uses ``/var/lib/kubelet`` as the default path. + (`PR #1384 `__) + +* Added the ``dcgmExporter.serviceMonitor.scrapeTimeout`` Helm value to configure the scrape timeout for the DCGM Exporter ``ServiceMonitor``. + (`PR #2404 `__) + +Fixed Issues +------------ + +* Fixed an issue where custom NVIDIADriver custom resources could remain stuck and fail to reconcile after a conflicting default NVIDIADriver custom resource was deleted. + The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. + (`PR #2258 `__) + + +---- + .. _v26.3.3: 26.3.3 From 03fed83e938d442a9386f5db7685ec786fd44f16 Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:05:41 -0400 Subject: [PATCH 3/7] update versions Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- openshift/versions1.json | 4 ++++ repo.toml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openshift/versions1.json b/openshift/versions1.json index a6293db72..cde5ec46a 100644 --- a/openshift/versions1.json +++ b/openshift/versions1.json @@ -1,6 +1,10 @@ [ { "preferred": "true", + "url": "../26.7/", + "version": "26.7" + }, + { "url": "../26.3/", "version": "26.3" }, diff --git a/repo.toml b/repo.toml index 4395991d1..c220f4aec 100644 --- a/repo.toml +++ b/repo.toml @@ -175,8 +175,8 @@ output_format = "linkcheck" docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" -version = "26.3" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch -source_substitutions = { minor_version = "26.3", version = "v26.3.3", recommended = "580.173.02", dra_version = "0.4.1" } +version = "26.7" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch +source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "580.173.02", dra_version = "0.4.1" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From 820711333c634fcf1c76af822a5734ddaea0ab49 Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:08:42 -0400 Subject: [PATCH 4/7] Update release notes for codedrop 2 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/cdi.rst | 92 ++++++++++++++++++++++++++++++++++ gpu-operator/release-notes.rst | 45 +++++++++++++++-- repo.toml | 2 +- 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/gpu-operator/cdi.rst b/gpu-operator/cdi.rst index acd15b8dc..b1bd35933 100644 --- a/gpu-operator/cdi.rst +++ b/gpu-operator/cdi.rst @@ -244,6 +244,98 @@ Disable the NRI Plugin by modifying the cluster policy: After disabling the NRI Plugin, the ``nvidia`` runtime class will be created. +.. _nri-cdi-devices: + +****************************************** +Requesting CDI Devices with the NRI Plugin +****************************************** + +When the NRI Plugin is enabled, a pod can request that the NRI Plugin inject specific CDI devices into one of its containers. +Add an annotation of the following form to the pod specification: + +.. code-block:: yaml + + metadata: + annotations: + nvidia.cdi.k8s.io/container.: "" + +Replace ```` with the name of the container in the pod that the device is injected into, and replace ```` with the CDI device to request. +The NRI Plugin supports two kinds of CDI devices: non-management devices and management devices. + +Requesting a Non-Management CDI Device +************************************** + +A non-management CDI device injects a single GPU, identified by its GPU UUID, into a container. +Pods can request a non-management CDI device from **any** namespace. + +Retrieve the GPU UUID from the ``/var/run/cdi/k8s.device-plugin.nvidia.com-gpu.json`` file on the GPU node. + +Request the device with an annotation of the form ``k8s.device-plugin.nvidia.com/gpu=``. +The following pod specification requests a specific GPU for the ``ubuntu`` container: + +.. code-block:: yaml + + apiVersion: v1 + kind: Pod + metadata: + name: ubuntu + labels: + app: ubuntu + annotations: + nvidia.cdi.k8s.io/container.ubuntu: "k8s.device-plugin.nvidia.com/gpu=GPU-1d7f7c5b-b2f6-148d-ff9a-d8bc4941885c" + spec: + containers: + - image: ubuntu + name: ubuntu + command: + - "sleep" + - "604800" + imagePullPolicy: IfNotPresent + restartPolicy: Always + +Requesting a Management CDI Device +********************************** + +A management CDI device provides management-level access to all GPUs on the node, bypassing GPU allocation through the Device Plugin or DRA Driver for GPUs. +This is the access that a GPU Management Container requires. +Request a management device with the ``management.nvidia.com/gpu=all`` annotation value. + +By default, only pods in the namespace where the GPU Operator is installed, which is the namespace where the NVIDIA Container Toolkit runs, are permitted to request management CDI devices. +To permit pods in other namespaces to request management CDI devices, add those namespaces to the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable on the NVIDIA Container Toolkit. +The value is a comma-separated list of namespaces. + +Set the environment variable through the ``toolkit.env`` Helm value at installation time, or by modifying the cluster policy on an existing installation: + +.. code-block:: console + + $ kubectl patch clusterpolicies.nvidia.com/cluster-policy --type='json' \ + -p='[{"op": "replace", "path": "/spec/toolkit/env", "value": [{"name": "NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES", "value": "test-namespace"}]}]' + +The following pod specification, in ``test-namespace``, requests a management CDI device for the ``ctr`` container. +For this pod to start, ``test-namespace`` must be included in the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable. + +.. code-block:: yaml + + apiVersion: v1 + kind: Pod + metadata: + name: cdi-nri-test + namespace: test-namespace + labels: + app: cdi-nri-test + annotations: + nvidia.cdi.k8s.io/container.ctr: "management.nvidia.com/gpu=all" + spec: + containers: + - name: ctr + image: ubuntu + imagePullPolicy: IfNotPresent + command: + - bash + - -c + - sleep infinity + + ************ Known Issues ************ diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index aa1569fa0..e6659a23d 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -38,14 +38,28 @@ Refer to the :ref:`GPU Operator Component Matrix` for a list of software compone 26.7.0 ======= -.. This 26.7.0 entry is in progress. Only changes confirmed against merged code are listed. - Pending items (NVIDIADriver CR migration, CDI/NRI default change, image hardening, - Ubuntu 26.04, new hardware, component versions) are added as they are verified on the - release branch. - New Features ------------ +* Updated software component versions: + + - NVIDIA GPU Driver 595.71.05 + - NVIDIA Container Toolkit v1.20.0 + - NVIDIA DCGM 4.6.0-1 + - NVIDIA MIG Manager for Kubernetes v0.14.3 + - NVIDIA GDRCopy Driver v2.6 + - Node Feature Discovery v0.19.0 + +* Added a ``nvidia.com/gpu.deploy.client`` node label that lets the GPU Operator manage third-party GPU client pods during driver upgrades and MIG configuration changes. + Advanced users who run their own GPU client workloads that hold GPU device handles (for example, a standalone NVIDIA DRA driver) can add ``nvidia.com/gpu.deploy.client=true`` to the ``nodeSelector`` of the workload's DaemonSet, Deployment, StatefulSet, or Job. + The GPU Operator then automatically restarts these pods during a driver upgrade or a MIG configuration change, so the operation can proceed without manual pod eviction. + (`PR #2607 `__) + +* Added support for restarting driver pods in place during patch upgrades when the driver configuration is unchanged. + Previously, a chart upgrade that changed only cosmetic pod-template metadata, such as the ``helm.sh/chart`` label, evicted running GPU workloads and drained the node. + The driver-upgrade controller now compares the driver configuration digest between the running pod and the desired DaemonSet, and when they match, it cordons the node and restarts the driver pod in place without evicting workloads or draining the node. + (`PR #2527 `__) + * Added an ``upgradePolicy`` field to the NVIDIA driver custom resource definition (CRD). You can now define a driver upgrade policy per NVIDIADriver custom resource. When the field is unset, the driver-upgrade controller falls back to the default upgrade policy that is defined in the Helm chart values. @@ -58,6 +72,19 @@ New Features * Added the ``dcgmExporter.serviceMonitor.scrapeTimeout`` Helm value to configure the scrape timeout for the DCGM Exporter ``ServiceMonitor``. (`PR #2404 `__) +* Added support for configuring the namespace that the GPU Operator uses for its leader election ``Lease`` object. + When unset, the operator continues to use the namespace that it runs in. + (`PR #1333 `__) + +* Changed the RHEL-specific driver container image tags to use only the major RHEL version instead of the full ``major.minor`` version. + (`PR #2497 `__) + +* Added the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable for the NVIDIA Container Toolkit. + When the NRI Plugin is enabled, this variable holds the list of namespaces whose pods are permitted to request management CDI devices. + By default, only pods in the namespace where the GPU Operator is installed can request management CDI devices. + Set the variable through the ``toolkit.env`` Helm value or the ClusterPolicy custom resource. + Refer to :ref:`Requesting CDI Devices with the NRI Plugin ` for more information. + Fixed Issues ------------ @@ -65,6 +92,14 @@ Fixed Issues The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. (`PR #2258 `__) +* Fixed an issue where the ``ClusterPolicy`` status could briefly report ``Ready`` during a driver upgrade before all driver pods were upgraded. + (`PR #2510 `__, `Issue #1567 `__) + +* Fixed an issue where the driver container's containerd configuration mounts were set up even when the NRI plugin was enabled. + On immutable hosts with read-only containerd paths, this prevented the NRI plugin pod from starting. + The GPU Operator no longer configures these mounts when NRI is enabled. + (`PR #2514 `__) + ---- diff --git a/repo.toml b/repo.toml index c220f4aec..e1f9ecba8 100644 --- a/repo.toml +++ b/repo.toml @@ -176,7 +176,7 @@ docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" version = "26.7" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch -source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "580.173.02", dra_version = "0.4.1" } +source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "595.71.05", dra_version = "0.4.1" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From ead9b08e1f511fdeb5a62845fad3cbb64531066b Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:33:27 -0400 Subject: [PATCH 5/7] docs: Daemon set mounts for OL nodes Signed-off-by: Mike McKiernan --- gpu-operator/platform-support.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index c9077752c..2317fd56c 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -477,6 +477,21 @@ Cloud Service Providers - 1.32---1.36 +Starting with version 26.7.0, the Operator can render driver workloads for nodes when Node Feature Discovery +reports ``feature.node.kubernetes.io/system-os_release.ID=ol``. +The Operator combines ``ol`` with the major component of the ``VERSION_ID`` label to form the operating system tag. +For example, ``VERSION_ID=9.7`` produces ``ol9``. +For tag-based driver images, the Operator appends the operating system tag to the configured driver +version, so a version such as ``580.126.20`` resolves to an image tag that ends in ``580.126.20-ol9``. +The Operator also mounts custom yum repository configuration in ``/etc/yum.repos.d`` and custom certificate +configuration in ``/etc/pki/ca-trust/extracted/pem`` on these nodes. + +This behavior provides operating system tag selection and manifest rendering only. +The change does not add an NVIDIA-published Oracle Linux driver container image or qualify Oracle Linux for the validated operating +system and container runtime support matrices on this page. +You are responsible for building, validating, publishing, and maintaining a custom driver image with the tag that the Operator constructs. +NVIDIA provides limited support for custom driver container images. + .. _supported-precompiled-drivers: ***************************** From 1d675e3a706831c9834a36f2255fc1c139f39a90 Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:49:00 -0400 Subject: [PATCH 6/7] docs: Sundry bug fixes Signed-off-by: Mike McKiernan --- gpu-operator/release-notes.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index e6659a23d..984a03ac3 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -88,6 +88,37 @@ New Features Fixed Issues ------------ +* Fixed an issue where host driver validation could incorrectly report that ``nvidia-smi`` was missing when ``/usr/bin`` was an absolute symlink. The validator now resolves the symlink within the mounted host file system. (`PR #2464 `__, `Issue #1357 `__) * Fixed host driver and vGPU Manager validation so that the validator finds ``nvidia-smi`` in additional host locations, including ``/usr/sbin``, ``/opt/bin``, and the Windows Subsystem for Linux driver path. + Symlinks are resolved within the mounted host file system. + (`PR #2611 `__, `Issue #2506 `__) + +* Fixed an issue where vGPU Manager validation on non-SR-IOV GPUs could wait for unavailable SR-IOV virtual functions and time out. + The validator now waits for mediated-device parents on non-SR-IOV GPUs and for virtual functions on SR-IOV GPUs. + (`PR #2502 `__, `Issue #2365 `__) + +* Fixed an issue where the driver pod failed to start on kernels without ``CONFIG_MEMORY_HOTPLUG`` because the ``/sys/devices/system/memory/auto_online_blocks`` file was unavailable. + The driver pod now mounts the stable ``/sys/devices/system`` parent directory. + (`PR #2517 `__, `Issue #2463 `__) + +* Fixed an issue where the NVIDIA vGPU Device Manager did not start when the vGPU Manager driver was preinstalled on the host and ``driver.enabled=false``. + The readiness gate now accepts the status file from either a container-managed or host-installed vGPU Manager. + (`PR #2599 `__) + +* Fixed an issue where setting ``devicePlugin.config.create=true`` with empty configuration data could leave device plugin pods referencing a missing ConfigMap. + The Helm chart now rejects the configuration when ``devicePlugin.config.name`` or ``devicePlugin.config.data`` is empty. + (`PR #2642 `__, `Issue #2641 `__) + +* Fixed an issue where the ``ClusterPolicy`` status could fluctuate during an NVIDIADriver rolling upgrade. + The status now remains ``notReady`` while an NVIDIADriver-owned node has a pending, in-progress, or failed upgrade. + (`PR #2665 `__) + +* Fixed an issue where multiple NVIDIADriver custom resources with ``spec.default=true`` could incorrectly report a ``ready`` status. + The conflicting resources now report ``notReady`` with a ``ReconcileFailed`` condition until the conflict is resolved. + (`PR #2678 `__) + +* Updated the bundled vGPU Device Manager configuration for vGPU 20.0 so that PCI device ID ``0x318210DE`` maps to the ``B300X-269C`` profile. + (`PR #2625 `__, `Issue #2231 `__) + * Fixed an issue where custom NVIDIADriver custom resources could remain stuck and fail to reconcile after a conflicting default NVIDIADriver custom resource was deleted. The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. (`PR #2258 `__) From 13d9b67fa7ea49ce6f35577c06a85c2b2837efee Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 7 Aug 2026 13:51:00 -0400 Subject: [PATCH 7/7] docs(fix): Markup formatting Signed-off-by: Mike McKiernan --- gpu-operator/release-notes.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 984a03ac3..da9b32b9f 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -88,7 +88,11 @@ New Features Fixed Issues ------------ -* Fixed an issue where host driver validation could incorrectly report that ``nvidia-smi`` was missing when ``/usr/bin`` was an absolute symlink. The validator now resolves the symlink within the mounted host file system. (`PR #2464 `__, `Issue #1357 `__) * Fixed host driver and vGPU Manager validation so that the validator finds ``nvidia-smi`` in additional host locations, including ``/usr/sbin``, ``/opt/bin``, and the Windows Subsystem for Linux driver path. +* Fixed an issue where host driver validation could incorrectly report that ``nvidia-smi`` was missing when ``/usr/bin`` was an absolute symlink. + The validator now resolves the symlink within the mounted host file system. + (`PR #2464 `__, `Issue #1357 `__) + +* Fixed host driver and vGPU Manager validation so that the validator finds ``nvidia-smi`` in additional host locations, including ``/usr/sbin``, ``/opt/bin``, and the Windows Subsystem for Linux driver path. Symlinks are resolved within the mounted host file system. (`PR #2611 `__, `Issue #2506 `__)