From ce6f270dee17eeee046cce67a56e08724d54413a Mon Sep 17 00:00:00 2001 From: b-reyes Date: Mon, 3 Aug 2026 10:14:26 -0600 Subject: [PATCH 1/2] set gpumem and gpuutil to zero, if a gpumem infinite value is detected when collecting the metrics for each job step --- bin/seff | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/seff b/bin/seff index 21bd38c..8e31a2f 100755 --- a/bin/seff +++ b/bin/seff @@ -136,6 +136,13 @@ for my $step (@{$job->{'steps'}}) { exists $gres_map{'gres/gpumem'} && exists $gres_map{'gres/gpuutil'}) { my $gpumem = Slurmdb::find_tres_count_in_string($step->{'stats'}{'tres_usage_in_max'}, $gres_map{'gres/gpumem'}); my $gpuutil = Slurmdb::find_tres_count_in_string($step->{'stats'}{'tres_usage_in_max'}, $gres_map{'gres/gpuutil'}); + + # Set values equal to zero, if GPU memory does not exist i.e. is infinite (necessary for GPU jobs that have dedicated CPU-only job steps) + if ($gpumem == INFINITE64) { + $gpumem = 0; + $gpuutil = 0; + } + push @gpumem_values, $gpumem; push @gpuutil_values, $gpuutil; } From 2f98152126c540b4075baffc0a15e2097fed8a24 Mon Sep 17 00:00:00 2001 From: b-reyes Date: Mon, 3 Aug 2026 11:01:42 -0600 Subject: [PATCH 2/2] modify logic for setting gpumem and gpuutil values to zero so that it is only done if non-infinite values are found (allows us to correctly identify incompatible GPUs) --- bin/seff | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/seff b/bin/seff index 8e31a2f..7672b8e 100755 --- a/bin/seff +++ b/bin/seff @@ -136,13 +136,6 @@ for my $step (@{$job->{'steps'}}) { exists $gres_map{'gres/gpumem'} && exists $gres_map{'gres/gpuutil'}) { my $gpumem = Slurmdb::find_tres_count_in_string($step->{'stats'}{'tres_usage_in_max'}, $gres_map{'gres/gpumem'}); my $gpuutil = Slurmdb::find_tres_count_in_string($step->{'stats'}{'tres_usage_in_max'}, $gres_map{'gres/gpuutil'}); - - # Set values equal to zero, if GPU memory does not exist i.e. is infinite (necessary for GPU jobs that have dedicated CPU-only job steps) - if ($gpumem == INFINITE64) { - $gpumem = 0; - $gpuutil = 0; - } - push @gpumem_values, $gpumem; push @gpuutil_values, $gpuutil; } @@ -167,6 +160,19 @@ for my $step (@{$job->{'steps'}}) { } } +# See if we have any non-infinite values found +my $have_real_gpumem = grep { $_ != INFINITE64 } @gpumem_values; +my $have_real_gpuutil = grep { $_ != INFINITE64 } @gpuutil_values; + +# If non-infinite values are found, then replace any infinite values with zero +# This logic is required for when you have CPU-only job steps in a GPU job +if ($have_real_gpumem) { + @gpumem_values = map { $_ == INFINITE64 ? 0 : $_ } @gpumem_values; +} +if ($have_real_gpuutil) { + @gpuutil_values = map { $_ == INFINITE64 ? 0 : $_ } @gpuutil_values; +} + my $cput = $tot_cpu_sec + int(($tot_cpu_usec / 1000000) + 0.5); if ($mydebug) {