From 68a6ec61756f62517443dcb80de6bcdeca0e5aaf Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:20:01 +0000 Subject: [PATCH 01/11] Add option to disable/control data validation via ALWAYS_VALIDATE ALWAYS_VALIDATE now supports three modes: 0 disables validation, 1 validates once at the end (new default), and 2 validates after each iteration. Updates the client help/print text and SmokeTest preset to match the new semantics. Co-authored-by: Cursor --- src/client/EnvVars.hpp | 9 +++++---- src/client/Presets/SmokeTest.hpp | 2 +- src/header/TransferBench.hpp | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index 34940b69..e3233c7a 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -79,7 +79,7 @@ class EnvVars int useInteractive; // Pause for user-input before starting transfer loop // Data options - int alwaysValidate; // Validate after each iteration instead of once after all iterations + int alwaysValidate; // 0 = disable validation, 1 = validate once after all iterations, 2 = validate after each iteration int blockBytes; // Each subexecutor, except the last, gets a multiple of this many bytes to copy int byteOffset; // Byte-offset for memory allocations vector fillPattern; // Pattern of floats used to fill source data @@ -149,7 +149,7 @@ class EnvVars else if (archName == "gfx942") defaultGfxUnroll = 4; else if (archName == "gfx950") defaultGfxUnroll = 4; - alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 0); + alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 1); blockBytes = GetEnvVar("BLOCK_BYTES" , 256); byteOffset = GetEnvVar("BYTE_OFFSET" , 0); fillCompress = GetEnvVarArray("FILL_COMPRESS" , {}); @@ -352,7 +352,7 @@ class EnvVars { printf("Environment variables (client):\n"); printf("======================\n"); - printf(" ALWAYS_VALIDATE - Validate after each iteration instead of once after all iterations\n"); + printf(" ALWAYS_VALIDATE - Data validation mode: 0=disabled, 1=validate once after all iterations (default), 2=validate after each iteration\n"); printf(" BLOCK_BYTES - Controls granularity of how work is divided across subExecutors\n"); printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4\n"); printf(" CU_MASK - CU mask for streams. Can specify ranges e.g '5,10-12,14'\n"); @@ -454,7 +454,8 @@ class EnvVars if (hideEnv) return; Print("ALWAYS_VALIDATE", alwaysValidate, - "Validating after %s", (alwaysValidate ? "each iteration" : "all iterations")); + "Validation %s", (alwaysValidate == 0 ? "disabled" : + alwaysValidate == 2 ? "after each iteration" : "after all iterations")); Print("BLOCK_BYTES", blockBytes, "Each CU gets a mulitple of %d bytes to copy", blockBytes); Print("BYTE_OFFSET", byteOffset, diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 4b3325b8..daf5d7d8 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -211,7 +211,7 @@ int SmokeTestPreset(EnvVars& ev, } // Modify defaults unless they were set - ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1); + ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 2); ev.numIterations = EnvVars::GetEnvVar("NUM_ITERATIONS", 2); ev.numWarmups = EnvVars::GetEnvVar("NUM_WARMUPS", 0); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 5f22f52c..461fc6b9 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -217,7 +217,7 @@ namespace TransferBench */ struct DataOptions { - int alwaysValidate = 0; ///< Validate after each iteration instead of once at end + int alwaysValidate = 1; ///< 0 = disable validation, 1 = validate once at end, 2 = validate after each iteration int blockBytes = 256; ///< Each subexecutor works on a multiple of this many bytes int byteOffset = 0; ///< Byte-offset for memory allocations vector fillPattern = {}; ///< Pattern of floats used to fill source data @@ -6012,7 +6012,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; double deltaSec = std::chrono::duration_cast>(cpuDelta).count() / cfg.general.numSubIterations; - if (cfg.data.alwaysValidate) { + if (cfg.data.alwaysValidate == 2) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } @@ -6087,7 +6087,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Validate results - if (!cfg.data.alwaysValidate) { + if (cfg.data.alwaysValidate == 1) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } From d24ba28a5f96c50d5d7787c73e5503d9810aabe6 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:27:09 +0000 Subject: [PATCH 02/11] Make interactive-mode validation a separate, gated stage When USE_INTERACTIVE=1, validation now runs as a distinct stage that requires an keypress instead of running as part of the transfer stage. The interactive validation stage respects ALWAYS_VALIDATE: it is skipped when validation is disabled (0), triggered by for validate-at-end (1), and skipped for validate-each-iteration (2) since that mode already validates inline every iteration. Co-authored-by: Cursor --- src/header/TransferBench.hpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 461fc6b9..746c11da 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6023,10 +6023,18 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Pause for interactive mode - run validation and show per-transfer result before prompt - if (cfg.general.useInteractive) { + // Pause for interactive mode - validation is a separate stage triggered by user input. + // Only for mode 1 (validate at end); mode 2 already validates inline each iteration, + // and mode 0 disables validation entirely. + if (cfg.general.useInteractive && cfg.data.alwaysValidate == 1) { if (localRank == 0) { - System::Get().Log("Transfers complete. Validation results:\n"); + System::Get().Log("Transfers complete. Hit to run validation: "); + fflush(stdout); + if (scanf("%*c") != 0) { + System::Get().Log("[ERROR] Unexpected input\n"); + exit(1); + } + System::Get().Log("\nValidation results:\n"); size_t initOffset = cfg.data.byteOffset / sizeof(float); int numPass = 0, numFail = 0; @@ -6074,13 +6082,6 @@ static bool IsConfiguredGid(union ibv_gid const& gid) if (anyLocalDst) { if (transferOk) ++numPass; else ++numFail; } } System::Get().Log(" Summary: %d PASS %d FAIL\n", numPass, numFail); - System::Get().Log("Hit to continue: "); - fflush(stdout); - if (scanf("%*c") != 0) { - System::Get().Log("[ERROR] Unexpected input\n"); - exit(1); - } - System::Get().Log("\n"); fflush(stdout); } System::Get().Barrier(); From 62b0ef15d7602265e369d52855be5299708bcc17 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:35:52 +0000 Subject: [PATCH 03/11] Rework ALWAYS_VALIDATE to -1/0/1 for backward compatibility Restore the original ALWAYS_VALIDATE semantics and add a disable option: -1 = disable validation entirely (init + transfers only) 0 = validate once after all iterations (default) 1 = validate after each iteration This keeps the previous 0/1 behavior intact while adding -1 as a performance-debug option that skips validation. Interactive mode runs the Enter-gated validation stage only for mode 0; mode 1 validates inline each iteration and mode -1 skips validation. SmokeTest preset now defaults to 1 to preserve its validate-each-iteration behavior. Co-authored-by: Cursor --- src/client/EnvVars.hpp | 10 +++++----- src/client/Presets/SmokeTest.hpp | 2 +- src/header/TransferBench.hpp | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index e3233c7a..bd8a88e4 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -79,7 +79,7 @@ class EnvVars int useInteractive; // Pause for user-input before starting transfer loop // Data options - int alwaysValidate; // 0 = disable validation, 1 = validate once after all iterations, 2 = validate after each iteration + int alwaysValidate; // -1 = disable validation, 0 = validate once after all iterations, 1 = validate after each iteration int blockBytes; // Each subexecutor, except the last, gets a multiple of this many bytes to copy int byteOffset; // Byte-offset for memory allocations vector fillPattern; // Pattern of floats used to fill source data @@ -149,7 +149,7 @@ class EnvVars else if (archName == "gfx942") defaultGfxUnroll = 4; else if (archName == "gfx950") defaultGfxUnroll = 4; - alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 1); + alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 0); blockBytes = GetEnvVar("BLOCK_BYTES" , 256); byteOffset = GetEnvVar("BYTE_OFFSET" , 0); fillCompress = GetEnvVarArray("FILL_COMPRESS" , {}); @@ -352,7 +352,7 @@ class EnvVars { printf("Environment variables (client):\n"); printf("======================\n"); - printf(" ALWAYS_VALIDATE - Data validation mode: 0=disabled, 1=validate once after all iterations (default), 2=validate after each iteration\n"); + printf(" ALWAYS_VALIDATE - Data validation mode: -1=disabled, 0=validate once after all iterations (default), 1=validate after each iteration\n"); printf(" BLOCK_BYTES - Controls granularity of how work is divided across subExecutors\n"); printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4\n"); printf(" CU_MASK - CU mask for streams. Can specify ranges e.g '5,10-12,14'\n"); @@ -454,8 +454,8 @@ class EnvVars if (hideEnv) return; Print("ALWAYS_VALIDATE", alwaysValidate, - "Validation %s", (alwaysValidate == 0 ? "disabled" : - alwaysValidate == 2 ? "after each iteration" : "after all iterations")); + "Validation %s", (alwaysValidate < 0 ? "disabled" : + alwaysValidate == 1 ? "after each iteration" : "after all iterations")); Print("BLOCK_BYTES", blockBytes, "Each CU gets a mulitple of %d bytes to copy", blockBytes); Print("BYTE_OFFSET", byteOffset, diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index daf5d7d8..4b3325b8 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -211,7 +211,7 @@ int SmokeTestPreset(EnvVars& ev, } // Modify defaults unless they were set - ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 2); + ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1); ev.numIterations = EnvVars::GetEnvVar("NUM_ITERATIONS", 2); ev.numWarmups = EnvVars::GetEnvVar("NUM_WARMUPS", 0); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 746c11da..fbc45eb0 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -217,7 +217,7 @@ namespace TransferBench */ struct DataOptions { - int alwaysValidate = 1; ///< 0 = disable validation, 1 = validate once at end, 2 = validate after each iteration + int alwaysValidate = 0; ///< -1 = disable validation, 0 = validate once at end, 1 = validate after each iteration int blockBytes = 256; ///< Each subexecutor works on a multiple of this many bytes int byteOffset = 0; ///< Byte-offset for memory allocations vector fillPattern = {}; ///< Pattern of floats used to fill source data @@ -6012,7 +6012,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; double deltaSec = std::chrono::duration_cast>(cpuDelta).count() / cfg.general.numSubIterations; - if (cfg.data.alwaysValidate == 2) { + if (cfg.data.alwaysValidate == 1) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } @@ -6024,9 +6024,9 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Pause for interactive mode - validation is a separate stage triggered by user input. - // Only for mode 1 (validate at end); mode 2 already validates inline each iteration, - // and mode 0 disables validation entirely. - if (cfg.general.useInteractive && cfg.data.alwaysValidate == 1) { + // Only for mode 0 (validate at end); mode 1 already validates inline each iteration, + // and mode -1 disables validation entirely. + if (cfg.general.useInteractive && cfg.data.alwaysValidate == 0) { if (localRank == 0) { System::Get().Log("Transfers complete. Hit to run validation: "); fflush(stdout); @@ -6088,7 +6088,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Validate results - if (cfg.data.alwaysValidate == 1) { + if (cfg.data.alwaysValidate == 0) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } From 266f6a47e1dd876a43b216aee5a3069d52aab3be Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:44:42 +0000 Subject: [PATCH 04/11] Address Copilot review: validate env input and dedupe validation - Normalize ALWAYS_VALIDATE on read: warn and clamp values outside {-1, 0, 1} so runtime behavior matches the documented modes, instead of silently disabling validation for out-of-range values. - Avoid validating twice in interactive mode: the interactive stage now records failures into errResults and the end-of-run ValidateAllTransfers is skipped for ranks already validated by that stage. Co-authored-by: Cursor --- src/client/EnvVars.hpp | 14 +++++++++++++- src/client/Presets/SmokeTest.hpp | 2 +- src/header/TransferBench.hpp | 20 +++++++++++++++----- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index bd8a88e4..e73ed45b 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -149,7 +149,7 @@ class EnvVars else if (archName == "gfx942") defaultGfxUnroll = 4; else if (archName == "gfx950") defaultGfxUnroll = 4; - alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 0); + alwaysValidate = NormalizeValidateMode(GetEnvVar("ALWAYS_VALIDATE", 0)); blockBytes = GetEnvVar("BLOCK_BYTES" , 256); byteOffset = GetEnvVar("BYTE_OFFSET" , 0); fillCompress = GetEnvVarArray("FILL_COMPRESS" , {}); @@ -588,6 +588,18 @@ class EnvVars return defaultValue; } + // Validates ALWAYS_VALIDATE is one of the supported modes (-1/0/1); warns and clamps otherwise + static int NormalizeValidateMode(int value) + { + if (value < -1 || value > 1) { + int clamped = (value < -1) ? -1 : 1; + printf("[WARN] ALWAYS_VALIDATE=%d is invalid; expected -1 (disabled), 0 (validate at end), " + "or 1 (validate each iteration). Clamping to %d\n", value, clamped); + value = clamped; + } + return value; + } + // Returns comma-split tokens for varname, or an empty optional if unset/empty (use default). static std::optional> GetEnvTokens(std::string const& varname) { diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 4b3325b8..15311be1 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -211,7 +211,7 @@ int SmokeTestPreset(EnvVars& ev, } // Modify defaults unless they were set - ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1); + ev.alwaysValidate = EnvVars::NormalizeValidateMode(EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1)); ev.numIterations = EnvVars::GetEnvVar("NUM_ITERATIONS", 2); ev.numWarmups = EnvVars::GetEnvVar("NUM_WARMUPS", 0); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index fbc45eb0..2ef52f63 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6025,9 +6025,13 @@ static bool IsConfiguredGid(union ibv_gid const& gid) // Pause for interactive mode - validation is a separate stage triggered by user input. // Only for mode 0 (validate at end); mode 1 already validates inline each iteration, - // and mode -1 disables validation entirely. + // and mode -1 disables validation entirely. When the interactive stage validates a rank's + // local destinations here, the end-of-run ValidateAllTransfers is skipped for that rank to + // avoid validating twice. + bool interactiveValidated = false; if (cfg.general.useInteractive && cfg.data.alwaysValidate == 0) { if (localRank == 0) { + interactiveValidated = true; System::Get().Log("Transfers complete. Hit to run validation: "); fflush(stdout); if (scanf("%*c") != 0) { @@ -6069,12 +6073,18 @@ static bool IsConfiguredGid(union ibv_gid const& gid) size_t firstErr = 0; for (; firstErr < N; firstErr++) if (output[firstErr] != expected[firstErr]) break; - if (firstErr < N) + if (firstErr < N) { System::Get().Log(" DST[%d]=FAIL(first mismatch idx=%zu exp=%.5f got=%.5f)", dstIdx, firstErr, expected[firstErr], output[firstErr]); - else + errResults.push_back({ERR_FATAL, + "Transfer %d: Unexpected mismatch at index %lu of destination %d on rank %d: Expected %10.5f Actual: %10.5f", + transferIdx, firstErr, dstIdx, t.dsts[dstIdx].memRank, expected[firstErr], output[firstErr]}); + } else { System::Get().Log(" DST[%d]=FAIL(bitwise mismatch, no float-level diff found)", dstIdx); + errResults.push_back({ERR_FATAL, + "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}); + } transferOk = false; } } @@ -6087,8 +6097,8 @@ static bool IsConfiguredGid(union ibv_gid const& gid) System::Get().Barrier(); } - // Validate results - if (cfg.data.alwaysValidate == 0) { + // Validate results (skip on ranks already validated by the interactive stage above) + if (cfg.data.alwaysValidate == 0 && !interactiveValidated) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } From d37ba21125bcfb6c2a1c3c2ad59f35b69ce785b2 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:46:32 +0000 Subject: [PATCH 05/11] Use getchar()/EOF check for interactive validation prompt scanf("%*c") returns 0 on success and EOF on failure, so the previous != 0 check was effectively an EOF check with a misleading message. Replace with getchar() and report EOF explicitly. Co-authored-by: Cursor --- src/header/TransferBench.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 2ef52f63..be3bc562 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6034,8 +6034,8 @@ static bool IsConfiguredGid(union ibv_gid const& gid) interactiveValidated = true; System::Get().Log("Transfers complete. Hit to run validation: "); fflush(stdout); - if (scanf("%*c") != 0) { - System::Get().Log("[ERROR] Unexpected input\n"); + if (getchar() == EOF) { + System::Get().Log("[ERROR] Unexpected EOF while waiting for input\n"); exit(1); } System::Get().Log("\nValidation results:\n"); From d3afbcd4d45784c1d8be23e895660466d8400f8f Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:50:45 +0000 Subject: [PATCH 06/11] Show interactive validation results for ALWAYS_VALIDATE=1 In interactive mode, the per-transfer validation summary is now displayed for mode 1 (validate each iteration) as well, without requiring an keypress since validation already ran inline. Mode 0 keeps its -gated validation path and remains the authoritative validation for its rank; mode 1 only displays results and does not re-record errors (already captured inline). Mode -1 still shows nothing. Co-authored-by: Cursor --- src/header/TransferBench.hpp | 45 +++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index be3bc562..8235543a 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6023,22 +6023,29 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Pause for interactive mode - validation is a separate stage triggered by user input. - // Only for mode 0 (validate at end); mode 1 already validates inline each iteration, - // and mode -1 disables validation entirely. When the interactive stage validates a rank's - // local destinations here, the end-of-run ValidateAllTransfers is skipped for that rank to - // avoid validating twice. + // Show per-transfer validation results in interactive mode (skipped for mode -1, disabled). + // For mode 0 (validate at end) this stage is the validation path: it is gated behind an + // keypress, records failures into errResults, and lets the end-of-run + // ValidateAllTransfers be skipped for the rank to avoid validating twice. + // For mode 1 (validate each iteration) validation already ran inline; this stage only + // displays the results and does not require an keypress. bool interactiveValidated = false; - if (cfg.general.useInteractive && cfg.data.alwaysValidate == 0) { + if (cfg.general.useInteractive && cfg.data.alwaysValidate != -1) { + // Mode 0 uses this stage as the authoritative validation path + bool const isValidationPath = (cfg.data.alwaysValidate == 0); if (localRank == 0) { - interactiveValidated = true; - System::Get().Log("Transfers complete. Hit to run validation: "); - fflush(stdout); - if (getchar() == EOF) { - System::Get().Log("[ERROR] Unexpected EOF while waiting for input\n"); - exit(1); + if (isValidationPath) { + interactiveValidated = true; + System::Get().Log("Transfers complete. Hit to run validation: "); + fflush(stdout); + if (getchar() == EOF) { + System::Get().Log("[ERROR] Unexpected EOF while waiting for input\n"); + exit(1); + } + System::Get().Log("\nValidation results:\n"); + } else { + System::Get().Log("Validation results:\n"); } - System::Get().Log("\nValidation results:\n"); size_t initOffset = cfg.data.byteOffset / sizeof(float); int numPass = 0, numFail = 0; @@ -6076,14 +6083,16 @@ static bool IsConfiguredGid(union ibv_gid const& gid) if (firstErr < N) { System::Get().Log(" DST[%d]=FAIL(first mismatch idx=%zu exp=%.5f got=%.5f)", dstIdx, firstErr, expected[firstErr], output[firstErr]); - errResults.push_back({ERR_FATAL, - "Transfer %d: Unexpected mismatch at index %lu of destination %d on rank %d: Expected %10.5f Actual: %10.5f", - transferIdx, firstErr, dstIdx, t.dsts[dstIdx].memRank, expected[firstErr], output[firstErr]}); + if (isValidationPath) + errResults.push_back({ERR_FATAL, + "Transfer %d: Unexpected mismatch at index %lu of destination %d on rank %d: Expected %10.5f Actual: %10.5f", + transferIdx, firstErr, dstIdx, t.dsts[dstIdx].memRank, expected[firstErr], output[firstErr]}); } else { System::Get().Log(" DST[%d]=FAIL(bitwise mismatch, no float-level diff found)", dstIdx); - errResults.push_back({ERR_FATAL, - "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}); + if (isValidationPath) + errResults.push_back({ERR_FATAL, + "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}); } transferOk = false; } From 50f9f26424998c3fe32e79b3b2ec44d4848f64f1 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 21:52:52 +0000 Subject: [PATCH 07/11] Condense verbose comments introduced in this branch Co-authored-by: Cursor --- src/client/EnvVars.hpp | 2 +- src/header/TransferBench.hpp | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index e73ed45b..c7f2af36 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -588,7 +588,7 @@ class EnvVars return defaultValue; } - // Validates ALWAYS_VALIDATE is one of the supported modes (-1/0/1); warns and clamps otherwise + // Clamps ALWAYS_VALIDATE to a supported mode (-1/0/1), warning on invalid input static int NormalizeValidateMode(int value) { if (value < -1 || value > 1) { diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 8235543a..6c3b8f9a 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6023,15 +6023,10 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Show per-transfer validation results in interactive mode (skipped for mode -1, disabled). - // For mode 0 (validate at end) this stage is the validation path: it is gated behind an - // keypress, records failures into errResults, and lets the end-of-run - // ValidateAllTransfers be skipped for the rank to avoid validating twice. - // For mode 1 (validate each iteration) validation already ran inline; this stage only - // displays the results and does not require an keypress. + // Interactive per-transfer validation display (skipped for mode -1) bool interactiveValidated = false; if (cfg.general.useInteractive && cfg.data.alwaysValidate != -1) { - // Mode 0 uses this stage as the authoritative validation path + // Mode 0 validates here (-gated); mode 1 already validated inline, only display bool const isValidationPath = (cfg.data.alwaysValidate == 0); if (localRank == 0) { if (isValidationPath) { From 414bb88af68cf0b1eee0690f33c792d129d5b1ff Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 22:07:18 +0000 Subject: [PATCH 08/11] Use range-based ALWAYS_VALIDATE and skip dstReference when disabled Per maintainer review, switch to range-based semantics: <0 = disable validation 0 = validate once at end (default) >0 = validate after each iteration Any positive value now enables per-iteration validation (preserving the original truthy behavior), so the explicit clamp/normalize helper is removed. When validation is disabled (<0), the dstReference expected-result arrays are no longer allocated or computed, saving setup time. Co-authored-by: Cursor --- src/client/EnvVars.hpp | 20 +++--------- src/client/Presets/SmokeTest.hpp | 2 +- src/header/TransferBench.hpp | 56 +++++++++++++++----------------- 3 files changed, 31 insertions(+), 47 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index c7f2af36..6fd27d22 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -79,7 +79,7 @@ class EnvVars int useInteractive; // Pause for user-input before starting transfer loop // Data options - int alwaysValidate; // -1 = disable validation, 0 = validate once after all iterations, 1 = validate after each iteration + int alwaysValidate; // <0 = disable validation, 0 = validate once after all iterations, >0 = validate after each iteration int blockBytes; // Each subexecutor, except the last, gets a multiple of this many bytes to copy int byteOffset; // Byte-offset for memory allocations vector fillPattern; // Pattern of floats used to fill source data @@ -149,7 +149,7 @@ class EnvVars else if (archName == "gfx942") defaultGfxUnroll = 4; else if (archName == "gfx950") defaultGfxUnroll = 4; - alwaysValidate = NormalizeValidateMode(GetEnvVar("ALWAYS_VALIDATE", 0)); + alwaysValidate = GetEnvVar("ALWAYS_VALIDATE", 0); blockBytes = GetEnvVar("BLOCK_BYTES" , 256); byteOffset = GetEnvVar("BYTE_OFFSET" , 0); fillCompress = GetEnvVarArray("FILL_COMPRESS" , {}); @@ -352,7 +352,7 @@ class EnvVars { printf("Environment variables (client):\n"); printf("======================\n"); - printf(" ALWAYS_VALIDATE - Data validation mode: -1=disabled, 0=validate once after all iterations (default), 1=validate after each iteration\n"); + printf(" ALWAYS_VALIDATE - Data validation mode: <0=disabled, 0=validate once after all iterations (default), >0=validate after each iteration\n"); printf(" BLOCK_BYTES - Controls granularity of how work is divided across subExecutors\n"); printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4\n"); printf(" CU_MASK - CU mask for streams. Can specify ranges e.g '5,10-12,14'\n"); @@ -455,7 +455,7 @@ class EnvVars Print("ALWAYS_VALIDATE", alwaysValidate, "Validation %s", (alwaysValidate < 0 ? "disabled" : - alwaysValidate == 1 ? "after each iteration" : "after all iterations")); + alwaysValidate > 0 ? "after each iteration" : "after all iterations")); Print("BLOCK_BYTES", blockBytes, "Each CU gets a mulitple of %d bytes to copy", blockBytes); Print("BYTE_OFFSET", byteOffset, @@ -588,18 +588,6 @@ class EnvVars return defaultValue; } - // Clamps ALWAYS_VALIDATE to a supported mode (-1/0/1), warning on invalid input - static int NormalizeValidateMode(int value) - { - if (value < -1 || value > 1) { - int clamped = (value < -1) ? -1 : 1; - printf("[WARN] ALWAYS_VALIDATE=%d is invalid; expected -1 (disabled), 0 (validate at end), " - "or 1 (validate each iteration). Clamping to %d\n", value, clamped); - value = clamped; - } - return value; - } - // Returns comma-split tokens for varname, or an empty optional if unset/empty (use default). static std::optional> GetEnvTokens(std::string const& varname) { diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 15311be1..4b3325b8 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -211,7 +211,7 @@ int SmokeTestPreset(EnvVars& ev, } // Modify defaults unless they were set - ev.alwaysValidate = EnvVars::NormalizeValidateMode(EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1)); + ev.alwaysValidate = EnvVars::GetEnvVar("ALWAYS_VALIDATE", 1); ev.numIterations = EnvVars::GetEnvVar("NUM_ITERATIONS", 2); ev.numWarmups = EnvVars::GetEnvVar("NUM_WARMUPS", 0); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 6c3b8f9a..d611fe45 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -217,7 +217,7 @@ namespace TransferBench */ struct DataOptions { - int alwaysValidate = 0; ///< -1 = disable validation, 0 = validate once at end, 1 = validate after each iteration + int alwaysValidate = 0; ///< <0 = disable validation, 0 = validate once at end, >0 = validate after each iteration int blockBytes = 256; ///< Each subexecutor works on a multiple of this many bytes int byteOffset = 0; ///< Byte-offset for memory allocations vector fillPattern = {}; ///< Pattern of floats used to fill source data @@ -5878,24 +5878,30 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Prepare reference src/dst arrays - only once for largest size + // Prepare reference src/dst arrays - only once for largest size. + // dstReference (expected results) is only needed when validation is enabled. + bool const validateEnabled = (cfg.data.alwaysValidate >= 0); size_t maxN = maxNumBytes / sizeof(float); - vector outputBuffer(maxN); - vector> dstReference(maxNumSrcs + 1, vector(maxN)); + vector outputBuffer(validateEnabled ? maxN : 0); + vector> dstReference; { size_t initOffset = cfg.data.byteOffset / sizeof(float); vector> srcReference(maxNumSrcs, vector(maxN)); - memset(dstReference[0].data(), MEMSET_CHAR, maxNumBytes); + if (validateEnabled) { + dstReference.assign(maxNumSrcs + 1, vector(maxN)); + memset(dstReference[0].data(), MEMSET_CHAR, maxNumBytes); + } for (int numSrcs = 0; numSrcs < maxNumSrcs; numSrcs++) { PrepareReference(cfg, srcReference[numSrcs], numSrcs); - for (int i = 0; i < maxN; i++) { - dstReference[numSrcs+1][i] = (numSrcs == 0 ? 0 : dstReference[numSrcs][i]) + srcReference[numSrcs][i]; - } + if (validateEnabled) + for (int i = 0; i < maxN; i++) + dstReference[numSrcs+1][i] = (numSrcs == 0 ? 0 : dstReference[numSrcs][i]) + srcReference[numSrcs][i]; } // Release un-used partial sums - for (int numSrcs = 0; numSrcs < minNumSrcs; numSrcs++) - dstReference[numSrcs].clear(); + if (validateEnabled) + for (int numSrcs = 0; numSrcs < minNumSrcs; numSrcs++) + dstReference[numSrcs].clear(); // Initialize all src memory buffers (if on local rank) bool const verbose = System::Get().IsVerbose(); @@ -6012,7 +6018,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; double deltaSec = std::chrono::duration_cast>(cpuDelta).count() / cfg.general.numSubIterations; - if (cfg.data.alwaysValidate == 1) { + if (cfg.data.alwaysValidate > 0) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } @@ -6023,14 +6029,12 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Interactive per-transfer validation display (skipped for mode -1) - bool interactiveValidated = false; - if (cfg.general.useInteractive && cfg.data.alwaysValidate != -1) { - // Mode 0 validates here (-gated); mode 1 already validated inline, only display - bool const isValidationPath = (cfg.data.alwaysValidate == 0); + // Interactive per-transfer validation display (skipped when validation disabled). Actual + // validation is done by ValidateAllTransfers (mode 0) or inline each iteration (mode >0); + // this only displays. + if (cfg.general.useInteractive && cfg.data.alwaysValidate >= 0) { if (localRank == 0) { - if (isValidationPath) { - interactiveValidated = true; + if (cfg.data.alwaysValidate == 0) { System::Get().Log("Transfers complete. Hit to run validation: "); fflush(stdout); if (getchar() == EOF) { @@ -6075,20 +6079,12 @@ static bool IsConfiguredGid(union ibv_gid const& gid) size_t firstErr = 0; for (; firstErr < N; firstErr++) if (output[firstErr] != expected[firstErr]) break; - if (firstErr < N) { + if (firstErr < N) System::Get().Log(" DST[%d]=FAIL(first mismatch idx=%zu exp=%.5f got=%.5f)", dstIdx, firstErr, expected[firstErr], output[firstErr]); - if (isValidationPath) - errResults.push_back({ERR_FATAL, - "Transfer %d: Unexpected mismatch at index %lu of destination %d on rank %d: Expected %10.5f Actual: %10.5f", - transferIdx, firstErr, dstIdx, t.dsts[dstIdx].memRank, expected[firstErr], output[firstErr]}); - } else { + else System::Get().Log(" DST[%d]=FAIL(bitwise mismatch, no float-level diff found)", dstIdx); - if (isValidationPath) - errResults.push_back({ERR_FATAL, - "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}); - } transferOk = false; } } @@ -6101,8 +6097,8 @@ static bool IsConfiguredGid(union ibv_gid const& gid) System::Get().Barrier(); } - // Validate results (skip on ranks already validated by the interactive stage above) - if (cfg.data.alwaysValidate == 0 && !interactiveValidated) { + // Validate results + if (cfg.data.alwaysValidate == 0) { ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer), errResults); } From 8a1efdf4e7bdae066dc6fc8b2dbcd350faa6dfa6 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 22:14:03 +0000 Subject: [PATCH 09/11] Log when validation is disabled and unify interactive prefix - Print "Validation disabled (ALWAYS_VALIDATE < 0)" in the interactive stage (in place of results) and in Utils::PrintResults for SHOW_DETAILS. - Show the "Transfers complete." prefix for per-iteration mode (>0) in interactive output, matching the validate-at-end (0) path. Co-authored-by: Cursor --- src/client/Utilities.hpp | 2 ++ src/header/TransferBench.hpp | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/client/Utilities.hpp b/src/client/Utilities.hpp index 9730ace6..7c0f9fbd 100644 --- a/src/client/Utilities.hpp +++ b/src/client/Utilities.hpp @@ -551,6 +551,8 @@ namespace TransferBench::Utils if (!RankDoesOutput()) return; if (!ev.outputToCsv) printf("Test %d:\n", testNum); + if (ev.alwaysValidate < 0 && !ev.outputToCsv) + printf("Validation disabled (ALWAYS_VALIDATE < 0)\n"); bool isMultiRank = TransferBench::GetNumRanks() > 1; diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index d611fe45..9ba8b1c8 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6043,7 +6043,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } System::Get().Log("\nValidation results:\n"); } else { - System::Get().Log("Validation results:\n"); + System::Get().Log("Transfers complete.\nValidation results:\n"); } size_t initOffset = cfg.data.byteOffset / sizeof(float); @@ -6095,6 +6095,10 @@ static bool IsConfiguredGid(union ibv_gid const& gid) fflush(stdout); } System::Get().Barrier(); + } else if (cfg.general.useInteractive) { + if (localRank == 0) + System::Get().Log("Transfers complete. Validation disabled (ALWAYS_VALIDATE < 0)\n"); + System::Get().Barrier(); } // Validate results From 92a22194501154788c98c71c45970e50f1b01b0e Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 22:17:38 +0000 Subject: [PATCH 10/11] Avoid duplicate validation-disabled message in interactive mode Suppress the PrintResults "validation disabled" message when interactive mode is active, since the interactive stage already prints it. Co-authored-by: Cursor --- src/client/Utilities.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/Utilities.hpp b/src/client/Utilities.hpp index 7c0f9fbd..dafa7541 100644 --- a/src/client/Utilities.hpp +++ b/src/client/Utilities.hpp @@ -551,7 +551,7 @@ namespace TransferBench::Utils if (!RankDoesOutput()) return; if (!ev.outputToCsv) printf("Test %d:\n", testNum); - if (ev.alwaysValidate < 0 && !ev.outputToCsv) + if (ev.alwaysValidate < 0 && !ev.outputToCsv && !ev.useInteractive) printf("Validation disabled (ALWAYS_VALIDATE < 0)\n"); bool isMultiRank = TransferBench::GetNumRanks() > 1; From e95c0f4c986b7068c39d6181b576591aa5696286 Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Tue, 14 Jul 2026 22:31:47 +0000 Subject: [PATCH 11/11] Coordinate interactive EOF abort and fix display comment - Replace exit(1) on rank-0 EOF at the validation prompt with a broadcast so all ranks abort together (return ERR_FATAL) instead of leaving other ranks hung at the following Barrier. - Correct the interactive-block comment: it performs its own comparison pass to display PASS/FAIL, not merely display. Co-authored-by: Cursor --- src/header/TransferBench.hpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 9ba8b1c8..e75c535d 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -6029,23 +6029,32 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Interactive per-transfer validation display (skipped when validation disabled). Actual - // validation is done by ValidateAllTransfers (mode 0) or inline each iteration (mode >0); - // this only displays. + // Interactive per-transfer PASS/FAIL display (skipped when validation disabled). This does its + // own comparison pass to show results; authoritative error reporting still comes from + // ValidateAllTransfers (mode 0) or the inline per-iteration validation (mode >0). if (cfg.general.useInteractive && cfg.data.alwaysValidate >= 0) { + bool inputOk = true; if (localRank == 0) { if (cfg.data.alwaysValidate == 0) { System::Get().Log("Transfers complete. Hit to run validation: "); fflush(stdout); if (getchar() == EOF) { System::Get().Log("[ERROR] Unexpected EOF while waiting for input\n"); - exit(1); + inputOk = false; + } else { + System::Get().Log("\nValidation results:\n"); } - System::Get().Log("\nValidation results:\n"); } else { System::Get().Log("Transfers complete.\nValidation results:\n"); } + } + // Coordinate any EOF abort so no rank is left waiting at the Barrier below + System::Get().Broadcast(0, sizeof(inputOk), &inputOk); + if (!inputOk) + ERR_APPEND((ErrResult{ERR_FATAL, "Unexpected EOF while waiting for interactive input"}), errResults); + + if (localRank == 0) { size_t initOffset = cfg.data.byteOffset / sizeof(float); int numPass = 0, numFail = 0; for (auto rss : transferResources) {