Skip to content

[debug] option to disable validation #334

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nileshnegi merged 11 commits into from
Jul 15, 2026
Merged

[debug] option to disable validation #334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
68a6ec6
Add option to disable/control data validation via ALWAYS_VALIDATE
nileshnegi Jul 14, 2026
d24ba28
Make interactive-mode validation a separate, gated stage
nileshnegi Jul 14, 2026
62b0ef1
Rework ALWAYS_VALIDATE to -1/0/1 for backward compatibility
nileshnegi Jul 14, 2026
266f6a4
Address Copilot review: validate env input and dedupe validation
nileshnegi Jul 14, 2026
d37ba21
Use getchar()/EOF check for interactive validation prompt
nileshnegi Jul 14, 2026
d3afbcd
Show interactive validation results for ALWAYS_VALIDATE=1
nileshnegi Jul 14, 2026
50f9f26
Condense verbose comments introduced in this branch
nileshnegi Jul 14, 2026
414bb88
Use range-based ALWAYS_VALIDATE and skip dstReference when disabled
nileshnegi Jul 14, 2026
8a1efdf
Log when validation is disabled and unify interactive prefix
nileshnegi Jul 14, 2026
92a2219
Avoid duplicate validation-disabled message in interactive mode
nileshnegi Jul 14, 2026
e95c0f4
Coordinate interactive EOF abort and fix display comment
nileshnegi Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/client/EnvVars.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class EnvVars
int useInteractive; // Pause for user-input before starting transfer loop

// Data options
int alwaysValidate; // Validate after each iteration instead of once after all iterations
int alwaysValidate; // <0 = disable validation, 0 = validate once after all iterations, >0 = validate after each iteration
int blockBytes; // Each subexecutor, except the last, gets a multiple of this many bytes to copy
int byteOffset; // Byte-offset for memory allocations
vector<float> fillPattern; // Pattern of floats used to fill source data
Expand Down Expand Up @@ -149,7 +149,7 @@ class EnvVars
else if (archName == "gfx942") defaultGfxUnroll = 4;
else if (archName == "gfx950") defaultGfxUnroll = 4;

alwaysValidate = GetEnvVar("ALWAYS_VALIDATE" , 0);
alwaysValidate = GetEnvVar("ALWAYS_VALIDATE", 0);
blockBytes = GetEnvVar("BLOCK_BYTES" , 256);
byteOffset = GetEnvVar("BYTE_OFFSET" , 0);
fillCompress = GetEnvVarArray("FILL_COMPRESS" , {});
Expand Down Expand Up @@ -352,7 +352,7 @@ class EnvVars
{
printf("Environment variables (client):\n");
printf("======================\n");
printf(" ALWAYS_VALIDATE - Validate after each iteration instead of once after all iterations\n");
printf(" ALWAYS_VALIDATE - Data validation mode: <0=disabled, 0=validate once after all iterations (default), >0=validate after each iteration\n");
printf(" BLOCK_BYTES - Controls granularity of how work is divided across subExecutors\n");
printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4\n");
printf(" CU_MASK - CU mask for streams. Can specify ranges e.g '5,10-12,14'\n");
Expand Down Expand Up @@ -454,7 +454,8 @@ class EnvVars
if (hideEnv) return;

Print("ALWAYS_VALIDATE", alwaysValidate,
"Validating after %s", (alwaysValidate ? "each iteration" : "all iterations"));
"Validation %s", (alwaysValidate < 0 ? "disabled" :
alwaysValidate > 0 ? "after each iteration" : "after all iterations"));
Print("BLOCK_BYTES", blockBytes,
"Each CU gets a mulitple of %d bytes to copy", blockBytes);
Print("BYTE_OFFSET", byteOffset,
Expand Down
2 changes: 2 additions & 0 deletions src/client/Utilities.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,8 @@ namespace TransferBench::Utils
if (!RankDoesOutput()) return;

if (!ev.outputToCsv) printf("Test %d:\n", testNum);
if (ev.alwaysValidate < 0 && !ev.outputToCsv && !ev.useInteractive)
printf("Validation disabled (ALWAYS_VALIDATE < 0)\n");

bool isMultiRank = TransferBench::GetNumRanks() > 1;

Expand Down
68 changes: 46 additions & 22 deletions src/header/TransferBench.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ namespace TransferBench
*/
struct DataOptions
{
int alwaysValidate = 0; ///< Validate after each iteration instead of once at end
int alwaysValidate = 0; ///< <0 = disable validation, 0 = validate once at end, >0 = validate after each iteration
int blockBytes = 256; ///< Each subexecutor works on a multiple of this many bytes
int byteOffset = 0; ///< Byte-offset for memory allocations
vector<float> fillPattern = {}; ///< Pattern of floats used to fill source data
Expand Down Expand Up @@ -5878,24 +5878,30 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
}
}

// Prepare reference src/dst arrays - only once for largest size
// Prepare reference src/dst arrays - only once for largest size.
// dstReference (expected results) is only needed when validation is enabled.
bool const validateEnabled = (cfg.data.alwaysValidate >= 0);
size_t maxN = maxNumBytes / sizeof(float);
vector<float> outputBuffer(maxN);
vector<vector<float>> dstReference(maxNumSrcs + 1, vector<float>(maxN));
vector<float> outputBuffer(validateEnabled ? maxN : 0);
vector<vector<float>> dstReference;
{
size_t initOffset = cfg.data.byteOffset / sizeof(float);
vector<vector<float>> srcReference(maxNumSrcs, vector<float>(maxN));
memset(dstReference[0].data(), MEMSET_CHAR, maxNumBytes);

if (validateEnabled) {
dstReference.assign(maxNumSrcs + 1, vector<float>(maxN));
memset(dstReference[0].data(), MEMSET_CHAR, maxNumBytes);
}
for (int numSrcs = 0; numSrcs < maxNumSrcs; numSrcs++) {
PrepareReference(cfg, srcReference[numSrcs], numSrcs);
for (int i = 0; i < maxN; i++) {
dstReference[numSrcs+1][i] = (numSrcs == 0 ? 0 : dstReference[numSrcs][i]) + srcReference[numSrcs][i];
}
if (validateEnabled)
for (int i = 0; i < maxN; i++)
dstReference[numSrcs+1][i] = (numSrcs == 0 ? 0 : dstReference[numSrcs][i]) + srcReference[numSrcs][i];
}
// Release un-used partial sums
for (int numSrcs = 0; numSrcs < minNumSrcs; numSrcs++)
dstReference[numSrcs].clear();
if (validateEnabled)
for (int numSrcs = 0; numSrcs < minNumSrcs; numSrcs++)
dstReference[numSrcs].clear();

// Initialize all src memory buffers (if on local rank)
bool const verbose = System::Get().IsVerbose();
Expand Down Expand Up @@ -6012,7 +6018,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart;
double deltaSec = std::chrono::duration_cast<std::chrono::duration<double>>(cpuDelta).count() / cfg.general.numSubIterations;

if (cfg.data.alwaysValidate) {
if (cfg.data.alwaysValidate > 0) {
ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer),
errResults);
}
Expand All @@ -6023,11 +6029,32 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
}
}

// Pause for interactive mode - run validation and show per-transfer result before prompt
if (cfg.general.useInteractive) {
// Interactive per-transfer PASS/FAIL display (skipped when validation disabled). This does its
// own comparison pass to show results; authoritative error reporting still comes from
// ValidateAllTransfers (mode 0) or the inline per-iteration validation (mode >0).
if (cfg.general.useInteractive && cfg.data.alwaysValidate >= 0) {
Comment thread
nileshnegi marked this conversation as resolved.
bool inputOk = true;
if (localRank == 0) {
System::Get().Log("Transfers complete. Validation results:\n");
if (cfg.data.alwaysValidate == 0) {
System::Get().Log("Transfers complete. Hit <Enter> to run validation: ");
fflush(stdout);
if (getchar() == EOF) {
System::Get().Log("[ERROR] Unexpected EOF while waiting for input\n");
inputOk = false;
} else {
System::Get().Log("\nValidation results:\n");
}
Comment thread
nileshnegi marked this conversation as resolved.
} else {
System::Get().Log("Transfers complete.\nValidation results:\n");
}
Comment thread
gilbertlee-amd marked this conversation as resolved.
}

// Coordinate any EOF abort so no rank is left waiting at the Barrier below
System::Get().Broadcast(0, sizeof(inputOk), &inputOk);
if (!inputOk)
ERR_APPEND((ErrResult{ERR_FATAL, "Unexpected EOF while waiting for interactive input"}), errResults);

if (localRank == 0) {
size_t initOffset = cfg.data.byteOffset / sizeof(float);
int numPass = 0, numFail = 0;
for (auto rss : transferResources) {
Expand Down Expand Up @@ -6074,20 +6101,17 @@ static bool IsConfiguredGid(union ibv_gid const& gid)
if (anyLocalDst) { if (transferOk) ++numPass; else ++numFail; }
}
System::Get().Log(" Summary: %d PASS %d FAIL\n", numPass, numFail);
System::Get().Log("Hit <Enter> to continue: ");
fflush(stdout);
if (scanf("%*c") != 0) {
System::Get().Log("[ERROR] Unexpected input\n");
exit(1);
}
System::Get().Log("\n");
fflush(stdout);
}
System::Get().Barrier();
} else if (cfg.general.useInteractive) {
if (localRank == 0)
System::Get().Log("Transfers complete. Validation disabled (ALWAYS_VALIDATE < 0)\n");
System::Get().Barrier();
}

// Validate results
if (!cfg.data.alwaysValidate) {
if (cfg.data.alwaysValidate == 0) {
ERR_APPEND(ValidateAllTransfers(cfg, transfers, transferResources, dstReference, outputBuffer),
errResults);
}
Expand Down
Loading