Replace per-iteration std::async with a persistent thread pool

CHANGELOG.md

@@ -3,6 +3,10 @@
 Documentation for TransferBench is available at
 [https://rocm.docs.amd.com/projects/TransferBench](https://rocm.docs.amd.com/projects/TransferBench).
 
+## v1.69.00
+### Fixed
+- Fix for non-zero byte offsets used with DMA executor
+
 ## v1.68.00
 ### Fixed
 - Improper draining of writes that could artificially inflate transfer timing

src/client/Client.cpp

@@ -259,7 +259,7 @@ void DisplayVersion()
 #ifndef TB_GIT_COMMIT
 #define TB_GIT_COMMIT "unknown"
 #endif
-  Print("TransferBench v%s.%s (%s:%s)%s%s\n", VERSION, CLIENT_VERSION, TB_GIT_BRANCH, TB_GIT_COMMIT, support.c_str(), multiNodeMode.c_str());
+  Print("TransferBench v%s.%sC (%s:%s)%s%s\n", VERSION, CLIENT_VERSION, TB_GIT_BRANCH, TB_GIT_COMMIT, support.c_str(), multiNodeMode.c_str());
   Print("=============================================================================================================\n");
 }
 

src/client/EnvVars.hpp

@@ -79,7 +79,7 @@ class EnvVars
   int useInteractive;                // Pause for user-input before starting transfer loop
 
   // Data options
-  int alwaysValidate;                // Validate after each iteration instead of once after all iterations
+  int alwaysValidate;                // <0 = disable validation, 0 = validate once after all iterations, >0 = validate after each iteration
   int blockBytes;                    // Each subexecutor, except the last, gets a multiple of this many bytes to copy
   int byteOffset;                    // Byte-offset for memory allocations
   vector<float> fillPattern;         // Pattern of floats used to fill source data
@@ -149,7 +149,7 @@ class EnvVars
     else if (archName == "gfx942") defaultGfxUnroll = 4;
     else if (archName == "gfx950") defaultGfxUnroll = 4;
 
-    alwaysValidate    = GetEnvVar("ALWAYS_VALIDATE"     , 0);
+    alwaysValidate    = GetEnvVar("ALWAYS_VALIDATE", 0);
     blockBytes        = GetEnvVar("BLOCK_BYTES"         , 256);
     byteOffset        = GetEnvVar("BYTE_OFFSET"         , 0);
     fillCompress      = GetEnvVarArray("FILL_COMPRESS"  , {});
@@ -352,7 +352,7 @@ class EnvVars
   {
     printf("Environment variables (client):\n");
     printf("======================\n");
-    printf(" ALWAYS_VALIDATE     - Validate after each iteration instead of once after all iterations\n");
+    printf(" ALWAYS_VALIDATE     - Data validation mode: <0=disabled, 0=validate once after all iterations (default), >0=validate after each iteration\n");
     printf(" BLOCK_BYTES         - Controls granularity of how work is divided across subExecutors\n");
     printf(" BYTE_OFFSET         - Initial byte-offset for memory allocations.  Must be multiple of 4\n");
     printf(" CU_MASK             - CU mask for streams. Can specify ranges e.g '5,10-12,14'\n");
@@ -454,7 +454,8 @@ class EnvVars
     if (hideEnv) return;
 
     Print("ALWAYS_VALIDATE", alwaysValidate,
-          "Validating after %s", (alwaysValidate ? "each iteration" : "all iterations"));
+          "Validation %s", (alwaysValidate < 0 ? "disabled" :
+                            alwaysValidate > 0 ? "after each iteration" : "after all iterations"));
     Print("BLOCK_BYTES", blockBytes,
           "Each CU gets a mulitple of %d bytes to copy", blockBytes);
     Print("BYTE_OFFSET", byteOffset,

src/client/Utilities.hpp

@@ -551,6 +551,8 @@ namespace TransferBench::Utils
     if (!RankDoesOutput()) return;
 
     if (!ev.outputToCsv) printf("Test %d:\n", testNum);
+    if (ev.alwaysValidate < 0 && !ev.outputToCsv && !ev.useInteractive)
+      printf("Validation disabled (ALWAYS_VALIDATE < 0)\n");
 
     bool isMultiRank = TransferBench::GetNumRanks() > 1;