[TRTLLMINF-133][infra] Move check testlist stage earlier in CI

jenkins/L0_MergeRequest.groovy

@@ -571,6 +571,23 @@ def launchReleaseCheck(pipeline, globalVars)
     })
 }
 
+def launchTestListCheck(pipeline, globalVars)
+{
+    def key = "Check Test List"
+    def image = globalVars["LLM_DOCKER_IMAGE"]
+    trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(image, "package"), "trt-llm", {
+        stage("[${key}] Run") {
+            echoNodeAndGpuInfo(pipeline, key)
+            sh "git config --global --add safe.directory \"*\""
+            trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true)
+
+            def llmPath = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim()
+            sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmPath} LLM_BACKEND_ROOT=${llmPath}/triton_backend " +
+               "python3 ${llmPath}/scripts/check_test_list.py --l0 --qa --waive --validate --check-duplicate-waives"
+        }
+    })
+}
+
 def getGitlabMRChangedFile(pipeline, function, filePath="") {
     def result = null
     def pageId = 0
@@ -1613,6 +1630,15 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                 launchReleaseCheck(this, globalVars)
             }
         },
+        "Check Test List": {
+            script {
+                if (GEN_POST_MERGE_BUILDS_ONLY) {
+                    echo "Skipping Check Test List (GenPostMergeBuilds mode: builds only)"
+                    return
+                }
+                launchTestListCheck(this, globalVars)
+            }
+        },
         "x86_64-Linux": {
             script {
                 // CBTS deliberately does NOT short-circuit at the arch / Build
@@ -2048,15 +2074,25 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
         echo "Will run job to build ngc containers and running in-pipeline scanning for them"
     }
 
+    def alwaysFailFastStages = ["Release-Check", "Check Test List"] as Set
     parallelJobs = stages.collectEntries{key, value -> [key, {
         script {
             stage(key) {
-                value()
+                if (enableFailFast || key in alwaysFailFastStages) {
+                    value()
+                } else {
+                    // Avoid interrupting other stages on failure.
+                    catchError(catchInterruptions: false) {
+                        value()
+                    }
+                }
             }
         }
     }]}
 
-    parallelJobs.failFast = enableFailFast
+    // With --disable-fail-fast, ordinary build/test failures are suppressed above after
+    // marking their stage and build failed, so those branches do not trigger this fail-fast.
+    parallelJobs.failFast = true
     pipeline.parallel parallelJobs
 }
 
@@ -2148,11 +2184,20 @@ pipeline {
             steps {
                 script {
                     if (isReleaseCheckMode) {
-                        stage("Release-Check") {
-                            script {
-                                launchReleaseCheck(this, globalVars)
+                        def releaseCheckStages = [
+                            "Release-Check": {
+                                stage("Release-Check") {
+                                    launchReleaseCheck(this, globalVars)
+                                }
+                            },
+                            "Check Test List": {
+                                stage("Check Test List") {
+                                    launchTestListCheck(this, globalVars)
+                                }
                             }
-                        }
+                        ]
+                        releaseCheckStages.failFast = true
+                        parallel releaseCheckStages
                     } else {
                         // globalVars[CACHED_CHANGED_FILE_LIST] is only used in setupPipelineEnvironment
                         // Remove it to workaround the "Argument list too long" error

jenkins/L0_Test.groovy

@@ -3539,30 +3539,6 @@ def runLLMAgentFlowTest(pipeline, stageName)
     sh "cd ${WORKSPACE}/${stageName} && sed -i 's/testsuite name=\"pytest\"/testsuite name=\"${stageName}\"/g' results.xml || true"
 }
 
-def launchTestListCheck(pipeline)
-{
-    stageName = "Test List Check"
-    trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(LLM_DOCKER_IMAGE, "a10"), "trt-llm", {
-        try {
-            echoNodeAndGpuInfo(pipeline, stageName)
-            sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m"
-            // download TRT-LLM tarfile
-            def tarName = BUILD_CONFIGS[VANILLA_CONFIG][TARNAME]
-            def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}"
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "pwd && wget -nv ${llmTarfile} && ls -alh")
-            sh "tar -zxf ${tarName}"
-            def llmPath = sh (script: "realpath .", returnStdout: true).trim()
-            def llmSrc = "${llmPath}/TensorRT-LLM/src"
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install -r ${llmSrc}/requirements-dev.txt")
-            sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive"
-        } catch (InterruptedException e) {
-            throw e
-        } catch (Exception e) {
-            throw e
-        }
-    })
-}
-
 def generateTimeoutTestResultXml(pipeline, stageName) {
     def scriptPath = sh(
         script: "find . -name generate_timeout_xml.py | head -n 1 | xargs realpath",
@@ -3771,7 +3747,8 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu
         }
     }
 
-    sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7"
+    def ciVersions = readProperties file: "${llmSrc}/jenkins/ci_versions.properties"
+    sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==${ciVersions.TRT_TEST_DB_VERSION}"
     // CBTS Layer 3: download the pre-built cbts_test_db/ tarball that the
     // orchestrator uploaded to Artifactory (see getCbtsResult in
     // L0_MergeRequest.groovy). This avoids re-running main.py locally and
@@ -6458,25 +6435,6 @@ pipeline {
                 }
             }
         }
-        stage("Check Test List")
-        {
-            when {
-                expression {
-                    // Only run the test list validation when necessary
-                    globalVars[RUN_MODE] != "nightly_release" &&
-                    env.targetArch == X86_64_TRIPLE &&
-                    testFilter[ONLY_ONE_GROUP_CHANGED] != "Docs" &&
-                    !(env.JOB_NAME ==~ /.*Multi-GPU.*/) &&
-                    !(env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/)
-                }
-            }
-            steps
-            {
-                script {
-                    launchTestListCheck(this)
-                }
-            }
-        }
         stage("Test") {
             steps {
                 script {

jenkins/ci_versions.properties

@@ -0,0 +1,7 @@
+# CI tool versions shared across Jenkins pipelines and Python scripts.
+# Format: KEY=VALUE (no quotes, no spaces around '=').
+#
+# Consumed by:
+#   - jenkins/L0_Test.groovy      (readProperties)
+#   - scripts/check_test_list.py  (key=value parse)
+TRT_TEST_DB_VERSION=1.8.5+bc6df7

scripts/check_test_list.py

@@ -17,6 +17,12 @@
 --waive:    Check only the tests in $LLM_ROOT/tests/integration/test_list/waives.txt.
 --validate: Run AST-based validation of test list entries against source files.
 
+Collection stub (``--l0`` / ``--qa`` / ``--waive``):
+  These modes run ``pytest --co`` with the plugin
+  ``tests/integration/defs/stubify_bindings.py`` to avoid compiling or
+  downloading TRT-LLM C++ binaries. Instead, the plugin creates a pure-Python
+  stub of the compiled modules.
+
 Note:
 All the perf tests will be excluded since they are generated dynamically.
 """
@@ -523,23 +529,63 @@ def validate_test_lists(test_lists_dir: str, test_base_dir: str):
 
 
 # =============================================================================
-# L0 / QA / Waive verification (runtime, requires pytest + model weights)
+# L0 / QA / Waive verification (runtime pytest --co with bindings collection stub)
 # =============================================================================
 
 
-def install_python_dependencies(llm_src):
+def _get_trt_test_db_version() -> str:
+    """Read TRT_TEST_DB_VERSION from jenkins/ci_versions.properties."""
+    props_file = Path(
+        __file__).resolve().parent.parent / "jenkins" / "ci_versions.properties"
+    with open(props_file) as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("TRT_TEST_DB_VERSION="):
+                return line.split("=", 1)[1]
+    raise RuntimeError(f"TRT_TEST_DB_VERSION not found in {props_file}")
+
+
+def install_python_dependencies(llm_src: str) -> None:
+    """Install Python deps for collection — no TRT-LLM wheel or compile."""
     subprocess.run(f"cd {llm_src} && pip3 install -r requirements-dev.txt",
                    shell=True,
                    check=True)
+
+    trt_test_db_ver = _get_trt_test_db_version()
     subprocess.run(
-        f"pip3 install --force-reinstall --no-deps {llm_src}/../tensorrt_llm-*.whl",
+        f"pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple "
+        f"--ignore-installed trt-test-db=={trt_test_db_ver}",
         shell=True,
         check=True)
+
+
+def _collection_pytest_env(llm_src: str) -> dict[str, str]:
+    """Env for stubbed ``pytest --co``: PYTHONPATH + placeholder models root."""
+    # The stubify_bindings plugin needs to be in the PYTHONPATH so it can be imported by pytest.
+    defs_dir = os.path.join(llm_src, "tests", "integration", "defs")
+    existing = os.environ.get("PYTHONPATH", "")
+    pythonpath = os.pathsep.join(p for p in (llm_src, defs_dir, existing) if p)
+    return {
+        **os.environ,
+        "PYTHONPATH": pythonpath,
+        "TRT_LLM_NO_LIB_INIT": "1",
+        # Collection only needs llm_models_root() to be a directory.
+        # Fixtures and weight loads do not run under pytest --co.
+        "LLM_MODELS_ROOT": "/tmp",
+    }
+
+
+def _run_collection_pytest(llm_src: str, test_list: str) -> None:
+    """Run pytest --co with the collection bindings stub plugin."""
+    env = _collection_pytest_env(llm_src)
     subprocess.run(
-        "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple "
-        "--ignore-installed trt-test-db==1.8.5+bc6df7",
+        f"cd {llm_src}/tests/integration/defs && "
+        f"pytest -p stubify_bindings --test-list={test_list} "
+        f"--output-dir={llm_src} -s --co -q",
         shell=True,
-        check=True)
+        check=True,
+        env=env,
+    )
 
 
 def verify_l0_test_lists(llm_src):
@@ -591,11 +637,7 @@ def verify_l0_test_lists(llm_src):
     with open(test_list, "w") as f:
         f.writelines(f"{line}\n" for line in sorted(cleaned_lines))
 
-    subprocess.run(
-        f"cd {llm_src}/tests/integration/defs && "
-        f"pytest --test-list={test_list} --output-dir={llm_src} -s --co -q",
-        shell=True,
-        check=True)
+    _run_collection_pytest(llm_src, test_list)
 
 
 def verify_qa_test_lists(llm_src):
@@ -605,11 +647,7 @@ def verify_qa_test_lists(llm_src):
     test_def_files = subprocess.check_output(
         f"ls -d {test_qa_path}/*.txt", shell=True).decode().strip().split('\n')
     for test_def_file in test_def_files:
-        subprocess.run(
-            f"cd {llm_src}/tests/integration/defs && "
-            f"pytest --test-list={test_def_file} --output-dir={llm_src} -s --co -q",
-            shell=True,
-            check=True)
+        _run_collection_pytest(llm_src, test_def_file)
         # append all the test_def_file to qa_test.txt
         with open(f"{llm_src}/qa_test.txt", "a") as f:
             with open(test_def_file, "r") as test_file:
@@ -720,11 +758,7 @@ def verify_waive_list(llm_src, args):
     with open(tmp_waives_file, "w") as f:
         f.writelines(f"{line}\n" for line in sorted(processed_lines))
 
-    subprocess.run(
-        f"cd {llm_src}/tests/integration/defs && "
-        f"pytest --test-list={tmp_waives_file} --output-dir={llm_src} -s --co -q",
-        shell=True,
-        check=True)
+    _run_collection_pytest(llm_src, tmp_waives_file)
 
 
 def main():
@@ -765,7 +799,6 @@ def main():
     script_dir = os.path.dirname(os.path.realpath(__file__))
     llm_src = os.path.abspath(os.path.join(script_dir, "../"))
 
-    # Only skip installing dependencies if ONLY --check-duplicates or --validate is used
     if args.l0 or args.qa or args.waive:
         install_python_dependencies(llm_src)