Reduce scan startup time in large repositories

.github/workflows/pr-preview.yml

@@ -261,6 +261,10 @@ jobs:
       - name: Build and push Docker preview
         uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
         with:
+          # Build from the checked-out workspace, not the default Git context. The
+          # wheel is only present here as a downloaded artifact, and this also keeps
+          # Dockerfile.preview on the default-branch checkout rather than the PR ref.
+          context: .
           file: Dockerfile.preview
           push: true
           pull: true

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2.6.5
+
+### Changed: faster local scan setup for large repositories
+
+- Replaced repeated per-pattern recursive manifest globs with one streaming
+  filesystem walk per scan root. Excluded directories and `.git` are pruned
+  before descent, reducing filesystem metadata work without building a
+  repository-sized in-memory file index.
+- Removed the unconditional `git fetch --all` from CLI initialization. Pull
+  request scans now use local refs first and fetch only a required base or head
+  ref when the checkout does not contain enough history.
+- Added native Buildkite commit, branch, pull-request range, and GitHub SCM
+  configuration fallbacks so Buildkite jobs no longer need GitHub
+  Actions-shaped environment-variable shims.
+- Added INFO-level timings for CLI run registration, organization setup, Git
+  initialization and fetches, changed-file detection, supported-pattern lookup,
+  and manifest discovery.
+- Supported manifest patterns are cached for each CLI invocation once the API
+  returns them, so a transient lookup failure no longer keeps the run on the
+  smaller local fallback pattern set. Manifest results from `--sub-path` routing
+  are reused during scan creation.
+
 ## 2.6.4
 
 ### Changed: bump pinned @coana-tech/cli to 15.10.13

benchmarks/manifest_discovery.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Compare legacy per-pattern rglob discovery with the single-pass walker.
+
+This is an opt-in developer benchmark, not a timing assertion in the test
+suite. It creates a synthetic monorepo so filesystem or CI-agent changes do not
+make regular tests flaky.
+"""
+
+import argparse
+import tempfile
+import time
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+from socketsecurity.core import Core
+from socketsecurity.core.socket_config import SocketConfig
+from socketsecurity.core.utils import socket_globs
+
+
+def seed_tree(root: Path, directories: int, files_per_directory: int) -> None:
+    for directory_index in range(directories):
+        directory = root / "packages" / f"package-{directory_index:05d}"
+        directory.mkdir(parents=True)
+        (directory / "package.json").write_text("{}\n", encoding="utf-8")
+        for file_index in range(files_per_directory):
+            (directory / f"source-{file_index:03d}.txt").write_text(
+                "not a manifest\n",
+                encoding="utf-8",
+            )
+
+    # These trees model the expensive directories that the new walker prunes
+    # before descent rather than visiting once for every manifest pattern.
+    for excluded in (".git/objects", "node_modules/example", ".venv/site-packages"):
+        directory = root / excluded
+        directory.mkdir(parents=True)
+        for index in range(files_per_directory * 10):
+            (directory / f"object-{index:05d}").write_text("x", encoding="utf-8")
+
+
+def legacy_discover(root: Path) -> set[str]:
+    results = set()
+    excluded_dirs = SocketConfig(api_key="benchmark").excluded_dirs
+    for ecosystem_patterns in socket_globs.values():
+        for details in ecosystem_patterns.values():
+            for pattern in Core.expand_brace_pattern(details["pattern"]):
+                insensitive = Core.to_case_insensitive_regex(pattern)
+                for candidate in root.rglob(insensitive):
+                    if candidate.is_file() and not Core.is_excluded(
+                            str(candidate),
+                            excluded_dirs,
+                    ):
+                        results.add(candidate.as_posix())
+    return results
+
+
+def new_core() -> Core:
+    core = Core.__new__(Core)
+    core.config = SocketConfig(api_key="benchmark")
+    core.cli_config = SimpleNamespace(exclude_paths=None)
+    core.sdk = MagicMock()
+    core._supported_patterns = socket_globs
+    return core
+
+
+def timed(function, root: Path) -> tuple[set[str], float]:
+    start = time.perf_counter()
+    results = set(function(root))
+    return results, time.perf_counter() - start
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--directories", type=int, default=500)
+    parser.add_argument("--files-per-directory", type=int, default=20)
+    args = parser.parse_args()
+
+    with tempfile.TemporaryDirectory(prefix="socket-manifest-benchmark-") as temp:
+        root = Path(temp)
+        seed_tree(root, args.directories, args.files_per_directory)
+        legacy_results, legacy_seconds = timed(legacy_discover, root)
+        new_results, new_seconds = timed(
+            lambda path: new_core().find_files(str(path)),
+            root,
+        )
+
+    if legacy_results != new_results:
+        raise SystemExit(
+            "Manifest result mismatch: "
+            f"legacy={len(legacy_results)}, single_pass={len(new_results)}"
+        )
+
+    speedup = legacy_seconds / new_seconds if new_seconds else float("inf")
+    print(f"Manifests: {len(new_results)}")
+    print(f"Legacy per-pattern rglob: {legacy_seconds:.3f}s")
+    print(f"Single-pass walk: {new_seconds:.3f}s")
+    print(f"Speedup: {speedup:.1f}x")
+
+
+if __name__ == "__main__":
+    main()

docs/ci-cd.md

@@ -81,6 +81,19 @@ steps:
       SOCKET_SECURITY_API_TOKEN: "${SOCKET_SECURITY_API_TOKEN}"
 ```
 
+The CLI reads Buildkite's native `BUILDKITE_COMMIT`, `BUILDKITE_BRANCH`,
+`BUILDKITE_PULL_REQUEST`, and `BUILDKITE_PULL_REQUEST_BASE_BRANCH` variables.
+For pull-request builds, ensure the checkout contains the base branch and the
+checked-out head commit. The CLI uses those local refs first and performs a
+targeted fetch only when a required ref or its comparison history is missing;
+it does not fetch every remote ref and tag during startup.
+
+When `--scm github` is used from Buildkite, the CLI also derives GitHub comment
+context from `BUILDKITE_REPO`, `BUILDKITE_BUILD_CHECKOUT_PATH`, and the variables
+above. Set `GH_API_TOKEN` to a GitHub token with the required repository access.
+GitHub Enterprise users should also set `GITHUB_API_URL`; GitHub.com defaults to
+`https://api.github.com`.
+
 #### Merge-base baselines in Buildkite (dynamic pipelines)
 
 Notes for using `--base-commit-sha` (see the

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.6.4"
+version = "2.6.5"
 requires-python = ">= 3.11"
 license = {"file" = "LICENSE"}
 dependencies = [

socketsecurity/__init__.py

@@ -1,3 +1,3 @@
 __author__ = 'socket.dev'
-__version__ = '2.6.4'
+__version__ = '2.6.5'
 USER_AGENT = f'SocketPythonCLI/{__version__}'