From 876630d53913302486602f14ef2f45c3ed6614bc Mon Sep 17 00:00:00 2001 From: jdalton Date: Wed, 29 Jul 2026 21:48:53 -0400 Subject: [PATCH 1/4] feat(bindings): upstream provenance pins + lock-step gate for well-known bindings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the socket-registry fleet's upstream-reference technique to well_known_bindings.toml: every third-party npm package with a bundled native wrapper records which upstream release it ports (version, tarball sha256, repo, git ref) and when it was last reviewed against it (ported-at, date). The lock-step rule — ported-at == version — means a pin bump can't outrun the wrapper review it demands. Enforced twice: the toml parser inside perry refuses to load a skewed table, and scripts/binding_pins.mjs --check gates it in CI (offline). Node builtins (node-builtin=true), aliases (alias-of), and perry-owned packages are exempt from pinning; a distinct package sharing a wrapper crate (redis, iovalkey) still carries its own pin. scripts/binding_pins.mjs provisions pins from the npm registry (--set/--backfill), gates them offline (--check), flags newly-soaked upstream releases (--check --refresh --soak-days N), and materializes an upstream at its pin for port review (--materialize). All 34 third-party bindings backfilled at their current latest. Also adds an iovalkey binding — the Valkey fork of ioredis (valkey-io/iovalkey), a distinct npm package served by the existing perry-ext-ioredis surface — wired through NATIVE_MODULES, the codegen module-normalization + Redis-constructor lists, and stdlib feature gating, and annotates every NATIVE_MODULES entry. 758 tests pass; binding_pins.mjs --check green. --- .github/workflows/test.yml | 9 + changelog.d/0000-binding-upstream-lockstep.md | 1 + crates/perry-api-manifest/src/entries.rs | 219 +++++----- .../perry-codegen/src/lower_call/builtin.rs | 2 +- .../src/lower_call/native_module_dispatch.rs | 5 +- .../perry/src/commands/compile/well_known.rs | 205 ++++++++++ crates/perry/src/commands/stdlib_features.rs | 2 +- crates/perry/well_known_bindings.toml | 259 ++++++++++++ docs/src/SUMMARY.md | 1 + docs/src/native-libraries/upstream-pins.md | 81 ++++ scripts/binding_pins.mjs | 386 ++++++++++++++++++ 11 files changed, 1062 insertions(+), 108 deletions(-) create mode 100644 changelog.d/0000-binding-upstream-lockstep.md create mode 100644 docs/src/native-libraries/upstream-pins.md create mode 100644 scripts/binding_pins.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cc540c849..80d7140c8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,6 +147,15 @@ jobs: - name: File size limit run: ./scripts/check_file_size.sh + # Well-known binding provenance pins: every third-party binding in + # well_known_bindings.toml must carry an [bindings..upstream] + # pin, and the lock-step rule (ported-at == version) must hold, so a + # pin bump can't outrun the wrapper review it demands. Offline/CI-safe + # (no network); the weekly update runs `--check --refresh` to surface + # newly-soaked upstream releases as advisories. + - name: Binding upstream pins (lock-step) + run: node scripts/binding_pins.mjs --check + # GC write-barrier store-site inventory: every raw heap-slot store in # perry-codegen / perry-runtime / perry-stdlib must be barriered or # carry a justified GC_STORE_AUDIT(...) marker (or a justified entry diff --git a/changelog.d/0000-binding-upstream-lockstep.md b/changelog.d/0000-binding-upstream-lockstep.md new file mode 100644 index 0000000000..a9a78656ca --- /dev/null +++ b/changelog.d/0000-binding-upstream-lockstep.md @@ -0,0 +1 @@ +**Well-known bindings now carry upstream provenance pins with a lock-step review gate.** Every third-party npm package with a bundled native wrapper in `well_known_bindings.toml` records the exact upstream release it ports (version, tarball sha256, repo, git ref) and the date/release it was last reviewed against. The lock-step rule — `ported-at` must equal `version` — means a pin bump can't outrun the wrapper review it demands; it's enforced by `node scripts/binding_pins.mjs --check` in CI and by the perry binary's own table parser. Node builtins, aliases, and perry-owned packages are exempt. New `scripts/binding_pins.mjs` provisions pins (`--set`/`--backfill`), gates them offline (`--check`), flags newly-soaked upstream releases (`--check --refresh`), and materializes an upstream at its pin for review (`--materialize`). Adapted from the socket-registry fleet's upstream-reference technique. Also adds an `iovalkey` binding (the Valkey fork of ioredis) served by the existing `perry-ext-ioredis` surface. diff --git a/crates/perry-api-manifest/src/entries.rs b/crates/perry-api-manifest/src/entries.rs index a9fa434d24..591d1fdfd9 100644 --- a/crates/perry-api-manifest/src/entries.rs +++ b/crates/perry-api-manifest/src/entries.rs @@ -28,123 +28,132 @@ use crate::{ApiEntry, ApiKind, ApiSource, ParamSpec, TypeSpec}; /// answer module-resolution questions without depending on /// `perry-hir`. Order matches the original list to keep diffs minimal. pub const NATIVE_MODULES: &[&str] = &[ - "mysql2", - "mysql2/promise", - "pg", - "uuid", - "bcrypt", - "argon2", - "ioredis", - "axios", - "node-fetch", - "ws", - "zlib", - "crypto", - "dotenv", - "dotenv/config", - "jsonwebtoken", - "nanoid", - "slugify", - "validator", - "ethers", - "mongodb", - "better-sqlite3", - "sqlite", - "tursodb", - "iroh", + // ── Third-party npm packages (native wrappers; see well_known_bindings.toml) ── + "mysql2", // MySQL/MariaDB client + "mysql2/promise", // mysql2's promise-API subpath + "pg", // PostgreSQL client + "uuid", // RFC-4122 UUID generation + "bcrypt", // bcrypt password hashing (replaces the N-API addon) + "argon2", // Argon2 password hashing (replaces the N-API addon) + "ioredis", // Redis/Valkey client + // iovalkey: the Valkey fork of ioredis (valkey-io/iovalkey), served by the + // same perry-ext-ioredis surface — see well_known_bindings.toml. + "iovalkey", + "axios", // HTTP client (routes onto the native fetch/http stack) + "node-fetch", // WHATWG fetch client + "ws", // WebSocket client/server + "zlib", // (Node builtin) gzip/deflate/brotli/zstd compression + "crypto", // (Node builtin) hashing, HMAC, cipher, sign/verify, WebCrypto + "dotenv", // .env file loader + "dotenv/config", // dotenv's auto-load-on-import subpath + "jsonwebtoken", // JWT sign/verify + "nanoid", // compact URL-safe ID generation + "slugify", // string → URL slug + "validator", // string validators/sanitizers + "ethers", // Ethereum library (utils/wallet/ABI) + "mongodb", // MongoDB driver + "better-sqlite3", // synchronous SQLite (replaces the N-API addon) + "sqlite", // node:sqlite builtin surface + "tursodb", // Turso/libSQL client (legacy in-tree; now @perryts/tursodb) + "iroh", // iroh p2p (legacy in-tree; now @perryts/iroh) // #6562: Bun FFI (C-ABI). The `bun:` prefix is part of the specifier // (unlike `node:`, which is stripped) — `import { dlopen } from "bun:ffi"`. "bun:ffi", - "node-cron", - "nodemailer", - "http", - "https", - "http2", - "inspector", - "inspector/promises", - "events", - "domain", - "os", - "buffer", - "assert", - "assert/strict", - "test", - "child_process", - "dns", - "dns/promises", - "dgram", - "net", - "tls", - "stream", - "streams", - "fs", - "module", - "path", - "path/posix", - "path/win32", - "console", - "constants", - "util", - "util/types", - "dns", - "dns/promises", - "url", - "lru-cache", - "commander", - "decimal.js", - "bignumber.js", - "exponential-backoff", - "lodash", - "dayjs", - "date-fns", - "moment", - "sharp", - "cheerio", - "cron", - "fastify", - "async_hooks", + "node-cron", // cron-style scheduler (npm node-cron; aliases `cron`) + "nodemailer", // SMTP email sending + // ── Node.js builtin modules ── + "http", // HTTP client + server + "https", // HTTPS client + server + "http2", // HTTP/2 client + server + "inspector", // V8 inspector protocol + "inspector/promises", // inspector's promise-API subpath + "events", // EventEmitter + "domain", // (legacy) error-domain grouping + "os", // OS info (platform, cpus, hostname, …) + "buffer", // Buffer / Blob + "assert", // assertions + "assert/strict", // assert in strict mode + "test", // node:test runner surface + "child_process", // spawn/exec subprocesses + "dns", // DNS resolution + "dns/promises", // dns promise-API subpath + "dgram", // UDP sockets + "net", // TCP sockets + servers + "tls", // TLS/SSL sockets + "stream", // streams (Readable/Writable/Transform) + "streams", // WHATWG web-streams surface + "fs", // filesystem + "module", // module system introspection (createRequire, …) + "path", // path manipulation (host flavor) + "path/posix", // path, POSIX semantics + "path/win32", // path, Windows semantics + "console", // console.* logging + "constants", // (legacy) OS/fs/crypto constant tables + "util", // promisify, inspect, TextEncoder, … + "util/types", // runtime type predicates (isDate, …) + "dns", // (duplicate of the dns entry above — kept for parity) + "dns/promises", // (duplicate — kept for parity) + "url", // URL / URLSearchParams + // ── More third-party npm packages ── + "lru-cache", // LRU cache + "commander", // CLI argument parser + "decimal.js", // arbitrary-precision decimals + "bignumber.js", // arbitrary-precision big numbers + "exponential-backoff", // retry-with-backoff helper + "lodash", // general utility library + "dayjs", // date/time library + "date-fns", // functional date utilities + "moment", // (legacy) date/time library + "sharp", // image processing (replaces the N-API addon) + "cheerio", // server-side jQuery-style HTML parsing + "cron", // cron scheduler (aliases node-cron) + "fastify", // HTTP server framework + // ── Node.js builtins (cont.) ── + "async_hooks", // async context tracking // #2875: internal module backing DisposableStack/AsyncDisposableStack // instance-method dispatch (no JS import surface). "__disposable__", - "readline", - "repl", - "sea", - "string_decoder", - "querystring", - "cluster", - "tty", - "wasi", - "perf_hooks", - "v8", - "vm", - "process", + "readline", // line-by-line stdin reading + "repl", // REPL surface + "sea", // single-executable-application API + "string_decoder", // incremental byte→string decoding + "querystring", // (legacy) query-string encode/decode + "cluster", // worker-process clustering + "tty", // terminal I/O + "wasi", // WebAssembly System Interface + "perf_hooks", // performance measurement + "v8", // V8-compat introspection surface + "vm", // script compilation/eval sandboxes + "process", // the process object as an importable module + // ── perry-owned builtins (Perry-native; don't resolve under Node/Bun) ── // Bare `perry` builtin — embedded-asset introspection (#5731): // `embeddedFiles`, `readEmbedded`, `isStandaloneExecutable`. "perry", - "perry/tui", - "perry/yoga", - "perry/ui", - "perry/system", - "perry/plugin", - "perry/widget", - "perry/i18n", - "worker_threads", - "perry/thread", + "perry/tui", // terminal-UI framework + "perry/yoga", // Yoga flexbox layout + "perry/ui", // native UI (AppKit/UIKit/Win32/GTK4/…) + "perry/system", // OS integration (keychain, notifications, …) + "perry/plugin", // compile-time plugin surface + "perry/widget", // home-screen widgets (WidgetKit/Glance) + "perry/i18n", // internationalization runtime + "worker_threads", // (Node builtin) OS-thread workers + "perry/thread", // perry-native threading (parallelMap/spawn) // `perry/gc` — explicit GC control (collect / minor / idleHint). // Served entirely by perry-runtime; a no-op-style Perry-native // surface like `perry/thread` (doesn't resolve under Node/Bun). "perry/gc", - "perry/updater", - "perry/container", - "perry/container-compose", - "perry/compose", - "perry/workloads", - "perry/media", - "perry/audio", - "perry/background", - "redis", - "rate-limiter-flexible", - "fetch", + "perry/updater", // auto-update client (@perry/updater signer side) + "perry/container", // container runtime surface + "perry/container-compose", // docker-compose-style orchestration + "perry/compose", // compose helpers + "perry/workloads", // workload scheduling + "perry/media", // media (video/image) surface + "perry/audio", // audio surface + "perry/background", // background-task surface + // ── More third-party npm packages ── + "redis", // npm `redis` client (aliases ioredis) + "rate-limiter-flexible", // rate limiting + "fetch", // bare-name alias for the node-fetch surface // `@perryts/pdf` — official PDF creation package (#516). // Bundled wrapper lives in `crates/perry-ext-pdf`; the producer // side companion to the existing PdfView widget. d.ts at @@ -167,7 +176,7 @@ pub const NATIVE_MODULES: &[&str] = &[ // the API-identical @lydell fork (opencode's static import) resolve to // the one perry-runtime implementation — no N-API addon involved. "node-pty", - "@lydell/node-pty", + "@lydell/node-pty", // API-identical node-pty fork (see above) ]; /// Node built-in submodules that Perry routes through the diff --git a/crates/perry-codegen/src/lower_call/builtin.rs b/crates/perry-codegen/src/lower_call/builtin.rs index d06ef28eec..e94416c701 100644 --- a/crates/perry-codegen/src/lower_call/builtin.rs +++ b/crates/perry-codegen/src/lower_call/builtin.rs @@ -48,7 +48,7 @@ pub(super) fn lower_builtin_new( "Client" | "Pool" => Some(&["pg"]), "Database" => Some(&["better-sqlite3"]), "DatabaseSync" | "Session" | "StatementSync" => Some(&["sqlite", "node:sqlite"]), - "Redis" => Some(&["ioredis", "redis"]), + "Redis" => Some(&["ioredis", "redis", "iovalkey"]), "MongoClient" => Some(&["mongodb"]), "Decimal" => Some(&["decimal.js"]), "RateLimiterMemory" => Some(&["rate-limiter-flexible"]), diff --git a/crates/perry-codegen/src/lower_call/native_module_dispatch.rs b/crates/perry-codegen/src/lower_call/native_module_dispatch.rs index 77d514999f..064fe7c176 100644 --- a/crates/perry-codegen/src/lower_call/native_module_dispatch.rs +++ b/crates/perry-codegen/src/lower_call/native_module_dispatch.rs @@ -36,7 +36,10 @@ pub fn native_module_lookup( // here so call-site lookups find the right runtime fns regardless // of which alias the user imported from. let normalized = match module { - "redis" => "ioredis", + // `redis` and `iovalkey` (the Valkey fork of ioredis) share the + // perry-ext-ioredis staticlib and its `js_ioredis_*` dispatch rows; + // normalize both to `ioredis` so call-site lookups resolve. + "redis" | "iovalkey" => "ioredis", "sys" => "util", // #6563: @lydell/node-pty is an API-identical fork of node-pty // (opencode imports the fork, kimi-code the original); both route to diff --git a/crates/perry/src/commands/compile/well_known.rs b/crates/perry/src/commands/compile/well_known.rs index c4cf0005ea..2f0f1af4f6 100644 --- a/crates/perry/src/commands/compile/well_known.rs +++ b/crates/perry/src/commands/compile/well_known.rs @@ -28,6 +28,55 @@ pub struct WellKnownBinding { /// GitHub issue tracking the migration. Surfaced in error /// messages when the bundled `.a` is absent. pub tracking: Option, + /// Upstream provenance pin — which release of the npm package this + /// wrapper ports, and when it was last reviewed against it. See + /// `docs/src/native-libraries/upstream-pins.md` and the lock-step + /// gate in `scripts/binding_pins.mjs`. `None` for entries exempt + /// from pinning (`node_builtin`, `alias_of`, and perry-owned + /// packages). + pub upstream: Option, + /// `true` when this binding ports a Node.js **builtin** module + /// (`node:zlib`/`net`/`http`/…) rather than a third-party npm + /// package. Its upstream is Node core, not an npm dist, so it + /// carries no npm provenance pin. + pub node_builtin: bool, + /// When set, this row is an **alias** for another binding (a + /// package subpath like `mysql2/promise`, or a bare-name alias like + /// `fetch` → `node-fetch`) and shares that binding's provenance + /// instead of carrying its own pin. + pub alias_of: Option, +} + +/// Provenance pin for a binding's upstream npm package — the same record +/// shape as an upstream reference submodule's `.gitmodules` block +/// (pinned release + content hash + review stamp), carried as toml +/// fields since the upstream here is an npm dist, not a vendored tree. +/// +/// The **lock-step rule**: `ported_at` must equal `version`. Re-pinning +/// an upstream release without re-reviewing the wrapper against the +/// upstream diff reds the `binding_pins.mjs --check` gate until +/// `ported_at` advances with the review — an upstream release can never +/// go silently stale, and a pin bump can never outrun its port. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpstreamPin { + /// Pinned upstream npm release (an immutable published version, + /// e.g. `"17.4.2"`) — the analogue of a release-tag pin. + pub version: String, + /// SHA-256 of the npm registry tarball for `version` — the + /// content hash of record (npm's own `dist.integrity` is sha512; + /// this is computed from the tarball bytes at pin time so it can + /// be independently re-verified with `shasum -a 256`). + pub sha256: String, + /// Upstream source repository URL, when the package declares one. + pub repo: Option, + /// Upstream git commit for the release (`gitHead` from the npm + /// registry when the publisher recorded it) — empty when unknown. + pub git_ref: Option, + /// Release the wrapper was last **reviewed** against. Lock-step: + /// must equal `version`. + pub ported_at: String, + /// ISO date (YYYY-MM-DD) of that review. + pub date: String, } /// Parse the embedded toml on first call; reuse on subsequent ones. @@ -157,6 +206,57 @@ fn parse_well_known_toml(raw: &str) -> Result .and_then(|v| v.as_str()) .map(String::from); + let node_builtin = entry_table + .get("node-builtin") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let alias_of = entry_table + .get("alias-of") + .and_then(|v| v.as_str()) + .map(String::from); + + let upstream = match entry_table.get("upstream") { + None => None, + Some(value) => { + let up = value + .as_table() + .ok_or_else(|| format!("[bindings.{}.upstream] is not a table", pkg_name))?; + let required = |field: &str| -> Result { + up.get(field) + .and_then(|v| v.as_str()) + .map(String::from) + .ok_or_else(|| { + format!( + "[bindings.{}.upstream] missing required `{}` field", + pkg_name, field + ) + }) + }; + let version = required("version")?; + let ported_at = required("ported-at")?; + // Parse-time lock-step backstop. The authoritative gate is + // `scripts/binding_pins.mjs --check` (CI); failing here too + // means a skewed pin can't even ship inside the binary. + if ported_at != version { + return Err(format!( + "[bindings.{}.upstream] lock-step violation: ported-at ({}) \ + != version ({}) — re-review the wrapper against the \ + upstream diff and advance ported-at with the review", + pkg_name, ported_at, version + )); + } + Some(UpstreamPin { + version, + sha256: required("sha256")?, + repo: up.get("repo").and_then(|v| v.as_str()).map(String::from), + git_ref: up.get("ref").and_then(|v| v.as_str()).map(String::from), + ported_at, + date: required("date")?, + }) + } + }; + out.insert( pkg_name.clone(), WellKnownBinding { @@ -164,6 +264,9 @@ fn parse_well_known_toml(raw: &str) -> Result krate, lib, tracking, + upstream, + node_builtin, + alias_of, }, ); } @@ -349,4 +452,106 @@ mod tests { missing_stdlib.join("\n") ); } + + #[test] + fn upstream_pin_parses() { + let raw = r#" + [bindings.foo] + crate = "perry-ext-foo" + lib = "perry_ext_foo" + + [bindings.foo.upstream] + version = "1.2.3" + sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + repo = "https://github.com/example/foo" + ref = "0123456789012345678901234567890123456789" + ported-at = "1.2.3" + date = "2026-07-29" + "#; + let parsed = parse_well_known_toml(raw).expect("pinned entry must parse"); + let pin = parsed["foo"] + .upstream + .as_ref() + .expect("upstream pin present"); + assert_eq!(pin.version, "1.2.3"); + assert_eq!(pin.ported_at, "1.2.3"); + assert_eq!(pin.date, "2026-07-29"); + assert_eq!(pin.repo.as_deref(), Some("https://github.com/example/foo")); + } + + /// The lock-step rule enforced at parse time: a pin bump + /// (`version`) that outruns its review (`ported-at`) must refuse + /// to load — the authoritative CI gate is binding_pins.mjs + /// --check, but a skewed pin must not even ship inside the binary. + #[test] + fn upstream_pin_rejects_lock_step_violation() { + let raw = r#" + [bindings.foo] + crate = "perry-ext-foo" + lib = "perry_ext_foo" + + [bindings.foo.upstream] + version = "2.0.0" + sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ported-at = "1.2.3" + date = "2026-07-29" + "#; + let err = parse_well_known_toml(raw).expect_err("skewed pin must reject"); + assert!(err.contains("lock-step"), "got: {}", err); + assert!(err.contains("ported-at"), "got: {}", err); + } + + #[test] + fn upstream_pin_rejects_missing_required_field() { + let raw = r#" + [bindings.foo] + crate = "perry-ext-foo" + lib = "perry_ext_foo" + + [bindings.foo.upstream] + version = "1.2.3" + ported-at = "1.2.3" + date = "2026-07-29" + "#; + let err = parse_well_known_toml(raw).expect_err("missing sha256 must reject"); + assert!(err.contains("sha256"), "got: {}", err); + } + + /// Every shipped binding must carry an upstream pin — the port map + /// is TOTAL, so a new binding can't land silently unpinned. (Same + /// rule as `every_entry_references_a_workspace_crate` above.) + #[test] + fn every_entry_carries_an_upstream_pin() { + let unpinned: Vec<&str> = iter_well_known() + .filter(|b| b.upstream.is_none()) + // Exempt: Node builtins (upstream is Node core, not npm), aliases + // (share the aliased binding's pin), and perry-owned packages. + .filter(|b| !b.node_builtin && b.alias_of.is_none()) + .filter(|b| !b.package.starts_with("@perryts/") && !b.package.starts_with("perry/")) + .map(|b| b.package.as_str()) + .collect(); + assert!( + unpinned.is_empty(), + "bindings without an [bindings..upstream] pin — provision one with \ + `node scripts/binding_pins.mjs --set `:\n {}", + unpinned.join("\n ") + ); + } + + /// Every `alias-of` must point at a binding that actually exists — + /// a dangling alias would resolve to a real crate (via its own + /// crate/lib fields) but claim provenance from a phantom parent. + #[test] + fn alias_of_targets_exist() { + for b in iter_well_known() { + if let Some(target) = &b.alias_of { + assert!( + lookup_well_known(target).is_some(), + "binding `{}` is alias-of `{}`, which is not a known binding", + b.package, + target + ); + } + } + } } diff --git a/crates/perry/src/commands/stdlib_features.rs b/crates/perry/src/commands/stdlib_features.rs index 5438864028..ed3c9de799 100644 --- a/crates/perry/src/commands/stdlib_features.rs +++ b/crates/perry/src/commands/stdlib_features.rs @@ -86,7 +86,7 @@ pub fn module_to_features(module: &str) -> &'static [&'static str] { // `database-redis` umbrella retained for backwards-compat; // per-binding gate is `bundled-ioredis` (v0.5.565) so the // well-known flip can route to perry-ext-ioredis. - "ioredis" | "redis" => &["bundled-ioredis"], + "ioredis" | "redis" | "iovalkey" => &["bundled-ioredis"], // `database-mongodb` umbrella retained for backwards-compat; // per-binding gate is `bundled-mongodb` (v0.5.568) so the // well-known flip can route to perry-ext-mongodb. diff --git a/crates/perry/well_known_bindings.toml b/crates/perry/well_known_bindings.toml index 9c83b201ed..2ec5634d1f 100644 --- a/crates/perry/well_known_bindings.toml +++ b/crates/perry/well_known_bindings.toml @@ -41,55 +41,125 @@ lib = "perry_ext_dotenv" # when the bundled .a is missing at link time. tracking = "#466" +[bindings.dotenv.upstream] +version = "17.4.2" +sha256 = "8648852be8209110b34dca75dcc3ed12ce7fae9fcc8edd1ef9e180e708af1398" +repo = "https://github.com/motdotla/dotenv" +ref = "a61f616a3160bb6e6f22ff55f08b7eba3a3fab68" +ported-at = "17.4.2" +date = "2026-07-30" [bindings.nanoid] crate = "perry-ext-nanoid" lib = "perry_ext_nanoid" tracking = "#466" +[bindings.nanoid.upstream] +version = "6.0.0" +sha256 = "5cade80a39ccf4fd174c8e412eca13accfce36c9c2a0982b4ea23403d729a8d8" +repo = "https://github.com/ai/nanoid" +ref = "4dacb107b54ffd0e1abfe91b7ef452e0fd5a8e12" +ported-at = "6.0.0" +date = "2026-07-30" [bindings.uuid] crate = "perry-ext-uuid" lib = "perry_ext_uuid" tracking = "#466" +[bindings.uuid.upstream] +version = "14.0.1" +sha256 = "e062c57ed120ea135478f305da04802bf69eab23fbc0ca2bae6989b6855390c4" +repo = "https://github.com/uuidjs/uuid" +ref = "70177807e9229dfacde2038dc1e722f1828f358a" +ported-at = "14.0.1" +date = "2026-07-30" [bindings.slugify] crate = "perry-ext-slugify" lib = "perry_ext_slugify" tracking = "#466" +[bindings.slugify.upstream] +version = "1.6.9" +sha256 = "19ab247286f687aa8803b3615eab0779f517904ec96298e5005ff4de77ff9948" +repo = "https://github.com/simov/slugify" +ref = "78c6ee658d61d81a3a33bc1f558b50013be9ccf6" +ported-at = "1.6.9" +date = "2026-07-30" [bindings.bcrypt] crate = "perry-ext-bcrypt" lib = "perry_ext_bcrypt" tracking = "#466" +[bindings.bcrypt.upstream] +version = "6.0.0" +sha256 = "3b2478cda06f066c744e06cd0b68f55e14cc546c19ee2b80be600dfc65b5ae20" +repo = "https://github.com/kelektiv/node.bcrypt.js" +ported-at = "6.0.0" +date = "2026-07-30" [bindings.argon2] crate = "perry-ext-argon2" lib = "perry_ext_argon2" tracking = "#466" +[bindings.argon2.upstream] +version = "0.45.1" +sha256 = "4acffbd5e5a87fe8fad926afc9a9ac450d69b7082686dd2836be0d8309dbd8aa" +repo = "https://github.com/ranisalt/node-argon2" +ref = "786de7152f95881b0683aea1d2ca60ed0d6d9e2f" +ported-at = "0.45.1" +date = "2026-07-30" [bindings.jsonwebtoken] crate = "perry-ext-jsonwebtoken" lib = "perry_ext_jsonwebtoken" tracking = "#466" +[bindings.jsonwebtoken.upstream] +version = "9.0.3" +sha256 = "d9af2628a7a4dda25acf1e19c7ecc2468e1e9e8d4619fe2cae829e89d96f6b82" +repo = "https://github.com/auth0/node-jsonwebtoken" +ref = "ed59e76ea37a80f54b833668c02a5271984dcba3" +ported-at = "9.0.3" +date = "2026-07-30" [bindings.validator] crate = "perry-ext-validator" lib = "perry_ext_validator" tracking = "#466" +[bindings.validator.upstream] +version = "13.15.35" +sha256 = "f9a6b506bd9eda8df9d2a4120613426948d9f66cde1b6d5fad3406758d2f81f4" +repo = "https://github.com/validatorjs/validator.js" +ref = "7a8079709cd4cb27b2a1846e6f6508d68c9d928f" +ported-at = "13.15.35" +date = "2026-07-30" [bindings.lru-cache] crate = "perry-ext-lru-cache" lib = "perry_ext_lru_cache" tracking = "#466" +[bindings.lru-cache.upstream] +version = "11.5.2" +sha256 = "e46c8eaafc64f168603aebd39cfd0e987bec39a93ade280653e2331fd2516a22" +repo = "https://github.com/isaacs/node-lru-cache" +ref = "16b3a916662ab449d496b7b4b4f04132565d1d28" +ported-at = "11.5.2" +date = "2026-07-30" [bindings.better-sqlite3] crate = "perry-ext-better-sqlite3" lib = "perry_ext_better_sqlite3" tracking = "#466" +[bindings.better-sqlite3.upstream] +version = "13.0.2" +sha256 = "785493c44913bd6415eb4ef244f97b4774a8ba1ee7c45137f05c3ad09282bc75" +repo = "https://github.com/WiseLibs/better-sqlite3" +ref = "569e85ad031515d665c18c601da20d1915675ee5" +ported-at = "13.0.2" +date = "2026-07-30" [bindings.zlib] crate = "perry-ext-zlib" lib = "perry_ext_zlib" tracking = "#466" +node-builtin = true [bindings.exponential-backoff] crate = "perry-ext-exponential-backoff" @@ -108,140 +178,319 @@ tracking = "#466" # its job: bundling the wrappers that *port* perry-stdlib's existing # in-tree implementations, not net-new functionality. +[bindings.exponential-backoff.upstream] +version = "3.1.3" +sha256 = "2ede00f3e458fa991d304dd8a1fed812d686ac5d213c1d6704963c3674913526" +repo = "https://github.com/coveooss/exponential-backoff" +ref = "5622170828bd91dc149585bfa3d82c8972a75ac5" +ported-at = "3.1.3" +date = "2026-07-30" [bindings.axios] crate = "perry-ext-axios" lib = "perry_ext_axios" tracking = "#466" +[bindings.axios.upstream] +version = "1.19.0" +sha256 = "a511049fdaec40a320368b3ee965079b3e14481f82d052584f746bbdc3f01ede" +repo = "https://github.com/axios/axios" +ref = "311fcc5c8d989b7248f05d390bb83bfbfb009977" +ported-at = "1.19.0" +date = "2026-07-30" [bindings.events] crate = "perry-ext-events" lib = "perry_ext_events" tracking = "#466" +node-builtin = true [bindings."decimal.js"] crate = "perry-ext-decimal" lib = "perry_ext_decimal" tracking = "#466" +[bindings."decimal.js".upstream] +version = "10.6.0" +sha256 = "433210122e8149972f64054c54b0ce0872e5f15a3c5835aab57ff1a8e6d071b1" +repo = "https://github.com/MikeMcl/decimal.js" +ref = "f1ee2f404d6bf96d59c04db80c1f404742afa3fa" +ported-at = "10.6.0" +date = "2026-07-30" [bindings."bignumber.js"] crate = "perry-ext-decimal" lib = "perry_ext_decimal" tracking = "#466" +[bindings."bignumber.js".upstream] +version = "11.1.5" +sha256 = "b98a3d093fdbd8854b32a631229dfb68366f29ad5496c680434a64f762aad321" +repo = "https://github.com/MikeMcl/bignumber.js" +ref = "275cd85181041d7b6bebf8ef9b70ee0b1bf056b0" +ported-at = "11.1.5" +date = "2026-07-30" [bindings.dayjs] crate = "perry-ext-dayjs" lib = "perry_ext_dayjs" tracking = "#466" +[bindings.dayjs.upstream] +version = "1.11.21" +sha256 = "290ec8bf81878ae4927581d9be74125f07722ea8bbea2ff77a2226dce12df876" +repo = "https://github.com/iamkun/dayjs" +ref = "a25f01e8154290b203ab480ffeedc60c80bd0710" +ported-at = "1.11.21" +date = "2026-07-30" [bindings."date-fns"] crate = "perry-ext-dayjs" lib = "perry_ext_dayjs" tracking = "#466" +[bindings.date-fns.upstream] +version = "4.4.0" +sha256 = "eb106d1e9276213d6144b221c103e4abb7d92186734f7505f5a3860427b41a06" +repo = "https://github.com/date-fns/date-fns" +ported-at = "4.4.0" +date = "2026-07-30" [bindings.moment] crate = "perry-ext-moment" lib = "perry_ext_moment" tracking = "#466" +[bindings.moment.upstream] +version = "2.30.1" +sha256 = "52219a9fee5e1faade4c72536c173c54cedd5e2619272dd0c251a30aeafcde8c" +repo = "https://github.com/moment/moment" +ported-at = "2.30.1" +date = "2026-07-30" [bindings.cheerio] crate = "perry-ext-cheerio" lib = "perry_ext_cheerio" tracking = "#466" +[bindings.cheerio.upstream] +version = "1.2.0" +sha256 = "56024eb0f315c373a4be57b26f3d956e9618cd2778ad283960c09cc1d4e820ab" +repo = "https://github.com/cheeriojs/cheerio" +ref = "e3c7aaf9ed64fe3cb9a181e58a41c0fdd6dbfbfc" +ported-at = "1.2.0" +date = "2026-07-30" [bindings.sharp] crate = "perry-ext-sharp" lib = "perry_ext_sharp" tracking = "#466" +[bindings.sharp.upstream] +version = "0.35.3" +sha256 = "53637f5503f81d10b02097eca6f94c44e69d92ba7ef759f268a0c4ea1d06ae54" +repo = "https://github.com/lovell/sharp" +ref = "1018449164723ba0203c1beffaba0e21f7829c18" +ported-at = "0.35.3" +date = "2026-07-30" [bindings."rate-limiter-flexible"] crate = "perry-ext-ratelimit" lib = "perry_ext_ratelimit" tracking = "#466" +[bindings.rate-limiter-flexible.upstream] +version = "11.2.0" +sha256 = "db92ca4b73bea8ccac0ed67b37b6970170dea15a063f539a5b354bb2a36661d0" +repo = "https://github.com/animir/node-rate-limiter-flexible" +ref = "2c12f60043c4bc2d0a7b7d05392824027d810b75" +ported-at = "11.2.0" +date = "2026-07-30" [bindings.commander] crate = "perry-ext-commander" lib = "perry_ext_commander" tracking = "#466" +[bindings.commander.upstream] +version = "15.0.0" +sha256 = "632c1e039b31e98fa79c4fae5b10a5ffbbf9df0f21c9ffb3d74e95734b30696f" +repo = "https://github.com/tj/commander.js" +ref = "ba6d13ddb4243e5913367734f8c159089ffe7834" +ported-at = "15.0.0" +date = "2026-07-30" [bindings.ethers] crate = "perry-ext-ethers" lib = "perry_ext_ethers" tracking = "#466" +[bindings.ethers.upstream] +version = "6.17.0" +sha256 = "17355f81284ba8431953c77af803b122baf4670cdbf84907adf7cafcd745052b" +repo = "https://github.com/ethers-io/ethers.js" +ref = "b48bfe34b11e059c4418718f9cad4e6cbe4c0680" +ported-at = "6.17.0" +date = "2026-07-30" [bindings.cron] crate = "perry-ext-cron" lib = "perry_ext_cron" tracking = "#466" +[bindings.cron.upstream] +version = "4.4.0" +sha256 = "868d5acfd1ff068de40d25a8d78cc1678305c7bc9192edd7e2b0d083346bcbdf" +repo = "https://github.com/kelektiv/node-cron" +ref = "d2f18cf7c56721913607c8159abe9f18c72decba" +ported-at = "4.4.0" +date = "2026-07-30" [bindings."node-cron"] crate = "perry-ext-cron" lib = "perry_ext_cron" tracking = "#466" +[bindings.node-cron.upstream] +version = "4.6.0" +sha256 = "78766bf4f4de32b009259e4e780a9ac8cd7155da970052427e68518cbeecfd32" +repo = "https://github.com/node-cron/node-cron" +ref = "0be2ca03ffebc79fa823081b8a44bff6f731a583" +ported-at = "4.6.0" +date = "2026-07-30" [bindings.ioredis] crate = "perry-ext-ioredis" lib = "perry_ext_ioredis" tracking = "#466" +[bindings.ioredis.upstream] +version = "5.11.1" +sha256 = "56b4e71ea1c54d22161a83c5d941f22c9ea53f23ecfbf332e33b7a1157b0f22b" +repo = "https://github.com/luin/ioredis" +ref = "fb224a7609b6d25959e06e31fdab2460d1f75691" +ported-at = "5.11.1" +date = "2026-07-30" + +# iovalkey is the Valkey project's fork of ioredis (valkey-io/iovalkey) — same +# RESP client API, separately published and versioned. It shares the ioredis +# wrapper crate (same `js_ioredis_*` surface) but is a distinct npm package, so +# it carries its own provenance pin, exactly like the `redis` row below. +[bindings.iovalkey] +crate = "perry-ext-ioredis" +lib = "perry_ext_ioredis" +tracking = "#466" + +[bindings.iovalkey.upstream] +version = "0.4.0" +sha256 = "466e118e5a069cec6625fb66cb4a2af7e7019581d038f8d99de6b3d4c2a487f2" +repo = "https://github.com/valkey-io/iovalkey" +ref = "96aa5a84b8e72da784445066c2765a555976d3eb" +ported-at = "0.4.0" +date = "2026-07-30" [bindings.redis] crate = "perry-ext-ioredis" lib = "perry_ext_ioredis" tracking = "#466" +[bindings.redis.upstream] +version = "6.1.0" +sha256 = "44bf885feeaf6ac356acfb315227999dfaa58aa7c5da226493c25bf7926fd2e5" +repo = "https://github.com/redis/node-redis" +ref = "d400bc9c1e7b17013c53015f098274a25aa70640" +ported-at = "6.1.0" +date = "2026-07-30" [bindings.pg] crate = "perry-ext-pg" lib = "perry_ext_pg" tracking = "#466" +[bindings.pg.upstream] +version = "8.22.0" +sha256 = "2f8b273b9b93b8712251cbe2b0f05378f4da2187eab5348c3c6c34d02622a55e" +repo = "https://github.com/brianc/node-postgres" +ref = "b617619f9fb6fbd231731823e2732a2927ded4be" +ported-at = "8.22.0" +date = "2026-07-30" [bindings.mysql2] crate = "perry-ext-mysql2" lib = "perry_ext_mysql2" tracking = "#466" +[bindings.mysql2.upstream] +version = "3.23.2" +sha256 = "5ebc7c22dd90fe3c7c69b9b1343208d71cc8e9395b433d16eb8be63306218a78" +repo = "https://github.com/sidorares/node-mysql2" +ref = "a98730277381005777f70aabfcefa017c1b970a3" +ported-at = "3.23.2" +date = "2026-07-30" [bindings."mysql2/promise"] crate = "perry-ext-mysql2" lib = "perry_ext_mysql2" tracking = "#466" +# Subpath of the mysql2 package — not separately published, so it carries no +# pin of its own; its provenance is mysql2's. +alias-of = "mysql2" [bindings.nodemailer] crate = "perry-ext-nodemailer" lib = "perry_ext_nodemailer" tracking = "#466" +[bindings.nodemailer.upstream] +version = "9.0.3" +sha256 = "f3da7499531fec3e8d7c59c9b1f240832c501b7a1e2ac01935c64baa66df52b4" +repo = "https://github.com/nodemailer/nodemailer" +ref = "1f61eb49b4316589045576752c721bd89918ef21" +ported-at = "9.0.3" +date = "2026-07-30" [bindings.mongodb] crate = "perry-ext-mongodb" lib = "perry_ext_mongodb" tracking = "#466" +[bindings.mongodb.upstream] +version = "7.5.0" +sha256 = "4d93c312a54bcfcc8f858f3f92de329b807fca53b4f1a456cea2b95e1cb1cdaa" +repo = "https://github.com/mongodb/node-mongodb-native" +ref = "c4368315b2ab08789f8ae505fce6721549f3bf9d" +ported-at = "7.5.0" +date = "2026-07-30" [bindings."node-fetch"] crate = "perry-ext-fetch" lib = "perry_ext_fetch" tracking = "#466" +[bindings.node-fetch.upstream] +version = "3.3.2" +sha256 = "615af90e363f8f276b4b54f8e6c163cf3686dce1d8867dd7e52cbed4d38d2dab" +repo = "https://github.com/node-fetch/node-fetch" +ref = "8b3320d2a7c07bce4afc6b2bf6c3bbddda85b01f" +ported-at = "3.3.2" +date = "2026-07-30" [bindings.fetch] crate = "perry-ext-fetch" lib = "perry_ext_fetch" tracking = "#466" +# Bare `fetch` is an alias for the node-fetch surface (same wrapper crate); it +# is not itself an npm package, so it inherits node-fetch's provenance. +alias-of = "node-fetch" [bindings.ws] crate = "perry-ext-ws" lib = "perry_ext_ws" tracking = "#466" +[bindings.ws.upstream] +version = "8.21.1" +sha256 = "bb0f7e58ba1f64746672734d36175fe185f226491e336abc0743e2a8f4472ec1" +repo = "https://github.com/websockets/ws" +ref = "ae1de54330cef77e487548890fabfeb9aae1d83d" +ported-at = "8.21.1" +date = "2026-07-30" [bindings.net] crate = "perry-ext-net" lib = "perry_ext_net" tracking = "#466" +node-builtin = true [bindings.http] crate = "perry-ext-http" lib = "perry_ext_http" tracking = "#466" +node-builtin = true [bindings.https] crate = "perry-ext-http" lib = "perry_ext_http" tracking = "#466" +node-builtin = true # Server-side `node:http2` lives in perry-ext-http's internal server # module, so the staticlib binding stays uniform across http/https/http2. @@ -249,11 +498,13 @@ tracking = "#466" crate = "perry-ext-http" lib = "perry_ext_http" tracking = "#577" +node-builtin = true [bindings.streams] crate = "perry-ext-streams" lib = "perry_ext_streams" tracking = "#466" +node-builtin = true [bindings.fastify] crate = "perry-ext-fastify" @@ -275,6 +526,14 @@ tracking = "#466" # function FFI: createPdf / pdfAddText / pdfAddLine / pdfNewPage / # pdfSave. Page units = PDF points (1/72"), origin = bottom-left # per the PDF standard. + +[bindings.fastify.upstream] +version = "5.10.0" +sha256 = "765b9bbdc03029d65b2fc544306ce950029c3470806d2ddee47a454dec0a67dc" +repo = "https://github.com/fastify/fastify" +ref = "c47975e8a4c0cb1cb066749b948c0e6e22c97663" +ported-at = "5.10.0" +date = "2026-07-30" [bindings."@perryts/pdf"] crate = "perry-ext-pdf" lib = "perry_ext_pdf" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 8489bd9a3a..d6ca8147b5 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -25,6 +25,7 @@ - [Authoring a Native Binding](native-libraries/authoring-guide.md) - [`perry-ffi` ABI Reference](native-libraries/abi.md) - [Manifest Schema (spec v1)](native-libraries/manifest-v1.md) + - [Upstream Provenance Pins](native-libraries/upstream-pins.md) # Multi-Threading diff --git a/docs/src/native-libraries/upstream-pins.md b/docs/src/native-libraries/upstream-pins.md new file mode 100644 index 0000000000..789ebb62e8 --- /dev/null +++ b/docs/src/native-libraries/upstream-pins.md @@ -0,0 +1,81 @@ +# Well-known binding upstream pins + +Every third-party npm package that ships a bundled native wrapper in +`crates/perry/well_known_bindings.toml` carries a **provenance pin**: the exact +upstream release the wrapper ports, its tarball content hash, and the date the +wrapper was last reviewed against it. This is the same discipline the +socket-registry fleet applies to its vendored upstream references, adapted for +npm dists. + +```toml +[bindings.ioredis] +crate = "perry-ext-ioredis" +lib = "perry_ext_ioredis" +tracking = "#466" + +[bindings.ioredis.upstream] +version = "5.11.1" # pinned npm release (immutable dist) +sha256 = "56b4e71e…" # sha256 of the registry tarball at pin time +repo = "https://github.com/luin/ioredis" +ref = "fb224a76…" # publisher's gitHead for the release, when known +ported-at = "5.11.1" # release the wrapper was last REVIEWED against +date = "2026-07-30" +``` + +## The lock-step rule + +**`ported-at` must equal `version`.** Re-pinning a binding to a newer upstream +release without re-reviewing the wrapper against the upstream diff reds the +`binding_pins.mjs --check` gate — and the perry binary itself refuses to load a +skewed table. An upstream release can never go silently stale, and a pin bump +can never outrun the review it demands: bumping `version` forces you to advance +`ported-at`, which forces the review. + +## Exempt entries + +Three kinds of row carry no pin: + +- **Node builtins** (`node-builtin = true`): `zlib`, `events`, `net`, `http`, + `https`, `http2`, `streams`. Their upstream is Node core, not an npm dist. +- **Aliases** (`alias-of = ""`): a package subpath (`mysql2/promise`) + or a bare-name alias (`fetch` → `node-fetch`) that shares its target's + provenance. +- **Perry-owned packages** (`@perryts/*`, `perry/*`). + +Note that a distinct npm package served by a shared wrapper crate is **not** an +alias — `redis` and `iovalkey` both use `perry-ext-ioredis` but are separately +published and versioned, so each carries its own pin. + +## Tooling — `scripts/binding_pins.mjs` + +```sh +# Provision or bump one pin to a specific version (default: latest stable) +node scripts/binding_pins.mjs --set ioredis 5.11.1 + +# Provision every currently-unpinned binding at its latest stable +node scripts/binding_pins.mjs --backfill + +# Offline gate (CI): pins present, lock-stepped, crates exist. Exit 1 on any +# violation. No network. +node scripts/binding_pins.mjs --check + +# Advisory: additionally flag pins whose upstream has a newer stable release +# that has soaked >= N days (default 7). Network. Run in the weekly update. +node scripts/binding_pins.mjs --check --refresh --soak-days 7 + +# Materialize the upstream repo at the pinned ref into gitignored upstream/ +# for port review (diff the old pin against a candidate new tag) +node scripts/binding_pins.mjs --materialize ioredis +``` + +Never hand-edit `version` / `sha256` / `ref` — the tarball hash can't be +recomputed at edit time. Use `--set`, which fetches the registry tarball, +hashes it, records the publisher's `gitHead`, and stamps `ported-at`/`date`. + +## Cadence + +The `--check` gate runs on every PR (offline). The weekly dependency-update +job runs `--check --refresh`, so a newly-soaked upstream release surfaces as an +actionable advisory — re-pin with `--set`, review the wrapper against the diff +(`--materialize` helps), and land the bump with `ported-at` advanced. This +mirrors the fleet's `vendor-actions.mts --check` weekly cadence. diff --git a/scripts/binding_pins.mjs b/scripts/binding_pins.mjs new file mode 100644 index 0000000000..c7fc757c96 --- /dev/null +++ b/scripts/binding_pins.mjs @@ -0,0 +1,386 @@ +#!/usr/bin/env node +// Upstream provenance pins for well_known_bindings.toml — provisioning, +// lock-step gate, and porting helpers. +// +// Ported from the socket-registry fleet's upstream-reference technique +// (gitmodules-hash.mts provisions pins; vendor-actions.mts --check reds +// when a pin falls behind its latest soaked upstream release). The +// upstream of a perry binding is an npm dist rather than a vendored git +// tree, so the pin lives as toml fields instead of a .gitmodules block, +// but the records and rules are the same: +// +// [bindings..upstream] +// version = "1.2.3" # pinned npm release (immutable dist) +// sha256 = "<64hex>" # sha256 of the registry tarball at pin time +// repo = "" # upstream source repo, when declared +// ref = "<40hex>" # gitHead recorded by the publisher, when known +// ported-at = "1.2.3" # release the wrapper was last REVIEWED against +// date = "YYYY-MM-DD" +// +// THE LOCK-STEP RULE: ported-at must equal version. Re-pinning an +// upstream without re-reviewing the wrapper against the upstream diff +// reds the --check gate (and the parser inside perry itself) until +// ported-at advances with the review. An upstream release can never go +// silently stale; a pin bump can never outrun its port. +// +// Modes: +// --set [version] provision/update one pin (default: latest) +// --backfill provision every unpinned binding at latest +// --check offline gate: pins present, lock-stepped, +// crates exist. Exit 1 on violation. CI-safe. +// --check --refresh [--soak-days N] +// network advisory: exit 1 when a pinned +// upstream has a newer stable release that +// has soaked >= N days (default 7) +// --materialize shallow-clone the upstream repo at the +// pinned ref into gitignored upstream/ +// for port review (diff old pin vs new tag) +// +// Never hand-edit `version`/`sha256`/`ref` — the tarball hash cannot be +// recomputed at edit time. Use --set. + +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const TOML_PATH = path.join(ROOT, 'crates', 'perry', 'well_known_bindings.toml'); +const REGISTRY = 'https://registry.npmjs.org'; +const DEFAULT_SOAK_DAYS = 7; + +// Perry's own packages have no third-party upstream to pin. +const SELF_OWNED = (name) => name.startsWith('@perryts/') || name.startsWith('perry/'); + +// A binding is exempt from carrying its own npm provenance pin when it is +// perry-owned, a Node builtin (upstream is Node core, not an npm dist), or an +// alias/subpath of another binding (it shares that binding's pin). +const isExempt = (b) => + SELF_OWNED(b.name) || b.fields['node-builtin'] === 'true' || Boolean(b.fields['alias-of']); + +// --------------------------------------------------------------------------- +// Minimal structural toml handling. The bindings file is machine-managed and +// regular ([bindings.] blocks with flat string fields plus an optional +// [bindings..upstream] sub-block), so we parse/rewrite it structurally +// instead of pulling a toml dependency into the repo's script surface. +// --------------------------------------------------------------------------- + +function parseBindings(raw) { + const bindings = new Map(); + let current = null; + let section = null; // 'binding' | 'upstream' + for (const line of raw.split('\n')) { + const header = line.match(/^\[bindings\.(?:"([^"]+)"|([^.\]"]+))(\.upstream)?\]\s*$/); + if (header) { + const name = header[1] ?? header[2]; + if (header[3]) { + section = 'upstream'; + current = bindings.get(name); + if (current) current.upstream = {}; + } else { + section = 'binding'; + current = { name, fields: {}, upstream: null }; + bindings.set(name, current); + } + continue; + } + if (/^\[/.test(line)) { + current = null; + section = null; + continue; + } + // Quoted string values, plus bare booleans (`node-builtin = true`). + const kv = line.match(/^\s*([A-Za-z0-9_-]+)\s*=\s*(?:"([^"]*)"|(true|false))\s*$/); + if (kv && current) { + const value = kv[2] ?? kv[3]; + if (section === 'upstream' && current.upstream) { + current.upstream[kv[1]] = value; + } else if (section === 'binding') { + current.fields[kv[1]] = value; + } + } + } + return bindings; +} + +function upstreamBlockText(name, pin) { + const key = /^[A-Za-z0-9_-]+$/.test(name) ? name : `"${name}"`; + const lines = [`[bindings.${key}.upstream]`]; + lines.push(`version = "${pin.version}"`); + lines.push(`sha256 = "${pin.sha256}"`); + if (pin.repo) lines.push(`repo = "${pin.repo}"`); + if (pin.ref) lines.push(`ref = "${pin.ref}"`); + lines.push(`ported-at = "${pin['ported-at']}"`); + lines.push(`date = "${pin.date}"`); + return lines.join('\n'); +} + +// Insert or replace the upstream sub-block directly after the binding's +// own block, preserving every other byte of the file (comments included). +function writePin(raw, name, pin) { + const esc = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // A binding key may appear bare (`date-fns`) or quoted (`"decimal.js"`, + // and quoted-anyway `"date-fns"`); match whichever the file actually uses + // rather than assuming from the name's characters. + const keyPattern = `(?:${esc}|"${esc}")`; + const headerRe = new RegExp(`^\\[bindings\\.${keyPattern}\\]\\s*$`, 'm'); + const headerMatch = raw.match(headerRe); + if (!headerMatch) { + throw new Error(`no [bindings.${name}] block found in ${TOML_PATH}`); + } + const upstreamRe = new RegExp( + `\\n?\\[bindings\\.${keyPattern}\\.upstream\\]\\s*\\n(?:[ \\t]*[A-Za-z0-9_-]+\\s*=\\s*"[^"]*"\\s*\\n?)*`, + ); + const existing = raw.match(upstreamRe); + if (existing) { + return raw.replace(upstreamRe, `\n${upstreamBlockText(name, pin)}\n`); + } + // Append after the binding block: from the header, find the next section + // header (or EOF) and insert before it. + const start = headerMatch.index; + const rest = raw.slice(start + headerMatch[0].length); + const next = rest.search(/\n\[/); + const insertAt = next === -1 ? raw.length : start + headerMatch[0].length + next; + return `${raw.slice(0, insertAt)}\n\n${upstreamBlockText(name, pin)}${raw.slice(insertAt)}`; +} + +// --------------------------------------------------------------------------- +// npm registry +// --------------------------------------------------------------------------- + +async function fetchJson(url) { + const res = await fetch(url, { headers: { accept: 'application/json' } }); + if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); + return res.json(); +} + +async function fetchPackument(name) { + return fetchJson(`${REGISTRY}/${name.replace('/', '%2f')}`); +} + +function latestStable(packument) { + const version = packument['dist-tags']?.latest; + if (!version) throw new Error(`${packument.name}: no dist-tags.latest`); + return version; +} + +async function sha256OfTarball(url) { + const res = await fetch(url); + if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); + const bytes = Buffer.from(await res.arrayBuffer()); + return createHash('sha256').update(bytes).digest('hex'); +} + +function normalizeRepo(repository) { + const url = typeof repository === 'string' ? repository : repository?.url; + if (!url) return undefined; + return url + .replace(/^git\+/, '') + .replace(/^git:\/\//, 'https://') + .replace(/^ssh:\/\/git@/, 'https://') + .replace(/\.git$/, ''); +} + +async function provisionPin(name, requestedVersion) { + const packument = await fetchPackument(name); + const version = requestedVersion ?? latestStable(packument); + const manifest = packument.versions?.[version]; + if (!manifest) throw new Error(`${name}@${version}: version not in registry`); + const sha256 = await sha256OfTarball(manifest.dist.tarball); + return { + version, + sha256, + repo: normalizeRepo(manifest.repository), + ref: manifest.gitHead, + 'ported-at': version, + date: new Date().toISOString().slice(0, 10), + }; +} + +// --------------------------------------------------------------------------- +// modes +// --------------------------------------------------------------------------- + +async function modeSet(names, requestedVersion) { + let raw = fs.readFileSync(TOML_PATH, 'utf8'); + const bindings = parseBindings(raw); + for (const name of names) { + if (!bindings.has(name)) { + throw new Error(`no [bindings.${name}] block — add the binding row first`); + } + const pin = await provisionPin(name, requestedVersion); + raw = writePin(raw, name, pin); + console.log(`pinned ${name}@${pin.version} sha256:${pin.sha256.slice(0, 12)}…`); + } + fs.writeFileSync(TOML_PATH, raw); +} + +async function modeBackfill() { + const raw = fs.readFileSync(TOML_PATH, 'utf8'); + const bindings = parseBindings(raw); + const unpinned = [...bindings.values()].filter((b) => !b.upstream && !isExempt(b)); + if (unpinned.length === 0) { + console.log('all bindings pinned — nothing to backfill'); + return; + } + for (const b of unpinned) { + await modeSet([b.name]); + } +} + +function checkOffline(bindings) { + const failures = []; + for (const b of bindings.values()) { + // Aliases inherit their target's provenance — verify the target exists + // and is itself pinned (or exempt), then skip the pin requirement. + const aliasTarget = b.fields['alias-of']; + if (aliasTarget) { + const target = bindings.get(aliasTarget); + if (!target) { + failures.push(`${b.name}: alias-of \`${aliasTarget}\`, which is not a known binding`); + } else if (!target.upstream && !isExempt(target)) { + failures.push(`${b.name}: alias-of \`${aliasTarget}\`, which has no pin`); + } + continue; + } + if (isExempt(b)) continue; + if (!b.upstream) { + failures.push(`${b.name}: missing [bindings.${b.name}.upstream] pin`); + continue; + } + for (const field of ['version', 'sha256', 'ported-at', 'date']) { + if (!b.upstream[field]) failures.push(`${b.name}: upstream pin missing \`${field}\``); + } + if ( + b.upstream['ported-at'] && + b.upstream.version && + b.upstream['ported-at'] !== b.upstream.version + ) { + failures.push( + `${b.name}: LOCK-STEP violation — ported-at (${b.upstream['ported-at']}) != ` + + `version (${b.upstream.version}). Review the wrapper against the upstream ` + + `diff, then advance ported-at with the review.`, + ); + } + if (b.upstream.sha256 && !/^[0-9a-f]{64}$/.test(b.upstream.sha256)) { + failures.push(`${b.name}: sha256 is not 64 lowercase hex chars`); + } + if (b.upstream.date && !/^\d{4}-\d{2}-\d{2}$/.test(b.upstream.date)) { + failures.push(`${b.name}: date is not YYYY-MM-DD`); + } + const crateDir = path.join(ROOT, 'crates', b.fields.crate ?? ''); + if (!b.fields.crate || !fs.existsSync(crateDir)) { + failures.push(`${b.name}: crate \`${b.fields.crate}\` not found in workspace`); + } + } + return failures; +} + +async function checkRefresh(bindings, soakDays) { + const advisories = []; + const now = Date.now(); + for (const b of bindings.values()) { + if (isExempt(b) || !b.upstream?.version) continue; + let packument; + try { + packument = await fetchPackument(b.name); + } catch (err) { + advisories.push(`${b.name}: registry lookup failed (${err.message}) — skipping`); + continue; + } + const latest = latestStable(packument); + if (latest === b.upstream.version) continue; + const publishedAt = packument.time?.[latest]; + const soakedDays = publishedAt + ? Math.floor((now - Date.parse(publishedAt)) / 86_400_000) + : Infinity; + if (soakedDays >= soakDays) { + advisories.push( + `${b.name}: pinned ${b.upstream.version}, latest stable ${latest} ` + + `(soaked ${soakedDays}d >= ${soakDays}d) — re-pin, re-review, advance ported-at: ` + + `node scripts/binding_pins.mjs --set ${b.name}`, + ); + } + } + return advisories; +} + +function modeMaterialize(name) { + const bindings = parseBindings(fs.readFileSync(TOML_PATH, 'utf8')); + const b = bindings.get(name); + if (!b) throw new Error(`no [bindings.${name}] block`); + if (!b.upstream?.repo) throw new Error(`${name}: no upstream repo recorded in its pin`); + const dest = path.join(ROOT, 'upstream', name.replace('/', '__')); + const ref = b.upstream.ref; + fs.mkdirSync(path.dirname(dest), { recursive: true }); + if (fs.existsSync(dest)) { + console.log(`upstream/${name} already materialized — fetching pin`); + } else { + execFileSync('git', ['clone', '--filter=blob:none', '--no-checkout', b.upstream.repo, dest], { + stdio: 'inherit', + }); + } + const checkout = ref || `v${b.upstream.version}`; + try { + execFileSync('git', ['-C', dest, 'fetch', '--depth', '1', 'origin', checkout], { + stdio: 'inherit', + }); + execFileSync('git', ['-C', dest, 'checkout', '--detach', 'FETCH_HEAD'], { stdio: 'inherit' }); + } catch { + // Publishers without gitHead and without v-prefixed tags: try the bare version tag. + execFileSync('git', ['-C', dest, 'fetch', '--depth', '1', 'origin', b.upstream.version], { + stdio: 'inherit', + }); + execFileSync('git', ['-C', dest, 'checkout', '--detach', 'FETCH_HEAD'], { stdio: 'inherit' }); + } + console.log(`materialized upstream/${name} at ${checkout} (gitignored, review-only)`); +} + +// --------------------------------------------------------------------------- + +const args = process.argv.slice(2); +const has = (flag) => args.includes(flag); +const valueOf = (flag) => { + const i = args.indexOf(flag); + return i !== -1 ? args[i + 1] : undefined; +}; + +try { + if (has('--set')) { + const name = valueOf('--set'); + if (!name) throw new Error('--set requires a binding name'); + const maybeVersion = args[args.indexOf('--set') + 2]; + const version = + maybeVersion && !maybeVersion.startsWith('--') ? maybeVersion : undefined; + await modeSet([name], version); + } else if (has('--backfill')) { + await modeBackfill(); + } else if (has('--materialize')) { + const name = valueOf('--materialize'); + if (!name) throw new Error('--materialize requires a binding name'); + modeMaterialize(name); + } else if (has('--check')) { + const bindings = parseBindings(fs.readFileSync(TOML_PATH, 'utf8')); + const failures = checkOffline(bindings); + for (const f of failures) console.error(`FAIL ${f}`); + let advisories = []; + if (has('--refresh')) { + const soakDays = Number(valueOf('--soak-days') ?? DEFAULT_SOAK_DAYS); + advisories = await checkRefresh(bindings, soakDays); + for (const a of advisories) console.error(`STALE ${a}`); + } + if (failures.length || advisories.length) process.exit(1); + console.log( + `binding pins OK — ${[...bindings.values()].filter((b) => b.upstream).length} pinned, lock-step holds`, + ); + } else { + console.error( + 'usage: binding_pins.mjs --set [version] | --backfill | --check [--refresh [--soak-days N]] | --materialize ', + ); + process.exit(2); + } +} catch (err) { + console.error(`error: ${err.message}`); + process.exit(1); +} From 30958f96f823b0c3837a607582b7518e0aeda583 Mon Sep 17 00:00:00 2001 From: jdalton Date: Wed, 29 Jul 2026 21:49:59 -0400 Subject: [PATCH 2/4] docs: key the changelog fragment to PR #7031 --- ...ing-upstream-lockstep.md => 7031-binding-upstream-lockstep.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{0000-binding-upstream-lockstep.md => 7031-binding-upstream-lockstep.md} (100%) diff --git a/changelog.d/0000-binding-upstream-lockstep.md b/changelog.d/7031-binding-upstream-lockstep.md similarity index 100% rename from changelog.d/0000-binding-upstream-lockstep.md rename to changelog.d/7031-binding-upstream-lockstep.md From 5462879aadb12556acb3f78afb4a0954f40277a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 30 Jul 2026 08:37:04 +0200 Subject: [PATCH 3/4] fix(bindings): address provenance gate review --- .github/workflows/security-audit.yml | 6 +++++- external-tools.json | 2 +- scripts/binding_pins.mjs | 27 +++++++++++++++++++++++++-- scripts/soak/external-tools.mts | 15 +++++++++------ scripts/soak/external-tools.test.mts | 4 +++- scripts/soak/soak.mts | 7 ++++--- scripts/soak/soak.test.mts | 2 ++ 7 files changed, 49 insertions(+), 14 deletions(-) diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 15b4eae15e..2987b0ff24 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -20,6 +20,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 with: @@ -131,7 +133,7 @@ jobs: for skill in .claude/skills/*/; do echo "::group::skillspector ${skill}" uv tool run --python 3.12 \ - --from git+https://github.com/NVIDIA/skillspector@2eb84478 \ + --from git+https://github.com/NVIDIA/skillspector@2eb844780ab163f01468ecf142c40a2ec0fcaec0 \ skillspector scan "${skill}" --no-llm echo "::endgroup::" done @@ -145,6 +147,8 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 with: diff --git a/external-tools.json b/external-tools.json index 440998fc34..7f27a9f5b3 100644 --- a/external-tools.json +++ b/external-tools.json @@ -174,7 +174,7 @@ "description": "NVIDIA's third-party-skill security scanner (LangGraph-based; YARA + AST + OSV.dev CVE lookups + optional LLM analysis). No PyPI release / no GH tags upstream — pinned to a git SHA on main + installed via a locked uv project (pyproject.toml + uv.lock, `uv sync --locked`; the fleet uv pin + exclude-newer make it reproducible). Sibling to AgentShield: AgentShield audits the operator's .claude/ config; SkillSpector audits untrusted upstream skills before install.", "release": "uv-project", "repository": "github:NVIDIA/skillspector", - "version": "2eb84478", + "version": "2eb844780ab163f01468ecf142c40a2ec0fcaec0", "versionDate": "2026-05-18" } } diff --git a/scripts/binding_pins.mjs b/scripts/binding_pins.mjs index c7fc757c96..543ae4988b 100644 --- a/scripts/binding_pins.mjs +++ b/scripts/binding_pins.mjs @@ -49,6 +49,9 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const TOML_PATH = path.join(ROOT, 'crates', 'perry', 'well_known_bindings.toml'); const REGISTRY = 'https://registry.npmjs.org'; const DEFAULT_SOAK_DAYS = 7; +const REGISTRY_TIMEOUT_MS = 30_000; +const TARBALL_TIMEOUT_MS = 60_000; +const GIT_TIMEOUT_MS = 60_000; // Perry's own packages have no third-party upstream to pin. const SELF_OWNED = (name) => name.startsWith('@perryts/') || name.startsWith('perry/'); @@ -150,7 +153,17 @@ function writePin(raw, name, pin) { // --------------------------------------------------------------------------- async function fetchJson(url) { - const res = await fetch(url, { headers: { accept: 'application/json' } }); + let res; + try { + res = await fetch(url, { + headers: { accept: 'application/json' }, + signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS), + }); + } catch (err) { + throw new Error(`${url}: request failed or timed out after ${REGISTRY_TIMEOUT_MS}ms`, { + cause: err, + }); + } if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); return res.json(); } @@ -166,7 +179,14 @@ function latestStable(packument) { } async function sha256OfTarball(url) { - const res = await fetch(url); + let res; + try { + res = await fetch(url, { signal: AbortSignal.timeout(TARBALL_TIMEOUT_MS) }); + } catch (err) { + throw new Error(`${url}: download failed or timed out after ${TARBALL_TIMEOUT_MS}ms`, { + cause: err, + }); + } if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); const bytes = Buffer.from(await res.arrayBuffer()); return createHash('sha256').update(bytes).digest('hex'); @@ -319,18 +339,21 @@ function modeMaterialize(name) { } else { execFileSync('git', ['clone', '--filter=blob:none', '--no-checkout', b.upstream.repo, dest], { stdio: 'inherit', + timeout: GIT_TIMEOUT_MS, }); } const checkout = ref || `v${b.upstream.version}`; try { execFileSync('git', ['-C', dest, 'fetch', '--depth', '1', 'origin', checkout], { stdio: 'inherit', + timeout: GIT_TIMEOUT_MS, }); execFileSync('git', ['-C', dest, 'checkout', '--detach', 'FETCH_HEAD'], { stdio: 'inherit' }); } catch { // Publishers without gitHead and without v-prefixed tags: try the bare version tag. execFileSync('git', ['-C', dest, 'fetch', '--depth', '1', 'origin', b.upstream.version], { stdio: 'inherit', + timeout: GIT_TIMEOUT_MS, }); execFileSync('git', ['-C', dest, 'checkout', '--detach', 'FETCH_HEAD'], { stdio: 'inherit' }); } diff --git a/scripts/soak/external-tools.mts b/scripts/soak/external-tools.mts index fa092b71a3..62182a7800 100644 --- a/scripts/soak/external-tools.mts +++ b/scripts/soak/external-tools.mts @@ -105,8 +105,10 @@ export function checkPins(tools: Record): string[] { ...(pin.integrity ? [pin.integrity] : []), ...Object.values(pin.platforms ?? {}).map(p => p.integrity), ] - if (pin.release === 'asset' && integrities.length === 0) { - out.push(`${name}: release asset without any integrity pin`) + const downloadable = + pin.release === 'asset' || Boolean(pin.purl) || pin.repository?.startsWith('npm:') + if (downloadable && integrities.length === 0) { + out.push(`${name}: downloadable pin without any integrity pin`) } for (const sri of integrities) { if (!/^sha512-[A-Za-z0-9+/]+={0,2}$/.test(sri)) { @@ -551,10 +553,11 @@ if [ -n "\${${sentinel}:-}" ] || [ -z "$REAL" ] || ! command -v sfw >/dev/null 2 echo "${cmd}: not found" >&2; exit 127 fi export ${sentinel}=1 -# Enterprise sfw defaults to BLOCK for non-registry hosts, which breaks -# ordinary dev flows the day a Socket key lands; free tier hardcodes -# ignore and disregards the var, so setting it is always safe. -export SFW_UNKNOWN_HOST_ACTION=ignore +# Keep enterprise sfw's block-by-default posture. Operators can explicitly +# allow unknown hosts for a single invocation when the command needs them. +if [ "\${SFW_ALLOW_UNKNOWN_HOSTS:-}" = "1" ]; then + export SFW_UNKNOWN_HOST_ACTION=ignore +fi exec sfw '${cmd}' "$@" ` // Remove the handle before writing: writeFileSync FOLLOWS a symlink, so diff --git a/scripts/soak/external-tools.test.mts b/scripts/soak/external-tools.test.mts index 78a8751a1c..5c2622bc32 100644 --- a/scripts/soak/external-tools.test.mts +++ b/scripts/soak/external-tools.test.mts @@ -28,10 +28,12 @@ test('the repo external-tools.json passes checkPins', () => { assert.deepEqual(checkPins(tools), []) }) -test('checkPins flags missing pins, bad SRIs, and asset entries with no integrity', () => { +test('checkPins flags missing pins, bad SRIs, and downloadable entries with no integrity', () => { assert.equal(checkPins({ a: {} }).length, 1) assert.equal(checkPins({ a: { version: '1.0.0', integrity: 'sha256-abc' } }).length, 1) assert.equal(checkPins({ a: { version: '1.0.0', release: 'asset' } }).length, 1) + assert.equal(checkPins({ a: { purl: 'pkg:npm/a@1.0.0' } }).length, 1) + assert.equal(checkPins({ a: { version: '1.0.0', repository: 'npm:a' } }).length, 1) }) test('checkPins validates soakBypass dates and arithmetic; expiry is a warning, not a failure', () => { diff --git a/scripts/soak/soak.mts b/scripts/soak/soak.mts index a6322bac89..85d13a698b 100644 --- a/scripts/soak/soak.mts +++ b/scripts/soak/soak.mts @@ -71,7 +71,7 @@ export function checkCargoConfig(body: string, file: string): Finding[] { } export function checkNpmrc(body: string, file: string): Finding[] { - const days = /^min-release-age=(\d+)\s*$/m.exec(body)?.[1] + const days = /^min-release-age[ \t]*=[ \t]*(\d+)[ \t]*$/m.exec(body)?.[1] if (Number(days) === SOAK_DAYS) { return [] } @@ -435,8 +435,9 @@ export function fixCargoConfig(body: string): string { } export function fixNpmrc(body: string): string { - if (/^min-release-age=\d+\s*$/m.test(body)) { - return body.replace(/^min-release-age=\d+\s*$/m, `min-release-age=${SOAK_DAYS}`) + const key = /^min-release-age[ \t]*=[ \t]*\d+[ \t]*$/m + if (key.test(body)) { + return body.replace(key, `min-release-age=${SOAK_DAYS}`) } return `${body.trimEnd()}\nmin-release-age=${SOAK_DAYS}\n` } diff --git a/scripts/soak/soak.test.mts b/scripts/soak/soak.test.mts index 83b5b4bc05..53dd31cde8 100644 --- a/scripts/soak/soak.test.mts +++ b/scripts/soak/soak.test.mts @@ -47,10 +47,12 @@ test('cargo config: wrong window and missing unstable gate are findings', () => test('npmrc: window must match SOAK_DAYS and fix writes it', () => { assert.equal(checkNpmrc('min-release-age=7\n', 'n').length, 0) + assert.equal(checkNpmrc('min-release-age = 7\n', 'n').length, 0) assert.equal(checkNpmrc('min-release-age=3\n', 'n').length, 1) assert.equal(checkNpmrc('# nothing\n', 'n').length, 1) assert.match(fixNpmrc('# nothing\n'), /min-release-age=7/) assert.match(fixNpmrc('min-release-age=3\n'), /min-release-age=7/) + assert.equal(fixNpmrc('min-release-age = 3\n'), 'min-release-age=7\n') }) test('workspace yaml: clean fixture passes', () => { From d7ffb90315fd660d6a3b47368d77a6e45fa83f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 30 Jul 2026 08:47:09 +0200 Subject: [PATCH 4/4] fix(security): harden soaked tool workflows --- .github/workflows/security-audit.yml | 7 +++++- .github/workflows/soak-autofix.yml | 12 +++++++++- changelog.d/6912-security-tooling-soak.md | 2 +- external-tools.json | 4 ++-- scripts/soak/constants.mts | 3 ++- scripts/soak/external-tools.mts | 19 +++++++++++++++ scripts/soak/external-tools.test.mts | 15 ++++++++++++ scripts/soak/soak.mts | 29 ++++++++++++++++------- scripts/soak/soak.test.mts | 14 +++++++++++ 9 files changed, 90 insertions(+), 15 deletions(-) diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 2987b0ff24..82b77a11c0 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -130,10 +130,15 @@ jobs: - name: Scan each skill run: | set -euo pipefail + skillspector_ref="$(jq -er '.tools.skillspector.version' external-tools.json)" + soak_days="$(sed -nE 's/^export const SOAK_DAYS = ([0-9]+)$/\1/p' scripts/soak/constants.mts)" + test -n "$soak_days" + exclude_newer="$(date -u -d "${soak_days} days ago" '+%Y-%m-%dT%H:%M:%SZ')" for skill in .claude/skills/*/; do echo "::group::skillspector ${skill}" uv tool run --python 3.12 \ - --from git+https://github.com/NVIDIA/skillspector@2eb844780ab163f01468ecf142c40a2ec0fcaec0 \ + --exclude-newer "$exclude_newer" \ + --from "git+https://github.com/NVIDIA/skillspector@${skillspector_ref}" \ skillspector scan "${skill}" --no-llm echo "::endgroup::" done diff --git a/.github/workflows/soak-autofix.yml b/.github/workflows/soak-autofix.yml index 2701ed3cf0..9aa044154e 100644 --- a/.github/workflows/soak-autofix.yml +++ b/.github/workflows/soak-autofix.yml @@ -70,6 +70,11 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" BRANCH="bot/soak-autofix" + expected_oid="" + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + git fetch origin "refs/heads/$BRANCH:refs/remotes/origin/$BRANCH" + expected_oid="$(git rev-parse "refs/remotes/origin/$BRANCH")" + fi git checkout -B "$BRANCH" git add -A git commit -m "chore(soak): prune expired soak annotations (automated) @@ -78,7 +83,12 @@ jobs: external-tools.mts --fix. Windows that cleared have soaked; their bypass annotations are dead weight the gates would otherwise fail on." - git push -f origin "$BRANCH" + if [ -n "$expected_oid" ]; then + git push --force-with-lease="refs/heads/$BRANCH:$expected_oid" \ + origin "HEAD:refs/heads/$BRANCH" + else + git push origin "HEAD:refs/heads/$BRANCH" + fi if [ -z "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number')" ]; then gh pr create --head "$BRANCH" \ --title "chore(soak): prune expired soak annotations (automated)" \ diff --git a/changelog.d/6912-security-tooling-soak.md b/changelog.d/6912-security-tooling-soak.md index 34075b6410..efe4877c7d 100644 --- a/changelog.d/6912-security-tooling-soak.md +++ b/changelog.d/6912-security-tooling-soak.md @@ -1 +1 @@ -**Supply-chain soak + pinned security tooling:** port the wheelhouse/nub security stack (nubjs/nub#442). A 7-day release-age soak (`SOAK_DAYS` in `scripts/soak/constants.mts`, `npm run soak` parity gate + `soak:fix` fixer) now governs every dependency surface — root `.npmrc` `min-release-age`, `tools/pnpm-workspace.yaml` `minimumReleaseAge` with dated auto-expiring exclusions, `tools/taze.config.mts` `maturityPeriod` (soaked `npm run deps:update` updater), `.cargo/config.toml` `min-publish-age` (nightly-only; inert on stable), and per-block dependabot `cooldown`. `external-tools.json` pins pnpm 11.15.1 / npm 12.0.1 / sfw 1.14.0 (dated soakBypass) / zizmor 1.28.0 / agentshield 1.4.0 / skillspector `@2eb84478` exact with sha512 SRI; `npm run tools:install` installs them into a local rack and writes Socket Firewall shims for npm/yarn/pnpm/pip/uv/cargo (recursion-sentinel, fail-open, clobber-guarded; the pnpm/npm shims wrap the rack-pinned binaries). CI grows a zizmor workflow running the rack-pinned binary (gate at high with a documented ratchet config in `.github/zizmor.yml`) plus always-run `soak-gate`, `agent-scan` (AgentShield over `.claude/`), and `skills-scan` (NVIDIA SkillSpector over `.claude/skills/`) jobs in security-audit.yml. New `soak` skill documents the workflow. +**Supply-chain soak + pinned security tooling:** port the wheelhouse/nub security stack (nubjs/nub#442). A 7-day release-age soak (`SOAK_DAYS` in `scripts/soak/constants.mts`, `npm run soak` parity gate + `soak:fix` fixer) now governs every dependency surface — root `.npmrc` `min-release-age`, `tools/pnpm-workspace.yaml` `minimumReleaseAge` with dated auto-expiring exclusions, `tools/taze.config.mts` `maturityPeriod` (soaked `npm run deps:update` updater), `.cargo/config.toml` `min-publish-age` (nightly-only; inert on stable), and per-block dependabot `cooldown`. `external-tools.json` pins pnpm 11.15.1 / npm 12.0.1 / sfw 1.14.0 (dated soakBypass) / zizmor 1.28.0 / agentshield 1.4.0 with SHA-512 SRI, and SkillSpector to a full Git SHA with a seven-day uv dependency cutoff; `npm run tools:install` installs them into a local rack and writes Socket Firewall shims for npm/yarn/pnpm/pip/uv/cargo (recursion-sentinel, fail-open, clobber-guarded; the pnpm/npm shims wrap the rack-pinned binaries). CI grows a zizmor workflow running the rack-pinned binary (gate at high with a documented ratchet config in `.github/zizmor.yml`) plus always-run `soak-gate`, `agent-scan` (AgentShield over `.claude/`), and `skills-scan` (NVIDIA SkillSpector over `.claude/skills/`) jobs in security-audit.yml. New `soak` skill documents the workflow. diff --git a/external-tools.json b/external-tools.json index 7f27a9f5b3..77799d445f 100644 --- a/external-tools.json +++ b/external-tools.json @@ -47,7 +47,7 @@ }, "npm": { "notes": [ - "npm 12 (min-release-age support). ONE platform-agnostic registry tarball, single integrity", + "npm 11.10+ supports min-release-age; npm 11.17+ adds min-release-age-exclude. ONE platform-agnostic registry tarball, single integrity", "Installed from the pinned tarball only — never npm install -g npm, no self-update path" ], "description": "npm — pinned, SRI-verified registry tarball; installed without self-update", @@ -171,7 +171,7 @@ "integrity": "sha512-R98OO1Ujyk2lezDLb+iQmMhF6FwTJCHajy3G4FCB6x7wkSTqR9f8+eAelC5KDzYDsGSbc0sOZvjXOOPRBtMpDg==" }, "skillspector": { - "description": "NVIDIA's third-party-skill security scanner (LangGraph-based; YARA + AST + OSV.dev CVE lookups + optional LLM analysis). No PyPI release / no GH tags upstream — pinned to a git SHA on main + installed via a locked uv project (pyproject.toml + uv.lock, `uv sync --locked`; the fleet uv pin + exclude-newer make it reproducible). Sibling to AgentShield: AgentShield audits the operator's .claude/ config; SkillSpector audits untrusted upstream skills before install.", + "description": "NVIDIA's third-party-skill security scanner (LangGraph-based; YARA + AST + OSV.dev CVE lookups + optional LLM analysis). No PyPI release / no GH tags upstream — pinned to a full git SHA on main; CI reads this ref directly and applies the fleet SOAK_DAYS cutoff with uv --exclude-newer. Sibling to AgentShield: AgentShield audits the operator's .claude/ config; SkillSpector audits untrusted upstream skills before install.", "release": "uv-project", "repository": "github:NVIDIA/skillspector", "version": "2eb844780ab163f01468ecf142c40a2ec0fcaec0", diff --git a/scripts/soak/constants.mts b/scripts/soak/constants.mts index 116e3a3d5f..66d2f5a4f2 100644 --- a/scripts/soak/constants.mts +++ b/scripts/soak/constants.mts @@ -8,7 +8,8 @@ * - `.cargo/config.toml` -> `global-min-publish-age = " days"` (cargo -Zmin-publish-age) * - `rust-toolchain.toml` -> dated nightly adopted only once >= SOAK_DAYS old * - `pnpm-workspace.yaml` -> `minimumReleaseAge: ` (aube reads minutes) - * - `.npmrc` -> `min-release-age=` (npm >= 11.17, days) + * - `.npmrc` -> `min-release-age=` (npm >= 11.10, days; + * annotated excludes require npm >= 11.17) * - `taze.config.mts` -> `maturityPeriod: SOAK_DAYS` (imports this) * * The data files can't import this module, so `scripts/soak/soak.mts` diff --git a/scripts/soak/external-tools.mts b/scripts/soak/external-tools.mts index 62182a7800..c79aec6220 100644 --- a/scripts/soak/external-tools.mts +++ b/scripts/soak/external-tools.mts @@ -295,6 +295,17 @@ export function extractArchive(name: string, destDir: string, asset: string, buf export function linkHandle(target: string, name: string): void { mkdirSync(BIN_DIR, { recursive: true }) const handle = path.join(BIN_DIR, name) + if (process.platform !== 'win32' && isManagedSfwShim(handle)) { + const body = readFileSync(handle, 'utf8') + const updated = body.replace(/^REAL='[^']*'$/m, `REAL='${target}'`) + if (updated !== body) { + writeFileSync(handle, updated) + } + console.warn( + `[external-tools] preserving managed sfw shim ${handle} while updating its rack target`, + ) + return + } // force also removes a DANGLING handle (existsSync would report false // for one and a bare symlink would then throw EEXIST). rmSync(handle, { force: true }) @@ -328,6 +339,14 @@ export function linkHandle(target: string, name: string): void { symlinkSync(target, handle) } +export function isManagedSfwShim(handle: string): boolean { + try { + return readFileSync(handle, 'utf8').includes('# sfw shim for ') + } catch { + return false + } +} + async function installAssetTool(name: string, pin: ToolPin): Promise { const plat = pin.platforms?.[platformKey()] if (!plat) { diff --git a/scripts/soak/external-tools.test.mts b/scripts/soak/external-tools.test.mts index 5c2622bc32..09b71495a5 100644 --- a/scripts/soak/external-tools.test.mts +++ b/scripts/soak/external-tools.test.mts @@ -13,6 +13,7 @@ import { checkPins, download, extractArchive, + isManagedSfwShim, installTool, main, pruneExpiredSoakBypasses, @@ -36,6 +37,20 @@ test('checkPins flags missing pins, bad SRIs, and downloadable entries with no i assert.equal(checkPins({ a: { version: '1.0.0', repository: 'npm:a' } }).length, 1) }) +test('managed sfw shim detection distinguishes wrappers from ordinary handles', () => { + const dir = mkdtempSync(path.join(os.tmpdir(), 'perry-sfw-shim-')) + try { + const handle = path.join(dir, 'npm') + writeFileSync(handle, '#!/bin/sh\n# sfw shim for npm — managed\n') + assert.equal(isManagedSfwShim(handle), true) + writeFileSync(handle, '#!/bin/sh\nexec npm \"$@\"\n') + assert.equal(isManagedSfwShim(handle), false) + assert.equal(isManagedSfwShim(path.join(dir, 'missing')), false) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + test('checkPins validates soakBypass dates and arithmetic; expiry is a warning, not a failure', () => { const pub = addDaysIso(todayIso(), -1) const good = { diff --git a/scripts/soak/soak.mts b/scripts/soak/soak.mts index 85d13a698b..ba693c6778 100644 --- a/scripts/soak/soak.mts +++ b/scripts/soak/soak.mts @@ -47,7 +47,8 @@ export interface Finding { export function checkCargoConfig(body: string, file: string): Finding[] { const out: Finding[] = [] - const age = /^global-min-publish-age\s*=\s*"([^"]*)"/m.exec(body)?.[1] + const registry = /^\[registry\][^[]*/ms.exec(body)?.[0] ?? '' + const age = /^global-min-publish-age\s*=\s*"([^"]*)"/m.exec(registry)?.[1] const wanted = `${SOAK_DAYS} days` if (age !== wanted) { out.push({ @@ -428,10 +429,20 @@ export function fixDependabotCooldown(body: string): string { } export function fixCargoConfig(body: string): string { - return body.replace( - /^(global-min-publish-age\s*=\s*)"[^"]*"/m, - `$1"${SOAK_DAYS} days"`, - ) + const registrySection = /^\[registry\][^[]*/ms + const key = /^(global-min-publish-age\s*=\s*)"[^"]*"/m + if (registrySection.test(body)) { + return body.replace(registrySection, section => { + if (key.test(section)) { + return section.replace(key, `$1"${SOAK_DAYS} days"`) + } + return section.replace( + /^\[registry\][ \t]*$/m, + `[registry]\nglobal-min-publish-age = "${SOAK_DAYS} days"`, + ) + }) + } + return `${body.trimEnd()}\n\n[registry]\nglobal-min-publish-age = "${SOAK_DAYS} days"\n` } export function fixNpmrc(body: string): string { @@ -443,10 +454,10 @@ export function fixNpmrc(body: string): string { } export function fixWorkspaceYaml(body: string): string { - let out = body.replace( - /^(minimumReleaseAge:\s*)\d+\s*$/m, - `$1${SOAK_MINUTES}`, - ) + const key = /^(minimumReleaseAge:\s*)\d+[ \t]*$/m + let out = key.test(body) + ? body.replace(key, `$1${SOAK_MINUTES}`) + : `minimumReleaseAge: ${SOAK_MINUTES}\n${body}` // Prune expired pins together with their annotation line. const today = todayIso() const lines = out.split('\n') diff --git a/scripts/soak/soak.test.mts b/scripts/soak/soak.test.mts index 53dd31cde8..a99c09409b 100644 --- a/scripts/soak/soak.test.mts +++ b/scripts/soak/soak.test.mts @@ -43,6 +43,14 @@ test('cargo config: wrong window and missing unstable gate are findings', () => assert.equal(checkCargoConfig(good, 'c').length, 0) assert.equal(checkCargoConfig(good.replace('7 days', '3 days'), 'c').length, 1) assert.equal(checkCargoConfig('[registry]\nglobal-min-publish-age = "7 days"\n', 'c').length, 1) + assert.equal( + checkCargoConfig( + 'global-min-publish-age = "7 days"\n\n[unstable]\nmin-publish-age = true\n', + 'c', + ).length, + 1, + ) + assert.match(fixCargoConfig('[unstable]\nmin-publish-age = true\n'), /\[registry\]\n/) }) test('npmrc: window must match SOAK_DAYS and fix writes it', () => { @@ -64,6 +72,12 @@ test('workspace yaml: wrong minutes value is a finding', () => { assert.equal(checkWorkspaceYaml(bad, 'y').filter(f => f.what.includes('minimumReleaseAge')).length, 1) }) +test('workspace yaml fixer inserts a missing minimumReleaseAge key', () => { + const fixed = fixWorkspaceYaml('catalog:\n taze: 19.14.1\n') + assert.match(fixed, /^minimumReleaseAge: 10080$/m) + assert.equal(checkWorkspaceYaml(fixed, 'y').length, 0) +}) + test('excludes: flow-style list is rejected outright', () => { const flow = "minimumReleaseAge: 10080\nminimumReleaseAgeExclude: ['left-pad@1.3.0']\n" const findings = checkExcludeAnnotations(flow, 'y')