From 663a3ea3672de4a3b3fe6e58929fa6a2e9cef534 Mon Sep 17 00:00:00 2001 From: Jeff Larson Date: Mon, 6 Jul 2026 21:27:51 -0700 Subject: [PATCH] fix(signature): keep the TUF/tough atomic temp writes off /tmp (JEF-377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sigstore-rs 0.14 builds its tough TUF client without a datastore path, so tough falls back to TempDir::new() — a /tmp/.tmp/ dir — and writes latest_known_time.json plus refreshed root/timestamp/snapshot metadata there on every trust-root load. That churns the operator's prompt AND masquerades as a drop-and-execute IOC. The cache_dir we pass to SigstoreTrustRoot::new only backs the trusted_root.json checkout, not the datastore, and sigstore-rs exposes no API to redirect it — so $TMPDIR is the only lever. - Pin $TMPDIR to protector's TUF cache dir at single-threaded startup (before the tokio runtime spawns any worker thread; set_var is only sound single-threaded in edition 2024). An explicit operator $TMPDIR still wins. New tuf_tmpdir module with a pure, unit-tested resolver. - Move the chart's TUF cache off /tmp onto a dedicated protector-owned tuf-cache emptyDir at /var/lib/protector/tuf, so both the checkout AND (via the pin) tough's datastore land in a stable, attributable path. - Document the OnceCell fetch-once-and-stick semantics: get_or_try_init caches only success, so a transient TUF blip retries but steady state does no TUF writes — no per-verify or interval refresh. NON-GOAL respected: /tmp write observation is untouched — /tmp writes remain IOCs the agent keeps flagging; this only stops protector's OWN benign TUF plumbing. Closes JEF-377 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01VtjoJttCvBY4dzCoE4f9vP --- charts/protector/templates/deployment.yaml | 20 +++- engine/src/main.rs | 26 ++++- engine/src/policies/signature/cosign.rs | 8 ++ engine/src/policies/signature/mod.rs | 1 + engine/src/policies/signature/tuf_tmpdir.rs | 101 ++++++++++++++++++++ 5 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 engine/src/policies/signature/tuf_tmpdir.rs diff --git a/charts/protector/templates/deployment.yaml b/charts/protector/templates/deployment.yaml index a0c8944..d400a88 100644 --- a/charts/protector/templates/deployment.yaml +++ b/charts/protector/templates/deployment.yaml @@ -216,10 +216,14 @@ spec: value: /etc/protector/tls/tls.crt - name: PROTECTOR_TLS_KEY value: /etc/protector/tls/tls.key - # sigstore TUF cache — must be writable under readOnlyRootFilesystem, - # so it points into the /tmp emptyDir below. + # sigstore TUF cache — must be writable under readOnlyRootFilesystem, so it points + # into the dedicated `tuf-cache` emptyDir below. Kept OFF /tmp (JEF-377): the engine + # pins $TMPDIR to this dir at startup so the tough TUF client's atomic temp writes + # (latest_known_time.json + refreshed metadata) land here — a stable, attributable, + # protector-owned path — instead of a `/tmp/.tmp/` dir that both churns the + # prompt and masquerades as a drop-and-execute IOC the agent must keep flagging. - name: PROTECTOR_TUF_CACHE - value: /tmp/sigstore + value: /var/lib/protector/tuf - name: PROTECTOR_GATED_PREFIXES value: {{ .Values.signature.gatedPrefixes | quote }} - name: PROTECTOR_IDENTITY_REGEXP @@ -341,6 +345,10 @@ spec: readOnly: true - name: tmp mountPath: /tmp + # Writable sigstore TUF cache, kept OFF /tmp (JEF-377) so the tough client's atomic + # temp writes are attributable to protector, not IOC-shaped /tmp/.tmp/ paths. + - name: tuf-cache + mountPath: /var/lib/protector/tuf {{- if .Values.ingestAuth.enabled }} - name: ingest-auth mountPath: /etc/protector/ingest @@ -368,6 +376,12 @@ spec: secretName: {{ include "protector.servingCertName" . }} - name: tmp emptyDir: {} + # Dedicated writable scratch for the sigstore TUF cache, deliberately separate from the + # /tmp emptyDir so protector's own TUF metadata writes never look like a /tmp drop IOC + # (JEF-377). Ephemeral per pod — tough deletes its datastore after each load, and the + # trusted_root.json checkout is cheap to re-fetch. + - name: tuf-cache + emptyDir: {} {{- if .Values.ingestAuth.enabled }} # The ingest bearer token (Fix A), surfaced as a file the engine reads once. - name: ingest-auth diff --git a/engine/src/main.rs b/engine/src/main.rs index 3ed8869..eed58bf 100644 --- a/engine/src/main.rs +++ b/engine/src/main.rs @@ -204,8 +204,23 @@ fn build_webhook_signing_observer( } } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { + // Route the sigstore/tough TUF client's atomic temp writes (latest_known_time.json + the + // refreshed root/timestamp/snapshot metadata) OFF /tmp and onto protector's own TUF cache + // dir (JEF-377). sigstore-rs 0.14 gives tough no datastore path, so it defaults to a + // `/tmp/.tmp/` TempDir that both churns the prompt and masquerades as a drop-and-execute + // IOC; $TMPDIR is the only lever. This MUST run before the tokio runtime spawns any worker + // thread — `set_var` is only sound single-threaded (edition 2024). See `tuf_tmpdir`. + protector::policies::signature::tuf_tmpdir::pin_from_env(); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .context("building the tokio runtime")? + .block_on(run()) +} + +async fn run() -> Result<()> { // Logging + (when OTEL_EXPORTER_OTLP_ENDPOINT is set) OTLP export of traces and // engine metrics to the node-local collector, like the cluster's other services. let telemetry = protector::telemetry::init(env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); @@ -254,7 +269,12 @@ async fn main() -> Result<()> { "signature gating off (PROTECTOR_GATED_PREFIXES empty) — no images are signature-checked" ); } - let tuf_cache = PathBuf::from(env_or("PROTECTOR_TUF_CACHE", "/tmp/sigstore")); + // Same default the startup `$TMPDIR` pin uses (JEF-377), so tough's atomic temp writes land + // in exactly this cache dir rather than under /tmp/.tmp/. + let tuf_cache = PathBuf::from(env_or( + "PROTECTOR_TUF_CACHE", + protector::policies::signature::tuf_tmpdir::DEFAULT_TUF_CACHE, + )); // 20s (was 5s): a keyless Fulcio+Rekor+TUF verify of a first-party signed image on the arm64 // engine — especially with a cold TUF cache after a restart — routinely needs >5s, so a 5s // budget left those images stuck in "checking" forever (JEF-326). Still env-overridable, and diff --git a/engine/src/policies/signature/cosign.rs b/engine/src/policies/signature/cosign.rs index 89db212..9620761 100644 --- a/engine/src/policies/signature/cosign.rs +++ b/engine/src/policies/signature/cosign.rs @@ -102,6 +102,14 @@ impl CosignChecker { } /// Get (or lazily fetch) the sigstore TUF trust root. + /// + /// Fetch-once-and-stick (JEF-377): `get_or_try_init` caches only a *successful* init — on + /// `Err` the cell stays empty, so a transient TUF/registry blip is retried on the next call + /// rather than being frozen, and a success is reused for the process lifetime. There is no + /// per-verify or interval TUF refresh: the one fetch here (and its one-time tough temp write) + /// happens on the first verification and never again while it holds, so steady state does no + /// TUF writes at all. (The tough temp write itself is kept off /tmp via the startup `$TMPDIR` + /// pin — see `tuf_tmpdir`.) async fn trust_root(&self) -> Result<&SigstoreTrustRoot> { self.trust_root .get_or_try_init(|| async { diff --git a/engine/src/policies/signature/mod.rs b/engine/src/policies/signature/mod.rs index 5599ac0..a8f1678 100644 --- a/engine/src/policies/signature/mod.rs +++ b/engine/src/policies/signature/mod.rs @@ -15,6 +15,7 @@ mod cosign; pub mod posture; pub mod provenance; pub mod rekor; +pub mod tuf_tmpdir; #[cfg(test)] mod tests; diff --git a/engine/src/policies/signature/tuf_tmpdir.rs b/engine/src/policies/signature/tuf_tmpdir.rs new file mode 100644 index 0000000..95e6d78 --- /dev/null +++ b/engine/src/policies/signature/tuf_tmpdir.rs @@ -0,0 +1,101 @@ +//! Keep the sigstore TUF client's atomic temp writes OFF `/tmp` (JEF-377). +//! +//! sigstore-rs 0.14 builds its `tough` TUF client via `RepositoryLoader` WITHOUT calling +//! `.datastore(...)`, so `tough` falls back to `Datastore::new(None)` → `TempDir::new()` — a +//! random dir under `$TMPDIR` (default `/tmp`). On every trust-root load it writes +//! `latest_known_time.json` plus the refreshed root/timestamp/snapshot metadata there. That +//! churns the operator's prompt AND masquerades as a `/tmp/.tmp/…` drop-and-execute IOC. +//! The `cache_dir` we pass to [`SigstoreTrustRoot::new`](sigstore::trust::sigstore::SigstoreTrustRoot::new) +//! only backs the final `trusted_root.json` checkout read/write — NOT the tough datastore — and +//! sigstore-rs 0.14 exposes no API to redirect the datastore, so the ONLY lever is `$TMPDIR`. +//! +//! We pin `$TMPDIR` to protector's own TUF cache dir at single-threaded startup so tough's temp +//! writes land in a stable, attributable, protector-owned dir alongside the rest of the cache. +//! +//! NON-GOAL (hard): this does NOT suppress `/tmp` write *observation* anywhere — `/tmp` writes +//! are IOCs (drop-and-execute) the agent MUST keep seeing. This only stops protector's OWN benign +//! TUF plumbing from writing to `/tmp`. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +/// Default TUF cache dir when `PROTECTOR_TUF_CACHE` is unset. Kept identical to `main`'s default +/// (a single source of truth) so [`pin_from_env`] pins `$TMPDIR` to exactly the dir the +/// `CosignChecker` will use as its cache. In-cluster the chart points this OFF `/tmp` at a +/// protector-owned volume; locally it stays a writable `/tmp` subdir. +pub const DEFAULT_TUF_CACHE: &str = "/tmp/sigstore"; + +/// Decide the dir to pin as `$TMPDIR` for the TUF/tough temp writes, or `None` to leave the +/// process `$TMPDIR` untouched. An explicit, non-empty operator `$TMPDIR` always wins (the escape +/// hatch); otherwise we route tough's temp files into `cache_dir` so `latest_known_time.json` +/// lands beside the rest of the TUF cache instead of under `/tmp/.tmp/`. +fn resolve_tmpdir(existing: Option, cache_dir: &Path) -> Option { + match existing { + Some(v) if !v.is_empty() => None, + _ => Some(cache_dir.to_path_buf()), + } +} + +/// Pin `$TMPDIR` (see [`resolve_tmpdir`]) so the sigstore/tough TUF client writes its atomic temp +/// files into protector's TUF cache dir, not `/tmp`. +/// +/// MUST be called from `main` BEFORE the async runtime spawns any worker thread: +/// [`std::env::set_var`] is only sound while the process is single-threaded (edition 2024). A +/// best-effort no-op if the dir can't be created — the worst case is the pre-JEF-377 behavior (a +/// `/tmp` temp write), never a startup failure (the `CosignChecker` cache-dir create surfaces any +/// real misconfiguration with context). +pub fn pin_from_env() { + let cache_dir = std::env::var_os("PROTECTOR_TUF_CACHE") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(DEFAULT_TUF_CACHE)); + let Some(target) = resolve_tmpdir(std::env::var_os("TMPDIR"), &cache_dir) else { + return; + }; + // Point $TMPDIR at the dir only if we can actually create it — pointing it at a dir that + // doesn't exist would make tough's `TempDir::new` fail. If create fails (read-only fs / bad + // path), leave $TMPDIR alone and fall back to the prior behavior. + if std::fs::create_dir_all(&target).is_err() { + return; + } + // SAFETY: called from `main` before the tokio runtime is built, so the process is still + // single-threaded and no other thread can be reading or writing the environment concurrently. + unsafe { + std::env::set_var("TMPDIR", &target); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unset_tmpdir_routes_tough_temp_into_the_cache_dir() { + // No operator $TMPDIR ⇒ we pin it to the TUF cache dir so tough's atomic temp writes + // (latest_known_time.json) land there, not under /tmp/.tmp/. + let cache = Path::new("/var/lib/protector/tuf"); + assert_eq!( + resolve_tmpdir(None, cache), + Some(PathBuf::from("/var/lib/protector/tuf")) + ); + } + + #[test] + fn empty_tmpdir_is_treated_as_unset() { + // An empty $TMPDIR is not a usable dir — treat it like unset and pin the cache dir. + let cache = Path::new("/var/lib/protector/tuf"); + assert_eq!( + resolve_tmpdir(Some(OsString::from("")), cache), + Some(PathBuf::from("/var/lib/protector/tuf")) + ); + } + + #[test] + fn an_explicit_operator_tmpdir_wins() { + // If the operator set $TMPDIR deliberately, respect it (the escape hatch) — don't override. + let cache = Path::new("/var/lib/protector/tuf"); + assert_eq!( + resolve_tmpdir(Some(OsString::from("/custom/scratch")), cache), + None + ); + } +}