ci: dependency curation fixes + devbox/nix CI toolchain (Artifactory-only)

.github/actions/curation-audit/action.yml

@@ -0,0 +1,113 @@
+name: 'Curation Audit'
+description: 'On install failure, run a JFrog curation audit against just the packages yarn actually 403d, for a clear policy reason instead of a raw 403'
+
+inputs:
+  install-log:
+    description: 'Path to the captured yarn install output'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Extract 403d packages from the install log
+      shell: bash
+      id: extract
+      env:
+        INSTALL_LOG: ${{ inputs.install-log }}
+      run: |
+        set -euo pipefail
+        WORKDIR=$(mktemp -d)
+        echo "AUDIT_WORKDIR=$WORKDIR" >> "$GITHUB_ENV"
+
+        # yarn logs each curation 403 as three lines, and prints the same
+        # failure twice: once in the resolution tree (with a │/└ prefix) and
+        # once more in the final summary (no prefix) - the prefix is
+        # optional here so both forms normalize to the same name@version
+        # and collapse into one entry via sort -u below.
+        #   ➤ YN0035: │ <name>@npm:<version>: The remote server failed to provide...
+        #   ➤ YN0035: │   Response Code: 403 (Forbidden)
+        sed -E 's/\x1b\[[0-9;]*m//g' "$INSTALL_LOG" \
+          | grep "npm:.*: The remote server failed" \
+          | sed -E 's/.*YN0035: ([│└] )?//' \
+          | sed -E 's/(@npm:[^:]+): The remote.*/\1/' \
+          | sed -E 's/@npm:/@/' \
+          | sort -u > "$WORKDIR/packages.txt" || true
+
+        count=$(wc -l < "$WORKDIR/packages.txt" | tr -d ' ')
+        echo "count=$count" >> "$GITHUB_OUTPUT"
+        echo "Found $count curation-blocked package(s):"
+        cat "$WORKDIR/packages.txt"
+
+        host=$(echo "${ARTIFACTORY_URL}" | sed -E 's#https?://##')
+        cat > "$WORKDIR/base.npmrc" <<EOF
+        registry=${ARTIFACTORY_URL}/artifactory/api/npm/virtual-npm-thirdparty/
+        //${host}/artifactory/api/npm/virtual-npm-thirdparty/:_authToken=${YARN_NPM_AUTH_TOKEN}
+        EOF
+
+        cat > "$WORKDIR/check_one.sh" <<'SCRIPT'
+        #!/usr/bin/env bash
+        set -uo pipefail
+
+        pkg="$1"
+        name="${pkg%@*}"
+        version="${pkg##*@}"
+
+        tmpdir=$(mktemp -d)
+        cp "${AUDIT_WORKDIR}/base.npmrc" "$tmpdir/.npmrc"
+        python3 -c '
+        import json, sys
+        name, version, out = sys.argv[1], sys.argv[2], sys.argv[3]
+        json.dump({"name": "curation-audit-synthetic", "version": "1.0.0", "private": True, "dependencies": {name: version}}, open(out, "w"))
+        ' "$name" "$version" "$tmpdir/package.json"
+
+        out=$(cd "$tmpdir" && jf ca --run-native --legacy-peer-deps --format=table 2>&1)
+        exit_status=$?
+        rm -rf "$tmpdir"
+
+        # jf's own exit code doesn't distinguish a real curation block from noise
+        # (e.g. npm ls ELSPROBLEMS when a package's own peer deps aren't declared
+        # in this single-package synthetic project) - only treat it as a real
+        # block when the output actually says so.
+        if echo "$out" | grep -qE "download blocking policy|Found [1-9][0-9]* blocked packages?"; then
+          echo "::group::BLOCKED: $pkg"
+          echo "$out"
+          echo "::endgroup::"
+          exit 1
+        elif [ $exit_status -ne 0 ]; then
+          echo "::group::Audit inconclusive for $pkg (non-curation resolution error, ignoring)"
+          echo "$out"
+          echo "::endgroup::"
+        fi
+        exit 0
+        SCRIPT
+        chmod +x "$WORKDIR/check_one.sh"
+
+    - name: Install jf CLI
+      if: steps.extract.outputs.count != '0'
+      shell: bash
+      run: |
+        curl -fkL https://getcli.jfrog.io/v2-jf | sh
+        sudo mv jf /usr/local/bin/jf
+
+    - name: Configure jf against the SDK build lane
+      if: steps.extract.outputs.count != '0'
+      shell: bash
+      run: |
+        jf c add curation-audit-server \
+          --url="${ARTIFACTORY_URL}" \
+          --access-token="${YARN_NPM_AUTH_TOKEN}" \
+          --interactive=false \
+          --overwrite
+        jf c use curation-audit-server
+
+    - name: Curation audit each blocked package
+      if: steps.extract.outputs.count != '0'
+      shell: bash
+      run: |
+        set -uo pipefail
+        xargs -P 8 -n 1 "${AUDIT_WORKDIR}/check_one.sh" < "${AUDIT_WORKDIR}/packages.txt"
+        exit_status=$?
+        if [ $exit_status -ne 0 ]; then
+          echo "::error::Curation blocked one or more packages - see the BLOCKED groups above for the policy reason on each."
+          exit 1
+        fi

.github/actions/devbox-bootstrap/action.yml

@@ -0,0 +1,122 @@
+name: 'Devbox Bootstrap (Artifactory-only)'
+description: 'Install nix (via apt) and a patched devbox, resolving everything through the SDK build lane Artifactory instance only - no nixos.org, no get.jetify.com, no direct archive.ubuntu.com'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Diagnose the new repos' auth scheme and URL layout
+      shell: bash
+      run: |
+        set -uo pipefail
+        AUTH=(-H "Authorization: Bearer ${YARN_NPM_AUTH_TOKEN}")
+
+        check() {
+          local desc="$1" url="$2"
+          shift 2
+          local code
+          code=$(curl -sS -o /tmp/probe-body -w "%{http_code}" "$@" "$url")
+          echo "$desc -> HTTP $code ($url)"
+          if [ "$code" != "401" ] && [ "$code" != "404" ]; then
+            head -c 300 /tmp/probe-body; echo
+          fi
+        }
+
+        check "debian, bearer, dists/jammy/InRelease (virtual)"  "${ARTIFACTORY_URL}/artifactory/virtual-debian-thirdparty/dists/jammy/InRelease" "${AUTH[@]}"
+        check "debian, bearer, dists/jammy/InRelease (remote)"   "${ARTIFACTORY_URL}/artifactory/remote-debian-thirdparty-filtered/dists/jammy/InRelease" "${AUTH[@]}"
+        check "debian, token-as-user basic, dists/jammy/InRelease (remote)" "${ARTIFACTORY_URL}/artifactory/remote-debian-thirdparty-filtered/dists/jammy/InRelease" -u "${YARN_NPM_AUTH_TOKEN}:"
+
+        check "nix, bearer, nix-cache-info"                  "${ARTIFACTORY_URL}/artifactory/virtual-nix-thirdparty/nix-cache-info" "${AUTH[@]}"
+        check "nix, token-as-user basic, nix-cache-info"     "${ARTIFACTORY_URL}/artifactory/virtual-nix-thirdparty/nix-cache-info" -u "${YARN_NPM_AUTH_TOKEN}:"
+
+        echo "--- storage API listing (introspects actual repo contents) ---"
+        check "storage api, debian remote"  "${ARTIFACTORY_URL}/artifactory/api/storage/remote-debian-thirdparty-filtered" "${AUTH[@]}"
+
+    - name: Point apt + nix at the Artifactory pull-through caches
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        host=$(echo "${ARTIFACTORY_URL}" | sed -E 's#https?://##; s#/+$##')
+        codename=$(. /etc/os-release && echo "${VERSION_CODENAME}")
+        echo "Detected Ubuntu codename: ${codename}"
+
+        # Shared credentials for both apt and nix: any username works, the
+        # Artifactory access token is the password (standard JFrog convention).
+        sudo mkdir -p /etc/apt/auth.conf.d
+        printf 'machine %s\nlogin artifactory-build-token\npassword %s\n' \
+          "${host}" "${YARN_NPM_AUTH_TOKEN}" | sudo tee /etc/apt/auth.conf.d/twilio-artifactory.conf > /dev/null
+        sudo chmod 600 /etc/apt/auth.conf.d/twilio-artifactory.conf
+
+        sudo mkdir -p /etc/nix
+        printf 'machine %s\nlogin artifactory-build-token\npassword %s\n' \
+          "${host}" "${YARN_NPM_AUTH_TOKEN}" | sudo tee /etc/nix/netrc > /dev/null
+        sudo chmod 600 /etc/nix/netrc
+
+        # Debian/Ubuntu pull-through cache (remote-debian-thirdparty-filtered):
+        # mirrors archive.ubuntu.com's main archive only (no -security, a
+        # separate upstream host) - fine for the one-off nix-bin install below.
+        printf 'deb %s/artifactory/virtual-debian-thirdparty %s main universe\ndeb %s/artifactory/virtual-debian-thirdparty %s-updates main universe\n' \
+          "${ARTIFACTORY_URL}" "${codename}" "${ARTIFACTORY_URL}" "${codename}" \
+          | sudo tee /etc/apt/sources.list.d/twilio-artifactory.list > /dev/null
+        sudo rm -f /etc/apt/sources.list
+        sudo rm -f /etc/apt/sources.list.d/ubuntu.sources
+
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends nix-bin ca-certificates
+
+        # nix-bin's postinst creates /nix owned by root. We run nix DAEMONLESS
+        # and as the normal (non-root) runner user for every step after this
+        # one, so hand the store to that user now instead of prefixing every
+        # later nix/devbox/yarn invocation with sudo.
+        sudo chown -R "$(id -u):$(id -g)" /nix
+
+        # Nix global config: substituter is the nix pull-through cache
+        # (remote-nix-thirdparty, mirroring cache.nixos.org byte-for-byte, so
+        # the standard public key still verifies it). flake-registry empty:
+        # channels.nixos.org is not proxied - devbox.lock resolves straight to
+        # substitutable store paths without it. build-users-group empty +
+        # sandbox off: daemonless single-user nix has no nixbld group, and the
+        # one local build we do (patched devbox) can't use nix's sandbox
+        # without the namespaces/privileges a plain runner VM doesn't have.
+        printf '%s\n' \
+          'experimental-features = nix-command flakes' \
+          "substituters = ${ARTIFACTORY_URL}/artifactory/api/nix/virtual-nix-thirdparty" \
+          'trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=' \
+          'netrc-file = /etc/nix/netrc' \
+          'flake-registry = ' \
+          'build-users-group = ' \
+          'sandbox = false' \
+          | sudo tee /etc/nix/nix.conf > /dev/null
+
+        {
+          echo "NIX_REMOTE="
+          echo "DEVBOX_NIX_BINARY_CACHE=${ARTIFACTORY_URL}/artifactory/api/nix/virtual-nix-thirdparty"
+        } >> "$GITHUB_ENV"
+
+        echo "$HOME/.nix-profile/bin" >> "$GITHUB_PATH"
+        echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
+
+    - name: Install a patched devbox (honors DEVBOX_NIX_BINARY_CACHE)
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        # devbox is a nixpkgs package; `nixpkgs#devbox` needs the flake registry
+        # (channels.nixos.org, not proxied) and `github:NixOS/nixpkgs/<rev>`
+        # fetches from github (not proxied) even though the resulting NAR is in
+        # Artifactory - so name the nixpkgs SOURCE by its STORE PATH directly
+        # (content-addressed by revision, substitutable from Artifactory).
+        # overrideAttrs applies the same patch ajs-renderer already proved
+        # (jetify-com/devbox#2891, rebased onto nixpkgs' devbox 0.17.2), making
+        # devbox read its binary cache from DEVBOX_NIX_BINARY_CACHE instead of
+        # the hardcoded https://cache.nixos.org - without it, `devbox run` does
+        # a fatal narinfo HEAD to cache.nixos.org on every invocation.
+        NIXPKGS_SRC=/nix/store/p9rjycrjhya7jbi9759vf7xql9cm8xqh-source
+        PATCH="$(cd "${{ github.action_path }}/../../devbox-overlay" && pwd)/configurable-nix-binary-cache.patch"
+
+        nix-store --realise "${NIXPKGS_SRC}"
+        nix --extra-experimental-features 'nix-command flakes' \
+          profile install --impure --expr \
+          "let p = import ${NIXPKGS_SRC} {}; in p.devbox.overrideAttrs (o: { patches = (o.patches or []) ++ [ ${PATCH} ]; })"
+
+        devbox version

.github/devbox-overlay/configurable-nix-binary-cache.patch

@@ -0,0 +1,148 @@
+diff --git a/internal/devpkg/narinfo_cache.go b/internal/devpkg/narinfo_cache.go
+index e39fd2a..5c79300 100644
+--- a/internal/devpkg/narinfo_cache.go
++++ b/internal/devpkg/narinfo_cache.go
+@@ -15,15 +15,29 @@
+ 	"github.com/pkg/errors"
+ 	"go.jetify.com/devbox/internal/debug"
+ 	"go.jetify.com/devbox/internal/devbox/providers/nixcache"
++	"go.jetify.com/devbox/internal/envir"
+ 	"go.jetify.com/devbox/internal/goutil"
+ 	"go.jetify.com/devbox/internal/lock"
+ 	"go.jetify.com/devbox/internal/nix"
+ 	"golang.org/x/sync/errgroup"
+ )
+
+-// binaryCache is the store from which to fetch this package's binaries.
+-// It is used as FromStore in builtins.fetchClosure.
+-const binaryCache = "https://cache.nixos.org"
++// defaultBinaryCache is the public Nix binary cache that Devbox queries by
++// default for prebuilt package outputs.
++const defaultBinaryCache = "https://cache.nixos.org"
++
++// BinaryCache is the store from which to fetch a package's binaries. It is
++// used as the FromStore in builtins.fetchClosure and as the first cache
++// queried when checking whether a package's outputs are already built.
++//
++// It defaults to the public cache (https://cache.nixos.org) but can be
++// overridden with the DEVBOX_NIX_BINARY_CACHE environment variable. This is
++// useful in network-restricted environments where the public cache is
++// unreachable and an internal mirror/proxy serving the same store paths must
++// be used instead.
++func BinaryCache() string {
++	return envir.GetValueOrDefault(envir.DevboxNixBinaryCache, defaultBinaryCache)
++}
+
+ // useDefaultOutputs is a special value for the outputName parameter of
+ // fetchNarInfoStatusOnce, which indicates that the default outputs should be
+@@ -320,7 +334,7 @@ func(o *s3.Options) {
+ var nixCacheIsConfigured = goutil.OnceValueWithContext(nixcache.IsConfigured)
+
+ func readCaches(ctx context.Context) ([]string, error) {
+-	cacheURIs := []string{binaryCache}
++	cacheURIs := []string{BinaryCache()}
+ 	if !nixCacheIsConfigured.Do(ctx) {
+ 		return cacheURIs, nil
+ 	}
+diff --git a/internal/envir/env.go b/internal/envir/env.go
+index 5cf0c37..382ec20 100644
+--- a/internal/envir/env.go
++++ b/internal/envir/env.go
+@@ -6,6 +6,13 @@
+ const (
+ 	DevboxCache   = "DEVBOX_CACHE"
+ 	DevboxGateway = "DEVBOX_GATEWAY"
++	// DevboxNixBinaryCache overrides the default Nix binary cache that Devbox
++	// queries for prebuilt package outputs (https://cache.nixos.org). This is
++	// useful in network-restricted environments where the public cache is
++	// unreachable and an internal mirror/proxy must be used instead. The value
++	// should be a substituter URL serving the same store paths (e.g. an
++	// Artifactory/Nexus generic remote that mirrors cache.nixos.org).
++	DevboxNixBinaryCache = "DEVBOX_NIX_BINARY_CACHE"
+ 	// DevboxLatestVersion is the latest version available of the devbox CLI binary.
+ 	// NOTE: it should NOT start with v (like 0.4.8)
+ 	DevboxLatestVersion  = "DEVBOX_LATEST_VERSION"
+diff --git a/internal/shellgen/generate.go b/internal/shellgen/generate.go
+index 838152f..16c3052 100644
+--- a/internal/shellgen/generate.go
++++ b/internal/shellgen/generate.go
+@@ -17,6 +17,7 @@
+ 	"github.com/pkg/errors"
+ 	"go.jetify.com/devbox/internal/cuecfg"
+ 	"go.jetify.com/devbox/internal/debug"
++	"go.jetify.com/devbox/internal/devpkg"
+ 	"go.jetify.com/devbox/internal/redact"
+ )
+
+@@ -161,9 +162,41 @@ func toJSON(a any) string {
+ }
+
+ var templateFuncs = template.FuncMap{
+-	"json":     toJSON,
+-	"contains": strings.Contains,
+-	"debug":    debug.IsEnabled,
++	"json":              toJSON,
++	"contains":          strings.Contains,
++	"debug":             debug.IsEnabled,
++	"fetchClosureStore": fetchClosureStore,
++	"nixString":         nixString,
++}
++
++// fetchClosureStore returns the store URL to use as the fromStore of a
++// builtins.fetchClosure in the generated flake.
++//
++// fetchClosure only supports http(s) stores, so when a package's outputs were
++// found in an http(s) cache we use that cache directly (it's where the path
++// actually lives, which may be the public cache or a configured mirror via
++// DEVBOX_NIX_BINARY_CACHE). For s3 caches fetchClosure can't fetch directly, so
++// we fall back to the configured binary cache as a placeholder store — the path
++// is already built locally, so fetchClosure won't actually fetch from it.
++func fetchClosureStore(cacheURI string) string {
++	if strings.HasPrefix(cacheURI, "http://") || strings.HasPrefix(cacheURI, "https://") {
++		return cacheURI
++	}
++	return devpkg.BinaryCache()
++}
++
++// nixString renders s as a double-quoted Nix string literal, escaping the
++// characters that are special inside one: backslash, double quote, and the
++// "${" antiquotation (interpolation) start. This keeps values that originate
++// from configuration (e.g. a cache URL from DEVBOX_NIX_BINARY_CACHE) from
++// breaking flake evaluation or being interpreted as Nix interpolation.
++func nixString(s string) string {
++	r := strings.NewReplacer(
++		`\`, `\\`,
++		`"`, `\"`,
++		`${`, `\${`,
++	)
++	return `"` + r.Replace(s) + `"`
+ }
+
+ func makeFlakeFile(d devboxer, plan *flakePlan) error {
+diff --git a/internal/shellgen/tmpl/flake.nix.tmpl b/internal/shellgen/tmpl/flake.nix.tmpl
+index 3bd0513..cbe1321 100644
+--- a/internal/shellgen/tmpl/flake.nix.tmpl
++++ b/internal/shellgen/tmpl/flake.nix.tmpl
+@@ -42,13 +42,16 @@
+             {{ if $output.CacheURI -}}
+             (builtins.trace "downloading {{ $pkg.Versioned }}" (builtins.fetchClosure {
+               {{/*
+-                HACK HACK HACK! fetchClosure only supports http(s) caches and not
+-                s3 caches. Until we implement that, we put a fake store here.
+-                Since we pre-build everything, fetchClosure will not actually
+-                fetch anything and just use the local version. This may break
+-                if user somehow removes the local store path.
++                fetchClosure only supports http(s) caches and not s3 caches.
++                fetchClosureStore returns the http(s) cache the output was found
++                in (the public cache or a configured mirror via
++                DEVBOX_NIX_BINARY_CACHE), falling back to the default binary
++                cache for s3 caches. Since we pre-build everything, fetchClosure
++                will not actually fetch anything for an s3 cache and just use the
++                local version. This may break if the user somehow removes the
++                local store path.
+               */}}
+-              fromStore = "https://cache.nixos.org";
++              fromStore = {{ fetchClosureStore $output.CacheURI | nixString }};
+               fromPath = "{{ $pkg.InputAddressedPathForOutput $output.Name }}";
+               inputAddressed = true;
+             }))