Promote apt lock timeout and bounded boot upgrade to sandbox

debian/changelog

@@ -1,3 +1,9 @@
+puppet-code (0.1.0-1build322) noble; urgency=medium
+
+  * commit event. see changes history in git log
+
+ -- root <packager@infrahouse.com>  Sat, 01 Aug 2026 00:25:11 +0000
+
 puppet-code (0.1.0-1build321) noble; urgency=medium
 
   * commit event. see changes history in git log

environments/sandbox/modules/profile/files/github_runner/gha-boot-security-upgrade.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# One-shot security patching during runner provisioning, with a hard cumulative
+# time bound.
+#
+# Why a script instead of exec's tries/try_sleep: Puppet's `timeout` is
+# PER-ATTEMPT, so `tries` multiplies the worst case to
+# tries * timeout + (tries-1) * try_sleep with no cumulative cap. That matters
+# here because an overrun is not merely a slow run -- ih-puppet applies with
+# --detailed-exitcodes and exits 4/6 when a resource fails, which trips
+# ih-bootstrap.sh's `trap _ih_signal_abandon ERR` and ABANDONs the instance. So a
+# retry budget that can exceed the bootstrap lifecycle hook is a fleet-churn bug,
+# not a latency bug. Bounding total wall clock here lets one legitimately long
+# upgrade use the whole window while still capping the worst case.
+#
+# What actually needs retrying: both commands below can fail within seconds under
+# lock contention.
+#   - `apt-get update` takes /var/lib/apt/lists/lock, which DPkg::Lock::Timeout
+#     does NOT cover (measured: fails in ~1s even with the option set).
+#   - `unattended-upgrade` refuses to run concurrently with itself.
+# Contenders are routine: the Inspector and GuardDuty agents each dpkg-install
+# about a minute into every boot, squarely inside the provisioning window.
+#
+# Note ih-puppet already runs the catalog twice and only checks the second exit
+# code, so a transient failure gets one free retry above this script too.
+#
+# Usage: gha-boot-security-upgrade.sh [budget_seconds] [marker_path]
+
+# Deliberately no `set -e`: failures of the apt commands are expected and handled
+# by the retry loop below.
+set -uo pipefail
+
+BUDGET="${1:-480}"
+MARKER="${2:-/run/gha-boot-upgrade.done}"
+
+deadline=$(( $(date +%s) + BUDGET ))
+attempt=0
+
+while :; do
+    attempt=$(( attempt + 1 ))
+    remaining=$(( deadline - $(date +%s) ))
+
+    if [ "$remaining" -le 0 ]; then
+        echo "gha-boot-security-upgrade: ${BUDGET}s budget exhausted after ${attempt} attempt(s)" >&2
+        exit 1
+    fi
+
+    echo "gha-boot-security-upgrade: attempt ${attempt}, ${remaining}s of budget left"
+
+    # Each command is capped at the remaining budget so a single slow command
+    # cannot overshoot the deadline.
+    if timeout "$remaining" apt-get update -qq && timeout "$remaining" unattended-upgrade; then
+        # Written only on success, so a failed upgrade simply retries on the next
+        # Puppet apply. Lives on tmpfs so it clears on a real boot.
+        touch "$MARKER"
+        echo "gha-boot-security-upgrade: succeeded on attempt ${attempt}"
+        exit 0
+    fi
+
+    # Only sleep if there will still be budget to use afterwards.
+    if [ $(( deadline - $(date +%s) )) -gt 15 ]; then
+        sleep 15
+    fi
+done

environments/sandbox/modules/profile/manifests/apt_lock_timeout.pp

@@ -0,0 +1,47 @@
+# @summary: Make apt-get wait for the dpkg lock instead of failing outright.
+#
+# Declared by profile::repos with `stage => init` so the drop-in exists before any
+# Package resource in stage main. That ordering is the whole point: a drop-in
+# applied halfway through the catalog does not help the Package resources Puppet
+# already evaluated.
+#
+# Why this is needed at all: Ubuntu ships `binary::apt::DPkg::Lock::Timeout "120"`,
+# and that scope applies ONLY to the `apt` command. Puppet's package provider,
+# cloud-init and the AWS agents all shell out to `apt-get`, which inherits nothing
+# and fails instantly on a held lock. Measured on noble/apt 2.8.3 against a held
+# /var/lib/dpkg/lock-frontend: apt-get install exits 100 in 0s without the
+# unscoped key, and 0 in 45s (waiting out a 40s lock) with it.
+#
+# Lock contention here is routine, not hypothetical:
+#   - the Inspector and GuardDuty agents each dpkg-install ~1 min into every boot
+#   - profile::unattended_upgrades deliberately unmasks and STARTS the apt-daily
+#     timers mid-catalog, so an unattended-upgrade can begin during the run
+#
+# NOTE: the path is deliberately the same file infrahouse-ubuntu-pro writes from
+# its provision.sh. Two drop-ins both setting this key would resolve by lexical
+# filename order, which is a silent trap -- so Puppet converges the AMI's file
+# rather than racing a second one of its own.
+#
+# @param timeout
+#   Seconds apt-get waits for the dpkg lock. Sourced from the apt_lock_timeout
+#   custom fact (set via the cloud-init module's custom_facts), defaulting to the
+#   300 that current AMIs ship.
+#
+#   Interpolated as-is, so it makes no difference whether the fact arrives as an
+#   Integer or a String.
+#
+#   Keep it well inside the gha_runner bootstrap lifecycle hook (1200s,
+#   default_result ABANDON): a genuinely wedged lock costs this many seconds per
+#   Package resource, and the hook is not renewed during bootstrap.
+class profile::apt_lock_timeout (
+  $timeout = pick_default($facts['apt_lock_timeout'], 300),
+) {
+
+  file { '/etc/apt/apt.conf.d/99-lock-timeout':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => "DPkg::Lock::Timeout \"${timeout}\";\n",
+  }
+}

environments/sandbox/modules/profile/manifests/github_runner.pp

@@ -65,12 +65,35 @@
   # per boot rather than on every Puppet apply, and is written only on success,
   # so a failed upgrade simply retries on the next apply. This applies Ubuntu
   # security updates, which do not depend on the InfraHouse repos.
+  #
+  # The retry/bounding logic lives in the script rather than in exec's
+  # tries/try_sleep because exec's timeout is per-attempt, so tries would multiply
+  # the worst case with no cumulative cap. A resource failure here ABANDONs the
+  # instance (ih-puppet exits 4/6, ih-bootstrap's ERR trap signals ABANDON), so
+  # the total must stay inside the 1200s bootstrap hook budget -- nothing renews
+  # it, since gha-lifecycle-heartbeater.sh is a no-op outside Terminating:Wait.
+  $boot_upgrade_script = '/usr/local/bin/gha-boot-security-upgrade.sh'
+  $boot_upgrade_budget = 480
+
+  file { $boot_upgrade_script:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/profile/github_runner/gha-boot-security-upgrade.sh',
+  }
+
   exec { 'gha-boot-security-upgrade':
-    command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done',
+    command => "${boot_upgrade_script} ${boot_upgrade_budget}",
     path    => '/usr/bin:/bin:/usr/sbin:/sbin',
     unless  => 'test -f /run/gha-boot-upgrade.done',
-    timeout => 1200,
-    require => Class['profile::unattended_upgrades'],
+    # Slightly above the script's own budget so the script always gets to exit and
+    # log why it gave up, rather than being killed mid-report by Puppet.
+    timeout => $boot_upgrade_budget + 60,
+    require => [
+      Class['profile::unattended_upgrades'],
+      File[$boot_upgrade_script],
+    ],
   }
 
 }

environments/sandbox/modules/profile/manifests/repos.pp

@@ -7,4 +7,11 @@
       tries     => 5,
     },
   }
+
+  # Also in the init stage: the lock timeout has to be in place before anything in
+  # stage main starts installing packages. See the class for why apt-get needs an
+  # unscoped key of its own.
+  class { 'profile::apt_lock_timeout':
+    stage => init,
+  }
 }