From a155d7e3f018534eac8862f35368d8085979c369 Mon Sep 17 00:00:00 2001 From: yulia-ivashko Date: Thu, 6 Aug 2026 00:33:51 +0300 Subject: [PATCH 1/2] fix(kubernetes): report an unpullable image as its own cause, not as an isolation result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image the cluster cannot download and a cluster whose networking cannot be characterised are different problems with different owners, and the probe reported them identically: "network isolation could not be verified". Observed live, where a stale image reference pointing at a local registry that no longer runs produced a verdict that read as though something were wrong with the cluster's network, sending the reader to the wrong setting entirely. An image pull failure now travels as its own flag, so a surface can name the setting at fault instead of relaying a networking message. Concurrency for the RBAC preflight, unrelated but from the same observation: readiness ran twenty-six `auth can-i` probes one after another, each a process spawn and a round trip, measured at 221 ms apiece — six seconds before the settings page could say anything, paid even by someone using Docker, since readiness checks every provider the platform supports. They now run eight at a time, and every denial is collected rather than only the first, so one message names everything to request from an administrator. --- src/providers/kubernetes.js | 33 +++++++++++++++++-- src/providers/network-policy-enforcement.js | 11 +++++-- .../network-policy-enforcement.test.js | 5 ++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/providers/kubernetes.js b/src/providers/kubernetes.js index 2f16eef..285f519 100644 --- a/src/providers/kubernetes.js +++ b/src/providers/kubernetes.js @@ -47,9 +47,19 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { } throw new ProviderUnavailableError(`Kubernetes cluster is not reachable: ${cause instanceof Error ? cause.message : String(cause)}`, { provider, code: 'WORKSPACE_PROVIDER_CLUSTER_UNREACHABLE', cause }); }); - for (const [verb, resource] of requiredPermissions(policy)) { + // Each check is a process spawn and a round trip, and there are two dozen of them: + // run sequentially they dominated the time to display readiness. Every denial is + // collected rather than only the first, so one message names everything to request. + const denied = await mapWithConcurrency(requiredPermissions(policy), RBAC_PROBE_CONCURRENCY, async ([verb, resource]) => { const { stdout } = await kubectl(['auth', 'can-i', verb, resource, '-n', policy.kubernetes.namespace], { timeoutMs: 20_000 }); - if (stdout.trim() !== 'yes') throw new ProviderUnavailableError(`Kubernetes RBAC denies ${verb} ${resource} in namespace ${policy.kubernetes.namespace}`, { provider, code: 'WORKSPACE_PROVIDER_RBAC_DENIED' }); + return stdout.trim() === 'yes' ? null : `${verb} ${resource}`; + }); + const missing = denied.filter(Boolean); + if (missing.length > 0) { + throw new ProviderUnavailableError( + `Kubernetes RBAC denies ${missing.join(', ')} in namespace ${policy.kubernetes.namespace}`, + { provider, code: 'WORKSPACE_PROVIDER_RBAC_DENIED' }, + ); } // Validated after the environment checks because the DNS address is discovered from // the cluster, and an unreachable cluster is a more actionable answer than a policy gap. @@ -139,7 +149,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { } if (action === 'check-isolation') { const result = await checkNetworkPolicyEnforcement(kubectl, { context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: validateImage(policy, policy.defaultImage), force: true }); - return { action, verdict: result.verdict, diagnostics: result.diagnostics }; + return { action, verdict: result.verdict, diagnostics: result.diagnostics, imageUnavailable: result.imageUnavailable === true }; } throw new Error(`Unsupported Kubernetes setup action: ${action}`); } @@ -593,6 +603,23 @@ function portAvailable(port) { return new Promise((resolve) => { const server = createServer(); server.once('error', () => resolve(false)); server.listen(port, '127.0.0.1', () => server.close(() => resolve(true))); }); } +const RBAC_PROBE_CONCURRENCY = 8; + +/** Runs `task` over `items` with a bounded number in flight, preserving input order. */ +async function mapWithConcurrency(items, limit, task) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await task(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + function requiredPermissions(policy) { const resources = ['pods', 'secrets', 'serviceaccounts', 'persistentvolumeclaims', 'services', 'deployments.apps', 'networkpolicies.networking.k8s.io']; const result = resources.flatMap((resource) => ['create', 'get', 'delete'].map((verb) => [verb, resource])); diff --git a/src/providers/network-policy-enforcement.js b/src/providers/network-policy-enforcement.js index 2c79758..3c9cda1 100644 --- a/src/providers/network-policy-enforcement.js +++ b/src/providers/network-policy-enforcement.js @@ -122,7 +122,12 @@ async function runProbePod(kubectl, manifest, namespace, deadline, now) { const waiting = container?.state?.waiting; // An unpullable image never terminates, so treat it as a probe failure rather than // waiting out the whole deadline for a pod that cannot run. - if (waiting && /ErrImagePull|ImagePullBackOff|InvalidImageName|CreateContainerConfigError/i.test(waiting.reason ?? '')) { + if (waiting && /ErrImagePull|ImagePullBackOff|InvalidImageName/i.test(waiting.reason ?? '')) { + // Distinguished from other start failures because the cause is a setting the + // operator owns, not anything about the cluster's networking. + return { blocked: `the workspace image could not be pulled by the cluster`, imageUnavailable: true }; + } + if (waiting && /CreateContainerConfigError/i.test(waiting.reason ?? '')) { return { blocked: `probe pod ${name} cannot start: ${waiting.reason}${waiting.message ? ` (${waiting.message})` : ''}` }; } if (status.phase === 'Failed' && !terminated) return { blocked: `probe pod ${name} failed: ${status.reason ?? 'unknown reason'}` }; @@ -144,7 +149,7 @@ export async function probeNetworkPolicyEnforcement(kubectl, { namespace, image, try { const baseline = await runProbePod(kubectl, probePod(`${baseName}-baseline`, namespace, { ...labels, 'openchamber.io/probe-role': 'baseline' }, image, 'baseline'), namespace, deadline, now); created.push(['pod', `${baseName}-baseline`]); - if (baseline.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${baseline.blocked}.`] }; + if (baseline.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${baseline.blocked}.`], imageUnavailable: baseline.imageUnavailable === true }; if (baseline.exitCode !== 0) { return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, @@ -156,7 +161,7 @@ export async function probeNetworkPolicyEnforcement(kubectl, { namespace, image, created.push(['networkpolicy', baseName]); const restricted = await runProbePod(kubectl, probePod(`${baseName}-restricted`, namespace, restrictedLabels, image, 'restricted'), namespace, deadline, now); created.push(['pod', `${baseName}-restricted`]); - if (restricted.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${restricted.blocked}.`] }; + if (restricted.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${restricted.blocked}.`], imageUnavailable: restricted.imageUnavailable === true }; if (restricted.exitCode === 0) return { verdict: ENFORCEMENT_VERDICTS.NOT_ENFORCED, diagnostics: [] }; return { verdict: ENFORCEMENT_VERDICTS.ENFORCED, diagnostics: [] }; } finally { diff --git a/src/providers/network-policy-enforcement.test.js b/src/providers/network-policy-enforcement.test.js index d01da2b..fc33583 100644 --- a/src/providers/network-policy-enforcement.test.js +++ b/src/providers/network-policy-enforcement.test.js @@ -69,7 +69,10 @@ describe('kubernetes network policy enforcement probe', () => { const result = await probeNetworkPolicyEnforcement(kubectl, { namespace: 'workspaces', image: IMAGE }); expect(result.verdict).toBe(ENFORCEMENT_VERDICTS.INCONCLUSIVE); - expect(result.diagnostics[0]).toMatch(/ImagePullBackOff/); + // The cause is a setting the operator owns, so it is reported apart from anything + // the probe learned about the cluster's networking. + expect(result.imageUnavailable).toBe(true); + expect(result.diagnostics[0]).toMatch(/image could not be pulled/i); }); it('removes every probe resource it created, including after a verdict of no enforcement', async () => { From 0bacdda404371dc1b5ed932856541692f232c897 Mon Sep 17 00:00:00 2001 From: yulia-ivashko Date: Thu, 6 Aug 2026 00:38:18 +0300 Subject: [PATCH 2/2] perf(kubernetes): probe isolation with the gateway image rather than the workspace one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe only needs a runtime that can open a TCP connection, but it ran the workspace image: 1.32 GB, against the egress gateway's 329 MB. A cluster meeting either for the first time downloads it before the check can answer, so the check was four times slower than it needed to be on exactly the clusters where someone is most likely to run it — a new one, being set up. Managed egress already requires the gateway image, so nothing new is pulled in the default configuration. External egress keeps the workspace image, which is needed there regardless, and the reference is digest-validated either way. Verified live: enforced verdict unchanged, sixteen seconds against a cluster that already had both images. --- src/policy.js | 2 +- src/providers/kubernetes.js | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/policy.js b/src/policy.js index 2d897c8..84188b0 100644 --- a/src/policy.js +++ b/src/policy.js @@ -196,7 +196,7 @@ export function validateImage(policy, image) { return normalized; } -function validateGatewayImage(image) { +export function validateGatewayImage(image) { const normalized = String(image ?? '').trim(); if (!isPinnedImage(normalized)) throw new PolicyError('Managed egress requires a sha256 digest-pinned gateway image'); return normalized; diff --git a/src/providers/kubernetes.js b/src/providers/kubernetes.js index 285f519..b8550a5 100644 --- a/src/providers/kubernetes.js +++ b/src/providers/kubernetes.js @@ -7,7 +7,7 @@ import { commandExists, run, runJson, spawnBackground } from '../process.js'; import { CleanupError, OwnershipError, ProviderUnavailableError } from '../errors.js'; import { canonicalResourceRefs, createMetadata, deriveWorkspaceIdentity, labelHash, providerLabels, readCleanupMetadata, readMetadata, workspaceName, WORKSPACE_RUNTIME } from '../metadata.js'; import { createWorkspaceSecrets, getWorkspaceToken, rotateWorkspaceCredentials, selectGrantedCredentials } from '../auth.js'; -import { requireKubernetesEgress, validateImage } from '../policy.js'; +import { requireKubernetesEgress, validateGatewayImage, validateImage } from '../policy.js'; import { grantedEgressPolicy } from '../egress-domains.js'; import { waitForHttpHealth } from '../health.js'; import { KUBERNETES_TOKEN_FILE, KUBERNETES_TOKEN_MOUNT_PATH, PROVIDER_MODEL_AUTH_FILE, runtimeCommand, runtimeEnvironment } from '../runtime-command.js'; @@ -81,6 +81,17 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { return { provider, available: true, diagnostics: enforcement?.diagnostics ?? [], isolation: enforcement ? { verdict: enforcement.verdict } : null }; } +/** + * The image the isolation probe runs. The probe only needs a runtime that can open a + * TCP connection, so it uses the egress gateway image where managed egress already + * requires it: a quarter the size of the workspace image, which matters because a + * cluster seeing either for the first time must download it before answering. + */ + function isolationProbeImage() { + if (policy.egress.mode === 'managed' && policy.egress.gatewayImage) return validateGatewayImage(policy.egress.gatewayImage); + return validateImage(policy, policy.defaultImage); + } + /** * The address of the cluster's DNS service, which is different on every cluster and is * therefore discovered rather than asked for. The setting remains an override, and is @@ -148,7 +159,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { return { action, namespace, created: !exists }; } if (action === 'check-isolation') { - const result = await checkNetworkPolicyEnforcement(kubectl, { context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: validateImage(policy, policy.defaultImage), force: true }); + const result = await checkNetworkPolicyEnforcement(kubectl, { context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: isolationProbeImage(), force: true }); return { action, verdict: result.verdict, diagnostics: result.diagnostics, imageUnavailable: result.imageUnavailable === true }; } throw new Error(`Unsupported Kubernetes setup action: ${action}`); @@ -166,7 +177,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { // Every isolation guarantee this provider makes rests on the cluster enforcing the // NetworkPolicies it writes, and acceptance of the objects proves nothing. Verified // here rather than in preflight so listing and readiness stay cheap. - const enforcement = await requireNetworkPolicyEnforcement(kubectl, { provider, context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: validateImage(policy, policy.defaultImage) }); + const enforcement = await requireNetworkPolicyEnforcement(kubectl, { provider, context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: isolationProbeImage() }); const meta = readMetadata(info, provider, policy); const identity = identityFromMetadata(meta); const refs = canonicalResourceRefs(meta.providerResourceID, provider, policy);