diff --git a/src/policy.js b/src/policy.js index 2d897c8..84188b0 100644 --- a/src/policy.js +++ b/src/policy.js @@ -196,7 +196,7 @@ export function validateImage(policy, image) { return normalized; } -function validateGatewayImage(image) { +export function validateGatewayImage(image) { const normalized = String(image ?? '').trim(); if (!isPinnedImage(normalized)) throw new PolicyError('Managed egress requires a sha256 digest-pinned gateway image'); return normalized; diff --git a/src/providers/kubernetes.js b/src/providers/kubernetes.js index 2f16eef..b8550a5 100644 --- a/src/providers/kubernetes.js +++ b/src/providers/kubernetes.js @@ -7,7 +7,7 @@ import { commandExists, run, runJson, spawnBackground } from '../process.js'; import { CleanupError, OwnershipError, ProviderUnavailableError } from '../errors.js'; import { canonicalResourceRefs, createMetadata, deriveWorkspaceIdentity, labelHash, providerLabels, readCleanupMetadata, readMetadata, workspaceName, WORKSPACE_RUNTIME } from '../metadata.js'; import { createWorkspaceSecrets, getWorkspaceToken, rotateWorkspaceCredentials, selectGrantedCredentials } from '../auth.js'; -import { requireKubernetesEgress, validateImage } from '../policy.js'; +import { requireKubernetesEgress, validateGatewayImage, validateImage } from '../policy.js'; import { grantedEgressPolicy } from '../egress-domains.js'; import { waitForHttpHealth } from '../health.js'; import { KUBERNETES_TOKEN_FILE, KUBERNETES_TOKEN_MOUNT_PATH, PROVIDER_MODEL_AUTH_FILE, runtimeCommand, runtimeEnvironment } from '../runtime-command.js'; @@ -47,9 +47,19 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { } throw new ProviderUnavailableError(`Kubernetes cluster is not reachable: ${cause instanceof Error ? cause.message : String(cause)}`, { provider, code: 'WORKSPACE_PROVIDER_CLUSTER_UNREACHABLE', cause }); }); - for (const [verb, resource] of requiredPermissions(policy)) { + // Each check is a process spawn and a round trip, and there are two dozen of them: + // run sequentially they dominated the time to display readiness. Every denial is + // collected rather than only the first, so one message names everything to request. + const denied = await mapWithConcurrency(requiredPermissions(policy), RBAC_PROBE_CONCURRENCY, async ([verb, resource]) => { const { stdout } = await kubectl(['auth', 'can-i', verb, resource, '-n', policy.kubernetes.namespace], { timeoutMs: 20_000 }); - if (stdout.trim() !== 'yes') throw new ProviderUnavailableError(`Kubernetes RBAC denies ${verb} ${resource} in namespace ${policy.kubernetes.namespace}`, { provider, code: 'WORKSPACE_PROVIDER_RBAC_DENIED' }); + return stdout.trim() === 'yes' ? null : `${verb} ${resource}`; + }); + const missing = denied.filter(Boolean); + if (missing.length > 0) { + throw new ProviderUnavailableError( + `Kubernetes RBAC denies ${missing.join(', ')} in namespace ${policy.kubernetes.namespace}`, + { provider, code: 'WORKSPACE_PROVIDER_RBAC_DENIED' }, + ); } // Validated after the environment checks because the DNS address is discovered from // the cluster, and an unreachable cluster is a more actionable answer than a policy gap. @@ -71,6 +81,17 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { return { provider, available: true, diagnostics: enforcement?.diagnostics ?? [], isolation: enforcement ? { verdict: enforcement.verdict } : null }; } +/** + * The image the isolation probe runs. The probe only needs a runtime that can open a + * TCP connection, so it uses the egress gateway image where managed egress already + * requires it: a quarter the size of the workspace image, which matters because a + * cluster seeing either for the first time must download it before answering. + */ + function isolationProbeImage() { + if (policy.egress.mode === 'managed' && policy.egress.gatewayImage) return validateGatewayImage(policy.egress.gatewayImage); + return validateImage(policy, policy.defaultImage); + } + /** * The address of the cluster's DNS service, which is different on every cluster and is * therefore discovered rather than asked for. The setting remains an override, and is @@ -138,8 +159,8 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { return { action, namespace, created: !exists }; } if (action === 'check-isolation') { - const result = await checkNetworkPolicyEnforcement(kubectl, { context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: validateImage(policy, policy.defaultImage), force: true }); - return { action, verdict: result.verdict, diagnostics: result.diagnostics }; + const result = await checkNetworkPolicyEnforcement(kubectl, { context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: isolationProbeImage(), force: true }); + return { action, verdict: result.verdict, diagnostics: result.diagnostics, imageUnavailable: result.imageUnavailable === true }; } throw new Error(`Unsupported Kubernetes setup action: ${action}`); } @@ -156,7 +177,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { // Every isolation guarantee this provider makes rests on the cluster enforcing the // NetworkPolicies it writes, and acceptance of the objects proves nothing. Verified // here rather than in preflight so listing and readiness stay cheap. - const enforcement = await requireNetworkPolicyEnforcement(kubectl, { provider, context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: validateImage(policy, policy.defaultImage) }); + const enforcement = await requireNetworkPolicyEnforcement(kubectl, { provider, context: policy.kubernetes.context, namespace: policy.kubernetes.namespace, image: isolationProbeImage() }); const meta = readMetadata(info, provider, policy); const identity = identityFromMetadata(meta); const refs = canonicalResourceRefs(meta.providerResourceID, provider, policy); @@ -593,6 +614,23 @@ function portAvailable(port) { return new Promise((resolve) => { const server = createServer(); server.once('error', () => resolve(false)); server.listen(port, '127.0.0.1', () => server.close(() => resolve(true))); }); } +const RBAC_PROBE_CONCURRENCY = 8; + +/** Runs `task` over `items` with a bounded number in flight, preserving input order. */ +async function mapWithConcurrency(items, limit, task) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await task(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + function requiredPermissions(policy) { const resources = ['pods', 'secrets', 'serviceaccounts', 'persistentvolumeclaims', 'services', 'deployments.apps', 'networkpolicies.networking.k8s.io']; const result = resources.flatMap((resource) => ['create', 'get', 'delete'].map((verb) => [verb, resource])); diff --git a/src/providers/network-policy-enforcement.js b/src/providers/network-policy-enforcement.js index 2c79758..3c9cda1 100644 --- a/src/providers/network-policy-enforcement.js +++ b/src/providers/network-policy-enforcement.js @@ -122,7 +122,12 @@ async function runProbePod(kubectl, manifest, namespace, deadline, now) { const waiting = container?.state?.waiting; // An unpullable image never terminates, so treat it as a probe failure rather than // waiting out the whole deadline for a pod that cannot run. - if (waiting && /ErrImagePull|ImagePullBackOff|InvalidImageName|CreateContainerConfigError/i.test(waiting.reason ?? '')) { + if (waiting && /ErrImagePull|ImagePullBackOff|InvalidImageName/i.test(waiting.reason ?? '')) { + // Distinguished from other start failures because the cause is a setting the + // operator owns, not anything about the cluster's networking. + return { blocked: `the workspace image could not be pulled by the cluster`, imageUnavailable: true }; + } + if (waiting && /CreateContainerConfigError/i.test(waiting.reason ?? '')) { return { blocked: `probe pod ${name} cannot start: ${waiting.reason}${waiting.message ? ` (${waiting.message})` : ''}` }; } if (status.phase === 'Failed' && !terminated) return { blocked: `probe pod ${name} failed: ${status.reason ?? 'unknown reason'}` }; @@ -144,7 +149,7 @@ export async function probeNetworkPolicyEnforcement(kubectl, { namespace, image, try { const baseline = await runProbePod(kubectl, probePod(`${baseName}-baseline`, namespace, { ...labels, 'openchamber.io/probe-role': 'baseline' }, image, 'baseline'), namespace, deadline, now); created.push(['pod', `${baseName}-baseline`]); - if (baseline.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${baseline.blocked}.`] }; + if (baseline.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${baseline.blocked}.`], imageUnavailable: baseline.imageUnavailable === true }; if (baseline.exitCode !== 0) { return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, @@ -156,7 +161,7 @@ export async function probeNetworkPolicyEnforcement(kubectl, { namespace, image, created.push(['networkpolicy', baseName]); const restricted = await runProbePod(kubectl, probePod(`${baseName}-restricted`, namespace, restrictedLabels, image, 'restricted'), namespace, deadline, now); created.push(['pod', `${baseName}-restricted`]); - if (restricted.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${restricted.blocked}.`] }; + if (restricted.blocked) return { verdict: ENFORCEMENT_VERDICTS.INCONCLUSIVE, diagnostics: [`Network isolation could not be verified: ${restricted.blocked}.`], imageUnavailable: restricted.imageUnavailable === true }; if (restricted.exitCode === 0) return { verdict: ENFORCEMENT_VERDICTS.NOT_ENFORCED, diagnostics: [] }; return { verdict: ENFORCEMENT_VERDICTS.ENFORCED, diagnostics: [] }; } finally { diff --git a/src/providers/network-policy-enforcement.test.js b/src/providers/network-policy-enforcement.test.js index d01da2b..fc33583 100644 --- a/src/providers/network-policy-enforcement.test.js +++ b/src/providers/network-policy-enforcement.test.js @@ -69,7 +69,10 @@ describe('kubernetes network policy enforcement probe', () => { const result = await probeNetworkPolicyEnforcement(kubectl, { namespace: 'workspaces', image: IMAGE }); expect(result.verdict).toBe(ENFORCEMENT_VERDICTS.INCONCLUSIVE); - expect(result.diagnostics[0]).toMatch(/ImagePullBackOff/); + // The cause is a setting the operator owns, so it is reported apart from anything + // the probe learned about the cluster's networking. + expect(result.imageUnavailable).toBe(true); + expect(result.diagnostics[0]).toMatch(/image could not be pulled/i); }); it('removes every probe resource it created, including after a verdict of no enforcement', async () => {