From e6b2da5aaaf7f5812f8d3539c3048144d6eb5f3a Mon Sep 17 00:00:00 2001 From: yulia-ivashko Date: Wed, 5 Aug 2026 23:33:25 +0300 Subject: [PATCH] feat(kubernetes): read cluster facts from the cluster instead of asking for them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kubernetes was unusable until someone supplied `egress.dnsCIDRs` by hand: the policy check rejected an empty list outright, so the provider reported itself unavailable. That value is the cluster's own DNS service address — 10.96.0.10 on kind, 10.43.0.10 on k3s, different again elsewhere — which is machine data the cluster will state on request. Requiring a person to find it blocked the case Kubernetes exists for, connecting to a cluster somebody else runs, and a wrong value breaks name resolution inside the workspace in a way that is very hard to trace back. The provider now resolves it from the `kube-system` service labelled `k8s-app=kube-dns`, the selector every common distribution uses, covering both addresses of a dual-stack service and rejecting a headless one. A configured range stays authoritative and skips discovery entirely. When RBAC hides `kube-system`, the operator is asked — with an explicit code and a message that says what to request — because that is the one case where the machine genuinely cannot answer. Egress validation moves after the environment checks, since discovery needs a reachable cluster and "the cluster is unreachable" is a more actionable answer than "the egress policy is incomplete". Discovered ranges are validated exactly as configured ones are, and cached per context with an expiry so a cluster rebuilt under a familiar name is not answered from a stale entry. Also exposes a read-only view of kubeconfig contexts, so a surface can offer the clusters the host already knows instead of asking for a context name to be typed exactly. Only context names, their namespaces and which is current are returned: a kubeconfig also holds tokens, client certificates and server addresses, and none of that belongs in a settings surface. Verified live against two clusters with no DNS range configured, where the provider previously refused to start at all. --- src/operations.js | 6 ++ src/policy.js | 11 ++- src/providers/kubernetes-dns.test.js | 101 +++++++++++++++++++++++++++ src/providers/kubernetes.js | 71 ++++++++++++++++++- 4 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 src/providers/kubernetes-dns.test.js diff --git a/src/operations.js b/src/operations.js index c6cbe01..66ccf41 100644 --- a/src/operations.js +++ b/src/operations.js @@ -92,6 +92,12 @@ export function createWorkspaceProviderOperations(options = {}) { async validateProvider(kind) { return providers.get(parseProviderKind(kind)).validate(); }, + /** Read-only view of what the host already knows about reaching this provider. */ + async describeProvider(kind) { + const target = providers.get(parseProviderKind(kind)); + if (typeof target.describe !== 'function') return { provider: kind, contexts: [], currentContext: null }; + return target.describe(); + }, /** Completes one setup requirement for a provider that can do so itself. */ async prepareProvider(kind, action) { const target = providers.get(parseProviderKind(kind)); diff --git a/src/policy.js b/src/policy.js index 8a2933c..2d897c8 100644 --- a/src/policy.js +++ b/src/policy.js @@ -83,9 +83,14 @@ export function requireDockerEgress(policy) { throw new PolicyError('Docker external egress requires egress.proxyUrl'); } -export function requireKubernetesEgress(policy) { - if (policy.egress.dnsCIDRs.length === 0) throw new PolicyError('Kubernetes egress requires egress.dnsCIDRs for controlled DNS'); - for (const cidr of policy.egress.dnsCIDRs) validateCIDR(cidr, 'Workspace egress DNS CIDR'); +/** + * @param dnsCIDRs the DNS ranges actually in force — discovered from the cluster when the + * policy leaves them unset, so a discovered value is validated as strictly as a + * configured one. + */ +export function requireKubernetesEgress(policy, dnsCIDRs = policy.egress.dnsCIDRs) { + if (dnsCIDRs.length === 0) throw new PolicyError('Kubernetes egress requires egress.dnsCIDRs for controlled DNS'); + for (const cidr of dnsCIDRs) validateCIDR(cidr, 'Workspace egress DNS CIDR'); if (policy.egress.mode === 'managed') { validateGatewayImage(policy.egress.gatewayImage); return; diff --git a/src/providers/kubernetes-dns.test.js b/src/providers/kubernetes-dns.test.js new file mode 100644 index 0000000..792034f --- /dev/null +++ b/src/providers/kubernetes-dns.test.js @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createKubernetesProvider, resetKubernetesDiscoveryCache } from './kubernetes.js'; + +const BASE_POLICY = { + defaultProvider: 'kubernetes', + defaultImage: 'ghcr.io/openchamber/opencode-workspace@sha256:1111111111111111111111111111111111111111111111111111111111111111', + allowedImages: [], + requirePinnedImage: true, + modelAuth: 'explicit-opencode-auth-content', + egress: { + mode: 'managed', + gatewayImage: 'ghcr.io/openchamber/workspace-egress-gateway@sha256:2222222222222222222222222222222222222222222222222222222222222222', + allowedDomains: ['api.anthropic.com'], + allowedCIDRs: [], + allowedPorts: [443], + dnsCIDRs: [], + gatewayPolicy: { allowedDomains: ['api.anthropic.com'], allowedCIDRs: [], allowedPorts: [443] }, + }, + kubernetes: { + context: 'provided-cluster', namespace: 'workspaces', connectivity: 'port-forward', + storage: '8Gi', cpuRequest: '250m', memoryRequest: '512Mi', cpuLimit: '2', memoryLimit: '4Gi', + ingress: { ingressClassName: '', hostTemplate: '', pathTemplate: '/', tls: { mode: 'existing-secret', secretName: '' }, controllerNamespaceSelector: {}, controllerPodSelector: {}, annotations: {} }, + }, +}; + +/** Answers the preflight sequence so a test only states the DNS services the cluster has. */ +function clusterWith({ dnsServices, dnsError, policy = BASE_POLICY }) { + const calls = []; + const run = vi.fn(async (command, args) => { + calls.push(args.join(' ')); + const joined = args.join(' '); + if (joined.includes('version --client')) return { stdout: 'Client Version: v1.36.1' }; + if (joined.includes('get namespace')) return { stdout: 'namespace/workspaces' }; + if (joined.includes('auth can-i')) return { stdout: 'yes' }; + if (joined.includes('k8s-app=kube-dns')) { + if (dnsError) throw Object.assign(new Error('forbidden'), { stderr: 'services is forbidden' }); + return { stdout: JSON.stringify({ items: dnsServices }) }; + } + throw new Error(`unexpected kubectl call: ${joined}`); + }); + return { run, calls, policy }; +} + +vi.mock('../process.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, run: (...args) => globalThis.__kubectlRun(...args) }; +}); + +const validate = async (fixture) => { + globalThis.__kubectlRun = fixture.run; + return createKubernetesProvider({ policy: fixture.policy, sourceDirectory: process.cwd() }).validate(); +}; + +describe('kubernetes cluster DNS discovery', () => { + beforeEach(() => resetKubernetesDiscoveryCache()); + + it('discovers the cluster DNS address instead of demanding it be configured', async () => { + const fixture = clusterWith({ dnsServices: [{ spec: { clusterIP: '10.96.0.10' } }] }); + + await expect(validate(fixture)).resolves.toMatchObject({ available: true }); + expect(fixture.calls.some((call) => call.includes('k8s-app=kube-dns'))).toBe(true); + }); + + it('covers both addresses of a dual-stack DNS service', async () => { + const fixture = clusterWith({ dnsServices: [{ spec: { clusterIPs: ['10.96.0.10', 'fd00::10'] } }] }); + + await expect(validate(fixture)).resolves.toMatchObject({ available: true }); + }); + + it('keeps a configured range authoritative and does not query the cluster for it', async () => { + const policy = { ...BASE_POLICY, egress: { ...BASE_POLICY.egress, dnsCIDRs: ['10.43.0.10/32'] } }; + const fixture = clusterWith({ dnsServices: [{ spec: { clusterIP: '10.96.0.10' } }], policy }); + + await expect(validate(fixture)).resolves.toMatchObject({ available: true }); + expect(fixture.calls.some((call) => call.includes('k8s-app=kube-dns'))).toBe(false); + }); + + it('asks for the address only when the cluster will not reveal it', async () => { + const fixture = clusterWith({ dnsServices: [], dnsError: true }); + + await expect(validate(fixture)).rejects.toMatchObject({ code: 'WORKSPACE_PROVIDER_DNS_UNRESOLVED' }); + }); + + it('treats a headless DNS service as no answer rather than a usable range', async () => { + const fixture = clusterWith({ dnsServices: [{ spec: { clusterIP: 'None' } }] }); + + await expect(validate(fixture)).rejects.toMatchObject({ code: 'WORKSPACE_PROVIDER_DNS_UNRESOLVED' }); + }); + + it('reports an unreachable cluster before complaining about the egress policy', async () => { + const fixture = clusterWith({ dnsServices: [] }); + fixture.run.mockImplementation(async (command, args) => { + const joined = args.join(' '); + if (joined.includes('version --client')) return { stdout: 'Client Version: v1.36.1' }; + if (joined.includes('get namespace')) throw Object.assign(new Error('connection refused'), { stderr: 'Unable to connect to the server' }); + throw new Error(`unexpected kubectl call: ${joined}`); + }); + + await expect(validate(fixture)).rejects.toMatchObject({ code: 'WORKSPACE_PROVIDER_CLUSTER_UNREACHABLE' }); + }); +}); diff --git a/src/providers/kubernetes.js b/src/providers/kubernetes.js index edb273c..2f16eef 100644 --- a/src/providers/kubernetes.js +++ b/src/providers/kubernetes.js @@ -18,6 +18,15 @@ import { ARTIFACT_LIMITS, RUNTIME_ARTIFACT_SCRIPT } from '../artifact.js'; import { checkNetworkPolicyEnforcement, lastEnforcementVerdict, requireNetworkPolicyEnforcement } from './network-policy-enforcement.js'; const portForwards = new Map(); +// Cluster DNS is a property of the cluster, so it is cached per kubeconfig context, and +// expires so a cluster that is rebuilt under the same context name is not answered from +// a stale entry for the lifetime of the process. +const DNS_CACHE_TTL_MS = 10 * 60 * 1000; +const dnsCIDRCache = new Map(); + +export function resetKubernetesDiscoveryCache() { + dnsCIDRCache.clear(); +} export const KUBERNETES_SEED_EXTRACT_COMMAND = `set -eu; cat > /tmp/source.tar; tar --no-same-owner --no-overwrite-dir --strip-components=1 -xf /tmp/source.tar -C "$1"; mkdir -p "$1/.openchamber-runtime"; printf '%s' "$2" > "$1/.openchamber-runtime/source-generation"`; @@ -31,7 +40,6 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { // gap first hides the real reason this host cannot run Kubernetes workspaces. await kubectl(['version', '--client=true'], { timeoutMs: 15_000 }).catch((cause) => { throw new ProviderUnavailableError('kubectl is not available', { provider, code: 'WORKSPACE_PROVIDER_CLI_MISSING', cause }); }); if (!kubernetesConfigured(policy)) throw new ProviderUnavailableError('No Kubernetes configuration was found for this host', { provider, code: 'WORKSPACE_PROVIDER_NOT_CONFIGURED' }); - requireKubernetesEgress(policy); await kubectl(['get', 'namespace', policy.kubernetes.namespace, '-o', 'name'], { timeoutMs: 20_000 }).catch((cause) => { const text = `${cause?.stderr ?? ''} ${cause instanceof Error ? cause.message : String(cause)}`; if (/NotFound|not found/i.test(text)) { @@ -43,6 +51,9 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { const { stdout } = await kubectl(['auth', 'can-i', verb, resource, '-n', policy.kubernetes.namespace], { timeoutMs: 20_000 }); if (stdout.trim() !== 'yes') throw new ProviderUnavailableError(`Kubernetes RBAC denies ${verb} ${resource} in namespace ${policy.kubernetes.namespace}`, { provider, code: 'WORKSPACE_PROVIDER_RBAC_DENIED' }); } + // Validated after the environment checks because the DNS address is discovered from + // the cluster, and an unreachable cluster is a more actionable answer than a policy gap. + requireKubernetesEgress(policy, await resolveDnsCIDRs()); if (policy.kubernetes.connectivity === 'ingress') { await kubectl(['get', 'ingressclass', policy.kubernetes.ingress.ingressClassName, '-o', 'name'], { timeoutMs: 20_000 }); if (policy.kubernetes.ingress.tls.mode === 'existing-secret') await kubectl(['get', 'secret', policy.kubernetes.ingress.tls.secretName, '-n', policy.kubernetes.namespace, '-o', 'name'], { timeoutMs: 20_000 }); @@ -60,6 +71,60 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { return { provider, available: true, diagnostics: enforcement?.diagnostics ?? [], isolation: enforcement ? { verdict: enforcement.verdict } : null }; } + /** + * The address of the cluster's DNS service, which is different on every cluster and is + * therefore discovered rather than asked for. The setting remains an override, and is + * the only route left when RBAC hides `kube-system` from this account. + */ + async function resolveDnsCIDRs() { + if (policy.egress.dnsCIDRs.length > 0) return policy.egress.dnsCIDRs; + const cached = dnsCIDRCache.get(policy.kubernetes.context ?? ''); + if (cached && cached.expiresAt > Date.now()) return cached.cidrs; + let discovered = []; + try { + const { stdout } = await kubectl(['get', 'service', '-n', 'kube-system', '-l', 'k8s-app=kube-dns', '-o', 'json'], { timeoutMs: 20_000 }); + const services = JSON.parse(stdout).items ?? []; + const addresses = services.flatMap((item) => item?.spec?.clusterIPs ?? (item?.spec?.clusterIP ? [item.spec.clusterIP] : [])); + discovered = addresses.filter((address) => typeof address === 'string' && address && address !== 'None') + .map((address) => (address.includes(':') ? `${address}/128` : `${address}/32`)); + } catch { + discovered = []; + } + if (discovered.length === 0) { + throw new ProviderUnavailableError( + 'The cluster DNS address could not be determined, and no DNS range is configured. Ask your cluster administrator for the DNS service address and set it under Advanced.', + { provider, code: 'WORKSPACE_PROVIDER_DNS_UNRESOLVED' }, + ); + } + dnsCIDRCache.set(policy.kubernetes.context ?? '', { cidrs: discovered, expiresAt: Date.now() + DNS_CACHE_TTL_MS }); + return discovered; + } + + /** + * What the host already knows about reaching a cluster. kubeconfig is where the + * industry keeps this, and it binds cluster and namespace together, so it is read + * rather than retyped. Only names travel: a kubeconfig also holds tokens, client + * certificates and server addresses, and none of that belongs in a settings surface. + */ + async function describe() { + if (!kubernetesConfigured(policy)) return { provider, contexts: [], currentContext: null }; + try { + const { stdout } = await run('kubectl', ['config', 'view', '-o', 'json'], { timeoutMs: 20_000 }); + const config = JSON.parse(stdout); + const currentContext = typeof config['current-context'] === 'string' ? config['current-context'] : null; + const contexts = (Array.isArray(config.contexts) ? config.contexts : []) + .filter((entry) => typeof entry?.name === 'string' && entry.name) + .map((entry) => ({ + name: entry.name, + namespace: typeof entry.context?.namespace === 'string' ? entry.context.namespace : null, + current: entry.name === currentContext, + })); + return { provider, contexts, currentContext }; + } catch { + return { provider, contexts: [], currentContext: null }; + } + } + /** * Completes a setup requirement on the operator's behalf. Kept separate from preflight * so that inspecting readiness never changes the cluster. @@ -106,7 +171,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { const secrets = await createWorkspaceSecrets(meta.providerResourceID, grantedCredentials); const hostPort = await availableStablePort(meta.providerResourceID); await transaction.update({ hostPort, imageDigest: image }); - const manifests = buildManifests({ identity, refs, image, policy: { ...policy, egress: grantedEgressPolicy(policy, grantedCredentials) }, token: secrets.token, modelAuth: secrets.modelAuth }); + const manifests = buildManifests({ identity, refs, image, policy: { ...policy, egress: { ...grantedEgressPolicy(policy, grantedCredentials), dnsCIDRs: await resolveDnsCIDRs() } }, token: secrets.token, modelAuth: secrets.modelAuth }); for (const manifest of manifests.infrastructure) { const resource = `${manifest.kind.toLowerCase()}:${manifest.metadata.name}`; await transaction.create(resource, () => createManifest(kubectl, manifest), async () => { @@ -242,7 +307,7 @@ export function createKubernetesProvider({ policy, sourceDirectory }) { }); } - return { kind: provider, configure, create, target, remove, list, health, exportWorkspace, reconcile, rotateCredentials, validate: preflight, setup }; + return { kind: provider, configure, create, target, remove, list, health, exportWorkspace, reconcile, rotateCredentials, validate: preflight, setup, describe }; } export function buildManifests({ identity, refs, image, policy, token, modelAuth }) {