fix: ipam not releasing ips on vm deletion

README.md

@@ -30,5 +30,5 @@ Details, one-time machine setup, and caveats: [dev/README.md](dev/README.md)
   - VyOS (values in [dev/README.md](dev/README.md))
   - GitHub
   - Google
-  - Billing meter secret
+  - Internal cron secret
   - Cloudflare email key

apps/cron/.dev.vars.example

@@ -1 +1 @@
-BILLING_METER_SECRET=""
+INTERNAL_CRON_SECRET=""

apps/cron/src/index.ts

@@ -1,27 +1,29 @@
 interface Env {
 	DASHBOARD: Fetcher;
-	BILLING_METER_SECRET: string;
+	INTERNAL_CRON_SECRET: string;
 }
 
 const METER_PATH = '/api/internal/billing/meter';
+const IPAM_RECONCILE_PATH = '/api/internal/ipam/reconcile';
 
-async function runBillingMeter(env: Env): Promise<void> {
-	const response = await env.DASHBOARD.fetch(`https://dashboard.internal${METER_PATH}`, {
+async function runInternalCronRoute(env: Env, path: string, label: string): Promise<void> {
+	const response = await env.DASHBOARD.fetch(`https://dashboard.internal${path}`, {
 		method: 'POST',
-		headers: { authorization: `Bearer ${env.BILLING_METER_SECRET}` }
+		headers: { authorization: `Bearer ${env.INTERNAL_CRON_SECRET}` }
 	});
 
 	const body = await response.text();
 	if (!response.ok) {
-		throw new Error(`billing meter failed: ${response.status} ${response.statusText} ${body}`);
+		throw new Error(`${label} failed: ${response.status} ${response.statusText} ${body}`);
 	}
 
-	console.log(`billing meter ok: ${body}`);
+	console.log(`${label} ok: ${body}`);
 }
 
 export default {
 	async scheduled(_event, env, ctx): Promise<void> {
-		ctx.waitUntil(runBillingMeter(env));
+		ctx.waitUntil(runInternalCronRoute(env, METER_PATH, 'billing meter'));
+		ctx.waitUntil(runInternalCronRoute(env, IPAM_RECONCILE_PATH, 'ipam reconcile'));
 	},
 	async fetch(): Promise<Response> {
 		return new Response('stack-dashboard-cron: billing meter worker', { status: 200 });

apps/dashboard/.env.example

@@ -37,7 +37,9 @@ AUTUMN_ENABLED="false"
 AUTUMN_SECRET=""
 AUTUMN_DEFAULT_PLAN_ID="stack" # This should not be changed.
 AUTUMN_SERVER_ENTITY_FEATURE_ID="active_servers" # This should not be changed.
-BILLING_METER_SECRET="" # random strings
+
+# Internal cron jobs
+INTERNAL_CRON_SECRET="" # random strings
 
 # OAuth — GitHub (https://github.com/settings/developers)
 GITHUB_CLIENT_ID=""

apps/dashboard/src/app.d.ts

@@ -46,7 +46,7 @@ declare global {
 					connectionString: string;
 				};
 				FEATURE_FLAGS?: KVNamespace;
-				BILLING_METER_SECRET?: string;
+				INTERNAL_CRON_SECRET?: string;
 				GITHUB_CLIENT_ID?: string;
 				GITHUB_CLIENT_SECRET?: string;
 				PROXMOX_VPC?: Fetcher;

apps/dashboard/src/lib/remote/admin-users.remote.ts

@@ -284,7 +284,10 @@ async function deleteOrganizationResources(
 	}
 
 	for (const vm of projectVms.filter((item) => item.active)) {
-		const metered = await meterResourceThrough('vm', vm.id);
+		const metered = await meterResourceThrough('vm', vm.id).catch((err) => {
+			console.warn(`Failed to meter VM ${vm.id} during user delete`, err);
+			error(502, `Failed to meter VM "${vm.name}" during user delete`);
+		});
 		if (!metered?.event || metered.syncStatus === 'synced') {
 			await deleteProjectServerEntity(organizationId, vm.id).catch((err) => {
 				console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err);
@@ -295,7 +298,9 @@ async function deleteOrganizationResources(
 
 	await db.delete(volumes).where(eq(volumes.ownerProjectId, organizationId));
 	for (const vm of projectVms) {
-		await releaseVmNetworking(db, vm.id);
+		await releaseVmNetworking(db, vm.id).catch((err) => {
+			console.warn(`Failed to release networking for VM ${vm.id} during user delete`, err);
+		});
 	}
 	if (vmIds.length > 0) {
 		await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds));

apps/dashboard/src/lib/remote/projects.remote.ts

@@ -245,7 +245,10 @@ export const deleteProject = command(deleteParams, async (params) => {
 	}
 
 	for (const vm of projectVms.filter((item) => item.active)) {
-		const metered = await meterResourceThrough('vm', vm.id);
+		const metered = await meterResourceThrough('vm', vm.id).catch((err) => {
+			console.warn(`Failed to meter VM ${vm.id} during project delete`, err);
+			return null;
+		});
 		if (!metered?.event || metered.syncStatus === 'synced') {
 			await deleteProjectServerEntity(params.projectId, vm.id).catch((err) => {
 				console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err);
@@ -256,7 +259,9 @@ export const deleteProject = command(deleteParams, async (params) => {
 
 	await db.delete(volumes).where(eq(volumes.ownerProjectId, params.projectId));
 	for (const vm of projectVms) {
-		await releaseVmNetworking(db, vm.id);
+		await releaseVmNetworking(db, vm.id).catch((err) => {
+			console.warn(`Failed to release networking for VM ${vm.id} during project delete`, err);
+		});
 	}
 	if (vmIds.length > 0) {
 		await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds));

apps/dashboard/src/lib/server/env.ts

@@ -28,7 +28,7 @@ export type RuntimeEnv = {
 		connectionString: string;
 	};
 	FEATURE_FLAGS?: KVNamespace;
-	BILLING_METER_SECRET?: string;
+	INTERNAL_CRON_SECRET?: string;
 	GITHUB_CLIENT_ID?: string;
 	GITHUB_CLIENT_SECRET?: string;
 };
@@ -67,7 +67,7 @@ export function getRuntimeEnv(): RuntimeEnv {
 			AUTUMN_SERVER_ENTITY_FEATURE_ID: platformEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID,
 			HYPERDRIVE: platformEnv.HYPERDRIVE,
 			FEATURE_FLAGS: platformEnv.FEATURE_FLAGS,
-			BILLING_METER_SECRET: platformEnv.BILLING_METER_SECRET,
+			INTERNAL_CRON_SECRET: platformEnv.INTERNAL_CRON_SECRET,
 			GITHUB_CLIENT_ID: platformEnv.GITHUB_CLIENT_ID,
 			GITHUB_CLIENT_SECRET: platformEnv.GITHUB_CLIENT_SECRET
 		};
@@ -95,7 +95,7 @@ export function getRuntimeEnv(): RuntimeEnv {
 		AUTUMN_DEFAULT_PLAN_ID: privateEnv.AUTUMN_DEFAULT_PLAN_ID,
 		AUTUMN_SERVER_ENTITY_FEATURE_ID: privateEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID,
 		DATABASE_URL: required('DATABASE_URL', privateEnv.DATABASE_URL),
-		BILLING_METER_SECRET: privateEnv.BILLING_METER_SECRET,
+		INTERNAL_CRON_SECRET: privateEnv.INTERNAL_CRON_SECRET,
 		GITHUB_CLIENT_ID: privateEnv.GITHUB_CLIENT_ID,
 		GITHUB_CLIENT_SECRET: privateEnv.GITHUB_CLIENT_SECRET
 	};

apps/dashboard/src/lib/server/ipam.ts

@@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit';
 import { Address4, Address6 } from 'ip-address';
 import { and, asc, eq, isNotNull, sql } from 'drizzle-orm';
 import { initDrizzle } from '$lib/server/db';
-import { ipamAllocations, ipamPrefixes } from '$lib/server/db/schema';
+import { ipamAllocations, ipamPrefixes, vms } from '$lib/server/db/schema';
 import { isVyosConfigured, VyosClient } from '$lib/server/vyos';
 
 export type IpFamily = 'ipv4' | 'ipv6';
@@ -798,3 +798,26 @@ export async function releaseVmNetworking(db: QueryableDb, vmId: string) {
 	await deleteVyosDelegatedRoute(allocations);
 	await db.delete(ipamAllocations).where(eq(ipamAllocations.associatedVmId, vmId));
 }
+
+export async function reconcileOrphanedIpamAllocations(db: QueryableDb, limit = 200) {
+	const orphaned = await db
+		.selectDistinct({ vmId: ipamAllocations.associatedVmId })
+		.from(ipamAllocations)
+		.innerJoin(vms, eq(vms.id, ipamAllocations.associatedVmId))
+		.where(eq(vms.active, false))
+		.limit(limit);
+
+	let released = 0;
+	let failed = 0;
+	for (const { vmId } of orphaned) {
+		try {
+			await releaseVmNetworking(db, vmId);
+			released += 1;
+		} catch (err) {
+			failed += 1;
+			console.warn(`Failed to reconcile orphaned IPAM allocations for VM ${vmId}`, err);
+		}
+	}
+
+	return { checked: orphaned.length, released, failed };
+}

apps/dashboard/src/lib/server/vm-deletion.ts

@@ -63,17 +63,20 @@ async function deleteVmResources(row: DeletableVm): Promise<void> {
 			.returning({ id: vms.id });
 		if (claimed.length === 0) return;
 
-		const metered = await meterResourceThrough('vm', row.id);
+		await releaseVmNetworking(db, row.id).catch((err) => {
+			console.warn(`Failed to release networking for VM ${row.id}`, err);
+		});
+
+		const metered = await meterResourceThrough('vm', row.id).catch((err) => {
+			console.warn(`Failed to meter VM ${row.id} during deletion`, err);
+			return null;
+		});
 		if (row.ownerProjectId && (!metered?.event || metered.syncStatus === 'synced')) {
 			await deleteProjectServerEntity(row.ownerProjectId, row.id).catch((err) => {
 				console.warn(`Failed to delete Autumn entity for VM ${row.id}`, err);
 			});
 		}
 
-		await releaseVmNetworking(db, row.id).catch((err) => {
-			console.warn(`Failed to release networking for VM ${row.id}`, err);
-		});
-
 		console.log(`VM ${row.id} deletion completed`);
 	} catch (err) {
 		console.error(`VM ${row.id} deletion cleanup failed:`, err);

apps/dashboard/src/routes/api/internal/billing/meter/+server.ts

@@ -9,8 +9,8 @@ import { enforceProjectBillingGrace } from '$lib/server/billing/enforcement';
 import { getRuntimeEnv } from '$lib/server/env';
 
 export const POST: RequestHandler = async ({ request }) => {
-	const secret = getRuntimeEnv().BILLING_METER_SECRET;
-	if (!secret) error(503, 'Billing meter is not configured');
+	const secret = getRuntimeEnv().INTERNAL_CRON_SECRET;
+	if (!secret) error(503, 'Internal cron is not configured');
 
 	const authorization = request.headers.get('authorization');
 	if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized');

apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts

@@ -0,0 +1,17 @@
+import { error, json } from '@sveltejs/kit';
+import type { RequestHandler } from './$types';
+import { getRuntimeEnv } from '$lib/server/env';
+import { initDrizzle } from '$lib/server/db';
+import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam';
+
+export const POST: RequestHandler = async ({ request }) => {
+	const secret = getRuntimeEnv().INTERNAL_CRON_SECRET;
+	if (!secret) error(503, 'Internal cron is not configured');
+
+	const authorization = request.headers.get('authorization');
+	if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized');
+
+	const ipam = await reconcileOrphanedIpamAllocations(initDrizzle());
+
+	return json({ ipam });
+};