From 2d433fc7bd7289e156ae86600b56aa0bf49303c2 Mon Sep 17 00:00:00 2001 From: june-fish Date: Mon, 6 Jul 2026 15:28:08 -0700 Subject: [PATCH 1/5] fix: ip addresses not released on vm deletion --- apps/dashboard/src/lib/remote/admin-users.remote.ts | 9 +++++++-- apps/dashboard/src/lib/remote/projects.remote.ts | 9 +++++++-- apps/dashboard/src/lib/server/vm-deletion.ts | 13 ++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apps/dashboard/src/lib/remote/admin-users.remote.ts b/apps/dashboard/src/lib/remote/admin-users.remote.ts index 5cb26b5..4d0cc2b 100644 --- a/apps/dashboard/src/lib/remote/admin-users.remote.ts +++ b/apps/dashboard/src/lib/remote/admin-users.remote.ts @@ -284,7 +284,10 @@ async function deleteOrganizationResources( } for (const vm of projectVms.filter((item) => item.active)) { - const metered = await meterResourceThrough('vm', vm.id); + const metered = await meterResourceThrough('vm', vm.id).catch((err) => { + console.warn(`Failed to meter VM ${vm.id} during user delete`, err); + error(502, `Failed to meter VM "${vm.name}" during user delete`); + }); if (!metered?.event || metered.syncStatus === 'synced') { await deleteProjectServerEntity(organizationId, vm.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err); @@ -295,7 +298,9 @@ async function deleteOrganizationResources( await db.delete(volumes).where(eq(volumes.ownerProjectId, organizationId)); for (const vm of projectVms) { - await releaseVmNetworking(db, vm.id); + await releaseVmNetworking(db, vm.id).catch((err) => { + console.warn(`Failed to release networking for VM ${vm.id} during user delete`, err); + }); } if (vmIds.length > 0) { await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds)); diff --git a/apps/dashboard/src/lib/remote/projects.remote.ts b/apps/dashboard/src/lib/remote/projects.remote.ts index a76e6ce..3d6a4ce 100644 --- a/apps/dashboard/src/lib/remote/projects.remote.ts +++ b/apps/dashboard/src/lib/remote/projects.remote.ts @@ -245,7 +245,10 @@ export const deleteProject = command(deleteParams, async (params) => { } for (const vm of projectVms.filter((item) => item.active)) { - const metered = await meterResourceThrough('vm', vm.id); + const metered = await meterResourceThrough('vm', vm.id).catch((err) => { + console.warn(`Failed to meter VM ${vm.id} during project delete`, err); + return null; + }); if (!metered?.event || metered.syncStatus === 'synced') { await deleteProjectServerEntity(params.projectId, vm.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err); @@ -256,7 +259,9 @@ export const deleteProject = command(deleteParams, async (params) => { await db.delete(volumes).where(eq(volumes.ownerProjectId, params.projectId)); for (const vm of projectVms) { - await releaseVmNetworking(db, vm.id); + await releaseVmNetworking(db, vm.id).catch((err) => { + console.warn(`Failed to release networking for VM ${vm.id} during project delete`, err); + }); } if (vmIds.length > 0) { await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds)); diff --git a/apps/dashboard/src/lib/server/vm-deletion.ts b/apps/dashboard/src/lib/server/vm-deletion.ts index 0a36bb1..8e38ba1 100644 --- a/apps/dashboard/src/lib/server/vm-deletion.ts +++ b/apps/dashboard/src/lib/server/vm-deletion.ts @@ -63,17 +63,20 @@ async function deleteVmResources(row: DeletableVm): Promise { .returning({ id: vms.id }); if (claimed.length === 0) return; - const metered = await meterResourceThrough('vm', row.id); + await releaseVmNetworking(db, row.id).catch((err) => { + console.warn(`Failed to release networking for VM ${row.id}`, err); + }); + + const metered = await meterResourceThrough('vm', row.id).catch((err) => { + console.warn(`Failed to meter VM ${row.id} during deletion`, err); + return null; + }); if (row.ownerProjectId && (!metered?.event || metered.syncStatus === 'synced')) { await deleteProjectServerEntity(row.ownerProjectId, row.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${row.id}`, err); }); } - await releaseVmNetworking(db, row.id).catch((err) => { - console.warn(`Failed to release networking for VM ${row.id}`, err); - }); - console.log(`VM ${row.id} deletion completed`); } catch (err) { console.error(`VM ${row.id} deletion cleanup failed:`, err); From 38913be2385eb49acb42b52e03548e6564d2535b Mon Sep 17 00:00:00 2001 From: june-fish Date: Mon, 6 Jul 2026 15:29:16 -0700 Subject: [PATCH 2/5] feat: reconcileOrphanedIpamAllocations --- apps/dashboard/src/lib/server/ipam.ts | 27 +++++++++++++++++-- .../api/internal/billing/meter/+server.ts | 8 +++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/lib/server/ipam.ts b/apps/dashboard/src/lib/server/ipam.ts index cde8c11..8bccbc6 100644 --- a/apps/dashboard/src/lib/server/ipam.ts +++ b/apps/dashboard/src/lib/server/ipam.ts @@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit'; import { Address4, Address6 } from 'ip-address'; import { and, asc, eq, isNotNull, sql } from 'drizzle-orm'; import { initDrizzle } from '$lib/server/db'; -import { ipamAllocations, ipamPrefixes } from '$lib/server/db/schema'; +import { ipamAllocations, ipamPrefixes, vms } from '$lib/server/db/schema'; import { isVyosConfigured, VyosClient } from '$lib/server/vyos'; export type IpFamily = 'ipv4' | 'ipv6'; @@ -788,7 +788,7 @@ export async function allocateVmNetworking( return allocations; } catch (err) { - await releaseVmNetworking(db, params.vmId).catch(() => {}); + await releaseVmNetworking(db, params.vmId).catch(() => { }); throw err; } } @@ -798,3 +798,26 @@ export async function releaseVmNetworking(db: QueryableDb, vmId: string) { await deleteVyosDelegatedRoute(allocations); await db.delete(ipamAllocations).where(eq(ipamAllocations.associatedVmId, vmId)); } + +export async function reconcileOrphanedIpamAllocations(db: QueryableDb, limit = 200) { + const orphaned = await db + .selectDistinct({ vmId: ipamAllocations.associatedVmId }) + .from(ipamAllocations) + .innerJoin(vms, eq(vms.id, ipamAllocations.associatedVmId)) + .where(eq(vms.active, false)) + .limit(limit); + + let released = 0; + let failed = 0; + for (const { vmId } of orphaned) { + try { + await releaseVmNetworking(db, vmId); + released += 1; + } catch (err) { + failed += 1; + console.warn(`Failed to reconcile orphaned IPAM allocations for VM ${vmId}`, err); + } + } + + return { checked: orphaned.length, released, failed }; +} diff --git a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts index 043e960..3192723 100644 --- a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts +++ b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts @@ -7,6 +7,8 @@ import { } from '$lib/server/billing/autumn'; import { enforceProjectBillingGrace } from '$lib/server/billing/enforcement'; import { getRuntimeEnv } from '$lib/server/env'; +import { initDrizzle } from '$lib/server/db'; +import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam'; export const POST: RequestHandler = async ({ request }) => { const secret = getRuntimeEnv().BILLING_METER_SECRET; @@ -24,6 +26,10 @@ export const POST: RequestHandler = async ({ request }) => { console.error('Billing grace enforcement failed', err); return { checked: 0, suspended: 0, failed: true }; }); + const ipam = await reconcileOrphanedIpamAllocations(initDrizzle()).catch((err) => { + console.error('IPAM reconciliation failed', err); + return { checked: 0, released: 0, failed: 0 }; + }); - return json({ metered, synced, cancellations, enforcement }); + return json({ metered, synced, cancellations, enforcement, ipam }); }; From f7512faa36947d9613fb84ef3c9356b3e80789f5 Mon Sep 17 00:00:00 2001 From: june-fish Date: Mon, 6 Jul 2026 15:45:47 -0700 Subject: [PATCH 3/5] refactor: split off ipam reconcile --- apps/cron/src/index.ts | 12 +++++++----- .../api/internal/billing/meter/+server.ts | 8 +------- .../api/internal/ipam/reconcile/+server.ts | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts diff --git a/apps/cron/src/index.ts b/apps/cron/src/index.ts index 2329a98..aa2f0bc 100644 --- a/apps/cron/src/index.ts +++ b/apps/cron/src/index.ts @@ -4,24 +4,26 @@ interface Env { } const METER_PATH = '/api/internal/billing/meter'; +const IPAM_RECONCILE_PATH = '/api/internal/ipam/reconcile'; -async function runBillingMeter(env: Env): Promise { - const response = await env.DASHBOARD.fetch(`https://dashboard.internal${METER_PATH}`, { +async function runInternalCronRoute(env: Env, path: string, label: string): Promise { + const response = await env.DASHBOARD.fetch(`https://dashboard.internal${path}`, { method: 'POST', headers: { authorization: `Bearer ${env.BILLING_METER_SECRET}` } }); const body = await response.text(); if (!response.ok) { - throw new Error(`billing meter failed: ${response.status} ${response.statusText} ${body}`); + throw new Error(`${label} failed: ${response.status} ${response.statusText} ${body}`); } - console.log(`billing meter ok: ${body}`); + console.log(`${label} ok: ${body}`); } export default { async scheduled(_event, env, ctx): Promise { - ctx.waitUntil(runBillingMeter(env)); + ctx.waitUntil(runInternalCronRoute(env, METER_PATH, 'billing meter')); + ctx.waitUntil(runInternalCronRoute(env, IPAM_RECONCILE_PATH, 'ipam reconcile')); }, async fetch(): Promise { return new Response('stack-dashboard-cron: billing meter worker', { status: 200 }); diff --git a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts index 3192723..043e960 100644 --- a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts +++ b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts @@ -7,8 +7,6 @@ import { } from '$lib/server/billing/autumn'; import { enforceProjectBillingGrace } from '$lib/server/billing/enforcement'; import { getRuntimeEnv } from '$lib/server/env'; -import { initDrizzle } from '$lib/server/db'; -import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam'; export const POST: RequestHandler = async ({ request }) => { const secret = getRuntimeEnv().BILLING_METER_SECRET; @@ -26,10 +24,6 @@ export const POST: RequestHandler = async ({ request }) => { console.error('Billing grace enforcement failed', err); return { checked: 0, suspended: 0, failed: true }; }); - const ipam = await reconcileOrphanedIpamAllocations(initDrizzle()).catch((err) => { - console.error('IPAM reconciliation failed', err); - return { checked: 0, released: 0, failed: 0 }; - }); - return json({ metered, synced, cancellations, enforcement, ipam }); + return json({ metered, synced, cancellations, enforcement }); }; diff --git a/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts new file mode 100644 index 0000000..fa0add0 --- /dev/null +++ b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts @@ -0,0 +1,17 @@ +import { error, json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getRuntimeEnv } from '$lib/server/env'; +import { initDrizzle } from '$lib/server/db'; +import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam'; + +export const POST: RequestHandler = async ({ request }) => { + const secret = getRuntimeEnv().BILLING_METER_SECRET; + if (!secret) error(503, 'Internal cron is not configured'); + + const authorization = request.headers.get('authorization'); + if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized'); + + const ipam = await reconcileOrphanedIpamAllocations(initDrizzle()); + + return json({ ipam }); +}; From 960568272c16daa3135530e9e871cc57bd443f9c Mon Sep 17 00:00:00 2001 From: june-fish Date: Mon, 6 Jul 2026 15:54:05 -0700 Subject: [PATCH 4/5] refactor: change cron secret name TODO: wrangler secret put INTERNAL_CRON_SECRET with the same value and remove the old value --- README.md | 2 +- apps/cron/.dev.vars.example | 2 +- apps/cron/src/index.ts | 4 ++-- apps/dashboard/.env.example | 4 +++- apps/dashboard/src/app.d.ts | 2 +- apps/dashboard/src/lib/server/env.ts | 6 +++--- .../src/routes/api/internal/billing/meter/+server.ts | 4 ++-- .../src/routes/api/internal/ipam/reconcile/+server.ts | 2 +- 8 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2e8b230..6ac834e 100644 --- a/README.md +++ b/README.md @@ -30,5 +30,5 @@ Details, one-time machine setup, and caveats: [dev/README.md](dev/README.md) - VyOS (values in [dev/README.md](dev/README.md)) - GitHub - Google - - Billing meter secret + - Internal cron secret - Cloudflare email key diff --git a/apps/cron/.dev.vars.example b/apps/cron/.dev.vars.example index ea83cb4..7750365 100644 --- a/apps/cron/.dev.vars.example +++ b/apps/cron/.dev.vars.example @@ -1 +1 @@ -BILLING_METER_SECRET="" +INTERNAL_CRON_SECRET="" diff --git a/apps/cron/src/index.ts b/apps/cron/src/index.ts index aa2f0bc..df07f3a 100644 --- a/apps/cron/src/index.ts +++ b/apps/cron/src/index.ts @@ -1,6 +1,6 @@ interface Env { DASHBOARD: Fetcher; - BILLING_METER_SECRET: string; + INTERNAL_CRON_SECRET: string; } const METER_PATH = '/api/internal/billing/meter'; @@ -9,7 +9,7 @@ const IPAM_RECONCILE_PATH = '/api/internal/ipam/reconcile'; async function runInternalCronRoute(env: Env, path: string, label: string): Promise { const response = await env.DASHBOARD.fetch(`https://dashboard.internal${path}`, { method: 'POST', - headers: { authorization: `Bearer ${env.BILLING_METER_SECRET}` } + headers: { authorization: `Bearer ${env.INTERNAL_CRON_SECRET}` } }); const body = await response.text(); diff --git a/apps/dashboard/.env.example b/apps/dashboard/.env.example index 51d99fe..ab564cf 100644 --- a/apps/dashboard/.env.example +++ b/apps/dashboard/.env.example @@ -37,7 +37,9 @@ AUTUMN_ENABLED="false" AUTUMN_SECRET="" AUTUMN_DEFAULT_PLAN_ID="stack" # This should not be changed. AUTUMN_SERVER_ENTITY_FEATURE_ID="active_servers" # This should not be changed. -BILLING_METER_SECRET="" # random strings + +# Internal cron jobs +INTERNAL_CRON_SECRET="" # random strings # OAuth — GitHub (https://github.com/settings/developers) GITHUB_CLIENT_ID="" diff --git a/apps/dashboard/src/app.d.ts b/apps/dashboard/src/app.d.ts index f56e247..974dd98 100644 --- a/apps/dashboard/src/app.d.ts +++ b/apps/dashboard/src/app.d.ts @@ -46,7 +46,7 @@ declare global { connectionString: string; }; FEATURE_FLAGS?: KVNamespace; - BILLING_METER_SECRET?: string; + INTERNAL_CRON_SECRET?: string; GITHUB_CLIENT_ID?: string; GITHUB_CLIENT_SECRET?: string; PROXMOX_VPC?: Fetcher; diff --git a/apps/dashboard/src/lib/server/env.ts b/apps/dashboard/src/lib/server/env.ts index 5a3e645..3b4b8a0 100644 --- a/apps/dashboard/src/lib/server/env.ts +++ b/apps/dashboard/src/lib/server/env.ts @@ -28,7 +28,7 @@ export type RuntimeEnv = { connectionString: string; }; FEATURE_FLAGS?: KVNamespace; - BILLING_METER_SECRET?: string; + INTERNAL_CRON_SECRET?: string; GITHUB_CLIENT_ID?: string; GITHUB_CLIENT_SECRET?: string; }; @@ -67,7 +67,7 @@ export function getRuntimeEnv(): RuntimeEnv { AUTUMN_SERVER_ENTITY_FEATURE_ID: platformEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID, HYPERDRIVE: platformEnv.HYPERDRIVE, FEATURE_FLAGS: platformEnv.FEATURE_FLAGS, - BILLING_METER_SECRET: platformEnv.BILLING_METER_SECRET, + INTERNAL_CRON_SECRET: platformEnv.INTERNAL_CRON_SECRET, GITHUB_CLIENT_ID: platformEnv.GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET: platformEnv.GITHUB_CLIENT_SECRET }; @@ -95,7 +95,7 @@ export function getRuntimeEnv(): RuntimeEnv { AUTUMN_DEFAULT_PLAN_ID: privateEnv.AUTUMN_DEFAULT_PLAN_ID, AUTUMN_SERVER_ENTITY_FEATURE_ID: privateEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID, DATABASE_URL: required('DATABASE_URL', privateEnv.DATABASE_URL), - BILLING_METER_SECRET: privateEnv.BILLING_METER_SECRET, + INTERNAL_CRON_SECRET: privateEnv.INTERNAL_CRON_SECRET, GITHUB_CLIENT_ID: privateEnv.GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET: privateEnv.GITHUB_CLIENT_SECRET }; diff --git a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts index 043e960..4c108d6 100644 --- a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts +++ b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts @@ -9,8 +9,8 @@ import { enforceProjectBillingGrace } from '$lib/server/billing/enforcement'; import { getRuntimeEnv } from '$lib/server/env'; export const POST: RequestHandler = async ({ request }) => { - const secret = getRuntimeEnv().BILLING_METER_SECRET; - if (!secret) error(503, 'Billing meter is not configured'); + const secret = getRuntimeEnv().INTERNAL_CRON_SECRET; + if (!secret) error(503, 'Internal cron is not configured'); const authorization = request.headers.get('authorization'); if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized'); diff --git a/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts index fa0add0..08f7f77 100644 --- a/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts +++ b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts @@ -5,7 +5,7 @@ import { initDrizzle } from '$lib/server/db'; import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam'; export const POST: RequestHandler = async ({ request }) => { - const secret = getRuntimeEnv().BILLING_METER_SECRET; + const secret = getRuntimeEnv().INTERNAL_CRON_SECRET; if (!secret) error(503, 'Internal cron is not configured'); const authorization = request.headers.get('authorization'); From 4fe9cc7ef4f670e42f08fc9bf2375b7497e3d7e4 Mon Sep 17 00:00:00 2001 From: june-fish Date: Tue, 14 Jul 2026 19:53:54 -0500 Subject: [PATCH 5/5] lint --- apps/dashboard/src/lib/server/ipam.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/lib/server/ipam.ts b/apps/dashboard/src/lib/server/ipam.ts index 8bccbc6..15ed61a 100644 --- a/apps/dashboard/src/lib/server/ipam.ts +++ b/apps/dashboard/src/lib/server/ipam.ts @@ -788,7 +788,7 @@ export async function allocateVmNetworking( return allocations; } catch (err) { - await releaseVmNetworking(db, params.vmId).catch(() => { }); + await releaseVmNetworking(db, params.vmId).catch(() => {}); throw err; } }