diff --git a/README.md b/README.md index 2e8b230..6ac834e 100644 --- a/README.md +++ b/README.md @@ -30,5 +30,5 @@ Details, one-time machine setup, and caveats: [dev/README.md](dev/README.md) - VyOS (values in [dev/README.md](dev/README.md)) - GitHub - Google - - Billing meter secret + - Internal cron secret - Cloudflare email key diff --git a/apps/cron/.dev.vars.example b/apps/cron/.dev.vars.example index ea83cb4..7750365 100644 --- a/apps/cron/.dev.vars.example +++ b/apps/cron/.dev.vars.example @@ -1 +1 @@ -BILLING_METER_SECRET="" +INTERNAL_CRON_SECRET="" diff --git a/apps/cron/src/index.ts b/apps/cron/src/index.ts index 2329a98..df07f3a 100644 --- a/apps/cron/src/index.ts +++ b/apps/cron/src/index.ts @@ -1,27 +1,29 @@ interface Env { DASHBOARD: Fetcher; - BILLING_METER_SECRET: string; + INTERNAL_CRON_SECRET: string; } const METER_PATH = '/api/internal/billing/meter'; +const IPAM_RECONCILE_PATH = '/api/internal/ipam/reconcile'; -async function runBillingMeter(env: Env): Promise { - const response = await env.DASHBOARD.fetch(`https://dashboard.internal${METER_PATH}`, { +async function runInternalCronRoute(env: Env, path: string, label: string): Promise { + const response = await env.DASHBOARD.fetch(`https://dashboard.internal${path}`, { method: 'POST', - headers: { authorization: `Bearer ${env.BILLING_METER_SECRET}` } + headers: { authorization: `Bearer ${env.INTERNAL_CRON_SECRET}` } }); const body = await response.text(); if (!response.ok) { - throw new Error(`billing meter failed: ${response.status} ${response.statusText} ${body}`); + throw new Error(`${label} failed: ${response.status} ${response.statusText} ${body}`); } - console.log(`billing meter ok: ${body}`); + console.log(`${label} ok: ${body}`); } export default { async scheduled(_event, env, ctx): Promise { - ctx.waitUntil(runBillingMeter(env)); + ctx.waitUntil(runInternalCronRoute(env, METER_PATH, 'billing meter')); + ctx.waitUntil(runInternalCronRoute(env, IPAM_RECONCILE_PATH, 'ipam reconcile')); }, async fetch(): Promise { return new Response('stack-dashboard-cron: billing meter worker', { status: 200 }); diff --git a/apps/dashboard/.env.example b/apps/dashboard/.env.example index 51d99fe..ab564cf 100644 --- a/apps/dashboard/.env.example +++ b/apps/dashboard/.env.example @@ -37,7 +37,9 @@ AUTUMN_ENABLED="false" AUTUMN_SECRET="" AUTUMN_DEFAULT_PLAN_ID="stack" # This should not be changed. AUTUMN_SERVER_ENTITY_FEATURE_ID="active_servers" # This should not be changed. -BILLING_METER_SECRET="" # random strings + +# Internal cron jobs +INTERNAL_CRON_SECRET="" # random strings # OAuth — GitHub (https://github.com/settings/developers) GITHUB_CLIENT_ID="" diff --git a/apps/dashboard/src/app.d.ts b/apps/dashboard/src/app.d.ts index f56e247..974dd98 100644 --- a/apps/dashboard/src/app.d.ts +++ b/apps/dashboard/src/app.d.ts @@ -46,7 +46,7 @@ declare global { connectionString: string; }; FEATURE_FLAGS?: KVNamespace; - BILLING_METER_SECRET?: string; + INTERNAL_CRON_SECRET?: string; GITHUB_CLIENT_ID?: string; GITHUB_CLIENT_SECRET?: string; PROXMOX_VPC?: Fetcher; diff --git a/apps/dashboard/src/lib/remote/admin-users.remote.ts b/apps/dashboard/src/lib/remote/admin-users.remote.ts index 5cb26b5..4d0cc2b 100644 --- a/apps/dashboard/src/lib/remote/admin-users.remote.ts +++ b/apps/dashboard/src/lib/remote/admin-users.remote.ts @@ -284,7 +284,10 @@ async function deleteOrganizationResources( } for (const vm of projectVms.filter((item) => item.active)) { - const metered = await meterResourceThrough('vm', vm.id); + const metered = await meterResourceThrough('vm', vm.id).catch((err) => { + console.warn(`Failed to meter VM ${vm.id} during user delete`, err); + error(502, `Failed to meter VM "${vm.name}" during user delete`); + }); if (!metered?.event || metered.syncStatus === 'synced') { await deleteProjectServerEntity(organizationId, vm.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err); @@ -295,7 +298,9 @@ async function deleteOrganizationResources( await db.delete(volumes).where(eq(volumes.ownerProjectId, organizationId)); for (const vm of projectVms) { - await releaseVmNetworking(db, vm.id); + await releaseVmNetworking(db, vm.id).catch((err) => { + console.warn(`Failed to release networking for VM ${vm.id} during user delete`, err); + }); } if (vmIds.length > 0) { await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds)); diff --git a/apps/dashboard/src/lib/remote/projects.remote.ts b/apps/dashboard/src/lib/remote/projects.remote.ts index a76e6ce..3d6a4ce 100644 --- a/apps/dashboard/src/lib/remote/projects.remote.ts +++ b/apps/dashboard/src/lib/remote/projects.remote.ts @@ -245,7 +245,10 @@ export const deleteProject = command(deleteParams, async (params) => { } for (const vm of projectVms.filter((item) => item.active)) { - const metered = await meterResourceThrough('vm', vm.id); + const metered = await meterResourceThrough('vm', vm.id).catch((err) => { + console.warn(`Failed to meter VM ${vm.id} during project delete`, err); + return null; + }); if (!metered?.event || metered.syncStatus === 'synced') { await deleteProjectServerEntity(params.projectId, vm.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${vm.id}`, err); @@ -256,7 +259,9 @@ export const deleteProject = command(deleteParams, async (params) => { await db.delete(volumes).where(eq(volumes.ownerProjectId, params.projectId)); for (const vm of projectVms) { - await releaseVmNetworking(db, vm.id); + await releaseVmNetworking(db, vm.id).catch((err) => { + console.warn(`Failed to release networking for VM ${vm.id} during project delete`, err); + }); } if (vmIds.length > 0) { await db.delete(ipAssignments).where(inArray(ipAssignments.associatedVmId, vmIds)); diff --git a/apps/dashboard/src/lib/server/env.ts b/apps/dashboard/src/lib/server/env.ts index 5a3e645..3b4b8a0 100644 --- a/apps/dashboard/src/lib/server/env.ts +++ b/apps/dashboard/src/lib/server/env.ts @@ -28,7 +28,7 @@ export type RuntimeEnv = { connectionString: string; }; FEATURE_FLAGS?: KVNamespace; - BILLING_METER_SECRET?: string; + INTERNAL_CRON_SECRET?: string; GITHUB_CLIENT_ID?: string; GITHUB_CLIENT_SECRET?: string; }; @@ -67,7 +67,7 @@ export function getRuntimeEnv(): RuntimeEnv { AUTUMN_SERVER_ENTITY_FEATURE_ID: platformEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID, HYPERDRIVE: platformEnv.HYPERDRIVE, FEATURE_FLAGS: platformEnv.FEATURE_FLAGS, - BILLING_METER_SECRET: platformEnv.BILLING_METER_SECRET, + INTERNAL_CRON_SECRET: platformEnv.INTERNAL_CRON_SECRET, GITHUB_CLIENT_ID: platformEnv.GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET: platformEnv.GITHUB_CLIENT_SECRET }; @@ -95,7 +95,7 @@ export function getRuntimeEnv(): RuntimeEnv { AUTUMN_DEFAULT_PLAN_ID: privateEnv.AUTUMN_DEFAULT_PLAN_ID, AUTUMN_SERVER_ENTITY_FEATURE_ID: privateEnv.AUTUMN_SERVER_ENTITY_FEATURE_ID, DATABASE_URL: required('DATABASE_URL', privateEnv.DATABASE_URL), - BILLING_METER_SECRET: privateEnv.BILLING_METER_SECRET, + INTERNAL_CRON_SECRET: privateEnv.INTERNAL_CRON_SECRET, GITHUB_CLIENT_ID: privateEnv.GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET: privateEnv.GITHUB_CLIENT_SECRET }; diff --git a/apps/dashboard/src/lib/server/ipam.ts b/apps/dashboard/src/lib/server/ipam.ts index cde8c11..15ed61a 100644 --- a/apps/dashboard/src/lib/server/ipam.ts +++ b/apps/dashboard/src/lib/server/ipam.ts @@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit'; import { Address4, Address6 } from 'ip-address'; import { and, asc, eq, isNotNull, sql } from 'drizzle-orm'; import { initDrizzle } from '$lib/server/db'; -import { ipamAllocations, ipamPrefixes } from '$lib/server/db/schema'; +import { ipamAllocations, ipamPrefixes, vms } from '$lib/server/db/schema'; import { isVyosConfigured, VyosClient } from '$lib/server/vyos'; export type IpFamily = 'ipv4' | 'ipv6'; @@ -798,3 +798,26 @@ export async function releaseVmNetworking(db: QueryableDb, vmId: string) { await deleteVyosDelegatedRoute(allocations); await db.delete(ipamAllocations).where(eq(ipamAllocations.associatedVmId, vmId)); } + +export async function reconcileOrphanedIpamAllocations(db: QueryableDb, limit = 200) { + const orphaned = await db + .selectDistinct({ vmId: ipamAllocations.associatedVmId }) + .from(ipamAllocations) + .innerJoin(vms, eq(vms.id, ipamAllocations.associatedVmId)) + .where(eq(vms.active, false)) + .limit(limit); + + let released = 0; + let failed = 0; + for (const { vmId } of orphaned) { + try { + await releaseVmNetworking(db, vmId); + released += 1; + } catch (err) { + failed += 1; + console.warn(`Failed to reconcile orphaned IPAM allocations for VM ${vmId}`, err); + } + } + + return { checked: orphaned.length, released, failed }; +} diff --git a/apps/dashboard/src/lib/server/vm-deletion.ts b/apps/dashboard/src/lib/server/vm-deletion.ts index 0a36bb1..8e38ba1 100644 --- a/apps/dashboard/src/lib/server/vm-deletion.ts +++ b/apps/dashboard/src/lib/server/vm-deletion.ts @@ -63,17 +63,20 @@ async function deleteVmResources(row: DeletableVm): Promise { .returning({ id: vms.id }); if (claimed.length === 0) return; - const metered = await meterResourceThrough('vm', row.id); + await releaseVmNetworking(db, row.id).catch((err) => { + console.warn(`Failed to release networking for VM ${row.id}`, err); + }); + + const metered = await meterResourceThrough('vm', row.id).catch((err) => { + console.warn(`Failed to meter VM ${row.id} during deletion`, err); + return null; + }); if (row.ownerProjectId && (!metered?.event || metered.syncStatus === 'synced')) { await deleteProjectServerEntity(row.ownerProjectId, row.id).catch((err) => { console.warn(`Failed to delete Autumn entity for VM ${row.id}`, err); }); } - await releaseVmNetworking(db, row.id).catch((err) => { - console.warn(`Failed to release networking for VM ${row.id}`, err); - }); - console.log(`VM ${row.id} deletion completed`); } catch (err) { console.error(`VM ${row.id} deletion cleanup failed:`, err); diff --git a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts index 043e960..4c108d6 100644 --- a/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts +++ b/apps/dashboard/src/routes/api/internal/billing/meter/+server.ts @@ -9,8 +9,8 @@ import { enforceProjectBillingGrace } from '$lib/server/billing/enforcement'; import { getRuntimeEnv } from '$lib/server/env'; export const POST: RequestHandler = async ({ request }) => { - const secret = getRuntimeEnv().BILLING_METER_SECRET; - if (!secret) error(503, 'Billing meter is not configured'); + const secret = getRuntimeEnv().INTERNAL_CRON_SECRET; + if (!secret) error(503, 'Internal cron is not configured'); const authorization = request.headers.get('authorization'); if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized'); diff --git a/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts new file mode 100644 index 0000000..08f7f77 --- /dev/null +++ b/apps/dashboard/src/routes/api/internal/ipam/reconcile/+server.ts @@ -0,0 +1,17 @@ +import { error, json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getRuntimeEnv } from '$lib/server/env'; +import { initDrizzle } from '$lib/server/db'; +import { reconcileOrphanedIpamAllocations } from '$lib/server/ipam'; + +export const POST: RequestHandler = async ({ request }) => { + const secret = getRuntimeEnv().INTERNAL_CRON_SECRET; + if (!secret) error(503, 'Internal cron is not configured'); + + const authorization = request.headers.get('authorization'); + if (authorization !== `Bearer ${secret}`) error(401, 'Unauthorized'); + + const ipam = await reconcileOrphanedIpamAllocations(initDrizzle()); + + return json({ ipam }); +};