diff --git a/.github/AGENTS.md b/.github/AGENTS.md index bef8ccbd..82105afb 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -7,11 +7,11 @@ CI/CD and PR automation. Two groups: **build/deploy** pipelines and the **PR rev | Workflow | Trigger | Does | |----------|---------|------| | `frontend-ci.yml` | PR to main/develop, merge_group | In `apps/frontend`: `npm ci` → typecheck → lint → build → test. **Required check `frontend-ci`.** | -| `lambda-tests.yml` | push/PR main/develop, merge_group | Discover lambdas (excl. `tools/`); matrix per lambda spins Postgres 16, seeds `db_setup.sql`, starts dev-server, health-checks, `npm test`. **Required check `lambda-tests`.** | +| `lambda-tests.yml` | push/PR main/develop, merge_group | Discover lambdas (excl. `tools/`); matrix per lambda spins Postgres 16, applies `db/migrations` + `db/seed.sql`, starts dev-server, health-checks, `npm test`. Also gates `migrations-fresh` (migrations apply clean from scratch, types not stale) and `migrations-guard` (applied migrations immutable, no unsafe SQL). **Required check `lambda-tests`.** | | `lambda-deploy.yml` | push to main, paths `apps/backend/lambdas/**` or `shared/types/**` | Detect changed lambdas (all if none); build `npm ci --legacy-peer-deps` + `npm run package` → `lambda.zip`; `aws lambda update-function-code --function-name branch-` (us-east-2). | | `frontend-deploy.yml` | push to main, paths `apps/frontend/**` | Build static export (`npm run build` → `out/`) with `NEXT_PUBLIC_API_BASE_URL`; `aws s3 sync` to the frontend bucket + CloudFront invalidation. `production` env, OIDC apply role. | | `lambda-readme.yml` | after `terraform-plan` completes, or manual | `node tools/lambda-cli.js generate-readme` (all), commit regenerated READMEs. | -| `regenerate-db-types.yaml` | `db_setup.sql` changes, after `lambda-readme`, or manual | Spin Postgres, apply schema, `kysely-codegen`, strip kysely import → local `ColumnType`, write `shared/types/db-types.d.ts`, `tsc --noEmit`, commit to PR branch (or comment "in sync"). | +| `regenerate-db-types.yaml` (`Schema Change Checks`) | `db/migrations/**` changes, or manual | Post a sticky migration checklist comment, spin Postgres, apply migrations, run `npm run types` (shared code path with local `make types`), `tsc --noEmit`, commit to PR branch (or comment "in sync"). | | `terraform-plan.yml` | PR main/develop, merge_group | Detect changed TF dirs; `fmt` + terraform-docs (auto-commit); per-dir `init`/`validate`/`plan`, post plan PR comment. **Required check `terraform-plan-summary`.** | | `terraform-apply.yml` | push to main `infrastructure/**/*.tf`, or manual (dir list) | Per-dir matrix, `production` env (approval gate), `plan` → `apply -auto-approve`. | diff --git a/.github/actions/slack-deploy/action.yml b/.github/actions/slack-deploy/action.yml index 0f12653b..57fb8d18 100644 --- a/.github/actions/slack-deploy/action.yml +++ b/.github/actions/slack-deploy/action.yml @@ -19,6 +19,10 @@ inputs: description: 'Completed step count so far (for step/complete)' required: false default: '0' + bar-offset: + description: '(step mode) steps already finished by non-matrix jobs, e.g. migrate' + required: false + default: '0' step-name: description: 'Past-tense label for a step, e.g. "Applied infrastructure/aws"' required: false @@ -91,6 +95,7 @@ runs: COMPONENT: ${{ inputs.component }} TOTAL: ${{ inputs.total-steps }} STEP_INDEX: ${{ inputs.step-index }} + BAR_OFFSET: ${{ inputs.bar-offset }} STEP_NAME: ${{ inputs.step-name }} STEP_STATUS: ${{ inputs.step-status }} DETAILS: ${{ inputs.details }} @@ -112,6 +117,10 @@ runs: const component = process.env.COMPONENT || 'app'; const total = parseInt(process.env.TOTAL || '1', 10) || 1; const done = parseInt(process.env.STEP_INDEX || '0', 10) || 0; + // liveDone() only counts jobs matching the given name, so legs finished + // by a differently-named job (the `migrate` job) have to be added back + // or the bar stalls one step behind for the whole run. + const barOffset = parseInt(process.env.BAR_OFFSET || '0', 10) || 0; const stepName = process.env.STEP_NAME || ''; const stepOk = (process.env.STEP_STATUS || 'success').toLowerCase() === 'success'; let rawDetails = process.env.DETAILS || ''; @@ -287,7 +296,7 @@ runs: const jobName = process.env.JOB_NAME || ''; let d = done; // +1 for this leg, which is finishing now but not yet marked completed. - if (jobName) { try { d = Math.min(total, (await liveDone(jobName)) + 1); } catch (err) { d = done; } } + if (jobName) { try { d = Math.min(total, barOffset + (await liveDone(jobName)) + 1); } catch (err) { d = done; } } await threadStep(d, stepName, stepOk, rawDetails); return; } diff --git a/.github/workflows/lambda-deploy.yml b/.github/workflows/lambda-deploy.yml index 40f63717..f012f7a9 100644 --- a/.github/workflows/lambda-deploy.yml +++ b/.github/workflows/lambda-deploy.yml @@ -6,7 +6,13 @@ on: paths: - 'apps/backend/lambdas/**' - 'shared/types/**' + - 'apps/backend/db/migrations/**' workflow_dispatch: + inputs: + migrations_only: + description: 'Apply pending DB migrations without redeploying lambdas' + type: boolean + default: false jobs: detect-changes: @@ -14,46 +20,66 @@ jobs: outputs: lambdas: ${{ steps.detect.outputs.lambdas }} has-changes: ${{ steps.detect.outputs.has-changes }} + migrate: ${{ steps.detect.outputs.migrate }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Detect changed lambdas + - name: Detect changed lambdas and migrations id: detect + env: + EVENT: ${{ github.event_name }} + MIGRATIONS_ONLY: ${{ inputs.migrations_only }} run: | - BASE_COMMIT="${{ github.event.before }}" + set -euo pipefail all_lambdas=$(ls -d apps/backend/lambdas/*/ | grep -Ev "(tools)/" | sed 's#/$##' | jq -R -s -c 'split("\n") | map(select(. != ""))') - + changed_files=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only HEAD) - - changed_lambdas=() - for lambda_path in apps/backend/lambdas/*/; do - lambda_name=$(basename "$lambda_path") - if [[ "$lambda_name" != "tools" ]]; then - if echo "$changed_files" | grep -q "^apps/backend/lambdas/$lambda_name/"; then + + migrate=false + code=false + grep -q '^apps/backend/db/migrations/' <<<"$changed_files" && migrate=true + grep -qE '^(apps/backend/lambdas/|shared/types/)' <<<"$changed_files" && code=true + + # workflow_dispatch has no meaningful diff: apply everything, or + # migrations only when explicitly asked (recovery / manual re-run). + if [[ "$EVENT" == "workflow_dispatch" ]]; then + migrate=true + if [[ "$MIGRATIONS_ONLY" == "true" ]]; then code=false; else code=true; fi + fi + + lambdas='[]' + if [[ "$code" == "true" ]]; then + changed_lambdas=() + for lambda_path in apps/backend/lambdas/*/; do + lambda_name=$(basename "$lambda_path") + [[ "$lambda_name" == "tools" ]] && continue + if grep -q "^apps/backend/lambdas/$lambda_name/" <<<"$changed_files"; then changed_lambdas+=("$lambda_path") fi + done + + if [[ ${#changed_lambdas[@]} -eq 0 ]]; then + # shared/types/** (or a dispatch) touched nothing lambda-specific: + # deploy all. Gated on `code` so a migrations-only push does NOT + # redeploy all six functions. + echo "No specific lambda changes detected, deploying all lambdas" + lambdas=$all_lambdas + else + lambdas=$(printf '%s\n' "${changed_lambdas[@]}" | sed 's#/$##' | jq -R -s -c 'split("\n") | map(select(. != ""))') fi - done - - if [[ ${#changed_lambdas[@]} -eq 0 ]]; then - echo "No specific lambda changes detected, deploying all lambdas" - lambdas=$all_lambdas - has_changes="true" - else - lambdas=$(printf '%s\n' "${changed_lambdas[@]}" | sed 's#/$##' | jq -R -s -c 'split("\n") | map(select(. != ""))') - has_changes="true" fi - + echo "lambdas=$lambdas" >> $GITHUB_OUTPUT - echo "has-changes=$has_changes" >> $GITHUB_OUTPUT + echo "has-changes=$code" >> $GITHUB_OUTPUT + echo "migrate=$migrate" >> $GITHUB_OUTPUT # Post the initial "Deploying backend" Slack message before builds run. prep: needs: detect-changes - if: needs.detect-changes.outputs.has-changes == 'true' + if: needs.detect-changes.outputs.has-changes == 'true' || needs.detect-changes.outputs.migrate == 'true' runs-on: ubuntu-latest permissions: contents: read @@ -63,11 +89,16 @@ jobs: total: ${{ steps.count.outputs.total }} steps: - uses: actions/checkout@v4 - - name: Count lambdas + - name: Count steps id: count env: LAMBDAS: ${{ needs.detect-changes.outputs.lambdas }} - run: echo "total=$(echo "$LAMBDAS" | jq 'length')" >> "$GITHUB_OUTPUT" + MIGRATE: ${{ needs.detect-changes.outputs.migrate }} + run: | + total=$(echo "$LAMBDAS" | jq 'length') + # The migrate job is its own leg in the Slack status bar. + [ "$MIGRATE" = "true" ] && total=$((total + 1)) + echo "total=$total" >> "$GITHUB_OUTPUT" - name: Start deploy notification id: slack uses: ./.github/actions/slack-deploy @@ -115,9 +146,187 @@ jobs: path: ${{ matrix.lambda }}/lambda.zip if-no-files-found: error + # Schema lands BEFORE the zip swap: `deploy` needs this job, so a failed + # migration leaves production running the OLD code against the OLD schema. + # The consequence for contributors is that only additive migrations are safe in + # a single PR -- see the expand/contract rules in apps/backend/db/README.md. + migrate: + needs: [detect-changes, prep] + if: needs.detect-changes.outputs.migrate == 'true' + runs-on: ubuntu-latest + # Separate from `production` purely so this job gets its own OIDC subject and + # can use the narrowly-scoped branch-ci-migrate role instead of the + # AdministratorAccess branch-ci-apply role. Intentionally has no required + # reviewers: a pending approval here would strand a merged PR with its schema + # applied and its lambda code undeployed. + environment: production-db + # kysely takes a pg advisory lock, but serialize anyway so two merges in a row + # queue instead of racing -- and never cancel a half-run migration. + concurrency: + group: db-migrate-prod + cancel-in-progress: false + permissions: + id-token: write # assume the OIDC migrate role + contents: read + actions: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install db tooling + run: npm ci --prefix apps/backend/db --no-audit --no-fund + + - name: Configure AWS credentials (scoped migrate role via OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::489881683177:role/branch-ci-migrate + aws-region: us-east-2 + + # Read the connection from the same place the lambdas read it, so CI can + # never migrate a database the code isn't talking to. Terraform is the + # authoritative writer of that env block (infrastructure/aws/lambda.tf). + - name: Resolve DB connection from the deployed lambda config + run: | + set -euo pipefail + cfg=$(aws lambda get-function-configuration \ + --function-name branch-auth \ + --query 'Environment.Variables' --output json) + for k in DB_HOST DB_PORT DB_USER DB_NAME; do + v=$(jq -er --arg k "$k" '.[$k]' <<<"$cfg") \ + || { echo "::error::$k missing from branch-auth config"; exit 1; } + echo "$k=$v" >> "$GITHUB_ENV" + done + pw=$(jq -er '.DB_PASSWORD' <<<"$cfg") + echo "::add-mask::$pw" + echo "DB_PASSWORD=$pw" >> "$GITHUB_ENV" + + # The runner reaches RDS over the public internet, so TLS is mandatory here + # even though the lambdas currently use ssl:false (they at least stay inside + # AWS). AWS publishes the CA bundle, so verification is free. + - name: Fetch RDS CA bundle + run: | + curl -fsSL -o "$RUNNER_TEMP/rds-ca.pem" \ + https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem + echo "DB_SSL=true" >> "$GITHUB_ENV" + echo "DB_SSL_CA=$RUNNER_TEMP/rds-ca.pem" >> "$GITHUB_ENV" + + - name: Pending migrations (read-only preflight) + id: status + run: | + set -o pipefail + npm run migrate:status --prefix apps/backend/db 2>&1 | tee status.log + + # 0000_baseline_schema is idempotent, but "idempotent" is one typo away from + # DROP SCHEMA. It is adopted once by hand against production (see + # apps/backend/db/README.md); if it still shows as pending here, we are + # pointed at a database we did not expect -- refuse rather than guess. + - name: Refuse to execute the baseline against production + run: | + if grep -E '0000_baseline_schema' status.log | grep -q 'PENDING'; then + echo "::error::0000_baseline_schema is PENDING on the target database. It was never adopted (or DB_HOST is wrong). Refusing to run -- see apps/backend/db/README.md." + exit 1 + fi + + - name: Snapshot production before migrating + id: snapshot + run: | + set -euo pipefail + INSTANCE=$(aws rds describe-db-instances \ + --query "DBInstances[?Endpoint.Address=='${DB_HOST}'].DBInstanceIdentifier | [0]" \ + --output text) + [ "$INSTANCE" != "None" ] || { echo "::error::no RDS instance matches DB_HOST"; exit 1; } + SNAP="branch-premigrate-$(date -u +%Y%m%dT%H%M%SZ)-${GITHUB_SHA::7}" + aws rds create-db-snapshot \ + --db-instance-identifier "$INSTANCE" --db-snapshot-identifier "$SNAP" \ + --tags Key=source,Value=github-actions Key=commit,Value="$GITHUB_SHA" + aws rds wait db-snapshot-available --db-snapshot-identifier "$SNAP" + echo "instance=$INSTANCE" >> "$GITHUB_OUTPUT" + echo "id=$SNAP" >> "$GITHUB_OUTPUT" + echo "Snapshot $SNAP is available" + + - name: Apply migrations + id: up + run: | + set -o pipefail + { echo "Pending before this run:"; cat status.log; echo; \ + npm run migrate --prefix apps/backend/db; } 2>&1 | tee migrate.log + + # Manual snapshots never expire on their own and survive instance deletion, + # so without this every deploy would leave one behind forever. Five is + # plenty: anything older than the last few migrations is better served by + # point-in-time recovery (backup_retention_period = 7 in + # infrastructure/aws/main.tf). + - name: Prune old pre-migration snapshots (keep 5) + if: always() + continue-on-error: true + run: | + aws rds describe-db-snapshots --snapshot-type manual \ + --query "reverse(sort_by(DBSnapshots[?starts_with(DBSnapshotIdentifier,'branch-premigrate-')],&SnapshotCreateTime))[5:].DBSnapshotIdentifier" \ + --output text | tr '\t' '\n' | while read -r s; do + [ -n "$s" ] && aws rds delete-db-snapshot --db-snapshot-identifier "$s" + done + + - name: Summarize + if: always() + run: | + { + echo "### Database migration" + echo "" + echo "Snapshot: \`${{ steps.snapshot.outputs.id || 'not taken' }}\`" + echo "" + echo '```' + cat status.log 2>/dev/null || echo '(no status captured)' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # Reuses the lambda-result-* artifact pattern so `notify` picks the + # migration up with no changes and fails the Slack bar even when `deploy` + # never ran. + - name: Save migrate result + if: always() + run: | + mkdir -p lambda-results + echo "migrate|${{ steps.up.outcome }}" > lambda-results/migrate.meta + cp migrate.log lambda-results/migrate.log 2>/dev/null \ + || cp status.log lambda-results/migrate.log 2>/dev/null \ + || echo "(no migrate output captured)" > lambda-results/migrate.log + + - name: Upload migrate result + if: always() + uses: actions/upload-artifact@v4 + with: + name: lambda-result-migrate + path: lambda-results/ + if-no-files-found: warn + + - name: Thread step (live) + if: always() + uses: ./.github/actions/slack-deploy + with: + mode: step + component: backend + total-steps: ${{ needs.prep.outputs.total }} + job-name: migrate + step-name: Applied DB migrations + step-status: ${{ steps.up.outcome }} + details-file: migrate.log + ts: ${{ needs.prep.outputs.ts }} + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy: - needs: [detect-changes, prep, build] - if: needs.detect-changes.outputs.has-changes == 'true' + needs: [detect-changes, prep, build, migrate] + # A skipped `needs` skips the dependent job unless the condition is explicit. + # Enumerate results rather than using always(), so a FAILED migration still + # blocks the zip swap while a SKIPPED one (code-only push) does not. + if: >- + !cancelled() + && needs.detect-changes.outputs.has-changes == 'true' + && needs.build.result == 'success' + && (needs.migrate.result == 'success' || needs.migrate.result == 'skipped') runs-on: ubuntu-latest environment: production permissions: @@ -189,6 +398,9 @@ jobs: component: backend total-steps: ${{ needs.prep.outputs.total }} job-name: deploy + # The migrate job already reported one completed leg under a different + # job name, so add it back or the bar stalls a step behind. + bar-offset: ${{ needs.detect-changes.outputs.migrate == 'true' && '1' || '0' }} step-name: Deployed branch-${{ steps.rname.outputs.name }} step-status: ${{ steps.deploy.outcome }} details-file: deploy.log @@ -198,8 +410,10 @@ jobs: notify: name: Deployment Notify - needs: [detect-changes, prep, deploy] - if: always() && needs.detect-changes.outputs.has-changes == 'true' + needs: [detect-changes, prep, migrate, deploy] + if: >- + always() && (needs.detect-changes.outputs.has-changes == 'true' + || needs.detect-changes.outputs.migrate == 'true') runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/lambda-tests.yml b/.github/workflows/lambda-tests.yml index 7a07a217..92676d9e 100644 --- a/.github/workflows/lambda-tests.yml +++ b/.github/workflows/lambda-tests.yml @@ -47,8 +47,15 @@ jobs: uses: actions/setup-node@v4 with: node-version: '20' - - name: Setup database schema - run: psql postgres://branch_dev:password@localhost:5432/branch_db -f apps/backend/db/db_setup.sql + # Build the schema the same way production does -- by applying + # db/migrations -- then load the dev seed rows the tests expect. The tests + # also call ensureSchema()/resetData() from db/testkit.ts themselves, so + # this doubles as a per-PR smoke test that migrations apply to a virgin DB. + - name: Apply migrations and seed + working-directory: apps/backend/db + run: npm ci --no-audit --no-fund && npm run migrate && npm run seed + env: + DATABASE_URL: postgres://branch_dev:password@localhost:5432/branch_db - name: Build shared lambda-auth package run: npm ci --prefix shared/lambda-auth && npm run build --prefix shared/lambda-auth - name: Install dependencies @@ -63,11 +70,167 @@ jobs: curl --fail --retry 5 --retry-delay 2 http://localhost:3000/${LAMBDA_NAME}/health npm test env: - DATABASE_URL: postgres://branch_dev:password@localhost:5432/branch_db?options=-csearch_path%3Dbranch + # No ?options=-csearch_path: the lambdas schema-qualify every query via + # the generated "branch.*" DB keys, and the migrator sets search_path on + # its own connections. + DATABASE_URL: postgres://branch_dev:password@localhost:5432/branch_db COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }} COGNITO_CLIENT_ID: ${{ secrets.COGNITO_CLIENT_ID }} AWS_REGION: us-east-2 - + + # Migrations must apply cleanly to an empty database, be idempotent, and agree + # with the committed types. + migrations-fresh: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: branch_dev + POSTGRES_PASSWORD: password + POSTGRES_DB: branch_db + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + env: + DATABASE_URL: postgres://branch_dev:password@localhost:5432/branch_db + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install db tooling + run: npm ci --prefix apps/backend/db --no-audit --no-fund + + - name: Apply every migration from scratch + run: npm run migrate --prefix apps/backend/db + + - name: Ledger must be clean afterwards + run: | + set -o pipefail + npm run migrate:status --prefix apps/backend/db | tee status.log + if grep -q 'PENDING' status.log; then + echo "::error::migrations still pending after a successful migrate run" + exit 1 + fi + + - name: Re-running migrations must be a no-op + run: npm run migrate --prefix apps/backend/db + + - name: Seeds must apply on top of the migrated schema + run: npm run seed --prefix apps/backend/db + + # Drift check: the committed types must be what these migrations produce. + - name: Committed types must match the migrations + run: | + cp shared/types/db-types.d.ts /tmp/committed.d.ts + npm run types --prefix apps/backend/db + if ! diff -u /tmp/committed.d.ts shared/types/db-types.d.ts; then + echo "::error::shared/types/db-types.d.ts is stale. The Schema Change Checks workflow pushes a fix to this PR automatically -- wait for its commit, then this check goes green. Or run 'make types' in apps/backend and commit the result." + exit 1 + fi + + # Guards that need no database: migration files are append-only, correctly + # named, and free of statements that would break the currently deployed code. + migrations-guard: + runs-on: ubuntu-latest + env: + DIR: apps/backend/db/migrations + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # push-to-main has nothing meaningful to compare against; resolving to "" + # makes the guards no-op WITHOUT skipping the job, so the lambda-tests gate + # (which requires result == 'success') still passes. + - name: Resolve base sha + id: base + run: | + case "${{ github.event_name }}" in + pull_request) base='${{ github.event.pull_request.base.sha }}' ;; + merge_group) base='${{ github.event.merge_group.base_sha }}' ;; + *) base='' ;; + esac + echo "sha=$base" >> "$GITHUB_OUTPUT" + + - name: Applied migrations are immutable + if: steps.base.outputs.sha != '' + env: + BASE: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + MB=$(git merge-base "$BASE" HEAD) + # Anything but A(dded) under migrations/ is a violation: M modified, + # D deleted, R renamed, C copied, T typechange. --find-renames is + # explicit so a rename reports R (not D+A) regardless of the runner's + # diff.renames default. + violations=$(git diff --name-status --find-renames "$MB" HEAD -- "$DIR" | grep -vE '^A[[:space:]]' || true) + if [ -n "$violations" ]; then + echo "::error::Migrations already on main can never be modified, renamed or deleted -- someone has already run them. Add a NEW migration that fixes the old one." + printf '%s\n' "$violations" + exit 1 + fi + + - name: New migrations must be named correctly + if: steps.base.outputs.sha != '' + env: + BASE: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + export LC_ALL=C + MB=$(git merge-base "$BASE" HEAD) + added=$(git diff --diff-filter=A --name-only "$MB" HEAD -- "$DIR" | xargs -r -n1 basename | sort || true) + last=$(git ls-tree -r --name-only "$MB" -- "$DIR" | xargs -r -n1 basename | sort | tail -1 || true) + fail=0 + for f in $added; do + if ! echo "$f" | grep -qE '^(0000_baseline_schema|[0-9]{14}_[a-z0-9_]+)\.sql$'; then + echo "::error file=$DIR/$f::name must be YYYYMMDDHHMMSS_lower_snake_case.sql -- use 'make new-migration NAME=...' so it is generated for you" + fail=1 + fi + # Out-of-order merges are allowed at runtime (the migrator sets + # allowUnorderedMigrations), so this is advice, not a failure. + if [ -n "${last:-}" ] && [ "$f" \< "$last" ]; then + echo "::warning file=$DIR/$f::sorts before $last, which is already on main, so it will be applied out of order. Harmless, but renaming it to a current timestamp keeps the history readable." + fi + done + exit $fail + + - name: No statements that would break the deployed code + if: steps.base.outputs.sha != '' + env: + BASE: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + MB=$(git merge-base "$BASE" HEAD) + files=$(git diff --diff-filter=A --name-only "$MB" HEAD -- "$DIR" || true) + fail=0 + for f in $files; do + [ -f "$f" ] || continue + # '-- allow-destructive: ' opts a file out, deliberately loudly. + if grep -qi '^-- allow-destructive:' "$f"; then + echo "::warning file=$f::opted out of the destructive-SQL check" + continue + fi + if grep -inE '\b(DROP[[:space:]]+SCHEMA|DROP[[:space:]]+DATABASE|TRUNCATE|DROP[[:space:]]+TABLE|DROP[[:space:]]+COLUMN)\b' "$f"; then + echo "::error file=$f::destructive statement. Migrations run against production BEFORE the new lambda code deploys, so the currently deployed code would hit the changed schema. See the expand/contract rules in apps/backend/db/README.md. Add '-- allow-destructive: ' only for a contract-phase migration whose expand phase is already live." + fail=1 + fi + if grep -inE 'CONCURRENTLY' "$f"; then + echo "::error file=$f::CONCURRENTLY cannot run inside a transaction, and the migrator wraps the run in one. This database is tiny -- use a plain CREATE INDEX." + fail=1 + fi + if grep -inE '^[[:space:]]*INSERT[[:space:]]+INTO[[:space:]]+(branch\.)?users\b' "$f"; then + echo "::warning file=$f::inserting users in a migration puts rows in PRODUCTION. Seeded users with a NULL cognito_sub are claimable by POST /auth/register, so this can hand someone an account. Dev seed rows belong in apps/backend/db/seed.sql." + fi + done + exit $fail + # The discover job only globs apps/backend/lambdas/*/, so the shared auth # package -- the single most security-critical module in the backend -- would # never have its tests run without this job. @@ -89,7 +252,7 @@ jobs: # silently disable the gate rather than fail loudly. lambda-tests: name: lambda-tests - needs: [test, shared-auth] + needs: [test, shared-auth, migrations-fresh, migrations-guard] if: always() runs-on: ubuntu-latest steps: @@ -103,4 +266,15 @@ jobs: echo "shared/lambda-auth tests failed or were cancelled" exit 1 fi - echo "All lambda tests passed!" + # Folded into this gate rather than added to branch protection, so + # infrastructure/github/main.tf needs no change and there is no risk of + # wedging the merge queue on a mistyped required-check context. + if [ "${{ needs.migrations-fresh.result }}" != "success" ]; then + echo "Migrations do not apply cleanly from scratch, or the committed DB types are stale" + exit 1 + fi + if [ "${{ needs.migrations-guard.result }}" != "success" ]; then + echo "Migration guard failed (edited an applied migration, bad name, or unsafe SQL)" + exit 1 + fi + echo "All lambda and migration checks passed!" diff --git a/.github/workflows/preview-env.yml b/.github/workflows/preview-env.yml index b0d5ae47..38fe31d3 100644 --- a/.github/workflows/preview-env.yml +++ b/.github/workflows/preview-env.yml @@ -9,7 +9,12 @@ name: PR Preview Environment # Cost: ~$0. Per-PR lambdas + one REST API Gateway (all serverless-at-rest); # the frontend is a path prefix on the ONE shared CloudFront distribution # (infrastructure/preview-shared). Shared RDS + Cognito are reused as-is; -# migrations are never run here. See infrastructure/preview/*. +# migrations are never run here -- structurally, not just to save money: the +# generated DB types hardcode the `branch.` schema prefix +# (shared/types/db-types.d.ts), so a per-PR schema would need per-PR type +# regeneration and a rebuild of every lambda. The consequence is that a PR adding +# a migration CANNOT be fully previewed: its new columns don't exist yet, so +# endpoints touching them fail until the PR merges. See infrastructure/preview/*. on: pull_request: types: [labeled, synchronize, unlabeled, closed] @@ -232,7 +237,7 @@ jobs: `**Open:** ${url}`, `**API:** \`${process.env.API_URL}\``, '', - '_Shared RDS + Cognito (prod data); DB migrations are **not** applied here. New commits update this environment in place — a note is posted here on each update. Remove the `test-environment` label or close the PR to tear it down._', + '_Shared RDS + Cognito (prod data); DB migrations are **not** applied here — if this PR adds a migration, endpoints using the new columns will fail until it merges. New commits update this environment in place — a note is posted here on each update. Remove the `test-environment` label or close the PR to tear it down._', ].join('\n'); } else { body = `🌿 **Test environment updated in place** ✅ — Click [here](${url}) to open. updated for ${sha} · [logs](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`; diff --git a/.github/workflows/regenerate-db-types.yaml b/.github/workflows/regenerate-db-types.yaml index 69df1b0d..22e7b94f 100644 --- a/.github/workflows/regenerate-db-types.yaml +++ b/.github/workflows/regenerate-db-types.yaml @@ -1,9 +1,9 @@ -name: Generate Database Types For Schema Changes +name: Schema Change Checks on: pull_request: paths: - - apps/backend/db/db_setup.sql + - apps/backend/db/migrations/** workflow_dispatch: @@ -12,7 +12,8 @@ permissions: pull-requests: write env: - NODE_VERSION: '23' + # Matches the lambda runtime (nodejs20.x) and every other workflow. + NODE_VERSION: '20' DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb jobs: @@ -31,12 +32,73 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 + # Posted before anything can fail, so the guidance is on the PR even if a + # later step breaks. Sticky: updated in place on every push rather than + # re-commented, which would get the bot muted. + - name: Comment migration checklist on the PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const { execSync } = require('child_process'); + const base = context.payload.pull_request.base.sha; + const mergeBase = execSync(`git merge-base ${base} HEAD`).toString().trim(); + const added = execSync( + `git diff --diff-filter=A --name-only ${mergeBase} HEAD -- apps/backend/db/migrations`, + ).toString().trim().split('\n').filter(Boolean); + + const outside = execSync( + `git diff --name-only ${mergeBase} HEAD -- . ':(exclude)apps/backend/db'`, + ).toString().trim().split('\n').filter(Boolean); + + const MARKER = ''; + const list = added.length + ? added.map((f) => `- \`${f}\``).join('\n') + : '_(no newly added migration files in this PR)_'; + + let body = `${MARKER} + ### This PR contains a database migration + + ${list} + + **It will be applied to the production database automatically when this PR merges**, before the new lambda code is deployed. Please confirm before requesting review: + + - [ ] **Applied and tested locally.** \`cd apps/backend && make migrate\`, then run the affected lambda's tests (\`cd apps/backend/lambdas/ && npm test\`). \`make show-migrations\` shows what applied. + - [ ] **Safe for the code that is live right now.** During the deploy window -- and indefinitely if the deploy fails -- the currently deployed lambdas run against your new schema. Additive changes (\`CREATE TABLE\`, nullable \`ADD COLUMN\`, \`CREATE INDEX\`) are fine in one PR. \`DROP COLUMN\`, renames, \`ADD COLUMN NOT NULL\` with no default, and new \`UNIQUE\`/\`CHECK\`/\`FOREIGN KEY\` constraints need two merged PRs -- see the expand/contract rules in \`apps/backend/db/README.md\`. + - [ ] **Kept separate from unrelated changes.** A migration PR should ideally contain the migration, the code that needs it, and nothing else. It changes production state, it is the one thing here that redeploying cannot roll back, and a reviewer should be able to see the whole schema change without scrolling past unrelated work. + - [ ] **No already-merged migration was edited.** Fix an old migration by adding a new one; there is no \`down\`. + + \`shared/types/db-types.d.ts\` is regenerated and pushed to this branch automatically -- don't hand-edit it. Expect one red \`migrations-fresh\` check before that commit lands.`; + + if (outside.length > 15) { + body += `\n\nThis PR also changes ${outside.length} files outside \`apps/backend/db/\`. If any are unrelated to this schema change, consider splitting them into a separate PR.`; + } + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find((c) => c.body?.includes(MARKER)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: existing.id, body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: context.issue.number, body, + }); + } + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - + - name: Cache npm dependencies uses: actions/cache@v4 with: @@ -49,14 +111,12 @@ jobs: run: | echo "Installing project dependencies" npm ci --legacy-peer-deps - - echo "Installing kysely-codegen, typescript, and pg globally" - npm install -g kysely-codegen typescript pg - echo "Verifying installations" - which kysely-codegen || { echo "Error: kysely-codegen not found"; exit 1; } - tsc --version || { echo "Error: typescript not found"; exit 1; } - node -e "require('pg')" || { echo "Error: pg not found"; exit 1; } + # apps/backend/db pins kysely-codegen. Installing it globally instead + # would let CI generate types with a different version than a + # contributor's `make types` run. + echo "Installing db tooling" + npm ci --prefix apps/backend/db --no-audit --no-fund - name: Setup PostgreSQL uses: ikalnytskyi/action-setup-postgres@v6 @@ -74,78 +134,24 @@ jobs: env: PGPASSWORD: postgres - - name: Initialize database schema + - name: Build schema from migrations run: | - echo "Applying database schema" - psql -h localhost -U postgres -d testdb -f apps/backend/db/db_setup.sql - + echo "Applying db/migrations" + npm run migrate --prefix apps/backend/db + echo "Verifying schema application" - psql -h localhost -U postgres -d testdb -c "\dt" + psql -h localhost -U postgres -d testdb -c "\dt branch.*" env: PGPASSWORD: postgres + DATABASE_URL: ${{ env.DATABASE_URL }} - - name: Generate types with kysely-codegen - run: | - echo "Starting type generation..." - cd apps/backend/db - - # Run kysely-codegen - kysely-codegen \ - --url "$DATABASE_URL" \ - --out-file db-types.d.ts \ - --dialect postgres || { - echo "Error: kysely-codegen failed to generate types" - exit 1 - } - - # Verify the output file was created - if [ ! -f "db-types.d.ts" ]; then - echo "Error: db-types.d.ts was not generated" - exit 1 - fi - - echo "Type generation successful" - - - name: Copy generated types to shared DTOs package - run: | - echo "Copying generated types to shared/types" - - SOURCE_FILE="apps/backend/db/db-types.d.ts" - TARGET_FILE="shared/types/db-types.d.ts" - - # verify the source file exists and is not empty - if [ ! -s "$SOURCE_FILE" ]; then - echo "Error: Generated types file not found or empty at $SOURCE_FILE" - exit 1 - fi - - # Replace the kysely import with a local, structurally identical - # ColumnType so the shared package stays dependency-free and the - # lambdas stay fully self contained. - node -e ' - const fs = require("fs"); - const [src, target] = process.argv.slice(1); - const kyselyImport = `import type { ColumnType } from "kysely";`; - const localColumnType = [ - "/**", - " * Structurally identical to kysely'"'"'s ColumnType, defined locally so this", - " * package has no dependencies and lambdas stay fully self contained.", - " */", - "type ColumnType = {", - " readonly __select__: SelectType;", - " readonly __insert__: InsertType;", - " readonly __update__: UpdateType;", - "};", - ].join("\n"); - const contents = fs.readFileSync(src, "utf8"); - if (!contents.includes(kyselyImport)) { - console.error("Error: expected kysely ColumnType import not found in " + src); - process.exit(1); - } - fs.writeFileSync(target, contents.replace(kyselyImport, localColumnType)); - ' "$SOURCE_FILE" "$TARGET_FILE" - - echo "Wrote $TARGET_FILE" + # Both the codegen invocation and the kysely-ColumnType rewrite live in + # apps/backend/db/src/, so this produces byte-identical output to a + # contributor's `make types` -- rather than two copies kept in sync by hand. + - name: Generate types + run: npm run types --prefix apps/backend/db + env: + DATABASE_URL: ${{ env.DATABASE_URL }} - name: Verify TypeScript compilation run: | @@ -295,7 +301,7 @@ jobs: echo "---" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Workflow Trigger:** Schema change detected in:" >> $GITHUB_STEP_SUMMARY - echo "- \`apps/backend/db/db_setup.sql\`" >> $GITHUB_STEP_SUMMARY + echo "- \`apps/backend/db/migrations/**\`" >> $GITHUB_STEP_SUMMARY diff --git a/AGENTS.md b/AGENTS.md index 32d59af9..de053b68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ BRANCH is a non-profit accounting platform (projects, donors, donations, expendi Two `file:`-linked packages dedupe code across lambdas: -- **`@branch/types`** (`shared/types/`) — types only, no runtime. Exports DB row types (`DB`, `BranchUsers`, ...) + auth DTOs (`AuthContext`, `AuthenticatedUser`, `AccessLevel`, `AuthorizationCheck`). `db-types.d.ts` is **generated** from `apps/backend/db/db_setup.sql` by the `regenerate-db-types` workflow — never hand-edit it. +- **`@branch/types`** (`shared/types/`) — types only, no runtime. Exports DB row types (`DB`, `BranchUsers`, ...) + auth DTOs (`AuthContext`, `AuthenticatedUser`, `AccessLevel`, `AuthorizationCheck`). `db-types.d.ts` is **generated** from `apps/backend/db/migrations/**` by the `Schema Change Checks` workflow (or locally by `make types`) — never hand-edit it. - **`@branch/lambda-auth`** (`shared/lambda-auth/`) — runtime auth: `authenticateRequest(db, event)`, `extractToken(event)`, `checkAuthorization(ctx, level, resourceUserId?)`. Lambdas wrap it in their local `auth.ts`. ## Root commands @@ -49,5 +49,5 @@ Per-app build/test/dev commands live in each app's `package.json` and AGENTS.md. - TypeScript strict everywhere. Prettier + ESLint enforced on push (husky + lint-staged). - Squash-merge only; 2 approvals required; PR merge queue. Branch protection requires `frontend-ci`, `lambda-tests`, `terraform-plan-summary` checks to pass. -- Schema changes go in `apps/backend/db/db_setup.sql`; DB types regenerate automatically via workflow. +- Schema changes are migrations: `make new-migration NAME=x` in `apps/backend`, write SQL, `make migrate`. Applied to prod automatically on merge, before the lambda deploy — so additive changes only in a single PR. See `apps/backend/db/README.md`. - Don't commit secrets — all secrets flow through Infisical → GitHub Actions / Terraform. diff --git a/apps/backend/AGENTS.md b/apps/backend/AGENTS.md index e4158926..32d7740b 100644 --- a/apps/backend/AGENTS.md +++ b/apps/backend/AGENTS.md @@ -8,7 +8,7 @@ See `lambdas/AGENTS.md` for the per-lambda code conventions and the `lambda-cli` | Service | Port | Domain | |---------|------|--------| -| postgres | 5432 | DB (schema auto-init from `db/db_setup.sql`) | +| postgres | 5432 | DB (schema applied by the one-shot `migrator` service from `db/migrations`) | | users | 3001 | user management | | projects | 3002 | projects, memberships, dashboard | | donors | 3003 | donors + donations | @@ -42,10 +42,10 @@ http://localhost:3000//health ## Database -- Schema in `db/db_setup.sql` (schema `branch`, with seed data). Tables: `users`, `projects`, `project_memberships` (roles: PI/Accountant/Staff/Admin), `donors`, `project_donations`, `expenditures` (status: approved/pending/denied/needs_more_info), `reports` (report_type: technical/narrative). +- Schema built from `db/migrations/*.sql` (Postgres schema `branch`); dev/test rows in `db/seed.sql`. Tables: `users`, `projects`, `project_memberships` (roles: PI/Accountant/Staff/Admin), `donors`, `project_donations`, `expenditures` (status: approved/pending/denied/needs_more_info), `reports` (report_type: technical/narrative). - Kysely connects via `db.ts` in each lambda (`Kysely` + `pg.Pool`, env `DB_HOST/DB_PORT/DB_USER/DB_PASSWORD/DB_NAME`). Query with the `branch.` schema prefix: `db.selectFrom('branch.users')`. -- **Changing the schema:** edit `db_setup.sql`. The `regenerate-db-types` workflow regenerates `shared/types/db-types.d.ts` (via `kysely-codegen`) on push. Never hand-edit `db-types.d.ts`. -- Root `package.json` has TypeORM migration scripts (`migration:generate` etc.) — legacy NestJS scaffolding, **not** used by the lambdas. Lambdas use Kysely + the raw SQL setup file. +- **Changing the schema:** `make new-migration NAME=add_thing` → write SQL → `make migrate` (applies it and regenerates `shared/types/db-types.d.ts`). `make show-migrations` shows applied vs pending; the ledger is `branch.kysely_migration`. Forward-only — never edit a merged migration, and never hand-edit `db-types.d.ts`. Migrations apply to prod automatically on merge **before** the lambda deploy, so a single PR may only contain additive changes; see `db/README.md` for expand/contract. +- The migration runner is kysely's `Migrator` over plain `.sql` files (`db/src/`). Root `package.json` still carries NestJS/TypeORM **dependencies** from the original scaffold, but nothing uses them; its dead `migration:*` scripts have been removed. ## Shared packages @@ -74,7 +74,7 @@ Automatic on push to `main` touching `apps/backend/lambdas/**` or `shared/types/ **`branch.users.is_admin` is the single source of truth for admin.** There is no promotion from a Cognito group, and no pre-token-generation trigger, so `is_admin` is not a JWT claim — `GET /auth/me` is the only way a client can learn it. -**A `branch.users` row with `cognito_sub IS NULL` is a pending invitation**, created by the `db_setup.sql` seeds or by admin `POST /users`. `POST /auth/register` claims such a row (setting `cognito_sub`, never touching `is_admin`) instead of returning 409. Registration only 409s when the row is already claimed. +**A `branch.users` row with `cognito_sub IS NULL` is a pending invitation**, created by the `db/seed.sql` rows or by admin `POST /users`. `POST /auth/register` claims such a row (setting `cognito_sub`, never touching `is_admin`) instead of returning 409. Registration only 409s when the row is already claimed. **Bootstrapping the first admin** is a manual SQL statement in every environment, because `is_admin` can only be set by an existing admin: `make grant-admin EMAIL=…` locally, or the equivalent `UPDATE` against RDS in production. diff --git a/apps/backend/BACKEND_ANALYSIS.md b/apps/backend/BACKEND_ANALYSIS.md index 4914dc00..c5af9550 100644 --- a/apps/backend/BACKEND_ANALYSIS.md +++ b/apps/backend/BACKEND_ANALYSIS.md @@ -43,7 +43,8 @@ apps/backend/ ├── docker-compose.yml # Orchestrates all services ├── Makefile # Developer convenience commands ├── db/ -│ └── db_setup.sql # Schema + seed data (single file) +│ ├── migrations/ # Schema migrations, applied in filename order +│ └── seed.sql # Dev/test seed data └── lambdas/ ├── auth/ # Authentication (Cognito + DB) ├── donors/ # Donor management (read-only) @@ -324,17 +325,19 @@ This file is labeled as a "unit test" but doesn't mock the database — it conne ## Database & Schema Issues -### 20. Destructive Schema Reset on Every Init +### 20. Destructive Schema Reset on Every Init — RESOLVED -**File:** `db/db_setup.sql`, line 1: +`db/db_setup.sql` opened with `DROP SCHEMA IF EXISTS branch CASCADE;`, there was no +migration system, and nothing applied the schema to production at all (RDS was built +by hand). -```sql -DROP SCHEMA IF EXISTS branch CASCADE; -``` - -This drops the entire schema and recreates it. There is no migration system in place. This is dangerous if accidentally run against a production database. - -**Recommendation:** Implement a proper migration system (e.g., Kysely migrations, Flyway, or a simple numbered migration approach). +**Resolved:** migrations are now plain `.sql` files in `db/migrations/`, applied by +kysely's `Migrator` and tracked in `branch.kysely_migration`. Locally: +`make new-migration NAME=x` → `make migrate`. In CI, the `migrate` job in +`lambda-deploy.yml` snapshots RDS and applies pending migrations on merge to main, +**before** the lambda zips deploy, so a failed migration blocks the code deploy. PR +gates (`migrations-fresh`, `migrations-guard`) reject destructive statements, edits to +already-merged migrations, and stale generated types. See `db/README.md`. ### 21. Missing Database Indexes @@ -446,6 +449,6 @@ This is likely a mistake. The `aws-lambda` npm package is a deprecated Lambda ru | 17 | Use environment variables in test database connections | Small | | 18 | Add tests for donors service and expand coverage | Medium | | 19 | Mock database in projects "unit" tests | Small | -| 20 | Implement database migration system | Medium-Large | +| 20 | ~~Implement database migration system~~ (done — `db/migrations/`, `db/README.md`) | Medium-Large | | 24-25 | Fix Dockerfile for production use | Medium | | 26 | Align dependency versions across services | Small | diff --git a/apps/backend/Makefile b/apps/backend/Makefile index 937be50e..60095c09 100644 --- a/apps/backend/Makefile +++ b/apps/backend/Makefile @@ -1,12 +1,20 @@ -.PHONY: up down logs build clean restart ps health +.PHONY: up down logs build clean restart ps health migrate show-migrations new-migration seed types db-reset # Default environment file ENV_FILE ?= .env -# Start all services (foreground, Ctrl+C stops everything) +# Run container commands that write into the repo as the invoking user, so +# generated files aren't left root-owned. npm needs a writable cache to match. +DOCKER_USER = --user "$(shell id -u):$(shell id -g)" -e npm_config_cache=/tmp/.npm -e HOME=/tmp + +# Start all services (foreground, Ctrl+C stops everything). +# +# --abort-on-container-failure, not --abort-on-container-exit: the `migrator` +# service is one-shot and exits 0 as soon as migrations are applied, which +# --abort-on-container-exit would treat as a reason to tear down the whole stack. up: @echo "Starting all services..." - docker compose --env-file $(ENV_FILE) up --build --abort-on-container-exit + docker compose --env-file $(ENV_FILE) up --build --abort-on-container-failure # Stop all services down: @@ -34,7 +42,7 @@ build: restart: @echo "Restarting all services..." docker compose down - docker compose --env-file $(ENV_FILE) up -d --build + docker compose --env-file $(ENV_FILE) up -d --build --abort-on-container-failure # List running containers ps: @@ -86,17 +94,87 @@ grant-admin: docker compose exec -T postgres psql -U branch_dev -d branch_db \ -c "UPDATE branch.users SET is_admin = TRUE WHERE email = lower('$(EMAIL)');" -# Reset database (WARNING: destroys all data) +# --------------------------------------------------------------------------- +# Migrations. Schema changes live in db/migrations/*.sql and are applied to +# production automatically on merge -- see apps/backend/db/README.md. +# --------------------------------------------------------------------------- + +# Apply pending migrations, seed if the database is empty, regenerate the shared +# DB types, then show what's applied. +# +# THE command to run after adding or pulling a migration. Safe any time: with +# nothing pending it just prints status. Regenerating types is folded in here on +# purpose, so it cannot be forgotten. +migrate: + @docker compose --env-file $(ENV_FILE) build migrator + @docker compose --env-file $(ENV_FILE) run --rm migrator npm run migrate + @docker compose --env-file $(ENV_FILE) run --rm migrator npm run seed -- --if-empty + @$(MAKE) types + @$(MAKE) show-migrations + @echo "" + @echo "Commit db/migrations/*.sql together with shared/types/db-types.d.ts." + +# List every migration and whether it has been applied. Read-only. +# +# The source of truth is the branch.kysely_migration table -- one row per applied +# migration -- diffed against the .sql files in db/migrations. +show-migrations: + @docker compose --env-file $(ENV_FILE) run --rm migrator npm run migrate:status + +# Create an empty timestamped migration file, then say what to do next. +# usage: make new-migration NAME=add_expenditure_project_id_index +# +# Runs in the container (so Docker is the only prerequisite) with db/migrations +# bind-mounted, so the new file lands in the repo. DOCKER_USER keeps it owned by +# you rather than root. +new-migration: + @test -n "$(NAME)" || (echo "usage: make new-migration NAME=add_expenditure_index" && exit 1) + @docker compose --env-file $(ENV_FILE) run --rm --no-deps $(DOCKER_USER) \ + -v "$(CURDIR)/db/migrations:/db/migrations" \ + migrator npm run --silent migrate:new -- $(NAME) + @echo "" + @echo "Now edit the file above, then run 'make migrate'." + +# Truncate every table and re-apply db/seed.sql. Leaves the schema alone. +seed: + @docker compose --env-file $(ENV_FILE) run --rm migrator npm run seed + +# Regenerate shared/types/db-types.d.ts from the local database. Called for you by +# `make migrate`. +# +# Runs in the container on purpose: on the host, localhost:5432 may resolve to a +# separate postgres install (homebrew, Postgres.app) instead of the compose one, +# and you would silently generate types from the wrong database. Inside the +# container DB_HOST=postgres is unambiguous. +types: + @docker compose --env-file $(ENV_FILE) run --rm $(DOCKER_USER) \ + -v "$(CURDIR)/../../shared/types:/shared/types" \ + -e TYPES_OUT=/shared/types/db-types.d.ts \ + migrator npm run --silent types + +# Rebuild schema "branch" from the migration files and reseed, in place. +# Use `make down-v` if you need to destroy the volume itself. db-reset: - @echo "WARNING: This will destroy all data in the database!" + @echo "WARNING: this drops and rebuilds schema \"branch\" -- all local data is lost!" @read -p "Are you sure? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1 - docker compose down -v - docker compose --env-file $(ENV_FILE) up -d postgres - @echo "Database reset complete. Run 'make up' to start all services." + @docker compose --env-file $(ENV_FILE) build migrator + @docker compose --env-file $(ENV_FILE) run --rm migrator npm run reset + @$(MAKE) types + @echo "Schema rebuilt from db/migrations and reseeded." # Help help: @echo "Available commands:" + @echo "" + @echo " Migrations (see apps/backend/db/README.md):" + @echo " make migrate - Apply pending migrations, seed if empty, regenerate types" + @echo " make show-migrations - Show which migrations are applied and which are pending" + @echo " make new-migration NAME= - Create a new empty migration file" + @echo " make seed - Truncate every table and re-apply db/seed.sql" + @echo " make types - Regenerate shared/types/db-types.d.ts" + @echo " make db-reset - Rebuild schema from migrations + reseed (destroys data)" + @echo "" + @echo " Services:" @echo " make up - Start all services" @echo " make down - Stop all services" @echo " make down-v - Stop all services and remove volumes" @@ -109,5 +187,4 @@ help: @echo " make clean - Clean up Docker resources" @echo " make db-shell - Open PostgreSQL shell" @echo " make grant-admin EMAIL= - Promote a local user to admin" - @echo " make db-reset - Reset database (WARNING: destroys data)" @echo " make help - Show this help message" diff --git a/apps/backend/README.md b/apps/backend/README.md index 4bafdc46..c10e4c9b 100644 --- a/apps/backend/README.md +++ b/apps/backend/README.md @@ -53,7 +53,7 @@ The backend consists of microservices, each running as a separate container: | --------------------------------- | ------------------------------------------------------ | | `make up` | Start all services (builds if needed) | | `make down` | Stop all services | -| `make down-v` | Stop all services and remove volumes (resets database) | +| `make down-v` | Stop all services and remove volumes (destroys the DB) | | `make logs` | View logs for all services | | `make logs-service SERVICE=users` | View logs for a specific service | | `make build` | Build all images without starting | @@ -62,9 +62,22 @@ The backend consists of microservices, each running as a separate container: | `make health` | Check health of all services | | `make clean` | Clean up Docker resources | | `make db-shell` | Open PostgreSQL shell | -| `make db-reset` | Reset database (destroys all data) | | `make help` | Show all available commands | +### Schema migrations + +See [`db/README.md`](db/README.md) for the rules — migrations are applied to +production automatically on merge, so a single PR may only contain additive changes. + +| Command | Description | +| -------------------------------------- | ------------------------------------------------------------------------ | +| `make new-migration NAME=add_thing` | Create an empty timestamped migration in `db/migrations/` | +| `make migrate` | Apply pending migrations, seed if empty, regenerate DB types, show status | +| `make show-migrations` | Show which migrations are applied and which are pending | +| `make seed` | Truncate every table and re-apply `db/seed.sql` | +| `make types` | Regenerate `shared/types/db-types.d.ts` from the local database | +| `make db-reset` | Rebuild schema `branch` from migrations and reseed (destroys all data) | + ## Environment Variables Configure services via `.env` file or environment variables: @@ -187,7 +200,10 @@ apps/backend/ ├── .env.example # Example environment file ├── README.md # This file ├── db/ -│ └── db_setup.sql # Database schema and seed data +│ ├── migrations/ # Schema migrations (applied in filename order) +│ ├── seed.sql # Dev/test seed data (never applied to prod) +│ ├── testkit.ts # ensureSchema()/resetData() for lambda tests +│ └── src/ # Migration runner + type generator CLI └── lambdas/ ├── auth/ # Authentication service ├── donors/ # Donors service diff --git a/apps/backend/db/Dockerfile b/apps/backend/db/Dockerfile new file mode 100644 index 00000000..30b3f55f --- /dev/null +++ b/apps/backend/db/Dockerfile @@ -0,0 +1,18 @@ +# One-shot migration runner, built from the repo root like the lambdas. +# +# A Dockerfile rather than bind-mounting the repo into node:20-alpine and running +# `npm ci` on every `up`: that would clobber the host's apps/backend/db/node_modules +# with alpine-built artifacts and cost ~30s per start. This way docker caches the +# install layer. +FROM node:20-alpine + +WORKDIR /db + +COPY apps/backend/db/package.json apps/backend/db/package-lock.json ./ +RUN npm ci --no-audit --no-fund + +COPY apps/backend/db/tsconfig.json apps/backend/db/testkit.ts apps/backend/db/seed.sql ./ +COPY apps/backend/db/src ./src +COPY apps/backend/db/migrations ./migrations + +CMD ["npm", "run", "migrate"] diff --git a/apps/backend/db/README.md b/apps/backend/db/README.md new file mode 100644 index 00000000..01303f0a --- /dev/null +++ b/apps/backend/db/README.md @@ -0,0 +1,134 @@ +# Database migrations + +Schema changes are plain `.sql` files in `migrations/`, applied by [kysely's +`Migrator`](https://kysely.dev/docs/migrations). They are applied to **production +automatically when your PR merges**. + +## Changing the schema + +```bash +cd apps/backend +make up # if the stack isn't already running + +make new-migration NAME=add_expenditure_notes # creates migrations/_add_expenditure_notes.sql +# ...write your SQL... +make migrate # applies it, reseeds if empty, regenerates types, prints status +``` + +Then run the tests for any lambda you touched (`cd lambdas/ && npm test`) and +commit `migrations/*.sql` **together with** `shared/types/db-types.d.ts`. + +Other commands: `make show-migrations` (what's applied vs pending), `make seed` +(truncate + reseed), `make db-reset` (rebuild the schema from scratch), +`make types` (regenerate types only). + +## The one rule that matters + +**Migrations run against production BEFORE the new lambda code is deployed.** For +that window — and indefinitely, if the deploy fails — the **currently deployed +code runs against your new schema**. So: + +> Every migration must be safe for the code that is live right now. + +**Safe in a single PR (expand):** + +- `CREATE TABLE` +- `ADD COLUMN` that is nullable, or `NOT NULL DEFAULT ` +- `CREATE INDEX` (not `CONCURRENTLY` — see below) +- widening a type (`VARCHAR(100)` → `VARCHAR(200)`, `INT` → `BIGINT`) +- `ALTER TYPE ... ADD VALUE` +- backfilling data into columns the live code doesn't read + +**Requires two merged PRs (expand, then contract):** + +- `DROP COLUMN` / `DROP TABLE` — PR 1 stops the code using it and deploys; PR 2 drops it +- **renames** — PR 1 adds the new column, backfills, and writes both; PR 2 reads the + new one; PR 3 drops the old one +- `ADD COLUMN NOT NULL` with no default — the live code's `INSERT`s omit it and start + failing immediately +- adding `UNIQUE` / `CHECK` / `FOREIGN KEY` that live data or live code could violate +- narrowing a type, or `SET NOT NULL` on an existing nullable column + +CI rejects the destructive statements outright. If you genuinely are doing the +contract half of an expand/contract, add `-- allow-destructive: ` at the top +of the migration. + +**There is no `down`.** Production is forward-only: to undo a bad migration, write a +new one. Never edit a migration that has been merged — someone has already run it, +and CI will reject the change. + +## How it works + +- `migrations/*.sql` — applied in filename order. `make new-migration` generates a + UTC `YYYYMMDDHHMMSS_` prefix so concurrent PRs can't collide. +- `seed.sql` — dev/test data only, **never applied to production**. +- `testkit.ts` — `ensureSchema()` / `resetData()` used by the lambda tests. +- `src/` — the runner CLI, the type generator, and the shared post-processing that + keeps local and CI type output byte-identical. + +**What's applied is tracked in the database**, in `branch.kysely_migration` — one row +per applied migration (`name`, `timestamp`). "Pending" is just the `.sql` files on +disk minus the rows in that table. It's an ordinary table, so in any environment: + +```sql +select * from branch.kysely_migration order by name; +``` + +All pending migrations run inside a **single transaction** with +`search_path = branch, public`, so table names can be unqualified and a failure +part-way through rolls the whole run back. That's also why `CREATE INDEX +CONCURRENTLY` and `VACUUM` don't work here — they can't run in a transaction. This +database is tiny; a plain `CREATE INDEX` is fine. + +Out-of-order merges are allowed (`allowUnorderedMigrations`): if your migration +merges after someone whose timestamp is later, it simply applies late. The +alternative — the default — is a production deploy that fails with `corrupted +migrations` and can only be unblocked by hand-editing `kysely_migration` in RDS. + +## In CI + +| When | What happens | +| --- | --- | +| PR opened/updated with a migration | A bot posts a checklist comment. `migrations-fresh` applies every migration to an empty Postgres, checks idempotency, and verifies the committed types match. `migrations-guard` rejects edits to already-merged migrations, bad filenames, and unsafe SQL. | +| Types are stale | The `Schema Change Checks` workflow regenerates `shared/types/db-types.d.ts` and pushes it to your branch. **Expect one red `migrations-fresh` run before that commit lands** — that's normal, not a flake. | +| PR merges to main | The `migrate` job in `lambda-deploy.yml` snapshots RDS, applies pending migrations, and only then lets the lambda zips deploy. A failed migration blocks the code deploy. | +| Preview environments | Migrations are **never** applied there — previews share the production database. A PR that adds a migration cannot be fully previewed; endpoints using the new columns will fail until it merges. | + +If a migration fails in CI, production is left on the old schema **and** the old +code. Fix it forward with a new commit. + +Two rollback levers exist, both free at this database's size and both a ~20-minute +manual restore-into-a-new-instance procedure, not a button: + +- a `branch-premigrate--` snapshot taken before every migration run, tagged + with the commit. The five most recent are kept — manual snapshots never expire on + their own, so the job prunes older ones. +- 7 days of point-in-time recovery (`backup_retention_period` in + `infrastructure/aws/main.tf`), which can restore to any second in the window. + +## One-time: adopting an existing database + +`0000_baseline_schema.sql` is the schema as it existed before migrations, and is the +only migration allowed to use `IF NOT EXISTS` — that's what lets it be applied to a +database that already has these tables. + +`IF NOT EXISTS` skips the **entire** `CREATE TABLE` when the table exists, so it +cannot detect a column or constraint that differs. Before running the migrator +against a pre-existing database, diff it: + +```bash +# use a pg_dump matching the server's major version +pg_dump --schema-only --schema=branch --no-owner --no-privileges --no-comments \ + -T 'branch.kysely_migration*' -d "$URL" | grep -v '^--' +``` + +Run that against a local database with only the baseline applied, and against the +target; `diff -u` the two. Once it's empty, run `npm run migrate` against the target +with a human watching — the baseline no-ops and records itself in the ledger. + +If the schemas genuinely diverge, use `npm run db -- stamp 0000_baseline_schema` to +record it without executing, then write a follow-up migration reconciling the +difference. + +The `migrate` job refuses to run if `0000_baseline_schema` is still pending, on the +assumption that it means `DB_HOST` is pointing somewhere unexpected. diff --git a/apps/backend/db/db_setup.sql b/apps/backend/db/db_setup.sql deleted file mode 100644 index 16163e8c..00000000 --- a/apps/backend/db/db_setup.sql +++ /dev/null @@ -1,123 +0,0 @@ -DROP SCHEMA IF EXISTS branch CASCADE; -CREATE SCHEMA branch; -SET search_path TO branch; - -CREATE TABLE users ( - user_id SERIAL PRIMARY KEY, - cognito_sub VARCHAR(255) UNIQUE, - name VARCHAR(100) NOT NULL, - email VARCHAR(150) UNIQUE NOT NULL, - is_admin BOOLEAN DEFAULT FALSE, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - profile_image TEXT -); - -CREATE TABLE projects ( - project_id SERIAL PRIMARY KEY, - name VARCHAR(100) NOT NULL, - description TEXT NOT NULL, - total_budget NUMERIC(12,2), - start_date DATE, - end_date DATE, - currency VARCHAR(10) DEFAULT 'USD', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE project_memberships ( - membership_id SERIAL PRIMARY KEY, - project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, - user_id INT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, - role VARCHAR(30) NOT NULL CHECK (role IN ('PI', 'Accountant', 'Staff', 'Admin')), - start_date DATE, - hours NUMERIC(6,2), - UNIQUE (project_id, user_id) -); - -CREATE TABLE donors ( - donor_id SERIAL PRIMARY KEY, - organization VARCHAR(150) NOT NULL, - contact_name VARCHAR(100), - contact_email VARCHAR(150), - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE project_donations ( - donation_id SERIAL PRIMARY KEY, - donor_id INT NOT NULL REFERENCES donors(donor_id) ON DELETE CASCADE, - project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, - amount NUMERIC(12,2) NOT NULL CHECK (amount >= 0), - donated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE (donor_id, project_id) -); - -CREATE TABLE expenditures ( - expenditure_id SERIAL PRIMARY KEY, - project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, - entered_by INT REFERENCES users(user_id) ON DELETE SET NULL, - amount NUMERIC(12,2) NOT NULL CHECK (amount >= 0), - category VARCHAR(50), - description TEXT, - status VARCHAR(15) NOT NULL DEFAULT 'pending' CHECK (status IN ('approved', 'pending', 'denied', 'needs_more_info')), - receipt_url TEXT, - spent_on DATE NOT NULL DEFAULT CURRENT_DATE, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE reports ( - report_id SERIAL PRIMARY KEY, - project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, - title VARCHAR(255) NOT NULL, - object_url TEXT NOT NULL, - report_type TEXT NOT NULL DEFAULT 'technical' CHECK (report_type IN ('technical', 'narrative')), - date_created DATE NOT NULL DEFAULT CURRENT_DATE -); - --- These seeded admins intentionally have cognito_sub = NULL. A NULL cognito_sub --- means "pending invitation": POST /auth/register signs the email up in Cognito --- and CLAIMS this row (setting cognito_sub) rather than returning 409, which --- preserves user_id and is_admin. The same mechanism backs admin-created users --- (POST /users), which also insert without a cognito_sub. --- --- To sign in as one of these locally you must control the mailbox to receive the --- Cognito verification code. Otherwise register your own email and run --- `make grant-admin EMAIL=you@example.com`. -INSERT INTO users (name, email, is_admin) VALUES -('Ashley Duggan', 'ashley@branch.org', TRUE), -('Renee Reddy', 'renee@branch.org', TRUE), -('Nour Shoreibah', 'nour@branch.org', TRUE); - -INSERT INTO projects (name, description, total_budget, start_date, end_date, currency) VALUES -('Clinician Communication Study', 'Study of clinician-patient communication patterns', 500000, '2025-01-01', '2026-01-01', 'USD'), -('Health Education Initiative', 'Community health education and outreach', 300000, '2025-03-01', '2026-03-01', 'USD'), -('Policy Advocacy Program', 'Advocacy and policy change efforts', 200000, '2025-06-01', '2026-06-01', 'USD'), -('Proj B', '', 2500.50, NULL, NULL, 'USD'); - -INSERT INTO donors (organization, contact_name, contact_email) VALUES -('NIH', 'Dr. Sarah Lee', 'sarah@nih.gov'), -('Harvard Medical', 'John Smith', 'john@harvard.edu'), -('Wellcome Trust', 'Anna Johnson', 'anna@wellcome.org'); - -INSERT INTO project_donations (donor_id, project_id, amount, donated_at) VALUES -(1, 1, 150000, '2025-01-10'), -(2, 2, 120000, '2025-03-15'), -(3, 3, 90000, '2025-06-20'); - -INSERT INTO project_memberships (project_id, user_id, role, start_date, hours) VALUES -(1, 1, 'PI', '2025-01-01', 100.00), -(1, 2, 'Accountant', '2025-02-01', 80.00), -(2, 3, 'Staff', '2025-03-15', 60.00); - -INSERT INTO expenditures (project_id, entered_by, amount, category, description, spent_on) VALUES -(1, 1, 5000, 'Travel', 'Domestic conference attendance', '2025-02-10'), -(1, 1, 4200, 'Travel Foreign', 'International collaborator meeting in London', '2025-03-22'), -(2, 2, 3000, 'General', 'Recording device supplies', '2025-04-05'), -(2, 2, 1500, 'Visitor / Honorarium', 'Guest lecturer honorarium', '2025-05-18'), -(3, 3, 2500, 'General', 'Educational materials', '2025-07-12'), -(3, 3, 1800, 'Travel', 'Local outreach travel', '2025-08-03'); - -INSERT INTO reports (project_id, title, object_url) VALUES -(1, 'Clinician Communication Study Report', 'https://s3.amazonaws.com/branch-reports/clinician_communication_study_report.pdf'), -(2, 'Health Education Initiative Report', 'https://s3.amazonaws.com/branch-reports/health_education_initiative_report.pdf'), -(3, 'Policy Advocacy Program Report', 'https://s3.amazonaws.com/branch-reports/policy_advocacy_program_report.pdf'), -(2, 'Research Program Reports', 'https://s3.amazonaws.com/branch-reports/research_program_reports.pdf'), -(3, 'Health Care Data Reports', 'https://s3.amazonaws.com/branch-reports/health_care_data_reports.pdf'); diff --git a/apps/backend/db/kysely-codegen.config.js b/apps/backend/db/kysely-codegen.config.js deleted file mode 100644 index 79e5fd5b..00000000 --- a/apps/backend/db/kysely-codegen.config.js +++ /dev/null @@ -1,4 +0,0 @@ -export default { - dialect: 'postgres', - outFile: './backend/db/types.d.ts', -} diff --git a/apps/backend/db/migrations/0000_baseline_schema.sql b/apps/backend/db/migrations/0000_baseline_schema.sql new file mode 100644 index 00000000..3233272b --- /dev/null +++ b/apps/backend/db/migrations/0000_baseline_schema.sql @@ -0,0 +1,89 @@ +-- 0000_baseline_schema +-- +-- The schema as it existed before migrations were introduced, transcribed from +-- the old apps/backend/db/db_setup.sql. Table bodies are copied verbatim, since +-- this file has to match a production database that was built by hand. +-- +-- A `0` prefix so it sorts before every future YYYYMMDDHHMMSS-prefixed migration +-- by construction. +-- +-- This is the ONLY migration allowed to use IF NOT EXISTS. It has to be, because +-- production already has these tables and this migration is how that database +-- gets adopted into the ledger. In a normal migration IF NOT EXISTS hides drift +-- instead of failing, so don't copy this pattern. +-- +-- Note IF NOT EXISTS skips the ENTIRE CREATE TABLE when the table exists, so it +-- cannot detect a column or constraint that differs. Adoption is therefore gated +-- on a pg_dump diff against production -- see apps/backend/db/README.md. + +CREATE SCHEMA IF NOT EXISTS branch; + +CREATE TABLE IF NOT EXISTS users ( + user_id SERIAL PRIMARY KEY, + cognito_sub VARCHAR(255) UNIQUE, + name VARCHAR(100) NOT NULL, + email VARCHAR(150) UNIQUE NOT NULL, + is_admin BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + profile_image TEXT +); + +CREATE TABLE IF NOT EXISTS projects ( + project_id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + description TEXT NOT NULL, + total_budget NUMERIC(12,2), + start_date DATE, + end_date DATE, + currency VARCHAR(10) DEFAULT 'USD', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS project_memberships ( + membership_id SERIAL PRIMARY KEY, + project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, + user_id INT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + role VARCHAR(30) NOT NULL CHECK (role IN ('PI', 'Accountant', 'Staff', 'Admin')), + start_date DATE, + hours NUMERIC(6,2), + UNIQUE (project_id, user_id) +); + +CREATE TABLE IF NOT EXISTS donors ( + donor_id SERIAL PRIMARY KEY, + organization VARCHAR(150) NOT NULL, + contact_name VARCHAR(100), + contact_email VARCHAR(150), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS project_donations ( + donation_id SERIAL PRIMARY KEY, + donor_id INT NOT NULL REFERENCES donors(donor_id) ON DELETE CASCADE, + project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, + amount NUMERIC(12,2) NOT NULL CHECK (amount >= 0), + donated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE (donor_id, project_id) +); + +CREATE TABLE IF NOT EXISTS expenditures ( + expenditure_id SERIAL PRIMARY KEY, + project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, + entered_by INT REFERENCES users(user_id) ON DELETE SET NULL, + amount NUMERIC(12,2) NOT NULL CHECK (amount >= 0), + category VARCHAR(50), + description TEXT, + status VARCHAR(15) NOT NULL DEFAULT 'pending' CHECK (status IN ('approved', 'pending', 'denied', 'needs_more_info')), + receipt_url TEXT, + spent_on DATE NOT NULL DEFAULT CURRENT_DATE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS reports ( + report_id SERIAL PRIMARY KEY, + project_id INT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE, + title VARCHAR(255) NOT NULL, + object_url TEXT NOT NULL, + report_type TEXT NOT NULL DEFAULT 'technical' CHECK (report_type IN ('technical', 'narrative')), + date_created DATE NOT NULL DEFAULT CURRENT_DATE +); diff --git a/apps/backend/db/package-lock.json b/apps/backend/db/package-lock.json index fa9871f4..649f6761 100644 --- a/apps/backend/db/package-lock.json +++ b/apps/backend/db/package-lock.json @@ -1,23 +1,30 @@ { - "name": "db", + "name": "@branch/db", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "db", + "name": "@branch/db", "version": "1.0.0", "license": "ISC", "dependencies": { "kysely": "^0.28.8", - "kysely-codegen": "^0.19.0", "pg": "^8.16.3" + }, + "devDependencies": { + "@types/node": "^20.11.30", + "@types/pg": "^8.15.6", + "kysely-codegen": "^0.19.0", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" } }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", @@ -32,15 +39,134 @@ "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -52,22 +178,32 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, "license": "Python-2.0" }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -78,6 +214,7 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -90,6 +227,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -99,6 +237,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -115,6 +254,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -127,18 +267,21 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, "license": "MIT" }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, "license": "MIT" }, "node_modules/cosmiconfig": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "dev": true, "license": "MIT", "dependencies": { "env-paths": "^2.2.1", @@ -161,10 +304,18 @@ } } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -174,6 +325,7 @@ "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -186,6 +338,7 @@ "version": "12.0.3", "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-12.0.3.tgz", "integrity": "sha512-uc47g4b+4k/M/SeaW1y4OApx+mtLWl92l5LMPP0GNXctZqELk+YGgOPIIC5elYmUH4OuoK3JLhuRUYegeySiFA==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "dotenv": "^16.4.5" @@ -201,6 +354,7 @@ "version": "16.6.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -213,6 +367,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -222,6 +377,7 @@ "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "dev": true, "license": "MIT", "dependencies": { "is-arrayish": "^0.2.1" @@ -231,6 +387,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.8.0" @@ -240,6 +397,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -252,12 +410,14 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, "license": "ISC" }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -267,6 +427,7 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/git-diff/-/git-diff-2.0.6.tgz", "integrity": "sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==", + "dev": true, "license": "ISC", "dependencies": { "chalk": "^2.3.2", @@ -283,6 +444,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, "license": "MIT", "dependencies": { "color-convert": "^1.9.0" @@ -295,6 +457,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^3.2.1", @@ -309,6 +472,7 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, "license": "MIT", "dependencies": { "color-name": "1.1.3" @@ -318,12 +482,14 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, "license": "MIT" }, "node_modules/git-diff/node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -333,6 +499,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^3.0.0" @@ -346,6 +513,7 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -366,6 +534,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -375,6 +544,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -387,6 +557,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", @@ -404,6 +575,7 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -414,12 +586,14 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, "license": "ISC" }, "node_modules/interpret": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.10" @@ -429,12 +603,14 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, "license": "MIT" }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -450,6 +626,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -459,12 +636,14 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -477,6 +656,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, "license": "MIT" }, "node_modules/kysely": { @@ -492,6 +672,7 @@ "version": "0.19.0", "resolved": "https://registry.npmjs.org/kysely-codegen/-/kysely-codegen-0.19.0.tgz", "integrity": "sha512-ZpdQQnpfY0kh45CA6yPA9vdFsBE+b06Fx7QVcbL5rX//yjbA0yYGZGhnH7GTd4P4BY/HIv5uAfuOD83JVZf95w==", + "dev": true, "license": "MIT", "dependencies": { "chalk": "4.1.2", @@ -559,12 +740,14 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, "license": "MIT" }, "node_modules/loglevel": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6.0" @@ -574,10 +757,18 @@ "url": "https://tidelift.com/funding/github/npm/loglevel" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -591,6 +782,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -603,6 +795,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -612,6 +805,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -621,6 +815,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, "license": "MIT", "dependencies": { "callsites": "^3.0.0" @@ -633,6 +828,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.0.0", @@ -651,6 +847,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -660,6 +857,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, "license": "MIT" }, "node_modules/pg": { @@ -755,12 +953,14 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -773,6 +973,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -821,6 +1022,7 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", + "dev": true, "dependencies": { "resolve": "^1.1.6" }, @@ -832,6 +1034,7 @@ "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.0", @@ -852,6 +1055,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -861,6 +1065,7 @@ "version": "0.8.5", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "glob": "^7.0.0", @@ -878,6 +1083,7 @@ "version": "1.1.8", "resolved": "https://registry.npmjs.org/shelljs.exec/-/shelljs.exec-1.1.8.tgz", "integrity": "sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==", + "dev": true, "license": "ISC", "engines": { "node": ">= 4.0.0" @@ -896,6 +1102,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -908,6 +1115,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -920,6 +1128,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -928,10 +1137,93 @@ "node": ">=8.0" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/ts-node/node_modules/diff": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, "license": "ISC" }, "node_modules/xtend": { @@ -943,10 +1235,21 @@ "node": ">=0.4" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/zod": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/apps/backend/db/package.json b/apps/backend/db/package.json index 99ce4d95..98472569 100644 --- a/apps/backend/db/package.json +++ b/apps/backend/db/package.json @@ -1,17 +1,27 @@ { - "name": "db", + "name": "@branch/db", "version": "1.0.0", - "main": "kysely-codegen.config.js", + "private": true, + "description": "Database migrations, seed data and type generation for BRANCH.", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "db": "ts-node --transpile-only src/cli.ts", + "migrate": "npm run db -- up", + "migrate:status": "npm run db -- status", + "migrate:new": "npm run db -- new", + "seed": "npm run db -- seed", + "reset": "npm run db -- reset", + "types": "ts-node --transpile-only src/generate-types.ts" }, - "keywords": [], - "author": "", "license": "ISC", - "description": "", "dependencies": { "kysely": "^0.28.8", - "kysely-codegen": "^0.19.0", "pg": "^8.16.3" + }, + "devDependencies": { + "@types/node": "^20.11.30", + "@types/pg": "^8.15.6", + "kysely-codegen": "^0.19.0", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" } } diff --git a/apps/backend/db/seed.sql b/apps/backend/db/seed.sql new file mode 100644 index 00000000..5deea79a --- /dev/null +++ b/apps/backend/db/seed.sql @@ -0,0 +1,58 @@ +-- Seed data for local development and tests. NEVER applied to production. +-- +-- Ids are load-bearing: tests assume users 1-3, projects 1-4 and donors 1-3 +-- exist, and that any row they create lands on 4+. resetData() in db/testkit.ts +-- TRUNCATEs with RESTART IDENTITY before running this file, so every sequence +-- starts at 1. Do not reorder or remove rows without checking the tests. +-- +-- Fully schema-qualified so it never depends on search_path. + +-- These seeded admins intentionally have cognito_sub = NULL. A NULL cognito_sub +-- means "pending invitation": POST /auth/register signs the email up in Cognito +-- and CLAIMS this row (setting cognito_sub) rather than returning 409, which +-- preserves user_id and is_admin. The same mechanism backs admin-created users +-- (POST /users), which also insert without a cognito_sub. +-- +-- To sign in as one of these locally you must control the mailbox to receive the +-- Cognito verification code. Otherwise register your own email and run +-- `make grant-admin EMAIL=you@example.com`. +INSERT INTO branch.users (name, email, is_admin) VALUES +('Ashley Duggan', 'ashley@branch.org', TRUE), +('Renee Reddy', 'renee@branch.org', TRUE), +('Nour Shoreibah', 'nour@branch.org', TRUE); + +INSERT INTO branch.projects (name, description, total_budget, start_date, end_date, currency) VALUES +('Clinician Communication Study', 'Study of clinician-patient communication patterns', 500000, '2025-01-01', '2026-01-01', 'USD'), +('Health Education Initiative', 'Community health education and outreach', 300000, '2025-03-01', '2026-03-01', 'USD'), +('Policy Advocacy Program', 'Advocacy and policy change efforts', 200000, '2025-06-01', '2026-06-01', 'USD'), +('Proj B', '', 2500.50, NULL, NULL, 'USD'); + +INSERT INTO branch.donors (organization, contact_name, contact_email) VALUES +('NIH', 'Dr. Sarah Lee', 'sarah@nih.gov'), +('Harvard Medical', 'John Smith', 'john@harvard.edu'), +('Wellcome Trust', 'Anna Johnson', 'anna@wellcome.org'); + +INSERT INTO branch.project_donations (donor_id, project_id, amount, donated_at) VALUES +(1, 1, 150000, '2025-01-10'), +(2, 2, 120000, '2025-03-15'), +(3, 3, 90000, '2025-06-20'); + +INSERT INTO branch.project_memberships (project_id, user_id, role, start_date, hours) VALUES +(1, 1, 'PI', '2025-01-01', 100.00), +(1, 2, 'Accountant', '2025-02-01', 80.00), +(2, 3, 'Staff', '2025-03-15', 60.00); + +INSERT INTO branch.expenditures (project_id, entered_by, amount, category, description, spent_on) VALUES +(1, 1, 5000, 'Travel', 'Domestic conference attendance', '2025-02-10'), +(1, 1, 4200, 'Travel Foreign', 'International collaborator meeting in London', '2025-03-22'), +(2, 2, 3000, 'General', 'Recording device supplies', '2025-04-05'), +(2, 2, 1500, 'Visitor / Honorarium', 'Guest lecturer honorarium', '2025-05-18'), +(3, 3, 2500, 'General', 'Educational materials', '2025-07-12'), +(3, 3, 1800, 'Travel', 'Local outreach travel', '2025-08-03'); + +INSERT INTO branch.reports (project_id, title, object_url) VALUES +(1, 'Clinician Communication Study Report', 'https://s3.amazonaws.com/branch-reports/clinician_communication_study_report.pdf'), +(2, 'Health Education Initiative Report', 'https://s3.amazonaws.com/branch-reports/health_education_initiative_report.pdf'), +(3, 'Policy Advocacy Program Report', 'https://s3.amazonaws.com/branch-reports/policy_advocacy_program_report.pdf'), +(2, 'Research Program Reports', 'https://s3.amazonaws.com/branch-reports/research_program_reports.pdf'), +(3, 'Health Care Data Reports', 'https://s3.amazonaws.com/branch-reports/health_care_data_reports.pdf'); diff --git a/apps/backend/db/src/cli.ts b/apps/backend/db/src/cli.ts new file mode 100644 index 00000000..1db07d8b --- /dev/null +++ b/apps/backend/db/src/cli.ts @@ -0,0 +1,231 @@ +/** + * db CLI -- the single entrypoint for everything schema related. + * + * npm run migrate apply every pending migration + * npm run migrate:status list applied/pending + * npm run migrate:new -- x scaffold migrations/_x.sql + * npm run seed truncate + re-apply seed.sql + * npm run seed -- --if-empty seed only an empty database + * npm run reset rebuild the schema from migrations, then seed + * npm run db -- stamp NAME record a migration as applied WITHOUT running it + * + * The Makefile in apps/backend wraps these; CI calls them directly. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { sql } from 'kysely'; +import type { PoolClient } from 'pg'; +import { SCHEMA, createPool, describeTarget } from './config'; +import { createDb, createMigrator } from './migrator'; +import { MIGRATIONS_DIR } from './provider'; +import { + migrationsFingerprint, + rebuildSchema, + resetData, + seedIfEmpty, + stampFingerprint, +} from '../testkit'; + +const TEMPLATE = `-- __NAME__ +-- +-- Every pending migration runs inside a SINGLE transaction, with +-- search_path = branch, public -- so table names can be unqualified, and +-- CREATE INDEX CONCURRENTLY / VACUUM will not work here. +-- +-- This migration is applied to PRODUCTION automatically when the PR merges, +-- BEFORE the new lambda code is deployed. It must be safe for the code that is +-- live right now: additive changes only. See apps/backend/db/README.md for the +-- expand/contract rules that destructive changes need. +-- +-- Forward-only: there is no rollback. Fix a mistake with a new migration, and +-- never edit a migration that has been merged -- someone has already run it. +-- Do not use IF NOT EXISTS: you want a failure, not silent drift. + +`; + +async function up(): Promise { + const db = createDb(); + try { + console.log(`migrating ${describeTarget()}`); + // migrateToLatest NEVER throws -- it returns the error. Not checking it is + // the classic way to ship a green deploy that migrated nothing. + const { error, results } = await createMigrator(db).migrateToLatest(); + + for (const result of results ?? []) { + console.log(`${result.status.padEnd(11)} ${result.migrationName}`); + } + + if (error) { + console.error('\nmigration failed, nothing was applied:'); + console.error(error); + process.exitCode = 1; + return; + } + + if (!results?.length) console.log('database is already up to date'); + + // Lets db/testkit.ts skip its rebuild when the live schema already matches + // the migration files on disk. Value is a hex digest, safe to inline. + await sql + .raw( + `comment on schema ${SCHEMA} is 'migrations:${migrationsFingerprint()}'`, + ) + .execute(db); + } finally { + await db.destroy(); + } +} + +async function status(): Promise { + const db = createDb(); + try { + const migrations = await createMigrator(db).getMigrations(); + if (!migrations.length) { + console.log(`no *.sql files in ${MIGRATIONS_DIR}`); + return; + } + + console.log(`${describeTarget()}\n`); + for (const migration of migrations) { + const applied = migration.executedAt?.toISOString() ?? 'PENDING'; + console.log(`${applied.padEnd(26)} ${migration.name}`); + } + + const pending = migrations.filter( + (migration) => !migration.executedAt, + ).length; + console.log(`\n${migrations.length - pending} applied, ${pending} pending`); + } finally { + await db.destroy(); + } +} + +function newMigration(name?: string): void { + if (!name || !/^[a-z0-9]+(_[a-z0-9]+)*$/.test(name)) { + console.error( + 'usage: npm run migrate:new -- add_expenditure_project_id_index\n' + + ' (lower_snake_case, letters and digits only)', + ); + process.exitCode = 1; + return; + } + + // UTC YYYYMMDDHHMMSS, generated so nobody hand-types one: collisions between + // concurrent PRs are then effectively impossible. + const stamp = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14); + const file = path.join(MIGRATIONS_DIR, `${stamp}_${name}.sql`); + + fs.writeFileSync(file, TEMPLATE.replace('__NAME__', `${stamp}_${name}`), { + flag: 'wx', + }); + console.log(`created ${path.relative(process.cwd(), file)}`); +} + +/** + * Record a migration as applied WITHOUT running it. Only for adopting a database + * whose schema was built by hand -- verify with pg_dump first. The table + * definitions below are copied from kysely's Migrator so a later migrate run + * finds exactly what it expects. + */ +async function stamp(name?: string): Promise { + if (!name) { + console.error('usage: npm run db -- stamp 0000_baseline_schema'); + process.exitCode = 1; + return; + } + + const db = createDb(); + try { + await sql.raw(`create schema if not exists ${SCHEMA}`).execute(db); + await sql + .raw( + `create table if not exists ${SCHEMA}.kysely_migration ( + name varchar(255) not null primary key, + "timestamp" varchar(255) not null)`, + ) + .execute(db); + await sql + .raw( + `create table if not exists ${SCHEMA}.kysely_migration_lock ( + id varchar(255) not null primary key, + is_locked integer not null default 0)`, + ) + .execute(db); + await sql + .raw( + `insert into ${SCHEMA}.kysely_migration_lock (id, is_locked) + values ('migration_lock', 0) on conflict (id) do nothing`, + ) + .execute(db); + await sql + .raw( + `insert into ${SCHEMA}.kysely_migration (name, "timestamp") + values ('${name}', '${new Date().toISOString()}')`, + ) + .execute(db); + + console.log( + `stamped ${name} as applied (not executed) on ${describeTarget()}`, + ); + } finally { + await db.destroy(); + } +} + +async function withClient( + fn: (client: PoolClient) => Promise, +): Promise { + const pool = createPool(); + const client = await pool.connect(); + try { + await fn(client); + } finally { + client.release(); + await pool.end(); + } +} + +async function main(): Promise { + const [command, ...args] = process.argv.slice(2); + + switch (command) { + case 'up': + return up(); + case 'status': + return status(); + case 'new': + return newMigration(args[0]); + case 'stamp': + return stamp(args[0]); + case 'seed': + return withClient(async (client) => { + if (args.includes('--if-empty')) { + console.log( + (await seedIfEmpty(client)) + ? 'seeded' + : 'database already has data, left alone', + ); + } else { + await resetData(client); + console.log('truncated and reseeded'); + } + }); + case 'reset': + return withClient(async (client) => { + await rebuildSchema(client); + await resetData(client); + await stampFingerprint(client); + console.log('schema rebuilt from migrations and reseeded'); + }); + default: + console.error( + 'usage: db ', + ); + process.exitCode = 1; + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/apps/backend/db/src/config.ts b/apps/backend/db/src/config.ts new file mode 100644 index 00000000..f856de51 --- /dev/null +++ b/apps/backend/db/src/config.ts @@ -0,0 +1,51 @@ +import fs from 'node:fs'; +import { Pool } from 'pg'; + +/** Every table lives in this schema; the generated DB types key off it. */ +export const SCHEMA = 'branch'; + +/** + * The one place that knows how to reach the database. + * + * DATABASE_URL wins (CI, production, `psql`-style one-offs); otherwise the + * discrete DB_* vars, which are the same ones the lambdas' db.ts reads, with the + * same docker-compose defaults. That means `make migrate` needs no configuration + * at all locally, and CI/prod only need to set what they already set. + */ +export function databaseUrl(): string { + if (process.env.DATABASE_URL) return process.env.DATABASE_URL; + + const user = encodeURIComponent(process.env.DB_USER ?? 'branch_dev'); + const password = encodeURIComponent(process.env.DB_PASSWORD ?? 'password'); + const host = process.env.DB_HOST ?? 'localhost'; + const port = process.env.DB_PORT ?? '5432'; + const name = process.env.DB_NAME ?? 'branch_db'; + + return `postgresql://${user}:${password}@${host}:${port}/${name}`; +} + +/** + * TLS settings. Local postgres speaks plaintext; RDS terminates TLS, and when we + * connect to it from a GitHub runner the traffic crosses the public internet, so + * CI sets DB_SSL_CA to the AWS bundle and gets full certificate verification. + * DB_SSL=true without a CA falls back to encrypted-but-unverified. + */ +function sslConfig(): false | { ca: string } | { rejectUnauthorized: false } { + if (process.env.DB_SSL_CA) { + return { ca: fs.readFileSync(process.env.DB_SSL_CA, 'utf8') }; + } + if (process.env.DB_SSL === 'true') { + return { rejectUnauthorized: false }; + } + return false; +} + +export function createPool(): Pool { + return new Pool({ connectionString: databaseUrl(), ssl: sslConfig() }); +} + +/** Host and database only -- safe to log, unlike databaseUrl(). */ +export function describeTarget(): string { + const url = new URL(databaseUrl()); + return `${url.hostname}:${url.port || '5432'}${url.pathname}`; +} diff --git a/apps/backend/db/src/generate-types.ts b/apps/backend/db/src/generate-types.ts new file mode 100644 index 00000000..c41dc5a6 --- /dev/null +++ b/apps/backend/db/src/generate-types.ts @@ -0,0 +1,72 @@ +/** + * Regenerates shared/types/db-types.d.ts from the live database. + * + * This is the ONLY place the kysely-codegen invocation lives -- + * .github/workflows/regenerate-db-types.yaml runs this same script, so local and + * CI output are identical by construction rather than by convention. + * + * npm run types (or `make types` / `make migrate` from apps/backend) + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createPool, databaseUrl } from './config'; +import { assertSchemaIsCurrent } from '../testkit'; +import { postprocess } from './postprocess-types'; + +/** + * Where to write the generated types. TYPES_OUT exists because `make types` runs + * this inside the migrator container, where the repo root is not an ancestor of + * __dirname -- the Makefile bind-mounts shared/types and points TYPES_OUT at it. + * CI and a direct `npm run types` use the repo-relative default. + */ +const OUT = process.env.TYPES_OUT + ? path.resolve(process.env.TYPES_OUT) + : path.resolve(__dirname, '../../../../shared/types/db-types.d.ts'); + +async function main(): Promise { + // Generating from a database that is behind db/migrations silently produces + // stale types, so refuse instead of guessing. + const pool = createPool(); + const client = await pool.connect(); + try { + await assertSchemaIsCurrent(client); + } finally { + client.release(); + await pool.end(); + } + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'branch-types-')); + const tmp = path.join(tmpDir, 'db-types.d.ts'); + + execFileSync( + path.resolve(__dirname, '../node_modules/.bin/kysely-codegen'), + [ + '--url', + databaseUrl(), + '--dialect', + 'postgres', + // Ignore anything created outside schema "branch". kysely already excludes + // kysely_migration / kysely_migration_lock by name. + // + // Deliberately NOT --default-schema: that would strip the `branch.` prefix + // from the generated DB keys and break every db.selectFrom('branch.users') + // in all six lambdas. + '--include-pattern', + 'branch.*', + '--out-file', + tmp, + ], + { stdio: 'inherit' }, + ); + + fs.writeFileSync(OUT, postprocess(fs.readFileSync(tmp, 'utf8'))); + fs.rmSync(tmpDir, { recursive: true, force: true }); + console.log(`wrote ${path.relative(process.cwd(), OUT)}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/apps/backend/db/src/migrator.ts b/apps/backend/db/src/migrator.ts new file mode 100644 index 00000000..b7d6393f --- /dev/null +++ b/apps/backend/db/src/migrator.ts @@ -0,0 +1,52 @@ +import { CompiledQuery, Kysely, Migrator, PostgresDialect } from 'kysely'; +import { SCHEMA, createPool } from './config'; +import { SqlFileMigrationProvider } from './provider'; + +export function createDb(): Kysely { + return new Kysely({ + dialect: new PostgresDialect({ + pool: createPool(), + // Awaited by PostgresDriver before the connection is handed out, so every + // migration runs with search_path=branch and may use unqualified table + // names. Doing this with a `SET` inside a migration file instead would + // leak the setting onto a pooled connection. + onCreateConnection: async (connection) => { + await connection.executeQuery( + CompiledQuery.raw(`set search_path to ${SCHEMA}, public`), + ); + }, + }), + }); +} + +export function createMigrator(db: Kysely): Migrator { + return new Migrator({ + db, + provider: new SqlFileMigrationProvider(), + + // Bookkeeping lives beside the tables it tracks: branch.kysely_migration + // (one row per applied migration) and branch.kysely_migration_lock. + // + // Keep the DEFAULT table names. kysely-codegen's introspector skips tables + // named kysely_migration / kysely_migration_lock in every schema, so they + // stay out of shared/types/db-types.d.ts for free. Renaming them to + // something like schema_migrations would leak the interfaces into the + // shared types package. + migrationTableSchema: SCHEMA, + + // Two contributors' migrations will sometimes merge out of timestamp order + // (Alice authors first, Bob merges first). With the default `false` that + // makes the next production deploy fail with "corrupted migrations", + // fixable only by renaming the file and hand-editing kysely_migration in + // RDS. With `true` the late migration is simply applied late. Deletions and + // renames of already-applied migrations are still rejected either way, which + // is the guard that actually matters. + allowUnorderedMigrations: true, + + // Byte-order comparison, matching the .sort() the provider uses to order + // files. The default is localeCompare, whose handling of `_` can disagree + // with byte order and raise a spurious ordering error when two migrations + // share a timestamp. + nameComparator: (a, b) => (a < b ? -1 : a > b ? 1 : 0), + }); +} diff --git a/apps/backend/db/src/postprocess-types.ts b/apps/backend/db/src/postprocess-types.ts new file mode 100644 index 00000000..785bae59 --- /dev/null +++ b/apps/backend/db/src/postprocess-types.ts @@ -0,0 +1,37 @@ +/** + * Post-processing applied to kysely-codegen's output before it is written to + * shared/types/db-types.d.ts. + * + * This used to be an inline `node -e` heredoc inside + * .github/workflows/regenerate-db-types.yaml. It lives here so the local + * `npm run types` and the workflow produce byte-identical output by + * construction, rather than by two copies staying in sync by discipline. + */ + +const KYSELY_IMPORT = 'import type { ColumnType } from "kysely";'; + +const LOCAL_COLUMN_TYPE = [ + '/**', + " * Structurally identical to kysely's ColumnType, defined locally so this", + ' * package has no dependencies and lambdas stay fully self contained.', + ' */', + 'type ColumnType = {', + ' readonly __select__: SelectType;', + ' readonly __insert__: InsertType;', + ' readonly __update__: UpdateType;', + '};', +].join('\n'); + +/** + * Swaps kysely's ColumnType import for a local copy so @branch/types stays + * dependency-free. Throws if the import is absent -- a silent no-op would ship a + * shared package that imports something it does not depend on. + */ +export function postprocess(generated: string): string { + if (!generated.includes(KYSELY_IMPORT)) { + throw new Error( + `expected ${JSON.stringify(KYSELY_IMPORT)} in the kysely-codegen output`, + ); + } + return generated.replace(KYSELY_IMPORT, LOCAL_COLUMN_TYPE); +} diff --git a/apps/backend/db/src/provider.ts b/apps/backend/db/src/provider.ts new file mode 100644 index 00000000..375ce792 --- /dev/null +++ b/apps/backend/db/src/provider.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { sql } from 'kysely'; +import type { Kysely, Migration, MigrationProvider } from 'kysely'; + +export const MIGRATIONS_DIR = path.join(__dirname, '..', 'migrations'); + +/** Byte-order sort, matching what kysely uses internally to order migrations. */ +export function migrationFilenames(dir: string = MIGRATIONS_DIR): string[] { + return fs + .readdirSync(dir) + .filter((file) => file.endsWith('.sql')) + .sort(); +} + +/** + * Serves every `*.sql` file in db/migrations to kysely's Migrator. The migration + * name is the filename without `.sql`, and kysely orders by that name, so the + * numeric prefix defines the order. + * + * Forward-only by design: kysely's migrateDown silently does nothing for a + * migration with no `down` (it leaves the tracking row in place), so we never + * define one and never expose a rollback command. Undo by writing a new + * migration. + */ +export class SqlFileMigrationProvider implements MigrationProvider { + constructor(private readonly dir: string = MIGRATIONS_DIR) {} + + async getMigrations(): Promise> { + const migrations: Record = {}; + + for (const file of migrationFilenames(this.dir)) { + const name = file.slice(0, -'.sql'.length); + if (name.length > 255) { + throw new Error( + `migration name is longer than kysely_migration.name (varchar(255)): ${file}`, + ); + } + + const contents = fs.readFileSync(path.join(this.dir, file), 'utf8'); + migrations[name] = { + // node-postgres sends parameterless queries over the simple query + // protocol, so one file may hold many `;`-separated statements. + up: async (db: Kysely) => { + await sql.raw(contents).execute(db); + }, + }; + } + + return migrations; + } +} diff --git a/apps/backend/db/testkit.ts b/apps/backend/db/testkit.ts new file mode 100644 index 00000000..0ec21657 --- /dev/null +++ b/apps/backend/db/testkit.ts @@ -0,0 +1,211 @@ +/** + * Shared Postgres fixture helpers for the lambda tests and the db CLI. + * + * Deliberately dependency-free -- node builtins only, and the caller injects + * anything with a node-postgres shaped `query(sql)` (a Client, a PoolClient). + * That is what lets every lambda import this file by relative path + * (`../../../db/testkit`) without adding a package or a build step, in a repo + * with no npm workspaces. + * + * Typical use in a test file: + * + * beforeAll(async () => { ...await ensureSchema(client) }); + * beforeEach(async () => { ...await resetData(client) }); + */ +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +export const SCHEMA = 'branch'; + +const MIGRATIONS_DIR = path.join(__dirname, 'migrations'); +const SEED_PATH = path.join(__dirname, 'seed.sql'); + +export interface Queryable { + query(sql: string): Promise<{ rows?: Array> }>; +} + +/** + * Byte-order sort, matching the sort kysely's Migrator uses to order migrations. + * If these two ever disagree, tests and production apply migrations in different + * orders. + */ +function migrationFiles(): string[] { + return fs + .readdirSync(MIGRATIONS_DIR) + .filter((file) => file.endsWith('.sql')) + .sort(); +} + +let allSql: string | undefined; +function allMigrationSql(): string { + allSql ??= migrationFiles() + .map( + (file) => + `-- ${file}\n${fs.readFileSync( + path.join(MIGRATIONS_DIR, file), + 'utf8', + )}`, + ) + .join('\n'); + return allSql; +} + +let seed: string | undefined; +function seedSql(): string { + seed ??= fs.readFileSync(SEED_PATH, 'utf8'); + return seed; +} + +/** Hex digest of every migration file, so it is safe to inline into SQL. */ +export function migrationsFingerprint(): string { + return crypto.createHash('sha256').update(allMigrationSql()).digest('hex'); +} + +/** + * Writes kysely's migration ledger as if every migration file had just been + * applied by the Migrator: branch.kysely_migration with one row per file, plus + * branch.kysely_migration_lock. The DDL is copied from kysely's Migrator so a + * later `npm run migrate` finds exactly what it expects. + * + * Without this, rebuildSchema() would leave a schema whose tables exist but + * whose ledger is empty -- and since the dev stack and the tests share one local + * database, the next `make migrate` would try to re-apply every migration and + * fail on "already exists". + */ +export async function stampLedger(client: Queryable): Promise { + await client.query( + `CREATE TABLE IF NOT EXISTS ${SCHEMA}.kysely_migration ( + name varchar(255) NOT NULL PRIMARY KEY, + "timestamp" varchar(255) NOT NULL)`, + ); + await client.query( + `CREATE TABLE IF NOT EXISTS ${SCHEMA}.kysely_migration_lock ( + id varchar(255) NOT NULL PRIMARY KEY, + is_locked integer NOT NULL DEFAULT 0)`, + ); + await client.query( + `INSERT INTO ${SCHEMA}.kysely_migration_lock (id, is_locked) + VALUES ('migration_lock', 0) ON CONFLICT (id) DO NOTHING`, + ); + + const now = new Date().toISOString(); + const rows = migrationFiles() + .map((file) => `('${file.slice(0, -'.sql'.length)}', '${now}')`) + .join(', '); + if (rows) { + await client.query( + `INSERT INTO ${SCHEMA}.kysely_migration (name, "timestamp") + VALUES ${rows} ON CONFLICT (name) DO NOTHING`, + ); + } +} + +/** + * Drops schema `branch` and applies every migration file, in order, then records + * them in the ledger so the result is indistinguishable from a real migration + * run. Used by the tests and by `make db-reset`. + */ +export async function rebuildSchema(client: Queryable): Promise { + await client.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await client.query(`CREATE SCHEMA ${SCHEMA}`); + // Set search_path on the connection rather than inside the migration text: a + // bare `SET` in a migration file would leak onto a pooled connection. + await client.query(`SET search_path TO ${SCHEMA}, public`); + try { + // A parameterless multi-statement query goes over the simple query protocol + // and runs as one implicit transaction -- the same all-or-nothing semantics + // kysely's Migrator gives us in production. + await client.query(allMigrationSql()); + } finally { + await client.query('RESET search_path'); + } + await stampLedger(client); + await stampFingerprint(client); +} + +/** + * Records which set of migration files built this schema, as a schema comment. + * A comment rather than a table so it stays invisible to kysely-codegen. + */ +export async function stampFingerprint(client: Queryable): Promise { + await client.query( + `COMMENT ON SCHEMA ${SCHEMA} IS 'migrations:${migrationsFingerprint()}'`, + ); +} + +async function schemaIsCurrent(client: Queryable): Promise { + const { rows } = await client.query( + `SELECT obj_description(oid, 'pg_namespace') AS comment + FROM pg_namespace WHERE nspname = '${SCHEMA}'`, + ); + return rows?.[0]?.comment === `migrations:${migrationsFingerprint()}`; +} + +/** + * Call once per test file, in `beforeAll`. Rebuilds the schema only when the + * migration files have changed since it was last built, so adding a migration + * needs no manual reset and the normal case costs one SELECT. + */ +export async function ensureSchema(client: Queryable): Promise { + if (!(await schemaIsCurrent(client))) await rebuildSchema(client); +} + +/** Fails loudly instead of rebuilding. Used by the type generator. */ +export async function assertSchemaIsCurrent(client: Queryable): Promise { + if (!(await schemaIsCurrent(client))) { + throw new Error( + `schema "${SCHEMA}" was not built from the current db/migrations. Run \`make migrate\` (or \`make db-reset\`) first.`, + ); + } +} + +let truncateStatement: string | undefined; +async function truncateAll(client: Queryable): Promise { + if (truncateStatement === undefined) { + // Discovered, never hardcoded: a table added by a future migration is reset + // automatically instead of leaking rows between tests. + const { rows } = await client.query( + `SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) AS t + FROM pg_tables + WHERE schemaname = '${SCHEMA}' AND tablename NOT LIKE 'kysely_migration%' + ORDER BY tablename`, + ); + const tables = (rows ?? []).map((row) => row.t as string); + if (tables.length === 0) { + throw new Error( + `no tables in schema "${SCHEMA}" -- call ensureSchema() first`, + ); + } + truncateStatement = `TRUNCATE TABLE ${tables.join( + ', ', + )} RESTART IDENTITY CASCADE`; + } + return truncateStatement; +} + +/** + * Call in `beforeEach`. Empties every table, restarts every SERIAL sequence at 1, + * and re-applies seed.sql -- so seeded ids stay deterministic (users 1-3, + * projects 1-4, donors 1-3) and rows a test creates land on 4+, exactly as they + * did when tests re-ran the whole of db_setup.sql. + * + * Not covered by TRUNCATE: standalone sequences (ones not owned by a serial + * column) and materialized views. There are none today. + */ +export async function resetData(client: Queryable): Promise { + await client.query(await truncateAll(client)); + await client.query(seedSql()); +} + +/** + * Seeds only a database that has never been seeded, so `make up` does not wipe a + * locally registered account (or a `make grant-admin` promotion) on restart. + * Returns whether it seeded. + */ +export async function seedIfEmpty(client: Queryable): Promise { + const { rows } = await client.query(`SELECT 1 FROM ${SCHEMA}.users LIMIT 1`); + if (rows?.length) return false; + await resetData(client); + return true; +} diff --git a/apps/backend/db/tsconfig.json b/apps/backend/db/tsconfig.json new file mode 100644 index 00000000..460db106 --- /dev/null +++ b/apps/backend/db/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "esModuleInterop": true, + "moduleResolution": "node", + "strict": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "noEmit": true, + "sourceMap": true + }, + "include": ["src/*.ts", "testkit.ts"], + "exclude": ["node_modules"] +} diff --git a/apps/backend/docker-compose.yml b/apps/backend/docker-compose.yml index 1eb64c9a..f049743b 100644 --- a/apps/backend/docker-compose.yml +++ b/apps/backend/docker-compose.yml @@ -12,7 +12,6 @@ services: - '${DB_PORT:-5432}:5432' volumes: - postgres_data:/var/lib/postgresql/data - - ./db/db_setup.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro healthcheck: test: [ @@ -23,6 +22,32 @@ services: timeout: 5s retries: 5 + # Applies every pending migration, then seeds only if the database is empty. + # + # Runs on every `make up`, unlike the old /docker-entrypoint-initdb.d mount of + # db_setup.sql, which only fired on a brand new volume -- so a schema change + # used to need `make down-v`. + # + # One-shot: it exits when done, and the lambdas wait for that via + # service_completed_successfully. That is also why the Makefile passes + # --abort-on-container-failure rather than --abort-on-container-exit; the + # latter would tear the whole stack down the moment this succeeds. + migrator: + build: + context: ../.. + dockerfile: apps/backend/db/Dockerfile + container_name: branch-migrator + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_USER: ${DB_USER:-branch_dev} + DB_PASSWORD: ${DB_PASSWORD:-password} + DB_NAME: ${DB_NAME:-branch_db} + command: sh -c "npm run migrate && npm run seed -- --if-empty" + depends_on: + postgres: + condition: service_healthy + # Users Service users: build: @@ -43,6 +68,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully # Projects Service projects: @@ -64,6 +91,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully # Donors Service donors: @@ -85,6 +114,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully # Expenditures Service expenditures: @@ -106,6 +137,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully # Reports Service reports: @@ -131,6 +164,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully # Auth Service auth: @@ -152,6 +187,8 @@ services: depends_on: postgres: condition: service_healthy + migrator: + condition: service_completed_successfully volumes: postgres_data: diff --git a/apps/backend/lambdas/AGENTS.md b/apps/backend/lambdas/AGENTS.md index 62f35751..46a32bc6 100644 --- a/apps/backend/lambdas/AGENTS.md +++ b/apps/backend/lambdas/AGENTS.md @@ -84,7 +84,7 @@ Convention: optional `page` + `limit` query params → `offset = (page-1)*limit` ## Testing - `*.unit.test.ts` — mock `../auth` and (sometimes) `../db`; test routing/validation/authz branches. No DB needed. -- `*.e2e.test.ts` — seed Postgres from `db_setup.sql`, mock only `authenticateRequest` to inject an auth context, hit the real handler against the real DB. +- `*.e2e.test.ts` — `ensureSchema()` in `beforeAll` builds schema `branch` from `db/migrations` if stale; `resetData()` in `beforeEach` truncates with `RESTART IDENTITY` and re-applies `db/seed.sql` (so seeded ids stay 1-3 and created rows land on 4+). Both from `db/testkit.ts`. Mock only `authenticateRequest` to inject an auth context, hit the real handler against the real DB. - Pattern: `jest.mock('../auth')` then `mockAuthenticateRequest.mockResolvedValueOnce(adminCtx)`. Build events with a `createEvent(method, path, body?, query?)` helper. - Per-lambda scripts: `npm test` runs jest (some use `--forceExit`); `npm run test:e2e` / the `test` script use `start-server-and-test` against `//health` on port 3000. CI: `.github/workflows/lambda-tests.yml` spins up Postgres per lambda matrix. diff --git a/apps/backend/lambdas/auth/handler.ts b/apps/backend/lambdas/auth/handler.ts index 120b2425..b22cd06a 100644 --- a/apps/backend/lambdas/auth/handler.ts +++ b/apps/backend/lambdas/auth/handler.ts @@ -652,7 +652,7 @@ async function handleRegister(event: any): Promise { } // A branch.users row with cognito_sub IS NULL is a PENDING INVITATION, not a - // conflict. Two paths create them: the db_setup.sql seeds and admin + // conflict. Two paths create them: the db/seed.sql rows and admin // POST /users. Before claim-on-register both were permanently unable to sign // in -- registration 409'd on the email, and lambda-auth's authenticateRequest // can never match a NULL cognito_sub. diff --git a/apps/backend/lambdas/auth/test/auth.e2e.test.ts b/apps/backend/lambdas/auth/test/auth.e2e.test.ts index d2d6e391..f609ef61 100644 --- a/apps/backend/lambdas/auth/test/auth.e2e.test.ts +++ b/apps/backend/lambdas/auth/test/auth.e2e.test.ts @@ -1,6 +1,5 @@ -import fs from 'fs'; -import path from 'path'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; const pool = new Pool({ host: 'localhost', @@ -11,15 +10,23 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); beforeEach(async () => { const testName = expect.getState().currentTestName; if (testName && (testName.includes('duplicate') || testName.includes('normalization') || testName.includes('register with valid'))) { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/donors/test/donors.test.ts b/apps/backend/lambdas/donors/test/donors.test.ts index 5a9c4492..4451d690 100644 --- a/apps/backend/lambdas/donors/test/donors.test.ts +++ b/apps/backend/lambdas/donors/test/donors.test.ts @@ -1,6 +1,5 @@ -import fs from 'fs'; -import path from 'path'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; import { handler } from '../handler'; jest.mock('../auth'); import { authenticateRequest } from '../auth'; @@ -15,8 +14,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + const authenticatedUser = { isAuthenticated: true, @@ -55,7 +63,7 @@ describe("Donor API with data", () => { beforeEach(async () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/expenditures/test/expenditures.e2e.test.ts b/apps/backend/lambdas/expenditures/test/expenditures.e2e.test.ts index ec47fb98..8db5552c 100644 --- a/apps/backend/lambdas/expenditures/test/expenditures.e2e.test.ts +++ b/apps/backend/lambdas/expenditures/test/expenditures.e2e.test.ts @@ -1,7 +1,6 @@ -import { describe, test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { describe, test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; // mock auth only for now jest.mock('../auth'); @@ -35,8 +34,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + // Helper to create Lambda events function postEvent(body: Record) { @@ -159,7 +167,7 @@ describe('Expenditures integration tests', () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/projects/auth.ts b/apps/backend/lambdas/projects/auth.ts index ea9f5879..105c2256 100644 --- a/apps/backend/lambdas/projects/auth.ts +++ b/apps/backend/lambdas/projects/auth.ts @@ -85,7 +85,7 @@ export async function canCreateProject(userId: number): Promise { * Admin-only, deliberately stricter than canEditProject. * * Deleting a project cascades to project_memberships, project_donations, - * expenditures and reports (ON DELETE CASCADE in db/db_setup.sql), destroying + * expenditures and reports (ON DELETE CASCADE, see db/migrations/), destroying * financial history. A PI may edit a project but must not be able to erase it. */ export async function canDeleteProject(userId: number): Promise { diff --git a/apps/backend/lambdas/projects/test/crud.test.ts b/apps/backend/lambdas/projects/test/crud.test.ts index c69a5189..9eca6801 100644 --- a/apps/backend/lambdas/projects/test/crud.test.ts +++ b/apps/backend/lambdas/projects/test/crud.test.ts @@ -1,7 +1,6 @@ -import { test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; jest.mock('../auth', () => ({ ...jest.requireActual('../auth'), @@ -28,8 +27,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + function getEvent(rawPath: string) { return { @@ -63,7 +71,7 @@ beforeEach(async () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/projects/test/example.test.ts b/apps/backend/lambdas/projects/test/example.test.ts index f38c43e0..37223067 100644 --- a/apps/backend/lambdas/projects/test/example.test.ts +++ b/apps/backend/lambdas/projects/test/example.test.ts @@ -1,7 +1,6 @@ -import { test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; jest.mock('../auth', () => ({ ...jest.requireActual('../auth'), @@ -28,8 +27,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + function getDonorsEvent(rawPath: string, queryStringParameters: Record = {}) { return { @@ -46,7 +54,7 @@ beforeEach(async () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/projects/test/projects.e2e.test.ts b/apps/backend/lambdas/projects/test/projects.e2e.test.ts index 9101789f..565e64bb 100644 --- a/apps/backend/lambdas/projects/test/projects.e2e.test.ts +++ b/apps/backend/lambdas/projects/test/projects.e2e.test.ts @@ -1,7 +1,6 @@ -import { describe, test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { describe, test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; jest.mock('../auth', () => ({ ...jest.requireActual('../auth'), @@ -34,8 +33,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + // Non-admin users inserted by the Authorization block after each reseed. // The seed creates users 1-3, so these deterministically become 4 and 5. @@ -55,7 +63,7 @@ beforeEach(async () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } @@ -308,7 +316,7 @@ describe('GET /dashboard (e2e)', () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/projects/test/projects.unit.test.ts b/apps/backend/lambdas/projects/test/projects.unit.test.ts index fd1972d7..24fc0857 100644 --- a/apps/backend/lambdas/projects/test/projects.unit.test.ts +++ b/apps/backend/lambdas/projects/test/projects.unit.test.ts @@ -1,7 +1,6 @@ -import { test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; jest.mock('../auth', () => ({ ...jest.requireActual('../auth'), @@ -23,8 +22,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + const adminUser = { isAuthenticated: true as const, @@ -46,7 +54,7 @@ beforeEach(async () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/reports/test/reports.e2e.test.ts b/apps/backend/lambdas/reports/test/reports.e2e.test.ts index a4c1d3d5..7b931e34 100644 --- a/apps/backend/lambdas/reports/test/reports.e2e.test.ts +++ b/apps/backend/lambdas/reports/test/reports.e2e.test.ts @@ -1,7 +1,6 @@ -import { describe, test, expect, beforeEach, afterAll, jest } from '@jest/globals'; -import fs from 'fs'; -import path from 'path'; +import { describe, test, expect, beforeAll, beforeEach, afterAll, jest } from '@jest/globals'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; jest.mock('../auth'); jest.mock('@aws-sdk/client-s3', () => ({ @@ -28,8 +27,17 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); + function getEvent(queryStringParameters?: Record) { return { @@ -57,7 +65,7 @@ describe('Reports e2e tests', () => { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/apps/backend/lambdas/users/test/users.test.ts b/apps/backend/lambdas/users/test/users.test.ts index e6b67618..e7f34a0b 100644 --- a/apps/backend/lambdas/users/test/users.test.ts +++ b/apps/backend/lambdas/users/test/users.test.ts @@ -1,6 +1,5 @@ -import fs from 'fs'; -import path from 'path'; import { Pool } from 'pg'; +import { ensureSchema, resetData } from '../../../db/testkit'; import { handler } from '../handler'; import { authenticateRequest, checkAuthorization } from '../auth'; @@ -34,15 +33,23 @@ const pool = new Pool({ ssl: false, }); -const seedSqlPath = path.resolve(__dirname, '../../../db/db_setup.sql'); -const seedSql = fs.readFileSync(seedSqlPath, 'utf8'); +// Build schema "branch" from db/migrations if it isn't already current. Cheap +// (one SELECT) unless a migration was added since the schema was last built. +beforeAll(async () => { + const client = await pool.connect(); + try { + await ensureSchema(client); + } finally { + client.release(); + } +}); beforeEach(async () => { jest.clearAllMocks(); try { const client = await pool.connect(); try { - await client.query(seedSql); + await resetData(client); } finally { client.release(); } diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md index 0c28f58a..890a502f 100644 --- a/infrastructure/aws/README.md +++ b/infrastructure/aws/README.md @@ -41,9 +41,11 @@ No modules. | [aws_db_instance.branch_rds](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/db_instance) | resource | | [aws_iam_openid_connect_provider.github](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_openid_connect_provider) | resource | | [aws_iam_role.ci_apply](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role) | resource | +| [aws_iam_role.ci_migrate](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role) | resource | | [aws_iam_role.ci_plan](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role) | resource | | [aws_iam_role.ci_preview](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role) | resource | | [aws_iam_role.lambda_role](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.ci_migrate](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.ci_plan_state_lock](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.ci_preview](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.lambda_cognito_admin](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/iam_role_policy) | resource | @@ -68,6 +70,7 @@ No modules. | [archive_file.lambda_placeholder](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.ci_apply_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ci_migrate_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_plan_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_preview_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.frontend_bucket](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | @@ -89,6 +92,7 @@ No modules. |------|-------------| | [api\_gateway\_url](#output\_api\_gateway\_url) | The URL of the API Gateway | | [ci\_apply\_role\_arn](#output\_ci\_apply\_role\_arn) | OIDC role for terraform-apply / lambda-deploy (write, production env only) | +| [ci\_migrate\_role\_arn](#output\_ci\_migrate\_role\_arn) | OIDC role for the db migrate job (production-db env only) | | [ci\_plan\_role\_arn](#output\_ci\_plan\_role\_arn) | OIDC role for terraform-plan (read-only) | | [ci\_preview\_role\_arn](#output\_ci\_preview\_role\_arn) | OIDC role for preview-env.yml (scoped write, preview env only) | | [cognito\_client\_id](#output\_cognito\_client\_id) | Cognito User Pool Client ID (public client, generate\_secret = false) | diff --git a/infrastructure/aws/main.tf b/infrastructure/aws/main.tf index 90e19c4a..a7aa5976 100644 --- a/infrastructure/aws/main.tf +++ b/infrastructure/aws/main.tf @@ -13,6 +13,15 @@ resource "aws_db_instance" "branch_rds" { password = data.infisical_secrets.rds_folder.secrets["password"].value skip_final_snapshot = true + # Set explicitly because CI now applies schema migrations to this instance + # automatically (the `migrate` job in .github/workflows/lambda-deploy.yml). That + # job takes a named pre-migration snapshot, but point-in-time recovery is the + # backstop for everything else, and this was previously left to whatever the API + # defaulted to. Backup storage up to allocated_storage (10 GB) is free, so 7 + # days of PITR costs nothing. + backup_retention_period = 7 + backup_window = "07:00-08:00" # UTC — off-hours for a US-based club + # The lambdas in lambda.tf have no vpc_config, so they run outside any VPC and # had no route to this instance while it was private -- every DB-backed request # hung until the 30s lambda timeout. Putting them in the VPC instead is the diff --git a/infrastructure/aws/oidc.tf b/infrastructure/aws/oidc.tf index 03aa63d8..f0d9461a 100644 --- a/infrastructure/aws/oidc.tf +++ b/infrastructure/aws/oidc.tf @@ -226,6 +226,88 @@ resource "aws_iam_role_policy" "ci_preview" { }) } +# --------------------------------------------------------------------------- +# Migrate role — assumable ONLY from the `production-db` environment. +# Powers the `migrate` job in .github/workflows/lambda-deploy.yml, which applies +# apps/backend/db/migrations to the production RDS instance before the new lambda +# code is deployed. +# +# Deliberately NOT branch-ci-apply: applying schema needs to read one lambda's +# config and snapshot one database, nothing else. The job reads the DB connection +# off the deployed branch-auth function rather than holding a copy of the +# credentials, so there is no second place for the prod password to live and +# rotating it in Infisical needs no CI change. +# --------------------------------------------------------------------------- +data "aws_iam_policy_document" "ci_migrate_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.github.arn] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${local.github_repo}:environment:production-db"] + } + } +} + +resource "aws_iam_role" "ci_migrate" { + name = "branch-ci-migrate" + assume_role_policy = data.aws_iam_policy_document.ci_migrate_assume.json +} + +resource "aws_iam_role_policy" "ci_migrate" { + name = "db-migrate" + role = aws_iam_role.ci_migrate.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + # The single source of truth for DB_HOST/USER/PASSWORD/NAME, written by + # lambda.tf. Reading it means CI can never migrate a database the code is + # not talking to. + { + Sid = "ReadDbConnectionFromDeployedLambda" + Effect = "Allow" + Action = ["lambda:GetFunctionConfiguration"] + Resource = aws_lambda_function.functions["auth"].arn + }, + # DescribeDBInstances is on "*" because branch_rds has no explicit + # `identifier`, so its id is an AWS-generated terraform- string that + # cannot be written down here; the job resolves it by matching DB_HOST. + { + Sid = "ResolveInstanceAndSnapshots" + Effect = "Allow" + Action = ["rds:DescribeDBInstances", "rds:DescribeDBSnapshots"] + Resource = "*" + }, + # Pre-migration snapshot, plus pruning old ones. Backup storage under the + # instance's allocated_storage is free, so this costs nothing and is the + # only rollback lever that does not require reasoning about timestamps. + { + Sid = "PreMigrationSnapshots" + Effect = "Allow" + Action = [ + "rds:CreateDBSnapshot", + "rds:DeleteDBSnapshot", + "rds:AddTagsToResource", + ] + Resource = [ + aws_db_instance.branch_rds.arn, + "arn:aws:rds:us-east-2:${data.aws_caller_identity.current.account_id}:snapshot:branch-premigrate-*", + ] + }, + ] + }) +} + output "ci_plan_role_arn" { description = "OIDC role for terraform-plan (read-only)" value = aws_iam_role.ci_plan.arn @@ -240,3 +322,8 @@ output "ci_preview_role_arn" { description = "OIDC role for preview-env.yml (scoped write, preview env only)" value = aws_iam_role.ci_preview.arn } + +output "ci_migrate_role_arn" { + description = "OIDC role for the db migrate job (production-db env only)" + value = aws_iam_role.ci_migrate.arn +} diff --git a/infrastructure/github/README.md b/infrastructure/github/README.md index fd7fc9fb..cec8db2b 100644 --- a/infrastructure/github/README.md +++ b/infrastructure/github/README.md @@ -35,6 +35,7 @@ No modules. | [github_repository.branch](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository) | resource | | [github_repository_collaborator.collaborators](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_collaborator) | resource | | [github_repository_environment.preview](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_environment) | resource | +| [github_repository_environment.production_db](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_environment) | resource | | [github_repository_file.bot_config_json](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_file) | resource | | [github_repository_file.bot_state_json](https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository_file) | resource | | [infisical_secrets.cognito_folder](https://registry.terraform.io/providers/infisical/infisical/latest/docs/data-sources/secrets) | data source | diff --git a/infrastructure/github/main.tf b/infrastructure/github/main.tf index c4d8a657..89de2527 100644 --- a/infrastructure/github/main.tf +++ b/infrastructure/github/main.tf @@ -57,6 +57,26 @@ resource "github_repository_environment" "preview" { environment = "preview" } +# Deployment environment for the `migrate` job in +# .github/workflows/lambda-deploy.yml, which applies apps/backend/db/migrations to +# the production database. +# +# It exists ONLY to give that job its own OIDC subject +# (repo:...:environment:production-db) so it can assume the narrowly-scoped +# branch-ci-migrate role instead of the AdministratorAccess branch-ci-apply role. +# +# Intentionally has NO required reviewers, for the same reason as `preview` but a +# sharper one: `deploy` depends on `migrate`, so a pending approval would leave +# main merged with its schema applied and its lambda code undeployed — silently, +# until someone noticed. The PR already passed 2 approvals and code-owner review, +# and the mechanical gates (migrations-fresh, migrations-guard) plus the +# pre-migration snapshot are what actually catch bad migrations. Add +# `reviewers { users = [...] }` HERE, not to `production`, if that changes. +resource "github_repository_environment" "production_db" { + repository = github_repository.branch.name + environment = "production-db" +} + resource "github_repository_collaborator" "collaborators" { for_each = { for c in var.repository_collaborators : c.username => c } repository = github_repository.branch.name diff --git a/infrastructure/preview/variables.tf b/infrastructure/preview/variables.tf index 4ee11a13..3a0e3db1 100644 --- a/infrastructure/preview/variables.tf +++ b/infrastructure/preview/variables.tf @@ -8,7 +8,10 @@ variable "pr_number" { # shared RDS endpoint. Passing it in keeps prod as the single source of truth # for DB creds / Cognito ids / reports bucket rather than duplicating them here. # Preview envs deliberately reuse the shared RDS + Cognito pool (data risk is -# accepted); migrations are NEVER run from this module. +# accepted); migrations are NEVER run from this module -- the generated DB types +# hardcode the `branch.` schema prefix, so a per-PR schema would require per-PR +# type regeneration and rebuilding every lambda. A PR that adds a migration +# therefore cannot be fully previewed. variable "lambda_env" { description = "Environment variables applied to every preview lambda (superset across services)." type = map(string) diff --git a/package.json b/package.json index 9930bd3c..cb619120 100644 --- a/package.json +++ b/package.json @@ -9,13 +9,9 @@ "lint": "eslint apps/frontend --ext .ts,.tsx --fix && eslint apps/backend --ext .ts,.tsx --fix", "prepush": "npm run format:check && npm run lint:check", "prepush:fix": "npm run format && npm run lint", - "prepare": "husky install", - "typeorm": "ts-node -r tsconfig-paths/register --project apps/backend/tsconfig.app.json ./node_modules/typeorm/cli.js", - "migration:generate": "npm run typeorm -- migration:generate apps/backend/src/migrations/$npm_config_name -d apps/backend/src/data-source.ts", - "migration:run": "npm run typeorm -- migration:run -d apps/backend/src/data-source.ts", - "migration:revert": "npm run typeorm -- migration:revert -d apps/backend/src/data-source.ts", - "migration:create": "npm run typeorm -- migration:create apps/backend/src/migrations/$npm_config_name" + "prepare": "husky install" }, + "//": "Migrations live in apps/backend/db. Run them with `make migrate` from apps/backend, or `npm run migrate --prefix apps/backend/db`. The TypeORM migration:* scripts that used to be here were dead -- they pointed at apps/backend/src/data-source.ts, which does not exist.", "private": true, "dependencies": { "@aws-sdk/client-cognito-identity-provider": "^3.410.0", diff --git a/shared/types/README.md b/shared/types/README.md index fccc673f..89e7b95d 100644 --- a/shared/types/README.md +++ b/shared/types/README.md @@ -6,7 +6,7 @@ Shared type definitions for the Branch lambdas. This replaces the per-lambda cop | File | Contents | |------|----------| -| `db-types.d.ts` | Kysely row types generated from `apps/backend/db/db_setup.sql` (`DB`, `BranchUsers`, `BranchProjects`, ...) | +| `db-types.d.ts` | Kysely row types generated from `apps/backend/db/migrations/**` (`DB`, `BranchUsers`, `BranchProjects`, ...) | | `auth-types.d.ts` | Auth DTOs (`AuthenticatedUser`, `AuthContext`, `AccessLevel`, `AuthorizationCheck`) | ## How it works @@ -32,7 +32,7 @@ Note: `db-types.d.ts` defines a local `ColumnType` that is structurally identica ## Regenerating DB types -`db-types.d.ts` is regenerated automatically by the `regenerate-db-types` GitHub workflow whenever `apps/backend/db/db_setup.sql` changes. Do not edit it manually. +`db-types.d.ts` is regenerated automatically by the `Schema Change Checks` GitHub workflow whenever `apps/backend/db/migrations/**` changes, and locally by `make migrate` / `make types` in `apps/backend`. Do not edit it manually. ## New lambdas