Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 84 additions & 82 deletions package-lock.json

Large diffs are not rendered by default.

8 changes: 7 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"start": "vp dev",
"typecheck": "tsc -b",
"format:check": "vp fmt --check .",
"lint": "vp lint src/app src/worker scripts test vite.config.ts vitest.config.ts vitest.app.config.ts vitest.migration.config.ts --deny-warnings",
"lint": "vp lint src processor scripts test vite.config.ts vitest.config.ts vitest.app.config.ts vitest.migration.config.ts --deny-warnings",
"test": "node scripts/run-worker-tests.mjs && vp test run --config vitest.app.config.ts && vp test run --config vitest.migration.config.ts",
"test:worker": "node scripts/run-worker-tests.mjs",
"test:app": "vp test run --config vitest.app.config.ts",
Expand Down Expand Up @@ -43,20 +43,26 @@
"devDependencies": {
"@cloudflare/vite-plugin": "^1.50.0",
"@cloudflare/vitest-pool-workers": "^0.20.1",
"@oxlint/plugins": "1.78.0",
"@testing-library/react": "^16.3.2",
"@testing-library/user-event": "^14.6.1",
"@types/node": "^26.1.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.5",
"jsdom": "^26.1.0",
"oxlint": "1.78.0",
Comment thread
HazAT marked this conversation as resolved.
"tsx": "^4.23.5",
"typescript": "^7.0.2",
"vite": "^8.2.0",
"vite-plus": "^0.2.7",
"vitest": "^4.1.10",
"wrangler": "^4.118.0"
},
"overrides": {
"@oxlint/plugins": "$@oxlint/plugins",
"oxlint": "$oxlint"
},
"engines": {
"node": ">=24.11.0"
},
Expand Down
4 changes: 2 additions & 2 deletions processor/video-processor.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,8 @@ async function processRequest(payload) {
function validatePayload(value) {
if (
!value ||
typeof value !== 'object' ||
typeof value.videoId !== 'string' ||
Object.prototype.toString.call(value) !== '[object Object]' ||
Object.prototype.toString.call(value.videoId) !== '[object String]' ||
!/^[a-zA-Z0-9-]{1,128}$/.test(value.videoId) ||
!Number.isInteger(value.attempt) ||
value.attempt < 1
Expand Down
27 changes: 18 additions & 9 deletions scripts/migrate/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ interface Arguments {
async function main() {
const args = parseArguments(process.argv.slice(2));
assertExplicitDestination(args.target, args.environment, args.confirmation);
const database = JSON.parse(await readFile(args.database, 'utf8')) as unknown;
const database = JSON.parse(await readFile(args.database, 'utf8'));
const manifest = args.storageManifest
? await readStorageManifest(args.storageManifest)
: [];
Expand Down Expand Up @@ -92,13 +92,22 @@ function destination(args: Arguments): ImportOptions {
};
}

function parseArguments(argv: string[]): Arguments {
const command = argv.shift() as Arguments['command'] | undefined;
if (!command || !['validate', 'dry-run', 'import', 'reconcile'].includes(command)) {
throw new Error(
'Usage: cli.ts <validate|dry-run|import|reconcile> --database <export.json>',
);
function parseCommand(value: string | undefined): Arguments['command'] {
switch (value) {
case 'validate':
case 'dry-run':
case 'import':
case 'reconcile':
return value;
default:
throw new Error(
'Usage: cli.ts <validate|dry-run|import|reconcile> --database <export.json>',
);
}
}

function parseArguments(argv: string[]): Arguments {
const command = parseCommand(argv.shift());
const flags = new Map<string, string>();
for (let index = 0; index < argv.length; index += 2) {
const flag = argv[index];
Expand Down Expand Up @@ -144,7 +153,7 @@ async function output(report: MigrationReport, filename?: string) {
if (filename) console.log(`Report: ${path.resolve(filename)}`);
}

main().catch((error: unknown) => {
console.error(error instanceof Error ? error.message : error);
main().catch((cause: unknown) => {
console.error(cause instanceof Error ? cause.message : cause);
process.exitCode = 1;
});
19 changes: 10 additions & 9 deletions scripts/migrate/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {mkdtemp, rm, writeFile} from 'node:fs/promises';
import {tmpdir} from 'node:os';
import path from 'node:path';

import {isJsonNumber, isJsonString} from '../../src/shared/json';
import type {MigrationData, MigrationReport} from './types';

export type Destination = 'local' | 'cloudflare';
Expand Down Expand Up @@ -180,19 +181,20 @@ export function migrationSql(data: MigrationData, destination: Destination = 'lo
return `${statements.join('\n')}\n`;
}

function sql(strings: TemplateStringsArray, ...values: unknown[]) {
type SqlValue = boolean | null | number | string | undefined;

function sql(strings: TemplateStringsArray, ...values: SqlValue[]) {
return strings.reduce(
(result, part, index) =>
result + part + (index < values.length ? quote(values[index]) : ''),
'',
);
}

function quote(value: unknown) {
function quote(value: SqlValue) {
if (value === null || value === undefined) return 'NULL';
if (typeof value === 'number') return String(value);
if (typeof value !== 'string')
throw new TypeError('SQL values must be strings or numbers');
if (isJsonNumber(value)) return String(value);
if (!isJsonString(value)) throw new TypeError('SQL values must be strings or numbers');
return `'${value.replaceAll("'", "''")}'`;
}

Expand Down Expand Up @@ -228,8 +230,7 @@ function wrangler(args: string[], config?: string) {
);
}

function commandError(error: unknown) {
if (error && typeof error === 'object' && 'stderr' in error)
return String(error.stderr).trim();
return error instanceof Error ? error.message : String(error);
function commandError(cause: unknown) {
if (cause instanceof Error && 'stderr' in cause) return String(cause.stderr).trim();
return cause instanceof Error ? cause.message : String(cause);
}
157 changes: 114 additions & 43 deletions scripts/migrate/reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
} from './types';
import type {Destination, ImportOptions} from './import';

const tables: Record<EntityName, string> = {
const tables = {
users: 'users',
years: 'years',
groups: 'groups',
Expand All @@ -22,12 +22,21 @@ const tables: Record<EntityName, string> = {
votes: 'votes',
awards: 'awards',
media: 'media',
};
} satisfies Record<EntityName, string>;

export function transformedCounts(data: MigrationData): Record<EntityName, number> {
return Object.fromEntries(
entityNames.map((name) => [name, data[name].length]),
) as Record<EntityName, number>;
export function transformedCounts(data: MigrationData) {
return {
users: data.users.length,
years: data.years.length,
groups: data.groups.length,
projects: data.projects.length,
projectMembers: data.projectMembers.length,
awardCategories: data.awardCategories.length,
projectNominations: data.projectNominations.length,
votes: data.votes.length,
awards: data.awards.length,
media: data.media.length,
} satisfies Record<EntityName, number>;
}

export function destinationCounts(options: ImportOptions, data: MigrationData) {
Expand All @@ -38,11 +47,11 @@ export function destinationCounts(options: ImportOptions, data: MigrationData) {
const counts: Partial<Record<EntityName, number>> = {};
const sourceCounts: Partial<Record<EntityName, number>> = {};
for (const {row} of parsed.flatMap(({results}) => results)) {
const value = JSON.parse(row) as {
const value: {
entity: EntityName;
kind: 'all' | 'source';
count: number;
};
} = JSON.parse(row);
const target = value.kind === 'all' ? counts : sourceCounts;
target[value.entity] = (target[value.entity] ?? 0) + value.count;
}
Expand All @@ -55,7 +64,8 @@ function executeLocalCountQueries(options: ImportOptions, data: MigrationData) {
writeFileSync(sqlFile, destinationCountSql(data), {mode: 0o600});
try {
const output = wranglerCountCommand(options, ['--file', sqlFile]);
return parseWranglerJson(output) as Array<{results: Array<{row: string}>}>;
const parsed: Array<{results: Array<{row: string}>}> = parseWranglerJson(output);
return parsed;
} finally {
rmSync(directory, {recursive: true, force: true});
}
Expand All @@ -64,7 +74,8 @@ function executeLocalCountQueries(options: ImportOptions, data: MigrationData) {
function executeRemoteCountQueries(options: ImportOptions, data: MigrationData) {
return destinationCountStatements(data).flatMap((statement) => {
const output = wranglerCountCommand(options, ['--command', statement]);
return parseWranglerJson(output) as Array<{results: Array<{row: string}>}>;
const parsed: Array<{results: Array<{row: string}>}> = parseWranglerJson(output);
return parsed;
});
}

Expand Down Expand Up @@ -129,41 +140,101 @@ export function reconcileCounts(report: MigrationReport) {
}
}

function sourceCountQueries(name: EntityName, data: MigrationData) {
const rows = data[name] as unknown as Array<Record<string, unknown>>;
if (!rows.length) {
return [`SELECT json_object('entity','${name}','kind','source','count',0) row`];
function sourceCountQueries(name: EntityName, data: MigrationData): string[] {
switch (name) {
case 'users':
return singleSourceCountQueries(
name,
'source_uid',
data.users.map((row) => row.sourceUid),
);
case 'years':
return singleSourceCountQueries(
name,
'id',
data.years.map((row) => row.id),
);
case 'groups':
return singleSourceCountQueries(
name,
'source_id',
data.groups.map((row) => row.sourceId),
);
case 'projects':
return singleSourceCountQueries(
name,
'source_id',
data.projects.map((row) => row.sourceId),
);
case 'projectMembers':
return pairSourceCountQueries(
name,
'user_id',
data.projectMembers.map((row) => [row.projectId, row.userId]),
);
case 'awardCategories':
return singleSourceCountQueries(
name,
'source_id',
data.awardCategories.map((row) => row.sourceId),
);
case 'projectNominations':
return pairSourceCountQueries(
name,
'award_category_id',
data.projectNominations.map((row) => [row.projectId, row.awardCategoryId]),
);
case 'votes':
return singleSourceCountQueries(
name,
'source_id',
data.votes.map((row) => row.sourceId),
);
case 'awards':
return singleSourceCountQueries(
name,
'source_id',
data.awards.map((row) => row.sourceId),
);
case 'media':
return singleSourceCountQueries(
name,
'source_id',
data.media.map((row) => row.sourceId),
);
}
return chunks(rows, 100).map((chunk) => sourceCountQuery(name, chunk));
}

function sourceCountQuery(name: EntityName, rows: Array<Record<string, unknown>>) {
const column =
name === 'projectMembers' || name === 'projectNominations'
? null
: name === 'years'
? 'id'
: name === 'users'
? 'source_uid'
: 'source_id';
const sourceProperty =
name === 'years' ? 'id' : name === 'users' ? 'sourceUid' : 'sourceId';
if (column) {
const values = rows.map((row) => `(${quote(row[sourceProperty])})`).join(',');
return `WITH expected(value) AS (VALUES ${values})
}

function singleSourceCountQueries(name: EntityName, column: string, values: string[]) {
if (!values.length) return [emptySourceCountQuery(name)];
return chunks(values, 100).map((chunk) => {
const expected = chunk.map((value) => `(${quote(value)})`).join(',');
return `WITH expected(value) AS (VALUES ${expected})
SELECT json_object('entity','${name}','kind','source','count',COUNT(*)) row
FROM ${tables[name]} destination JOIN expected ON destination.${column}=expected.value`;
}
const secondColumn = name === 'projectMembers' ? 'user_id' : 'award_category_id';
const secondProperty = name === 'projectMembers' ? 'userId' : 'awardCategoryId';
const values = rows
.map((row) => `(${quote(row.projectId)},${quote(row[secondProperty])})`)
.join(',');
return `WITH expected(project_id, related_id) AS (VALUES ${values})
SELECT json_object('entity','${name}','kind','source','count',COUNT(*)) row
FROM ${tables[name]} destination JOIN expected
ON destination.project_id=expected.project_id
AND destination.${secondColumn}=expected.related_id`;
});
}

function pairSourceCountQueries(
name: EntityName,
secondColumn: string,
values: Array<[string, string]>,
) {
if (!values.length) return [emptySourceCountQuery(name)];
return chunks(values, 100).map((chunk) => {
const expected = chunk
.map(([projectId, relatedId]) => `(${quote(projectId)},${quote(relatedId)})`)
.join(',');
return `WITH expected(project_id, related_id) AS (VALUES ${expected})
SELECT json_object('entity','${name}','kind','source','count',COUNT(*)) row
FROM ${tables[name]} destination JOIN expected
ON destination.project_id=expected.project_id
AND destination.${secondColumn}=expected.related_id`;
});
}

function emptySourceCountQuery(name: EntityName) {
return `SELECT json_object('entity','${name}','kind','source','count',0) row`;
}

function chunks<T>(values: T[], size: number) {
Expand All @@ -172,7 +243,7 @@ function chunks<T>(values: T[], size: number) {
);
}

function quote(value: unknown) {
function quote<T>(value: T) {
return `'${String(value).replaceAll("'", "''")}'`;
}

Expand Down
Loading
Loading