Skip to content

feat(export): drive meta export table list from generated manifest #1362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pyramation merged 1 commit into from
Jul 12, 2026
Merged

feat(export): drive meta export table list from generated manifest #1362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions pgpm/export/__tests__/cross-flow-parity.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ const SITE_ID = 'c1000001-0000-0000-0000-000000000001';
const DOMAIN_ID = 'c2000001-0000-0000-0000-000000000001';
const API_SCHEMA_ID = 'c3000001-0000-0000-0000-000000000001';
const INDEX_ID = 'd0000001-0000-0000-0000-000000000001';
const RLS_FUNCTION_ID = 'd1000001-0000-0000-0000-000000000001';
const POLICY_ID = 'd1000001-0000-0000-0000-000000000001';
const CORS_SETTINGS_ID = 'd2000001-0000-0000-0000-000000000001';
const USER_AUTH_MODULE_ID = 'd3000001-0000-0000-0000-000000000001';

Expand Down Expand Up @@ -390,7 +390,7 @@ describe('Cross-flow parity: exportMeta vs exportGraphQLMeta', () => {
predicates jsonb,
is_unique boolean
);
CREATE TABLE metaschema_public.rls_function (
CREATE TABLE metaschema_public.policy (
id uuid PRIMARY KEY,
database_id uuid,
schema_id uuid,
Expand Down Expand Up @@ -478,11 +478,11 @@ describe('Cross-flow parity: exportMeta vs exportGraphQLMeta', () => {
VALUES ($1, $2, $3, $4, 'users_pkey', 'btree', ARRAY[$5]::uuid[], '{"columns": ["id"]}'::jsonb, true)
`, [INDEX_ID, DATABASE_ID, SCHEMA_ID_PUB, TABLE_ID_USERS, FIELD_ID_1]);

// rls_function table — tests boolean and int columns
// policy table — tests boolean and int columns
await pg.query(`
INSERT INTO metaschema_public.rls_function (id, database_id, schema_id, table_id, role_name, command, function_name, is_using, force_enabled, priority)
INSERT INTO metaschema_public.policy (id, database_id, schema_id, table_id, role_name, command, function_name, is_using, force_enabled, priority)
VALUES ($1, $2, $3, $4, 'authenticated', 'SELECT', 'check_owner', true, true, 10)
`, [RLS_FUNCTION_ID, DATABASE_ID, SCHEMA_ID_PUB, TABLE_ID_USERS]);
`, [POLICY_ID, DATABASE_ID, SCHEMA_ID_PUB, TABLE_ID_USERS]);

// cors_settings table — tests text[] columns
await pg.query(`
Expand Down Expand Up @@ -660,7 +660,7 @@ describe('Cross-flow parity: exportMeta vs exportGraphQLMeta', () => {
expect(gqlResult['index']?.trim()).toBe(sqlResult['index']?.trim());
});

it('rls_function table with boolean and int columns should be identical across both flows', async () => {
it('policy table with boolean and int columns should be identical across both flows', async () => {
const sqlResult = await exportMeta({
opts: { pg: dbConfig },
dbname: dbConfig.database,
Expand All @@ -673,9 +673,9 @@ describe('Cross-flow parity: exportMeta vs exportGraphQLMeta', () => {
database_id: DATABASE_ID
});

expect(sqlResult['rls_function']).toBeDefined();
expect(gqlResult['rls_function']).toBeDefined();
expect(gqlResult['rls_function']?.trim()).toBe(sqlResult['rls_function']?.trim());
expect(sqlResult['policy']).toBeDefined();
expect(gqlResult['policy']).toBeDefined();
expect(gqlResult['policy']?.trim()).toBe(sqlResult['policy']?.trim());
});

it('cors_settings table with text[] columns should be identical across both flows', async () => {
Expand Down
9 changes: 4 additions & 5 deletions pgpm/export/__tests__/export-meta.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ describe('Export Meta Config Validation', () => {
describe('metaschema_public tables', () => {
const required = [
'database', 'schema', 'table', 'field',
'policy', 'index', 'trigger', 'trigger_function', 'rls_function',
'policy', 'index', 'trigger', 'trigger_function',
'foreign_key_constraint', 'primary_key_constraint', 'unique_constraint',
'check_constraint', 'full_text_search', 'schema_grant', 'table_grant',
'default_privilege'
Expand All @@ -32,7 +32,7 @@ describe('Export Meta Config Validation', () => {
const required = [
'domains', 'sites', 'apis', 'apps',
'site_modules', 'site_themes', 'site_metadata',
'api_modules', 'api_extensions', 'api_schemas'
'api_modules', 'api_schemas'
];

it('should include all required services_public tables in config', () => {
Expand All @@ -46,16 +46,15 @@ describe('Export Meta Config Validation', () => {
describe('metaschema_modules_public tables', () => {
const required = [
'rls_module', 'user_auth_module', 'memberships_module',
'permissions_module', 'limits_module', 'levels_module',
'permissions_module', 'limits_module',
'events_module',
'users_module', 'hierarchy_module', 'membership_types_module',
'invites_module', 'emails_module', 'sessions_module',
'user_state_module', 'profiles_module', 'config_secrets_user_module',
'user_credentials_module', 'user_settings_module',
'connected_accounts_module', 'phone_numbers_module',
'crypto_addresses_module', 'crypto_auth_module',
'field_module', 'table_module',
'secure_table_provision', 'uuid_module', 'default_ids_module',
'secure_table_provision', 'default_ids_module',
'denormalized_table_field',
'relation_provision', 'entity_type_provision',
'rate_limits_module', 'storage_module',
Expand Down
57 changes: 15 additions & 42 deletions pgpm/export/__tests__/export-utils.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import path from 'path';
import {
META_TABLE_CONFIG,
META_TABLE_ORDER,
META_TABLE_OVERRIDES,
DB_REQUIRED_EXTENSIONS,
SERVICE_REQUIRED_EXTENSIONS,
META_COMMON_HEADER,
Expand Down Expand Up @@ -100,56 +101,28 @@ describe('META_TABLE_CONFIG and META_TABLE_ORDER consistency', () => {
// =============================================================================

describe('SQL and GraphQL flow table parity', () => {
let sqlFlowTables: string[];
let graphqlFlowTables: string[];
let sqlSource: string;
let gqlSource: string;

beforeAll(() => {
// Extract queryAndParse keys from export-meta.ts
const sqlSource = readFileSync(
join(__dirname, '../src/export-meta.ts'),
'utf-8'
);
sqlFlowTables = [...sqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]);

// Extract queryAndParse keys from export-graphql-meta.ts
const gqlSource = readFileSync(
join(__dirname, '../src/export-graphql-meta.ts'),
'utf-8'
);
graphqlFlowTables = [...gqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]);
sqlSource = readFileSync(join(__dirname, '../src/export-meta.ts'), 'utf-8');
gqlSource = readFileSync(join(__dirname, '../src/export-graphql-meta.ts'), 'utf-8');
});

it('both flows should query the same set of tables', () => {
const sqlSet = new Set(sqlFlowTables);
const gqlSet = new Set(graphqlFlowTables);

const inSqlNotGql = sqlFlowTables.filter(t => !gqlSet.has(t));
const inGqlNotSql = graphqlFlowTables.filter(t => !sqlSet.has(t));

expect(inSqlNotGql).toEqual([]);
expect(inGqlNotSql).toEqual([]);
it('both flows should iterate the generated manifest via META_TABLE_ORDER', () => {
expect(sqlSource).toContain('META_TABLE_ORDER');
expect(gqlSource).toContain('META_TABLE_ORDER');
});

it('all queried tables should have entries in META_TABLE_CONFIG', () => {
const configKeys = new Set(Object.keys(META_TABLE_CONFIG));

const sqlMissing = sqlFlowTables.filter(t => !configKeys.has(t));
const gqlMissing = graphqlFlowTables.filter(t => !configKeys.has(t));

expect(sqlMissing).toEqual([]);
expect(gqlMissing).toEqual([]);
it('neither flow should have hardcoded per-table queryAndParse calls', () => {
expect([...sqlSource.matchAll(/queryAndParse\('(\w+)'/g)]).toEqual([]);
expect([...gqlSource.matchAll(/queryAndParse\('(\w+)'/g)]).toEqual([]);
});

it('every key in META_TABLE_CONFIG should be queried by both flows', () => {
const sqlSet = new Set(sqlFlowTables);
const gqlSet = new Set(graphqlFlowTables);
const configKeys = Object.keys(META_TABLE_CONFIG);

const notQueriedBySql = configKeys.filter(k => !sqlSet.has(k));
const notQueriedByGql = configKeys.filter(k => !gqlSet.has(k));

expect(notQueriedBySql).toEqual([]);
expect(notQueriedByGql).toEqual([]);
it('every override key should exist in the generated manifest config', () => {
const configKeys = new Set(Object.keys(META_TABLE_CONFIG));
const orphaned = Object.keys(META_TABLE_OVERRIDES).filter(k => !configKeys.has(k));
expect(orphaned).toEqual([]);
});
});

Expand Down
121 changes: 12 additions & 109 deletions pgpm/export/src/export-graphql-meta.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { Parser } from 'csv-to-pg';
import { toSnakeCase } from 'inflekt';

import { FieldType, META_TABLE_CONFIG, TableConfig } from './export-utils';
import { FieldType, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils';
import { GraphQLClient } from './graphql-client';
import {
buildFieldsFragment,
Expand Down Expand Up @@ -244,114 +244,17 @@ export const exportGraphQLMeta = async ({

// Batch queries by schema group — independent HTTP requests run in parallel
// within each group for significant speedup over sequential awaits.

// metaschema_public tables
await Promise.all([
queryAndParse('database'),
queryAndParse('schema'),
queryAndParse('function'),
queryAndParse('spatial_relation'),
queryAndParse('table'),
queryAndParse('field'),
queryAndParse('policy'),
queryAndParse('index'),
queryAndParse('trigger'),
queryAndParse('trigger_function'),
queryAndParse('rls_function'),
queryAndParse('foreign_key_constraint'),
queryAndParse('primary_key_constraint'),
queryAndParse('unique_constraint'),
queryAndParse('check_constraint'),
queryAndParse('full_text_search'),
queryAndParse('schema_grant'),
queryAndParse('table_grant'),
queryAndParse('default_privilege')
]);

// services_public tables
await Promise.all([
queryAndParse('domains'),
queryAndParse('sites'),
queryAndParse('apis'),
queryAndParse('apps'),
queryAndParse('site_modules'),
queryAndParse('site_themes'),
queryAndParse('site_metadata'),
queryAndParse('api_modules'),
queryAndParse('api_extensions'),
queryAndParse('api_schemas'),
queryAndParse('database_settings'),
queryAndParse('api_settings'),
queryAndParse('rls_settings'),
queryAndParse('cors_settings'),
queryAndParse('pubkey_settings'),
queryAndParse('webauthn_settings')
]);

// metaschema_modules_public tables
await Promise.all([
queryAndParse('rls_module'),
queryAndParse('user_auth_module'),
queryAndParse('memberships_module'),
queryAndParse('permissions_module'),
queryAndParse('limits_module'),
queryAndParse('levels_module'),
queryAndParse('events_module'),
queryAndParse('users_module'),
queryAndParse('hierarchy_module'),
queryAndParse('membership_types_module'),
queryAndParse('invites_module'),
queryAndParse('emails_module'),
queryAndParse('sessions_module'),
queryAndParse('user_state_module'),
queryAndParse('profiles_module'),
queryAndParse('config_secrets_user_module'),
queryAndParse('user_credentials_module'),
queryAndParse('user_settings_module'),
queryAndParse('connected_accounts_module'),
queryAndParse('phone_numbers_module'),
queryAndParse('crypto_addresses_module'),
queryAndParse('crypto_auth_module'),
queryAndParse('field_module'),
queryAndParse('table_module'),
queryAndParse('secure_table_provision'),
queryAndParse('uuid_module'),
queryAndParse('default_ids_module'),
queryAndParse('denormalized_table_field'),
queryAndParse('relation_provision'),
queryAndParse('entity_type_provision'),
queryAndParse('rate_limits_module'),
queryAndParse('storage_module'),
queryAndParse('billing_module'),
queryAndParse('billing_provider_module'),
queryAndParse('devices_module'),
queryAndParse('identity_providers_module'),
queryAndParse('integration_providers_module'),
queryAndParse('notifications_module'),
queryAndParse('plans_module'),
queryAndParse('realtime_module'),
queryAndParse('session_secrets_module'),
queryAndParse('infra_secrets_module'),
queryAndParse('infra_config_module'),
queryAndParse('internal_secrets_module'),
queryAndParse('i18n_module'),
queryAndParse('agent_module'),
queryAndParse('function_module'),
queryAndParse('namespace_module'),
queryAndParse('merkle_store_module'),
queryAndParse('graph_module'),
queryAndParse('graph_execution_module'),
queryAndParse('function_deployment_module'),
queryAndParse('function_invocation_module'),
queryAndParse('compute_log_module'),
queryAndParse('db_usage_module'),
queryAndParse('storage_log_module'),
queryAndParse('transfer_log_module'),
queryAndParse('webauthn_auth_module'),
queryAndParse('webauthn_credentials_module'),
queryAndParse('inference_log_module'),
queryAndParse('rate_limit_meters_module')
]);
// Tables come from the generated manifest (see export-utils.ts); groups run
// sequentially in manifest schema order.
const keysBySchema = new Map<string, string[]>();
for (const key of META_TABLE_ORDER) {
const schema = META_TABLE_CONFIG[key].schema;
if (!keysBySchema.has(schema)) keysBySchema.set(schema, []);
keysBySchema.get(schema).push(key);
}
for (const keys of keysBySchema.values()) {
await Promise.all(keys.map(key => queryAndParse(key)));
}

return sql;
};
Loading
Loading