diff --git a/pgpm/core/src/core/class/pgpm.ts b/pgpm/core/src/core/class/pgpm.ts index ec87911a92..49668823c2 100644 --- a/pgpm/core/src/core/class/pgpm.ts +++ b/pgpm/core/src/core/class/pgpm.ts @@ -1,7 +1,6 @@ import { loadConfigSyncFromDir, resolvePgpmPath,walkUp } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import { errors, PgpmOptions, PgpmWorkspaceConfig } from '@pgpmjs/types'; -import yanse from 'yanse'; import { execSync } from 'child_process'; import fs from 'fs'; import * as glob from 'glob'; @@ -10,24 +9,24 @@ import { parse } from 'parse-package-name'; import path, { dirname, resolve } from 'path'; import { getPgPool } from 'pg-cache'; import { PgConfig } from 'pg-env'; +import yanse from 'yanse'; -import { DEFAULT_TEMPLATE_REPO, DEFAULT_TEMPLATE_TTL_MS, DEFAULT_TEMPLATE_TOOL_NAME, scaffoldTemplate } from '../template-scaffold'; import { getAvailableExtensions } from '../../extensions/extensions'; import { generatePlan, writePlan, writePlanFile } from '../../files'; -import { Tag, ExtendedPlanFile, Change } from '../../files/types'; -import { parsePlanFile } from '../../files/plan/parser'; -import { isValidTagName, isValidChangeName, parseReference } from '../../files/plan/validators'; -import { getNow as getPlanTimestamp } from '../../files/plan/generator'; -import { resolveTagToChangeName } from '../../resolution/resolve'; import { ExtensionInfo, getExtensionInfo, getExtensionName, getInstalledExtensions, parseControlFile, - writeExtensions, + writeExtensions } from '../../files'; import { generateControlFileContent, writeExtensionMakefile } from '../../files/extension/writer'; +import { getNow as getPlanTimestamp } from '../../files/plan/generator'; +import { parsePlanFile } from '../../files/plan/parser'; +import { isValidChangeName, isValidTagName, parseReference } from '../../files/plan/validators'; +import { Change, Tag } from '../../files/types'; +import { PackageAnalysisIssue, PackageAnalysisResult, RenameOptions } from '../../files/types'; import { PgpmMigrate } from '../../migrate/client'; import { getExtensionsAndModules, @@ -37,10 +36,9 @@ import { ModuleMap } from '../../modules/modules'; import { packageModule } from '../../packaging/package'; -import { resolveExtensionDependencies, resolveDependencies } from '../../resolution/deps'; -import { PackageAnalysisIssue, PackageAnalysisResult, RenameOptions } from '../../files/types'; - +import { resolveDependencies,resolveExtensionDependencies } from '../../resolution/deps'; import { parseTarget } from '../../utils/target-utils'; +import { DEFAULT_TEMPLATE_REPO, DEFAULT_TEMPLATE_TOOL_NAME, DEFAULT_TEMPLATE_TTL_MS, scaffoldTemplate } from '../template-scaffold'; const logger = new Logger('pgpm'); @@ -51,27 +49,10 @@ const logger = new Logger('pgpm'); */ const EXTENSIONS_DIR = 'extensions'; -function getUTCTimestamp(d: Date = new Date()): string { - return ( - d.getUTCFullYear() + - '-' + String(d.getUTCMonth() + 1).padStart(2, '0') + - '-' + String(d.getUTCDate()).padStart(2, '0') + - 'T' + String(d.getUTCHours()).padStart(2, '0') + - ':' + String(d.getUTCMinutes()).padStart(2, '0') + - ':' + String(d.getUTCSeconds()).padStart(2, '0') + - 'Z' - ); -} - function sortObjectByKey>(obj: T): T { return Object.fromEntries(Object.entries(obj).sort(([a], [b]) => a.localeCompare(b))) as T; } -const getNow = () => - process.env.NODE_ENV === 'test' - ? getUTCTimestamp(new Date('2017-08-11T08:11:51Z')) - : getUTCTimestamp(new Date()); - /** * Truncates workspace extensions to include only modules from the target onwards. @@ -95,7 +76,7 @@ const truncateExtensionsToTarget = ( resolved: workspaceExtensions.resolved.slice(targetIndex), external: workspaceExtensions.external }; -} +}; export enum PackageContext { Outside = 'outside', @@ -484,7 +465,7 @@ export class PgpmPackage { cacheTtlMs: options.cacheTtlMs ?? DEFAULT_TEMPLATE_TTL_MS, toolName: options.toolName ?? DEFAULT_TEMPLATE_TOOL_NAME, cwd: this.cwd, - prompter: options.prompter, + prompter: options.prompter }); // Only create pgpm files (pgpm.plan, .control, deploy/revert/verify dirs) for pgpm-managed modules @@ -808,7 +789,7 @@ export class PgpmPackage { } if (!isValidChangeName(changeName)) { - throw errors.INVALID_NAME({ name: changeName, type: 'change', rules: "Change names must follow Sqitch naming rules" }); + throw errors.INVALID_NAME({ name: changeName, type: 'change', rules: 'Change names must follow Sqitch naming rules' }); } if (!this.isInWorkspace() && !this.isInModule()) { @@ -1125,7 +1106,7 @@ ${dependencies.length > 0 ? dependencies.map(dep => `-- requires: ${dep}`).join( getInstalledModules(): { installed: string[]; installedVersions: Record; - } { + } { this.ensureModule(); this.ensureWorkspace(); diff --git a/pgpm/core/src/files/plan/generator.ts b/pgpm/core/src/files/plan/generator.ts index be3355edf6..7a87a93007 100644 --- a/pgpm/core/src/files/plan/generator.ts +++ b/pgpm/core/src/files/plan/generator.ts @@ -1,9 +1,15 @@ import fs from 'fs'; /** - * Get a UTC timestamp string + * Fixed epoch timestamp used for deterministic plan generation. + * Can be overridden with the PGPM_FROZEN_TIME environment variable. */ -function getUTCTimestamp(d: Date = new Date()): string { +const DEFAULT_FROZEN_TIME = '2017-08-11T08:11:51Z'; + +/** + * Get a UTC timestamp string from a Date object. + */ +function getUTCTimestamp(d: Date): string { return ( d.getUTCFullYear() + '-' + String(d.getUTCMonth() + 1).padStart(2, '0') + @@ -16,12 +22,15 @@ function getUTCTimestamp(d: Date = new Date()): string { } /** - * Get a timestamp for the plan file + * Get a timestamp for the plan file. + * Uses the PGPM_FROZEN_TIME env var if present, otherwise the fixed epoch. */ export function getNow(): string { - return process.env.NODE_ENV === 'test' - ? getUTCTimestamp(new Date('2017-08-11T08:11:51Z')) - : getUTCTimestamp(new Date()); + const frozen = process.env.PGPM_FROZEN_TIME; + if (frozen) { + return getUTCTimestamp(new Date(frozen)); + } + return DEFAULT_FROZEN_TIME; } export interface PlanEntry { @@ -36,6 +45,26 @@ export interface GeneratePlanOptions { entries: PlanEntry[]; } +/** + * Sort dependencies by their order in the supplied plan, so prerequisites + * (parent migrations) appear before their dependents in the bracket. + * Internal dependencies are sorted by their plan index, and external + * dependencies (not in the plan) preserve their original order. + */ +const sortDependencies = (deps: string[], order: Map): string[] => { + const originalOrder = new Map(deps.map((dep, index) => [dep, index])); + return [...deps].sort((a, b) => { + const aIndex = order.get(a); + const bIndex = order.get(b); + const aOriginal = originalOrder.get(a)!; + const bOriginal = originalOrder.get(b)!; + if (aIndex !== undefined && bIndex !== undefined) return aIndex - bIndex || aOriginal - bOriginal; + if (aIndex !== undefined) return -1; + if (bIndex !== undefined) return 1; + return aOriginal - bOriginal; + }); +}; + /** * Generate a plan file content from structured data */ @@ -49,11 +78,15 @@ export function generatePlan(options: GeneratePlanOptions): string { `%uri=${uri || moduleName}` ]; + // Build an order map from the entry list so brackets can be sorted by plan order + const entryOrder = new Map(entries.map((entry, index) => [entry.change, index])); + // Generate the plan entries entries.forEach(entry => { - if (entry.dependencies && entry.dependencies.length > 0) { + const deps = sortDependencies(entry.dependencies || [], entryOrder); + if (deps.length > 0) { planfile.push( - `${entry.change} [${entry.dependencies.join(' ')}] ${now} constructive ${entry.comment ? ` # ${entry.comment}` : ''}` + `${entry.change} [${deps.join(' ')}] ${now} constructive ${entry.comment ? ` # ${entry.comment}` : ''}` ); } else { planfile.push( diff --git a/pgpm/core/src/files/plan/writer.ts b/pgpm/core/src/files/plan/writer.ts index 0f2ce31e64..8424e29225 100644 --- a/pgpm/core/src/files/plan/writer.ts +++ b/pgpm/core/src/files/plan/writer.ts @@ -1,8 +1,28 @@ import fs from 'fs'; import path from 'path'; -import { Change, PlanFile, PgpmRow, Tag, ExtendedPlanFile } from '../types'; import { parseAuthor } from '../../utils/author'; +import { Change, ExtendedPlanFile,PgpmRow, Tag } from '../types'; + +/** + * Sort dependencies by their order in the supplied plan, so prerequisites + * (parent migrations) appear before their dependents in the bracket. + * Internal dependencies are sorted by their plan index, and external + * dependencies (not in the plan) preserve their original order. + */ +const sortDependencies = (deps: string[], order: Map): string[] => { + const originalOrder = new Map(deps.map((dep, index) => [dep, index])); + return [...deps].sort((a, b) => { + const aIndex = order.get(a); + const bIndex = order.get(b); + const aOriginal = originalOrder.get(a)!; + const bOriginal = originalOrder.get(b)!; + if (aIndex !== undefined && bIndex !== undefined) return aIndex - bIndex || aOriginal - bOriginal; + if (aIndex !== undefined) return -1; + if (bIndex !== undefined) return 1; + return aOriginal - bOriginal; + }); +}; export interface PlanWriteOptions { outdir: string; @@ -26,6 +46,9 @@ export function writePgpmPlan(rows: PgpmRow[], opts: PlanWriteOptions): void { const duplicates: Record = {}; + // Build an order map from the row list so brackets can be sorted by plan order + const rowOrder = new Map(rows.map((row, index) => [row.deploy, index])); + const plan = opts.replacer(`%syntax-version=1.0.0 %project=constructive-extension-name %uri=constructive-extension-name @@ -39,7 +62,7 @@ ${rows duplicates[row.deploy] = true; if (row.deps?.length) { - return `${row.deploy} [${row.deps.join(' ')}] ${date()} ${authorName} <${authorEmail}> # add ${row.name}`; + return `${row.deploy} [${sortDependencies(row.deps, rowOrder).join(' ')}] ${date()} ${authorName} <${authorEmail}> # add ${row.name}`; } return `${row.deploy} ${date()} ${authorName} <${authorEmail}> # add ${row.name}`; }) @@ -62,42 +85,45 @@ export function writePlanFile(planPath: string, plan: ExtendedPlanFile): void { */ export function generatePlanFileContent(plan: ExtendedPlanFile): string { const { package: packageName, uri, changes, tags } = plan; - + let content = `%syntax-version=1.0.0\n`; content += `%project=${packageName}\n`; - + if (uri) { content += `%uri=${uri}\n`; } - + content += `\n`; - + + // Build an order map from the change list so brackets can be sorted by plan order + const changeOrder = new Map(changes.map((change, index) => [change.name, index])); + // Add changes and their associated tags for (const change of changes) { - content += generateChangeLineContent(change); + content += generateChangeLineContent(change, changeOrder); content += `\n`; - + const associatedTags = tags.filter(tag => tag.change === change.name); for (const tag of associatedTags) { content += generateTagLineContent(tag); content += `\n`; } } - + return content; } /** * Generate a line for a change in a plan file */ -export function generateChangeLineContent(change: Change): string { +export function generateChangeLineContent(change: Change, changeOrder?: Map): string { const { name, dependencies, timestamp, planner, email, comment } = change; - + let line = name; - - // Add dependencies if present + + // Add dependencies if present, sorted by plan order if (dependencies && dependencies.length > 0) { - line += ` [${dependencies.join(' ')}]`; + line += ` [${sortDependencies(dependencies, changeOrder || new Map()).join(' ')}]`; } // Add timestamp if present diff --git a/pgpm/export/src/export-graphql-meta.ts b/pgpm/export/src/export-graphql-meta.ts index c55bf0b963..4576c33832 100644 --- a/pgpm/export/src/export-graphql-meta.ts +++ b/pgpm/export/src/export-graphql-meta.ts @@ -7,8 +7,7 @@ */ import { Parser } from 'csv-to-pg'; import { toSnakeCase } from 'inflekt'; - -import { FieldType, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils'; +import { FieldType, getTimestampDefaultColumnsForTable, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils'; import { GraphQLClient } from './graphql-client'; import { buildFieldsFragment, @@ -94,7 +93,22 @@ const buildDynamicFieldsFromGraphQL = async ( } } - // Omit columns that are marked as columnDefaults — their DDL DEFAULT (e.g. + // Omit timestamptz columns whose default supplies the current time. Their + // DDL DEFAULT will apply at deploy time, keeping generated INSERT rows + // free of literal environment-specific timestamps. + const timestampDefaultColumns = getTimestampDefaultColumnsForTable(tableConfig.schema, tableConfig.table); + if (timestampDefaultColumns.length > 0) { + tableConfig.columnDefaults = tableConfig.columnDefaults || {}; + for (const colName of timestampDefaultColumns) { + if (dynamicFields[colName] === 'timestamptz') { + delete dynamicFields[colName]; + enumFields.delete(colName); + tableConfig.columnDefaults[colName] = ''; + } + } + } + + // Omit columns that are explicitly marked as columnDefaults — their DDL DEFAULT (e.g. // current_database()) will supply the correct value at deploy time, so the // exported INSERT must not hardcode an environment-specific literal. if (tableConfig.columnDefaults) { diff --git a/pgpm/export/src/export-meta.ts b/pgpm/export/src/export-meta.ts index fac7a7bb04..dc526576f7 100644 --- a/pgpm/export/src/export-meta.ts +++ b/pgpm/export/src/export-meta.ts @@ -1,40 +1,24 @@ import { PgpmOptions } from '@pgpmjs/types'; import { Parser } from 'csv-to-pg'; -import { getPgPool } from 'pg-cache'; import type { Pool } from 'pg'; +import { getPgPool } from 'pg-cache'; -import { FieldType, TableConfig, META_TABLE_CONFIG, META_TABLE_ORDER, mapPgTypeToFieldType } from './export-utils'; - -/** - * Query actual columns from information_schema for a given table. - * Returns a map of column_name -> udt_name (PostgreSQL type). - */ -const getTableColumns = async (pool: Pool, schemaName: string, tableName: string): Promise> => { - const result = await pool.query(` - SELECT column_name, udt_name - FROM information_schema.columns - WHERE table_schema = $1 AND table_name = $2 - ORDER BY ordinal_position - `, [schemaName, tableName]); - - const columns = new Map(); - for (const row of result.rows) { - columns.set(row.column_name, row.udt_name); - } - return columns; -}; +import { FieldType, getTableColumnsWithDefaults, isTimestampDefaultColumn,mapPgTypeToFieldType, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils'; /** * Build dynamic fields config from the database via information_schema. * - All fields are derived from `information_schema.columns` + `mapPgTypeToFieldType`. * - `typeOverrides` from the config are applied on top for special types * (image, upload, url) that cannot be inferred from PG types alone. + * - `timestamptz` columns with non-deterministic timestamp defaults (now(), + * CURRENT_TIMESTAMP, clock_timestamp(), or now() + interval) are omitted so + * their DDL DEFAULT supplies the value at deploy time. */ const buildDynamicFields = async ( pool: Pool, tableConfig: TableConfig ): Promise> => { - const actualColumns = await getTableColumns(pool, tableConfig.schema, tableConfig.table); + const actualColumns = await getTableColumnsWithDefaults(pool, tableConfig.schema, tableConfig.table); if (actualColumns.size === 0) { // Table doesn't exist, return empty fields @@ -42,10 +26,16 @@ const buildDynamicFields = async ( } const dynamicFields: Record = {}; + const timestampDefaultColumns: string[] = []; // Derive all fields from information_schema - for (const [columnName, udtName] of actualColumns) { - dynamicFields[columnName] = mapPgTypeToFieldType(udtName); + for (const [columnName, { udt_name, column_default }] of actualColumns) { + const fieldType = mapPgTypeToFieldType(udt_name); + dynamicFields[columnName] = fieldType; + + if (fieldType === 'timestamptz' && isTimestampDefaultColumn(column_default)) { + timestampDefaultColumns.push(columnName); + } } // Apply type overrides (image, upload, url) @@ -57,9 +47,20 @@ const buildDynamicFields = async ( } } - // Omit columns that are marked as columnDefaults — their DDL DEFAULT (e.g. - // current_database()) will supply the correct value at deploy time, so the - // exported INSERT must not hardcode an environment-specific literal. + // Omit timestamptz columns whose default supplies the current time. Their + // DDL DEFAULT will apply at deploy time, keeping generated INSERT rows + // free of literal environment-specific timestamps. + if (timestampDefaultColumns.length > 0) { + tableConfig.columnDefaults = tableConfig.columnDefaults || {}; + for (const colName of timestampDefaultColumns) { + delete dynamicFields[colName]; + tableConfig.columnDefaults[colName] = actualColumns.get(colName)?.column_default || ''; + } + } + + // Omit columns that are explicitly marked as columnDefaults — their DDL + // DEFAULT (e.g. current_database()) will supply the correct value at deploy + // time, so the exported INSERT must not hardcode an environment-specific literal. if (tableConfig.columnDefaults) { for (const colName of Object.keys(tableConfig.columnDefaults)) { delete dynamicFields[colName]; diff --git a/pgpm/export/src/export-utils.ts b/pgpm/export/src/export-utils.ts index 158ac26ed1..d0037b4f71 100644 --- a/pgpm/export/src/export-utils.ts +++ b/pgpm/export/src/export-utils.ts @@ -1,12 +1,13 @@ +import { getMissingInstallableModules, parseAuthor,PgpmPackage } from '@pgpmjs/core'; import { mkdirSync, rmSync } from 'fs'; import { sync as glob } from 'glob'; -import { Inquirerer } from 'inquirerer'; import { toSnakeCase } from 'inflekt'; +import { Inquirerer } from 'inquirerer'; import path from 'path'; +import type { Pool } from 'pg'; -import { PgpmPackage, getMissingInstallableModules, parseAuthor } from '@pgpmjs/core'; -import { lookupByPgUdt } from './type-map'; import metaExportTables from './meta-export-tables.json'; +import { lookupByPgUdt } from './type-map'; // ============================================================================= // Shared constants @@ -53,6 +54,54 @@ export const mapPgTypeToFieldType = (udtName: string): FieldType => { return entry?.fieldType ?? 'text'; }; +export interface TableColumnInfo { + udt_name: string; + column_default: string | null; +} + +/** + * Query actual columns from information_schema for a given table. + * Returns a map of column_name -> { udt_name, column_default }. + */ +export const getTableColumnsWithDefaults = async ( + pool: Pool, + schemaName: string, + tableName: string +): Promise> => { + const result = await pool.query( + ` + SELECT column_name, udt_name, column_default + FROM information_schema.columns + WHERE table_schema = $1 AND table_name = $2 + ORDER BY ordinal_position + `, + [schemaName, tableName] + ); + + const columns = new Map(); + for (const row of result.rows) { + columns.set(row.column_name, { + udt_name: row.udt_name, + column_default: row.column_default + }); + } + return columns; +}; + +/** + * Detect whether a column default expression is a non-deterministic timestamp + * default (now(), CURRENT_TIMESTAMP, clock_timestamp(), or now() + interval). + * Exported INSERT rows must omit these columns so the DDL DEFAULT applies at + * deploy time, keeping generated migrations deterministic. + */ +export const isTimestampDefaultColumn = (columnDefault: string | null): boolean => { + if (!columnDefault) return false; + const normalized = columnDefault.trim().toLowerCase(); + return /^(now\s*\(\s*\)|current_timestamp(?:\s*\(\s*\d*\s*\))?|clock_timestamp\s*\(\s*\))(?:\s*\+\s*'[^']*'\s*::\s*interval)?$/i.test( + normalized + ); +}; + /** * Required extensions for service/meta exports. * Includes native PostgreSQL extensions and pgpm modules for metadata management. @@ -190,6 +239,18 @@ export const META_TABLE_CONFIG: Record = Object.fromEntries ]) ); +/** + * Per-table timestamptz columns whose DDL default is non-deterministic + * (now(), CURRENT_TIMESTAMP, clock_timestamp(), now() + interval). + * Generated by constructive-db and propagated here via constructive-hub. + * The GraphQL export path uses this to omit those columns without a DB pool. + */ +export const META_TABLE_TIMESTAMP_DEFAULTS: Record = + (metaExportTables as { timestampDefaultColumns?: Record }).timestampDefaultColumns || {}; + +export const getTimestampDefaultColumnsForTable = (schema: string, table: string): string[] => + META_TABLE_TIMESTAMP_DEFAULTS[`${schema}.${table}`] || []; + // ============================================================================= // Shared interfaces // ============================================================================= diff --git a/pgpm/export/src/meta-export-tables.json b/pgpm/export/src/meta-export-tables.json index d43db2390c..e52fc23108 100644 --- a/pgpm/export/src/meta-export-tables.json +++ b/pgpm/export/src/meta-export-tables.json @@ -281,6 +281,11 @@ "schema": "metaschema_modules_public", "table": "infra_secrets_module" }, + { + "key": "integration_providers_module", + "schema": "metaschema_modules_public", + "table": "integration_providers_module" + }, { "key": "internal_secrets_module", "schema": "metaschema_modules_public", @@ -436,11 +441,6 @@ "schema": "metaschema_public", "table": "embedding_chunks" }, - { - "key": "partition", - "schema": "metaschema_public", - "table": "partition" - }, { "key": "spatial_relation", "schema": "metaschema_public", @@ -501,5 +501,90 @@ "schema": "metaschema_modules_public", "table": "graph_execution_module" } - ] + ], + "timestampDefaultColumns": { + "metaschema_modules_public.db_preset_module": [ + "created_at" + ], + "metaschema_modules_public.graph_execution_module": [ + "created_at" + ], + "metaschema_modules_public.graph_module": [ + "created_at" + ], + "metaschema_modules_public.hierarchy_module": [ + "created_at" + ], + "metaschema_modules_public.merkle_store_module": [ + "created_at" + ], + "metaschema_public.check_constraint": [ + "created_at", + "updated_at" + ], + "metaschema_public.database": [ + "created_at", + "updated_at" + ], + "metaschema_public.embedding_chunks": [ + "created_at", + "updated_at" + ], + "metaschema_public.field": [ + "created_at", + "updated_at" + ], + "metaschema_public.foreign_key_constraint": [ + "created_at", + "updated_at" + ], + "metaschema_public.full_text_search": [ + "created_at", + "updated_at" + ], + "metaschema_public.index": [ + "created_at", + "updated_at" + ], + "metaschema_public.policy": [ + "created_at", + "updated_at" + ], + "metaschema_public.primary_key_constraint": [ + "created_at", + "updated_at" + ], + "metaschema_public.schema": [ + "created_at", + "updated_at" + ], + "metaschema_public.schema_grant": [ + "created_at", + "updated_at" + ], + "metaschema_public.spatial_relation": [ + "created_at", + "updated_at" + ], + "metaschema_public.table": [ + "created_at", + "updated_at" + ], + "metaschema_public.table_grant": [ + "created_at", + "updated_at" + ], + "metaschema_public.trigger": [ + "created_at", + "updated_at" + ], + "metaschema_public.trigger_function": [ + "created_at", + "updated_at" + ], + "metaschema_public.unique_constraint": [ + "created_at", + "updated_at" + ] + } }