diff --git a/pgpm/export/__tests__/export-utils.test.ts b/pgpm/export/__tests__/export-utils.test.ts index 367c84915..0b760e72c 100644 --- a/pgpm/export/__tests__/export-utils.test.ts +++ b/pgpm/export/__tests__/export-utils.test.ts @@ -13,6 +13,8 @@ import path from 'path'; import { META_TABLE_CONFIG, META_TABLE_ORDER, + EXPORT_SCHEMAS, + EXPORT_BLACKLIST, DB_REQUIRED_EXTENSIONS, SERVICE_REQUIRED_EXTENSIONS, META_COMMON_HEADER, @@ -67,16 +69,21 @@ describe('META_TABLE_CONFIG and META_TABLE_ORDER consistency', () => { expect(duplicates).toEqual([]); }); - it('every config entry should have a valid schema', () => { - const validSchemas = ['metaschema_public', 'services_public', 'metaschema_modules_public']; + it('every config entry should have a valid schema from EXPORT_SCHEMAS', () => { + const validSchemas = new Set(EXPORT_SCHEMAS as unknown as string[]); for (const [key, config] of Object.entries(META_TABLE_CONFIG)) { - expect(validSchemas).toContain(config.schema); + expect(validSchemas.has(config.schema)).toBe(true); expect(config.table).toBeTruthy(); expect(Object.keys(config.fields).length).toBeGreaterThan(0); } }); + it('no table in META_TABLE_ORDER should be in EXPORT_BLACKLIST', () => { + const blacklisted = (META_TABLE_ORDER as unknown as string[]).filter(t => EXPORT_BLACKLIST.has(t)); + expect(blacklisted).toEqual([]); + }); + it('every config entry should have an id field of type uuid', () => { for (const [key, config] of Object.entries(META_TABLE_CONFIG)) { expect(config.fields).toHaveProperty('id'); @@ -94,61 +101,36 @@ describe('META_TABLE_CONFIG and META_TABLE_ORDER consistency', () => { }); // ============================================================================= -// Cross-flow table parity: export-meta.ts and export-graphql-meta.ts -// must query exactly the same set of tables +// Cross-flow table parity: both flows loop over META_TABLE_ORDER, +// so parity is guaranteed by construction. Verify both files import it. // ============================================================================= describe('SQL and GraphQL flow table parity', () => { - let sqlFlowTables: string[]; - let graphqlFlowTables: string[]; + let sqlSource: string; + let gqlSource: string; beforeAll(() => { - // Extract queryAndParse keys from export-meta.ts - const sqlSource = readFileSync( - join(__dirname, '../src/export-meta.ts'), - 'utf-8' - ); - sqlFlowTables = [...sqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]); - - // Extract queryAndParse keys from export-graphql-meta.ts - const gqlSource = readFileSync( - join(__dirname, '../src/export-graphql-meta.ts'), - 'utf-8' - ); - graphqlFlowTables = [...gqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]); + sqlSource = readFileSync(join(__dirname, '../src/export-meta.ts'), 'utf-8'); + gqlSource = readFileSync(join(__dirname, '../src/export-graphql-meta.ts'), 'utf-8'); }); - it('both flows should query the same set of tables', () => { - const sqlSet = new Set(sqlFlowTables); - const gqlSet = new Set(graphqlFlowTables); - - const inSqlNotGql = sqlFlowTables.filter(t => !gqlSet.has(t)); - const inGqlNotSql = graphqlFlowTables.filter(t => !sqlSet.has(t)); - - expect(inSqlNotGql).toEqual([]); - expect(inGqlNotSql).toEqual([]); + it('SQL flow should iterate META_TABLE_ORDER', () => { + expect(sqlSource).toContain('META_TABLE_ORDER'); + expect(sqlSource).toMatch(/for\s*\(\s*const\s+\w+\s+of\s+META_TABLE_ORDER\s*\)/); }); - it('all queried tables should have entries in META_TABLE_CONFIG', () => { - const configKeys = new Set(Object.keys(META_TABLE_CONFIG)); - - const sqlMissing = sqlFlowTables.filter(t => !configKeys.has(t)); - const gqlMissing = graphqlFlowTables.filter(t => !configKeys.has(t)); - - expect(sqlMissing).toEqual([]); - expect(gqlMissing).toEqual([]); + it('GraphQL flow should iterate META_TABLE_ORDER', () => { + expect(gqlSource).toContain('META_TABLE_ORDER'); + expect(gqlSource).toMatch(/for\s*\(\s*const\s+\w+\s+of\s+META_TABLE_ORDER\s*\)/); }); - it('every key in META_TABLE_CONFIG should be queried by both flows', () => { - const sqlSet = new Set(sqlFlowTables); - const gqlSet = new Set(graphqlFlowTables); - const configKeys = Object.keys(META_TABLE_CONFIG); - - const notQueriedBySql = configKeys.filter(k => !sqlSet.has(k)); - const notQueriedByGql = configKeys.filter(k => !gqlSet.has(k)); + it('SQL flow should support auto-discovery via EXPORT_SCHEMAS', () => { + expect(sqlSource).toContain('EXPORT_SCHEMAS'); + expect(sqlSource).toContain('information_schema.tables'); + }); - expect(notQueriedBySql).toEqual([]); - expect(notQueriedByGql).toEqual([]); + it('SQL flow should respect EXPORT_BLACKLIST', () => { + expect(sqlSource).toContain('EXPORT_BLACKLIST'); }); }); diff --git a/pgpm/export/src/export-graphql-meta.ts b/pgpm/export/src/export-graphql-meta.ts index b000f4a61..20f5de48f 100644 --- a/pgpm/export/src/export-graphql-meta.ts +++ b/pgpm/export/src/export-graphql-meta.ts @@ -7,7 +7,7 @@ */ import { Parser } from 'csv-to-pg'; -import { FieldType, META_TABLE_CONFIG } from './export-utils'; +import { FieldType, META_TABLE_CONFIG, META_TABLE_ORDER } from './export-utils'; import { GraphQLClient } from './graphql-client'; import { buildFieldsFragment, @@ -116,95 +116,20 @@ export const exportGraphQLMeta = async ({ } }; - // Batch queries by schema group — independent HTTP requests run in parallel - // within each group for significant speedup over sequential awaits. - - // metaschema_public tables - await Promise.all([ - queryAndParse('database'), - queryAndParse('schema'), - queryAndParse('function'), - queryAndParse('spatial_relation'), - queryAndParse('table'), - queryAndParse('field'), - queryAndParse('policy'), - queryAndParse('index'), - queryAndParse('trigger'), - queryAndParse('trigger_function'), - queryAndParse('rls_function'), - queryAndParse('foreign_key_constraint'), - queryAndParse('primary_key_constraint'), - queryAndParse('unique_constraint'), - queryAndParse('check_constraint'), - queryAndParse('full_text_search'), - queryAndParse('schema_grant'), - queryAndParse('table_grant'), - queryAndParse('default_privilege') - ]); - - // services_public tables - await Promise.all([ - queryAndParse('domains'), - queryAndParse('sites'), - queryAndParse('apis'), - queryAndParse('apps'), - queryAndParse('site_modules'), - queryAndParse('site_themes'), - queryAndParse('site_metadata'), - queryAndParse('api_modules'), - queryAndParse('api_extensions'), - queryAndParse('api_schemas'), - queryAndParse('database_settings'), - queryAndParse('api_settings'), - queryAndParse('rls_settings'), - queryAndParse('cors_settings'), - queryAndParse('pubkey_settings'), - queryAndParse('webauthn_settings') - ]); - - // metaschema_modules_public tables - await Promise.all([ - queryAndParse('rls_module'), - queryAndParse('user_auth_module'), - queryAndParse('memberships_module'), - queryAndParse('permissions_module'), - queryAndParse('limits_module'), - queryAndParse('levels_module'), - queryAndParse('users_module'), - queryAndParse('hierarchy_module'), - queryAndParse('membership_types_module'), - queryAndParse('invites_module'), - queryAndParse('emails_module'), - queryAndParse('sessions_module'), - queryAndParse('secrets_module'), - queryAndParse('profiles_module'), - queryAndParse('encrypted_secrets_module'), - queryAndParse('connected_accounts_module'), - queryAndParse('phone_numbers_module'), - queryAndParse('crypto_addresses_module'), - queryAndParse('crypto_auth_module'), - queryAndParse('field_module'), - queryAndParse('table_module'), - queryAndParse('table_template_module'), - queryAndParse('secure_table_provision'), - queryAndParse('uuid_module'), - queryAndParse('default_ids_module'), - queryAndParse('denormalized_table_field'), - queryAndParse('relation_provision'), - queryAndParse('entity_type_provision'), - queryAndParse('rate_limits_module'), - queryAndParse('storage_module'), - queryAndParse('billing_module'), - queryAndParse('billing_provider_module'), - queryAndParse('devices_module'), - queryAndParse('identity_providers_module'), - queryAndParse('notifications_module'), - queryAndParse('plans_module'), - queryAndParse('realtime_module'), - queryAndParse('session_secrets_module'), - queryAndParse('webauthn_auth_module'), - queryAndParse('webauthn_credentials_module') - ]); + // Group tables by schema for parallel execution within each group. + // Uses META_TABLE_ORDER as the single source of truth for which tables to export. + const by_schema = new Map(); + for (const key of META_TABLE_ORDER) { + const config = META_TABLE_CONFIG[key]; + if (!config) continue; + const group = by_schema.get(config.schema) || []; + group.push(key); + by_schema.set(config.schema, group); + } + + for (const [, keys] of by_schema) { + await Promise.all(keys.map(key => queryAndParse(key))); + } return sql; }; diff --git a/pgpm/export/src/export-meta.ts b/pgpm/export/src/export-meta.ts index 265929446..9f65e5e7f 100644 --- a/pgpm/export/src/export-meta.ts +++ b/pgpm/export/src/export-meta.ts @@ -3,7 +3,15 @@ import { Parser } from 'csv-to-pg'; import { getPgPool } from 'pg-cache'; import type { Pool } from 'pg'; -import { FieldType, TableConfig, META_TABLE_CONFIG } from './export-utils'; +import { + FieldType, + TableConfig, + META_TABLE_CONFIG, + META_TABLE_ORDER, + EXPORT_SCHEMAS, + EXPORT_BLACKLIST, + mapPgTypeToFieldType +} from './export-utils'; /** * Query actual columns from information_schema for a given table. @@ -127,91 +135,75 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams }; // ============================================================================= - // metaschema_public tables + // Phase 1: Export all configured tables in META_TABLE_ORDER // ============================================================================= - await queryAndParse('database', `SELECT * FROM metaschema_public.database WHERE id = $1 ORDER BY id`); - await queryAndParse('schema', `SELECT * FROM metaschema_public.schema WHERE database_id = $1 ORDER BY id`); - await queryAndParse('function', `SELECT * FROM metaschema_public.function WHERE database_id = $1 ORDER BY id`); - await queryAndParse('spatial_relation', `SELECT * FROM metaschema_public.spatial_relation WHERE database_id = $1 ORDER BY id`); - await queryAndParse('table', `SELECT * FROM metaschema_public.table WHERE database_id = $1 ORDER BY id`); - await queryAndParse('field', `SELECT * FROM metaschema_public.field WHERE database_id = $1 ORDER BY id`); - await queryAndParse('policy', `SELECT * FROM metaschema_public.policy WHERE database_id = $1 ORDER BY id`); - await queryAndParse('index', `SELECT * FROM metaschema_public.index WHERE database_id = $1 ORDER BY id`); - await queryAndParse('trigger', `SELECT * FROM metaschema_public.trigger WHERE database_id = $1 ORDER BY id`); - await queryAndParse('trigger_function', `SELECT * FROM metaschema_public.trigger_function WHERE database_id = $1 ORDER BY id`); - await queryAndParse('rls_function', `SELECT * FROM metaschema_public.rls_function WHERE database_id = $1 ORDER BY id`); - await queryAndParse('foreign_key_constraint', `SELECT * FROM metaschema_public.foreign_key_constraint WHERE database_id = $1 ORDER BY id`); - await queryAndParse('primary_key_constraint', `SELECT * FROM metaschema_public.primary_key_constraint WHERE database_id = $1 ORDER BY id`); - await queryAndParse('unique_constraint', `SELECT * FROM metaschema_public.unique_constraint WHERE database_id = $1 ORDER BY id`); - await queryAndParse('check_constraint', `SELECT * FROM metaschema_public.check_constraint WHERE database_id = $1 ORDER BY id`); - await queryAndParse('full_text_search', `SELECT * FROM metaschema_public.full_text_search WHERE database_id = $1 ORDER BY id`); - await queryAndParse('schema_grant', `SELECT * FROM metaschema_public.schema_grant WHERE database_id = $1 ORDER BY id`); - await queryAndParse('table_grant', `SELECT * FROM metaschema_public.table_grant WHERE database_id = $1 ORDER BY id`); - await queryAndParse('default_privilege', `SELECT * FROM metaschema_public.default_privilege WHERE database_id = $1 ORDER BY id`); + for (const key of META_TABLE_ORDER) { + const config = META_TABLE_CONFIG[key]; + if (!config) continue; + const filter_col = key === 'database' ? 'id' : 'database_id'; + await queryAndParse( + key, + `SELECT * FROM ${config.schema}.${config.table} WHERE ${filter_col} = $1 ORDER BY id` + ); + } // ============================================================================= - // services_public tables + // Phase 2: Auto-discover tables not yet in META_TABLE_ORDER + // New tables in the export schemas are automatically picked up without + // any config changes. Blacklisted tables are skipped. // ============================================================================= - await queryAndParse('domains', `SELECT * FROM services_public.domains WHERE database_id = $1 ORDER BY id`); - await queryAndParse('sites', `SELECT * FROM services_public.sites WHERE database_id = $1 ORDER BY id`); - await queryAndParse('apis', `SELECT * FROM services_public.apis WHERE database_id = $1 ORDER BY id`); - await queryAndParse('apps', `SELECT * FROM services_public.apps WHERE database_id = $1 ORDER BY id`); - await queryAndParse('site_modules', `SELECT * FROM services_public.site_modules WHERE database_id = $1 ORDER BY id`); - await queryAndParse('site_themes', `SELECT * FROM services_public.site_themes WHERE database_id = $1 ORDER BY id`); - await queryAndParse('site_metadata', `SELECT * FROM services_public.site_metadata WHERE database_id = $1 ORDER BY id`); - await queryAndParse('api_modules', `SELECT * FROM services_public.api_modules WHERE database_id = $1 ORDER BY id`); - await queryAndParse('api_extensions', `SELECT * FROM services_public.api_extensions WHERE database_id = $1 ORDER BY id`); - await queryAndParse('api_schemas', `SELECT * FROM services_public.api_schemas WHERE database_id = $1 ORDER BY id`); - await queryAndParse('database_settings', `SELECT * FROM services_public.database_settings WHERE database_id = $1 ORDER BY id`); - await queryAndParse('api_settings', `SELECT * FROM services_public.api_settings WHERE database_id = $1 ORDER BY id`); - await queryAndParse('rls_settings', `SELECT * FROM services_public.rls_settings WHERE database_id = $1 ORDER BY id`); - await queryAndParse('cors_settings', `SELECT * FROM services_public.cors_settings WHERE database_id = $1 ORDER BY id`); - await queryAndParse('pubkey_settings', `SELECT * FROM services_public.pubkey_settings WHERE database_id = $1 ORDER BY id`); - await queryAndParse('webauthn_settings', `SELECT * FROM services_public.webauthn_settings WHERE database_id = $1 ORDER BY id`); + const configured_tables = new Set(META_TABLE_ORDER as unknown as string[]); - // ============================================================================= - // metaschema_modules_public tables - // ============================================================================= - await queryAndParse('rls_module', `SELECT * FROM metaschema_modules_public.rls_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('user_auth_module', `SELECT * FROM metaschema_modules_public.user_auth_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('memberships_module', `SELECT * FROM metaschema_modules_public.memberships_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('permissions_module', `SELECT * FROM metaschema_modules_public.permissions_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('limits_module', `SELECT * FROM metaschema_modules_public.limits_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('levels_module', `SELECT * FROM metaschema_modules_public.levels_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('users_module', `SELECT * FROM metaschema_modules_public.users_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('hierarchy_module', `SELECT * FROM metaschema_modules_public.hierarchy_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('membership_types_module', `SELECT * FROM metaschema_modules_public.membership_types_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('invites_module', `SELECT * FROM metaschema_modules_public.invites_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('emails_module', `SELECT * FROM metaschema_modules_public.emails_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('sessions_module', `SELECT * FROM metaschema_modules_public.sessions_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('secrets_module', `SELECT * FROM metaschema_modules_public.secrets_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('profiles_module', `SELECT * FROM metaschema_modules_public.profiles_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('encrypted_secrets_module', `SELECT * FROM metaschema_modules_public.encrypted_secrets_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('connected_accounts_module', `SELECT * FROM metaschema_modules_public.connected_accounts_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('phone_numbers_module', `SELECT * FROM metaschema_modules_public.phone_numbers_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('crypto_addresses_module', `SELECT * FROM metaschema_modules_public.crypto_addresses_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('crypto_auth_module', `SELECT * FROM metaschema_modules_public.crypto_auth_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('field_module', `SELECT * FROM metaschema_modules_public.field_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('table_module', `SELECT * FROM metaschema_modules_public.table_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('table_template_module', `SELECT * FROM metaschema_modules_public.table_template_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('secure_table_provision', `SELECT * FROM metaschema_modules_public.secure_table_provision WHERE database_id = $1 ORDER BY id`); - await queryAndParse('uuid_module', `SELECT * FROM metaschema_modules_public.uuid_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('default_ids_module', `SELECT * FROM metaschema_modules_public.default_ids_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('denormalized_table_field', `SELECT * FROM metaschema_modules_public.denormalized_table_field WHERE database_id = $1 ORDER BY id`); - await queryAndParse('relation_provision', `SELECT * FROM metaschema_modules_public.relation_provision WHERE database_id = $1 ORDER BY id`); - await queryAndParse('entity_type_provision', `SELECT * FROM metaschema_modules_public.entity_type_provision WHERE database_id = $1 ORDER BY id`); - await queryAndParse('rate_limits_module', `SELECT * FROM metaschema_modules_public.rate_limits_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('storage_module', `SELECT * FROM metaschema_modules_public.storage_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('billing_module', `SELECT * FROM metaschema_modules_public.billing_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('billing_provider_module', `SELECT * FROM metaschema_modules_public.billing_provider_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('devices_module', `SELECT * FROM metaschema_modules_public.devices_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('identity_providers_module', `SELECT * FROM metaschema_modules_public.identity_providers_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('notifications_module', `SELECT * FROM metaschema_modules_public.notifications_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('plans_module', `SELECT * FROM metaschema_modules_public.plans_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('realtime_module', `SELECT * FROM metaschema_modules_public.realtime_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('session_secrets_module', `SELECT * FROM metaschema_modules_public.session_secrets_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('webauthn_auth_module', `SELECT * FROM metaschema_modules_public.webauthn_auth_module WHERE database_id = $1 ORDER BY id`); - await queryAndParse('webauthn_credentials_module', `SELECT * FROM metaschema_modules_public.webauthn_credentials_module WHERE database_id = $1 ORDER BY id`); + const discovered = await pool.query(` + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = ANY($1) AND table_type = 'BASE TABLE' + ORDER BY table_schema, table_name + `, [EXPORT_SCHEMAS as unknown as string[]]); + + for (const row of discovered.rows) { + const table_name: string = row.table_name; + const table_schema: string = row.table_schema; + + if (configured_tables.has(table_name) || EXPORT_BLACKLIST.has(table_name)) { + continue; + } + + // Auto-infer field types from information_schema + const columns = await getTableColumns(pool, table_schema, table_name); + if (columns.size === 0) continue; + + const fields: Record = {}; + for (const [col_name, udt_name] of columns) { + fields[col_name] = mapPgTypeToFieldType(udt_name); + } + + const auto_config: TableConfig = { schema: table_schema, table: table_name, fields }; + const filter_col = columns.has('database_id') ? 'database_id' : 'id'; + + try { + const auto_parser = new Parser({ + schema: auto_config.schema, + table: auto_config.table, + fields + }); + + const result = await pool.query( + `SELECT * FROM ${table_schema}.${table_name} WHERE ${filter_col} = $1 ORDER BY id`, + [database_id] + ); + if (result.rows.length) { + const parsed = await auto_parser.parse(result.rows); + if (parsed) { + sql[table_name] = parsed; + } + } + } catch (err: unknown) { + const pg_error = err as { code?: string }; + if (pg_error.code === '42P01') continue; + throw err; + } + } return sql; }; diff --git a/pgpm/export/src/export-utils.ts b/pgpm/export/src/export-utils.ts index 37d6eb6ae..675b96ae7 100644 --- a/pgpm/export/src/export-utils.ts +++ b/pgpm/export/src/export-utils.ts @@ -40,11 +40,32 @@ export const DB_REQUIRED_EXTENSIONS = [ 'pgpm-ltree-helpers' ] as const; +/** + * Schemas that participate in meta/module export. + * Tables in these schemas are candidates for auto-discovery. + */ +export const EXPORT_SCHEMAS = [ + 'metaschema_public', + 'services_public', + 'metaschema_modules_public' +] as const; + +/** + * Tables excluded from auto-discovery. + * These exist in the export schemas but should not be exported. + */ +export const EXPORT_BLACKLIST = new Set([ + 'node_type_registry', + 'blueprint', + 'blueprint_construction', + 'blueprint_template' +]); + /** * Map PostgreSQL data types to FieldType values. * Uses udt_name from information_schema which gives the base type name. */ -const mapPgTypeToFieldType = (udtName: string): FieldType => { +export const mapPgTypeToFieldType = (udtName: string): FieldType => { switch (udtName) { case 'uuid': return 'uuid';