Skip to content

feat(pgpm/export): loop-based export + auto-discover tables #1137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pyramation wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(pgpm/export): loop-based export + auto-discover tables #1137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 28 additions & 46 deletions pgpm/export/__tests__/export-utils.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import path from 'path';
import {
META_TABLE_CONFIG,
META_TABLE_ORDER,
EXPORT_SCHEMAS,
EXPORT_BLACKLIST,
DB_REQUIRED_EXTENSIONS,
SERVICE_REQUIRED_EXTENSIONS,
META_COMMON_HEADER,
Expand Down Expand Up @@ -67,16 +69,21 @@ describe('META_TABLE_CONFIG and META_TABLE_ORDER consistency', () => {
expect(duplicates).toEqual([]);
});

it('every config entry should have a valid schema', () => {
const validSchemas = ['metaschema_public', 'services_public', 'metaschema_modules_public'];
it('every config entry should have a valid schema from EXPORT_SCHEMAS', () => {
const validSchemas = new Set<string>(EXPORT_SCHEMAS as unknown as string[]);

for (const [key, config] of Object.entries(META_TABLE_CONFIG)) {
expect(validSchemas).toContain(config.schema);
expect(validSchemas.has(config.schema)).toBe(true);
expect(config.table).toBeTruthy();
expect(Object.keys(config.fields).length).toBeGreaterThan(0);
}
});

it('no table in META_TABLE_ORDER should be in EXPORT_BLACKLIST', () => {
const blacklisted = (META_TABLE_ORDER as unknown as string[]).filter(t => EXPORT_BLACKLIST.has(t));
expect(blacklisted).toEqual([]);
});

it('every config entry should have an id field of type uuid', () => {
for (const [key, config] of Object.entries(META_TABLE_CONFIG)) {
expect(config.fields).toHaveProperty('id');
Expand All @@ -94,61 +101,36 @@ describe('META_TABLE_CONFIG and META_TABLE_ORDER consistency', () => {
});

// =============================================================================
// Cross-flow table parity: export-meta.ts and export-graphql-meta.ts
// must query exactly the same set of tables
// Cross-flow table parity: both flows loop over META_TABLE_ORDER,
// so parity is guaranteed by construction. Verify both files import it.
// =============================================================================

describe('SQL and GraphQL flow table parity', () => {
let sqlFlowTables: string[];
let graphqlFlowTables: string[];
let sqlSource: string;
let gqlSource: string;

beforeAll(() => {
// Extract queryAndParse keys from export-meta.ts
const sqlSource = readFileSync(
join(__dirname, '../src/export-meta.ts'),
'utf-8'
);
sqlFlowTables = [...sqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]);

// Extract queryAndParse keys from export-graphql-meta.ts
const gqlSource = readFileSync(
join(__dirname, '../src/export-graphql-meta.ts'),
'utf-8'
);
graphqlFlowTables = [...gqlSource.matchAll(/queryAndParse\('(\w+)'/g)].map(m => m[1]);
sqlSource = readFileSync(join(__dirname, '../src/export-meta.ts'), 'utf-8');
gqlSource = readFileSync(join(__dirname, '../src/export-graphql-meta.ts'), 'utf-8');
});

it('both flows should query the same set of tables', () => {
const sqlSet = new Set(sqlFlowTables);
const gqlSet = new Set(graphqlFlowTables);

const inSqlNotGql = sqlFlowTables.filter(t => !gqlSet.has(t));
const inGqlNotSql = graphqlFlowTables.filter(t => !sqlSet.has(t));

expect(inSqlNotGql).toEqual([]);
expect(inGqlNotSql).toEqual([]);
it('SQL flow should iterate META_TABLE_ORDER', () => {
expect(sqlSource).toContain('META_TABLE_ORDER');
expect(sqlSource).toMatch(/for\s*\(\s*const\s+\w+\s+of\s+META_TABLE_ORDER\s*\)/);
});

it('all queried tables should have entries in META_TABLE_CONFIG', () => {
const configKeys = new Set(Object.keys(META_TABLE_CONFIG));

const sqlMissing = sqlFlowTables.filter(t => !configKeys.has(t));
const gqlMissing = graphqlFlowTables.filter(t => !configKeys.has(t));

expect(sqlMissing).toEqual([]);
expect(gqlMissing).toEqual([]);
it('GraphQL flow should iterate META_TABLE_ORDER', () => {
expect(gqlSource).toContain('META_TABLE_ORDER');
expect(gqlSource).toMatch(/for\s*\(\s*const\s+\w+\s+of\s+META_TABLE_ORDER\s*\)/);
});

it('every key in META_TABLE_CONFIG should be queried by both flows', () => {
const sqlSet = new Set(sqlFlowTables);
const gqlSet = new Set(graphqlFlowTables);
const configKeys = Object.keys(META_TABLE_CONFIG);

const notQueriedBySql = configKeys.filter(k => !sqlSet.has(k));
const notQueriedByGql = configKeys.filter(k => !gqlSet.has(k));
it('SQL flow should support auto-discovery via EXPORT_SCHEMAS', () => {
expect(sqlSource).toContain('EXPORT_SCHEMAS');
expect(sqlSource).toContain('information_schema.tables');
});

expect(notQueriedBySql).toEqual([]);
expect(notQueriedByGql).toEqual([]);
it('SQL flow should respect EXPORT_BLACKLIST', () => {
expect(sqlSource).toContain('EXPORT_BLACKLIST');
});
});

Expand Down
105 changes: 15 additions & 90 deletions pgpm/export/src/export-graphql-meta.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
*/
import { Parser } from 'csv-to-pg';

import { FieldType, META_TABLE_CONFIG } from './export-utils';
import { FieldType, META_TABLE_CONFIG, META_TABLE_ORDER } from './export-utils';
import { GraphQLClient } from './graphql-client';
import {
buildFieldsFragment,
Expand Down Expand Up @@ -116,95 +116,20 @@ export const exportGraphQLMeta = async ({
}
};

// Batch queries by schema group — independent HTTP requests run in parallel
// within each group for significant speedup over sequential awaits.

// metaschema_public tables
await Promise.all([
queryAndParse('database'),
queryAndParse('schema'),
queryAndParse('function'),
queryAndParse('spatial_relation'),
queryAndParse('table'),
queryAndParse('field'),
queryAndParse('policy'),
queryAndParse('index'),
queryAndParse('trigger'),
queryAndParse('trigger_function'),
queryAndParse('rls_function'),
queryAndParse('foreign_key_constraint'),
queryAndParse('primary_key_constraint'),
queryAndParse('unique_constraint'),
queryAndParse('check_constraint'),
queryAndParse('full_text_search'),
queryAndParse('schema_grant'),
queryAndParse('table_grant'),
queryAndParse('default_privilege')
]);

// services_public tables
await Promise.all([
queryAndParse('domains'),
queryAndParse('sites'),
queryAndParse('apis'),
queryAndParse('apps'),
queryAndParse('site_modules'),
queryAndParse('site_themes'),
queryAndParse('site_metadata'),
queryAndParse('api_modules'),
queryAndParse('api_extensions'),
queryAndParse('api_schemas'),
queryAndParse('database_settings'),
queryAndParse('api_settings'),
queryAndParse('rls_settings'),
queryAndParse('cors_settings'),
queryAndParse('pubkey_settings'),
queryAndParse('webauthn_settings')
]);

// metaschema_modules_public tables
await Promise.all([
queryAndParse('rls_module'),
queryAndParse('user_auth_module'),
queryAndParse('memberships_module'),
queryAndParse('permissions_module'),
queryAndParse('limits_module'),
queryAndParse('levels_module'),
queryAndParse('users_module'),
queryAndParse('hierarchy_module'),
queryAndParse('membership_types_module'),
queryAndParse('invites_module'),
queryAndParse('emails_module'),
queryAndParse('sessions_module'),
queryAndParse('secrets_module'),
queryAndParse('profiles_module'),
queryAndParse('encrypted_secrets_module'),
queryAndParse('connected_accounts_module'),
queryAndParse('phone_numbers_module'),
queryAndParse('crypto_addresses_module'),
queryAndParse('crypto_auth_module'),
queryAndParse('field_module'),
queryAndParse('table_module'),
queryAndParse('table_template_module'),
queryAndParse('secure_table_provision'),
queryAndParse('uuid_module'),
queryAndParse('default_ids_module'),
queryAndParse('denormalized_table_field'),
queryAndParse('relation_provision'),
queryAndParse('entity_type_provision'),
queryAndParse('rate_limits_module'),
queryAndParse('storage_module'),
queryAndParse('billing_module'),
queryAndParse('billing_provider_module'),
queryAndParse('devices_module'),
queryAndParse('identity_providers_module'),
queryAndParse('notifications_module'),
queryAndParse('plans_module'),
queryAndParse('realtime_module'),
queryAndParse('session_secrets_module'),
queryAndParse('webauthn_auth_module'),
queryAndParse('webauthn_credentials_module')
]);
// Group tables by schema for parallel execution within each group.
// Uses META_TABLE_ORDER as the single source of truth for which tables to export.
const by_schema = new Map<string, string[]>();
for (const key of META_TABLE_ORDER) {
const config = META_TABLE_CONFIG[key];
if (!config) continue;
const group = by_schema.get(config.schema) || [];
group.push(key);
by_schema.set(config.schema, group);
}

for (const [, keys] of by_schema) {
await Promise.all(keys.map(key => queryAndParse(key)));
}

return sql;
};
Loading
Loading