diff --git a/esbuild.package.js b/esbuild.package.js index 9cf72dc6..c95c4aaa 100644 --- a/esbuild.package.js +++ b/esbuild.package.js @@ -16,6 +16,7 @@ esbuild path.resolve(__dirname, 'forward_engineering', 'api.js'), path.resolve(__dirname, 'forward_engineering', 'ddlProvider.js'), path.resolve(__dirname, 'forward_engineering', 'dbtProvider.js'), + path.resolve(__dirname, 'forward_engineering', 'dataHubProvider.js'), path.resolve(__dirname, 'reverse_engineering', 'api.js'), ], bundle: true, diff --git a/forward_engineering/config.json b/forward_engineering/config.json index 511cb29f..ef7e55f1 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -444,5 +444,65 @@ } ] } - ] + ], + "dataHub": { + "urn": "urn:li:dataPlatform:snowflake", + "containers": [ + { + "type": "Database", + "source": "container", + "properties": { + "name": "database" + } + }, + "Schema" + ], + "datasets": ["Table", "View"], + "typeMapping": { + "DATE": "DateType", + "BIGINT": "NumberType", + "BINARY": "BytesType", + "BOOLEAN": "BooleanType", + "CHAR": "StringType", + "CHARACTER": "StringType", + "DATETIME": "TimeType", + "DEC": "NumberType", + "DECIMAL": "NumberType", + "NUMERIC": "NumberType", + "DOUBLE": "NumberType", + "DOUBLE PRECISION": "NumberType", + "FIXED": "NumberType", + "FLOAT": "NumberType", + "FLOAT4": "NumberType", + "FLOAT8": "NumberType", + "INT": "NumberType", + "INTEGER": "NumberType", + "NUMBER": "NumberType", + "OBJECT": "RecordType", + "REAL": "NumberType", + "BYTEINT": "NumberType", + "SMALLINT": "NumberType", + "STRING": "StringType", + "TEXT": "StringType", + "TIME": "TimeType", + "TIMESTAMP": "TimeType", + "TIMESTAMP_TZ": "TimeType", + "TIMESTAMP_LTZ": "TimeType", + "TIMESTAMP_NTZ": "TimeType", + "TINYINT": "NumberType", + "VARBINARY": "BytesType", + "VARCHAR": "StringType", + "CHARACTER VARYING": "StringType", + "VARIANT": "RecordType", + "ARRAY": "ArrayType", + "GEOGRAPHY": "NullType" + }, + "options": { + "convertUrnsToLowerCase": { + "use": true, + "default": true + } + }, + "viewLanguage": "SQL" + } } diff --git a/forward_engineering/dataHubProvider.js b/forward_engineering/dataHubProvider.js new file mode 100644 index 00000000..dceef76c --- /dev/null +++ b/forward_engineering/dataHubProvider.js @@ -0,0 +1,100 @@ +/** + * @typedef {import('./types').AppInstance} AppInstance + * @typedef {import('./types').ColumnDefinition} ColumnDefinition + * @typedef {import('./types').ConstraintDto} ConstraintDto + * @typedef {import('./types').JsonSchema} JsonSchema + */ +const { toLower } = require('lodash'); + +const types = require('./configs/types'); +const defaultTypes = require('./configs/defaultTypes'); +const getKeyHelper = require('./helpers/keyHelper'); +const getColumnDefinitionHelper = require('./helpers/columnDefinitionHelper'); +const { createView, hydrateView, hydrateViewColumn } = require('./helpers/viewHelper'); +const { FORMATS } = require('./helpers/constants'); + +class DataHubProvider { + /** + * @type {AppInstance} + */ + #appInstance; + + /** + * @param {{ appInstance: AppInstance }} + */ + constructor({ appInstance }) { + this.#appInstance = appInstance; + } + + /** + * @param {{ appInstance }} + * @returns {DataHubProvider} + */ + static createDataHubProvider({ appInstance }) { + return new DataHubProvider({ appInstance }); + } + + /** + * @param {string} type + * @returns {string | undefined} + */ + getDefaultType(type) { + return defaultTypes[type]; + } + + /** + * @returns {Record} + */ + getTypesDescriptors() { + return types; + } + + /** + * @param {string} type + * @returns {boolean} + */ + hasType(type) { + return Object.keys(types).map(toLower).includes(toLower(type)); + } + + /** + * @param {{ columnDefinition: ColumnDefinition; }} + * @returns {{ type: string; }} + */ + decorateType({ type, columnDefinition }) { + const columnDefinitionHelper = getColumnDefinitionHelper(this.#appInstance); + + return columnDefinitionHelper.decorateType(type, columnDefinition); + } + + createView(viewData, dbData, isActivated) { + const { getViewSelectStatement } = require('./helpers/tableHelper')(this.#appInstance); + const keyHelper = require('./helpers/keyHelper')(this.#appInstance); + + return createView({ + viewData, + isActivated, + scriptFormat: FORMATS.SNOWSIGHT, + getViewSelectStatement, + keyHelper, + }); + } + + hydrateView({ viewData, entityData }) { + return hydrateView({ viewData, entityData }); + } + + hydrateViewColumn(data) { + return hydrateViewColumn(data); + } + + getPlatformSchema() { + return { + 'com.linkedin.schema.MySqlDDL': { + tableSchema: '', + }, + }; + } +} + +module.exports = DataHubProvider; diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 331f5736..5793c027 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -56,6 +56,7 @@ const { prepareObjectTagsData, isEmptyTags, } = require('./helpers/tagHelper'); +const { createView, hydrateView, hydrateViewColumn } = require('./helpers/viewHelper'); const DEFAULT_SNOWFLAKE_SEQUENCE_START = 1; const DEFAULT_SNOWFLAKE_SEQUENCE_INCREMENT = 1; @@ -582,76 +583,12 @@ module.exports = (baseProvider, options, app) => { }, createView(viewData, dbData, isActivated) { - const orReplace = preSpace(viewData.orReplace && 'OR REPLACE'); - const ifNotExist = preSpace(viewData.ifNotExist && 'IF NOT EXISTS'); - const { columnList, tableColumns, tables } = viewData.keys.reduce( - (result, key) => { - result.columnList.push({ - name: `${getName(viewData.isCaseSensitive, key.alias || key.name)}`, - isActivated: key.isActivated, - comment: preSpace( - key.definition?.description && - `COMMENT ${escapeString(scriptFormat, key.definition.description)}`, - ), - }); - result.tableColumns.push({ - name: `${getName(viewData.isCaseSensitive, key.entityName)}.${getName(viewData.isCaseSensitive, key.name)}`, - isActivated: key.isActivated, - }); - - if (key.entityName) { - const tableName = getFullName(key.dbName, key.entityName); - - if (!result.tables.includes(tableName)) { - result.tables.push(tableName); - } - } - - return result; - }, - { - columnList: [], - tableColumns: [], - tables: [], - }, - ); - - if (isEmpty(tables) && !viewData.selectStatement) { - return ''; - } - - const viewColumns = viewColumnsToString(tableColumns, isActivated); - const selectStatement = getViewSelectStatement({ - tables, + return createView({ viewData, - viewColumns, - }); - - const tagStatement = getTagStatement({ - tags: viewData.viewTags, - isCaseSensitive: viewData.isCaseSensitive, - indent: '', - }); - - const clustering = viewData.materialized - ? keyHelper.getClusteringKey({ - clusteringKey: viewData.clusteringKey, - isParentActivated: isActivated, - }) - : undefined; - - return assignTemplates(templates.createView, { - orReplace, - ifNotExist, - secure: preSpace(viewData.secure && 'SECURE'), - materialized: preSpace(viewData.materialized && 'MATERIALIZED'), - name: viewData.fullName, - column_list: viewColumnsToString(columnList, isActivated), - copy_grants: viewData.copyGrants ? 'COPY GRANTS\n' : '', - comment: viewData.comment ? `COMMENT=${escapeString(scriptFormat, viewData.comment)}\n` : '', - select_statement: selectStatement, - tag: tagStatement ? tagStatement + '\n' : '', - clustering, + isActivated, + scriptFormat, + getViewSelectStatement, + keyHelper, }); }, @@ -1006,39 +943,11 @@ module.exports = (baseProvider, options, app) => { }, hydrateView({ viewData, entityData }) { - const firstTab = entityData[0]; - const { databaseName, schemaName } = viewData.schemaData; - const viewName = getName(firstTab.isCaseSensitive, viewData.name); - const fullName = getFullName(getFullName(databaseName, schemaName), viewName); - - return { - ...viewData, - orReplace: firstTab.orReplace, - ifNotExist: firstTab.ifNotExist, - name: getName(firstTab.isCaseSensitive, viewData.name), - selectStatement: firstTab.selectStatement, - isCaseSensitive: firstTab.isCaseSensitive, - copyGrants: firstTab.copyGrants, - comment: firstTab.description, - secure: firstTab.secure, - materialized: firstTab.materialized, - fullName, - clusteringKey: firstTab.clusteringKey, - viewTags: firstTab.viewTags ?? [], - }; + return hydrateView({ viewData, entityData }); }, hydrateViewColumn(data) { - if (!data.entityName) { - return data; - } - - return { - ...data, - name: getName(data.definition?.isCaseSensitive, data.name), - dbName: getName(head(data.containerData)?.isCaseSensitive, data.dbName), - entityName: getName(head(data.entityData)?.isCaseSensitive, data.entityName), - }; + return hydrateViewColumn(data); }, commentIfDeactivated(statement, data, isPartOfLine) { diff --git a/forward_engineering/helpers/viewHelper.js b/forward_engineering/helpers/viewHelper.js new file mode 100644 index 00000000..251ddbc5 --- /dev/null +++ b/forward_engineering/helpers/viewHelper.js @@ -0,0 +1,122 @@ +const { isEmpty, head } = require('lodash'); +const { preSpace } = require('../utils/preSpace'); +const { getName, getFullName, viewColumnsToString } = require('./general'); +const { escapeString } = require('../utils/escapeString'); +const { getTagStatement } = require('./tagHelper'); +const assignTemplates = require('../utils/assignTemplates'); +const templates = require('../configs/templates'); + +const createView = ({ viewData, isActivated, scriptFormat, getViewSelectStatement, keyHelper }) => { + const orReplace = preSpace(viewData.orReplace && 'OR REPLACE'); + const ifNotExist = preSpace(viewData.ifNotExist && 'IF NOT EXISTS'); + const { columnList, tableColumns, tables } = viewData.keys.reduce( + (result, key) => { + result.columnList.push({ + name: `${getName(viewData.isCaseSensitive, key.alias || key.name)}`, + isActivated: key.isActivated, + comment: preSpace( + key.definition?.description && `COMMENT ${escapeString(scriptFormat, key.definition.description)}`, + ), + }); + result.tableColumns.push({ + name: `${getName(viewData.isCaseSensitive, key.entityName)}.${getName(viewData.isCaseSensitive, key.name)}`, + isActivated: key.isActivated, + }); + + if (key.entityName) { + const tableName = getFullName(key.dbName, key.entityName); + + if (!result.tables.includes(tableName)) { + result.tables.push(tableName); + } + } + + return result; + }, + { + columnList: [], + tableColumns: [], + tables: [], + }, + ); + + if (isEmpty(tables) && !viewData.selectStatement) { + return ''; + } + + const viewColumns = viewColumnsToString(tableColumns, isActivated); + const selectStatement = getViewSelectStatement({ + tables, + viewData, + viewColumns, + }); + + const tagStatement = getTagStatement({ + tags: viewData.viewTags, + isCaseSensitive: viewData.isCaseSensitive, + indent: '', + }); + + const clustering = viewData.materialized + ? keyHelper.getClusteringKey({ + clusteringKey: viewData.clusteringKey, + isParentActivated: isActivated, + }) + : undefined; + + return assignTemplates(templates.createView, { + orReplace, + ifNotExist, + secure: preSpace(viewData.secure && 'SECURE'), + materialized: preSpace(viewData.materialized && 'MATERIALIZED'), + name: viewData.fullName, + column_list: viewColumnsToString(columnList, isActivated), + copy_grants: viewData.copyGrants ? 'COPY GRANTS\n' : '', + comment: viewData.comment ? `COMMENT=${escapeString(scriptFormat, viewData.comment)}\n` : '', + select_statement: selectStatement, + tag: tagStatement ? tagStatement + '\n' : '', + clustering, + }); +}; + +const hydrateView = ({ viewData, entityData }) => { + const firstTab = entityData[0]; + const { databaseName, schemaName } = viewData.schemaData; + const viewName = getName(firstTab.isCaseSensitive, viewData.name); + const fullName = getFullName(getFullName(databaseName, schemaName), viewName); + + return { + ...viewData, + orReplace: firstTab.orReplace, + ifNotExist: firstTab.ifNotExist, + name: getName(firstTab.isCaseSensitive, viewData.name), + selectStatement: firstTab.selectStatement, + isCaseSensitive: firstTab.isCaseSensitive, + copyGrants: firstTab.copyGrants, + comment: firstTab.description, + secure: firstTab.secure, + materialized: firstTab.materialized, + fullName, + clusteringKey: firstTab.clusteringKey, + viewTags: firstTab.viewTags ?? [], + }; +}; + +const hydrateViewColumn = data => { + if (!data.entityName) { + return data; + } + + return { + ...data, + name: getName(data.definition?.isCaseSensitive, data.name), + dbName: getName(head(data.containerData)?.isCaseSensitive, data.dbName), + entityName: getName(head(data.entityData)?.isCaseSensitive, data.entityName), + }; +}; + +module.exports = { + createView, + hydrateView, + hydrateViewColumn, +}; diff --git a/package.json b/package.json index 7e8e502b..66387d43 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,8 @@ "FEScriptCommentsSupported": true, "disableJsonDataMaxLength": true, "discoverRelationships": true, - "enableKeysMultipleAbrr": true + "enableKeysMultipleAbrr": true, + "enableDataHub": true } }, "description": "Hackolade plugin for Snowflake",