diff --git a/common/protos/index.ts b/common/protos/index.ts index 2bf2777b2..11642835a 100644 --- a/common/protos/index.ts +++ b/common/protos/index.ts @@ -32,6 +32,7 @@ const Struct = google.protobuf.Struct; // Save references to the original generated methods const originalVerify = Struct.verify; +const originalFromObject = Struct.fromObject; // Monkey Patching Methods Struct.verify = function (object: any) { @@ -46,6 +47,18 @@ Struct.verify = function (object: any) { return originalVerify.call(this, object); }; +Struct.fromObject = function (object: any) { + if (object && typeof object === "object" && !("fields" in object)) { + const fields: { [key: string]: any } = {}; + for (const [k, v] of Object.entries(object)) { + fields[k] = unknownToValueShallow(v); + } + Object.keys(object).forEach(key => delete object[key]); + object.fields = fields; + } + return originalFromObject.call(this, object); +}; + // This is a minimalist Typescript equivalent for the validation part of Profobuf's JsonFormat's // mergeMessage method: // https://github.com/protocolbuffers/protobuf/blob/670e0c2a0d0b64c994f743a73ee9b8926c47580d/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java#L1455 @@ -63,8 +76,12 @@ export function verifyObjectMatchesProto( throw ReferenceError(`Expected a top-level object, but found an array`); } - // Calling toObject on the object/JSON creates a version only contains the valid proto fields. - protoType.verify(object); + // Calling fromObject on the object/JSON triggers Struct.fromObject to format Struct plain objects. + try { + protoType.fromObject(object); + } catch { + // fromObject may throw TypeError on invalid field types. Ignore it and let checkFields throw formatted ReferenceError. + } const proto = protoType.create(object); const protoCastObject = protoType.toObject(proto); diff --git a/core/actions/assertion_test.ts b/core/actions/assertion_test.ts index 9deeedad8..91abf197d 100644 --- a/core/actions/assertion_test.ts +++ b/core/actions/assertion_test.ts @@ -183,6 +183,40 @@ SELECT 1` }); }); + test("assertions can be configured with a plain object for extraProperties", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/assertion.sqlx"), + `config { + type: "assertion", + metadata: { + extraProperties: { + priority: "high" + } + } +} +SELECT 1` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + asPlainObject(result.compile.compiledGraph.assertions[0].actionDescriptor.metadata) + ).deep.equals({ + extraProperties: { + fields: { + priority: { stringValue: "high" } + } + } + }); + }); + ["table", "view", "incremental"].forEach(tableType => { [`"fieldValue"`, `["fieldValue"]`].forEach(uniqueKeyField => { test(`for ${tableType} built-in assertions uniqueKey with value ${uniqueKeyField}`, () => { diff --git a/core/actions/incremental_table_test.ts b/core/actions/incremental_table_test.ts index b80ceab43..9f6708ba2 100644 --- a/core/actions/incremental_table_test.ts +++ b/core/actions/incremental_table_test.ts @@ -211,6 +211,69 @@ SELECT 1` }); }); + test("onSchemaChange combined with metadata.extraProperties as a plain object", () => { + const tableName = "on_schema_change_with_metadata"; + const tableContent = ` +config { + type: "incremental", + onSchemaChange: "IGNORE", + metadata: { + extraProperties: { + priority: "high" + } + } +} + +SELECT 1`; + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, `definitions/${tableName}.sqlx`), + tableContent + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + target: { + database: "defaultProject", + schema: "defaultDataset", + name: tableName + }, + canonicalTarget: { + database: "defaultProject", + schema: "defaultDataset", + name: tableName + }, + type: "incremental", + disabled: false, + protected: false, + hermeticity: "NON_HERMETIC", + onSchemaChange: "IGNORE", + enumType: "INCREMENTAL", + fileName: `definitions/${tableName}.sqlx`, + query: "\n\n\nSELECT 1", + incrementalQuery: "\n\n\nSELECT 1", + incrementalStrategy: "INCREMENTAL_STRATEGY_UNSPECIFIED", + actionDescriptor: { + metadata: { + extraProperties: { + fields: { + priority: { stringValue: "high" } + } + } + } + } + } + ]); + }); + test("sqlx minimal config", () => { const minimalIncrementalTableName = "minimal_incremental"; const minimalIncrementalTableContent = ` diff --git a/core/actions/view_test.ts b/core/actions/view_test.ts index cc4c3fbdd..5227b84d8 100644 --- a/core/actions/view_test.ts +++ b/core/actions/view_test.ts @@ -187,6 +187,40 @@ SELECT 1` ); }); }); + + test("views can be configured with a plain object for extraProperties", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/view.sqlx"), + `config { + type: "view", + metadata: { + extraProperties: { + priority: "high" + } + } +} +SELECT 1` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + asPlainObject(result.compile.compiledGraph.tables[0].actionDescriptor.metadata) + ).deep.equals({ + extraProperties: { + fields: { + priority: { stringValue: "high" } + } + } + }); + }); }); test("action config options", () => {