Fix incremental table compilation failing on metadata.extraProperties

common/protos/index.ts

@@ -32,6 +32,7 @@ const Struct = google.protobuf.Struct;
 
 // Save references to the original generated methods
 const originalVerify = Struct.verify;
+const originalFromObject = Struct.fromObject;
 
 // Monkey Patching Methods
 Struct.verify = function (object: any) {
@@ -46,6 +47,18 @@ Struct.verify = function (object: any) {
   return originalVerify.call(this, object);
 };
 
+Struct.fromObject = function (object: any) {
+  if (object && typeof object === "object" && !("fields" in object)) {
+    const fields: { [key: string]: any } = {};
+    for (const [k, v] of Object.entries(object)) {
+      fields[k] = unknownToValueShallow(v);
+    }
+    Object.keys(object).forEach(key => delete object[key]);
+    object.fields = fields;
+  }
+  return originalFromObject.call(this, object);
+};
+
 // This is a minimalist Typescript equivalent for the validation part of Profobuf's JsonFormat's
 // mergeMessage method:
 // https://github.com/protocolbuffers/protobuf/blob/670e0c2a0d0b64c994f743a73ee9b8926c47580d/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java#L1455
@@ -63,8 +76,12 @@ export function verifyObjectMatchesProto<Proto>(
     throw ReferenceError(`Expected a top-level object, but found an array`);
   }
 
-  // Calling toObject on the object/JSON creates a version only contains the valid proto fields.
-  protoType.verify(object);
+  // Calling fromObject on the object/JSON triggers Struct.fromObject to format Struct plain objects.
+  try {
+    protoType.fromObject(object);
+  } catch {
+    // fromObject may throw TypeError on invalid field types. Ignore it and let checkFields throw formatted ReferenceError.
+  }
   const proto = protoType.create(object);
   const protoCastObject = protoType.toObject(proto);
 

core/actions/assertion_test.ts

@@ -183,6 +183,40 @@ SELECT 1`
       });
     });
 
+    test("assertions can be configured with a plain object for extraProperties", () => {
+      const projectDir = tmpDirFixture.createNewTmpDir();
+      fs.writeFileSync(
+        path.join(projectDir, "workflow_settings.yaml"),
+        VALID_WORKFLOW_SETTINGS_YAML
+      );
+      fs.mkdirSync(path.join(projectDir, "definitions"));
+      fs.writeFileSync(
+        path.join(projectDir, "definitions/assertion.sqlx"),
+        `config {
+    type: "assertion",
+    metadata: {
+        extraProperties: {
+            priority: "high"
+        }
+    }
+}
+SELECT 1`
+      );
+
+      const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+      expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
+      expect(
+        asPlainObject(result.compile.compiledGraph.assertions[0].actionDescriptor.metadata)
+      ).deep.equals({
+        extraProperties: {
+          fields: {
+            priority: { stringValue: "high" }
+          }
+        }
+      });
+    });
+
     ["table", "view", "incremental"].forEach(tableType => {
       [`"fieldValue"`, `["fieldValue"]`].forEach(uniqueKeyField => {
         test(`for ${tableType} built-in assertions uniqueKey with value ${uniqueKeyField}`, () => {

core/actions/incremental_table_test.ts

@@ -211,6 +211,69 @@ SELECT 1`
       });
     });
 
+    test("onSchemaChange combined with metadata.extraProperties as a plain object", () => {
+      const tableName = "on_schema_change_with_metadata";
+      const tableContent = `
+config {
+  type: "incremental",
+  onSchemaChange: "IGNORE",
+  metadata: {
+    extraProperties: {
+      priority: "high"
+    }
+  }
+}
+
+SELECT 1`;
+      const projectDir = tmpDirFixture.createNewTmpDir();
+      fs.writeFileSync(
+        path.join(projectDir, "workflow_settings.yaml"),
+        VALID_WORKFLOW_SETTINGS_YAML
+      );
+      fs.mkdirSync(path.join(projectDir, "definitions"));
+      fs.writeFileSync(
+        path.join(projectDir, `definitions/${tableName}.sqlx`),
+        tableContent
+      );
+
+      const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+      expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
+      expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([
+        {
+          target: {
+            database: "defaultProject",
+            schema: "defaultDataset",
+            name: tableName
+          },
+          canonicalTarget: {
+            database: "defaultProject",
+            schema: "defaultDataset",
+            name: tableName
+          },
+          type: "incremental",
+          disabled: false,
+          protected: false,
+          hermeticity: "NON_HERMETIC",
+          onSchemaChange: "IGNORE",
+          enumType: "INCREMENTAL",
+          fileName: `definitions/${tableName}.sqlx`,
+          query: "\n\n\nSELECT 1",
+          incrementalQuery: "\n\n\nSELECT 1",
+          incrementalStrategy: "INCREMENTAL_STRATEGY_UNSPECIFIED",
+          actionDescriptor: {
+            metadata: {
+              extraProperties: {
+                fields: {
+                  priority: { stringValue: "high" }
+                }
+              }
+            }
+          }
+        }
+      ]);
+    });
+
     test("sqlx minimal config", () => {
       const minimalIncrementalTableName = "minimal_incremental";
       const minimalIncrementalTableContent = `

core/actions/view_test.ts

@@ -187,6 +187,40 @@ SELECT 1`
         );
       });
     });
+
+    test("views can be configured with a plain object for extraProperties", () => {
+      const projectDir = tmpDirFixture.createNewTmpDir();
+      fs.writeFileSync(
+        path.join(projectDir, "workflow_settings.yaml"),
+        VALID_WORKFLOW_SETTINGS_YAML
+      );
+      fs.mkdirSync(path.join(projectDir, "definitions"));
+      fs.writeFileSync(
+        path.join(projectDir, "definitions/view.sqlx"),
+        `config {
+    type: "view",
+    metadata: {
+        extraProperties: {
+            priority: "high"
+        }
+    }
+}
+SELECT 1`
+      );
+
+      const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+      expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
+      expect(
+        asPlainObject(result.compile.compiledGraph.tables[0].actionDescriptor.metadata)
+      ).deep.equals({
+        extraProperties: {
+          fields: {
+            priority: { stringValue: "high" }
+          }
+        }
+      });
+    });
   });
 
   test("action config options", () => {