Map DateTime to google.protobuf.Timestamp and document temporal string formats

elasticgraph-proto_ingestion/README.md

@@ -94,27 +94,100 @@ ElasticGraph.define_schema do |schema|
 end
 ```
 
+A custom scalar can also map to an externally defined proto type. Pass `import:` with the path of
+the proto file that defines the type:
+
+```ruby
+# in config/schema/duration.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.scalar_type "Duration" do |t|
+    t.mapping type: "keyword"
+    t.json_schema type: "string"
+    t.protobuf type: "google.protobuf.Duration", import: "google/protobuf/duration.proto"
+  end
+end
+```
+
+The generated `schema.proto` then contains `import "google/protobuf/duration.proto";`. ElasticGraph
+emits the import only when a generated message uses the scalar. The `import:` value must be the
+path of a `.proto` file.
+
+`field_comment:` documents the expected format on each generated field. This is useful when the
+proto type is wider than the ElasticGraph type:
+
+```ruby
+# in config/schema/phone_number.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.scalar_type "PhoneNumber" do |t|
+    t.mapping type: "keyword"
+    t.json_schema type: "string"
+    t.protobuf type: "string", field_comment: "Must be an E.164 phone number."
+  end
+end
+```
+
+A `PhoneNumber` field then renders as:
+
+```protobuf
+// Must be an E.164 phone number.
+string phone_number = 1;
+```
+
+The comment goes above the field, below the field's own doc comment, because proto compilers
+attach these leading comments to the code they generate for the field. A `field_comment:` can
+span multiple lines. The option is named `field_comment:` rather than `comment:` because a scalar
+type has no proto representation of its own to comment on; the comment applies to each field of
+that type.
+
+### Overriding a Built-in Scalar
+
+Use `on_built_in_types` to change the protobuf type of a built-in scalar. For example, map
+`DateTime` to `string` to keep the original UTC offset of each event:
+
+```ruby
+# in config/schema/protobuf.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.on_built_in_types do |type|
+    type.protobuf type: "string", field_comment: "Must be formatted as an ISO 8601 timestamp." if type.name == "DateTime"
+  end
+end
+```
+
+Each call to `protobuf` replaces the full protobuf configuration. The override above omits
+`import:`, so `schema.proto` no longer imports `google/protobuf/timestamp.proto`. An override that
+omits `field_comment:` likewise drops the built-in comment.
+
 ## Type Mappings
 
 The generated `schema.proto` uses these built-in scalar mappings:
 
-| ElasticGraph Type | Protobuf Type |
-|-------------------|------------|
-| `Boolean`         | `bool`     |
-| `Cursor`          | `string`   |
-| `Date`            | `string`   |
-| `DateTime`        | `string`   |
-| `Float`           | `double`   |
-| `ID`              | `string`   |
-| `Int`             | `int32`    |
-| `JsonSafeLong`    | `int64`    |
-| `LocalTime`       | `string`   |
-| `LongString`      | `int64`    |
-| `String`          | `string`   |
-| `TimeZone`        | `string`   |
-| `Untyped`         | `string`   |
+| ElasticGraph Type | Protobuf Type               |
+|-------------------|-----------------------------|
+| `Boolean`         | `bool`                      |
+| `Cursor`          | `string`                    |
+| `Date`            | `string`                    |
+| `DateTime`        | `google.protobuf.Timestamp` |
+| `Float`           | `double`                    |
+| `ID`              | `string`                    |
+| `Int`             | `int32`                     |
+| `JsonSafeLong`    | `int64`                     |
+| `LocalTime`       | `string`                    |
+| `LongString`      | `int64`                     |
+| `String`          | `string`                    |
+| `TimeZone`        | `string`                    |
+| `Untyped`         | `string`                    |
 
 Additionally:
+- `DateTime` uses the [well-known `Timestamp` type](https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp);
+  `schema.proto` imports `google/protobuf/timestamp.proto` automatically. Note that a `Timestamp`
+  is a UTC instant, so a publisher's original UTC offset is not preserved.
+- `string`-typed temporal scalars (`Date`, `LocalTime`, `TimeZone`) are wider than the
+  ElasticGraph types they carry, so generated fields of these types document the expected format
+  in a comment above the field (e.g. `// Must be formatted as an ISO 8601 date, e.g. "2024-11-25".`).
+  Values are validated when events are ingested, just as with JSON ingestion.
 - List types become `repeated` fields.
 - Lists of lists (e.g. `[[Float!]!]!`) are not supported because Protocol Buffers cannot represent
   them directly. Schema artifact generation raises an error identifying the unsupported field.

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -48,6 +48,7 @@ def to_proto
           sections = [
             %(syntax = "proto3";),
             "package #{@package_name};",
+            *render_imports(types),
             render_definitions(types)
           ]
 
@@ -107,6 +108,15 @@ def render_definitions(types)
             .join("\n\n")
         end
 
+        # Every type reports the proto file it needs imported, or `nil` when it needs none. Today only
+        # scalar types map to an externally defined proto type, but enum and object types can start
+        # requiring an import without any change here.
+        def render_imports(types)
+          imports = types.filter_map(&:protobuf_import).uniq.sort
+
+          imports.empty? ? [] : [imports.map { |import| %(import "#{import}";) }.join("\n")]
+        end
+
         def validate_unique_enum_value_prefixes(types)
           enum_type_by_prefix = {} # : ::Hash[::String, untyped]
 

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb

@@ -67,6 +67,21 @@ def proto_type_reference(package_name)
             ".#{package_name}.#{proto_name}"
           end
 
+          # Enum types render their own protobuf definition, so they never require an import.
+          #
+          # @return [nil]
+          def protobuf_import
+            nil
+          end
+
+          # Enum values are self-describing, so fields of this type get no format comment.
+          # Only scalar types document a format.
+          #
+          # @return [nil]
+          def protobuf_field_comment
+            nil
+          end
+
           # Returns the package-level prefix applied to this enum's protobuf values.
           #
           # @return [String]

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb

@@ -66,6 +66,21 @@ def proto_type_reference(package_name)
             ".#{package_name}.#{proto_name}"
           end
 
+          # Messages render their own protobuf definition, so they never require an import.
+          #
+          # @return [nil]
+          def protobuf_import
+            nil
+          end
+
+          # Messages carry their documentation on the message definition itself, so fields of this
+          # type get no format comment. Only scalar types document a format.
+          #
+          # @return [nil]
+          def protobuf_field_comment
+            nil
+          end
+
           private
 
           def render_proto_message(schema, message_name, package_name)
@@ -75,7 +90,7 @@ def render_proto_message(schema, message_name, package_name)
             active_field_names = fields.map { |schema_field, _| schema_field.name }
             documentation = ProtoDocumentation.comment_lines_for(doc_comment).map { |line| "#{line}\n" }.join
             field_definitions = fields.map do |schema_field, field|
-              repeated, field_type = proto_field_type_for(
+              repeated, field_type, field_comment = proto_field_type_for(
                 field.type,
                 package_name: package_name,
                 context_field_name: field.name
@@ -87,12 +102,9 @@ def render_proto_message(schema, message_name, package_name)
               )
               label = "repeated " if repeated
               line = "  #{label}#{field_type} #{schema_field.name} = #{field_number};"
-              field_documentation = ProtoDocumentation
-                .comment_lines_for(schema_field.doc_comment, indent: "  ")
-                .map { |comment_line| "#{comment_line}\n" }
-                .join
+              comment_lines = field_comment_lines_for(schema_field.doc_comment, field_comment)
 
-              "#{field_documentation}#{line}"
+              [*comment_lines, line].join("\n")
             end
             schema.reserved_field_numbers_for(message_name, active_field_names).each do |field_name, field_number|
               field_definitions << "  reserved #{field_number}; // Previously used by #{field_name}."
@@ -150,6 +162,19 @@ def proto_fields
             end
           end
 
+          # Renders a field's documentation and its type's format comment as the `//` lines that go
+          # above the field. Proto compilers attach these leading comments to the code they generate
+          # for the field, whereas a trailing comment on the field line is usually discarded.
+          def field_comment_lines_for(doc_comment, field_comment)
+            doc_lines = ProtoDocumentation.comment_lines_for(doc_comment, indent: "  ")
+            return doc_lines unless field_comment
+
+            format_lines = ProtoDocumentation.comment_lines_for(field_comment, indent: "  ")
+            return format_lines if doc_lines.empty?
+
+            doc_lines + ["  //"] + format_lines
+          end
+
           def proto_field_type_for(type_ref, package_name:, context_field_name:)
             list_depth, base_type_ref = ObjectInterfaceAndUnionExtension.list_depth_and_base_type(type_ref)
 
@@ -160,7 +185,7 @@ def proto_field_type_for(type_ref, package_name:, context_field_name:)
             end
 
             proto_type = _ = base_type_ref.resolved
-            [list_depth == 1, proto_type.proto_type_reference(package_name)]
+            [list_depth == 1, proto_type.proto_type_reference(package_name), proto_type.protobuf_field_comment]
           end
         end
       end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb

@@ -14,33 +14,57 @@ module SchemaDefinition
       module SchemaElements
         # Extends ScalarType with proto field type conversion.
         module ScalarTypeExtension
-          # Default protobuf types applied to ElasticGraph's built-in scalar types as they are constructed.
-          BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME = {
-            "Boolean" => "bool",
-            "Cursor" => "string",
-            "Date" => "string",
-            "DateTime" => "string",
-            "Float" => "double",
-            "ID" => "string",
-            "Int" => "int32",
-            "JsonSafeLong" => "int64",
-            "LocalTime" => "string",
-            "LongString" => "int64",
-            "String" => "string",
-            "TimeZone" => "string",
-            "Untyped" => "string"
-          }.freeze
+          # Default protobuf options applied to ElasticGraph's built-in scalar types as they are constructed.
+          BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME = {
+            "Boolean" => {type: "bool"},
+            "Cursor" => {type: "string"},
+            "Date" => {type: "string", field_comment: %(Must be formatted as an ISO 8601 date, e.g. "2024-11-25".)},
+            "DateTime" => {type: "google.protobuf.Timestamp", import: "google/protobuf/timestamp.proto"},
+            "Float" => {type: "double"},
+            "ID" => {type: "string"},
+            "Int" => {type: "int32"},
+            "JsonSafeLong" => {type: "int64"},
+            "LocalTime" => {type: "string", field_comment: %(Must be formatted as an ISO 8601 local time, e.g. "14:23:12".)},
+            "LongString" => {type: "int64"},
+            "String" => {type: "string"},
+            "TimeZone" => {type: "string", field_comment: %(Must be an IANA time zone identifier, e.g. "America/Los_Angeles".)},
+            "Untyped" => {type: "string"}
+          }.freeze # : ::Hash[::String, {type: ::String, ?import: ::String, ?field_comment: ::String}]
+
+          # An `import` is rendered as `import "PATH";`, so a quote or newline in the path would
+          # produce invalid proto. `protoc` also requires the path to name a `.proto` file.
+          VALID_PROTOBUF_IMPORT_PATH = %r{\A[\w./-]+\.proto\z}
 
           # Configured protobuf type (e.g. string, int64, bool).
           # @dynamic protobuf_type
           attr_reader :protobuf_type
 
-          # Configures the protobuf type for this scalar type.
+          # Proto file to import for the configured protobuf type, if it is externally defined.
+          # @dynamic protobuf_import
+          attr_reader :protobuf_import
+
+          # Comment rendered above each generated proto field of this scalar type.
+          # @dynamic protobuf_field_comment
+          attr_reader :protobuf_field_comment
+
+          # Configures the protobuf type for this scalar type. Each call replaces the full protobuf
+          # configuration, so an override that omits `import:` or `field_comment:` clears the value
+          # configured by a prior call.
           #
-          # @param type [String] protobuf scalar type name
+          # @param type [String] protobuf type name
+          # @param import [String, nil] proto file to import for an externally defined type
+          # @param field_comment [String, nil] comment rendered above each generated field of this type
           # @return [void]
-          def protobuf(type:)
+          # @raise [Errors::SchemaError] when `import` is not a `.proto` file path
+          def protobuf(type:, import: nil, field_comment: nil)
+            if import && !VALID_PROTOBUF_IMPORT_PATH.match?(import)
+              raise Errors::SchemaError, "`protobuf` import for `#{name}` must be the path of a `.proto` file, " \
+                "but got: #{import.inspect}."
+            end
+
             @protobuf_type = type
+            @protobuf_import = import
+            @protobuf_field_comment = field_comment
           end
 
           # Applies any built-in protobuf type, yields for further configuration, and validates the result.
@@ -50,8 +74,8 @@ def protobuf(type:)
           # @raise [Errors::SchemaError] when a protobuf type is missing
           def initialize_proto_extension
             original_name = type_ref.with_reverted_override.name
-            if (proto_type = BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME[original_name])
-              protobuf type: proto_type
+            if (proto_options = BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME[original_name])
+              protobuf(**proto_options)
             end
 
             yield

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs

@@ -34,6 +34,7 @@ module ElasticGraph
 
         def proto_types: () -> ::Array[untyped]
         def render_definitions: (::Array[untyped] types) -> ::String
+        def render_imports: (::Array[untyped] types) -> ::Array[::String]
         def validate_unique_enum_value_prefixes: (::Array[untyped] types) -> void
         def previous_field_names_for: (::String, ::String) -> ::Array[::String]
         def previous_field_names_by_type_name_and_field_name: () -> ::Hash[::String, ::Hash[::String, ::Array[::String]]]

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs

@@ -11,6 +11,8 @@ module ElasticGraph
           def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void
           def proto_name: () -> ::String
           def proto_type_reference: (::String package_name) -> ::String
+          def protobuf_import: () -> nil
+          def protobuf_field_comment: () -> nil
           def proto_enum_value_prefix: () -> ::String
           def to_proto: (Schema schema, ::String package_name) -> ::String
           def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType]

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs

@@ -10,6 +10,8 @@ module ElasticGraph
 
           def proto_name: () -> ::String
           def proto_type_reference: (::String package_name) -> ::String
+          def protobuf_import: () -> nil
+          def protobuf_field_comment: () -> nil
           def to_proto: (Schema schema, ::String package_name) -> ::String
           def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType]
           def self.list_depth_and_base_type: (
@@ -24,11 +26,12 @@ module ElasticGraph
             ::ElasticGraph::SchemaDefinition::SchemaElements::Field,
             ::ElasticGraph::SchemaDefinition::Indexing::Field
           ]]
+          def field_comment_lines_for: (::String? doc_comment, ::String? field_comment) -> ::Array[::String]
           def proto_field_type_for: (
             ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference,
             package_name: ::String,
             context_field_name: ::String
-          ) -> [bool, ::String]
+          ) -> [bool, ::String, ::String?]
         end
       end
     end

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs

@@ -3,11 +3,14 @@ module ElasticGraph
     module SchemaDefinition
       module SchemaElements
         module ScalarTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType
-          BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME: ::Hash[::String, ::String]
+          BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME: ::Hash[::String, {type: ::String, ?import: ::String, ?field_comment: ::String}]
+          VALID_PROTOBUF_IMPORT_PATH: ::Regexp
 
           attr_reader protobuf_type: ::String?
+          attr_reader protobuf_import: ::String?
+          attr_reader protobuf_field_comment: ::String?
 
-          def protobuf: (type: ::String) -> void
+          def protobuf: (type: ::String, ?import: ::String?, ?field_comment: ::String?) -> void
           def initialize_proto_extension: () { () -> void } -> void
           def proto_name: () -> ::String
           def proto_type_reference: (::String package_name) -> ::String