diff --git a/.generator/schemas/v1/openapi.yaml b/.generator/schemas/v1/openapi.yaml index 8f404467bc2..bbfcb6da104 100644 --- a/.generator/schemas/v1/openapi.yaml +++ b/.generator/schemas/v1/openapi.yaml @@ -6215,6 +6215,38 @@ components: type: string x-enum-varnames: - DECODER_PROCESSOR + LogsExcludeAttributeProcessor: + description: |- + Use this processor to remove an attribute from a log during processing. + The processor strips the specified attribute from the log event, which is useful + when the attribute contains sensitive data or is no longer needed downstream. + properties: + attribute_to_exclude: + description: Name of the log attribute to remove from the log event. + example: foo + type: string + is_enabled: + default: false + description: Whether or not the processor is enabled. + type: boolean + name: + description: Name of the processor. + type: string + type: + $ref: "#/components/schemas/LogsExcludeAttributeProcessorType" + required: + - type + - attribute_to_exclude + type: object + LogsExcludeAttributeProcessorType: + default: exclude-attribute + description: Type of logs exclude attribute processor. + enum: + - exclude-attribute + example: exclude-attribute + type: string + x-enum-varnames: + - EXCLUDE_ATTRIBUTE LogsExclusion: description: Represents the index exclusion filter object from configuration API. properties: @@ -6822,6 +6854,7 @@ components: - $ref: "#/components/schemas/LogsArrayProcessor" - $ref: "#/components/schemas/LogsDecoderProcessor" - $ref: "#/components/schemas/LogsSchemaProcessor" + - $ref: "#/components/schemas/LogsExcludeAttributeProcessor" LogsQueryCompute: description: Define computation for a log query. properties: diff --git a/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessor.java b/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessor.java new file mode 100644 index 00000000000..94029117a07 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessor.java @@ -0,0 +1,240 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v1.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Use this processor to remove an attribute from a log during processing. The processor strips the + * specified attribute from the log event, which is useful when the attribute contains sensitive + * data or is no longer needed downstream. + */ +@JsonPropertyOrder({ + LogsExcludeAttributeProcessor.JSON_PROPERTY_ATTRIBUTE_TO_EXCLUDE, + LogsExcludeAttributeProcessor.JSON_PROPERTY_IS_ENABLED, + LogsExcludeAttributeProcessor.JSON_PROPERTY_NAME, + LogsExcludeAttributeProcessor.JSON_PROPERTY_TYPE +}) +@jakarta.annotation.Generated( + value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") +public class LogsExcludeAttributeProcessor { + @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_ATTRIBUTE_TO_EXCLUDE = "attribute_to_exclude"; + private String attributeToExclude; + + public static final String JSON_PROPERTY_IS_ENABLED = "is_enabled"; + private Boolean isEnabled = false; + + public static final String JSON_PROPERTY_NAME = "name"; + private String name; + + public static final String JSON_PROPERTY_TYPE = "type"; + private LogsExcludeAttributeProcessorType type = + LogsExcludeAttributeProcessorType.EXCLUDE_ATTRIBUTE; + + public LogsExcludeAttributeProcessor() {} + + @JsonCreator + public LogsExcludeAttributeProcessor( + @JsonProperty(required = true, value = JSON_PROPERTY_ATTRIBUTE_TO_EXCLUDE) + String attributeToExclude, + @JsonProperty(required = true, value = JSON_PROPERTY_TYPE) + LogsExcludeAttributeProcessorType type) { + this.attributeToExclude = attributeToExclude; + this.type = type; + this.unparsed |= !type.isValid(); + } + + public LogsExcludeAttributeProcessor attributeToExclude(String attributeToExclude) { + this.attributeToExclude = attributeToExclude; + return this; + } + + /** + * Name of the log attribute to remove from the log event. + * + * @return attributeToExclude + */ + @JsonProperty(JSON_PROPERTY_ATTRIBUTE_TO_EXCLUDE) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public String getAttributeToExclude() { + return attributeToExclude; + } + + public void setAttributeToExclude(String attributeToExclude) { + this.attributeToExclude = attributeToExclude; + } + + public LogsExcludeAttributeProcessor isEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled; + return this; + } + + /** + * Whether or not the processor is enabled. + * + * @return isEnabled + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_IS_ENABLED) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Boolean getIsEnabled() { + return isEnabled; + } + + public void setIsEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled; + } + + public LogsExcludeAttributeProcessor name(String name) { + this.name = name; + return this; + } + + /** + * Name of the processor. + * + * @return name + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_NAME) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public LogsExcludeAttributeProcessor type(LogsExcludeAttributeProcessorType type) { + this.type = type; + this.unparsed |= !type.isValid(); + return this; + } + + /** + * Type of logs exclude attribute processor. + * + * @return type + */ + @JsonProperty(JSON_PROPERTY_TYPE) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public LogsExcludeAttributeProcessorType getType() { + return type; + } + + public void setType(LogsExcludeAttributeProcessorType type) { + if (!type.isValid()) { + this.unparsed = true; + } + this.type = type; + } + + /** + * A container for additional, undeclared properties. This is a holder for any undeclared + * properties as specified with the 'additionalProperties' keyword in the OAS document. + */ + private Map additionalProperties; + + /** + * Set the additional (undeclared) property with the specified name and value. If the property + * does not already exist, create it otherwise replace it. + * + * @param key The arbitrary key to set + * @param value The associated value + * @return LogsExcludeAttributeProcessor + */ + @JsonAnySetter + public LogsExcludeAttributeProcessor putAdditionalProperty(String key, Object value) { + if (this.additionalProperties == null) { + this.additionalProperties = new HashMap(); + } + this.additionalProperties.put(key, value); + return this; + } + + /** + * Return the additional (undeclared) property. + * + * @return The additional properties + */ + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + /** + * Return the additional (undeclared) property with the specified name. + * + * @param key The arbitrary key to get + * @return The specific additional property for the given key + */ + public Object getAdditionalProperty(String key) { + if (this.additionalProperties == null) { + return null; + } + return this.additionalProperties.get(key); + } + + /** Return true if this LogsExcludeAttributeProcessor object is equal to o. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogsExcludeAttributeProcessor logsExcludeAttributeProcessor = (LogsExcludeAttributeProcessor) o; + return Objects.equals(this.attributeToExclude, logsExcludeAttributeProcessor.attributeToExclude) + && Objects.equals(this.isEnabled, logsExcludeAttributeProcessor.isEnabled) + && Objects.equals(this.name, logsExcludeAttributeProcessor.name) + && Objects.equals(this.type, logsExcludeAttributeProcessor.type) + && Objects.equals( + this.additionalProperties, logsExcludeAttributeProcessor.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(attributeToExclude, isEnabled, name, type, additionalProperties); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LogsExcludeAttributeProcessor {\n"); + sb.append(" attributeToExclude: ").append(toIndentedString(attributeToExclude)).append("\n"); + sb.append(" isEnabled: ").append(toIndentedString(isEnabled)).append("\n"); + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" type: ").append(toIndentedString(type)).append("\n"); + sb.append(" additionalProperties: ") + .append(toIndentedString(additionalProperties)) + .append("\n"); + sb.append('}'); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessorType.java b/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessorType.java new file mode 100644 index 00000000000..07dc9822a30 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v1/model/LogsExcludeAttributeProcessorType.java @@ -0,0 +1,58 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v1.model; + +import com.datadog.api.client.ModelEnum; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Type of logs exclude attribute processor. */ +@JsonSerialize( + using = LogsExcludeAttributeProcessorType.LogsExcludeAttributeProcessorTypeSerializer.class) +public class LogsExcludeAttributeProcessorType extends ModelEnum { + + private static final Set allowedValues = + new HashSet(Arrays.asList("exclude-attribute")); + + public static final LogsExcludeAttributeProcessorType EXCLUDE_ATTRIBUTE = + new LogsExcludeAttributeProcessorType("exclude-attribute"); + + LogsExcludeAttributeProcessorType(String value) { + super(value, allowedValues); + } + + public static class LogsExcludeAttributeProcessorTypeSerializer + extends StdSerializer { + public LogsExcludeAttributeProcessorTypeSerializer(Class t) { + super(t); + } + + public LogsExcludeAttributeProcessorTypeSerializer() { + this(null); + } + + @Override + public void serialize( + LogsExcludeAttributeProcessorType value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + jgen.writeObject(value.value); + } + } + + @JsonCreator + public static LogsExcludeAttributeProcessorType fromValue(String value) { + return new LogsExcludeAttributeProcessorType(value); + } +} diff --git a/src/main/java/com/datadog/api/client/v1/model/LogsProcessor.java b/src/main/java/com/datadog/api/client/v1/model/LogsProcessor.java index 0b1bfba06cc..199b9fcba3b 100644 --- a/src/main/java/com/datadog/api/client/v1/model/LogsProcessor.java +++ b/src/main/java/com/datadog/api/client/v1/model/LogsProcessor.java @@ -965,6 +965,51 @@ public LogsProcessor deserialize(JsonParser jp, DeserializationContext ctxt) log.log(Level.FINER, "Input data does not match schema 'LogsSchemaProcessor'", e); } + // deserialize LogsExcludeAttributeProcessor + try { + boolean attemptParsing = true; + // ensure that we respect type coercion as set on the client ObjectMapper + if (LogsExcludeAttributeProcessor.class.equals(Integer.class) + || LogsExcludeAttributeProcessor.class.equals(Long.class) + || LogsExcludeAttributeProcessor.class.equals(Float.class) + || LogsExcludeAttributeProcessor.class.equals(Double.class) + || LogsExcludeAttributeProcessor.class.equals(Boolean.class) + || LogsExcludeAttributeProcessor.class.equals(String.class)) { + attemptParsing = typeCoercion; + if (!attemptParsing) { + attemptParsing |= + ((LogsExcludeAttributeProcessor.class.equals(Integer.class) + || LogsExcludeAttributeProcessor.class.equals(Long.class)) + && token == JsonToken.VALUE_NUMBER_INT); + attemptParsing |= + ((LogsExcludeAttributeProcessor.class.equals(Float.class) + || LogsExcludeAttributeProcessor.class.equals(Double.class)) + && (token == JsonToken.VALUE_NUMBER_FLOAT + || token == JsonToken.VALUE_NUMBER_INT)); + attemptParsing |= + (LogsExcludeAttributeProcessor.class.equals(Boolean.class) + && (token == JsonToken.VALUE_FALSE || token == JsonToken.VALUE_TRUE)); + attemptParsing |= + (LogsExcludeAttributeProcessor.class.equals(String.class) + && token == JsonToken.VALUE_STRING); + } + } + if (attemptParsing) { + tmp = tree.traverse(jp.getCodec()).readValueAs(LogsExcludeAttributeProcessor.class); + // TODO: there is no validation against JSON schema constraints + // (min, max, enum, pattern...), this does not perform a strict JSON + // validation, which means the 'match' count may be higher than it should be. + if (!((LogsExcludeAttributeProcessor) tmp).unparsed) { + deserialized = tmp; + match++; + } + log.log(Level.FINER, "Input data matches schema 'LogsExcludeAttributeProcessor'"); + } + } catch (Exception e) { + // deserialization failed, continue + log.log(Level.FINER, "Input data does not match schema 'LogsExcludeAttributeProcessor'", e); + } + LogsProcessor ret = new LogsProcessor(); if (match == 1) { ret.setActualInstance(deserialized); @@ -1093,6 +1138,11 @@ public LogsProcessor(LogsSchemaProcessor o) { setActualInstance(o); } + public LogsProcessor(LogsExcludeAttributeProcessor o) { + super("oneOf", Boolean.FALSE); + setActualInstance(o); + } + static { schemas.put("LogsGrokParser", new GenericType() {}); schemas.put("LogsDateRemapper", new GenericType() {}); @@ -1116,6 +1166,8 @@ public LogsProcessor(LogsSchemaProcessor o) { schemas.put("LogsArrayProcessor", new GenericType() {}); schemas.put("LogsDecoderProcessor", new GenericType() {}); schemas.put("LogsSchemaProcessor", new GenericType() {}); + schemas.put( + "LogsExcludeAttributeProcessor", new GenericType() {}); JSON.registerDescendants(LogsProcessor.class, Collections.unmodifiableMap(schemas)); } @@ -1131,7 +1183,7 @@ public Map getSchemas() { * LogsUserAgentParser, LogsCategoryProcessor, LogsArithmeticProcessor, * LogsStringBuilderProcessor, LogsPipelineProcessor, LogsGeoIPParser, LogsLookupProcessor, * ReferenceTableLogsLookupProcessor, LogsTraceRemapper, LogsSpanRemapper, LogsArrayProcessor, - * LogsDecoderProcessor, LogsSchemaProcessor + * LogsDecoderProcessor, LogsSchemaProcessor, LogsExcludeAttributeProcessor * *

It could be an instance of the 'oneOf' schemas. The oneOf child schemas may themselves be a * composed schema (allOf, anyOf, oneOf). @@ -1219,6 +1271,10 @@ public void setActualInstance(Object instance) { super.setActualInstance(instance); return; } + if (JSON.isInstanceOf(LogsExcludeAttributeProcessor.class, instance, new HashSet>())) { + super.setActualInstance(instance); + return; + } if (JSON.isInstanceOf(UnparsedObject.class, instance, new HashSet>())) { super.setActualInstance(instance); @@ -1230,7 +1286,8 @@ public void setActualInstance(Object instance) { + " LogsUserAgentParser, LogsCategoryProcessor, LogsArithmeticProcessor," + " LogsStringBuilderProcessor, LogsPipelineProcessor, LogsGeoIPParser," + " LogsLookupProcessor, ReferenceTableLogsLookupProcessor, LogsTraceRemapper," - + " LogsSpanRemapper, LogsArrayProcessor, LogsDecoderProcessor, LogsSchemaProcessor"); + + " LogsSpanRemapper, LogsArrayProcessor, LogsDecoderProcessor, LogsSchemaProcessor," + + " LogsExcludeAttributeProcessor"); } /** @@ -1239,14 +1296,14 @@ public void setActualInstance(Object instance) { * LogsURLParser, LogsUserAgentParser, LogsCategoryProcessor, LogsArithmeticProcessor, * LogsStringBuilderProcessor, LogsPipelineProcessor, LogsGeoIPParser, LogsLookupProcessor, * ReferenceTableLogsLookupProcessor, LogsTraceRemapper, LogsSpanRemapper, LogsArrayProcessor, - * LogsDecoderProcessor, LogsSchemaProcessor + * LogsDecoderProcessor, LogsSchemaProcessor, LogsExcludeAttributeProcessor * * @return The actual instance (LogsGrokParser, LogsDateRemapper, LogsStatusRemapper, * LogsServiceRemapper, LogsMessageRemapper, LogsAttributeRemapper, LogsURLParser, * LogsUserAgentParser, LogsCategoryProcessor, LogsArithmeticProcessor, * LogsStringBuilderProcessor, LogsPipelineProcessor, LogsGeoIPParser, LogsLookupProcessor, * ReferenceTableLogsLookupProcessor, LogsTraceRemapper, LogsSpanRemapper, LogsArrayProcessor, - * LogsDecoderProcessor, LogsSchemaProcessor) + * LogsDecoderProcessor, LogsSchemaProcessor, LogsExcludeAttributeProcessor) */ @Override public Object getActualInstance() { @@ -1473,4 +1530,16 @@ public LogsDecoderProcessor getLogsDecoderProcessor() throws ClassCastException public LogsSchemaProcessor getLogsSchemaProcessor() throws ClassCastException { return (LogsSchemaProcessor) super.getActualInstance(); } + + /** + * Get the actual instance of `LogsExcludeAttributeProcessor`. If the actual instance is not + * `LogsExcludeAttributeProcessor`, the ClassCastException will be thrown. + * + * @return The actual instance of `LogsExcludeAttributeProcessor` + * @throws ClassCastException if the instance is not `LogsExcludeAttributeProcessor` + */ + public LogsExcludeAttributeProcessor getLogsExcludeAttributeProcessor() + throws ClassCastException { + return (LogsExcludeAttributeProcessor) super.getActualInstance(); + } } diff --git a/src/test/resources/com/datadog/api/client/v1/api/logs_pipelines.feature b/src/test/resources/com/datadog/api/client/v1/api/logs_pipelines.feature index 00b7369e231..1ab4401b0fd 100644 --- a/src/test/resources/com/datadog/api/client/v1/api/logs_pipelines.feature +++ b/src/test/resources/com/datadog/api/client/v1/api/logs_pipelines.feature @@ -21,138 +21,138 @@ Feature: Logs Pipelines And a valid "appKeyAuth" key in the system And an instance of "LogsPipelines" API - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Create a pipeline returns "Bad Request" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "", "processors": [{"grok": {"match_rules": "rule_name_1 foo\nrule_name_2 bar", "support_rules": "rule_name_1 foo\nrule_name_2 bar"}, "is_enabled": false, "samples": [], "source": "message", "type": "grok-parser"}], "tags": []} When the request is sent Then the response status is 400 Bad Request - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Create a pipeline returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "", "processors": [{"grok": {"match_rules": "rule_name_1 foo\nrule_name_2 bar", "support_rules": "rule_name_1 foo\nrule_name_2 bar"}, "is_enabled": false, "samples": [], "source": "message", "type": "grok-parser"}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Array Processor Append Operation returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppend", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_to_array", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps"}}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Array Processor Append Operation with preserve_source false returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppendNoPreserve", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_and_remove_source", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps", "preserve_source": false}}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Array Processor Append Operation with preserve_source true returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppendPreserve", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_and_keep_source", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps", "preserve_source": true}}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Array Processor Length Operation returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayLength", "processors": [{"type": "array-processor", "is_enabled": true, "name": "count_tags", "operation": {"type": "length", "source": "tags", "target": "tagCount"}}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Array Processor Select Operation returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArraySelect", "processors": [{"type": "array-processor", "is_enabled": true, "name": "extract_referrer", "operation": {"type": "select", "source": "httpRequest.headers", "target": "referrer", "filter": "name:Referrer", "value_to_extract": "value"}}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Decoder Processor returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testDecoderProcessor", "processors": [{"type": "decoder-processor", "is_enabled": true, "name": "test_decoder", "source": "encoded.field", "target": "decoded.field", "binary_to_text_encoding": "base16", "input_representation": "utf_8"}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Schema Processor and preserve_source false returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid", "preserve_source": false}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name", "preserve_source": false}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api", "preserve_source": false}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user", "preserve_source": false}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor", "preserve_source": false}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud", "preserve_source": false}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request", "preserve_source": false}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata", "preserve_source": false}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time", "preserve_source": false}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint", "preserve_source": false}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity", "preserve_source": false}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id", "preserve_source": false}]}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Schema Processor and preserve_source true returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid", "preserve_source": true}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name", "preserve_source": true}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api", "preserve_source": true}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user", "preserve_source": true}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor", "preserve_source": true}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud", "preserve_source": true}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request", "preserve_source": true}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata", "preserve_source": true}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time", "preserve_source": true}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint", "preserve_source": true}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity", "preserve_source": true}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id", "preserve_source": true}]}], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with Span Id Remapper returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipeline", "processors": [{"type": "span-id-remapper", "is_enabled" : true, "name" : "test_filter", "sources" : [ "dd.span_id"] }], "tags": []} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with nested pipeline processor returns "OK" response Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testPipelineWithNested", "processors": [{"type": "pipeline", "is_enabled": true, "name": "nested_pipeline_with_metadata", "filter": {"query": "env:production"}, "tags": ["env:prod", "type:nested"], "description": "This is a nested pipeline for production logs"}], "tags": ["team:test"], "description": "Pipeline containing nested processor with tags and description"} When the request is sent Then the response status is 200 OK - @team:DataDog/event-platform-experience + @team:DataDog/logs-onboarding Scenario: Create a pipeline with schema processor Given new "CreateLogsPipeline" request And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid"}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name"}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api"}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user"}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor"}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud"}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request"}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata"}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time"}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint"}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity"}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id"}]}], "tags": []} When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Delete a pipeline returns "Bad Request" response Given new "DeleteLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 400 Bad Request - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Delete a pipeline returns "OK" response Given new "DeleteLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Get a pipeline returns "Bad Request" response Given new "GetLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 400 Bad Request - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Get a pipeline returns "OK" response Given new "GetLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Get all pipelines returns "OK" response Given new "ListLogsPipelines" request When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Get pipeline order returns "OK" response Given new "GetLogsPipelineOrder" request When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Update a pipeline returns "Bad Request" response Given new "UpdateLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" @@ -160,7 +160,7 @@ Feature: Logs Pipelines When the request is sent Then the response status is 400 Bad Request - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Update a pipeline returns "OK" response Given new "UpdateLogsPipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" @@ -168,21 +168,21 @@ Feature: Logs Pipelines When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Update pipeline order returns "Bad Request" response Given new "UpdateLogsPipelineOrder" request And body with value {"pipeline_ids": ["tags", "org_ids", "products"]} When the request is sent Then the response status is 400 Bad Request - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Update pipeline order returns "OK" response Given new "UpdateLogsPipelineOrder" request And body with value {"pipeline_ids": ["tags", "org_ids", "products"]} When the request is sent Then the response status is 200 OK - @generated @skip @team:DataDog/event-platform-experience + @generated @skip @team:DataDog/logs-onboarding Scenario: Update pipeline order returns "Unprocessable Entity" response Given new "UpdateLogsPipelineOrder" request And body with value {"pipeline_ids": ["tags", "org_ids", "products"]}