Audit and fix all filter plugin docs for consistency, accuracy, and completeness.

pipeline/filters/aws-metadata.md

@@ -8,19 +8,20 @@ The plugin supports the following configuration parameters:
 
 | Key | Description | Default |
 | :--- | :--- | :--- |
-| `imds_version` | Specify which version of the instance metadata service to use. Valid values are `v1` and `v2`. | `v2` |
+| `account_id` | The account ID for the current EC2 instance. | `false` |
+| `ami_id` | The EC2 instance image ID. | `false` |
 | `az` | The [availability zone](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html), such as `us-east-1a`. | `true` |
 | `ec2_instance_id` | The EC2 instance ID. | `true` |
 | `ec2_instance_type` | The EC2 instance type. | `false` |
-| `private_ip` | The EC2 instance private IP. | `false` |
-| `ami_id` | The EC2 instance image ID. | `false` |
-| `account_id` | The account ID for the current EC2 instance. | `false` |
+| `enable_entity` | Enables entity prefix for fields used for constructing entity. | `false` |
 | `hostname` | The hostname for the current EC2 instance. | `false` |
-| `vpc_id` | The VPC ID for the current EC2 instance. | `false` |
+| `imds_version` | Specify which version of the instance metadata service to use. Valid values are `v1` and `v2`. | `v2` |
+| `private_ip` | The EC2 instance private IP. | `false` |
+| `retry_interval_s` | Defines minimum duration in seconds between retries for fetching EC2 instance tags. | `300` |
 | `tags_enabled` | Specifies whether to attach EC2 instance tags. The EC2 instance must have the [`instance-metadata-tags`](https://docs.aws.amazon.com/cli/latest/reference/ec2/modify-instance-metadata-options.html) option enabled, which is disabled by default. | `false` |
-| `tags_include` | Defines a list of specific EC2 tag keys to inject into the logs. Tag keys must be comma-separated (`,`). Tags not included in this list will be ignored. Example: `Name,tag1,tag2`. | _none_ |
 | `tags_exclude` | Defines a list of specific EC2 tag keys not to inject into the logs. Tag keys must be comma-separated (`,`). Tags not included in this list will be injected into the logs. If both `tags_include` and `tags_exclude` are specified, the configuration is invalid and the plugin fails. Example: `Name,tag1,tag2` | _none_ |
-| `retry_interval_s` | Defines minimum duration between retries for fetching EC2 instance tags. | `300` |
+| `tags_include` | Defines a list of specific EC2 tag keys to inject into the logs. Tag keys must be comma-separated (`,`). Tags not included in this list will be ignored. Example: `Name,tag1,tag2`. | _none_ |
+| `vpc_id` | The VPC ID for the current EC2 instance. | `false` |
 
 If you run Fluent Bit in a container, you might need to use instance metadata v1. The plugin behaves the same regardless of which version is used.
 
@@ -49,54 +50,54 @@ The following is an example of a configuration file:
 
 ```yaml
 pipeline:
-    inputs:
-        - name: dummy
-          tag: dummy
+  inputs:
+    - name: dummy
+      tag: dummy
 
-    filters:
-        - name: aws
-          match: '*'
-          imds_version: v1
-          az: true
-          ec2_instance_id: true
-          ec2_instance_type: true
-          private_ip: true
-          ami_id: true
-          account_id: true
-          hostname: true
-          vpc_id: true
-          tags_enabled: true
+  filters:
+    - name: aws
+      match: '*'
+      imds_version: v1
+      az: true
+      ec2_instance_id: true
+      ec2_instance_type: true
+      private_ip: true
+      ami_id: true
+      account_id: true
+      hostname: true
+      vpc_id: true
+      tags_enabled: true
 
-    outputs:
-        - name: stdout
-          match: '*'
+  outputs:
+    - name: stdout
+      match: '*'
 ```
 
 {% endtab %}
 {% tab title="fluent-bit.conf" %}
 
 ```text
 [INPUT]
-    Name dummy
-    Tag dummy
+  Name dummy
+  Tag  dummy
 
 [FILTER]
-    Name aws
-    Match *
-    imds_version v1
-    az true
-    ec2_instance_id true
-    ec2_instance_type true
-    private_ip true
-    ami_id true
-    account_id true
-    hostname true
-    vpc_id true
-    tags_enabled true
+  Name              aws
+  Match             *
+  Imds_Version      v1
+  Az                true
+  Ec2_Instance_Id   true
+  Ec2_Instance_Type true
+  Private_Ip        true
+  Ami_Id            true
+  Account_Id        true
+  Hostname          true
+  Vpc_Id            true
+  Tags_Enabled      true
 
 [OUTPUT]
-    Name stdout
-    Match *
+  Name  stdout
+  Match *
 ```
 
 {% endtab %}
@@ -136,10 +137,10 @@ pipeline:
 
 ```text
 [FILTER]
-    Name aws
-    Match *
-    tags_enabled true
-    tags_include department,project
+    Name         aws
+    Match        *
+    Tags_Enabled true
+    Tags_Include department,project
 ```
 
 {% endtab %}
@@ -175,10 +176,10 @@ pipeline:
 
 ```text
 [FILTER]
-  Name aws
-  Match *
-  tags_enabled true
-  tags_exclude department
+  Name         aws
+  Match        *
+  Tags_Enabled true
+  Tags_Exclude department
 ```
 
 {% endtab %}

pipeline/filters/checklist.md

@@ -9,11 +9,11 @@ The plugin supports the following configuration parameters
 | Key | Description | Default |
 | :-- | :---------- | :------ |
 | `file` | The single value file that Fluent Bit will use as a lookup table to determine if the specified `lookup_key` exists. | _none_ |
-| `lookup_key` | The specific key to look up and determine if it exists. Supports [record accessor](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md). | _none_ |
-| `record` | The record to add if the `lookup_key` is found in the specified `file`. You can add multiple record parameters. | _none_ |
-| `mode` | Set the check mode. `exact` and `partial` are supported. | `exact`|
-| `print_query_time` | Print to stdout the elapsed query time for every matched record. | `false` |
 | `ignore_case` | Compare strings by ignoring case. | `false` |
+| `lookup_key` | The specific key to look up and determine if it exists. Supports [record accessor](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md). | `log` |
+| `mode` | Set the check mode. `exact` and `partial` are supported. | `exact` |
+| `print_query_time` | Print to stdout the elapsed query time for every matched record. | `false` |
+| `record` | The record to add if the `lookup_key` is found in the specified `file`. You can add multiple record parameters. | _none_ |
 
 ## Example configuration
 
@@ -37,7 +37,6 @@ pipeline:
       record:
         - ioc abc
         - badurl null
-      log_level: debug
 
   outputs:
     - name: stdout
@@ -62,7 +61,6 @@ pipeline:
   lookup_key $remote_addr
   record     ioc    abc
   record     badurl null
-  log_level  debug
 
 [OUTPUT]
   name       stdout

pipeline/filters/ecs-metadata.md

@@ -10,16 +10,19 @@ The plugin supports the following configuration parameters:
 
 | Key | Description | Default |
 | :--- | :--- | :--- |
-| `Add` | Similar to the `ADD` option in the [modify filter](../filters/modify.md). You can specify it multiple times. It takes two arguments: a `KEY` name and `VALUE`. The value uses Fluent Bit [`record_accessor`](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md) syntax to create a template that uses ECS Metadata values. See the list of supported metadata templating keys. This option lets you control both the key names for metadata and the format for metadata values. | _none_ |
-| `ECS_Tag_Prefix` | Similar to the `Kube_Tag_Prefix` option in the [Kubernetes filter](../filters/kubernetes.md) and performs the same function. The full log tag should be prefixed with this string and after the prefix the filter must find the next characters in the tag to be the Docker Container Short ID (the first 12 characters of the full container ID). The filter uses this to identify which container the log came from so it can find which task it's a part of. See the design section for more information. If not specified, it defaults to empty string, meaning that the tag must be prefixed with the 12 character container short ID. If you want to attach cluster metadata to system or OS logs from processes that don't run as part of containers or ECS Tasks, don't set this parameter and enable the `Cluster_Metadata_Only` option | empty string |
-| `Cluster_Metadata_Only` | When enabled, the plugin will only attempt to attach cluster metadata values. Use to attach cluster metadata to system or OS logs from processes that don't run as part of containers or ECS Tasks. | `Off` |
-| `ECS_Meta_Cache_TTL` | The filter builds a hash table in memory mapping each unique container short ID to its metadata. This option sets a max `TTL` for objects in the hash table. You should set this if you have frequent container or task restarts. For example, if your cluster runs short running batch jobs that complete in less than 10 minutes, there is no reason to keep any stored metadata longer than 10 minutes. You would therefore set this parameter to `10m`. | `1h` |
+| `add` | Similar to the `add` option in the [modify filter](../filters/modify.md). You can specify it multiple times. It takes two arguments: a `KEY` name and `VALUE`. The value uses Fluent Bit [`record_accessor`](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md) syntax to create a template that uses ECS Metadata values. See the list of supported metadata templating keys. This option lets you control both the key names for metadata and the format for metadata values. | _none_ |
+| `agent_endpoint_retries` | Number of retries for failed metadata requests to the ECS Agent Introspection endpoint. The most common cause of failed metadata requests is that the container the metadata request was made for isn't part of an ECS Task. | `2` |
+| `cluster_metadata_only` | When enabled, the plugin only attempts to attach cluster metadata values. Use to attach cluster metadata to system or OS logs from processes that don't run as part of containers or ECS Tasks. | `false` |
+| `ecs_meta_cache_ttl` | The filter builds a hash table in memory mapping each unique container short ID to its metadata. This option sets a max `TTL` for objects in the hash table. Set this if you have frequent container or task restarts. For example, if your cluster runs short-running batch jobs that complete in less than 10 minutes, set this parameter to `10m`. | `1h` |
+| `ecs_meta_host` | The host name at which the ECS Agent Introspection endpoint is reachable. | `127.0.0.1` |
+| `ecs_meta_port` | The port at which the ECS Agent Introspection endpoint is reachable. | `51678` |
+| `ecs_tag_prefix` | Similar to the `kube_tag_prefix` option in the [Kubernetes filter](../filters/kubernetes.md) and performs the same function. The full log tag should be prefixed with this string and after the prefix the filter must find the next characters in the tag to be the Docker Container Short ID (the first 12 characters of the full container ID). The filter uses this to identify which container the log came from so it can find which task it's a part of. If not specified, defaults to empty string, meaning that the tag must be prefixed with the 12-character container short ID. If you want to attach cluster metadata to system or OS logs from processes that don't run as part of containers or ECS Tasks, don't set this parameter and enable `cluster_metadata_only`. | `""` |
 
-### Supported templating variables for the `ADD` option
+### Supported templating variables for the `add` option
 
-The following template variables can be used for values with the `ADD` option. See the tutorial in the sections following for examples.
+The following template variables can be used for values with the `add` option. See the tutorial in the sections following for examples.
 
-| Variable | Description | Supported with `Cluster_Metadata_Only` on |
+| Variable | Description | Supported with `cluster_metadata_only` on |
 | :--- | :--- | :--- |
 | `$ClusterName` | The ECS cluster name. Fluent Bit is running on EC2 instances that are part of this cluster. | `Yes` |
 | `$ContainerInstanceArn` | The full ARN of the ECS EC2 Container Instance. This is the instance that Fluent Bit is running on. | `Yes` |
@@ -35,8 +38,7 @@ The following template variables can be used for values with the `ADD` option. S
 
 ### Configuration file
 
-The following configurations assume a properly configured parsers file and 'storage.path' variable defined in the services
-section of the Fluent Bit configuration (not shown).
+The following configurations assume a properly configured parsers file and `storage.path` variable defined in the services section of the Fluent Bit configuration (not shown).
 
 #### Example 1: Attach Task ID and cluster name to container logs
 
@@ -98,9 +100,9 @@ pipeline:
 [FILTER]
   Name ecs
   Match *
-  ECS_Tag_Prefix ecs.var.lib.docker.containers.
-  ADD ecs_task_id $TaskID
-  ADD cluster $ClusterName
+  ecs_tag_prefix ecs.var.lib.docker.containers.
+  add ecs_task_id $TaskID
+  add cluster $ClusterName
 
 [OUTPUT]
   Name stdout
@@ -117,7 +119,7 @@ The output log should be similar to:
 {
   "date":1665003546.0,
   "log":"some message from your container",
-  "ecs_task_id" "1234567890abcdefghijklmnop",
+  "ecs_task_id": "1234567890abcdefghijklmnop",
   "cluster": "your_cluster_name",
 }
 ```
@@ -180,8 +182,8 @@ pipeline:
 [FILTER]
   Name ecs
   Match *
-  ECS_Tag_Prefix ecs.var.lib.docker.containers.
-  ADD resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName
+  ecs_tag_prefix ecs.var.lib.docker.containers.
+  add resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName
 
 [OUTPUT]
   Name stdout
@@ -198,16 +200,15 @@ The output log would be similar to:
 {
   "date":1665003546.0,
   "log":"some message from your container",
-  "resource" "cluster.family.1234567890abcdefghijklmnop.app",
+  "resource": "cluster.family.1234567890abcdefghijklmnop.app",
 }
 ```
 
-The template variables in the value for the `resource` key are separated by dot characters. Only dots and commas
- (`.` and `,`) can come after a template variable. For more information, see the [Record accessor limitation's section](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md#limitations-of-record_accessor-templating).
+The template variables in the value for the `resource` key are separated by dot characters. Only dots and commas (`.` and `,`) can come after a template variable. For more information, see the [Record accessor limitation's section](../../administration/configuring-fluent-bit/classic-mode/record-accessor.md#limitations-of-record_accessor-templating).
 
 #### Example 3: Attach cluster metadata to non-container logs
 
-This example shows a use case for the `Cluster_Metadata_Only` option attaching cluster metadata to ECS Agent logs.
+This example shows a use case for the `cluster_metadata_only` option attaching cluster metadata to ECS Agent logs.
 
 {% tabs %}
 {% tab title="fluent-bit.yaml" %}
@@ -259,8 +260,8 @@ pipeline:
 [FILTER]
   Name ecs
   Match *
-  Cluster_Metadata_Only On
-  ADD cluster $ClusterName
+  cluster_metadata_only on
+  add cluster $ClusterName
 
 [OUTPUT]
   Name stdout

pipeline/filters/expect.md

@@ -12,12 +12,12 @@ For a detailed explanation of its usage and use cases, see [Validating and your
 
 The plugin supports the following configuration parameters:
 
-| Property | Description |
-| :--- | :--- |
-| `key_exists` | Check if a key with a given name exists in the record. |
-| `key_not_exists` | Check if a key doesn't exist in the record. |
-| `key_val_is_null` | Check that the value of the key is `NULL`. |
-| `key_val_is_not_null` | Check that the value of the key is `NOT NULL`. |
-| `key_val_eq` | Check that the value of the key equals the given value in the configuration. |
-| `action` | Action to take when a rule doesn't match. Available options are `warn` , `exit` or `result_key`. On `warn`, a warning message is sent to the logging layer when a mismatch of the `key*` rules is found. Using `exit` makes Fluent Bit exit with status code `255`. `result_key` adds a matching result to each record. |
-| `result_key` | Specify a key name for the matching result added when `action` is set to `result_key`. |
+| Key | Description | Default |
+| :--- | :--- | :--- |
+| `action` | Action to take when a rule doesn't match. Available options are `warn`, `exit` or `result_key`. On `warn`, a warning message is sent to the logging layer when a mismatch of the `key*` rules is found. Using `exit` makes Fluent Bit exit with status code `255`. `result_key` adds a matching result to each record. | `warn` |
+| `key_exists` | Check if a key with a given name exists in the record. | _none_ |
+| `key_not_exists` | Check if a key doesn't exist in the record. | _none_ |
+| `key_val_eq` | Check that the value of the key equals the given value in the configuration. | _none_ |
+| `key_val_is_not_null` | Check that the value of the key is `NOT NULL`. | _none_ |
+| `key_val_is_null` | Check that the value of the key is `NULL`. | _none_ |
+| `result_key` | Specify a key name for the matching result added when `action` is set to `result_key`. | `matched` |

pipeline/filters/geoip2-filter.md

@@ -15,7 +15,7 @@ This plugin supports the following configuration parameters:
 | Key | Description |
 | :--- | :--- |
 | `database` | Path to the GeoIP2 database. |
-| `lookup_key` | Field name to process. |
+| `lookup_key` | Field name to process. Multiple `lookup_key` entries are supported. |
 | `record` | Defines the `KEY LOOKUP_KEY VALUE` triplet. |
 
 ## Get started