diff --git a/CHANGELOG.md b/CHANGELOG.md index 897a1a8bf49..14f8c485ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ * [ENHANCEMENT] Ring: Add ring metric to count number of duplicate tokens. #7626 * [ENHANCEMENT] Metrics: Add native histogram support to all remaining production histograms, enabling dual-format (classic + native) exposition across all Cortex components. #7636 * [ENHANCEMENT] Ring: Cache `ShuffleShardWithLookback` subrings. The cached entry is invalidated on topology change or once `now` reaches the earliest `RegisteredTimestamp + lookbackPeriod` of any included instance. #7628 +* [ENHANCEMENT] Update prometheus alertmanager version to v0.33.0. #7647 * [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370 * [BUGFIX] Metrics Helper: Fix non-deterministic bucket order in merged histograms by sorting buckets after map iteration, matching Prometheus client library behavior. #7380 * [BUGFIX] Distributor: Return HTTP 401 Unauthorized when tenant ID resolution fails in the Prometheus Remote Write 2.0 path. #7389 diff --git a/go.mod b/go.mod index f4aeefd800b..e67ee0b1f24 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/felixge/fgprof v0.9.5 github.com/go-kit/log v0.2.1 github.com/go-openapi/strfmt v0.26.3 - github.com/go-openapi/swag v0.25.5 // indirect + github.com/go-openapi/swag v0.26.0 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/gogo/protobuf v1.3.2 github.com/gogo/status v1.1.1 @@ -39,7 +39,7 @@ require ( github.com/opentracing-contrib/go-stdlib v1.1.1 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/alertmanager v0.32.1 + github.com/prometheus/alertmanager v0.33.0 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.69.0 @@ -129,7 +129,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sns v1.39.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sns v1.39.17 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect @@ -140,9 +140,9 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect - github.com/coder/quartz v0.3.0 // indirect + github.com/coder/quartz v0.3.1 // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/cristalhq/hedgedhttp v0.9.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dennwc/varint v1.0.0 // indirect @@ -154,7 +154,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fsnotify/fsnotify v1.10.0 // indirect github.com/go-chi/chi/v5 v5.2.4 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logfmt/logfmt v0.6.1 // indirect @@ -165,18 +165,18 @@ require ( github.com/go-openapi/jsonpointer v0.22.5 // indirect github.com/go-openapi/jsonreference v0.21.5 // indirect github.com/go-openapi/loads v0.23.3 // indirect - github.com/go-openapi/runtime v0.29.3 // indirect + github.com/go-openapi/runtime v0.29.4 // indirect github.com/go-openapi/spec v0.22.4 // indirect - github.com/go-openapi/swag/cmdutils v0.25.5 // indirect + github.com/go-openapi/swag/cmdutils v0.26.0 // indirect github.com/go-openapi/swag/conv v0.26.1 // indirect - github.com/go-openapi/swag/fileutils v0.25.5 // indirect - github.com/go-openapi/swag/jsonname v0.25.5 // indirect - github.com/go-openapi/swag/loading v0.25.5 // indirect - github.com/go-openapi/swag/mangling v0.25.5 // indirect - github.com/go-openapi/swag/netutils v0.25.5 // indirect - github.com/go-openapi/swag/stringutils v0.25.5 // indirect + github.com/go-openapi/swag/fileutils v0.26.0 // indirect + github.com/go-openapi/swag/jsonname v0.26.0 // indirect + github.com/go-openapi/swag/loading v0.26.0 // indirect + github.com/go-openapi/swag/mangling v0.26.0 // indirect + github.com/go-openapi/swag/netutils v0.26.0 // indirect + github.com/go-openapi/swag/stringutils v0.26.0 // indirect github.com/go-openapi/swag/typeutils v0.26.1 // indirect - github.com/go-openapi/swag/yamlutils v0.25.5 // indirect + github.com/go-openapi/swag/yamlutils v0.26.0 // indirect github.com/go-openapi/validate v0.25.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gobwas/glob v0.2.3 // indirect @@ -234,12 +234,12 @@ require ( github.com/parquet-go/bitpack v1.0.0 // indirect github.com/parquet-go/jsonlite v1.0.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-community/prom-label-proxy v0.11.1 // indirect - github.com/prometheus/exporter-toolkit v0.15.1 // indirect + github.com/prometheus/exporter-toolkit v0.16.0 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/sigv4 v0.4.1 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect @@ -254,6 +254,9 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tinylib/msgp v1.6.1 // indirect + github.com/twmb/franz-go v1.21.2 // indirect + github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect + github.com/twmb/franz-go/plugin/kslog v1.0.0 // indirect github.com/twpayne/go-geom v1.6.1 // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vimeo/galaxycache v1.3.1 // indirect @@ -273,13 +276,13 @@ require ( go.opentelemetry.io/collector/semconv v0.128.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.66.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect diff --git a/go.sum b/go.sum index 50d53bbd501..f5077743dcc 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/aws/aws-sdk-go-v2/service/lightsail v1.50.4 h1:/1o2AYwHJojUDeMvQNyJiK github.com/aws/aws-sdk-go-v2/service/lightsail v1.50.4/go.mod h1:Nn2xx6HojGuNMtUFxxz/nyNLSS+tHMRsMhe3+W3wB5k= github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.15 h1:rOWMUrXJPcTXnk75ja6Bxv1P+j83dPhIWjfJ2cujj34= -github.com/aws/aws-sdk-go-v2/service/sns v1.39.15/go.mod h1:4exx1wZR0pe+WcMbas8OZ2krRrBbW7IUUvLXCCQbjkg= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.17 h1:synXIPC/L4Cc489P0XDcrVJzHSLj7krKRpFLalbGM2k= +github.com/aws/aws-sdk-go-v2/service/sns v1.39.17/go.mod h1:4ABZnI23uNK37waIjGwkubnCwGhepIt9x1GvASfljJA= github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= @@ -249,8 +249,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/coder/quartz v0.3.0 h1:bUoSEJ77NBfKtUqv6CPSC0AS8dsjqAqqAv7bN02m1mg= -github.com/coder/quartz v0.3.0/go.mod h1:BgE7DOj/8NfvRgvKw0jPLDQH/2Lya2kxcTaNJ8X0rZk= +github.com/coder/quartz v0.3.1 h1:JMJLj4Xj4NLSrUC1R/g/Hn0y9fkyOvb8tf6P0j+kPn0= +github.com/coder/quartz v0.3.1/go.mod h1:BgE7DOj/8NfvRgvKw0jPLDQH/2Lya2kxcTaNJ8X0rZk= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -258,8 +258,8 @@ github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmC github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= -github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cortexproject/promqlsmith v0.0.0-20260205231645-0c8ef5fe46a5 h1:MTGag0oh+M3hyWWg7a1Bju2VPeFLjG2l+LWLhUCFTYI= github.com/cortexproject/promqlsmith v0.0.0-20260205231645-0c8ef5fe46a5/go.mod h1:GrzWqtj9aJG+n4Cl1O30Zdk8cIgjqSUc2l5nZV8BnaA= github.com/cortexproject/weaveworks-common v0.0.0-20260508185357-75366cf4edfb h1:3Kr+IwRd0ybfFhUYpzdkmm6jIyRWonIWMsToRcycrL4= @@ -339,8 +339,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.10.0 h1:Xx/5Ydg9CeBDX/wi4VJqStNtohYjitZhhlHt4h3St1M= +github.com/fsnotify/fsnotify v1.10.0/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 h1:TufioMBjkJ6/Oqmlye/ReuxHFS35HyLmypj/BNy/8GY= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474/go.mod h1:PQwxF4UU8wuL+srGxr3BOhIW5zXqgucwVlO/nPZLsxw= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= @@ -381,40 +381,40 @@ github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= github.com/go-openapi/loads v0.23.3 h1:g5Xap1JfwKkUnZdn+S0L3SzBDpcTIYzZ5Qaag0YDkKQ= github.com/go-openapi/loads v0.23.3/go.mod h1:NOH07zLajXo8y55hom0omlHWDVVvCwBM/S+csCK8LqA= -github.com/go-openapi/runtime v0.29.3 h1:h5twGaEqxtQg40ePiYm9vFFH1q06Czd7Ot6ufdK0w/Y= -github.com/go-openapi/runtime v0.29.3/go.mod h1:8A1W0/L5eyNJvKciqZtvIVQvYO66NlB7INMSZ9bw/oI= +github.com/go-openapi/runtime v0.29.4 h1:k2lDxrGoSAJRdhFG2tONKMpkizY/4X1cciSdtzk4Jjo= +github.com/go-openapi/runtime v0.29.4/go.mod h1:K0k/2raY6oqXJnZAgWJB2i/12QKrhUKpZcH4PfV9P18= github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ= github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/strfmt v0.26.3 h1:rzmslHarJgBbf2qfGge+X3htclQfmXqBZMm0Too0HhU= github.com/go-openapi/strfmt v0.26.3/go.mod h1:a5nsUw0oRpQzZeOwx8bi6cKbzFZslpbCKt1LEot+KnQ= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU= -github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA= -github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c= -github.com/go-openapi/swag/cmdutils v0.25.5/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag v0.26.0 h1:GVDXCmfvhfu1BxiHo8/FA+BbKmhecHnG3varjON5/RI= +github.com/go-openapi/swag v0.26.0/go.mod h1:82g3193sZJRbocs7bNCqGfIgq8pkuwVwCfhKIRlEQF0= +github.com/go-openapi/swag/cmdutils v0.26.0 h1:iowihOcvq7y4egO8cOq0dmfohz6wfeQ63U1EnuhO2TU= +github.com/go-openapi/swag/cmdutils v0.26.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= github.com/go-openapi/swag/conv v0.26.1 h1:slr5FVkg9Wc3Y5zcwenD8Sd/PQ94b2I/QJI7N7KTBpg= github.com/go-openapi/swag/conv v0.26.1/go.mod h1:mvQXgPptZk9GTrFgGwWvT4q+dN+zQej9JfmGwnipz1A= -github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk= -github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc= -github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= -github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/fileutils v0.26.0 h1:WJoPRvsA7QRiiWluowkLJa9jaYR7FCuxmDvnCgaRRxU= +github.com/go-openapi/swag/fileutils v0.26.0/go.mod h1:0WDJ7lp67eNjPMO50wAWYlKvhOb6CQ37rzR7wrgI8Tc= +github.com/go-openapi/swag/jsonname v0.26.0 h1:gV1NFX9M8avo0YSpmWogqfQISigCmpaiNci8cGECU5w= +github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTeIz0ARZxHkAPktJK2M= github.com/go-openapi/swag/jsonutils v0.26.1 h1:2hdBfFkHg+7Wrz2VsCbeyR6hzkRDs7AztnMR2u84yOY= github.com/go-openapi/swag/jsonutils v0.26.1/go.mod h1:U+RMJH3wa+6BRiphuRtIyI8fW9HPFqFQ4sHk2oRx0UQ= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.1 h1:1CD7NiLLb/TXl3tOnFYU4b+mNfb5rtgHkaA+q7RMYYQ= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.1/go.mod h1:ZWafc8nMdYzTE3uYY6W86f0n46+IF0g4uUyRhJw/kXc= -github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU= -github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g= -github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw= -github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY= -github.com/go-openapi/swag/netutils v0.25.5 h1:LZq2Xc2QI8+7838elRAaPCeqJnHODfSyOa7ZGfxDKlU= -github.com/go-openapi/swag/netutils v0.25.5/go.mod h1:lHbtmj4m57APG/8H7ZcMMSWzNqIQcu0RFiXrPUara14= -github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M= -github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII= +github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= +github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= +github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= +github.com/go-openapi/swag/mangling v0.26.0/go.mod h1:jifS7W9vbg+pw63bT+GI53otluMQL3CeemuyCHKwVx0= +github.com/go-openapi/swag/netutils v0.26.0 h1:CmZp+ZT7HrmFwrC3GdGsXBq2+42T1bjKBapcqVpIs3c= +github.com/go-openapi/swag/netutils v0.26.0/go.mod h1:5iK+Ok3ZohWWex1C50BFTPexi03UaPwjW4Oj8kgrpwo= +github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg= +github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE= github.com/go-openapi/swag/typeutils v0.26.1 h1:yg42FgMzRR6PVQ3M3qHz1s+Y6/P4HoJ3cBarXa3OVnU= github.com/go-openapi/swag/typeutils v0.26.1/go.mod h1:VfnV+oUtSP2vCSCn2aJgnr8OevUYemyIzzS1VOzS10o= -github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ= -github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ= +github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= +github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= github.com/go-openapi/testify/enable/yaml/v2 v2.5.1 h1:q9NtHwK4qHF7yZziBPvZyv7zWAIk8ok88Gh2mR6Jpc8= github.com/go-openapi/testify/enable/yaml/v2 v2.5.1/go.mod h1:JW0MXIotCYps/XsgJnG3a8Q7rE5xAiBwoOD5OfaIQBk= github.com/go-openapi/testify/v2 v2.5.1 h1:TMdhCaw8fUNraVSf3Omoob1dO/AzBfhtFAPW0an6sBo= @@ -849,8 +849,8 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -871,8 +871,8 @@ github.com/prometheus-community/parquet-common v0.0.0-20251211092633-65ebeae24e9 github.com/prometheus-community/parquet-common v0.0.0-20251211092633-65ebeae24e94/go.mod h1:gewN7ZuOXJh0X2I57iGHyDLbLvL891P2Ynko2QM5axY= github.com/prometheus-community/prom-label-proxy v0.11.1 h1:jX+m+BQCNM0z3/P6V6jVxbiDKgugvk91SaICD6bVhT4= github.com/prometheus-community/prom-label-proxy v0.11.1/go.mod h1:uTeQW+wZ/VPV1LL3IPfvUE++wR2nPLex+Y4RE38Cpis= -github.com/prometheus/alertmanager v0.32.1 h1:BQ3jHXNq2A7VSD9Kh0Qx+kXbifNbHSDuKVbMmdRHHJ0= -github.com/prometheus/alertmanager v0.32.1/go.mod h1:0Dy9faTtMgpVYxJVxV0o65elTxHnSRCF/7gy5BKGZiE= +github.com/prometheus/alertmanager v0.33.0 h1:AAVa3wpCsaDxisTUUPXx+1qhnA2mx0f8Cc+smpAtN7w= +github.com/prometheus/alertmanager v0.33.0/go.mod h1:V06Uc8EZ5X5wLOJRGhtXx+EE2LgrinFIADbKWMVm1RY= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -895,8 +895,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.69.0 h1:OA85nJQS/T/MaYh/Q2CcgDKSGWqNIgrBDvDH85CuiNk= github.com/prometheus/common v0.69.0/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= -github.com/prometheus/exporter-toolkit v0.15.1 h1:XrGGr/qWl8Gd+pqJqTkNLww9eG8vR/CoRk0FubOKfLE= -github.com/prometheus/exporter-toolkit v0.15.1/go.mod h1:P/NR9qFRGbCFgpklyhix9F6v6fFr/VQB/CVsrMDGKo4= +github.com/prometheus/exporter-toolkit v0.16.0 h1:xT/j7L2XKF+VJd6B4fpUw6xWabHrSmsUf6mYmFqyu0s= +github.com/prometheus/exporter-toolkit v0.16.0/go.mod h1:d1EL8Z9674xQe/iWhwP2wDyCEoBPbXVeqDbqAUsgJWY= github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -997,6 +997,14 @@ github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77ro github.com/tjhop/slog-gokit v0.2.0 h1:tUNkuukDjpswQ2abhsugEobRRxN1aHEW8h4rvwdHMqU= github.com/tjhop/slog-gokit v0.2.0/go.mod h1:yA48zAHvV+Sg4z4VRyeFyFUNNXd3JY5Zg84u3USICq0= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/franz-go v1.21.2 h1:WrvV/spF48JzcRylqDQy02Vm6V6W4lhtD9Y4BOYNMu4= +github.com/twmb/franz-go v1.21.2/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260515175617-8268a5d078c0 h1:YWmvjmcidrKLLgObwU1k7K9KuesdYTV33OTIS0Ltj8o= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260515175617-8268a5d078c0/go.mod h1:9j4VxU2ng6tHgD4lIkNJ5OJ3D6vgPhhIp3tBa7dJgLA= +github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= +github.com/twmb/franz-go/pkg/kmsg v1.13.1/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= +github.com/twmb/franz-go/plugin/kslog v1.0.0 h1:I64oEmF+0PDvmyLgwrlOtg4mfpSE9GwlcLxM4af2t60= +github.com/twmb/franz-go/plugin/kslog v1.0.0/go.mod h1:8pMjK3OJJJNNYddBSbnXZkIK5dCKFIk9GcVVCDgvnQc= github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -1087,10 +1095,10 @@ go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.66.0 h1:U++6AfUpXXSILim4iH6Jb2oeK/mp7J4lNzzyO8Cx4Zw= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.66.0/go.mod h1:HVNUDNMGMeykut/2GZ++AZjglCqew/+Hf4lxRVqFFxQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0 h1:PnV4kVnw0zOmwwFkAzCN5O07fw1YOIQor120zrh0AVo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0/go.mod h1:ofAwF4uinaf8SXdVzzbL4OsxJ3VfeEg3f/F6CeF49/Y= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 h1:cuXaPAfIoJKsYjBjPSb2nKZEmgM43zVr25l37IxhKME= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0/go.mod h1:BuzhPofpCzlDi/Q/Xjg54M4/3oWqqyDe2Zeq7A2I0QE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 h1:cxOVDJ30qfzV27G5p9WMtJUB/3cXC0iL+u9EV1fSOws= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0/go.mod h1:Y+xiUbWetg65vAroDZcIzJ5wyPNWRH32EoIV9rIaa0g= go.opentelemetry.io/contrib/propagators/aws v1.44.0 h1:Rtvfd6nTbAF2csjiw41m1DfuqC5TneXs+gB84ZA3gq4= @@ -1109,8 +1117,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUY go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index b9b776cbfbb..39d79e13ff0 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -21,8 +21,10 @@ import ( "github.com/prometheus/alertmanager/cluster/clusterpb" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/dispatch" + "github.com/prometheus/alertmanager/eventrecorder" "github.com/prometheus/alertmanager/featurecontrol" "github.com/prometheus/alertmanager/inhibit" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/nflog" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/notify/discord" @@ -47,7 +49,6 @@ import ( "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/timeinterval" - "github.com/prometheus/alertmanager/types" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" commoncfg "github.com/prometheus/common/config" @@ -106,8 +107,7 @@ type Alertmanager struct { persister *statePersister nflog *nflog.Log silences *silence.Silences - alertMarker types.AlertMarker - groupMarker types.GroupMarker + groupMarker marker.GroupMarker alerts *mem.Alerts dispatcher *dispatch.Dispatcher inhibitor *inhibit.Inhibitor @@ -223,9 +223,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { am.wg.Go(func() { am.nflog.Maintenance(maintenancePeriod, notificationFile, am.stop, nil) }) - memMarker := types.NewMarker(reg) - am.alertMarker = memMarker - am.groupMarker = memMarker + am.groupMarker = marker.NewGroupMarker() silencesFile := filepath.Join(cfg.TenantDataDir, silencesSnapshot) @@ -264,7 +262,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { return nil, errors.Wrap(err, "error parsing the feature flag list") } - am.pipelineBuilder = notify.NewPipelineBuilder(am.registry, featureConfig) + am.pipelineBuilder = notify.NewPipelineBuilder(am.registry, featureConfig, eventrecorder.NopRecorder()) am.wg.Go(func() { am.silences.Maintenance(maintenancePeriod, silencesFile, am.stop, nil) @@ -274,16 +272,15 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { if am.cfg.Limits != nil { callback = newAlertsLimiter(am.cfg.UserID, am.cfg.Limits, reg) } - am.alerts, err = mem.NewAlerts(context.Background(), am.alertMarker, am.cfg.GCInterval, 0, callback, util_log.GoKitLogToSlog(am.logger), am.registry, nil) + am.alerts, err = mem.NewAlerts(context.Background(), am.cfg.GCInterval, 0, callback, util_log.GoKitLogToSlog(am.logger), eventrecorder.NopRecorder(), am.registry, nil) if err != nil { return nil, fmt.Errorf("failed to create alerts: %v", err) } am.api, err = api.New(api.Options{ - Alerts: am.alerts, - Silences: am.silences, - AlertStatusFunc: am.alertMarker.Status, - GroupMutedFunc: am.groupMarker.Muted, + Alerts: am.alerts, + Silences: am.silences, + GroupMutedFunc: am.groupMarker.Muted, // Cortex should not expose cluster information back to its tenants. Peer: &NilPeer{}, Registry: am.registry, @@ -302,7 +299,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { ui.Register(router) am.mux = am.api.Register(router, am.cfg.ExternalURL.Path) - am.dispatcherMetrics = dispatch.NewDispatcherMetrics(true, am.registry) + am.dispatcherMetrics = dispatch.NewDispatcherMetrics(true, am.registry, nil) //TODO: From this point onward, the alertmanager _might_ receive requests - we need to make sure we've settled and are ready. return am, nil @@ -355,7 +352,7 @@ func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config, rawCfg s am.dispatcher.Stop() } - am.inhibitor = inhibit.NewInhibitor(am.alerts, conf.InhibitRules, am.alertMarker, util_log.GoKitLogToSlog(log.With(am.logger, "component", "inhibitor"))) + am.inhibitor = inhibit.NewInhibitor(am.alerts, conf.InhibitRules, util_log.GoKitLogToSlog(log.With(am.logger, "component", "inhibitor")), eventrecorder.NopRecorder()) waitFunc := clusterWait(am.state.Position, am.cfg.PeerTimeout) @@ -398,7 +395,7 @@ func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config, rawCfg s integrationsMap, waitFunc, am.inhibitor, - silence.NewSilencer(am.silences, am.alertMarker, util_log.GoKitLogToSlog(am.logger)), + silence.NewSilencer(am.silences, util_log.GoKitLogToSlog(am.logger), eventrecorder.NopRecorder()), timeinterval.NewIntervener(timeIntervals), am.groupMarker, am.nflog, @@ -414,6 +411,7 @@ func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config, rawCfg s defaultMaintenanceInterval, // TODO: add to config &dispatcherLimits{tenant: am.cfg.UserID, limits: am.cfg.Limits}, util_log.GoKitLogToSlog(log.With(am.logger, "component", "dispatcher")), + eventrecorder.NopRecorder(), am.dispatcherMetrics, ) diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index 8cb6cd95fd0..ee27b551730 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -15,6 +15,12 @@ import ( "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" amcommoncfg "github.com/prometheus/alertmanager/config/common" + "github.com/prometheus/alertmanager/notify/discord" + "github.com/prometheus/alertmanager/notify/incidentio" + "github.com/prometheus/alertmanager/notify/jira" + "github.com/prometheus/alertmanager/notify/mattermost" + "github.com/prometheus/alertmanager/notify/msteams" + "github.com/prometheus/alertmanager/notify/webhook" "github.com/prometheus/alertmanager/template" amtracing "github.com/prometheus/alertmanager/tracing" commoncfg "github.com/prometheus/common/config" @@ -344,33 +350,33 @@ var noopValidator = func(any) error { return nil } // - nil: receiver config with no file-based fields to validate // - noopValidator: non-receiver config type to ignore var configValidators = map[reflect.Type]func(any) error{ - reflect.TypeFor[config.GlobalConfig](): func(v any) error { return validateGlobalConfig(v.(config.GlobalConfig)) }, - reflect.TypeFor[commoncfg.HTTPClientConfig](): func(v any) error { return validateReceiverHTTPConfig(v.(commoncfg.HTTPClientConfig)) }, - reflect.TypeFor[commoncfg.TLSConfig](): func(v any) error { return validateReceiverTLSConfig(v.(commoncfg.TLSConfig)) }, - reflect.TypeFor[config.OpsGenieConfig](): func(v any) error { return validateOpsGenieConfig(v.(config.OpsGenieConfig)) }, - reflect.TypeFor[config.SlackConfig](): func(v any) error { return validateSlackConfig(v.(config.SlackConfig)) }, - reflect.TypeFor[config.VictorOpsConfig](): func(v any) error { return validateVictorOpsConfig(v.(config.VictorOpsConfig)) }, - reflect.TypeFor[config.PagerdutyConfig](): func(v any) error { return validatePagerdutyConfig(v.(config.PagerdutyConfig)) }, - reflect.TypeFor[config.WebhookConfig](): func(v any) error { return validateWebhookConfig(v.(config.WebhookConfig)) }, - reflect.TypeFor[config.PushoverConfig](): func(v any) error { return validatePushOverConfig(v.(config.PushoverConfig)) }, - reflect.TypeFor[config.TelegramConfig](): func(v any) error { return validateTelegramConfig(v.(config.TelegramConfig)) }, - reflect.TypeFor[config.MSTeamsConfig](): func(v any) error { return validateMSTeamsConfig(v.(config.MSTeamsConfig)) }, - reflect.TypeFor[config.MSTeamsV2Config](): func(v any) error { return validateMSTeamsV2Config(v.(config.MSTeamsV2Config)) }, - reflect.TypeFor[config.RocketchatConfig](): func(v any) error { return validateRocketChatConfig(v.(config.RocketchatConfig)) }, - reflect.TypeFor[config.DiscordConfig](): func(v any) error { return validateDiscordConfig(v.(config.DiscordConfig)) }, - reflect.TypeFor[config.EmailConfig](): func(v any) error { return validateEmailConfig(v.(config.EmailConfig)) }, - reflect.TypeFor[config.IncidentioConfig](): func(v any) error { return validateIncidentIOConfig(v.(config.IncidentioConfig)) }, - reflect.TypeFor[config.MattermostConfig](): func(v any) error { return validateMattermostConfig(v.(config.MattermostConfig)) }, - reflect.TypeFor[config.WechatConfig](): func(v any) error { return validateWeChatConfig(v.(config.WechatConfig)) }, - reflect.TypeFor[config.WebexConfig](): nil, // No file-based fields to validate - reflect.TypeFor[config.SNSConfig](): nil, // No file-based fields to validate - reflect.TypeFor[config.JiraConfig](): nil, // No file-based fields to validate + reflect.TypeFor[config.GlobalConfig](): func(v any) error { return validateGlobalConfig(v.(config.GlobalConfig)) }, + reflect.TypeFor[commoncfg.HTTPClientConfig](): func(v any) error { return validateReceiverHTTPConfig(v.(commoncfg.HTTPClientConfig)) }, + reflect.TypeFor[commoncfg.TLSConfig](): func(v any) error { return validateReceiverTLSConfig(v.(commoncfg.TLSConfig)) }, + reflect.TypeFor[config.OpsGenieConfig](): func(v any) error { return validateOpsGenieConfig(v.(config.OpsGenieConfig)) }, + reflect.TypeFor[config.SlackConfig](): func(v any) error { return validateSlackConfig(v.(config.SlackConfig)) }, + reflect.TypeFor[config.VictorOpsConfig](): func(v any) error { return validateVictorOpsConfig(v.(config.VictorOpsConfig)) }, + reflect.TypeFor[config.PagerdutyConfig](): func(v any) error { return validatePagerdutyConfig(v.(config.PagerdutyConfig)) }, + reflect.TypeFor[webhook.WebhookConfig](): func(v any) error { return validateWebhookConfig(v.(webhook.WebhookConfig)) }, + reflect.TypeFor[config.PushoverConfig](): func(v any) error { return validatePushOverConfig(v.(config.PushoverConfig)) }, + reflect.TypeFor[config.TelegramConfig](): func(v any) error { return validateTelegramConfig(v.(config.TelegramConfig)) }, + reflect.TypeFor[msteams.MSTeamsConfig](): func(v any) error { return validateMSTeamsConfig(v.(msteams.MSTeamsConfig)) }, + reflect.TypeFor[config.MSTeamsV2Config](): func(v any) error { return validateMSTeamsV2Config(v.(config.MSTeamsV2Config)) }, + reflect.TypeFor[config.RocketchatConfig](): func(v any) error { return validateRocketChatConfig(v.(config.RocketchatConfig)) }, + reflect.TypeFor[discord.DiscordConfig](): func(v any) error { return validateDiscordConfig(v.(discord.DiscordConfig)) }, + reflect.TypeFor[config.EmailConfig](): func(v any) error { return validateEmailConfig(v.(config.EmailConfig)) }, + reflect.TypeFor[incidentio.IncidentioConfig](): func(v any) error { return validateIncidentIOConfig(v.(incidentio.IncidentioConfig)) }, + reflect.TypeFor[mattermost.MattermostConfig](): func(v any) error { return validateMattermostConfig(v.(mattermost.MattermostConfig)) }, + reflect.TypeFor[config.WechatConfig](): func(v any) error { return validateWeChatConfig(v.(config.WechatConfig)) }, + reflect.TypeFor[config.WebexConfig](): nil, // No file-based fields to validate + reflect.TypeFor[config.SNSConfig](): nil, // No file-based fields to validate + reflect.TypeFor[jira.JiraConfig](): nil, // No file-based fields to validate // Non-receiver config types (ignored during validation) reflect.TypeFor[config.Config](): noopValidator, reflect.TypeFor[amcommoncfg.NotifierConfig](): noopValidator, reflect.TypeFor[amtracing.TracingConfig](): noopValidator, reflect.TypeFor[config.ThreadingConfig](): noopValidator, - reflect.TypeFor[config.JiraFieldConfig](): noopValidator, + reflect.TypeFor[jira.JiraFieldConfig](): noopValidator, } // validateAlertmanagerConfig recursively scans the input config looking for data types for which @@ -563,7 +569,7 @@ func validatePagerdutyConfig(cfg config.PagerdutyConfig) error { // validateWebhookConfig validates the Webhook config and returns an error if it contains // settings not allowed by Cortex. -func validateWebhookConfig(cfg config.WebhookConfig) error { +func validateWebhookConfig(cfg webhook.WebhookConfig) error { if cfg.URLFile != "" { return errWebhookURLFileNotAllowed } @@ -598,7 +604,7 @@ func validateTelegramConfig(cfg config.TelegramConfig) error { // validateMSTeamsConfig validates the MSTeams Config and returns an error if it contains // settings not allowed by Cortex. -func validateMSTeamsConfig(cfg config.MSTeamsConfig) error { +func validateMSTeamsConfig(cfg msteams.MSTeamsConfig) error { if cfg.WebhookURLFile != "" { return errMSTeamsWebhookUrlFileNotAllowed } @@ -630,7 +636,7 @@ func validateRocketChatConfig(cfg config.RocketchatConfig) error { // validateDiscordConfig validates the Discord Config and returns an error if it contains // settings not allowed by Cortex. -func validateDiscordConfig(cfg config.DiscordConfig) error { +func validateDiscordConfig(cfg discord.DiscordConfig) error { if cfg.WebhookURLFile != "" { return errDiscordWebhookUrlFileNotAllowed } @@ -651,7 +657,7 @@ func validateEmailConfig(cfg config.EmailConfig) error { // validateIncidentIOConfig validates the IncidentIO Config and returns an error if it contains // settings not allowed by Cortex. -func validateIncidentIOConfig(cfg config.IncidentioConfig) error { +func validateIncidentIOConfig(cfg incidentio.IncidentioConfig) error { if cfg.URLFile != "" { return errIncidentIOURLFileNotAllowed } @@ -665,7 +671,7 @@ func validateIncidentIOConfig(cfg config.IncidentioConfig) error { // validateMatterMostConfig validates the Mattermost Config and returns an error if it contains // settings not allowed by Cortex. -func validateMattermostConfig(cfg config.MattermostConfig) error { +func validateMattermostConfig(cfg mattermost.MattermostConfig) error { if cfg.WebhookURLFile != "" { return errMatterMostWebhookUrlFileNotAllowed } diff --git a/pkg/alertmanager/api_test.go b/pkg/alertmanager/api_test.go index f3bbb5d1dee..70c9f893354 100644 --- a/pkg/alertmanager/api_test.go +++ b/pkg/alertmanager/api_test.go @@ -13,6 +13,7 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/notify/webhook" "github.com/prometheus/client_golang/prometheus" commoncfg "github.com/prometheus/common/config" "github.com/stretchr/testify/assert" @@ -1326,7 +1327,7 @@ func TestValidateAlertmanagerConfig(t *testing.T) { input: config.Config{ Receivers: []config.Receiver{{ Name: "test", - WebhookConfigs: []*config.WebhookConfig{{ + WebhookConfigs: []*webhook.WebhookConfig{{ HTTPConfig: &commoncfg.HTTPClientConfig{ BasicAuth: &commoncfg.BasicAuth{ PasswordFile: "/secrets", diff --git a/pkg/alertmanager/merger/v2_alert_groups_test.go b/pkg/alertmanager/merger/v2_alert_groups_test.go index db6bfd1d791..69981534ded 100644 --- a/pkg/alertmanager/merger/v2_alert_groups_test.go +++ b/pkg/alertmanager/merger/v2_alert_groups_test.go @@ -61,7 +61,7 @@ func v2group(label, receiver string, alerts ...*v2_models.GettableAlert) *v2_mod return &v2_models.AlertGroup{ Alerts: alerts, Labels: v2_models.LabelSet{"some-label": label}, - Receiver: &v2_models.Receiver{Name: &receiver}, + Receiver: &v2_models.ReceiverReference{Name: &receiver}, } } diff --git a/pkg/alertmanager/merger/v2_alerts_test.go b/pkg/alertmanager/merger/v2_alerts_test.go index e782ea4a931..788bb992b5e 100644 --- a/pkg/alertmanager/merger/v2_alerts_test.go +++ b/pkg/alertmanager/merger/v2_alerts_test.go @@ -68,7 +68,7 @@ func v2alert(fingerprint, annotation, updatedAt string) *v2_models.GettableAlert }, EndsAt: v2ParseTime("2020-01-01T12:00:00.000Z"), Fingerprint: &fingerprint, - Receivers: []*v2_models.Receiver{ + Receivers: []*v2_models.ReceiverReference{ { Name: &receiver, }, diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 6f709cb2af1..3fe9788bed4 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/alertmanager/cluster" "github.com/prometheus/alertmanager/cluster/clusterpb" amconfig "github.com/prometheus/alertmanager/config" + amcommoncfg "github.com/prometheus/alertmanager/config/common" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/httpgrpc" @@ -878,7 +879,7 @@ func (am *MultitenantAlertmanager) setConfig(cfg alertspb.AlertConfigDesc) error if err != nil { return err } - userAmConfig.Receivers[i].WebhookConfigs[j].URL = amconfig.SecretTemplateURL(u.String()) + userAmConfig.Receivers[i].WebhookConfigs[j].URL = amcommoncfg.SecretTemplateURL(u.String()) } } } diff --git a/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.br b/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.br new file mode 100644 index 00000000000..b86353762a5 Binary files /dev/null and b/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.br differ diff --git a/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.gz b/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.gz new file mode 100644 index 00000000000..5793cfff77a Binary files /dev/null and b/pkg/alertmanager/ui/app/dist/assets/index-BUvG_Zbo.js.gz differ diff --git a/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.br b/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.br deleted file mode 100644 index 5dc5646a667..00000000000 Binary files a/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.br and /dev/null differ diff --git a/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.gz b/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.gz deleted file mode 100644 index 1ddbfeade6a..00000000000 Binary files a/pkg/alertmanager/ui/app/dist/assets/index-BrEcN0Zb.js.gz and /dev/null differ diff --git a/pkg/alertmanager/ui/app/dist/index.html.br b/pkg/alertmanager/ui/app/dist/index.html.br index a4e83da49be..471891610fa 100644 Binary files a/pkg/alertmanager/ui/app/dist/index.html.br and b/pkg/alertmanager/ui/app/dist/index.html.br differ diff --git a/pkg/alertmanager/ui/app/dist/index.html.gz b/pkg/alertmanager/ui/app/dist/index.html.gz index d89a8a0b0e5..652524ea6a6 100644 Binary files a/pkg/alertmanager/ui/app/dist/index.html.gz and b/pkg/alertmanager/ui/app/dist/index.html.gz differ diff --git a/pkg/alertmanager/ui/update-am-ui.sh b/pkg/alertmanager/ui/update-am-ui.sh index 68fbb0baa26..c4459244e8a 100755 --- a/pkg/alertmanager/ui/update-am-ui.sh +++ b/pkg/alertmanager/ui/update-am-ui.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Usage: ./update-am-ui.sh [version] -# Example: ./update-am-ui.sh 0.32.1 +# Example: ./update-am-ui.sh 0.33.0 set -euo pipefail -VERSION="${1:-0.32.1}" +VERSION="${1:-0.33.0}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TARGET_DIR="${SCRIPT_DIR}/app/dist" TARBALL="alertmanager-web-ui-${VERSION}.tar.gz" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/sns/CHANGELOG.md b/vendor/github.com/aws/aws-sdk-go-v2/service/sns/CHANGELOG.md index 6597c0c7cba..b7623b6050c 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/sns/CHANGELOG.md +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/sns/CHANGELOG.md @@ -1,3 +1,13 @@ +# v1.39.17 (2026-04-29) + +* **Dependency Update**: Update to smithy-go v1.25.1. +* **Dependency Update**: Updated to the latest SDK module versions + +# v1.39.16 (2026-04-17) + +* **Dependency Update**: Bump smithy-go to 1.25.0 to support endpointBdd trait +* **Dependency Update**: Updated to the latest SDK module versions + # v1.39.15 (2026-03-26) * **Bug Fix**: Fix a bug where a recorded clock skew could persist on the client even if the client and server clock ended up realigning. diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/sns/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/service/sns/go_module_metadata.go index 756552500a9..88f8840943f 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/sns/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/sns/go_module_metadata.go @@ -3,4 +3,4 @@ package sns // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.39.15" +const goModuleVersion = "1.39.17" diff --git a/vendor/github.com/coder/quartz/LICENSE b/vendor/github.com/coder/quartz/LICENSE index f7c5d7fee65..aee8c1d627f 100644 --- a/vendor/github.com/coder/quartz/LICENSE +++ b/vendor/github.com/coder/quartz/LICENSE @@ -1,121 +1,18 @@ -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be - protected by copyright and related or neighboring rights ("Copyright and - Related Rights"). Copyright and Related Rights include, but are not - limited to, the following: - -i. the right to reproduce, adapt, distribute, perform, display, -communicate, and translate a Work; -ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or -likeness depicted in a Work; -iv. rights protecting against unfair competition in regards to a Work, -subject to the limitations in paragraph 4(a), below; -v. rights protecting the extraction, dissemination, use and reuse of data -in a Work; -vi. database rights (such as those arising under Directive 96/9/EC of the -European Parliament and of the Council of 11 March 1996 on the legal -protection of databases, and under any national implementation -thereof, including any amended or successor version of such -directive); and -vii. other similar, equivalent or corresponding rights throughout the -world based on applicable law or treaty, and any national -implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention - of, applicable law, Affirmer hereby overtly, fully, permanently, - irrevocably and unconditionally waives, abandons, and surrenders all of - Affirmer's Copyright and Related Rights and associated claims and causes - of action, whether now known or unknown (including existing as well as - future claims and causes of action), in the Work (i) in all territories - worldwide, (ii) for the maximum duration provided by applicable law or - treaty (including future time extensions), (iii) in any current or future - medium and for any number of copies, and (iv) for any purpose whatsoever, - including without limitation commercial, advertising or promotional - purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each - member of the public at large and to the detriment of Affirmer's heirs and - successors, fully intending that such Waiver shall not be subject to - revocation, rescission, cancellation, termination, or any other legal or - equitable action to disrupt the quiet enjoyment of the Work by the public - as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason - be judged legally invalid or ineffective under applicable law, then the - Waiver shall be preserved to the maximum extent permitted taking into - account Affirmer's express Statement of Purpose. In addition, to the - extent the Waiver is so judged Affirmer hereby grants to each affected - person a royalty-free, non transferable, non sublicensable, non exclusive, - irrevocable and unconditional license to exercise Affirmer's Copyright and - Related Rights in the Work (i) in all territories worldwide, (ii) for the - maximum duration provided by applicable law or treaty (including future - time extensions), (iii) in any current or future medium and for any number - of copies, and (iv) for any purpose whatsoever, including without - limitation commercial, advertising or promotional purposes (the - "License"). The License shall be deemed effective as of the date CC0 was - applied by Affirmer to the Work. Should any part of the License for any - reason be judged legally invalid or ineffective under applicable law, such - partial invalidity or ineffectiveness shall not invalidate the remainder - of the License, and in such case Affirmer hereby affirms that he or she - will not (i) exercise any of his or her remaining Copyright and Related - Rights in the Work or (ii) assert any associated claims and causes of - action with respect to the Work, in either case contrary to Affirmer's - express Statement of Purpose. - -4. Limitations and Disclaimers. - -a. No trademark or patent rights held by Affirmer are waived, abandoned, -surrendered, licensed or otherwise affected by this document. -b. Affirmer offers the Work as-is and makes no representations or -warranties of any kind concerning the Work, express, implied, -statutory or otherwise, including without limitation warranties of -title, merchantability, fitness for a particular purpose, non -infringement, or the absence of latent or other defects, accuracy, or -the present or absence of errors, whether or not discoverable, all to -the greatest extent permissible under applicable law. -c. Affirmer disclaims responsibility for clearing rights of other persons -that may apply to the Work or any use thereof, including without -limitation any person's Copyright and Related Rights in the Work. -Further, Affirmer disclaims responsibility for obtaining any necessary -consents, permissions or other rights required for any use of the -Work. -d. Affirmer understands and acknowledges that Creative Commons is not a -party to this document and has no duty or obligation with respect to -this CC0 or use of the Work. \ No newline at end of file +MIT No Attribution + +Copyright (c) Coder Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/coder/quartz/mock.go b/vendor/github.com/coder/quartz/mock.go index d168849d602..7255aff99bd 100644 --- a/vendor/github.com/coder/quartz/mock.go +++ b/vendor/github.com/coder/quartz/mock.go @@ -6,14 +6,27 @@ import ( "fmt" "slices" "sync" - "testing" "time" ) +// TestingT is the minimal interface required from a testing framework for the Mock. +type TestingT interface { + Helper() + + Log(...any) + Logf(string, ...any) + Error(...any) + Errorf(string, ...any) + Fatal(...any) + Fatalf(string, ...any) + + Cleanup(func()) +} + // Mock is the testing implementation of Clock. It tracks a time that monotonically increases // during a test, triggering any timers or tickers automatically. type Mock struct { - tb testing.TB + tb TestingT logger Logger mu sync.Mutex testOver bool @@ -215,7 +228,7 @@ func (m *Mock) matchCallLocked(c *apiCall) { // If multiple timers or tickers trigger simultaneously, they are all run on separate // go routines. type AdvanceWaiter struct { - tb testing.TB + tb TestingT ch chan struct{} } @@ -467,7 +480,7 @@ func (m *Mock) WithLogger(l Logger) *Mock { // NewMock creates a new Mock with the time set to midnight UTC on Jan 1, 2024. // You may re-set the time earlier than this, but only before timers or tickers // are created. -func NewMock(tb testing.TB) *Mock { +func NewMock(tb TestingT) *Mock { cur, err := time.Parse(time.RFC3339, "2024-01-01T00:00:00Z") if err != nil { panic(err) @@ -668,7 +681,7 @@ type Call struct { Duration time.Duration Tags []string - tb testing.TB + tb TestingT apiCall *apiCall trap *Trap } @@ -738,6 +751,10 @@ type Trap struct { unreleasedCalls int } +func (t *Trap) String() string { + return fmt.Sprintf("Trap %s(..., %v)", t.fn.String(), t.tags) +} + func (t *Trap) catch(c *apiCall) { select { case t.calls <- c: @@ -761,9 +778,15 @@ func (t *Trap) matches(c *apiCall) bool { func (t *Trap) Close() { t.mock.mu.Lock() defer t.mock.mu.Unlock() + select { + case <-t.done: + t.mock.tb.Logf("%s already Closed()", t) + return // already closed + default: + } if t.unreleasedCalls != 0 { t.mock.tb.Helper() - t.mock.tb.Errorf("trap Closed() with %d unreleased calls", t.unreleasedCalls) + t.mock.tb.Errorf("%s Closed() with %d unreleased calls", t, t.unreleasedCalls) } for i, tr := range t.mock.traps { if t == tr { @@ -809,7 +832,7 @@ func (t *Trap) MustWait(ctx context.Context) *Call { t.mock.tb.Helper() c, err := t.Wait(ctx) if err != nil { - t.mock.tb.Fatalf("context expired while waiting for trap: %s", err.Error()) + t.mock.tb.Fatalf("context expired while waiting for %s: %s", t, err.Error()) } return c } diff --git a/vendor/github.com/coreos/go-systemd/v22/activation/files.go b/vendor/github.com/coreos/go-systemd/v22/activation/files.go new file mode 100644 index 00000000000..8ac7fa3fa41 --- /dev/null +++ b/vendor/github.com/coreos/go-systemd/v22/activation/files.go @@ -0,0 +1,29 @@ +// Copyright 2026 RedHat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package activation + +import "os" + +// FilesWithNames maps fd names to a set of os.File pointers. +func FilesWithNames() map[string][]*os.File { + files := Files(true) + filesWithNames := map[string][]*os.File{} + + for _, f := range files { + filesWithNames[f.Name()] = append(filesWithNames[f.Name()], f) + } + + return filesWithNames +} diff --git a/vendor/github.com/coreos/go-systemd/v22/activation/files_windows.go b/vendor/github.com/coreos/go-systemd/v22/activation/files_stub.go similarity index 97% rename from vendor/github.com/coreos/go-systemd/v22/activation/files_windows.go rename to vendor/github.com/coreos/go-systemd/v22/activation/files_stub.go index d391bf00c5e..5ea78db417d 100644 --- a/vendor/github.com/coreos/go-systemd/v22/activation/files_windows.go +++ b/vendor/github.com/coreos/go-systemd/v22/activation/files_stub.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !unix + package activation import "os" diff --git a/vendor/github.com/coreos/go-systemd/v22/activation/files_unix.go b/vendor/github.com/coreos/go-systemd/v22/activation/files_unix.go index 7031f281a07..113cfef4d17 100644 --- a/vendor/github.com/coreos/go-systemd/v22/activation/files_unix.go +++ b/vendor/github.com/coreos/go-systemd/v22/activation/files_unix.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !windows +//go:build unix // Package activation implements primitives for systemd socket activation. package activation @@ -51,7 +51,7 @@ func Files(unsetEnv bool) []*os.File { } nfds, err := strconv.Atoi(os.Getenv("LISTEN_FDS")) - if err != nil || nfds == 0 { + if err != nil || nfds <= 0 { return nil } diff --git a/vendor/github.com/coreos/go-systemd/v22/activation/listeners.go b/vendor/github.com/coreos/go-systemd/v22/activation/listeners.go index 3dbe2b08776..108562bfc51 100644 --- a/vendor/github.com/coreos/go-systemd/v22/activation/listeners.go +++ b/vendor/github.com/coreos/go-systemd/v22/activation/listeners.go @@ -45,12 +45,7 @@ func ListenersWithNames() (map[string][]net.Listener, error) { for _, f := range files { if pc, err := net.FileListener(f); err == nil { - current, ok := listeners[f.Name()] - if !ok { - listeners[f.Name()] = []net.Listener{pc} - } else { - listeners[f.Name()] = append(current, pc) - } + listeners[f.Name()] = append(listeners[f.Name()], pc) f.Close() } } diff --git a/vendor/github.com/coreos/go-systemd/v22/journal/journal_unix.go b/vendor/github.com/coreos/go-systemd/v22/journal/journal_unix.go index 6266e16e573..55cb9eb26a1 100644 --- a/vendor/github.com/coreos/go-systemd/v22/journal/journal_unix.go +++ b/vendor/github.com/coreos/go-systemd/v22/journal/journal_unix.go @@ -106,7 +106,7 @@ func fdIsJournalStream(fd int) (bool, error) { var expectedStat syscall.Stat_t _, err := fmt.Sscanf(journalStream, "%d:%d", &expectedStat.Dev, &expectedStat.Ino) if err != nil { - return false, fmt.Errorf("failed to parse JOURNAL_STREAM=%q: %v", journalStream, err) + return false, fmt.Errorf("failed to parse JOURNAL_STREAM=%q: %w", journalStream, err) } var stat syscall.Stat_t diff --git a/vendor/github.com/fsnotify/fsnotify/.cirrus.yml b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml deleted file mode 100644 index 7f257e99ac9..00000000000 --- a/vendor/github.com/fsnotify/fsnotify/.cirrus.yml +++ /dev/null @@ -1,14 +0,0 @@ -freebsd_task: - name: 'FreeBSD' - freebsd_instance: - image_family: freebsd-14-2 - install_script: - - pkg update -f - - pkg install -y go - test_script: - # run tests as user "cirrus" instead of root - - pw useradd cirrus -m - - chown -R cirrus:cirrus . - - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... - - sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... - - FSNOTIFY_DEBUG=1 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race -v ./... diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index 6468d2cf400..4ed4864f475 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +1.10.0 2026-04-30 +----------------- +This version of fsnotify needs Go 1.23. + +### Changes and fixes + +- inotify: improve initialization error message ([#731]) + +- inotify: send Rename event if recursive watch is renamed ([#696]) + +- inotify: avoid copying event buffers when reading names ([#741]) + +- kqueue: skip dangling symlinks (ENOENT) in watchDirectoryFiles, so a + bad entry no longer aborts Watcher.Add for the whole directory ([#748]) + +- kqueue: drop watches directly in Close() to fix a file descriptor leak + when recycling watchers ([#740]) + +- windows: fix nil pointer dereference in remWatch ([#736]) + +- windows: lock watch field updates against concurrent WatchList to fix + a race introduced in v1.9.0 ([#709], [#749]) + + +[#696]: https://github.com/fsnotify/fsnotify/pull/696 +[#709]: https://github.com/fsnotify/fsnotify/pull/709 +[#731]: https://github.com/fsnotify/fsnotify/pull/731 +[#736]: https://github.com/fsnotify/fsnotify/pull/736 +[#740]: https://github.com/fsnotify/fsnotify/pull/740 +[#741]: https://github.com/fsnotify/fsnotify/pull/741 +[#748]: https://github.com/fsnotify/fsnotify/pull/748 +[#749]: https://github.com/fsnotify/fsnotify/pull/749 + + 1.9.0 2024-04-04 ---------------- diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md index 4cc40fa597d..cd0ee612da6 100644 --- a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -77,6 +77,8 @@ End-of-line escapes with `\` are not supported. debug [yes/no] # Enable/disable FSNOTIFY_DEBUG (tests are run in parallel by default, so -parallel=1 is probably a good idea). + state # Print internal state to stderr (exact output differs + # per backend). print [any strings] # Print text to stdout; for debugging. touch path diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index 1f4eb583d50..d8441aa0614 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -1,7 +1,7 @@ fsnotify is a Go library to provide cross-platform filesystem notifications on Windows, Linux, macOS, BSD, and illumos. -Go 1.17 or newer is required; the full documentation is at +Go 1.23 or newer is required; the full documentation is at https://pkg.go.dev/github.com/fsnotify/fsnotify --- @@ -12,7 +12,7 @@ Platform support: | :-------------------- | :--------- | :------------------------------------------------------------------------ | | inotify | Linux | Supported | | kqueue | BSD, macOS | Supported | -| ReadDirectoryChangesW | Windows | Supported | +| ReadDirectoryChangesW | Windows | Supported ([excluding `Chmod` operations][#487]) | | FEN | illumos | Supported | | fanotify | Linux 5.9+ | [Not yet](https://github.com/fsnotify/fsnotify/issues/114) | | FSEvents | macOS | [Needs support in x/sys/unix][fsevents] | @@ -22,6 +22,7 @@ Platform support: Linux and illumos should include Android and Solaris, but these are currently untested. +[#487]: https://github.com/fsnotify/fsnotify/issues/487 [fsevents]: https://github.com/fsnotify/fsnotify/issues/11#issuecomment-1279133120 [usn]: https://github.com/fsnotify/fsnotify/issues/53#issuecomment-1279829847 @@ -126,7 +127,7 @@ settings* until we have a native FSEvents implementation (see [#11]). ### Watching a file doesn't work well Watching individual files (rather than directories) is generally not recommended as many programs (especially editors) update files atomically: it will write to -a temporary file which is then moved to to destination, overwriting the original +a temporary file which is then moved to a destination, overwriting the original (or some variant thereof). The watcher on the original file is now lost, as that no longer exists. @@ -151,26 +152,25 @@ This is the event that inotify sends, so not much can be changed about this. The `fs.inotify.max_user_watches` sysctl variable specifies the upper limit for the number of watches per user, and `fs.inotify.max_user_instances` specifies the maximum number of inotify instances per user. Every Watcher you create is an -"instance", and every path you add is a "watch". +"instance", and every path you add is a "watch". Reaching the limit will result +in a "no space left on device" or "too many open files" error. These are also exposed in `/proc` as `/proc/sys/fs/inotify/max_user_watches` and -`/proc/sys/fs/inotify/max_user_instances` +`/proc/sys/fs/inotify/max_user_instances`. The default values differ per distro +and available memory. To increase them you can use `sysctl` or write the value to proc file: - # The default values on Linux 5.18 - sysctl fs.inotify.max_user_watches=124983 - sysctl fs.inotify.max_user_instances=128 + sysctl fs.inotify.max_user_watches=200000 + sysctl fs.inotify.max_user_instances=256 To make the changes persist on reboot edit `/etc/sysctl.conf` or `/usr/lib/sysctl.d/50-default.conf` (details differ per Linux distro; check your distro's documentation): - fs.inotify.max_user_watches=124983 - fs.inotify.max_user_instances=128 + fs.inotify.max_user_watches=200000 + fs.inotify.max_user_instances=256 -Reaching the limit will result in a "no space left on device" or "too many open -files" error. ### kqueue (macOS, all BSD systems) kqueue requires opening a file descriptor for every file that's being watched; diff --git a/vendor/github.com/fsnotify/fsnotify/backend_fen.go b/vendor/github.com/fsnotify/fsnotify/backend_fen.go index 57fc6928484..e43c6d088cc 100644 --- a/vendor/github.com/fsnotify/fsnotify/backend_fen.go +++ b/vendor/github.com/fsnotify/fsnotify/backend_fen.go @@ -158,7 +158,9 @@ func (w *fen) readEvents() { pevents := make([]unix.PortEvent, 8) for { - count, err := w.port.Get(pevents, 1, nil) + count, err := internal.IgnoringEINTR(func() (int, error) { + return w.port.Get(pevents, 1, nil) + }) if err != nil && err != unix.ETIME { // Interrupted system call (count should be 0) ignore and continue if errors.Is(err, unix.EINTR) && count == 0 { diff --git a/vendor/github.com/fsnotify/fsnotify/backend_inotify.go b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go index a36cb89d736..cdb7812ac12 100644 --- a/vendor/github.com/fsnotify/fsnotify/backend_inotify.go +++ b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go @@ -55,10 +55,10 @@ type ( path map[string]uint32 // pathname → wd } watch struct { - wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) - flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) - path string // Watch path. - recurse bool // Recursion with ./...? + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) + path string // Watch path. + watchFlags watchFlag } koekje struct { cookie uint32 @@ -66,6 +66,9 @@ type ( } ) +func (w watch) byUser() bool { return w.watchFlags&flagByUser != 0 } +func (w watch) recurse() bool { return w.watchFlags&flagRecurse != 0 } + func newWatches() *watches { return &watches{ wd: make(map[uint32]*watch), @@ -87,13 +90,13 @@ func (w *watches) removePath(path string) ([]uint32, error) { } watch := w.wd[wd] - if recurse && !watch.recurse { + if recurse && !watch.recurse() { return nil, fmt.Errorf("can't use /... with non-recursive watch %q", path) } delete(w.path, path) delete(w.wd, wd) - if !watch.recurse { + if !watch.recurse() { return []uint32{wd}, nil } @@ -139,7 +142,7 @@ func newBackend(ev chan Event, errs chan error) (backend, error) { // I/O operations won't terminate on close. fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) if fd == -1 { - return nil, errno + return nil, fmt.Errorf("couldn't initialize inotify: %w", errno) } w := &inotify{ @@ -188,11 +191,8 @@ func (w *inotify) AddWith(path string, opts ...addOpt) error { return fmt.Errorf("%w: %s", xErrUnsupported, with.op) } - add := func(path string, with withOpts, recurse bool) error { + add := func(path string, with withOpts, wf watchFlag) error { var flags uint32 - if with.noFollow { - flags |= unix.IN_DONT_FOLLOW - } if with.op.Has(Create) { flags |= unix.IN_CREATE } @@ -220,7 +220,7 @@ func (w *inotify) AddWith(path string, opts ...addOpt) error { if with.op.Has(xUnportableCloseRead) { flags |= unix.IN_CLOSE_NOWRITE } - return w.register(path, flags, recurse) + return w.register(path, flags, wf) } w.mu.Lock() @@ -248,14 +248,18 @@ func (w *inotify) AddWith(path string, opts ...addOpt) error { w.sendEvent(Event{Name: root, Op: Create}) } - return add(root, with, true) + wf := flagRecurse + if root == path { + wf |= flagByUser + } + return add(root, with, wf) }) } - return add(path, with, false) + return add(path, with, 0) } -func (w *inotify) register(path string, flags uint32, recurse bool) error { +func (w *inotify) register(path string, flags uint32, wf watchFlag) error { return w.watches.updatePath(path, func(existing *watch) (*watch, error) { if existing != nil { flags |= existing.flags | unix.IN_MASK_ADD @@ -272,10 +276,10 @@ func (w *inotify) register(path string, flags uint32, recurse bool) error { if existing == nil { return &watch{ - wd: uint32(wd), - path: path, - flags: flags, - recurse: recurse, + wd: uint32(wd), + path: path, + flags: flags, + watchFlags: wf, }, nil } @@ -425,11 +429,7 @@ func (w *inotify) handleEvent(inEvent *unix.InotifyEvent, buf *[65536]byte, offs nameLen = uint32(inEvent.Len) ) if nameLen > 0 { - /// Point "bytes" at the first byte of the filename - bb := *buf - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&bb[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] - /// The filename is padded with NULL bytes. TrimRight() gets rid of those. - name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\x00") + name += "/" + inotifyEventName(buf, offset, nameLen) } if debug { @@ -450,7 +450,9 @@ func (w *inotify) handleEvent(inEvent *unix.InotifyEvent, buf *[65536]byte, offs // We can't really update the state when a watched path is moved; only // IN_MOVE_SELF is sent and not IN_MOVED_{FROM,TO}. So remove the watch. if inEvent.Mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF { - if watch.recurse { // Do nothing + // Watch is set up as part of recurse: do nothing as the move gets + // registered from the parent directory. + if watch.recurse() && !watch.byUser() { return Event{}, true } @@ -460,6 +462,10 @@ func (w *inotify) handleEvent(inEvent *unix.InotifyEvent, buf *[65536]byte, offs return Event{}, false } } + + if watch.recurse() { + return Event{Name: watch.path, Op: Rename}, true + } } /// Skip if we're watching both this path and the parent; the parent will @@ -473,11 +479,11 @@ func (w *inotify) handleEvent(inEvent *unix.InotifyEvent, buf *[65536]byte, offs ev := w.newEvent(name, inEvent.Mask, inEvent.Cookie) // Need to update watch path for recurse. - if watch.recurse { + if watch.recurse() { isDir := inEvent.Mask&unix.IN_ISDIR == unix.IN_ISDIR /// New directory created: set up watch on it. if isDir && ev.Has(Create) { - err := w.register(ev.Name, watch.flags, true) + err := w.register(ev.Name, watch.flags, flagRecurse) if !w.sendError(err) { return Event{}, false } @@ -507,12 +513,13 @@ func (w *inotify) handleEvent(inEvent *unix.InotifyEvent, buf *[65536]byte, offs return ev, true } -func (w *inotify) isRecursive(path string) bool { - ww := w.watches.byPath(path) - if ww == nil { // path could be a file, so also check the Dir. - ww = w.watches.byPath(filepath.Dir(path)) +func inotifyEventName(buf *[65536]byte, offset, nameLen uint32) string { + start := int(offset + unix.SizeofInotifyEvent) + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[start]))[:nameLen:nameLen] + for nameLen > 0 && bytes[nameLen-1] == 0 { + nameLen-- } - return ww != nil && ww.recurse + return string(bytes[:nameLen]) } func (w *inotify) newEvent(name string, mask, cookie uint32) Event { @@ -578,6 +585,6 @@ func (w *inotify) state() { w.mu.Lock() defer w.mu.Unlock() for wd, ww := range w.watches.wd { - fmt.Fprintf(os.Stderr, "%4d: recurse=%t %q\n", wd, ww.recurse, ww.path) + fmt.Fprintf(os.Stderr, "%4d: %q watchFlags=0x%x\n", wd, ww.path, ww.watchFlags) } } diff --git a/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go index 340aeec061c..d2c8cfb6c6a 100644 --- a/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go +++ b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "sync" "time" @@ -245,9 +246,26 @@ func (w *kqueue) Close() error { return nil } + // Snapshot and drop all watches directly. w.Remove -> w.remove + // short-circuits on isClosed() (which is already true after + // w.shared.close() above), so calling Remove here in the happy path + // leaked every watched directory + file descriptor. On macOS a + // single directory watch opens an fd for every file in the dir, so + // long-running processes that recreate watchers (hot-reload dev + // servers, etc.) ran out of fds with EMFILE (#732). pathsToRemove := w.watches.listPaths(false) for _, name := range pathsToRemove { - w.Remove(name) + info, ok := w.watches.byPath(name) + if !ok { + // w.path has an entry for name but w.wd doesn't -- + // drop the stale lookup entry so the map state is + // consistent after Close. + w.watches.remove(0, name) + continue + } + _ = w.register([]int{info.wd}, unix.EV_DELETE, 0) + unix.Close(info.wd) + w.watches.remove(info.wd, name) } unix.Close(w.closepipe[1]) // Send "quit" message to readEvents @@ -376,19 +394,12 @@ func (w *kqueue) addWatch(name string, flags uint32, listDir bool) (string, erro } } - // Retry on EINTR; open() can return EINTR in practice on macOS. - // See #354, and Go issues 11180 and 39237. - for { - info.wd, err = unix.Open(name, openMode, 0) - if err == nil { - break - } - if errors.Is(err, unix.EINTR) { - continue - } + info.wd, err = internal.IgnoringEINTR(func() (int, error) { + return unix.Open(name, openMode, 0) + }) + if err != nil { return "", err } - info.isDir = fi.IsDir() } @@ -436,9 +447,10 @@ func (w *kqueue) readEvents() { eventBuffer := make([]unix.Kevent_t, 10) for { - kevents, err := w.read(eventBuffer) - // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != unix.EINTR { + kevents, err := internal.IgnoringEINTR(func() ([]unix.Kevent_t, error) { + return w.read(eventBuffer) + }) + if err != nil { if !w.sendError(fmt.Errorf("fsnotify.readEvents: %w", err)) { return } @@ -583,12 +595,14 @@ func (w *kqueue) watchDirectoryFiles(dirPath string) error { cleanPath, err := w.internalWatch(path, fi) if err != nil { - // No permission to read the file; that's not a problem: just skip. - // But do add it to w.fileExists to prevent it from being picked up - // as a "new" file later (it still shows up in the directory + // No permission, or the entry resolved to a missing target + // (e.g. a dangling symlink): not a problem, just skip. But + // do mark it as seen to prevent it from being picked up as + // a "new" file later (it still shows up in the directory // listing). switch { - case errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM): + case errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM) || + errors.Is(err, os.ErrNotExist): cleanPath = filepath.Clean(path) default: return fmt.Errorf("%q: %w", path, err) @@ -703,3 +717,19 @@ func (w *kqueue) xSupports(op Op) bool { } return true } + +func (w *kqueue) state() { + w.watches.mu.Lock() + defer w.watches.mu.Unlock() + + all := make([]int, 0, len(w.watches.wd)) + for wd := range w.watches.wd { + all = append(all, wd) + } + sort.Ints(all) + + for _, wd := range all { + ww := w.watches.wd[wd] + fmt.Fprintf(os.Stderr, "%4d %q linkname=%q\n", wd, ww.name, ww.linkName) + } +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_windows.go b/vendor/github.com/fsnotify/fsnotify/backend_windows.go index 3433642d641..8ef0eb0f621 100644 --- a/vendor/github.com/fsnotify/fsnotify/backend_windows.go +++ b/vendor/github.com/fsnotify/fsnotify/backend_windows.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "reflect" "runtime" "strings" "sync" @@ -359,22 +358,26 @@ func (w *readDirChangesW) addWatch(pathname string, flags uint64, bufsize int) e } else { windows.CloseHandle(ino.handle) } + w.mu.Lock() if pathname == dir { watchEntry.mask |= flags } else { watchEntry.names[filepath.Base(pathname)] |= flags } + w.mu.Unlock() err = w.startRead(watchEntry) if err != nil { return err } + w.mu.Lock() if pathname == dir { watchEntry.mask &= ^provisional } else { watchEntry.names[filepath.Base(pathname)] &= ^provisional } + w.mu.Unlock() return nil } @@ -394,8 +397,13 @@ func (w *readDirChangesW) remWatch(pathname string) error { w.mu.Lock() watch := w.watches.get(ino) w.mu.Unlock() + if watch == nil { + windows.CloseHandle(ino.handle) + return fmt.Errorf("%w: %s", ErrNonExistentWatch, pathname) + } if recurse && !watch.recurse { + windows.CloseHandle(ino.handle) return fmt.Errorf("can't use \\... with non-recursive watch %q", pathname) } @@ -403,16 +411,19 @@ func (w *readDirChangesW) remWatch(pathname string) error { if err != nil { w.sendError(os.NewSyscallError("CloseHandle", err)) } - if watch == nil { - return fmt.Errorf("%w: %s", ErrNonExistentWatch, pathname) - } if pathname == dir { - w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) + w.mu.Lock() + mask := watch.mask watch.mask = 0 + w.mu.Unlock() + w.sendEvent(watch.path, "", mask&sysFSIGNORED) } else { name := filepath.Base(pathname) - w.sendEvent(filepath.Join(watch.path, name), "", watch.names[name]&sysFSIGNORED) + w.mu.Lock() + mask := watch.names[name] delete(watch.names, name) + w.mu.Unlock() + w.sendEvent(filepath.Join(watch.path, name), "", mask&sysFSIGNORED) } return w.startRead(watch) @@ -420,17 +431,23 @@ func (w *readDirChangesW) remWatch(pathname string) error { // Must run within the I/O thread. func (w *readDirChangesW) deleteWatch(watch *watch) { - for name, mask := range watch.names { - if mask&provisional == 0 { - w.sendEvent(filepath.Join(watch.path, name), "", mask&sysFSIGNORED) + // Snapshot+clear under the lock so concurrent WatchList() readers see a + // consistent state. sendEvent must run outside the lock since it can + // block on the user-facing Events channel. + w.mu.Lock() + names := watch.names + watch.names = make(map[string]uint64) + mask := watch.mask + watch.mask = 0 + w.mu.Unlock() + + for name, m := range names { + if m&provisional == 0 { + w.sendEvent(filepath.Join(watch.path, name), "", m&sysFSIGNORED) } - delete(watch.names, name) } - if watch.mask != 0 { - if watch.mask&provisional == 0 { - w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) - } - watch.mask = 0 + if mask != 0 && mask&provisional == 0 { + w.sendEvent(watch.path, "", mask&sysFSIGNORED) } } @@ -457,9 +474,8 @@ func (w *readDirChangesW) startRead(watch *watch) error { } // We need to pass the array, rather than the slice. - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&watch.buf)) rdErr := windows.ReadDirectoryChanges(watch.ino.handle, - (*byte)(unsafe.Pointer(hdr.Data)), uint32(hdr.Len), + unsafe.SliceData(watch.buf), uint32(len(watch.buf)), watch.recurse, mask, nil, &watch.ov, 0) if rdErr != nil { err := os.NewSyscallError("ReadDirectoryChanges", rdErr) @@ -565,12 +581,7 @@ func (w *readDirChangesW) readEvents() { // Create a buf that is the size of the path name size := int(raw.FileNameLength / 2) - var buf []uint16 - // TODO: Use unsafe.Slice in Go 1.17; https://stackoverflow.com/questions/51187973 - sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) - sh.Len = size - sh.Cap = size + buf := unsafe.Slice(&raw.FileName, size) name := windows.UTF16ToString(buf) fullname := filepath.Join(watch.path, name) @@ -587,7 +598,9 @@ func (w *readDirChangesW) readEvents() { case windows.FILE_ACTION_RENAMED_OLD_NAME: watch.rename = name case windows.FILE_ACTION_RENAMED_NEW_NAME: - // Update saved path of all sub-watches. + // Update saved path of all sub-watches and rename the + // names entry under the lock so WatchList() can't observe + // a torn state. old := filepath.Join(watch.path, watch.rename) w.mu.Lock() for _, watchMap := range w.watches { @@ -597,21 +610,23 @@ func (w *readDirChangesW) readEvents() { } } } - w.mu.Unlock() - if watch.names[watch.rename] != 0 { watch.names[name] |= watch.names[watch.rename] delete(watch.names, watch.rename) mask = sysFSMOVESELF } + w.mu.Unlock() } if raw.Action != windows.FILE_ACTION_RENAMED_NEW_NAME { w.sendEvent(fullname, "", watch.names[name]&mask) } if raw.Action == windows.FILE_ACTION_REMOVED { - w.sendEvent(fullname, "", watch.names[name]&sysFSIGNORED) + w.mu.Lock() + ignored := watch.names[name] & sysFSIGNORED delete(watch.names, name) + w.mu.Unlock() + w.sendEvent(fullname, "", ignored) } if watch.rename != "" && raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index f64be4bf98e..c7a4cb309cc 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -51,26 +51,25 @@ import ( // The fs.inotify.max_user_watches sysctl variable specifies the upper limit // for the number of watches per user, and fs.inotify.max_user_instances // specifies the maximum number of inotify instances per user. Every Watcher you -// create is an "instance", and every path you add is a "watch". +// create is an "instance", and every path you add is a "watch". Reaching the +// limit will result in a "no space left on device" or "too many open files" +// error. // // These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and -// /proc/sys/fs/inotify/max_user_instances +// /proc/sys/fs/inotify/max_user_instances. The default values differ per distro +// and available memory. // // To increase them you can use sysctl or write the value to the /proc file: // -// # Default values on Linux 5.18 -// sysctl fs.inotify.max_user_watches=124983 -// sysctl fs.inotify.max_user_instances=128 +// sysctl fs.inotify.max_user_watches=200000 +// sysctl fs.inotify.max_user_instances=256 // // To make the changes persist on reboot edit /etc/sysctl.conf or // /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check // your distro's documentation): // -// fs.inotify.max_user_watches=124983 -// fs.inotify.max_user_instances=128 -// -// Reaching the limit will result in a "no space left on device" or "too many open -// files" error. +// fs.inotify.max_user_watches=200000 +// fs.inotify.max_user_instances=256 // // # kqueue notes (macOS, BSD) // @@ -220,7 +219,7 @@ const ( // File opened for reading was closed. // - // Only works on Linux and FreeBSD. + // Only works on Linux. xUnportableCloseRead ) @@ -410,7 +409,6 @@ type ( withOpts struct { bufsize int op Op - noFollow bool sendCreate bool } ) @@ -469,12 +467,6 @@ func withOps(op Op) addOpt { return func(opt *withOpts) { opt.op = op } } -// WithNoFollow disables following symlinks, so the symlinks themselves are -// watched. -func withNoFollow() addOpt { - return func(opt *withOpts) { opt.noFollow = true } -} - // "Internal" option for recursive watches on inotify. func withCreate() addOpt { return func(opt *withOpts) { opt.sendCreate = true } @@ -494,3 +486,13 @@ func recursivePath(path string) (string, bool) { } return path, false } + +type watchFlag uint8 + +const ( + // Added by user with Add(), rather than an internal watch. + flagByUser = watchFlag(0x01) + // Part of recursive watch; as the top-level path added by the user or an + // "internal" watch. + flagRecurse = watchFlag(0x02) +) diff --git a/vendor/github.com/fsnotify/fsnotify/internal/darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go index 0b01bc182a1..6721aa60968 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/darwin.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go @@ -15,25 +15,6 @@ var ( var maxfiles uint64 -func SetRlimit() { - // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ - var l syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) - if err == nil && l.Cur != l.Max { - l.Cur = l.Max - syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) - } - maxfiles = l.Cur - - if n, err := syscall.SysctlUint32("kern.maxfiles"); err == nil && uint64(n) < maxfiles { - maxfiles = uint64(n) - } - - if n, err := syscall.SysctlUint32("kern.maxfilesperproc"); err == nil && uint64(n) < maxfiles { - maxfiles = uint64(n) - } -} - func Maxfiles() uint64 { return maxfiles } func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go index 928319fb09a..7600180742b 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go @@ -6,52 +6,10 @@ var names = []struct { n string m uint32 }{ - {"NOTE_ABSOLUTE", unix.NOTE_ABSOLUTE}, {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, - {"NOTE_BACKGROUND", unix.NOTE_BACKGROUND}, - {"NOTE_CHILD", unix.NOTE_CHILD}, - {"NOTE_CRITICAL", unix.NOTE_CRITICAL}, {"NOTE_DELETE", unix.NOTE_DELETE}, - {"NOTE_EXEC", unix.NOTE_EXEC}, - {"NOTE_EXIT", unix.NOTE_EXIT}, - {"NOTE_EXITSTATUS", unix.NOTE_EXITSTATUS}, - {"NOTE_EXIT_CSERROR", unix.NOTE_EXIT_CSERROR}, - {"NOTE_EXIT_DECRYPTFAIL", unix.NOTE_EXIT_DECRYPTFAIL}, - {"NOTE_EXIT_DETAIL", unix.NOTE_EXIT_DETAIL}, - {"NOTE_EXIT_DETAIL_MASK", unix.NOTE_EXIT_DETAIL_MASK}, - {"NOTE_EXIT_MEMORY", unix.NOTE_EXIT_MEMORY}, - {"NOTE_EXIT_REPARENTED", unix.NOTE_EXIT_REPARENTED}, {"NOTE_EXTEND", unix.NOTE_EXTEND}, - {"NOTE_FFAND", unix.NOTE_FFAND}, - {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, - {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, - {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, - {"NOTE_FFNOP", unix.NOTE_FFNOP}, - {"NOTE_FFOR", unix.NOTE_FFOR}, - {"NOTE_FORK", unix.NOTE_FORK}, - {"NOTE_FUNLOCK", unix.NOTE_FUNLOCK}, - {"NOTE_LEEWAY", unix.NOTE_LEEWAY}, {"NOTE_LINK", unix.NOTE_LINK}, - {"NOTE_LOWAT", unix.NOTE_LOWAT}, - {"NOTE_MACHTIME", unix.NOTE_MACHTIME}, - {"NOTE_MACH_CONTINUOUS_TIME", unix.NOTE_MACH_CONTINUOUS_TIME}, - {"NOTE_NONE", unix.NOTE_NONE}, - {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, - {"NOTE_OOB", unix.NOTE_OOB}, - //{"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, -0x100000 (?!) - {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, - {"NOTE_REAP", unix.NOTE_REAP}, {"NOTE_RENAME", unix.NOTE_RENAME}, - {"NOTE_REVOKE", unix.NOTE_REVOKE}, - {"NOTE_SECONDS", unix.NOTE_SECONDS}, - {"NOTE_SIGNAL", unix.NOTE_SIGNAL}, - {"NOTE_TRACK", unix.NOTE_TRACK}, - {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, - {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, - {"NOTE_USECONDS", unix.NOTE_USECONDS}, - {"NOTE_VM_ERROR", unix.NOTE_VM_ERROR}, - {"NOTE_VM_PRESSURE", unix.NOTE_VM_PRESSURE}, - {"NOTE_VM_PRESSURE_SUDDEN_TERMINATE", unix.NOTE_VM_PRESSURE_SUDDEN_TERMINATE}, - {"NOTE_VM_PRESSURE_TERMINATE", unix.NOTE_VM_PRESSURE_TERMINATE}, {"NOTE_WRITE", unix.NOTE_WRITE}, } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go index 3186b0c3491..7600180742b 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go @@ -7,27 +7,9 @@ var names = []struct { m uint32 }{ {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, - {"NOTE_CHILD", unix.NOTE_CHILD}, {"NOTE_DELETE", unix.NOTE_DELETE}, - {"NOTE_EXEC", unix.NOTE_EXEC}, - {"NOTE_EXIT", unix.NOTE_EXIT}, {"NOTE_EXTEND", unix.NOTE_EXTEND}, - {"NOTE_FFAND", unix.NOTE_FFAND}, - {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, - {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, - {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, - {"NOTE_FFNOP", unix.NOTE_FFNOP}, - {"NOTE_FFOR", unix.NOTE_FFOR}, - {"NOTE_FORK", unix.NOTE_FORK}, {"NOTE_LINK", unix.NOTE_LINK}, - {"NOTE_LOWAT", unix.NOTE_LOWAT}, - {"NOTE_OOB", unix.NOTE_OOB}, - {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, - {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, {"NOTE_RENAME", unix.NOTE_RENAME}, - {"NOTE_REVOKE", unix.NOTE_REVOKE}, - {"NOTE_TRACK", unix.NOTE_TRACK}, - {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, - {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, {"NOTE_WRITE", unix.NOTE_WRITE}, } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go index f69fdb930f5..b9e45f55512 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go @@ -6,37 +6,15 @@ var names = []struct { n string m uint32 }{ - {"NOTE_ABSTIME", unix.NOTE_ABSTIME}, - {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, - {"NOTE_CHILD", unix.NOTE_CHILD}, - {"NOTE_CLOSE", unix.NOTE_CLOSE}, - {"NOTE_CLOSE_WRITE", unix.NOTE_CLOSE_WRITE}, {"NOTE_DELETE", unix.NOTE_DELETE}, - {"NOTE_EXEC", unix.NOTE_EXEC}, - {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_WRITE", unix.NOTE_WRITE}, {"NOTE_EXTEND", unix.NOTE_EXTEND}, - {"NOTE_FFAND", unix.NOTE_FFAND}, - {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, - {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, - {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, - {"NOTE_FFNOP", unix.NOTE_FFNOP}, - {"NOTE_FFOR", unix.NOTE_FFOR}, - {"NOTE_FILE_POLL", unix.NOTE_FILE_POLL}, - {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, {"NOTE_LINK", unix.NOTE_LINK}, - {"NOTE_LOWAT", unix.NOTE_LOWAT}, - {"NOTE_MSECONDS", unix.NOTE_MSECONDS}, - {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, - {"NOTE_OPEN", unix.NOTE_OPEN}, - {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, - {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, - {"NOTE_READ", unix.NOTE_READ}, {"NOTE_RENAME", unix.NOTE_RENAME}, {"NOTE_REVOKE", unix.NOTE_REVOKE}, - {"NOTE_SECONDS", unix.NOTE_SECONDS}, - {"NOTE_TRACK", unix.NOTE_TRACK}, - {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, - {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, - {"NOTE_USECONDS", unix.NOTE_USECONDS}, - {"NOTE_WRITE", unix.NOTE_WRITE}, + {"NOTE_OPEN", unix.NOTE_OPEN}, + {"NOTE_CLOSE", unix.NOTE_CLOSE}, + {"NOTE_CLOSE_WRITE", unix.NOTE_CLOSE_WRITE}, + {"NOTE_READ", unix.NOTE_READ}, } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go index 607e683bd73..5d8116436db 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go @@ -27,6 +27,6 @@ func Debug(name string, kevent *unix.Kevent_t) { if unknown > 0 { l = append(l, fmt.Sprintf("0x%x", unknown)) } - fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-60s → %q\n", + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-20s → %q\n", time.Now().Format("15:04:05.000000000"), mask, strings.Join(l, " | "), name) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go index e5b3b6f6943..7600180742b 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go @@ -7,19 +7,9 @@ var names = []struct { m uint32 }{ {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, - {"NOTE_CHILD", unix.NOTE_CHILD}, {"NOTE_DELETE", unix.NOTE_DELETE}, - {"NOTE_EXEC", unix.NOTE_EXEC}, - {"NOTE_EXIT", unix.NOTE_EXIT}, {"NOTE_EXTEND", unix.NOTE_EXTEND}, - {"NOTE_FORK", unix.NOTE_FORK}, {"NOTE_LINK", unix.NOTE_LINK}, - {"NOTE_LOWAT", unix.NOTE_LOWAT}, - {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, - {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, {"NOTE_RENAME", unix.NOTE_RENAME}, - {"NOTE_REVOKE", unix.NOTE_REVOKE}, - {"NOTE_TRACK", unix.NOTE_TRACK}, - {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, {"NOTE_WRITE", unix.NOTE_WRITE}, } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go index 1dd455bc5a4..871766d6997 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go @@ -7,22 +7,10 @@ var names = []struct { m uint32 }{ {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, - // {"NOTE_CHANGE", unix.NOTE_CHANGE}, // Not on 386? - {"NOTE_CHILD", unix.NOTE_CHILD}, {"NOTE_DELETE", unix.NOTE_DELETE}, - {"NOTE_EOF", unix.NOTE_EOF}, - {"NOTE_EXEC", unix.NOTE_EXEC}, - {"NOTE_EXIT", unix.NOTE_EXIT}, {"NOTE_EXTEND", unix.NOTE_EXTEND}, - {"NOTE_FORK", unix.NOTE_FORK}, {"NOTE_LINK", unix.NOTE_LINK}, - {"NOTE_LOWAT", unix.NOTE_LOWAT}, - {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, - {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, {"NOTE_RENAME", unix.NOTE_RENAME}, - {"NOTE_REVOKE", unix.NOTE_REVOKE}, - {"NOTE_TRACK", unix.NOTE_TRACK}, - {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, {"NOTE_TRUNCATE", unix.NOTE_TRUNCATE}, {"NOTE_WRITE", unix.NOTE_WRITE}, } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go index 5ac8b507978..758a2490525 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go @@ -15,17 +15,6 @@ var ( var maxfiles uint64 -func SetRlimit() { - // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ - var l syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) - if err == nil && l.Cur != l.Max { - l.Cur = l.Max - syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) - } - maxfiles = uint64(l.Cur) -} - func Maxfiles() uint64 { return maxfiles } func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, uint64(dev)) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix.go b/vendor/github.com/fsnotify/fsnotify/internal/unix.go index b251fb80386..9c66f5d30db 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/unix.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix.go @@ -15,17 +15,6 @@ var ( var maxfiles uint64 -func SetRlimit() { - // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ - var l syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) - if err == nil && l.Cur != l.Max { - l.Cur = l.Max - syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) - } - maxfiles = uint64(l.Cur) -} - func Maxfiles() uint64 { return maxfiles } func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix2.go b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go index 37dfeddc289..b2d89592f7c 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/unix2.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go @@ -2,6 +2,24 @@ package internal +import "syscall" + func HasPrivilegesForSymlink() bool { return true } + +// IgnoringEINTR makes a function call and repeats it if it returns an +// EINTR error. This appears to be required even though we install all +// signal handlers with SA_RESTART: see #22838, #38033, #38836, #40846. +// Also #20400 and #36644 are issues in which a signal handler is +// installed without setting SA_RESTART. None of these are the common case, +// but there are enough of them that it seems that we can't avoid +// an EINTR loop. +func IgnoringEINTR[T any](fn func() (T, error)) (T, error) { + for { + v, err := fn() + if err != syscall.EINTR { + return v, err + } + } +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/windows.go b/vendor/github.com/fsnotify/fsnotify/internal/windows.go index 896bc2e5a2f..e24d5692641 100644 --- a/vendor/github.com/fsnotify/fsnotify/internal/windows.go +++ b/vendor/github.com/fsnotify/fsnotify/internal/windows.go @@ -14,7 +14,6 @@ var ( ErrUnixEACCES = errors.New("dummy") ) -func SetRlimit() {} func Maxfiles() uint64 { return 1<<64 - 1 } func Mkfifo(path string, mode uint32) error { return errors.New("no FIFOs on Windows") } func Mknod(path string, mode uint32, dev int) error { return errors.New("no device nodes on Windows") } diff --git a/vendor/github.com/go-openapi/runtime/CONTRIBUTORS.md b/vendor/github.com/go-openapi/runtime/CONTRIBUTORS.md new file mode 100644 index 00000000000..541fd575e01 --- /dev/null +++ b/vendor/github.com/go-openapi/runtime/CONTRIBUTORS.md @@ -0,0 +1,81 @@ +# Contributors + +- Repository: ['go-openapi/runtime'] + +| Total Contributors | Total Contributions | +| --- | --- | +| 69 | 490 | + +| Username | All Time Contribution Count | All Commits | +| --- | --- | --- | +| @casualjim | 268 | | +| @fredbi | 69 | | +| @youyuanwu | 19 | | +| @josephwoodward | 13 | | +| @kenjones-cisco | 12 | | +| @GlenDC | 7 | | +| @elakito | 6 | | +| @moenning | 6 | | +| @mstoykov | 6 | | +| @ifraixedes | 5 | | +| @zeitlinger | 4 | | +| @jkawamoto | 3 | | +| @stoyanr | 3 | | +| @keramix | 2 | | +| @Equanox | 2 | | +| @ederavilaprado | 2 | | +| @nan0tube | 2 | | +| @thomdixon | 2 | | +| @deborggraever | 2 | | +| @MakarandNsd | 2 | | +| @Vadskye | 2 | | +| @jsilland | 2 | | +| @Kunde21 | 2 | | +| @bcomnes | 2 | | +| @galaxie | 2 | | +| @anfernee | 2 | | +| @wahabmk | 1 | | +| @vearutop | 1 | | +| @tschaub | 1 | | +| @pytlesk4 | 1 | | +| @tgraf | 1 | | +| @seanprince | 1 | | +| @rodriguise | 1 | | +| @petrkotas | 1 | | +| @maxatome | 1 | | +| @maxkarelov | 1 | | +| @aleksandr-vin | 1 | | +| @akutz | 1 | | +| @yabberyabber | 1 | | +| @elv-gilles | 1 | | +| @gregmarr | 1 | | +| @jwalter1-quest | 1 | | +| @s4s7 | 1 | | +| @stingshen | 1 | | +| @tamalsaha | 1 | | +| @tte | 1 | | +| @martian4202 | 1 | | +| @yan-zhuang | 1 | | +| @azylman | 1 | | +| @anasmuhmd | 1 | | +| @ArFe | 1 | | +| @CodeLingoBot | 1 | | +| @dlmiddlecote | 1 | | +| @danny-cheung | 1 | | +| @calavera | 1 | | +| @EdwardBetts | 1 | | +| @etsangsplk | 1 | | +| @ericzsplk | 1 | | +| @faguirre1 | 1 | | +| @florindragos | 1 | | +| @gbjk | 1 | | +| @taisho6339 | 1 | | +| @jbowes | 1 | | +| @JoakimSoderberg | 1 | | +| @robbert229 | 1 | | +| @jonathaningram | 1 | | +| @germanhs | 1 | | +| @pracucci | 1 | | +| @tooolbox | 1 | | + + _this file was generated by the [Contributors GitHub Action](https://github.com/github-community-projects/contributors)_ diff --git a/vendor/github.com/go-openapi/runtime/README.md b/vendor/github.com/go-openapi/runtime/README.md index dd7f5039a79..fa749062b34 100644 --- a/vendor/github.com/go-openapi/runtime/README.md +++ b/vendor/github.com/go-openapi/runtime/README.md @@ -101,7 +101,7 @@ Maintainers can cut a new release by either: [slack-badge]: https://img.shields.io/badge/slack-blue?link=https%3A%2F%2Fgoswagger.slack.com%2Farchives%2FC04R30YM [slack-url]: https://goswagger.slack.com/archives/C04R30YMU [discord-badge]: https://img.shields.io/discord/1446918742398341256?logo=discord&label=discord&color=blue -[discord-url]: https://discord.gg/twZ9BwT3 +[discord-url]: https://discord.gg/FfnFYaC3k5 [license-badge]: http://img.shields.io/badge/license-Apache%20v2-orange.svg diff --git a/vendor/github.com/go-openapi/runtime/go.work b/vendor/github.com/go-openapi/runtime/go.work index b4cd9e01e86..efa29dcac20 100644 --- a/vendor/github.com/go-openapi/runtime/go.work +++ b/vendor/github.com/go-openapi/runtime/go.work @@ -3,4 +3,4 @@ use ( ./client-middleware/opentracing ) -go 1.24.0 +go 1.25.0 diff --git a/vendor/github.com/go-openapi/runtime/go.work.sum b/vendor/github.com/go-openapi/runtime/go.work.sum index b24a8cfaf94..eaf8fddf6b9 100644 --- a/vendor/github.com/go-openapi/runtime/go.work.sum +++ b/vendor/github.com/go-openapi/runtime/go.work.sum @@ -1,3 +1,4 @@ +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/go-openapi/errors v0.22.2/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= github.com/go-openapi/jsonpointer v0.22.0/go.mod h1:xt3jV88UtExdIkkL7NloURjRQjbeUgcxFblMjq2iaiU= @@ -35,6 +36,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -53,6 +55,8 @@ golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= @@ -60,6 +64,8 @@ golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -88,6 +94,8 @@ golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -104,6 +112,8 @@ golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/vendor/github.com/go-openapi/swag/.gitignore b/vendor/github.com/go-openapi/swag/.gitignore index a0a95a96b3f..1680db44c01 100644 --- a/vendor/github.com/go-openapi/swag/.gitignore +++ b/vendor/github.com/go-openapi/swag/.gitignore @@ -4,4 +4,3 @@ Godeps .idea *.out .mcp.json -.claude/ diff --git a/vendor/github.com/go-openapi/swag/CONTRIBUTORS.md b/vendor/github.com/go-openapi/swag/CONTRIBUTORS.md index bc76fe820c0..286878acff9 100644 --- a/vendor/github.com/go-openapi/swag/CONTRIBUTORS.md +++ b/vendor/github.com/go-openapi/swag/CONTRIBUTORS.md @@ -4,11 +4,11 @@ | Total Contributors | Total Contributions | | --- | --- | -| 24 | 235 | +| 24 | 242 | | Username | All Time Contribution Count | All Commits | | --- | --- | --- | -| @fredbi | 105 | | +| @fredbi | 112 | | | @casualjim | 98 | | | @alexandear | 4 | | | @orisano | 3 | | @@ -33,4 +33,4 @@ | @davidalpert | 1 | | | @Xe | 1 | | - _this file was generated by the [Contributors GitHub Action](https://github.com/github/contributors)_ + _this file was generated by the [Contributors GitHub Action](https://github.com/github-community-projects/contributors)_ diff --git a/vendor/github.com/go-openapi/swag/README.md b/vendor/github.com/go-openapi/swag/README.md index 834eb2ffb9c..64f66710397 100644 --- a/vendor/github.com/go-openapi/swag/README.md +++ b/vendor/github.com/go-openapi/swag/README.md @@ -212,7 +212,7 @@ Maintainers can cut a new release by either: [slack-badge]: https://img.shields.io/badge/slack-blue?link=https%3A%2F%2Fgoswagger.slack.com%2Farchives%2FC04R30YM [slack-url]: https://goswagger.slack.com/archives/C04R30YMU [discord-badge]: https://img.shields.io/discord/1446918742398341256?logo=discord&label=discord&color=blue -[discord-url]: https://discord.gg/twZ9BwT3 +[discord-url]: https://discord.gg/FfnFYaC3k5 [license-badge]: http://img.shields.io/badge/license-Apache%20v2-orange.svg diff --git a/vendor/github.com/go-openapi/swag/SECURITY.md b/vendor/github.com/go-openapi/swag/SECURITY.md index 72296a83135..1fea2c5736a 100644 --- a/vendor/github.com/go-openapi/swag/SECURITY.md +++ b/vendor/github.com/go-openapi/swag/SECURITY.md @@ -6,14 +6,32 @@ This policy outlines the commitment and practices of the go-openapi maintainers | Version | Supported | | ------- | ------------------ | -| 0.25.x | :white_check_mark: | +| O.x | :white_check_mark: | + +## Vulnerability checks in place + +This repository uses automated vulnerability scans, at every merged commit and at least once a week. + +We use: + +* [`GitHub CodeQL`][codeql-url] +* [`trivy`][trivy-url] +* [`govulncheck`][govulncheck-url] + +Reports are centralized in github security reports and visible only to the maintainers. ## Reporting a vulnerability If you become aware of a security vulnerability that affects the current repository, -please report it privately to the maintainers. +**please report it privately to the maintainers** +rather than opening a publicly visible GitHub issue. + +Please follow the instructions provided by github to [Privately report a security vulnerability][github-guidance-url]. -Please follow the instructions provided by github to -[Privately report a security vulnerability](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability#privately-reporting-a-security-vulnerability). +> [!NOTE] +> On Github, navigate to the project's "Security" tab then click on "Report a vulnerability". -TL;DR: on Github, navigate to the project's "Security" tab then click on "Report a vulnerability". +[codeql-url]: https://github.com/github/codeql +[trivy-url]: https://trivy.dev/docs/latest/getting-started +[govulncheck-url]: https://go.dev/blog/govulncheck +[github-guidance-url]: https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability#privately-reporting-a-security-vulnerability diff --git a/vendor/github.com/go-openapi/swag/go.work b/vendor/github.com/go-openapi/swag/go.work index 1e537f0749b..8537cb2a76a 100644 --- a/vendor/github.com/go-openapi/swag/go.work +++ b/vendor/github.com/go-openapi/swag/go.work @@ -17,4 +17,4 @@ use ( ./yamlutils ) -go 1.24.0 +go 1.25.0 diff --git a/vendor/github.com/go-openapi/swag/jsonname/go_name_provider.go b/vendor/github.com/go-openapi/swag/jsonname/go_name_provider.go new file mode 100644 index 00000000000..adc4426873c --- /dev/null +++ b/vendor/github.com/go-openapi/swag/jsonname/go_name_provider.go @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 + +package jsonname + +import ( + "reflect" + "strings" + "sync" +) + +var _ providerIface = (*GoNameProvider)(nil) + +// GoNameProvider resolves json property names to go struct field names following +// the same rules as the standard library's [encoding/json] package. +// +// Contrary to [NameProvider], it considers exported fields without a json tag, +// and promotes fields from anonymous embedded struct types. +// +// Rules (aligned with encoding/json): +// +// - unexported fields are ignored; +// - a field tagged `json:"-"` is ignored; +// - a field tagged `json:"-,"` is kept under the json name "-" (stdlib quirk); +// - a field tagged `json:""` or with no json tag at all keeps its Go name as json name; +// - anonymous struct fields without an explicit json tag have their fields +// promoted into the parent, following breadth-first depth rules: +// a shallower field wins over a deeper one; at equal depth, a conflict +// discards all conflicting fields unless exactly one has an explicit json tag. +// +// This type is safe for concurrent use. +type GoNameProvider struct { + lock sync.Mutex + index map[reflect.Type]nameIndex +} + +// NewGoNameProvider creates a new [GoNameProvider]. +func NewGoNameProvider() *GoNameProvider { + return &GoNameProvider{ + index: make(map[reflect.Type]nameIndex), + } +} + +// GetJSONNames gets all the json property names for a type. +func (n *GoNameProvider) GetJSONNames(subject any) []string { + n.lock.Lock() + defer n.lock.Unlock() + + tpe := reflect.Indirect(reflect.ValueOf(subject)).Type() + names := n.nameIndexFor(tpe) + + res := make([]string, 0, len(names.jsonNames)) + for k := range names.jsonNames { + res = append(res, k) + } + + return res +} + +// GetJSONName gets the json name for a go property name. +func (n *GoNameProvider) GetJSONName(subject any, name string) (string, bool) { + tpe := reflect.Indirect(reflect.ValueOf(subject)).Type() + + return n.GetJSONNameForType(tpe, name) +} + +// GetJSONNameForType gets the json name for a go property name on a given type. +func (n *GoNameProvider) GetJSONNameForType(tpe reflect.Type, name string) (string, bool) { + n.lock.Lock() + defer n.lock.Unlock() + + names := n.nameIndexFor(tpe) + nme, ok := names.goNames[name] + + return nme, ok +} + +// GetGoName gets the go name for a json property name. +func (n *GoNameProvider) GetGoName(subject any, name string) (string, bool) { + tpe := reflect.Indirect(reflect.ValueOf(subject)).Type() + + return n.GetGoNameForType(tpe, name) +} + +// GetGoNameForType gets the go name for a given type for a json property name. +func (n *GoNameProvider) GetGoNameForType(tpe reflect.Type, name string) (string, bool) { + n.lock.Lock() + defer n.lock.Unlock() + + names := n.nameIndexFor(tpe) + nme, ok := names.jsonNames[name] + + return nme, ok +} + +func (n *GoNameProvider) nameIndexFor(tpe reflect.Type) nameIndex { + if names, ok := n.index[tpe]; ok { + return names + } + + names := buildGoNameIndex(tpe) + n.index[tpe] = names + + return names +} + +// fieldEntry captures a candidate field discovered while walking a struct +// along with the indirection path from the root type (used to resolve conflicts +// by depth in the same way encoding/json does). +type fieldEntry struct { + goName string + jsonName string + index []int + tagged bool +} + +func buildGoNameIndex(tpe reflect.Type) nameIndex { + fields := collectGoFields(tpe) + + idx := make(map[string]string, len(fields)) + reverseIdx := make(map[string]string, len(fields)) + for _, f := range fields { + idx[f.jsonName] = f.goName + reverseIdx[f.goName] = f.jsonName + } + + return nameIndex{jsonNames: idx, goNames: reverseIdx} +} + +// collectGoFields walks tpe breadth-first along anonymous struct fields, +// reproducing the field selection performed by encoding/json.typeFields. +func collectGoFields(tpe reflect.Type) []fieldEntry { + if tpe.Kind() != reflect.Struct { + return nil + } + + type queued struct { + typ reflect.Type + index []int + } + + current := []queued{} + next := []queued{{typ: tpe}} + visited := map[reflect.Type]bool{tpe: true} + + var ( + candidates []fieldEntry + count = map[string]int{} + nextCount = map[string]int{} + ) + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, count + for k := range nextCount { + delete(nextCount, k) + } + + for _, q := range current { + for i := 0; i < q.typ.NumField(); i++ { + sf := q.typ.Field(i) + + if sf.Anonymous { + ft := sf.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + if !sf.IsExported() && ft.Kind() != reflect.Struct { + continue + } + } else if !sf.IsExported() { + continue + } + + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + jsonName, _ := parseJSONTag(tag) + tagged := jsonName != "" + + ft := sf.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + + if sf.Anonymous && ft.Kind() == reflect.Struct && !tagged { + if visited[ft] { + continue + } + visited[ft] = true + + index := make([]int, len(q.index)+1) + copy(index, q.index) + index[len(q.index)] = i + next = append(next, queued{typ: ft, index: index}) + + continue + } + + name := jsonName + if name == "" { + name = sf.Name + } + + index := make([]int, len(q.index)+1) + copy(index, q.index) + index[len(q.index)] = i + + candidates = append(candidates, fieldEntry{ + goName: sf.Name, + jsonName: name, + index: index, + tagged: tagged, + }) + nextCount[name]++ + } + } + } + + return dominantFields(candidates) +} + +// dominantFields applies the Go encoding/json conflict resolution rules: +// at each JSON name, the shallowest field wins; at equal depth, a uniquely +// tagged candidate wins; otherwise all candidates for that name are dropped. +func dominantFields(candidates []fieldEntry) []fieldEntry { + byName := make(map[string][]fieldEntry, len(candidates)) + for _, c := range candidates { + byName[c.jsonName] = append(byName[c.jsonName], c) + } + + out := make([]fieldEntry, 0, len(byName)) + for _, group := range byName { + if len(group) == 1 { + out = append(out, group[0]) + + continue + } + + minDepth := len(group[0].index) + for _, c := range group[1:] { + if len(c.index) < minDepth { + minDepth = len(c.index) + } + } + + var shallow []fieldEntry + for _, c := range group { + if len(c.index) == minDepth { + shallow = append(shallow, c) + } + } + + if len(shallow) == 1 { + out = append(out, shallow[0]) + + continue + } + + var tagged []fieldEntry + for _, c := range shallow { + if c.tagged { + tagged = append(tagged, c) + } + } + if len(tagged) == 1 { + out = append(out, tagged[0]) + } + } + + return out +} + +// parseJSONTag returns the name component of a json struct tag and whether +// it carried any non-name option (kept for future-proofing, e.g. "omitempty"). +func parseJSONTag(tag string) (string, string) { + if tag == "" { + return "", "" + } + if idx := strings.IndexByte(tag, ','); idx >= 0 { + return tag[:idx], tag[idx+1:] + } + + return tag, "" +} diff --git a/vendor/github.com/go-openapi/swag/jsonname/ifaces.go b/vendor/github.com/go-openapi/swag/jsonname/ifaces.go new file mode 100644 index 00000000000..812ace5639f --- /dev/null +++ b/vendor/github.com/go-openapi/swag/jsonname/ifaces.go @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 + +package jsonname + +import "reflect" + +// providerIface is an unexported compile-time contract that every name provider +// in this package is expected to satisfy. +// It mirrors the interface declared by the main consumer of this module: [github.com/go-openapi/jsonpointer.NameProvider]. +type providerIface interface { + GetGoName(subject any, name string) (string, bool) + GetGoNameForType(tpe reflect.Type, name string) (string, bool) +} diff --git a/vendor/github.com/go-openapi/swag/jsonname/name_provider.go b/vendor/github.com/go-openapi/swag/jsonname/name_provider.go index 8eaf1bece8d..9f5da7a0165 100644 --- a/vendor/github.com/go-openapi/swag/jsonname/name_provider.go +++ b/vendor/github.com/go-openapi/swag/jsonname/name_provider.go @@ -12,6 +12,8 @@ import ( // DefaultJSONNameProvider is the default cache for types. var DefaultJSONNameProvider = NewNameProvider() +var _ providerIface = (*NameProvider)(nil) + // NameProvider represents an object capable of translating from go property names // to json property names. // diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go index 138083d9479..0956f34f246 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go @@ -1,7 +1,10 @@ // Package lz4block provides LZ4 BlockSize types and pools of buffers. package lz4block -import "sync" +import ( + "fmt" + "sync" +) const ( Block64Kb uint32 = 1 << (16 + iota*2) @@ -12,11 +15,11 @@ const ( ) var ( - BlockPool64K = sync.Pool{New: func() interface{} { return make([]byte, Block64Kb) }} - BlockPool256K = sync.Pool{New: func() interface{} { return make([]byte, Block256Kb) }} - BlockPool1M = sync.Pool{New: func() interface{} { return make([]byte, Block1Mb) }} - BlockPool4M = sync.Pool{New: func() interface{} { return make([]byte, Block4Mb) }} - BlockPool8M = sync.Pool{New: func() interface{} { return make([]byte, Block8Mb) }} + blockPool64K = sync.Pool{New: func() interface{} { return &[Block64Kb]byte{} }} + blockPool256K = sync.Pool{New: func() interface{} { return &[Block256Kb]byte{} }} + blockPool1M = sync.Pool{New: func() interface{} { return &[Block1Mb]byte{} }} + blockPool4M = sync.Pool{New: func() interface{} { return &[Block4Mb]byte{} }} + blockPool8M = sync.Pool{New: func() interface{} { return &[Block8Mb]byte{} }} ) func Index(b uint32) BlockSizeIndex { @@ -50,35 +53,39 @@ func (b BlockSizeIndex) IsValid() bool { } func (b BlockSizeIndex) Get() []byte { - var buf interface{} switch b { case 4: - buf = BlockPool64K.Get() + return blockPool64K.Get().(*[Block64Kb]byte)[:] case 5: - buf = BlockPool256K.Get() + return blockPool256K.Get().(*[Block256Kb]byte)[:] case 6: - buf = BlockPool1M.Get() + return blockPool1M.Get().(*[Block1Mb]byte)[:] case 7: - buf = BlockPool4M.Get() + return blockPool4M.Get().(*[Block4Mb]byte)[:] case 3: - buf = BlockPool8M.Get() + return blockPool8M.Get().(*[Block8Mb]byte)[:] + default: + panic(fmt.Errorf("invalid block index %d", b)) } - return buf.([]byte) } func Put(buf []byte) { // Safeguard: do not allow invalid buffers. switch c := cap(buf); uint32(c) { case Block64Kb: - BlockPool64K.Put(buf[:c]) + blockPool64K.Put((*[Block64Kb]byte)(buf[:c])) case Block256Kb: - BlockPool256K.Put(buf[:c]) + blockPool256K.Put((*[Block256Kb]byte)(buf[:c])) case Block1Mb: - BlockPool1M.Put(buf[:c]) + blockPool1M.Put((*[Block1Mb]byte)(buf[:c])) case Block4Mb: - BlockPool4M.Put(buf[:c]) + blockPool4M.Put((*[Block4Mb]byte)(buf[:c])) case Block8Mb: - BlockPool8M.Put(buf[:c]) + blockPool8M.Put((*[Block8Mb]byte)(buf[:c])) + case 0: + // Allow "returning" an empty buffer. + default: + panic(fmt.Errorf("invalid block size %d", c)) } } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go index 9f568fbb1af..e1d36058522 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go @@ -32,33 +32,7 @@ func decodeBlock(dst, src, dict []byte) (ret int) { // Literals. if lLen := b >> 4; lLen > 0 { - switch { - case lLen < 0xF && si+16 < uint(len(src)): - // Shortcut 1 - // if we have enough room in src and dst, and the literals length - // is small enough (0..14) then copy all 16 bytes, even if not all - // are part of the literals. - copy(dst[di:], src[si:si+16]) - si += lLen - di += lLen - if mLen := b & 0xF; mLen < 0xF { - // Shortcut 2 - // if the match length (4..18) fits within the literals, then copy - // all 18 bytes, even if not all are part of the literals. - mLen += 4 - if offset := u16(src[si:]); mLen <= offset && offset < di { - i := di - offset - // The remaining buffer may not hold 18 bytes. - // See https://github.com/pierrec/lz4/issues/51. - if end := i + 18; end <= uint(len(dst)) { - copy(dst[di:], dst[i:end]) - si += 2 - di += mLen - continue - } - } - } - case lLen == 0xF: + if lLen == 0xF { for { x := uint(src[si]) if lLen += x; int(lLen) < 0 { @@ -69,20 +43,27 @@ func decodeBlock(dst, src, dict []byte) (ret int) { break } } - fallthrough - default: + } + if lLen <= 16 && si+16 < uint(len(src)) { + // Shortcut 1: if we have enough room in src and dst, and the + // literal length is at most 16, then copy 16 bytes, even if not + // all are part of the literal. The compiler inlines this copy. + copy(dst[di:di+16], src[si:si+16]) + } else { copy(dst[di:di+lLen], src[si:si+lLen]) - si += lLen - di += lLen } + si += lLen + di += lLen } + // Match. mLen := b & 0xF if si == uint(len(src)) && mLen == 0 { break } else if si >= uint(len(src)) { return hasError } + mLen += minMatch offset := u16(src[si:]) if offset == 0 { @@ -90,9 +71,17 @@ func decodeBlock(dst, src, dict []byte) (ret int) { } si += 2 - // Match. - mLen += minMatch - if mLen == minMatch+0xF { + if mLen <= 16 { + // Shortcut 2: if the match length is at most 16 and we're far + // enough from the end of dst, copy 16 bytes unconditionally + // so that the compiler can inline the copy. + if mLen <= offset && offset < di && di+16 <= uint(len(dst)) { + i := di - offset + copy(dst[di:di+16], dst[i:i+16]) + di += mLen + continue + } + } else if mLen >= 15+minMatch { for { x := uint(src[si]) if mLen += x; int(mLen) < 0 { diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go index 710ea42812e..be116515217 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go @@ -16,4 +16,5 @@ const ( ErrOptionInvalidBlockSize Error = "lz4: invalid block size" ErrOptionNotApplicable Error = "lz4: option not applicable" ErrWriterNotClosed Error = "lz4: writer not closed" + ErrEndOfStream Error = "lz4: end of stream reached" ) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go index 04aaca8480d..52531dfeae1 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go @@ -143,7 +143,7 @@ func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { c <- nil // signal the collection loop that we are done <-c // wait for the collect loop to complete if f.isLegacy() && cum == cumx { - err = io.EOF + err = lz4errors.ErrEndOfStream } b.closeR(err) close(data) @@ -290,11 +290,11 @@ func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, erro // Only works in non concurrent mode, for concurrent mode // it is handled separately. // Linux kernel format appends the total uncompressed size at the end. - return 0, io.EOF + return 0, lz4errors.ErrEndOfStream } } else if x == 0 { // Marker for end of stream. - return 0, io.EOF + return 0, lz4errors.ErrEndOfStream } b.Size = DataBlockSize(x) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go index 18192a9433d..93c1f44b1b7 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" "io" - "io/ioutil" "github.com/pierrec/lz4/v4/internal/lz4block" "github.com/pierrec/lz4/v4/internal/lz4errors" @@ -96,7 +95,7 @@ newFrame: if err != nil { return err } - if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { + if _, err := io.CopyN(io.Discard, src, int64(skip)); err != nil { return err } goto newFrame diff --git a/vendor/github.com/pierrec/lz4/v4/reader.go b/vendor/github.com/pierrec/lz4/v4/reader.go index 275daad7cb9..094f4304790 100644 --- a/vendor/github.com/pierrec/lz4/v4/reader.go +++ b/vendor/github.com/pierrec/lz4/v4/reader.go @@ -63,7 +63,7 @@ func (r *Reader) Apply(options ...Option) (err error) { return } -// Size returns the size of the underlying uncompressed data, if set in the stream. +// Size returns the size of the current frame's uncompressed data, if set in the stream. func (r *Reader) Size() int { switch r.state.state { case readState, closedState: @@ -117,6 +117,7 @@ func (r *Reader) Read(buf []byte) (n int, err error) { for len(buf) > 0 { var bn int if r.idx == 0 { + read: if r.isNotConcurrent() { bn, err = r.read(buf) } else { @@ -129,12 +130,29 @@ func (r *Reader) Read(buf []byte) (n int, err error) { } switch err { case nil: + case lz4errors.ErrEndOfStream: + + // Read Checksum. + err = r.frame.CloseR(r.src) + if err != nil { + return + } + + //Check for new stream. + r.Reset(r.src) + if err = r.init(); r.state.next(err) { + return + } + + goto read case io.EOF: if er := r.frame.CloseR(r.src); er != nil { err = er } lz4block.Put(r.data) r.data = nil + // reset frame to release the buffer held by Block + r.frame.Reset(r.num) return default: return @@ -157,9 +175,9 @@ func (r *Reader) Read(buf []byte) (n int, err error) { } // read uncompresses the next block as follow: -// - if buf has enough room, the block is uncompressed into it directly -// and the lenght of used space is returned -// - else, the uncompress data is stored in r.data and 0 is returned +// - if buf has enough room, the block is uncompressed into it directly +// and the lenght of used space is returned +// - else, the uncompress data is stored in r.data and 0 is returned func (r *Reader) read(buf []byte) (int, error) { block := r.frame.Blocks.Block _, err := block.Read(r.frame, r.src, r.cum) @@ -189,6 +207,7 @@ func (r *Reader) read(buf []byte) (int, error) { } r.cum += uint32(len(dst)) if direct { + r.data = r.data[:0] return len(dst), nil } r.data = dst @@ -199,10 +218,8 @@ func (r *Reader) read(buf []byte) (int, error) { // initial state from NewReader, but instead reading from reader. // No access to reader is performed. func (r *Reader) Reset(reader io.Reader) { - if r.data != nil { - lz4block.Put(r.data) - r.data = nil - } + lz4block.Put(r.data) + r.data = nil r.frame.Reset(r.num) r.state.reset() r.src = reader @@ -212,6 +229,7 @@ func (r *Reader) Reset(reader io.Reader) { // WriteTo efficiently uncompresses the data from the Reader underlying source to w. func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { switch r.state.state { + case readState: case closedState, errorState: return 0, r.state.err case newState: @@ -232,6 +250,7 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { for { var bn int var dst []byte + read: if r.isNotConcurrent() { bn, err = r.read(data) dst = data[:bn] @@ -246,6 +265,21 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { } switch err { case nil: + case lz4errors.ErrEndOfStream: + // Read Checksum. + err = r.frame.CloseR(r.src) + if err != nil { + return + } + // Check for a new stream. + r.Reset(r.src) + if err = r.init(); r.state.next(err) { + if err == io.EOF { + err = nil + } + return + } + goto read case io.EOF: err = r.frame.CloseR(r.src) return diff --git a/vendor/github.com/pierrec/lz4/v4/state.go b/vendor/github.com/pierrec/lz4/v4/state.go index d94f04d05eb..532316eeca3 100644 --- a/vendor/github.com/pierrec/lz4/v4/state.go +++ b/vendor/github.com/pierrec/lz4/v4/state.go @@ -38,11 +38,15 @@ func (s *_State) reset() { s.err = nil } -// next sets the state to the next one unless it is passed a non nil error. -// It returns whether or not it is in error. +// next sets the state to the next one unless it is passed a non-nil error. +// It returns whether it is in error. func (s *_State) next(err error) bool { if err != nil { - s.err = fmt.Errorf("%s: %w", s.state, err) + if errors.Is(err, io.EOF) { + s.err = io.EOF + } else { + s.err = fmt.Errorf("%s: %w", s.state, err) + } s.state = errorState return true } @@ -61,15 +65,17 @@ func (s *_State) check(errp *error) { return } if err := *errp; err != nil { - s.err = fmt.Errorf("%w[%s]", err, s.state) - if !errors.Is(err, io.EOF) { - s.state = errorState + if errors.Is(err, io.EOF) { + s.err = io.EOF + } else { + s.err = fmt.Errorf("%w[%s]", err, s.state) } + s.state = errorState } } func (s *_State) fail() error { - s.state = errorState s.err = fmt.Errorf("%w[%s]", lz4errors.ErrInternalUnhandledState, s.state) + s.state = errorState return s.err } diff --git a/vendor/github.com/pierrec/lz4/v4/writer.go b/vendor/github.com/pierrec/lz4/v4/writer.go index 4358adee109..5637acde473 100644 --- a/vendor/github.com/pierrec/lz4/v4/writer.go +++ b/vendor/github.com/pierrec/lz4/v4/writer.go @@ -176,10 +176,8 @@ func (w *Writer) Close() error { } err := w.frame.CloseW(w.src, w.num) // It is now safe to free the buffer. - if w.data != nil { - lz4block.Put(w.data) - w.data = nil - } + lz4block.Put(w.data) + w.data = nil return err } @@ -190,6 +188,8 @@ func (w *Writer) Close() error { // // w.Close must be called before Reset or pending data may be dropped. func (w *Writer) Reset(writer io.Writer) { + lz4block.Put(w.data) + w.data = nil w.frame.Reset(w.num) w.state.reset() w.src = writer @@ -227,11 +227,13 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { return } n += int64(rn) - err = w.write(data[:rn], true) - if err != nil { - return + if rn > 0 { + err = w.write(data[:rn], true) + if err != nil { + return + } + w.handler(rn) } - w.handler(rn) if !done && !w.isNotConcurrent() { // The buffer will be returned automatically by go routines (safe=true) // so get a new one fo the next round. diff --git a/vendor/github.com/prometheus/alertmanager/alert/state.go b/vendor/github.com/prometheus/alertmanager/alert/state.go index b6cd363f471..02dd7881771 100644 --- a/vendor/github.com/prometheus/alertmanager/alert/state.go +++ b/vendor/github.com/prometheus/alertmanager/alert/state.go @@ -22,3 +22,26 @@ const ( AlertStateActive AlertState = "active" AlertStateSuppressed AlertState = "suppressed" ) + +// Compare returns -1 if s has lower priority than other, 0 if equal, +// and 1 if higher. Priority order: suppressed > active > unprocessed. +func (s AlertState) Compare(other AlertState) int { + p := func(st AlertState) int { + switch st { + case AlertStateSuppressed: + return 2 + case AlertStateActive: + return 1 + default: + return 0 + } + } + switch a, b := p(s), p(other); { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} diff --git a/vendor/github.com/prometheus/alertmanager/api/api.go b/vendor/github.com/prometheus/alertmanager/api/api.go index 2bb41095220..d7c10f66627 100644 --- a/vendor/github.com/prometheus/alertmanager/api/api.go +++ b/vendor/github.com/prometheus/alertmanager/api/api.go @@ -59,9 +59,6 @@ type Options struct { Alerts provider.Alerts // Silences to be used by the API. Mandatory. Silences *silence.Silences - // AlertStatusFunc is used be the API to retrieve the AlertStatus of an - // alert. Mandatory. - AlertStatusFunc func(model.Fingerprint) types.AlertStatus // GroupMutedFunc is used be the API to know if an alert is muted. // Mandatory. GroupMutedFunc func(routeID, groupKey string) ([]string, bool) @@ -95,9 +92,6 @@ func (o Options) validate() error { if o.Silences == nil { return errors.New("mandatory field Silences not set") } - if o.AlertStatusFunc == nil { - return errors.New("mandatory field AlertStatusFunc not set") - } if o.GroupMutedFunc == nil { return errors.New("mandatory field GroupMutedFunc not set") } @@ -125,7 +119,6 @@ func New(opts Options) (*API, error) { v2, err := apiv2.NewAPI( opts.Alerts, opts.GroupFunc, - opts.AlertStatusFunc, opts.GroupMutedFunc, opts.Silences, opts.Peer, diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/api.go b/vendor/github.com/prometheus/alertmanager/api/v2/api.go index feb9cd5b3d3..5088109333a 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/api.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/api.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "net/http" "regexp" "slices" @@ -35,6 +36,7 @@ import ( "github.com/rs/cors" "go.opentelemetry.io/otel/codes" + "github.com/prometheus/alertmanager/alert" "github.com/prometheus/alertmanager/api/metrics" open_api_models "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/api/v2/restapi" @@ -47,13 +49,14 @@ import ( "github.com/prometheus/alertmanager/cluster" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/dispatch" + "github.com/prometheus/alertmanager/eventrecorder" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/matcher/compat" "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/alertmanager/provider" "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/alertmanager/tracing" - "github.com/prometheus/alertmanager/types" ) var tracer = tracing.NewTracer("github.com/prometheus/alertmanager/api/v2") @@ -64,7 +67,6 @@ type API struct { silences *silence.Silences alerts provider.Alerts alertGroups groupsFn - getAlertStatus getAlertStatusFn groupMutedFunc groupMutedFunc uptime time.Time @@ -83,9 +85,8 @@ type API struct { } type ( - groupsFn func(context.Context, func(*dispatch.Route) bool, func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[prometheus_model.Fingerprint][]string, error) + groupsFn func(context.Context, func(*dispatch.Route) bool, func(*alert.Alert, time.Time) bool) (dispatch.AlertGroups, map[prometheus_model.Fingerprint][]string, error) groupMutedFunc func(routeID, groupKey string) ([]string, bool) - getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus setAlertStatusFn func(ctx context.Context, labels prometheus_model.LabelSet) ) @@ -93,7 +94,6 @@ type ( func NewAPI( alerts provider.Alerts, gf groupsFn, - asf getAlertStatusFn, gmf groupMutedFunc, silences *silence.Silences, peer cluster.ClusterPeer, @@ -102,7 +102,6 @@ func NewAPI( ) (*API, error) { api := API{ alerts: alerts, - getAlertStatus: asf, alertGroups: gf, groupMutedFunc: gmf, peer: peer, @@ -239,9 +238,20 @@ func (api *API) getReceiversHandler(params receiver_ops.GetReceiversParams) midd _, span := tracer.Start(params.HTTPRequest.Context(), "api.getReceiversHandler") defer span.End() + receiverMatchers, err := parseFilter(params.ReceiverMatchers) + if err != nil { + return receiver_ops.NewGetReceiversBadRequest().WithPayload( + fmt.Sprintf("failed to parse receiver_matchers param: %v", err.Error()), + ) + } + receivers := make([]*open_api_models.Receiver, 0, len(api.alertmanagerConfig.Receivers)) for i := range api.alertmanagerConfig.Receivers { - receivers = append(receivers, &open_api_models.Receiver{Name: &api.alertmanagerConfig.Receivers[i].Name}) + rcv := &api.alertmanagerConfig.Receivers[i] + if len(receiverMatchers) > 0 && !matchFilterLabels(receiverMatchers, rcv.Labels) { + continue + } + receivers = append(receivers, configReceiverToAPIReceiver(rcv)) } return receiver_ops.NewGetReceiversOK().WithPayload(receivers) @@ -278,13 +288,23 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re } } + receiverMatchers, err := parseFilter(params.ReceiverMatchers) + if err != nil { + logger.Debug("Failed to parse receiver matchers", "err", err) + return alert_ops.NewGetAlertsBadRequest().WithPayload( + fmt.Sprintf("failed to parse receiver_matchers param: %v", err.Error()), + ) + } + alerts := api.alerts.GetPending() defer alerts.Close() - alertFilter := api.alertFilter(matchers, *params.Silenced, *params.Inhibited, *params.Active) + tempMarker := marker.NewAlertMarker() + alertFilter := api.alertFilter(ctx, matchers, *params.Silenced, *params.Inhibited, *params.Active, tempMarker) now := time.Now() api.mtx.RLock() + rcvLabels := api.receiverLabelsMap() for a := range alerts.Next() { alert := a.Data if err = alerts.Err(); err != nil { @@ -304,11 +324,15 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re continue } + if len(receiverMatchers) > 0 && !receiversMatchLabels(receivers, receiverMatchers, rcvLabels) { + continue + } + if !alertFilter(alert, now) { continue } - openAlert := AlertToOpenAPIAlert(alert, api.getAlertStatus(alert.Fingerprint()), receivers, nil) + openAlert := AlertToOpenAPIAlert(alert, tempMarker.Status(alert.Fingerprint()), receivers, nil) res = append(res, openAlert) } @@ -328,7 +352,7 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re func (api *API) postAlertsHandler(params alert_ops.PostAlertsParams) middleware.Responder { logger := api.requestLogger(params.HTTPRequest) - ctx, span := tracer.Start(params.HTTPRequest.Context(), "api.postAlertsHandler") + ctx, span := tracer.Start(eventrecorder.WithEventRecording(params.HTTPRequest.Context()), "api.postAlertsHandler") defer span.End() alerts := OpenAPIAlertsToAlerts(ctx, params.Alerts) @@ -365,7 +389,7 @@ func (api *API) postAlertsHandler(params alert_ops.PostAlertsParams) middleware. // Make a best effort to insert all alerts that are valid. var ( - validAlerts = make([]*types.Alert, 0, len(alerts)) + validAlerts = make([]*alert.Alert, 0, len(alerts)) validationErrs error ) for _, a := range alerts { @@ -422,17 +446,44 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams } } - rf := func(receiverFilter *regexp.Regexp) func(r *dispatch.Route) bool { + receiverMatchers, err := parseFilter(params.ReceiverMatchers) + if err != nil { + logger.Debug("Failed to parse receiver matchers", "err", err) + return alertgroup_ops.NewGetAlertGroupsBadRequest().WithPayload( + fmt.Sprintf("failed to parse receiver_matchers param: %v", err.Error()), + ) + } + + api.mtx.RLock() + rcvLabels := api.receiverLabelsMap() + api.mtx.RUnlock() + + rf := func(receiverFilter *regexp.Regexp, receiverMatchers []*labels.Matcher, rcvLabels map[string]open_api_models.LabelSet) func(r *dispatch.Route) bool { return func(r *dispatch.Route) bool { receiver := r.RouteOpts.Receiver if receiverFilter != nil && !receiverFilter.MatchString(receiver) { return false } + if len(receiverMatchers) > 0 { + if lbls, ok := rcvLabels[receiver]; ok { + if !matchFilterLabels(receiverMatchers, lbls) { + return false + } + } else { + return false + } + } return true } - }(receiverFilter) + }(receiverFilter, receiverMatchers, rcvLabels) - af := api.alertFilter(matchers, *params.Silenced, *params.Inhibited, *params.Active) + // The alertFilter runs the inhibitor/silencer pipeline per alert to + // decide whether the alert should be included based on the + // silenced/inhibited/active query params. Passing a nil marker makes + // the filter use a fresh marker per alert, so the prediction for one + // alert does not leak into another (the same fingerprint may appear + // in multiple aggregation groups). + af := api.alertFilter(ctx, matchers, *params.Silenced, *params.Inhibited, *params.Active, nil) alertGroups, allReceivers, err := api.alertGroups(ctx, rf, af) if err != nil { message := "Failed to get alert groups" @@ -444,6 +495,15 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams res := make(open_api_models.AlertGroups, 0, len(alertGroups)) + // Snapshot setAlertStatus under the lock so we can predict the status + // for each alert without holding api.mtx. We predict at query time + // rather than reading the group's actual marker, so the response is + // stable across calls regardless of whether the notification pipeline + // has run in between (assuming no new alerts/silences/inhibits). + api.mtx.RLock() + setAlertStatus := api.setAlertStatus + api.mtx.RUnlock() + for _, alertGroup := range alertGroups { mutedBy, isMuted := api.groupMutedFunc(alertGroup.RouteID, alertGroup.GroupKey) if !*params.Muted && isMuted { @@ -451,7 +511,7 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams } ag := &open_api_models.AlertGroup{ - Receiver: &open_api_models.Receiver{Name: &alertGroup.Receiver}, + Receiver: &open_api_models.ReceiverReference{Name: &alertGroup.Receiver}, Labels: ModelLabelSetToAPILabelSet(alertGroup.Labels), Alerts: make([]*open_api_models.GettableAlert, 0, len(alertGroup.Alerts)), } @@ -459,7 +519,9 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams for _, alert := range alertGroup.Alerts { fp := alert.Fingerprint() receivers := allReceivers[fp] - status := api.getAlertStatus(fp) + // Predict status per (alert, group) using a fresh marker so + // writes don't leak across alerts or groups. + status := predictAlertStatus(ctx, setAlertStatus, alert) apiAlert := AlertToOpenAPIAlert(alert, status, receivers, mutedBy) ag.Alerts = append(ag.Alerts, apiAlert) } @@ -469,22 +531,55 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams return alertgroup_ops.NewGetAlertGroupsOK().WithPayload(res) } -func (api *API) alertFilter(matchers []*labels.Matcher, silenced, inhibited, active bool) func(a *types.Alert, now time.Time) bool { - return func(a *types.Alert, now time.Time) bool { - ctx, span := tracer.Start(context.Background(), "alertFilter") +// predictAlertStatus runs the silencer/inhibitor pipeline against a fresh +// AlertMarker to compute the alert's current status at query time. Using +// a fresh marker per call isolates the prediction so it does not leak +// across alerts or aggregation groups. +func predictAlertStatus(ctx context.Context, setAlertStatus setAlertStatusFn, a *alert.Alert) alert.AlertStatus { + m := marker.NewAlertMarker() + ctx = marker.WithContext(ctx, m) + setAlertStatus(ctx, a.Labels) + return m.Status(a.Fingerprint()) +} + +func (api *API) alertFilter(parent context.Context, matchers []*labels.Matcher, silenced, inhibited, active bool, m marker.AlertMarker) func(a *alert.Alert, now time.Time) bool { + // Snapshot the function pointer under the lock so the closure is safe + // to call without holding api.mtx (e.g. in getAlertGroupsHandler). + api.mtx.RLock() + setAlertStatus := api.setAlertStatus + api.mtx.RUnlock() + + return func(a *alert.Alert, now time.Time) bool { + ctx, span := tracer.Start(parent, "alertFilter") defer span.End() if !a.EndsAt.IsZero() && a.EndsAt.Before(now) { return false } + // Short-circuit on label matchers before the expensive + // silencer/inhibitor pipeline. + if !alertMatchesFilterLabels(&a.Alert, matchers) { + return false + } + // Set alert's current status based on its label set. - api.setAlertStatus(ctx, a.Labels) + // The inhibitor and silencer write to the marker via the context. + // When the caller does not provide a marker, use a fresh per-alert + // marker to avoid cross-alert contamination (e.g. the same + // fingerprint may appear in multiple aggregation groups for + // /alerts/groups). + predict := m + if predict == nil { + predict = marker.NewAlertMarker() + } + ctx = marker.WithContext(ctx, predict) + setAlertStatus(ctx, a.Labels) // Get alert's current status after seeing if it is suppressed. - status := api.getAlertStatus(a.Fingerprint()) + status := predict.Status(a.Fingerprint()) - if !active && status.State == types.AlertStateActive { + if !active && status.State == alert.AlertStateActive { return false } @@ -496,7 +591,7 @@ func (api *API) alertFilter(matchers []*labels.Matcher, silenced, inhibited, act return false } - return alertMatchesFilterLabels(&a.Alert, matchers) + return true } } @@ -540,6 +635,40 @@ func matchFilterLabels(matchers []*labels.Matcher, sms map[string]string) bool { return true } +// receiversMatchLabels returns true if at least one receiver's labels match all the given matchers. +func receiversMatchLabels(receivers []string, matchers []*labels.Matcher, rcvLabels map[string]open_api_models.LabelSet) bool { + for _, rcvName := range receivers { + if lbls, ok := rcvLabels[rcvName]; ok { + if matchFilterLabels(matchers, lbls) { + return true + } + } + } + return false +} + +// configReceiverToAPIReceiver converts a config.Receiver to an API model Receiver. +func configReceiverToAPIReceiver(rcv *config.Receiver) *open_api_models.Receiver { + apiLabels := make(open_api_models.LabelSet, len(rcv.Labels)) + maps.Copy(apiLabels, rcv.Labels) + return &open_api_models.Receiver{ + Name: &rcv.Name, + Labels: apiLabels, + } +} + +// receiverLabelsMap builds a lookup from receiver name to labels for the current config. +func (api *API) receiverLabelsMap() map[string]open_api_models.LabelSet { + m := make(map[string]open_api_models.LabelSet, len(api.alertmanagerConfig.Receivers)) + for i := range api.alertmanagerConfig.Receivers { + rcv := &api.alertmanagerConfig.Receivers[i] + apiLabels := make(open_api_models.LabelSet, len(rcv.Labels)) + maps.Copy(apiLabels, rcv.Labels) + m[rcv.Name] = apiLabels + } + return m +} + func (api *API) getSilencesHandler(params silence_ops.GetSilencesParams) middleware.Responder { logger := api.requestLogger(params.HTTPRequest) @@ -678,7 +807,7 @@ func (api *API) getSilenceHandler(params silence_ops.GetSilenceParams) middlewar func (api *API) deleteSilenceHandler(params silence_ops.DeleteSilenceParams) middleware.Responder { logger := api.requestLogger(params.HTTPRequest) - ctx, span := tracer.Start(params.HTTPRequest.Context(), "api.deleteSilenceHandler") + ctx, span := tracer.Start(eventrecorder.WithEventRecording(params.HTTPRequest.Context()), "api.deleteSilenceHandler") defer span.End() sid := params.SilenceID.String() @@ -695,7 +824,7 @@ func (api *API) deleteSilenceHandler(params silence_ops.DeleteSilenceParams) mid func (api *API) postSilencesHandler(params silence_ops.PostSilencesParams) middleware.Responder { logger := api.requestLogger(params.HTTPRequest) - ctx, span := tracer.Start(params.HTTPRequest.Context(), "api.postSilencesHandler") + ctx, span := tracer.Start(eventrecorder.WithEventRecording(params.HTTPRequest.Context()), "api.postSilencesHandler") defer span.End() sil, err := PostableSilenceToProto(params.Silence) diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/client/alert/get_alerts_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/client/alert/get_alerts_parameters.go index c574dec4232..512cedd650b 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/client/alert/get_alerts_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/client/alert/get_alerts_parameters.go @@ -104,6 +104,12 @@ type GetAlertsParams struct { */ Receiver *string + /* ReceiverMatchers. + + A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + */ + ReceiverMatchers []string + /* Silenced. Include silenced alerts in results. If false, excludes silenced alerts. Note that true (default) shows both silenced and non-silenced alerts. @@ -237,6 +243,17 @@ func (o *GetAlertsParams) SetReceiver(receiver *string) { o.Receiver = receiver } +// WithReceiverMatchers adds the receiverMatchers to the get alerts params +func (o *GetAlertsParams) WithReceiverMatchers(receiverMatchers []string) *GetAlertsParams { + o.SetReceiverMatchers(receiverMatchers) + return o +} + +// SetReceiverMatchers adds the receiverMatchers to the get alerts params +func (o *GetAlertsParams) SetReceiverMatchers(receiverMatchers []string) { + o.ReceiverMatchers = receiverMatchers +} + // WithSilenced adds the silenced to the get alerts params func (o *GetAlertsParams) WithSilenced(silenced *bool) *GetAlertsParams { o.SetSilenced(silenced) @@ -329,6 +346,17 @@ func (o *GetAlertsParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Reg } } + if o.ReceiverMatchers != nil { + + // binding items for receiver_matchers + joinedReceiverMatchers := o.bindParamReceiverMatchers(reg) + + // query array param receiver_matchers + if err := r.SetQueryParam("receiver_matchers", joinedReceiverMatchers...); err != nil { + return err + } + } + if o.Silenced != nil { // query param silenced @@ -385,3 +413,20 @@ func (o *GetAlertsParams) bindParamFilter(formats strfmt.Registry) []string { return filterIS } + +// bindParamGetAlerts binds the parameter receiver_matchers +func (o *GetAlertsParams) bindParamReceiverMatchers(formats strfmt.Registry) []string { + receiverMatchersIR := o.ReceiverMatchers + + var receiverMatchersIC []string + for _, receiverMatchersIIR := range receiverMatchersIR { // explode []string + + receiverMatchersIIV := receiverMatchersIIR // string as string + receiverMatchersIC = append(receiverMatchersIC, receiverMatchersIIV) + } + + // items.CollectionFormat: "multi" + receiverMatchersIS := swag.JoinByFormat(receiverMatchersIC, "multi") + + return receiverMatchersIS +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/client/alertgroup/get_alert_groups_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/client/alertgroup/get_alert_groups_parameters.go index 06bd5b7d457..280137fe25c 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/client/alertgroup/get_alert_groups_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/client/alertgroup/get_alert_groups_parameters.go @@ -112,6 +112,12 @@ type GetAlertGroupsParams struct { */ Receiver *string + /* ReceiverMatchers. + + A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + */ + ReceiverMatchers []string + /* Silenced. Include silenced alerts within the returned groups. If false, excludes silenced alerts from groups. Note that true (default) shows both silenced and non-silenced alerts. @@ -248,6 +254,17 @@ func (o *GetAlertGroupsParams) SetReceiver(receiver *string) { o.Receiver = receiver } +// WithReceiverMatchers adds the receiverMatchers to the get alert groups params +func (o *GetAlertGroupsParams) WithReceiverMatchers(receiverMatchers []string) *GetAlertGroupsParams { + o.SetReceiverMatchers(receiverMatchers) + return o +} + +// SetReceiverMatchers adds the receiverMatchers to the get alert groups params +func (o *GetAlertGroupsParams) SetReceiverMatchers(receiverMatchers []string) { + o.ReceiverMatchers = receiverMatchers +} + // WithSilenced adds the silenced to the get alert groups params func (o *GetAlertGroupsParams) WithSilenced(silenced *bool) *GetAlertGroupsParams { o.SetSilenced(silenced) @@ -346,6 +363,17 @@ func (o *GetAlertGroupsParams) WriteToRequest(r runtime.ClientRequest, reg strfm } } + if o.ReceiverMatchers != nil { + + // binding items for receiver_matchers + joinedReceiverMatchers := o.bindParamReceiverMatchers(reg) + + // query array param receiver_matchers + if err := r.SetQueryParam("receiver_matchers", joinedReceiverMatchers...); err != nil { + return err + } + } + if o.Silenced != nil { // query param silenced @@ -385,3 +413,20 @@ func (o *GetAlertGroupsParams) bindParamFilter(formats strfmt.Registry) []string return filterIS } + +// bindParamGetAlertGroups binds the parameter receiver_matchers +func (o *GetAlertGroupsParams) bindParamReceiverMatchers(formats strfmt.Registry) []string { + receiverMatchersIR := o.ReceiverMatchers + + var receiverMatchersIC []string + for _, receiverMatchersIIR := range receiverMatchersIR { // explode []string + + receiverMatchersIIV := receiverMatchersIIR // string as string + receiverMatchersIC = append(receiverMatchersIC, receiverMatchersIIV) + } + + // items.CollectionFormat: "multi" + receiverMatchersIS := swag.JoinByFormat(receiverMatchersIC, "multi") + + return receiverMatchersIS +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_parameters.go index b468b3c7efb..7fadf197f0c 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_parameters.go @@ -28,6 +28,7 @@ import ( "github.com/go-openapi/runtime" cr "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" ) // NewGetReceiversParams creates a new GetReceiversParams object, @@ -74,6 +75,13 @@ GetReceiversParams contains all the parameters to send to the API endpoint Typically these are written to a http.Request. */ type GetReceiversParams struct { + + /* ReceiverMatchers. + + A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + */ + ReceiverMatchers []string + timeout time.Duration Context context.Context HTTPClient *http.Client @@ -127,6 +135,17 @@ func (o *GetReceiversParams) SetHTTPClient(client *http.Client) { o.HTTPClient = client } +// WithReceiverMatchers adds the receiverMatchers to the get receivers params +func (o *GetReceiversParams) WithReceiverMatchers(receiverMatchers []string) *GetReceiversParams { + o.SetReceiverMatchers(receiverMatchers) + return o +} + +// SetReceiverMatchers adds the receiverMatchers to the get receivers params +func (o *GetReceiversParams) SetReceiverMatchers(receiverMatchers []string) { + o.ReceiverMatchers = receiverMatchers +} + // WriteToRequest writes these params to a swagger request func (o *GetReceiversParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { @@ -135,8 +154,36 @@ func (o *GetReceiversParams) WriteToRequest(r runtime.ClientRequest, reg strfmt. } var res []error + if o.ReceiverMatchers != nil { + + // binding items for receiver_matchers + joinedReceiverMatchers := o.bindParamReceiverMatchers(reg) + + // query array param receiver_matchers + if err := r.SetQueryParam("receiver_matchers", joinedReceiverMatchers...); err != nil { + return err + } + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } return nil } + +// bindParamGetReceivers binds the parameter receiver_matchers +func (o *GetReceiversParams) bindParamReceiverMatchers(formats strfmt.Registry) []string { + receiverMatchersIR := o.ReceiverMatchers + + var receiverMatchersIC []string + for _, receiverMatchersIIR := range receiverMatchersIR { // explode []string + + receiverMatchersIIV := receiverMatchersIIR // string as string + receiverMatchersIC = append(receiverMatchersIC, receiverMatchersIIV) + } + + // items.CollectionFormat: "multi" + receiverMatchersIS := swag.JoinByFormat(receiverMatchersIC, "multi") + + return receiverMatchersIS +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_responses.go b/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_responses.go index dd3850577cd..519fcfd36e9 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_responses.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/client/receiver/get_receivers_responses.go @@ -45,6 +45,12 @@ func (o *GetReceiversReader) ReadResponse(response runtime.ClientResponse, consu return nil, err } return result, nil + case 400: + result := NewGetReceiversBadRequest() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return nil, result default: return nil, runtime.NewAPIError("[GET /receivers] getReceivers", response, response.Code()) } @@ -117,3 +123,71 @@ func (o *GetReceiversOK) readResponse(response runtime.ClientResponse, consumer return nil } + +// NewGetReceiversBadRequest creates a GetReceiversBadRequest with default headers values +func NewGetReceiversBadRequest() *GetReceiversBadRequest { + return &GetReceiversBadRequest{} +} + +/* +GetReceiversBadRequest describes a response with status code 400, with default header values. + +Bad request +*/ +type GetReceiversBadRequest struct { + Payload string +} + +// IsSuccess returns true when this get receivers bad request response has a 2xx status code +func (o *GetReceiversBadRequest) IsSuccess() bool { + return false +} + +// IsRedirect returns true when this get receivers bad request response has a 3xx status code +func (o *GetReceiversBadRequest) IsRedirect() bool { + return false +} + +// IsClientError returns true when this get receivers bad request response has a 4xx status code +func (o *GetReceiversBadRequest) IsClientError() bool { + return true +} + +// IsServerError returns true when this get receivers bad request response has a 5xx status code +func (o *GetReceiversBadRequest) IsServerError() bool { + return false +} + +// IsCode returns true when this get receivers bad request response a status code equal to that given +func (o *GetReceiversBadRequest) IsCode(code int) bool { + return code == 400 +} + +// Code gets the status code for the get receivers bad request response +func (o *GetReceiversBadRequest) Code() int { + return 400 +} + +func (o *GetReceiversBadRequest) Error() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /receivers][%d] getReceiversBadRequest %s", 400, payload) +} + +func (o *GetReceiversBadRequest) String() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /receivers][%d] getReceiversBadRequest %s", 400, payload) +} + +func (o *GetReceiversBadRequest) GetPayload() string { + return o.Payload +} + +func (o *GetReceiversBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && !stderrors.Is(err, io.EOF) { + return err + } + + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/compat.go b/vendor/github.com/prometheus/alertmanager/api/v2/compat.go index c38105bd780..cb5560af058 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/compat.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/compat.go @@ -22,10 +22,10 @@ import ( prometheus_model "github.com/prometheus/common/model" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/prometheus/alertmanager/alert" open_api_models "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/silence/silencepb" - "github.com/prometheus/alertmanager/types" ) // GettableSilenceFromProto converts *silencepb.Silence to open_api_models.GettableSilence. @@ -139,17 +139,17 @@ func PostableSilenceToProto(s *open_api_models.PostableSilence) (*silencepb.Sile } // AlertToOpenAPIAlert converts internal alerts, alert types, and receivers to *open_api_models.GettableAlert. -func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers, mutedBy []string) *open_api_models.GettableAlert { - startsAt := strfmt.DateTime(alert.StartsAt) - updatedAt := strfmt.DateTime(alert.UpdatedAt) - endsAt := strfmt.DateTime(alert.EndsAt) +func AlertToOpenAPIAlert(source *alert.Alert, status alert.AlertStatus, receivers, mutedBy []string) *open_api_models.GettableAlert { + startsAt := strfmt.DateTime(source.StartsAt) + updatedAt := strfmt.DateTime(source.UpdatedAt) + endsAt := strfmt.DateTime(source.EndsAt) - apiReceivers := make([]*open_api_models.Receiver, 0, len(receivers)) + apiReceivers := make([]*open_api_models.ReceiverReference, 0, len(receivers)) for i := range receivers { - apiReceivers = append(apiReceivers, &open_api_models.Receiver{Name: &receivers[i]}) + apiReceivers = append(apiReceivers, &open_api_models.ReceiverReference{Name: &receivers[i]}) } - fp := alert.Fingerprint().String() + fp := source.Fingerprint().String() state := string(status.State) if len(mutedBy) > 0 { @@ -159,10 +159,10 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers aa := &open_api_models.GettableAlert{ Alert: open_api_models.Alert{ - GeneratorURL: strfmt.URI(alert.GeneratorURL), - Labels: ModelLabelSetToAPILabelSet(alert.Labels), + GeneratorURL: strfmt.URI(source.GeneratorURL), + Labels: ModelLabelSetToAPILabelSet(source.Labels), }, - Annotations: ModelLabelSetToAPILabelSet(alert.Annotations), + Annotations: ModelLabelSetToAPILabelSet(source.Annotations), StartsAt: &startsAt, UpdatedAt: &updatedAt, EndsAt: &endsAt, @@ -191,14 +191,14 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers return aa } -// OpenAPIAlertsToAlerts converts open_api_models.PostableAlerts to []*types.Alert. -func OpenAPIAlertsToAlerts(ctx context.Context, apiAlerts open_api_models.PostableAlerts) []*types.Alert { +// OpenAPIAlertsToAlerts converts open_api_models.PostableAlerts to []*alert.Alert. +func OpenAPIAlertsToAlerts(ctx context.Context, apiAlerts open_api_models.PostableAlerts) []*alert.Alert { _, span := tracer.Start(ctx, "OpenAPIAlertsToAlerts") defer span.End() - alerts := make([]*types.Alert, 0, len(apiAlerts)) + alerts := make([]*alert.Alert, 0, len(apiAlerts)) for _, apiAlert := range apiAlerts { - alerts = append(alerts, &types.Alert{ + alerts = append(alerts, &alert.Alert{ Alert: prometheus_model.Alert{ Labels: APILabelSetToModelLabelSet(apiAlert.Labels), Annotations: APILabelSetToModelLabelSet(apiAlert.Annotations), diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/models/alert_group.go b/vendor/github.com/prometheus/alertmanager/api/v2/models/alert_group.go index f5899195aec..6cad0ab2e4b 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/models/alert_group.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/models/alert_group.go @@ -45,7 +45,7 @@ type AlertGroup struct { // receiver // Required: true - Receiver *Receiver `json:"receiver"` + Receiver *ReceiverReference `json:"receiver"` } // Validate validates this alert group diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/models/gettable_alert.go b/vendor/github.com/prometheus/alertmanager/api/v2/models/gettable_alert.go index 4091b569cb3..8ca83284ec1 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/models/gettable_alert.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/models/gettable_alert.go @@ -50,7 +50,7 @@ type GettableAlert struct { // receivers // Required: true - Receivers []*Receiver `json:"receivers"` + Receivers []*ReceiverReference `json:"receivers"` // starts at // Required: true @@ -79,7 +79,7 @@ func (m *GettableAlert) UnmarshalJSON(raw []byte) error { Fingerprint *string `json:"fingerprint"` - Receivers []*Receiver `json:"receivers"` + Receivers []*ReceiverReference `json:"receivers"` StartsAt *strfmt.DateTime `json:"startsAt"` @@ -126,7 +126,7 @@ func (m GettableAlert) MarshalJSON() ([]byte, error) { Fingerprint *string `json:"fingerprint"` - Receivers []*Receiver `json:"receivers"` + Receivers []*ReceiverReference `json:"receivers"` StartsAt *strfmt.DateTime `json:"startsAt"` diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver.go b/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver.go index 8e1bf9ee457..6487936c281 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver.go @@ -21,6 +21,7 @@ package models import ( "context" + stderrors "errors" "github.com/go-openapi/errors" "github.com/go-openapi/strfmt" @@ -33,6 +34,9 @@ import ( // swagger:model receiver type Receiver struct { + // labels + Labels LabelSet `json:"labels,omitempty"` + // name // Required: true Name *string `json:"name"` @@ -42,6 +46,10 @@ type Receiver struct { func (m *Receiver) Validate(formats strfmt.Registry) error { var res []error + if err := m.validateLabels(formats); err != nil { + res = append(res, err) + } + if err := m.validateName(formats); err != nil { res = append(res, err) } @@ -52,6 +60,29 @@ func (m *Receiver) Validate(formats strfmt.Registry) error { return nil } +func (m *Receiver) validateLabels(formats strfmt.Registry) error { + if swag.IsZero(m.Labels) { // not required + return nil + } + + if m.Labels != nil { + if err := m.Labels.Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("labels") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("labels") + } + + return err + } + } + + return nil +} + func (m *Receiver) validateName(formats strfmt.Registry) error { if err := validate.Required("name", "body", m.Name); err != nil { @@ -61,8 +92,39 @@ func (m *Receiver) validateName(formats strfmt.Registry) error { return nil } -// ContextValidate validates this receiver based on context it is used +// ContextValidate validate this receiver based on the context it is used func (m *Receiver) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateLabels(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *Receiver) contextValidateLabels(ctx context.Context, formats strfmt.Registry) error { + + if swag.IsZero(m.Labels) { // not required + return nil + } + + if err := m.Labels.ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("labels") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("labels") + } + + return err + } + return nil } diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver_reference.go b/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver_reference.go new file mode 100644 index 00000000000..817c63fd3bc --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/api/v2/models/receiver_reference.go @@ -0,0 +1,85 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// Copyright Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ReceiverReference receiver reference +// +// swagger:model receiverReference +type ReceiverReference struct { + + // name + // Required: true + Name *string `json:"name"` +} + +// Validate validates this receiver reference +func (m *ReceiverReference) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateName(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ReceiverReference) validateName(formats strfmt.Registry) error { + + if err := validate.Required("name", "body", m.Name); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this receiver reference based on context it is used +func (m *ReceiverReference) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *ReceiverReference) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ReceiverReference) UnmarshalBinary(b []byte) error { + var res ReceiverReference + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/openapi.yaml b/vendor/github.com/prometheus/alertmanager/api/v2/openapi.yaml index 1f6cc17cd3e..1ee47c30dc5 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/openapi.yaml +++ b/vendor/github.com/prometheus/alertmanager/api/v2/openapi.yaml @@ -35,6 +35,8 @@ paths: - receiver operationId: getReceivers description: Get list of all receivers (name of notification integrations) + parameters: + - $ref: '#/parameters/receiverMatchers' responses: '200': description: Get receivers response @@ -42,6 +44,8 @@ paths: type: array items: $ref: '#/definitions/receiver' + '400': + $ref: '#/responses/BadRequest' /silences: get: tags: @@ -173,6 +177,7 @@ paths: description: A regex matching receivers to filter alerts by required: false type: string + - $ref: '#/parameters/receiverMatchers' responses: '200': description: Get alerts response @@ -241,6 +246,7 @@ paths: description: A regex matching receivers to filter alerts by required: false type: string + - '$ref': '#/parameters/receiverMatchers' responses: '200': description: Get alert groups response @@ -261,6 +267,16 @@ responses: schema: type: string +parameters: + receiverMatchers: + name: receiver_matchers + in: query + description: A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + required: false + type: array + collectionFormat: multi + items: + type: string definitions: alertmanagerStatus: @@ -436,7 +452,7 @@ definitions: receivers: type: array items: - $ref: '#/definitions/receiver' + $ref: '#/definitions/receiverReference' fingerprint: type: string startsAt: @@ -486,7 +502,7 @@ definitions: labels: $ref: '#/definitions/labelSet' receiver: - $ref: '#/definitions/receiver' + $ref: '#/definitions/receiverReference' alerts: type: array items: @@ -519,6 +535,15 @@ definitions: - inhibitedBy - mutedBy receiver: + type: object + properties: + name: + type: string + labels: + $ref: '#/definitions/labelSet' + required: + - name + receiverReference: type: object properties: name: diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/embedded_spec.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/embedded_spec.go index 2783b965f81..3e41640c95d 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/embedded_spec.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/embedded_spec.go @@ -101,6 +101,9 @@ func init() { "description": "A regex matching receivers to filter alerts by", "name": "receiver", "in": "query" + }, + { + "$ref": "#/parameters/receiverMatchers" } ], "responses": { @@ -199,6 +202,9 @@ func init() { "description": "A regex matching receivers to filter alerts by", "name": "receiver", "in": "query" + }, + { + "$ref": "#/parameters/receiverMatchers" } ], "responses": { @@ -224,6 +230,11 @@ func init() { "receiver" ], "operationId": "getReceivers", + "parameters": [ + { + "$ref": "#/parameters/receiverMatchers" + } + ], "responses": { "200": { "description": "Get receivers response", @@ -233,6 +244,9 @@ func init() { "$ref": "#/definitions/receiver" } } + }, + "400": { + "$ref": "#/responses/BadRequest" } } } @@ -425,7 +439,7 @@ func init() { "$ref": "#/definitions/labelSet" }, "receiver": { - "$ref": "#/definitions/receiver" + "$ref": "#/definitions/receiverReference" } } }, @@ -559,7 +573,7 @@ func init() { "receivers": { "type": "array", "items": { - "$ref": "#/definitions/receiver" + "$ref": "#/definitions/receiverReference" } }, "startsAt": { @@ -716,6 +730,20 @@ func init() { ] }, "receiver": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "labels": { + "$ref": "#/definitions/labelSet" + }, + "name": { + "type": "string" + } + } + }, + "receiverReference": { "type": "object", "required": [ "name" @@ -806,6 +834,18 @@ func init() { } } }, + "parameters": { + "receiverMatchers": { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "A matcher expression to filter by receiver labels. For example ` + "`" + `owner=\"my-team\"` + "`" + `. Can be repeated to apply multiple matchers.", + "name": "receiver_matchers", + "in": "query" + } + }, "responses": { "BadRequest": { "description": "Bad request", @@ -909,6 +949,16 @@ func init() { "description": "A regex matching receivers to filter alerts by", "name": "receiver", "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "A matcher expression to filter by receiver labels. For example ` + "`" + `owner=\"my-team\"` + "`" + `. Can be repeated to apply multiple matchers.", + "name": "receiver_matchers", + "in": "query" } ], "responses": { @@ -1019,6 +1069,16 @@ func init() { "description": "A regex matching receivers to filter alerts by", "name": "receiver", "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "A matcher expression to filter by receiver labels. For example ` + "`" + `owner=\"my-team\"` + "`" + `. Can be repeated to apply multiple matchers.", + "name": "receiver_matchers", + "in": "query" } ], "responses": { @@ -1050,6 +1110,18 @@ func init() { "receiver" ], "operationId": "getReceivers", + "parameters": [ + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "A matcher expression to filter by receiver labels. For example ` + "`" + `owner=\"my-team\"` + "`" + `. Can be repeated to apply multiple matchers.", + "name": "receiver_matchers", + "in": "query" + } + ], "responses": { "200": { "description": "Get receivers response", @@ -1059,6 +1131,12 @@ func init() { "$ref": "#/definitions/receiver" } } + }, + "400": { + "description": "Bad request", + "schema": { + "type": "string" + } } } } @@ -1266,7 +1344,7 @@ func init() { "$ref": "#/definitions/labelSet" }, "receiver": { - "$ref": "#/definitions/receiver" + "$ref": "#/definitions/receiverReference" } } }, @@ -1400,7 +1478,7 @@ func init() { "receivers": { "type": "array", "items": { - "$ref": "#/definitions/receiver" + "$ref": "#/definitions/receiverReference" } }, "startsAt": { @@ -1557,6 +1635,20 @@ func init() { ] }, "receiver": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "labels": { + "$ref": "#/definitions/labelSet" + }, + "name": { + "type": "string" + } + } + }, + "receiverReference": { "type": "object", "required": [ "name" @@ -1647,6 +1739,18 @@ func init() { } } }, + "parameters": { + "receiverMatchers": { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "A matcher expression to filter by receiver labels. For example ` + "`" + `owner=\"my-team\"` + "`" + `. Can be repeated to apply multiple matchers.", + "name": "receiver_matchers", + "in": "query" + } + }, "responses": { "BadRequest": { "description": "Bad request", diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_parameters.go index d7238d06127..930464e637b 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_parameters.go @@ -86,6 +86,12 @@ type GetAlertsParams struct { */ Receiver *string + /*A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + In: query + Collection Format: multi + */ + ReceiverMatchers []string + /*Include silenced alerts in results. If false, excludes silenced alerts. Note that true (default) shows both silenced and non-silenced alerts. In: query Default: true @@ -129,6 +135,11 @@ func (o *GetAlertsParams) BindRequest(r *http.Request, route *middleware.Matched res = append(res, err) } + qReceiverMatchers, qhkReceiverMatchers, _ := qs.GetOK("receiver_matchers") + if err := o.bindReceiverMatchers(qReceiverMatchers, qhkReceiverMatchers, route.Formats); err != nil { + res = append(res, err) + } + qSilenced, qhkSilenced, _ := qs.GetOK("silenced") if err := o.bindSilenced(qSilenced, qhkSilenced, route.Formats); err != nil { res = append(res, err) @@ -232,6 +243,28 @@ func (o *GetAlertsParams) bindReceiver(rawData []string, hasKey bool, formats st return nil } +// bindReceiverMatchers binds and validates array parameter ReceiverMatchers from query. +// +// Arrays are parsed according to CollectionFormat: "multi" (defaults to "csv" when empty). +func (o *GetAlertsParams) bindReceiverMatchers(rawData []string, hasKey bool, formats strfmt.Registry) error { + // CollectionFormat: multi + receiverMatchersIC := rawData + if len(receiverMatchersIC) == 0 { + return nil + } + + var receiverMatchersIR []string + for _, receiverMatchersIV := range receiverMatchersIC { + receiverMatchersI := receiverMatchersIV + + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersI) + } + + o.ReceiverMatchers = receiverMatchersIR + + return nil +} + // bindSilenced binds and validates parameter Silenced from query. func (o *GetAlertsParams) bindSilenced(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_urlbuilder.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_urlbuilder.go index a89a99c9b99..8f87a9de750 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_urlbuilder.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alert/get_alerts_urlbuilder.go @@ -29,12 +29,13 @@ import ( // GetAlertsURL generates an URL for the get alerts operation type GetAlertsURL struct { - Active *bool - Filter []string - Inhibited *bool - Receiver *string - Silenced *bool - Unprocessed *bool + Active *bool + Filter []string + Inhibited *bool + Receiver *string + ReceiverMatchers []string + Silenced *bool + Unprocessed *bool _basePath string // avoid unkeyed usage @@ -108,6 +109,20 @@ func (o *GetAlertsURL) Build() (*url.URL, error) { qs.Set("receiver", receiverQ) } + var receiverMatchersIR []string + for _, receiverMatchersI := range o.ReceiverMatchers { + receiverMatchersIS := receiverMatchersI + if receiverMatchersIS != "" { + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersIS) + } + } + + receiverMatchers := swag.JoinByFormat(receiverMatchersIR, "multi") + + for _, qsv := range receiverMatchers { + qs.Add("receiver_matchers", qsv) + } + var silencedQ string if o.Silenced != nil { silencedQ = swag.FormatBool(*o.Silenced) diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_parameters.go index 4a5652e4ba0..27f93edc98b 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_parameters.go @@ -92,6 +92,12 @@ type GetAlertGroupsParams struct { */ Receiver *string + /*A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + In: query + Collection Format: multi + */ + ReceiverMatchers []string + /*Include silenced alerts within the returned groups. If false, excludes silenced alerts from groups. Note that true (default) shows both silenced and non-silenced alerts. In: query Default: true @@ -134,6 +140,11 @@ func (o *GetAlertGroupsParams) BindRequest(r *http.Request, route *middleware.Ma res = append(res, err) } + qReceiverMatchers, qhkReceiverMatchers, _ := qs.GetOK("receiver_matchers") + if err := o.bindReceiverMatchers(qReceiverMatchers, qhkReceiverMatchers, route.Formats); err != nil { + res = append(res, err) + } + qSilenced, qhkSilenced, _ := qs.GetOK("silenced") if err := o.bindSilenced(qSilenced, qhkSilenced, route.Formats); err != nil { res = append(res, err) @@ -256,6 +267,28 @@ func (o *GetAlertGroupsParams) bindReceiver(rawData []string, hasKey bool, forma return nil } +// bindReceiverMatchers binds and validates array parameter ReceiverMatchers from query. +// +// Arrays are parsed according to CollectionFormat: "multi" (defaults to "csv" when empty). +func (o *GetAlertGroupsParams) bindReceiverMatchers(rawData []string, hasKey bool, formats strfmt.Registry) error { + // CollectionFormat: multi + receiverMatchersIC := rawData + if len(receiverMatchersIC) == 0 { + return nil + } + + var receiverMatchersIR []string + for _, receiverMatchersIV := range receiverMatchersIC { + receiverMatchersI := receiverMatchersIV + + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersI) + } + + o.ReceiverMatchers = receiverMatchersIR + + return nil +} + // bindSilenced binds and validates parameter Silenced from query. func (o *GetAlertGroupsParams) bindSilenced(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_urlbuilder.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_urlbuilder.go index d229a4a5b25..412b1aa35c7 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_urlbuilder.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup/get_alert_groups_urlbuilder.go @@ -29,12 +29,13 @@ import ( // GetAlertGroupsURL generates an URL for the get alert groups operation type GetAlertGroupsURL struct { - Active *bool - Filter []string - Inhibited *bool - Muted *bool - Receiver *string - Silenced *bool + Active *bool + Filter []string + Inhibited *bool + Muted *bool + Receiver *string + ReceiverMatchers []string + Silenced *bool _basePath string // avoid unkeyed usage @@ -116,6 +117,20 @@ func (o *GetAlertGroupsURL) Build() (*url.URL, error) { qs.Set("receiver", receiverQ) } + var receiverMatchersIR []string + for _, receiverMatchersI := range o.ReceiverMatchers { + receiverMatchersIS := receiverMatchersI + if receiverMatchersIS != "" { + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersIS) + } + } + + receiverMatchers := swag.JoinByFormat(receiverMatchersIR, "multi") + + for _, qsv := range receiverMatchers { + qs.Add("receiver_matchers", qsv) + } + var silencedQ string if o.Silenced != nil { silencedQ = swag.FormatBool(*o.Silenced) diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_parameters.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_parameters.go index b1a3cab7b63..6925912cf50 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_parameters.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_parameters.go @@ -23,7 +23,9 @@ import ( "net/http" "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" ) // NewGetReceiversParams creates a new GetReceiversParams object @@ -41,6 +43,12 @@ func NewGetReceiversParams() GetReceiversParams { type GetReceiversParams struct { // HTTP Request Object HTTPRequest *http.Request `json:"-"` + + /*A matcher expression to filter by receiver labels. For example `owner="my-team"`. Can be repeated to apply multiple matchers. + In: query + Collection Format: multi + */ + ReceiverMatchers []string } // BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface @@ -51,9 +59,36 @@ func (o *GetReceiversParams) BindRequest(r *http.Request, route *middleware.Matc var res []error o.HTTPRequest = r + qs := runtime.Values(r.URL.Query()) + qReceiverMatchers, qhkReceiverMatchers, _ := qs.GetOK("receiver_matchers") + if err := o.bindReceiverMatchers(qReceiverMatchers, qhkReceiverMatchers, route.Formats); err != nil { + res = append(res, err) + } if len(res) > 0 { return errors.CompositeValidationError(res...) } return nil } + +// bindReceiverMatchers binds and validates array parameter ReceiverMatchers from query. +// +// Arrays are parsed according to CollectionFormat: "multi" (defaults to "csv" when empty). +func (o *GetReceiversParams) bindReceiverMatchers(rawData []string, hasKey bool, formats strfmt.Registry) error { + // CollectionFormat: multi + receiverMatchersIC := rawData + if len(receiverMatchersIC) == 0 { + return nil + } + + var receiverMatchersIR []string + for _, receiverMatchersIV := range receiverMatchersIC { + receiverMatchersI := receiverMatchersIV + + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersI) + } + + o.ReceiverMatchers = receiverMatchersIR + + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_responses.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_responses.go index 0727afc9fbe..249b486b2fc 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_responses.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_responses.go @@ -74,3 +74,46 @@ func (o *GetReceiversOK) WriteResponse(rw http.ResponseWriter, producer runtime. panic(err) // let the recovery middleware deal with this } } + +// GetReceiversBadRequestCode is the HTTP code returned for type GetReceiversBadRequest +const GetReceiversBadRequestCode int = 400 + +/* +GetReceiversBadRequest Bad request + +swagger:response getReceiversBadRequest +*/ +type GetReceiversBadRequest struct { + + /* + In: Body + */ + Payload string `json:"body,omitempty"` +} + +// NewGetReceiversBadRequest creates GetReceiversBadRequest with default headers values +func NewGetReceiversBadRequest() *GetReceiversBadRequest { + + return &GetReceiversBadRequest{} +} + +// WithPayload adds the payload to the get receivers bad request response +func (o *GetReceiversBadRequest) WithPayload(payload string) *GetReceiversBadRequest { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get receivers bad request response +func (o *GetReceiversBadRequest) SetPayload(payload string) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetReceiversBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(400) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} diff --git a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_urlbuilder.go b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_urlbuilder.go index 3f7ba0ce6b8..0682e3dc319 100644 --- a/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_urlbuilder.go +++ b/vendor/github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver/get_receivers_urlbuilder.go @@ -23,11 +23,17 @@ import ( "errors" "net/url" golangswaggerpaths "path" + + "github.com/go-openapi/swag" ) // GetReceiversURL generates an URL for the get receivers operation type GetReceiversURL struct { + ReceiverMatchers []string + _basePath string + // avoid unkeyed usage + _ struct{} } // WithBasePath sets the base path for this url builder, only required when it's different from the @@ -57,6 +63,24 @@ func (o *GetReceiversURL) Build() (*url.URL, error) { } _result.Path = golangswaggerpaths.Join(_basePath, _path) + qs := make(url.Values) + + var receiverMatchersIR []string + for _, receiverMatchersI := range o.ReceiverMatchers { + receiverMatchersIS := receiverMatchersI + if receiverMatchersIS != "" { + receiverMatchersIR = append(receiverMatchersIR, receiverMatchersIS) + } + } + + receiverMatchers := swag.JoinByFormat(receiverMatchersIR, "multi") + + for _, qsv := range receiverMatchers { + qs.Add("receiver_matchers", qsv) + } + + _result.RawQuery = qs.Encode() + return &_result, nil } diff --git a/vendor/github.com/prometheus/alertmanager/config/config.go b/vendor/github.com/prometheus/alertmanager/config/config.go index 67d34eab087..aeea0560a3c 100644 --- a/vendor/github.com/prometheus/alertmanager/config/config.go +++ b/vendor/github.com/prometheus/alertmanager/config/config.go @@ -30,7 +30,14 @@ import ( "gopkg.in/yaml.v2" amcommoncfg "github.com/prometheus/alertmanager/config/common" + "github.com/prometheus/alertmanager/eventrecorder" "github.com/prometheus/alertmanager/matcher/compat" + "github.com/prometheus/alertmanager/notify/discord" + "github.com/prometheus/alertmanager/notify/incidentio" + "github.com/prometheus/alertmanager/notify/jira" + "github.com/prometheus/alertmanager/notify/mattermost" + "github.com/prometheus/alertmanager/notify/msteams" + "github.com/prometheus/alertmanager/notify/webhook" "github.com/prometheus/alertmanager/timeinterval" "github.com/prometheus/alertmanager/tracing" ) @@ -217,6 +224,20 @@ func resolveFilepaths(baseDir string, cfg *Config) { cfg.HTTPConfig.SetDirectory(baseDir) } } + + for i := range cfg.EventRecorder.FileOutputs { + cfg.EventRecorder.FileOutputs[i].Path = join(cfg.EventRecorder.FileOutputs[i].Path) + } + for _, out := range cfg.EventRecorder.WebhookOutputs { + if out.HTTPConfig != nil { + out.HTTPConfig.SetDirectory(baseDir) + } + } + for _, out := range cfg.EventRecorder.KafkaOutputs { + if out.TLSConfig != nil { + out.TLSConfig.SetDirectory(baseDir) + } + } } // MuteTimeInterval represents a named set of time intervals for which a route should be muted. @@ -268,6 +289,8 @@ type Config struct { TracingConfig tracing.TracingConfig `yaml:"tracing,omitempty" json:"tracing,omitempty"` + EventRecorder eventrecorder.Config `yaml:"event_recorder,omitempty" json:"event_recorder,omitempty"` + // original is the input from which the config was parsed. original string } @@ -925,25 +948,27 @@ func (r *Route) UnmarshalYAML(unmarshal func(any) error) error { type Receiver struct { // A unique identifier for this receiver. Name string `yaml:"name" json:"name"` - - DiscordConfigs []*DiscordConfig `yaml:"discord_configs,omitempty" json:"discord_configs,omitempty"` - EmailConfigs []*EmailConfig `yaml:"email_configs,omitempty" json:"email_configs,omitempty"` - IncidentioConfigs []*IncidentioConfig `yaml:"incidentio_configs,omitempty" json:"incidentio_configs,omitempty"` - PagerdutyConfigs []*PagerdutyConfig `yaml:"pagerduty_configs,omitempty" json:"pagerduty_configs,omitempty"` - SlackConfigs []*SlackConfig `yaml:"slack_configs,omitempty" json:"slack_configs,omitempty"` - WebhookConfigs []*WebhookConfig `yaml:"webhook_configs,omitempty" json:"webhook_configs,omitempty"` - OpsGenieConfigs []*OpsGenieConfig `yaml:"opsgenie_configs,omitempty" json:"opsgenie_configs,omitempty"` - WechatConfigs []*WechatConfig `yaml:"wechat_configs,omitempty" json:"wechat_configs,omitempty"` - PushoverConfigs []*PushoverConfig `yaml:"pushover_configs,omitempty" json:"pushover_configs,omitempty"` - VictorOpsConfigs []*VictorOpsConfig `yaml:"victorops_configs,omitempty" json:"victorops_configs,omitempty"` - SNSConfigs []*SNSConfig `yaml:"sns_configs,omitempty" json:"sns_configs,omitempty"` - TelegramConfigs []*TelegramConfig `yaml:"telegram_configs,omitempty" json:"telegram_configs,omitempty"` - WebexConfigs []*WebexConfig `yaml:"webex_configs,omitempty" json:"webex_configs,omitempty"` - MSTeamsConfigs []*MSTeamsConfig `yaml:"msteams_configs,omitempty" json:"msteams_configs,omitempty"` - MSTeamsV2Configs []*MSTeamsV2Config `yaml:"msteamsv2_configs,omitempty" json:"msteamsv2_configs,omitempty"` - JiraConfigs []*JiraConfig `yaml:"jira_configs,omitempty" json:"jira_configs,omitempty"` - RocketchatConfigs []*RocketchatConfig `yaml:"rocketchat_configs,omitempty" json:"rocketchat_configs,omitempty"` - MattermostConfigs []*MattermostConfig `yaml:"mattermost_configs,omitempty" json:"mattermost_configs,omitempty"` + // Labels attached to this receiver for querying and filtering. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` + + DiscordConfigs []*discord.DiscordConfig `yaml:"discord_configs,omitempty" json:"discord_configs,omitempty"` + EmailConfigs []*EmailConfig `yaml:"email_configs,omitempty" json:"email_configs,omitempty"` + IncidentioConfigs []*incidentio.IncidentioConfig `yaml:"incidentio_configs,omitempty" json:"incidentio_configs,omitempty"` + PagerdutyConfigs []*PagerdutyConfig `yaml:"pagerduty_configs,omitempty" json:"pagerduty_configs,omitempty"` + SlackConfigs []*SlackConfig `yaml:"slack_configs,omitempty" json:"slack_configs,omitempty"` + WebhookConfigs []*webhook.WebhookConfig `yaml:"webhook_configs,omitempty" json:"webhook_configs,omitempty"` + OpsGenieConfigs []*OpsGenieConfig `yaml:"opsgenie_configs,omitempty" json:"opsgenie_configs,omitempty"` + WechatConfigs []*WechatConfig `yaml:"wechat_configs,omitempty" json:"wechat_configs,omitempty"` + PushoverConfigs []*PushoverConfig `yaml:"pushover_configs,omitempty" json:"pushover_configs,omitempty"` + VictorOpsConfigs []*VictorOpsConfig `yaml:"victorops_configs,omitempty" json:"victorops_configs,omitempty"` + SNSConfigs []*SNSConfig `yaml:"sns_configs,omitempty" json:"sns_configs,omitempty"` + TelegramConfigs []*TelegramConfig `yaml:"telegram_configs,omitempty" json:"telegram_configs,omitempty"` + WebexConfigs []*WebexConfig `yaml:"webex_configs,omitempty" json:"webex_configs,omitempty"` + MSTeamsConfigs []*msteams.MSTeamsConfig `yaml:"msteams_configs,omitempty" json:"msteams_configs,omitempty"` + MSTeamsV2Configs []*MSTeamsV2Config `yaml:"msteamsv2_configs,omitempty" json:"msteamsv2_configs,omitempty"` + JiraConfigs []*jira.JiraConfig `yaml:"jira_configs,omitempty" json:"jira_configs,omitempty"` + RocketchatConfigs []*RocketchatConfig `yaml:"rocketchat_configs,omitempty" json:"rocketchat_configs,omitempty"` + MattermostConfigs []*mattermost.MattermostConfig `yaml:"mattermost_configs,omitempty" json:"mattermost_configs,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface for Receiver. @@ -955,5 +980,12 @@ func (c *Receiver) UnmarshalYAML(unmarshal func(any) error) error { if c.Name == "" { return errors.New("missing name in receiver") } + if c.Labels == nil { + c.Labels = make(map[string]string) + } + if v, ok := c.Labels["name"]; ok && v != c.Name { + return fmt.Errorf("receiver label \"name\" must match receiver name %q, got %q", c.Name, v) + } + c.Labels["name"] = c.Name return nil } diff --git a/vendor/github.com/prometheus/alertmanager/config/coordinator.go b/vendor/github.com/prometheus/alertmanager/config/coordinator.go index e9f3e21d8f4..3ec12bc100d 100644 --- a/vendor/github.com/prometheus/alertmanager/config/coordinator.go +++ b/vendor/github.com/prometheus/alertmanager/config/coordinator.go @@ -16,6 +16,7 @@ package config import ( "crypto/md5" "encoding/binary" + "errors" "log/slog" "sync" @@ -144,6 +145,38 @@ func (c *Coordinator) Reload() error { return nil } +// ApplyConfig accepts an already-loaded configuration, stores it, and +// notifies all subscribers. Use this for the initial load so the file +// is only read once. +func (c *Coordinator) ApplyConfig(conf *Config) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + if conf == nil { + c.configSuccessMetric.Set(0) + return errors.New("nil config passed to ApplyConfig") + } + + c.config = conf + + if err := c.notifySubscribers(); err != nil { + c.logger.Error( + "one or more config change subscribers failed to apply new config", + "file", c.configFilePath, + "err", err, + ) + c.configSuccessMetric.Set(0) + return err + } + + c.configSuccessMetric.Set(1) + c.configSuccessTimeMetric.SetToCurrentTime() + hash := md5HashAsMetricValue([]byte(c.config.original)) + c.configHashMetric.Set(hash) + + return nil +} + func md5HashAsMetricValue(data []byte) float64 { sum := md5.Sum(data) // We only want 48 bits as a float64 only has a 53 bit mantissa. diff --git a/vendor/github.com/prometheus/alertmanager/config/notifiers.go b/vendor/github.com/prometheus/alertmanager/config/notifiers.go index 33980fd1752..46626cb7a3c 100644 --- a/vendor/github.com/prometheus/alertmanager/config/notifiers.go +++ b/vendor/github.com/prometheus/alertmanager/config/notifiers.go @@ -26,24 +26,10 @@ import ( amcommoncfg "github.com/prometheus/alertmanager/config/common" - "github.com/prometheus/common/model" "github.com/prometheus/sigv4" ) var ( - // DefaultIncidentioConfig defines default values for Incident.io configurations. - DefaultIncidentioConfig = IncidentioConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - } - - // DefaultWebhookConfig defines default values for Webhook configurations. - DefaultWebhookConfig = WebhookConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - } // DefaultWebexConfig defines default values for Webex configurations. DefaultWebexConfig = WebexConfig{ @@ -53,15 +39,6 @@ var ( Message: `{{ template "webex.default.message" . }}`, } - // DefaultDiscordConfig defines default values for Discord configurations. - DefaultDiscordConfig = DiscordConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - Title: `{{ template "discord.default.title" . }}`, - Message: `{{ template "discord.default.message" . }}`, - } - // DefaultEmailConfig defines default values for Email configurations. DefaultEmailConfig = EmailConfig{ NotifierConfig: amcommoncfg.NotifierConfig{ @@ -188,15 +165,6 @@ var ( ParseMode: "HTML", } - DefaultMSTeamsConfig = MSTeamsConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - Title: `{{ template "msteams.default.title" . }}`, - Summary: `{{ template "msteams.default.summary" . }}`, - Text: `{{ template "msteams.default.text" . }}`, - } - DefaultMSTeamsV2Config = MSTeamsV2Config{ NotifierConfig: amcommoncfg.NotifierConfig{ VSendResolved: true, @@ -204,32 +172,6 @@ var ( Title: `{{ template "msteamsv2.default.title" . }}`, Text: `{{ template "msteamsv2.default.text" . }}`, } - - DefaultJiraConfig = JiraConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - APIType: "auto", - Summary: JiraFieldConfig{ - Template: `{{ template "jira.default.summary" . }}`, - }, - Description: JiraFieldConfig{ - Template: `{{ template "jira.default.description" . }}`, - }, - Priority: `{{ template "jira.default.priority" . }}`, - } - - DefaultMattermostConfig = MattermostConfig{ - NotifierConfig: amcommoncfg.NotifierConfig{ - VSendResolved: true, - }, - Username: `{{ template "mattermost.default.username" . }}`, - Color: `{{ template "mattermost.default.color" . }}`, - Text: `{{ template "mattermost.default.text" . }}`, - Title: `{{ template "mattermost.default.title" . }}`, - TitleLink: `{{ template "mattermost.default.titlelink" . }}`, - Fallback: `{{ template "mattermost.default.fallback" . }}`, - } ) // WebexConfig configures notifications via Webex. @@ -261,40 +203,6 @@ func (c *WebexConfig) UnmarshalYAML(unmarshal func(any) error) error { return nil } -// DiscordConfig configures notifications via Discord. -type DiscordConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` - WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` - - Content string `yaml:"content,omitempty" json:"content,omitempty"` - Title string `yaml:"title,omitempty" json:"title,omitempty"` - Message string `yaml:"message,omitempty" json:"message,omitempty"` - Username string `yaml:"username,omitempty" json:"username,omitempty"` - AvatarURL string `yaml:"avatar_url,omitempty" json:"avatar_url,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *DiscordConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultDiscordConfig - type plain DiscordConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - - if c.WebhookURL == nil && c.WebhookURLFile == "" { - return errors.New("one of webhook_url or webhook_url_file must be configured") - } - - if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { - return errors.New("at most one of webhook_url & webhook_url_file must be configured") - } - - return nil -} - // EmailConfig configures notifications via mail. type EmailConfig struct { amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` @@ -589,95 +497,6 @@ func (c *SlackConfig) UnmarshalYAML(unmarshal func(any) error) error { return nil } -// IncidentioConfig configures notifications via incident.io. -type IncidentioConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - - // URL to send POST request to. - URL *amcommoncfg.URL `yaml:"url" json:"url"` - URLFile string `yaml:"url_file" json:"url_file"` - - // AlertSourceToken is the key used to authenticate with the alert source in incident.io. - AlertSourceToken commoncfg.Secret `yaml:"alert_source_token,omitempty" json:"alert_source_token,omitempty"` - AlertSourceTokenFile string `yaml:"alert_source_token_file,omitempty" json:"alert_source_token_file,omitempty"` - - // MaxAlerts is the maximum number of alerts to be sent per incident.io message. - // Alerts exceeding this threshold will be truncated. Setting this to 0 - // allows an unlimited number of alerts. Note that if the payload exceeds - // incident.io's size limits, you will receive a 429 response and alerts - // will not be ingested. - MaxAlerts uint64 `yaml:"max_alerts" json:"max_alerts"` - - // Timeout is the maximum time allowed to invoke incident.io. Setting this to 0 - // does not impose a timeout. - Timeout time.Duration `yaml:"timeout" json:"timeout"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *IncidentioConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultIncidentioConfig - type plain IncidentioConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - if c.URL == nil && c.URLFile == "" { - return errors.New("one of url or url_file must be configured") - } - if c.URL != nil && c.URLFile != "" { - return errors.New("at most one of url & url_file must be configured") - } - if c.AlertSourceToken != "" && c.AlertSourceTokenFile != "" { - return errors.New("at most one of alert_source_token & alert_source_token_file must be configured") - } - if c.HTTPConfig != nil && c.HTTPConfig.Authorization != nil && (c.AlertSourceToken != "" || c.AlertSourceTokenFile != "") { - return errors.New("cannot specify alert_source_token or alert_source_token_file when using http_config.authorization") - } - - if (c.HTTPConfig != nil && c.HTTPConfig.Authorization == nil) && c.AlertSourceToken == "" && c.AlertSourceTokenFile == "" { - return errors.New("at least one of alert_source_token, alert_source_token_file or http_config.authorization must be configured") - } - return nil -} - -// WebhookConfig configures notifications via a generic webhook. -type WebhookConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - - // URL to send POST request to. - URL SecretTemplateURL `yaml:"url,omitempty" json:"url,omitempty"` - URLFile string `yaml:"url_file" json:"url_file"` - - // MaxAlerts is the maximum number of alerts to be sent per webhook message. - // Alerts exceeding this threshold will be truncated. Setting this to 0 - // allows an unlimited number of alerts. - MaxAlerts uint64 `yaml:"max_alerts" json:"max_alerts"` - - // Timeout is the maximum time allowed to invoke the webhook. Setting this to 0 - // does not impose a timeout. - Timeout time.Duration `yaml:"timeout" json:"timeout"` - Payload any `yaml:"payload,omitempty" json:"payload,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *WebhookConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultWebhookConfig - type plain WebhookConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - if c.URL == "" && c.URLFile == "" { - return errors.New("one of url or url_file must be configured") - } - if c.URL != "" && c.URLFile != "" { - return errors.New("at most one of url & url_file must be configured") - } - return nil -} - // WechatConfig configures notifications via Wechat. type WechatConfig struct { amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` @@ -908,6 +727,10 @@ type SNSConfig struct { Subject string `yaml:"subject,omitempty" json:"subject,omitempty"` Message string `yaml:"message,omitempty" json:"message,omitempty"` Attributes map[string]string `yaml:"attributes,omitempty" json:"attributes,omitempty"` + // UseAWSHTTPClient forces the AWS SDK's BuildableClient instead of + // alertmanager's tracing-wrapped HTTP client. Auto-enabled when AWS_CA_BUNDLE + // is set; set explicitly when configuring ca_bundle via shared AWS config. + UseAWSHTTPClient bool `yaml:"use_aws_http_client,omitempty" json:"use_aws_http_client,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -965,35 +788,6 @@ func (c *TelegramConfig) UnmarshalYAML(unmarshal func(any) error) error { return nil } -type MSTeamsConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` - WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` - - Title string `yaml:"title,omitempty" json:"title,omitempty"` - Summary string `yaml:"summary,omitempty" json:"summary,omitempty"` - Text string `yaml:"text,omitempty" json:"text,omitempty"` -} - -func (c *MSTeamsConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultMSTeamsConfig - type plain MSTeamsConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - - if c.WebhookURL == nil && c.WebhookURLFile == "" { - return errors.New("one of webhook_url or webhook_url_file must be configured") - } - - if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { - return errors.New("at most one of webhook_url & webhook_url_file must be configured") - } - - return nil -} - type MSTeamsV2Config struct { amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` @@ -1022,78 +816,6 @@ func (c *MSTeamsV2Config) UnmarshalYAML(unmarshal func(any) error) error { return nil } -type JiraFieldConfig struct { - // Template is the template string used to render the field. - Template string `yaml:"template,omitempty" json:"template,omitempty"` - // EnableUpdate indicates whether this field should be omitted when updating an existing issue. - EnableUpdate *bool `yaml:"enable_update,omitempty" json:"enable_update,omitempty"` -} - -type JiraConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - - APIURL *amcommoncfg.URL `yaml:"api_url,omitempty" json:"api_url,omitempty"` - APIType string `yaml:"api_type,omitempty" json:"api_type,omitempty"` - - Project string `yaml:"project,omitempty" json:"project,omitempty"` - Summary JiraFieldConfig `yaml:"summary,omitempty" json:"summary,omitempty"` - Description JiraFieldConfig `yaml:"description,omitempty" json:"description,omitempty"` - Labels []string `yaml:"labels,omitempty" json:"labels,omitempty"` - Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` - IssueType string `yaml:"issue_type,omitempty" json:"issue_type,omitempty"` - - ReopenTransition string `yaml:"reopen_transition,omitempty" json:"reopen_transition,omitempty"` - ResolveTransition string `yaml:"resolve_transition,omitempty" json:"resolve_transition,omitempty"` - WontFixResolution string `yaml:"wont_fix_resolution,omitempty" json:"wont_fix_resolution,omitempty"` - ReopenDuration model.Duration `yaml:"reopen_duration,omitempty" json:"reopen_duration,omitempty"` - - Fields map[string]any `yaml:"fields,omitempty" json:"custom_fields,omitempty"` -} - -func (f *JiraFieldConfig) EnableUpdateValue() bool { - if f.EnableUpdate == nil { - return true - } - return *f.EnableUpdate -} - -// Supports both the legacy string and the new object form. -func (f *JiraFieldConfig) UnmarshalYAML(unmarshal func(any) error) error { - // Try simple string first (backward compatibility). - var s string - if err := unmarshal(&s); err == nil { - f.Template = s - // DisableUpdate stays false by default. - return nil - } - - // Fallback to full object form. - type plain JiraFieldConfig - return unmarshal((*plain)(f)) -} - -func (c *JiraConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultJiraConfig - type plain JiraConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - - if c.Project == "" { - return errors.New("missing project in jira_config") - } - if c.IssueType == "" { - return errors.New("missing issue_type in jira_config") - } - if c.APIType != "auto" && - c.APIType != "cloud" && - c.APIType != "datacenter" { - return errors.New("unknown api_type on jira_config, must be auto, cloud or datacenter") - } - return nil -} - type RocketchatAttachmentField struct { Short *bool `json:"short"` Title string `json:"title,omitempty"` @@ -1161,107 +883,3 @@ func (c *RocketchatConfig) UnmarshalYAML(unmarshal func(any) error) error { } return nil } - -// MattermostPriority defines the priority for a mattermost notification. -type MattermostPriority struct { - Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` - RequestedAck bool `yaml:"requested_ack,omitempty" json:"requested_ack,omitempty"` - PersistentNotifications bool `yaml:"persistent_notifications,omitempty" json:"persistent_notifications,omitempty"` -} - -// MattermostProps defines additional properties for a mattermost notification. -// Only 'card' property takes effect now. -type MattermostProps struct { - Card string `yaml:"card,omitempty" json:"card,omitempty"` -} - -// MattermostField configures a single Mattermost field for Slack compatibility. -// See https://developers.mattermost.com/integrate/reference/message-attachments/#fields for more information. -type MattermostField struct { - Title string `yaml:"title,omitempty" json:"title,omitempty"` - Value string `yaml:"value,omitempty" json:"value,omitempty"` - Short *bool `yaml:"short,omitempty" json:"short,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface for MattermostField. -func (c *MattermostField) UnmarshalYAML(unmarshal func(any) error) error { - type plain MattermostField - if err := unmarshal((*plain)(c)); err != nil { - return err - } - if c.Title == "" { - return errors.New("missing title in Mattermost field configuration") - } - if c.Value == "" { - return errors.New("missing value in Mattermost field configuration") - } - return nil -} - -// MattermostAttachment defines an attachment for a Mattermost notification. -// See https://developers.mattermost.com/integrate/reference/message-attachments/#fields for more information. -type MattermostAttachment struct { - Fallback string `yaml:"fallback,omitempty" json:"fallback,omitempty"` - Color string `yaml:"color,omitempty" json:"color,omitempty"` - Pretext string `yaml:"pretext,omitempty" json:"pretext,omitempty"` - Text string `yaml:"text,omitempty" json:"text,omitempty"` - AuthorName string `yaml:"author_name,omitempty" json:"author_name,omitempty"` - AuthorLink string `yaml:"author_link,omitempty" json:"author_link,omitempty"` - AuthorIcon string `yaml:"author_icon,omitempty" json:"author_icon,omitempty"` - Title string `yaml:"title,omitempty" json:"title,omitempty"` - TitleLink string `yaml:"title_link,omitempty" json:"title_link,omitempty"` - Fields []*MattermostField `yaml:"fields,omitempty" json:"fields,omitempty"` - ThumbURL string `yaml:"thumb_url,omitempty" json:"thumb_url,omitempty"` - Footer string `yaml:"footer,omitempty" json:"footer,omitempty"` - FooterIcon string `yaml:"footer_icon,omitempty" json:"footer_icon,omitempty"` - ImageURL string `yaml:"image_url,omitempty" json:"image_url,omitempty"` -} - -// MattermostConfig configures notifications via Mattermost. -// See https://developers.mattermost.com/integrate/webhooks/incoming/ for more information. -type MattermostConfig struct { - amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` - - HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` - WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` - - Channel string `yaml:"channel,omitempty" json:"channel,omitempty"` - Username string `yaml:"username,omitempty" json:"username,omitempty"` - - Text string `yaml:"text,omitempty" json:"text,omitempty"` - Fallback string `yaml:"fallback,omitempty" json:"fallback,omitempty"` - Color string `yaml:"color,omitempty" json:"color,omitempty"` - Pretext string `yaml:"pretext,omitempty" json:"pretext,omitempty"` - AuthorName string `yaml:"author_name,omitempty" json:"author_name,omitempty"` - AuthorLink string `yaml:"author_link,omitempty" json:"author_link,omitempty"` - AuthorIcon string `yaml:"author_icon,omitempty" json:"author_icon,omitempty"` - Title string `yaml:"title,omitempty" json:"title,omitempty"` - TitleLink string `yaml:"title_link,omitempty" json:"title_link,omitempty"` - Fields []*MattermostField `yaml:"fields,omitempty" json:"fields,omitempty"` - ThumbURL string `yaml:"thumb_url,omitempty" json:"thumb_url,omitempty"` - Footer string `yaml:"footer,omitempty" json:"footer,omitempty"` - FooterIcon string `yaml:"footer_icon,omitempty" json:"footer_icon,omitempty"` - ImageURL string `yaml:"image_url,omitempty" json:"image_url,omitempty"` - IconURL string `yaml:"icon_url,omitempty" json:"icon_url,omitempty"` - IconEmoji string `yaml:"icon_emoji,omitempty" json:"icon_emoji,omitempty"` - Attachments []*MattermostAttachment `yaml:"attachments,omitempty" json:"attachments,omitempty"` - Type string `yaml:"type,omitempty" json:"type,omitempty"` - Props *MattermostProps `yaml:"props,omitempty" json:"props,omitempty"` - Priority *MattermostPriority `yaml:"priority,omitempty" json:"priority,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *MattermostConfig) UnmarshalYAML(unmarshal func(any) error) error { - *c = DefaultMattermostConfig - type plain MattermostConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - - if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { - return errors.New("at most one of webhook_url & webhook_url_file must be configured") - } - - return nil -} diff --git a/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go b/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go index 5990f135700..fea22a215a8 100644 --- a/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go +++ b/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go @@ -24,8 +24,7 @@ import ( "sync/atomic" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/google/uuid" "github.com/prometheus/common/model" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -34,7 +33,11 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/eventrecorder" + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/alertmanager/provider" "github.com/prometheus/alertmanager/store" "github.com/prometheus/alertmanager/tracing" @@ -50,63 +53,13 @@ const ( var tracer = tracing.NewTracer("github.com/prometheus/alertmanager/dispatch") -// DispatcherMetrics represents metrics associated to a dispatcher. -type DispatcherMetrics struct { - aggrGroups prometheus.Gauge - processingDuration prometheus.Summary - aggrGroupLimitReached prometheus.Counter - aggrGroupCreationRetries prometheus.Counter - aggrGroupCreationGivenUp prometheus.Counter -} - -// NewDispatcherMetrics returns a new registered DispatchMetrics. -func NewDispatcherMetrics(registerLimitMetrics bool, r prometheus.Registerer) *DispatcherMetrics { - if r == nil { - return nil - } - m := DispatcherMetrics{ - aggrGroups: promauto.With(r).NewGauge( - prometheus.GaugeOpts{ - Name: "alertmanager_dispatcher_aggregation_groups", - Help: "Number of active aggregation groups", - }, - ), - processingDuration: promauto.With(r).NewSummary( - prometheus.SummaryOpts{ - Name: "alertmanager_dispatcher_alert_processing_duration_seconds", - Help: "Summary of latencies for the processing of alerts.", - }, - ), - aggrGroupLimitReached: promauto.With(r).NewCounter( - prometheus.CounterOpts{ - Name: "alertmanager_dispatcher_aggregation_group_limit_reached_total", - Help: "Number of times when dispatcher failed to create new aggregation group due to limit.", - }, - ), - aggrGroupCreationRetries: promauto.With(r).NewCounter( - prometheus.CounterOpts{ - Name: "alertmanager_dispatcher_aggregation_group_creation_retries_total", - Help: "Number of CAS retries while creating aggregation groups under contention.", - }, - ), - aggrGroupCreationGivenUp: promauto.With(r).NewCounter( - prometheus.CounterOpts{ - Name: "alertmanager_dispatcher_aggregation_group_creation_given_up_total", - Help: "Number of alerts dropped because aggregation group creation exceeded the retry limit.", - }, - ), - } - - return &m -} - // Dispatcher sorts incoming alerts into aggregation groups and // assigns the correct notifiers to each. type Dispatcher struct { route *Route alerts provider.Alerts stage notify.Stage - marker types.GroupMarker + marker marker.GroupMarker metrics *DispatcherMetrics limits Limits propagator propagation.TextMapPropagator @@ -124,7 +77,8 @@ type Dispatcher struct { maintenanceInterval time.Duration concurrency int // Number of goroutines for alert ingestion - logger *slog.Logger + logger *slog.Logger + recorder eventrecorder.Recorder startTimer *time.Timer state atomic.Int32 @@ -149,16 +103,20 @@ func NewDispatcher( alerts provider.Alerts, route *Route, stage notify.Stage, - marker types.GroupMarker, + marker marker.GroupMarker, timeout func(time.Duration) time.Duration, maintenanceInterval time.Duration, limits Limits, logger *slog.Logger, + recorder eventrecorder.Recorder, metrics *DispatcherMetrics, ) *Dispatcher { if limits == nil { limits = nilLimits{} } + if metrics == nil { + metrics = newNoopDispatcherMetrics() + } // Calculate concurrency for ingestion. concurrency := min(max(runtime.GOMAXPROCS(0)/2, 2), 8) @@ -172,13 +130,19 @@ func NewDispatcher( maintenanceInterval: maintenanceInterval, concurrency: concurrency, logger: logger.With("component", "dispatcher"), + recorder: recorder, metrics: metrics, limits: limits, propagator: otel.GetTextMapPropagator(), } disp.state.Store(DispatcherStateUnknown) disp.loaded = make(chan struct{}) - disp.ctx, disp.cancel = context.WithCancel(context.Background()) + disp.ctx, disp.cancel = context.WithCancel(eventrecorder.WithEventRecording(context.Background())) + + if metrics != nil && metrics.alertsCollector != nil { + metrics.alertsCollector.dispatcher.Store(disp) + } + return disp } @@ -317,9 +281,14 @@ func (d *Dispatcher) doMaintenance() { ag := el.(*aggrGroup) if ag.destroyed() { ag.stop() - d.marker.DeleteByGroupKey(ag.routeID, ag.GroupKey()) deleted := d.routeGroupsSlice[i].groups.CompareAndDelete(ag.fingerprint(), ag) if deleted { + // TODO(ultrotter, siavash): + // Deletion from the marker should only happen if we really deleted the group. + // Fully fixing the case where a new group with the same fingerprint is created between + // CompareAndDelete and DeleteByGroupKey would require changes to the marker interface, + // so we leave it as a fix for after landing the pending marker changes. + d.marker.DeleteByGroupKey(ag.routeID, ag.GroupKey()) d.routeGroupsSlice[i].groupsLen.Add(-1) d.aggrGroupsNum.Add(-1) d.metrics.aggrGroups.Set(float64(d.aggrGroupsNum.Load())) @@ -340,11 +309,12 @@ func (d *Dispatcher) LoadingDone() <-chan struct{} { // AlertGroup represents how alerts exist within an aggrGroup. type AlertGroup struct { - Alerts types.AlertSlice - Labels model.LabelSet - Receiver string - GroupKey string - RouteID string + Alerts alert.AlertSlice + Labels model.LabelSet + Receiver string + GroupKey string + RouteID string + AlertStatuses map[model.Fingerprint]alert.AlertStatus } type AlertGroups []*AlertGroup @@ -359,7 +329,7 @@ func (ag AlertGroups) Less(i, j int) bool { func (ag AlertGroups) Len() int { return len(ag) } // Groups returns a slice of AlertGroups from the dispatcher's internal state. -func (d *Dispatcher) Groups(ctx context.Context, routeFilter func(*Route) bool, alertFilter func(*types.Alert, time.Time) bool) (AlertGroups, map[model.Fingerprint][]string, error) { +func (d *Dispatcher) Groups(ctx context.Context, routeFilter func(*Route) bool, alertFilter func(*alert.Alert, time.Time) bool) (AlertGroups, map[model.Fingerprint][]string, error) { select { case <-ctx.Done(): return nil, nil, ctx.Err() @@ -401,7 +371,7 @@ func (d *Dispatcher) Groups(ctx context.Context, routeFilter func(*Route) bool, } alerts := ag.alerts.List() - filteredAlerts := make([]*types.Alert, 0, len(alerts)) + filteredAlerts := make([]*alert.Alert, 0, len(alerts)) for _, a := range alerts { if !alertFilter(a, now) { continue @@ -424,6 +394,10 @@ func (d *Dispatcher) Groups(ctx context.Context, routeFilter func(*Route) bool, continue } alertGroup.Alerts = filteredAlerts + alertGroup.AlertStatuses = make(map[model.Fingerprint]alert.AlertStatus, len(filteredAlerts)) + for _, a := range filteredAlerts { + alertGroup.AlertStatuses[a.Fingerprint()] = ag.marker.Status(a.Fingerprint()) + } groups = append(groups, alertGroup) } @@ -452,11 +426,11 @@ func (d *Dispatcher) Stop() { // notifyFunc is a function that performs notification for the alert // with the given fingerprint. It aborts on context cancelation. // Returns false if notifying failed. -type notifyFunc func(context.Context, ...*types.Alert) bool +type notifyFunc func(context.Context, ...*alert.Alert) bool // groupAlert determines in which aggregation group the alert falls // and inserts it. -func (d *Dispatcher) groupAlert(ctx context.Context, alert *types.Alert, route *Route) { +func (d *Dispatcher) groupAlert(ctx context.Context, alert *alert.Alert, route *Route) { _, span := tracer.Start(ctx, "dispatch.Dispatcher.groupAlert", trace.WithAttributes( attribute.String("alerting.alert.name", alert.Name()), @@ -504,7 +478,7 @@ func (d *Dispatcher) groupAlert(ctx context.Context, alert *types.Alert, route * return } - ag := newAggrGroup(d.ctx, groupLabels, route, d.timeout, d.marker.(types.AlertMarker), d.logger) + ag := newAggrGroup(d.ctx, groupLabels, route, d.timeout, d.recorder, d.logger) // Insert the 1st alert in the group before starting the group's run() // function, to make sure that when the run() will be executed the 1st // alert is already there. @@ -516,6 +490,9 @@ func (d *Dispatcher) groupAlert(ctx context.Context, alert *types.Alert, route * // Try to store the new group in the map. If another goroutine has already created the same group, use the existing one. swapped := d.routeGroupsSlice[route.Idx].groups.CompareAndSwap(fp, el, ag) if swapped { + // Since we swapped the new group in, we need to cancel the old one, + // as doMaintenance will not be able to find it in the map anymore. + el.(*aggrGroup).cancel() // We swapped the new group in, we can break and start it. break } @@ -590,7 +567,7 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { if !ag.running.CompareAndSwap(false, true) { return // already running } - go ag.run(func(ctx context.Context, alerts ...*types.Alert) bool { + go ag.run(func(ctx context.Context, alerts ...*alert.Alert) bool { _, _, err := d.stage.Exec(ctx, d.logger, alerts...) if err != nil { logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "err", err) @@ -607,7 +584,7 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { }) } -func getGroupLabels(alert *types.Alert, route *Route) model.LabelSet { +func getGroupLabels(alert *alert.Alert, route *Route) model.LabelSet { capacity := len(route.RouteOpts.GroupBy) if route.RouteOpts.GroupByAll { capacity = len(alert.Labels) @@ -631,15 +608,18 @@ type aggrGroup struct { logger *slog.Logger routeID string routeKey string + matchers labels.Matchers - alerts *store.Alerts - marker types.AlertMarker - ctx context.Context - cancel func() - done chan struct{} - next *time.Timer - timeout func(time.Duration) time.Duration - running atomic.Bool + alerts *store.Alerts + marker marker.AlertMarker + recorder eventrecorder.Recorder + ctx context.Context + cancel func() + done chan struct{} + next *time.Timer + timeout func(time.Duration) time.Duration + running atomic.Bool + flushIdx uint64 } // newAggrGroup returns a new aggregation group. @@ -648,7 +628,7 @@ func newAggrGroup( labels model.LabelSet, r *Route, to func(time.Duration) time.Duration, - marker types.AlertMarker, + recorder eventrecorder.Recorder, logger *slog.Logger, ) *aggrGroup { if to == nil { @@ -658,14 +638,21 @@ func newAggrGroup( labels: labels, routeID: r.ID(), routeKey: r.Key(), + matchers: r.Matchers, opts: &r.RouteOpts, timeout: to, alerts: store.NewAlerts(), - marker: marker, + marker: marker.NewAlertMarker(), + recorder: recorder, done: make(chan struct{}), + flushIdx: 1, } ag.ctx, ag.cancel = context.WithCancel(ctx) + if id, err := uuid.NewRandom(); err == nil { + ag.ctx = notify.WithAggrGroupID(ag.ctx, id.String()) + } + ag.logger = logger.With("aggrGroup", ag.GroupKey()) // Set an initial one-time wait before flushing @@ -712,11 +699,16 @@ func (ag *aggrGroup) run(nf notifyFunc) { ctx = notify.WithMuteTimeIntervals(ctx, ag.opts.MuteTimeIntervals) ctx = notify.WithActiveTimeIntervals(ctx, ag.opts.ActiveTimeIntervals) ctx = notify.WithRouteID(ctx, ag.routeID) + ctx = notify.WithFlushID(ctx, ag.flushIdx) + ctx = notify.WithGroupMatchers(ctx, ag.matchers) + ctx = marker.WithContext(ctx, ag.marker) + + ag.flushIdx++ // Wait the configured interval before calling flush again. ag.resetTimer(ag.opts.GroupInterval) - ag.flush(func(alerts ...*types.Alert) bool { + ag.flush(func(alerts ...*alert.Alert) bool { ctx, span := tracer.Start(ctx, "dispatch.AggregationGroup.flush", trace.WithAttributes( attribute.String("alerting.aggregation_group.key", ag.GroupKey()), @@ -735,6 +727,11 @@ func (ag *aggrGroup) run(nf notifyFunc) { cancel() + // If destroyed, exit: this particular alert group won't be used anymore. + if ag.destroyed() { + return + } + case <-ag.ctx.Done(): return } @@ -755,7 +752,7 @@ func (ag *aggrGroup) resetTimer(t time.Duration) { // insert inserts the alert into the aggregation group. // Returns false if the aggregation group has been destroyed. -func (ag *aggrGroup) insert(ctx context.Context, alert *types.Alert) bool { +func (ag *aggrGroup) insert(ctx context.Context, alert *alert.Alert) bool { _, span := tracer.Start(ctx, "dispatch.AggregationGroup.insert", trace.WithAttributes( attribute.String("alerting.alert.name", alert.Name()), @@ -773,6 +770,8 @@ func (ag *aggrGroup) insert(ctx context.Context, alert *types.Alert) bool { span.SetStatus(codes.Error, message) span.RecordError(err) ag.logger.Error(message, "err", err) + } else { + ag.recorder.RecordEvent(ctx, notify.NewAlertGroupedEvent(ag.alertGroupInfo(), alert)) } return true } @@ -786,7 +785,7 @@ func (ag *aggrGroup) destroyed() bool { } // flush sends notifications for all new alerts. -func (ag *aggrGroup) flush(notify func(...*types.Alert) bool) { +func (ag *aggrGroup) flush(notify func(...*alert.Alert) bool) { if ag.empty() { return } @@ -812,6 +811,8 @@ func (ag *aggrGroup) flush(notify func(...*types.Alert) bool) { ag.logger.Debug("flushing", "numAlerts", len(alertsSlice), "alerts", alertsSlice) if notify(alertsSlice...) { + ag.recordResolvedEvents(resolvedSlice) + // Delete all resolved alerts as we just sent a notification for them, // and we don't want to send another one. However, we need to make sure // that each resolved alert has not fired again during the flush as then @@ -832,6 +833,25 @@ func (ag *aggrGroup) flush(notify func(...*types.Alert) bool) { } } +func (ag *aggrGroup) recordResolvedEvents(resolved types.AlertSlice) { + if len(resolved) == 0 { + return + } + groupInfo := ag.alertGroupInfo() + for _, a := range resolved { + ag.recorder.RecordEvent(ag.ctx, notify.NewAlertResolvedEvent(groupInfo, a)) + } +} + +func (ag *aggrGroup) alertGroupInfo() *eventrecorderpb.AlertGroupInfo { + return &eventrecorderpb.AlertGroupInfo{ + GroupKey: ag.GroupKey(), + GroupLabels: eventrecorder.LabelSetAsProto(ag.labels), + GroupId: notify.Key(ag.GroupKey()).Hash(), + ReceiverName: ag.opts.Receiver, + } +} + type nilLimits struct{} func (n nilLimits) MaxNumberOfAggregationGroups() int { return 0 } diff --git a/vendor/github.com/prometheus/alertmanager/dispatch/metric.go b/vendor/github.com/prometheus/alertmanager/dispatch/metric.go new file mode 100644 index 00000000000..f0f0fe2640a --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/dispatch/metric.go @@ -0,0 +1,208 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatch + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/featurecontrol" +) + +// DispatcherMetrics represents metrics associated to a dispatcher. +type DispatcherMetrics struct { + aggrGroups prometheus.Gauge + processingDuration prometheus.Summary + aggrGroupLimitReached prometheus.Counter + aggrGroupCreationRetries prometheus.Counter + aggrGroupCreationGivenUp prometheus.Counter + alertsCollector *alertStateCollector +} + +// newNoopDispatcherMetrics returns a DispatcherMetrics whose counters, +// gauges, and summaries silently discard observations. It is used when +// no prometheus.Registerer is provided. +func newNoopDispatcherMetrics() *DispatcherMetrics { + return &DispatcherMetrics{ + aggrGroups: prometheus.NewGauge(prometheus.GaugeOpts{}), + processingDuration: prometheus.NewSummary(prometheus.SummaryOpts{}), + aggrGroupLimitReached: prometheus.NewCounter(prometheus.CounterOpts{}), + aggrGroupCreationRetries: prometheus.NewCounter(prometheus.CounterOpts{}), + aggrGroupCreationGivenUp: prometheus.NewCounter(prometheus.CounterOpts{}), + } +} + +// NewDispatcherMetrics returns a new registered DispatchMetrics. +func NewDispatcherMetrics(_ bool, r prometheus.Registerer, ff featurecontrol.Flagger) *DispatcherMetrics { + if r == nil { + return newNoopDispatcherMetrics() + } + if ff == nil { + ff = featurecontrol.NoopFlags{} + } + + labels := []string{"state"} + if ff.EnableGroupKeyInMetrics() { + labels = append(labels, "group_key") + } + + collector := &alertStateCollector{ + desc: prometheus.NewDesc( + "alertmanager_alerts", + "How many alerts by state.", + labels, nil, + ), + enableGroupKey: ff.EnableGroupKeyInMetrics(), + } + r.MustRegister(collector) + + m := DispatcherMetrics{ + aggrGroups: promauto.With(r).NewGauge( + prometheus.GaugeOpts{ + Name: "alertmanager_dispatcher_aggregation_groups", + Help: "Number of active aggregation groups", + }, + ), + processingDuration: promauto.With(r).NewSummary( + prometheus.SummaryOpts{ + Name: "alertmanager_dispatcher_alert_processing_duration_seconds", + Help: "Summary of latencies for the processing of alerts.", + }, + ), + aggrGroupLimitReached: promauto.With(r).NewCounter( + prometheus.CounterOpts{ + Name: "alertmanager_dispatcher_aggregation_group_limit_reached_total", + Help: "Number of times when dispatcher failed to create new aggregation group due to limit.", + }, + ), + aggrGroupCreationRetries: promauto.With(r).NewCounter( + prometheus.CounterOpts{ + Name: "alertmanager_dispatcher_aggregation_group_creation_retries_total", + Help: "Number of CAS retries while creating aggregation groups under contention.", + }, + ), + aggrGroupCreationGivenUp: promauto.With(r).NewCounter( + prometheus.CounterOpts{ + Name: "alertmanager_dispatcher_aggregation_group_creation_given_up_total", + Help: "Number of alerts dropped because aggregation group creation exceeded the retry limit.", + }, + ), + alertsCollector: collector, + } + + return &m +} + +// alertStateCollector implements prometheus.Collector to collect alert count +// metrics by state from the dispatcher's aggregation groups. +type alertStateCollector struct { + desc *prometheus.Desc + dispatcher atomic.Pointer[Dispatcher] + enableGroupKey bool +} + +func (c *alertStateCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +func (c *alertStateCollector) Collect(ch chan<- prometheus.Metric) { + d := c.dispatcher.Load() + if d == nil { + return + } + if d.state.Load() != DispatcherStateRunning { + return + } + + if c.enableGroupKey { + labelValues := make([]string, 2) + for i := range d.routeGroupsSlice { + d.routeGroupsSlice[i].groups.Range(func(_, el any) bool { + ag := el.(*aggrGroup) + active, suppressed, unprocessed := ag.countAlertsByState() + labelValues[1] = ag.GroupKey() + labelValues[0] = string(alert.AlertStateActive) + c.emit(ch, float64(active), labelValues...) + labelValues[0] = string(alert.AlertStateSuppressed) + c.emit(ch, float64(suppressed), labelValues...) + labelValues[0] = string(alert.AlertStateUnprocessed) + c.emit(ch, float64(unprocessed), labelValues...) + return true + }) + } + return + } + + // Deduplicate by fingerprint for backward compatibility. + // The same alert can live in multiple aggregation groups with + // different per-group marker states. Use highest-priority state: + // suppressed > active > unprocessed. + seen := map[model.Fingerprint]alert.AlertState{} + for i := range d.routeGroupsSlice { + d.routeGroupsSlice[i].groups.Range(func(_, el any) bool { + ag := el.(*aggrGroup) + for _, a := range ag.alerts.List() { + fp := a.Fingerprint() + if !a.Resolved() { + state := ag.marker.Status(fp).State + if prev, ok := seen[fp]; !ok || state.Compare(prev) > 0 { + seen[fp] = state + } + } + } + return true + }) + } + var active, suppressed, unprocessed int + for _, state := range seen { + switch state { + case alert.AlertStateActive: + active++ + case alert.AlertStateSuppressed: + suppressed++ + default: + unprocessed++ + } + } + c.emit(ch, float64(active), string(alert.AlertStateActive)) + c.emit(ch, float64(suppressed), string(alert.AlertStateSuppressed)) + c.emit(ch, float64(unprocessed), string(alert.AlertStateUnprocessed)) +} + +// countAlertsByState counts non-resolved alerts in the group by their marker state. +func (ag *aggrGroup) countAlertsByState() (active, suppressed, unprocessed int) { + for _, a := range ag.alerts.List() { + if a.Resolved() { + continue + } + switch ag.marker.Status(a.Fingerprint()).State { + case alert.AlertStateActive: + active++ + case alert.AlertStateSuppressed: + suppressed++ + default: + unprocessed++ + } + } + return active, suppressed, unprocessed +} + +// emit sends a gauge metric with the given count and labels. +func (c *alertStateCollector) emit(ch chan<- prometheus.Metric, count float64, labelValues ...string) { + ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, count, labelValues...) +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/config.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/config.go new file mode 100644 index 00000000000..ff8ecf47f5b --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/config.go @@ -0,0 +1,60 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventrecorder + +// Config configures the event recorder feature. +// +// Outputs are grouped by type, one list per destination kind, mirroring +// the way receivers group their integrations (e.g. webhook_configs, +// email_configs). Every recorded event is fanned out to every output +// across all lists. +type Config struct { + FileOutputs []FileOutputConfig `yaml:"file_outputs,omitempty" json:"file_outputs,omitempty"` + WebhookOutputs []WebhookOutputConfig `yaml:"webhook_outputs,omitempty" json:"webhook_outputs,omitempty"` + KafkaOutputs []KafkaOutputConfig `yaml:"kafka_outputs,omitempty" json:"kafka_outputs,omitempty"` +} + +// totalOutputs returns the number of configured outputs across all +// destination kinds. +func (c Config) totalOutputs() int { + return len(c.FileOutputs) + len(c.WebhookOutputs) + len(c.KafkaOutputs) +} + +// configEqual compares two Config values by their semantically +// significant fields. Each per-type list is compared element-wise via +// the type's equal helper (defined alongside that output's +// implementation in file.go, webhook.go, kafka.go). +func configEqual(a, b Config) bool { + if len(a.FileOutputs) != len(b.FileOutputs) || + len(a.WebhookOutputs) != len(b.WebhookOutputs) || + len(a.KafkaOutputs) != len(b.KafkaOutputs) { + return false + } + for i := range a.FileOutputs { + if !a.FileOutputs[i].equal(b.FileOutputs[i]) { + return false + } + } + for i := range a.WebhookOutputs { + if !a.WebhookOutputs[i].equal(b.WebhookOutputs[i]) { + return false + } + } + for i := range a.KafkaOutputs { + if !a.KafkaOutputs[i].equal(b.KafkaOutputs[i]) { + return false + } + } + return true +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.pb.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.pb.go new file mode 100644 index 00000000000..fb508379795 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.pb.go @@ -0,0 +1,2096 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: eventrecorder.proto + +package eventrecorderpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// NotifyReason describes why a notification was sent for an aggregation +// group. +type NotifyReason int32 + +const ( + // Default / unknown reason. + NotifyReason_NOTIFY_REASON_UNSPECIFIED NotifyReason = 0 + // The group has never been notified before and contains at least one + // firing alert. + NotifyReason_NOTIFY_REASON_FIRST_NOTIFICATION NotifyReason = 1 + // New firing alerts have been added to the group since the last + // notification. + NotifyReason_NOTIFY_REASON_NEW_ALERTS_IN_GROUP NotifyReason = 2 + // Some alerts in the group have resolved since the last notification. + NotifyReason_NOTIFY_REASON_NEW_RESOLVED_ALERTS NotifyReason = 3 + // All alerts in the group have resolved. + NotifyReason_NOTIFY_REASON_ALL_ALERTS_RESOLVED NotifyReason = 4 + // The configured repeat interval has elapsed since the last + // notification. + NotifyReason_NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED NotifyReason = 5 +) + +// Enum value maps for NotifyReason. +var ( + NotifyReason_name = map[int32]string{ + 0: "NOTIFY_REASON_UNSPECIFIED", + 1: "NOTIFY_REASON_FIRST_NOTIFICATION", + 2: "NOTIFY_REASON_NEW_ALERTS_IN_GROUP", + 3: "NOTIFY_REASON_NEW_RESOLVED_ALERTS", + 4: "NOTIFY_REASON_ALL_ALERTS_RESOLVED", + 5: "NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED", + } + NotifyReason_value = map[string]int32{ + "NOTIFY_REASON_UNSPECIFIED": 0, + "NOTIFY_REASON_FIRST_NOTIFICATION": 1, + "NOTIFY_REASON_NEW_ALERTS_IN_GROUP": 2, + "NOTIFY_REASON_NEW_RESOLVED_ALERTS": 3, + "NOTIFY_REASON_ALL_ALERTS_RESOLVED": 4, + "NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED": 5, + } +) + +func (x NotifyReason) Enum() *NotifyReason { + p := new(NotifyReason) + *p = x + return p +} + +func (x NotifyReason) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NotifyReason) Descriptor() protoreflect.EnumDescriptor { + return file_eventrecorder_proto_enumTypes[0].Descriptor() +} + +func (NotifyReason) Type() protoreflect.EnumType { + return &file_eventrecorder_proto_enumTypes[0] +} + +func (x NotifyReason) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NotifyReason.Descriptor instead. +func (NotifyReason) EnumDescriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{0} +} + +// Type enumerates the supported matching operators. +type Matcher_Type int32 + +const ( + // Unspecified / unknown match type. + Matcher_TYPE_UNSPECIFIED Matcher_Type = 0 + // Exact string equality (=). + Matcher_TYPE_EQUAL Matcher_Type = 1 + // Regular expression match (=~). + Matcher_TYPE_REGEXP Matcher_Type = 2 + // Negated exact string equality (!=). + Matcher_TYPE_NOT_EQUAL Matcher_Type = 3 + // Negated regular expression match (!~). + Matcher_TYPE_NOT_REGEXP Matcher_Type = 4 +) + +// Enum value maps for Matcher_Type. +var ( + Matcher_Type_name = map[int32]string{ + 0: "TYPE_UNSPECIFIED", + 1: "TYPE_EQUAL", + 2: "TYPE_REGEXP", + 3: "TYPE_NOT_EQUAL", + 4: "TYPE_NOT_REGEXP", + } + Matcher_Type_value = map[string]int32{ + "TYPE_UNSPECIFIED": 0, + "TYPE_EQUAL": 1, + "TYPE_REGEXP": 2, + "TYPE_NOT_EQUAL": 3, + "TYPE_NOT_REGEXP": 4, + } +) + +func (x Matcher_Type) Enum() *Matcher_Type { + p := new(Matcher_Type) + *p = x + return p +} + +func (x Matcher_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Matcher_Type) Descriptor() protoreflect.EnumDescriptor { + return file_eventrecorder_proto_enumTypes[1].Descriptor() +} + +func (Matcher_Type) Type() protoreflect.EnumType { + return &file_eventrecorder_proto_enumTypes[1] +} + +func (x Matcher_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Matcher_Type.Descriptor instead. +func (Matcher_Type) EnumDescriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{15, 0} +} + +// Event is the top-level envelope written to each event recorder output. +// It wraps the specific event data with metadata about when and where +// the event was produced. +type Event struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The wall-clock time at which the event was recorded. + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,json=@timestamp,proto3" json:"timestamp,omitempty"` + // The hostname or address of the Alertmanager instance that produced + // the event. + Instance string `protobuf:"bytes,2,opt,name=instance,proto3" json:"instance,omitempty"` + // The event payload. Exactly one of the oneof fields inside EventData + // will be set. + Data *EventData `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + // The ordinal position of this instance among its HA cluster peers. + // Zero when clustering is disabled. + ClusterPosition uint32 `protobuf:"varint,4,opt,name=cluster_position,json=clusterPosition,proto3" json:"cluster_position,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Event) Reset() { + *x = Event{} + mi := &file_eventrecorder_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Event) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Event) ProtoMessage() {} + +func (x *Event) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Event.ProtoReflect.Descriptor instead. +func (*Event) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{0} +} + +func (x *Event) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *Event) GetInstance() string { + if x != nil { + return x.Instance + } + return "" +} + +func (x *Event) GetData() *EventData { + if x != nil { + return x.Data + } + return nil +} + +func (x *Event) GetClusterPosition() uint32 { + if x != nil { + return x.ClusterPosition + } + return 0 +} + +// EventData carries the payload for a single event recorder entry. +// Exactly one of the oneof fields will be populated. +type EventData struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to EventType: + // + // *EventData_AlertmanagerStartupEvent + // *EventData_AlertmanagerShutdownEvent + // *EventData_AlertCreated + // *EventData_AlertResolved + // *EventData_AlertGrouped + // *EventData_Notification + // *EventData_SilenceCreated + // *EventData_SilenceUpdated + // *EventData_SilenceMutedAlert + // *EventData_InhibitionMutedAlert + EventType isEventData_EventType `protobuf_oneof:"event_type"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EventData) Reset() { + *x = EventData{} + mi := &file_eventrecorder_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EventData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EventData) ProtoMessage() {} + +func (x *EventData) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EventData.ProtoReflect.Descriptor instead. +func (*EventData) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{1} +} + +func (x *EventData) GetEventType() isEventData_EventType { + if x != nil { + return x.EventType + } + return nil +} + +func (x *EventData) GetAlertmanagerStartupEvent() *AlertmanagerStartupEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_AlertmanagerStartupEvent); ok { + return x.AlertmanagerStartupEvent + } + } + return nil +} + +func (x *EventData) GetAlertmanagerShutdownEvent() *AlertmanagerShutdownEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_AlertmanagerShutdownEvent); ok { + return x.AlertmanagerShutdownEvent + } + } + return nil +} + +func (x *EventData) GetAlertCreated() *AlertCreatedEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_AlertCreated); ok { + return x.AlertCreated + } + } + return nil +} + +func (x *EventData) GetAlertResolved() *AlertResolvedEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_AlertResolved); ok { + return x.AlertResolved + } + } + return nil +} + +func (x *EventData) GetAlertGrouped() *AlertGroupedEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_AlertGrouped); ok { + return x.AlertGrouped + } + } + return nil +} + +func (x *EventData) GetNotification() *NotificationEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_Notification); ok { + return x.Notification + } + } + return nil +} + +func (x *EventData) GetSilenceCreated() *SilenceCreatedEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_SilenceCreated); ok { + return x.SilenceCreated + } + } + return nil +} + +func (x *EventData) GetSilenceUpdated() *SilenceUpdatedEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_SilenceUpdated); ok { + return x.SilenceUpdated + } + } + return nil +} + +func (x *EventData) GetSilenceMutedAlert() *SilenceMutedAlertEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_SilenceMutedAlert); ok { + return x.SilenceMutedAlert + } + } + return nil +} + +func (x *EventData) GetInhibitionMutedAlert() *InhibitionMutedAlertEvent { + if x != nil { + if x, ok := x.EventType.(*EventData_InhibitionMutedAlert); ok { + return x.InhibitionMutedAlert + } + } + return nil +} + +type isEventData_EventType interface { + isEventData_EventType() +} + +type EventData_AlertmanagerStartupEvent struct { + // Recorded when the Alertmanager process starts. + AlertmanagerStartupEvent *AlertmanagerStartupEvent `protobuf:"bytes,1,opt,name=alertmanager_startup_event,json=alertmanagerStartupEvent,proto3,oneof"` +} + +type EventData_AlertmanagerShutdownEvent struct { + // Recorded when the Alertmanager process shuts down gracefully. + AlertmanagerShutdownEvent *AlertmanagerShutdownEvent `protobuf:"bytes,2,opt,name=alertmanager_shutdown_event,json=alertmanagerShutdownEvent,proto3,oneof"` +} + +type EventData_AlertCreated struct { + // Recorded when a new alert is first inserted into the alert store. + AlertCreated *AlertCreatedEvent `protobuf:"bytes,3,opt,name=alert_created,json=alertCreated,proto3,oneof"` +} + +type EventData_AlertResolved struct { + // Recorded when an alert transitions to the resolved state and is + // removed from its aggregation group after successful notification. + AlertResolved *AlertResolvedEvent `protobuf:"bytes,4,opt,name=alert_resolved,json=alertResolved,proto3,oneof"` +} + +type EventData_AlertGrouped struct { + // Recorded when an alert is inserted into an aggregation group for + // the first time. + AlertGrouped *AlertGroupedEvent `protobuf:"bytes,5,opt,name=alert_grouped,json=alertGrouped,proto3,oneof"` +} + +type EventData_Notification struct { + // Recorded after a notification is successfully delivered to an + // integration (e.g., webhook, email, PagerDuty). + Notification *NotificationEvent `protobuf:"bytes,6,opt,name=notification,proto3,oneof"` +} + +type EventData_SilenceCreated struct { + // Recorded when a new silence is created. + SilenceCreated *SilenceCreatedEvent `protobuf:"bytes,7,opt,name=silence_created,json=silenceCreated,proto3,oneof"` +} + +type EventData_SilenceUpdated struct { + // Recorded when an existing silence is updated (e.g., extended or + // re-commented). + SilenceUpdated *SilenceUpdatedEvent `protobuf:"bytes,8,opt,name=silence_updated,json=silenceUpdated,proto3,oneof"` +} + +type EventData_SilenceMutedAlert struct { + // Recorded each time a silence actively suppresses an alert during + // the muting evaluation pass. + SilenceMutedAlert *SilenceMutedAlertEvent `protobuf:"bytes,9,opt,name=silence_muted_alert,json=silenceMutedAlert,proto3,oneof"` +} + +type EventData_InhibitionMutedAlert struct { + // Recorded each time one or more inhibition rules suppress an alert + // during the muting evaluation pass. + InhibitionMutedAlert *InhibitionMutedAlertEvent `protobuf:"bytes,10,opt,name=inhibition_muted_alert,json=inhibitionMutedAlert,proto3,oneof"` +} + +func (*EventData_AlertmanagerStartupEvent) isEventData_EventType() {} + +func (*EventData_AlertmanagerShutdownEvent) isEventData_EventType() {} + +func (*EventData_AlertCreated) isEventData_EventType() {} + +func (*EventData_AlertResolved) isEventData_EventType() {} + +func (*EventData_AlertGrouped) isEventData_EventType() {} + +func (*EventData_Notification) isEventData_EventType() {} + +func (*EventData_SilenceCreated) isEventData_EventType() {} + +func (*EventData_SilenceUpdated) isEventData_EventType() {} + +func (*EventData_SilenceMutedAlert) isEventData_EventType() {} + +func (*EventData_InhibitionMutedAlert) isEventData_EventType() {} + +// AlertmanagerStartupEvent is emitted once when the process starts. +type AlertmanagerStartupEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The semantic version of the Alertmanager binary (e.g., "0.28.0"). + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // Free-form build metadata such as Go version, branch, and revision. + BuildContext string `protobuf:"bytes,2,opt,name=build_context,json=buildContext,proto3" json:"build_context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertmanagerStartupEvent) Reset() { + *x = AlertmanagerStartupEvent{} + mi := &file_eventrecorder_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertmanagerStartupEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertmanagerStartupEvent) ProtoMessage() {} + +func (x *AlertmanagerStartupEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertmanagerStartupEvent.ProtoReflect.Descriptor instead. +func (*AlertmanagerStartupEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{2} +} + +func (x *AlertmanagerStartupEvent) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *AlertmanagerStartupEvent) GetBuildContext() string { + if x != nil { + return x.BuildContext + } + return "" +} + +// AlertmanagerShutdownEvent is emitted when the process shuts down +// gracefully. It carries no additional data. +type AlertmanagerShutdownEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertmanagerShutdownEvent) Reset() { + *x = AlertmanagerShutdownEvent{} + mi := &file_eventrecorder_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertmanagerShutdownEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertmanagerShutdownEvent) ProtoMessage() {} + +func (x *AlertmanagerShutdownEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertmanagerShutdownEvent.ProtoReflect.Descriptor instead. +func (*AlertmanagerShutdownEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{3} +} + +// LabelPair is a single key-value label. +type LabelPair struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The label name (e.g., "alertname"). + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // The label value (e.g., "HighMemoryUsage"). + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LabelPair) Reset() { + *x = LabelPair{} + mi := &file_eventrecorder_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LabelPair) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LabelPair) ProtoMessage() {} + +func (x *LabelPair) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LabelPair.ProtoReflect.Descriptor instead. +func (*LabelPair) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{4} +} + +func (x *LabelPair) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *LabelPair) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +// LabelSet is an ordered collection of label pairs. +type LabelSet struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The label pairs that make up this set. + Labels []*LabelPair `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LabelSet) Reset() { + *x = LabelSet{} + mi := &file_eventrecorder_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LabelSet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LabelSet) ProtoMessage() {} + +func (x *LabelSet) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LabelSet.ProtoReflect.Descriptor instead. +func (*LabelSet) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{5} +} + +func (x *LabelSet) GetLabels() []*LabelPair { + if x != nil { + return x.Labels + } + return nil +} + +// Alert represents a snapshot of an alert at the time the event was +// recorded. +type Alert struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The unique fingerprint derived from the alert's label set. + Fingerprint uint64 `protobuf:"varint,1,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` + // The value of the "alertname" label, provided for convenience. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // The full label set that identifies this alert. + Labels *LabelSet `protobuf:"bytes,3,opt,name=labels,proto3" json:"labels,omitempty"` + // Informational annotations attached to the alert (e.g., summary, + // description). + Annotations *LabelSet `protobuf:"bytes,4,opt,name=annotations,proto3" json:"annotations,omitempty"` + // The time at which the alert started firing. + StartsAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=starts_at,json=startsAt,proto3" json:"starts_at,omitempty"` + // The time at which the alert is considered resolved. For firing + // alerts this is typically set to a time in the future. + EndsAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=ends_at,json=endsAt,proto3" json:"ends_at,omitempty"` + // Whether the alert was resolved at the time the event was recorded. + Resolved bool `protobuf:"varint,7,opt,name=resolved,proto3" json:"resolved,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Alert) Reset() { + *x = Alert{} + mi := &file_eventrecorder_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Alert) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Alert) ProtoMessage() {} + +func (x *Alert) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Alert.ProtoReflect.Descriptor instead. +func (*Alert) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{6} +} + +func (x *Alert) GetFingerprint() uint64 { + if x != nil { + return x.Fingerprint + } + return 0 +} + +func (x *Alert) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Alert) GetLabels() *LabelSet { + if x != nil { + return x.Labels + } + return nil +} + +func (x *Alert) GetAnnotations() *LabelSet { + if x != nil { + return x.Annotations + } + return nil +} + +func (x *Alert) GetStartsAt() *timestamppb.Timestamp { + if x != nil { + return x.StartsAt + } + return nil +} + +func (x *Alert) GetEndsAt() *timestamppb.Timestamp { + if x != nil { + return x.EndsAt + } + return nil +} + +func (x *Alert) GetResolved() bool { + if x != nil { + return x.Resolved + } + return false +} + +// GroupedAlert is a reference to an alert within an aggregation group. +// It always carries the content hash; the full alert details are +// included when available. +type GroupedAlert struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A hash of the alert's label set, used for deduplication within the + // notification pipeline. + Hash uint64 `protobuf:"varint,1,opt,name=hash,proto3" json:"hash,omitempty"` + // The full alert details. May be absent when only the hash is needed + // (e.g., in firing/resolved lists on NotificationEvent). + Details *Alert `protobuf:"bytes,2,opt,name=details,proto3,oneof" json:"details,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GroupedAlert) Reset() { + *x = GroupedAlert{} + mi := &file_eventrecorder_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GroupedAlert) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GroupedAlert) ProtoMessage() {} + +func (x *GroupedAlert) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GroupedAlert.ProtoReflect.Descriptor instead. +func (*GroupedAlert) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{7} +} + +func (x *GroupedAlert) GetHash() uint64 { + if x != nil { + return x.Hash + } + return 0 +} + +func (x *GroupedAlert) GetDetails() *Alert { + if x != nil { + return x.Details + } + return nil +} + +// AlertGroupInfo describes the aggregation group context in which an +// alert is being processed. +type AlertGroupInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The composite key that uniquely identifies this aggregation group + // (encodes route and group label values). + GroupKey string `protobuf:"bytes,1,opt,name=group_key,json=groupKey,proto3" json:"group_key,omitempty"` + // The label set used to group alerts together within this route. + GroupLabels *LabelSet `protobuf:"bytes,2,opt,name=group_labels,json=groupLabels,proto3" json:"group_labels,omitempty"` + // A stable, shortened identifier derived from the group key (SHA-256 + // hex). + GroupId string `protobuf:"bytes,3,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"` + // The name of the receiver that this group routes to. + ReceiverName string `protobuf:"bytes,4,opt,name=receiver_name,json=receiverName,proto3" json:"receiver_name,omitempty"` + // The set of matchers defined on the route that matched these alerts. + Matchers []*Matcher `protobuf:"bytes,5,rep,name=matchers,proto3" json:"matchers,omitempty"` + // A UUID that uniquely identifies this aggregation group instance. + GroupUuid string `protobuf:"bytes,6,opt,name=group_uuid,json=groupUuid,proto3" json:"group_uuid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertGroupInfo) Reset() { + *x = AlertGroupInfo{} + mi := &file_eventrecorder_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertGroupInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertGroupInfo) ProtoMessage() {} + +func (x *AlertGroupInfo) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertGroupInfo.ProtoReflect.Descriptor instead. +func (*AlertGroupInfo) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{8} +} + +func (x *AlertGroupInfo) GetGroupKey() string { + if x != nil { + return x.GroupKey + } + return "" +} + +func (x *AlertGroupInfo) GetGroupLabels() *LabelSet { + if x != nil { + return x.GroupLabels + } + return nil +} + +func (x *AlertGroupInfo) GetGroupId() string { + if x != nil { + return x.GroupId + } + return "" +} + +func (x *AlertGroupInfo) GetReceiverName() string { + if x != nil { + return x.ReceiverName + } + return "" +} + +func (x *AlertGroupInfo) GetMatchers() []*Matcher { + if x != nil { + return x.Matchers + } + return nil +} + +func (x *AlertGroupInfo) GetGroupUuid() string { + if x != nil { + return x.GroupUuid + } + return "" +} + +// AlertCreatedEvent is emitted when a brand-new alert is inserted into +// the in-memory alert store. +type AlertCreatedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The newly created alert. + Alert *Alert `protobuf:"bytes,1,opt,name=alert,proto3" json:"alert,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertCreatedEvent) Reset() { + *x = AlertCreatedEvent{} + mi := &file_eventrecorder_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertCreatedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertCreatedEvent) ProtoMessage() {} + +func (x *AlertCreatedEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertCreatedEvent.ProtoReflect.Descriptor instead. +func (*AlertCreatedEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{9} +} + +func (x *AlertCreatedEvent) GetAlert() *Alert { + if x != nil { + return x.Alert + } + return nil +} + +// AlertResolvedEvent is emitted when an alert is removed from its +// aggregation group after a successful notification that included the +// resolution. +type AlertResolvedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The resolved alert, including its hash and full details. + Alert *GroupedAlert `protobuf:"bytes,1,opt,name=alert,proto3" json:"alert,omitempty"` + // The aggregation group from which the alert was resolved. + GroupInfo *AlertGroupInfo `protobuf:"bytes,2,opt,name=group_info,json=groupInfo,proto3" json:"group_info,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertResolvedEvent) Reset() { + *x = AlertResolvedEvent{} + mi := &file_eventrecorder_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertResolvedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertResolvedEvent) ProtoMessage() {} + +func (x *AlertResolvedEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertResolvedEvent.ProtoReflect.Descriptor instead. +func (*AlertResolvedEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{10} +} + +func (x *AlertResolvedEvent) GetAlert() *GroupedAlert { + if x != nil { + return x.Alert + } + return nil +} + +func (x *AlertResolvedEvent) GetGroupInfo() *AlertGroupInfo { + if x != nil { + return x.GroupInfo + } + return nil +} + +// AlertGroupedEvent is emitted the first time an alert is inserted into +// an aggregation group. +type AlertGroupedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The alert being grouped, including its hash and full details. + Alert *GroupedAlert `protobuf:"bytes,1,opt,name=alert,proto3" json:"alert,omitempty"` + // The aggregation group the alert was added to. + GroupInfo *AlertGroupInfo `protobuf:"bytes,2,opt,name=group_info,json=groupInfo,proto3" json:"group_info,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AlertGroupedEvent) Reset() { + *x = AlertGroupedEvent{} + mi := &file_eventrecorder_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AlertGroupedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AlertGroupedEvent) ProtoMessage() {} + +func (x *AlertGroupedEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AlertGroupedEvent.ProtoReflect.Descriptor instead. +func (*AlertGroupedEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{11} +} + +func (x *AlertGroupedEvent) GetAlert() *GroupedAlert { + if x != nil { + return x.Alert + } + return nil +} + +func (x *AlertGroupedEvent) GetGroupInfo() *AlertGroupInfo { + if x != nil { + return x.GroupInfo + } + return nil +} + +// Integration identifies a specific notification integration (e.g., +// the second PagerDuty receiver in a receiver definition). +type Integration struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The type of the integration (e.g., "webhook", "pagerduty"). + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The zero-based index of this integration within its receiver. + Index int64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Integration) Reset() { + *x = Integration{} + mi := &file_eventrecorder_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Integration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Integration) ProtoMessage() {} + +func (x *Integration) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Integration.ProtoReflect.Descriptor instead. +func (*Integration) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{12} +} + +func (x *Integration) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Integration) GetIndex() int64 { + if x != nil { + return x.Index + } + return 0 +} + +// NotificationEvent is emitted after a notification is successfully +// delivered to an integration. +type NotificationEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // All alerts that were included in the notification. + Alerts []*GroupedAlert `protobuf:"bytes,1,rep,name=alerts,proto3" json:"alerts,omitempty"` + // The subset of alerts that are currently firing. + FiringAlerts []*GroupedAlert `protobuf:"bytes,2,rep,name=firing_alerts,json=firingAlerts,proto3" json:"firing_alerts,omitempty"` + // The subset of alerts that are resolved. + ResolvedAlerts []*GroupedAlert `protobuf:"bytes,3,rep,name=resolved_alerts,json=resolvedAlerts,proto3" json:"resolved_alerts,omitempty"` + // Alerts that were muted (silenced or inhibited) during this flush + // cycle. + MutedAlerts []*GroupedAlert `protobuf:"bytes,4,rep,name=muted_alerts,json=mutedAlerts,proto3" json:"muted_alerts,omitempty"` + // The aggregation group context for this notification. + GroupInfo *AlertGroupInfo `protobuf:"bytes,5,opt,name=group_info,json=groupInfo,proto3" json:"group_info,omitempty"` + // The configured repeat interval for the aggregation group's route. + RepeatInterval *durationpb.Duration `protobuf:"bytes,6,opt,name=repeat_interval,json=repeatInterval,proto3" json:"repeat_interval,omitempty"` + // The reason the notification was triggered. + Reason NotifyReason `protobuf:"varint,7,opt,name=reason,proto3,enum=eventrecorderpb.NotifyReason" json:"reason,omitempty"` + // A monotonically increasing identifier for each flush cycle of the + // aggregation group. + FlushId uint64 `protobuf:"varint,8,opt,name=flush_id,json=flushId,proto3" json:"flush_id,omitempty"` + // The integration that delivered the notification. + Integration *Integration `protobuf:"bytes,9,opt,name=integration,proto3" json:"integration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NotificationEvent) Reset() { + *x = NotificationEvent{} + mi := &file_eventrecorder_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NotificationEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NotificationEvent) ProtoMessage() {} + +func (x *NotificationEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NotificationEvent.ProtoReflect.Descriptor instead. +func (*NotificationEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{13} +} + +func (x *NotificationEvent) GetAlerts() []*GroupedAlert { + if x != nil { + return x.Alerts + } + return nil +} + +func (x *NotificationEvent) GetFiringAlerts() []*GroupedAlert { + if x != nil { + return x.FiringAlerts + } + return nil +} + +func (x *NotificationEvent) GetResolvedAlerts() []*GroupedAlert { + if x != nil { + return x.ResolvedAlerts + } + return nil +} + +func (x *NotificationEvent) GetMutedAlerts() []*GroupedAlert { + if x != nil { + return x.MutedAlerts + } + return nil +} + +func (x *NotificationEvent) GetGroupInfo() *AlertGroupInfo { + if x != nil { + return x.GroupInfo + } + return nil +} + +func (x *NotificationEvent) GetRepeatInterval() *durationpb.Duration { + if x != nil { + return x.RepeatInterval + } + return nil +} + +func (x *NotificationEvent) GetReason() NotifyReason { + if x != nil { + return x.Reason + } + return NotifyReason_NOTIFY_REASON_UNSPECIFIED +} + +func (x *NotificationEvent) GetFlushId() uint64 { + if x != nil { + return x.FlushId + } + return 0 +} + +func (x *NotificationEvent) GetIntegration() *Integration { + if x != nil { + return x.Integration + } + return nil +} + +// Silence is a snapshot of a silence definition at the time the event +// was recorded. +type Silence struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The globally unique silence identifier (UUID). + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The matchers that define which alerts this silence suppresses. + // For silences with multiple matcher sets, this is the first set. + Matchers []*Matcher `protobuf:"bytes,2,rep,name=matchers,proto3" json:"matchers,omitempty"` + // Optional structured annotations on the silence (key-value pairs). + Annotations *LabelSet `protobuf:"bytes,3,opt,name=annotations,proto3" json:"annotations,omitempty"` + // The time at which the silence becomes active. + StartsAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=starts_at,json=startsAt,proto3" json:"starts_at,omitempty"` + // The time at which the silence expires. + EndsAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ends_at,json=endsAt,proto3" json:"ends_at,omitempty"` + // The last time the silence was created or updated. + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + // The author who created the silence. + CreatedBy string `protobuf:"bytes,7,opt,name=created_by,json=createdBy,proto3" json:"created_by,omitempty"` + // A human-readable comment explaining the silence. + Comment string `protobuf:"bytes,8,opt,name=comment,proto3" json:"comment,omitempty"` + // Additional matcher sets evaluated with OR logic. At least one + // matcher set must match for the silence to apply. + MatcherSets []*MatcherSet `protobuf:"bytes,9,rep,name=matcher_sets,json=matcherSets,proto3" json:"matcher_sets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Silence) Reset() { + *x = Silence{} + mi := &file_eventrecorder_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Silence) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Silence) ProtoMessage() {} + +func (x *Silence) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Silence.ProtoReflect.Descriptor instead. +func (*Silence) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{14} +} + +func (x *Silence) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Silence) GetMatchers() []*Matcher { + if x != nil { + return x.Matchers + } + return nil +} + +func (x *Silence) GetAnnotations() *LabelSet { + if x != nil { + return x.Annotations + } + return nil +} + +func (x *Silence) GetStartsAt() *timestamppb.Timestamp { + if x != nil { + return x.StartsAt + } + return nil +} + +func (x *Silence) GetEndsAt() *timestamppb.Timestamp { + if x != nil { + return x.EndsAt + } + return nil +} + +func (x *Silence) GetUpdatedAt() *timestamppb.Timestamp { + if x != nil { + return x.UpdatedAt + } + return nil +} + +func (x *Silence) GetCreatedBy() string { + if x != nil { + return x.CreatedBy + } + return "" +} + +func (x *Silence) GetComment() string { + if x != nil { + return x.Comment + } + return "" +} + +func (x *Silence) GetMatcherSets() []*MatcherSet { + if x != nil { + return x.MatcherSets + } + return nil +} + +// Matcher defines a single label matching rule. +type Matcher struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The matching operator to apply. + Type Matcher_Type `protobuf:"varint,1,opt,name=type,proto3,enum=eventrecorderpb.Matcher_Type" json:"type,omitempty"` + // The label name to match against. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // The value or pattern to match, interpreted according to type. + Pattern string `protobuf:"bytes,3,opt,name=pattern,proto3" json:"pattern,omitempty"` + // Human-readable string representation (e.g., "env=~prod.*"). + Rendered string `protobuf:"bytes,4,opt,name=rendered,proto3" json:"rendered,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Matcher) Reset() { + *x = Matcher{} + mi := &file_eventrecorder_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Matcher) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Matcher) ProtoMessage() {} + +func (x *Matcher) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Matcher.ProtoReflect.Descriptor instead. +func (*Matcher) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{15} +} + +func (x *Matcher) GetType() Matcher_Type { + if x != nil { + return x.Type + } + return Matcher_TYPE_UNSPECIFIED +} + +func (x *Matcher) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Matcher) GetPattern() string { + if x != nil { + return x.Pattern + } + return "" +} + +func (x *Matcher) GetRendered() string { + if x != nil { + return x.Rendered + } + return "" +} + +// MatcherSet is a conjunction of matchers: all matchers in the set must +// match for the set to match. +type MatcherSet struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The matchers that make up this set (evaluated with AND logic). + Matchers []*Matcher `protobuf:"bytes,1,rep,name=matchers,proto3" json:"matchers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MatcherSet) Reset() { + *x = MatcherSet{} + mi := &file_eventrecorder_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MatcherSet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MatcherSet) ProtoMessage() {} + +func (x *MatcherSet) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MatcherSet.ProtoReflect.Descriptor instead. +func (*MatcherSet) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{16} +} + +func (x *MatcherSet) GetMatchers() []*Matcher { + if x != nil { + return x.Matchers + } + return nil +} + +// SilenceCreatedEvent is emitted when a new silence is created. +type SilenceCreatedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The newly created silence. + Silence *Silence `protobuf:"bytes,1,opt,name=silence,proto3" json:"silence,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SilenceCreatedEvent) Reset() { + *x = SilenceCreatedEvent{} + mi := &file_eventrecorder_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SilenceCreatedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SilenceCreatedEvent) ProtoMessage() {} + +func (x *SilenceCreatedEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SilenceCreatedEvent.ProtoReflect.Descriptor instead. +func (*SilenceCreatedEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{17} +} + +func (x *SilenceCreatedEvent) GetSilence() *Silence { + if x != nil { + return x.Silence + } + return nil +} + +// SilenceUpdatedEvent is emitted when an existing silence is modified. +type SilenceUpdatedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The silence after the update. + Silence *Silence `protobuf:"bytes,1,opt,name=silence,proto3" json:"silence,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SilenceUpdatedEvent) Reset() { + *x = SilenceUpdatedEvent{} + mi := &file_eventrecorder_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SilenceUpdatedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SilenceUpdatedEvent) ProtoMessage() {} + +func (x *SilenceUpdatedEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SilenceUpdatedEvent.ProtoReflect.Descriptor instead. +func (*SilenceUpdatedEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{18} +} + +func (x *SilenceUpdatedEvent) GetSilence() *Silence { + if x != nil { + return x.Silence + } + return nil +} + +// MutedAlert identifies an alert that was suppressed by a silence or +// inhibition rule. +type MutedAlert struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The label set of the muted alert. + Labels *LabelSet `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels,omitempty"` + // The fingerprint of the muted alert. + Fingerprint uint64 `protobuf:"varint,2,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MutedAlert) Reset() { + *x = MutedAlert{} + mi := &file_eventrecorder_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MutedAlert) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MutedAlert) ProtoMessage() {} + +func (x *MutedAlert) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MutedAlert.ProtoReflect.Descriptor instead. +func (*MutedAlert) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{19} +} + +func (x *MutedAlert) GetLabels() *LabelSet { + if x != nil { + return x.Labels + } + return nil +} + +func (x *MutedAlert) GetFingerprint() uint64 { + if x != nil { + return x.Fingerprint + } + return 0 +} + +// SilenceMutedAlertEvent is emitted each time a silence suppresses an +// alert during the muting evaluation pass. +type SilenceMutedAlertEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The silence that suppressed the alert. + Silence *Silence `protobuf:"bytes,1,opt,name=silence,proto3" json:"silence,omitempty"` + // The alert that was suppressed. + MutedAlert *MutedAlert `protobuf:"bytes,2,opt,name=muted_alert,json=mutedAlert,proto3" json:"muted_alert,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SilenceMutedAlertEvent) Reset() { + *x = SilenceMutedAlertEvent{} + mi := &file_eventrecorder_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SilenceMutedAlertEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SilenceMutedAlertEvent) ProtoMessage() {} + +func (x *SilenceMutedAlertEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SilenceMutedAlertEvent.ProtoReflect.Descriptor instead. +func (*SilenceMutedAlertEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{20} +} + +func (x *SilenceMutedAlertEvent) GetSilence() *Silence { + if x != nil { + return x.Silence + } + return nil +} + +func (x *SilenceMutedAlertEvent) GetMutedAlert() *MutedAlert { + if x != nil { + return x.MutedAlert + } + return nil +} + +// InhibitRule is a snapshot of an inhibition rule definition. +type InhibitRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Matchers that identify source alerts (those that do the inhibiting). + SourceMatchers []*Matcher `protobuf:"bytes,1,rep,name=source_matchers,json=sourceMatchers,proto3" json:"source_matchers,omitempty"` + // Matchers that identify target alerts (those that get inhibited). + TargetMatchers []*Matcher `protobuf:"bytes,2,rep,name=target_matchers,json=targetMatchers,proto3" json:"target_matchers,omitempty"` + // Label names whose values must be equal between source and target + // alerts for the inhibition to take effect. + EqualLabels []string `protobuf:"bytes,3,rep,name=equal_labels,json=equalLabels,proto3" json:"equal_labels,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InhibitRule) Reset() { + *x = InhibitRule{} + mi := &file_eventrecorder_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InhibitRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InhibitRule) ProtoMessage() {} + +func (x *InhibitRule) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InhibitRule.ProtoReflect.Descriptor instead. +func (*InhibitRule) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{21} +} + +func (x *InhibitRule) GetSourceMatchers() []*Matcher { + if x != nil { + return x.SourceMatchers + } + return nil +} + +func (x *InhibitRule) GetTargetMatchers() []*Matcher { + if x != nil { + return x.TargetMatchers + } + return nil +} + +func (x *InhibitRule) GetEqualLabels() []string { + if x != nil { + return x.EqualLabels + } + return nil +} + +// InhibitionMutedAlertEvent is emitted when one or more inhibition +// rules suppress an alert. +type InhibitionMutedAlertEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The inhibition rules that matched. + InhibitRules []*InhibitRule `protobuf:"bytes,1,rep,name=inhibit_rules,json=inhibitRules,proto3" json:"inhibit_rules,omitempty"` + // The alert that was suppressed. + MutedAlert *MutedAlert `protobuf:"bytes,2,opt,name=muted_alert,json=mutedAlert,proto3" json:"muted_alert,omitempty"` + // The fingerprints of the source alerts that caused the inhibition. + InhibitingFingerprints []uint64 `protobuf:"varint,3,rep,packed,name=inhibiting_fingerprints,json=inhibitingFingerprints,proto3" json:"inhibiting_fingerprints,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InhibitionMutedAlertEvent) Reset() { + *x = InhibitionMutedAlertEvent{} + mi := &file_eventrecorder_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InhibitionMutedAlertEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InhibitionMutedAlertEvent) ProtoMessage() {} + +func (x *InhibitionMutedAlertEvent) ProtoReflect() protoreflect.Message { + mi := &file_eventrecorder_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InhibitionMutedAlertEvent.ProtoReflect.Descriptor instead. +func (*InhibitionMutedAlertEvent) Descriptor() ([]byte, []int) { + return file_eventrecorder_proto_rawDescGZIP(), []int{22} +} + +func (x *InhibitionMutedAlertEvent) GetInhibitRules() []*InhibitRule { + if x != nil { + return x.InhibitRules + } + return nil +} + +func (x *InhibitionMutedAlertEvent) GetMutedAlert() *MutedAlert { + if x != nil { + return x.MutedAlert + } + return nil +} + +func (x *InhibitionMutedAlertEvent) GetInhibitingFingerprints() []uint64 { + if x != nil { + return x.InhibitingFingerprints + } + return nil +} + +var File_eventrecorder_proto protoreflect.FileDescriptor + +const file_eventrecorder_proto_rawDesc = "" + + "\n" + + "\x13eventrecorder.proto\x12\x0feventrecorderpb\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xb9\x01\n" + + "\x05Event\x129\n" + + "\ttimestamp\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "@timestamp\x12\x1a\n" + + "\binstance\x18\x02 \x01(\tR\binstance\x12.\n" + + "\x04data\x18\x03 \x01(\v2\x1a.eventrecorderpb.EventDataR\x04data\x12)\n" + + "\x10cluster_position\x18\x04 \x01(\rR\x0fclusterPosition\"\x81\a\n" + + "\tEventData\x12i\n" + + "\x1aalertmanager_startup_event\x18\x01 \x01(\v2).eventrecorderpb.AlertmanagerStartupEventH\x00R\x18alertmanagerStartupEvent\x12l\n" + + "\x1balertmanager_shutdown_event\x18\x02 \x01(\v2*.eventrecorderpb.AlertmanagerShutdownEventH\x00R\x19alertmanagerShutdownEvent\x12I\n" + + "\ralert_created\x18\x03 \x01(\v2\".eventrecorderpb.AlertCreatedEventH\x00R\falertCreated\x12L\n" + + "\x0ealert_resolved\x18\x04 \x01(\v2#.eventrecorderpb.AlertResolvedEventH\x00R\ralertResolved\x12I\n" + + "\ralert_grouped\x18\x05 \x01(\v2\".eventrecorderpb.AlertGroupedEventH\x00R\falertGrouped\x12H\n" + + "\fnotification\x18\x06 \x01(\v2\".eventrecorderpb.NotificationEventH\x00R\fnotification\x12O\n" + + "\x0fsilence_created\x18\a \x01(\v2$.eventrecorderpb.SilenceCreatedEventH\x00R\x0esilenceCreated\x12O\n" + + "\x0fsilence_updated\x18\b \x01(\v2$.eventrecorderpb.SilenceUpdatedEventH\x00R\x0esilenceUpdated\x12Y\n" + + "\x13silence_muted_alert\x18\t \x01(\v2'.eventrecorderpb.SilenceMutedAlertEventH\x00R\x11silenceMutedAlert\x12b\n" + + "\x16inhibition_muted_alert\x18\n" + + " \x01(\v2*.eventrecorderpb.InhibitionMutedAlertEventH\x00R\x14inhibitionMutedAlertB\f\n" + + "\n" + + "event_type\"Y\n" + + "\x18AlertmanagerStartupEvent\x12\x18\n" + + "\aversion\x18\x01 \x01(\tR\aversion\x12#\n" + + "\rbuild_context\x18\x02 \x01(\tR\fbuildContext\"\x1b\n" + + "\x19AlertmanagerShutdownEvent\"3\n" + + "\tLabelPair\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\">\n" + + "\bLabelSet\x122\n" + + "\x06labels\x18\x01 \x03(\v2\x1a.eventrecorderpb.LabelPairR\x06labels\"\xb7\x02\n" + + "\x05Alert\x12 \n" + + "\vfingerprint\x18\x01 \x01(\x04R\vfingerprint\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x121\n" + + "\x06labels\x18\x03 \x01(\v2\x19.eventrecorderpb.LabelSetR\x06labels\x12;\n" + + "\vannotations\x18\x04 \x01(\v2\x19.eventrecorderpb.LabelSetR\vannotations\x127\n" + + "\tstarts_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\bstartsAt\x123\n" + + "\aends_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\x06endsAt\x12\x1a\n" + + "\bresolved\x18\a \x01(\bR\bresolved\"e\n" + + "\fGroupedAlert\x12\x12\n" + + "\x04hash\x18\x01 \x01(\x04R\x04hash\x125\n" + + "\adetails\x18\x02 \x01(\v2\x16.eventrecorderpb.AlertH\x00R\adetails\x88\x01\x01B\n" + + "\n" + + "\b_details\"\x80\x02\n" + + "\x0eAlertGroupInfo\x12\x1b\n" + + "\tgroup_key\x18\x01 \x01(\tR\bgroupKey\x12<\n" + + "\fgroup_labels\x18\x02 \x01(\v2\x19.eventrecorderpb.LabelSetR\vgroupLabels\x12\x19\n" + + "\bgroup_id\x18\x03 \x01(\tR\agroupId\x12#\n" + + "\rreceiver_name\x18\x04 \x01(\tR\freceiverName\x124\n" + + "\bmatchers\x18\x05 \x03(\v2\x18.eventrecorderpb.MatcherR\bmatchers\x12\x1d\n" + + "\n" + + "group_uuid\x18\x06 \x01(\tR\tgroupUuid\"A\n" + + "\x11AlertCreatedEvent\x12,\n" + + "\x05alert\x18\x01 \x01(\v2\x16.eventrecorderpb.AlertR\x05alert\"\x89\x01\n" + + "\x12AlertResolvedEvent\x123\n" + + "\x05alert\x18\x01 \x01(\v2\x1d.eventrecorderpb.GroupedAlertR\x05alert\x12>\n" + + "\n" + + "group_info\x18\x02 \x01(\v2\x1f.eventrecorderpb.AlertGroupInfoR\tgroupInfo\"\x88\x01\n" + + "\x11AlertGroupedEvent\x123\n" + + "\x05alert\x18\x01 \x01(\v2\x1d.eventrecorderpb.GroupedAlertR\x05alert\x12>\n" + + "\n" + + "group_info\x18\x02 \x01(\v2\x1f.eventrecorderpb.AlertGroupInfoR\tgroupInfo\"7\n" + + "\vIntegration\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05index\x18\x02 \x01(\x03R\x05index\"\xae\x04\n" + + "\x11NotificationEvent\x125\n" + + "\x06alerts\x18\x01 \x03(\v2\x1d.eventrecorderpb.GroupedAlertR\x06alerts\x12B\n" + + "\rfiring_alerts\x18\x02 \x03(\v2\x1d.eventrecorderpb.GroupedAlertR\ffiringAlerts\x12F\n" + + "\x0fresolved_alerts\x18\x03 \x03(\v2\x1d.eventrecorderpb.GroupedAlertR\x0eresolvedAlerts\x12@\n" + + "\fmuted_alerts\x18\x04 \x03(\v2\x1d.eventrecorderpb.GroupedAlertR\vmutedAlerts\x12>\n" + + "\n" + + "group_info\x18\x05 \x01(\v2\x1f.eventrecorderpb.AlertGroupInfoR\tgroupInfo\x12B\n" + + "\x0frepeat_interval\x18\x06 \x01(\v2\x19.google.protobuf.DurationR\x0erepeatInterval\x125\n" + + "\x06reason\x18\a \x01(\x0e2\x1d.eventrecorderpb.NotifyReasonR\x06reason\x12\x19\n" + + "\bflush_id\x18\b \x01(\x04R\aflushId\x12>\n" + + "\vintegration\x18\t \x01(\v2\x1c.eventrecorderpb.IntegrationR\vintegration\"\xae\x03\n" + + "\aSilence\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x124\n" + + "\bmatchers\x18\x02 \x03(\v2\x18.eventrecorderpb.MatcherR\bmatchers\x12;\n" + + "\vannotations\x18\x03 \x01(\v2\x19.eventrecorderpb.LabelSetR\vannotations\x127\n" + + "\tstarts_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\bstartsAt\x123\n" + + "\aends_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x06endsAt\x129\n" + + "\n" + + "updated_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\x12\x1d\n" + + "\n" + + "created_by\x18\a \x01(\tR\tcreatedBy\x12\x18\n" + + "\acomment\x18\b \x01(\tR\acomment\x12>\n" + + "\fmatcher_sets\x18\t \x03(\v2\x1b.eventrecorderpb.MatcherSetR\vmatcherSets\"\xee\x01\n" + + "\aMatcher\x121\n" + + "\x04type\x18\x01 \x01(\x0e2\x1d.eventrecorderpb.Matcher.TypeR\x04type\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x18\n" + + "\apattern\x18\x03 \x01(\tR\apattern\x12\x1a\n" + + "\brendered\x18\x04 \x01(\tR\brendered\"f\n" + + "\x04Type\x12\x14\n" + + "\x10TYPE_UNSPECIFIED\x10\x00\x12\x0e\n" + + "\n" + + "TYPE_EQUAL\x10\x01\x12\x0f\n" + + "\vTYPE_REGEXP\x10\x02\x12\x12\n" + + "\x0eTYPE_NOT_EQUAL\x10\x03\x12\x13\n" + + "\x0fTYPE_NOT_REGEXP\x10\x04\"B\n" + + "\n" + + "MatcherSet\x124\n" + + "\bmatchers\x18\x01 \x03(\v2\x18.eventrecorderpb.MatcherR\bmatchers\"I\n" + + "\x13SilenceCreatedEvent\x122\n" + + "\asilence\x18\x01 \x01(\v2\x18.eventrecorderpb.SilenceR\asilence\"I\n" + + "\x13SilenceUpdatedEvent\x122\n" + + "\asilence\x18\x01 \x01(\v2\x18.eventrecorderpb.SilenceR\asilence\"a\n" + + "\n" + + "MutedAlert\x121\n" + + "\x06labels\x18\x01 \x01(\v2\x19.eventrecorderpb.LabelSetR\x06labels\x12 \n" + + "\vfingerprint\x18\x02 \x01(\x04R\vfingerprint\"\x8a\x01\n" + + "\x16SilenceMutedAlertEvent\x122\n" + + "\asilence\x18\x01 \x01(\v2\x18.eventrecorderpb.SilenceR\asilence\x12<\n" + + "\vmuted_alert\x18\x02 \x01(\v2\x1b.eventrecorderpb.MutedAlertR\n" + + "mutedAlert\"\xb6\x01\n" + + "\vInhibitRule\x12A\n" + + "\x0fsource_matchers\x18\x01 \x03(\v2\x18.eventrecorderpb.MatcherR\x0esourceMatchers\x12A\n" + + "\x0ftarget_matchers\x18\x02 \x03(\v2\x18.eventrecorderpb.MatcherR\x0etargetMatchers\x12!\n" + + "\fequal_labels\x18\x03 \x03(\tR\vequalLabels\"\xd5\x01\n" + + "\x19InhibitionMutedAlertEvent\x12A\n" + + "\rinhibit_rules\x18\x01 \x03(\v2\x1c.eventrecorderpb.InhibitRuleR\finhibitRules\x12<\n" + + "\vmuted_alert\x18\x02 \x01(\v2\x1b.eventrecorderpb.MutedAlertR\n" + + "mutedAlert\x127\n" + + "\x17inhibiting_fingerprints\x18\x03 \x03(\x04R\x16inhibitingFingerprints*\xf3\x01\n" + + "\fNotifyReason\x12\x1d\n" + + "\x19NOTIFY_REASON_UNSPECIFIED\x10\x00\x12$\n" + + " NOTIFY_REASON_FIRST_NOTIFICATION\x10\x01\x12%\n" + + "!NOTIFY_REASON_NEW_ALERTS_IN_GROUP\x10\x02\x12%\n" + + "!NOTIFY_REASON_NEW_RESOLVED_ALERTS\x10\x03\x12%\n" + + "!NOTIFY_REASON_ALL_ALERTS_RESOLVED\x10\x04\x12)\n" + + "%NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED\x10\x05BBZ@github.com/prometheus/alertmanager/eventrecorder/eventrecorderpbb\x06proto3" + +var ( + file_eventrecorder_proto_rawDescOnce sync.Once + file_eventrecorder_proto_rawDescData []byte +) + +func file_eventrecorder_proto_rawDescGZIP() []byte { + file_eventrecorder_proto_rawDescOnce.Do(func() { + file_eventrecorder_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_eventrecorder_proto_rawDesc), len(file_eventrecorder_proto_rawDesc))) + }) + return file_eventrecorder_proto_rawDescData +} + +var file_eventrecorder_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_eventrecorder_proto_msgTypes = make([]protoimpl.MessageInfo, 23) +var file_eventrecorder_proto_goTypes = []any{ + (NotifyReason)(0), // 0: eventrecorderpb.NotifyReason + (Matcher_Type)(0), // 1: eventrecorderpb.Matcher.Type + (*Event)(nil), // 2: eventrecorderpb.Event + (*EventData)(nil), // 3: eventrecorderpb.EventData + (*AlertmanagerStartupEvent)(nil), // 4: eventrecorderpb.AlertmanagerStartupEvent + (*AlertmanagerShutdownEvent)(nil), // 5: eventrecorderpb.AlertmanagerShutdownEvent + (*LabelPair)(nil), // 6: eventrecorderpb.LabelPair + (*LabelSet)(nil), // 7: eventrecorderpb.LabelSet + (*Alert)(nil), // 8: eventrecorderpb.Alert + (*GroupedAlert)(nil), // 9: eventrecorderpb.GroupedAlert + (*AlertGroupInfo)(nil), // 10: eventrecorderpb.AlertGroupInfo + (*AlertCreatedEvent)(nil), // 11: eventrecorderpb.AlertCreatedEvent + (*AlertResolvedEvent)(nil), // 12: eventrecorderpb.AlertResolvedEvent + (*AlertGroupedEvent)(nil), // 13: eventrecorderpb.AlertGroupedEvent + (*Integration)(nil), // 14: eventrecorderpb.Integration + (*NotificationEvent)(nil), // 15: eventrecorderpb.NotificationEvent + (*Silence)(nil), // 16: eventrecorderpb.Silence + (*Matcher)(nil), // 17: eventrecorderpb.Matcher + (*MatcherSet)(nil), // 18: eventrecorderpb.MatcherSet + (*SilenceCreatedEvent)(nil), // 19: eventrecorderpb.SilenceCreatedEvent + (*SilenceUpdatedEvent)(nil), // 20: eventrecorderpb.SilenceUpdatedEvent + (*MutedAlert)(nil), // 21: eventrecorderpb.MutedAlert + (*SilenceMutedAlertEvent)(nil), // 22: eventrecorderpb.SilenceMutedAlertEvent + (*InhibitRule)(nil), // 23: eventrecorderpb.InhibitRule + (*InhibitionMutedAlertEvent)(nil), // 24: eventrecorderpb.InhibitionMutedAlertEvent + (*timestamppb.Timestamp)(nil), // 25: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 26: google.protobuf.Duration +} +var file_eventrecorder_proto_depIdxs = []int32{ + 25, // 0: eventrecorderpb.Event.timestamp:type_name -> google.protobuf.Timestamp + 3, // 1: eventrecorderpb.Event.data:type_name -> eventrecorderpb.EventData + 4, // 2: eventrecorderpb.EventData.alertmanager_startup_event:type_name -> eventrecorderpb.AlertmanagerStartupEvent + 5, // 3: eventrecorderpb.EventData.alertmanager_shutdown_event:type_name -> eventrecorderpb.AlertmanagerShutdownEvent + 11, // 4: eventrecorderpb.EventData.alert_created:type_name -> eventrecorderpb.AlertCreatedEvent + 12, // 5: eventrecorderpb.EventData.alert_resolved:type_name -> eventrecorderpb.AlertResolvedEvent + 13, // 6: eventrecorderpb.EventData.alert_grouped:type_name -> eventrecorderpb.AlertGroupedEvent + 15, // 7: eventrecorderpb.EventData.notification:type_name -> eventrecorderpb.NotificationEvent + 19, // 8: eventrecorderpb.EventData.silence_created:type_name -> eventrecorderpb.SilenceCreatedEvent + 20, // 9: eventrecorderpb.EventData.silence_updated:type_name -> eventrecorderpb.SilenceUpdatedEvent + 22, // 10: eventrecorderpb.EventData.silence_muted_alert:type_name -> eventrecorderpb.SilenceMutedAlertEvent + 24, // 11: eventrecorderpb.EventData.inhibition_muted_alert:type_name -> eventrecorderpb.InhibitionMutedAlertEvent + 6, // 12: eventrecorderpb.LabelSet.labels:type_name -> eventrecorderpb.LabelPair + 7, // 13: eventrecorderpb.Alert.labels:type_name -> eventrecorderpb.LabelSet + 7, // 14: eventrecorderpb.Alert.annotations:type_name -> eventrecorderpb.LabelSet + 25, // 15: eventrecorderpb.Alert.starts_at:type_name -> google.protobuf.Timestamp + 25, // 16: eventrecorderpb.Alert.ends_at:type_name -> google.protobuf.Timestamp + 8, // 17: eventrecorderpb.GroupedAlert.details:type_name -> eventrecorderpb.Alert + 7, // 18: eventrecorderpb.AlertGroupInfo.group_labels:type_name -> eventrecorderpb.LabelSet + 17, // 19: eventrecorderpb.AlertGroupInfo.matchers:type_name -> eventrecorderpb.Matcher + 8, // 20: eventrecorderpb.AlertCreatedEvent.alert:type_name -> eventrecorderpb.Alert + 9, // 21: eventrecorderpb.AlertResolvedEvent.alert:type_name -> eventrecorderpb.GroupedAlert + 10, // 22: eventrecorderpb.AlertResolvedEvent.group_info:type_name -> eventrecorderpb.AlertGroupInfo + 9, // 23: eventrecorderpb.AlertGroupedEvent.alert:type_name -> eventrecorderpb.GroupedAlert + 10, // 24: eventrecorderpb.AlertGroupedEvent.group_info:type_name -> eventrecorderpb.AlertGroupInfo + 9, // 25: eventrecorderpb.NotificationEvent.alerts:type_name -> eventrecorderpb.GroupedAlert + 9, // 26: eventrecorderpb.NotificationEvent.firing_alerts:type_name -> eventrecorderpb.GroupedAlert + 9, // 27: eventrecorderpb.NotificationEvent.resolved_alerts:type_name -> eventrecorderpb.GroupedAlert + 9, // 28: eventrecorderpb.NotificationEvent.muted_alerts:type_name -> eventrecorderpb.GroupedAlert + 10, // 29: eventrecorderpb.NotificationEvent.group_info:type_name -> eventrecorderpb.AlertGroupInfo + 26, // 30: eventrecorderpb.NotificationEvent.repeat_interval:type_name -> google.protobuf.Duration + 0, // 31: eventrecorderpb.NotificationEvent.reason:type_name -> eventrecorderpb.NotifyReason + 14, // 32: eventrecorderpb.NotificationEvent.integration:type_name -> eventrecorderpb.Integration + 17, // 33: eventrecorderpb.Silence.matchers:type_name -> eventrecorderpb.Matcher + 7, // 34: eventrecorderpb.Silence.annotations:type_name -> eventrecorderpb.LabelSet + 25, // 35: eventrecorderpb.Silence.starts_at:type_name -> google.protobuf.Timestamp + 25, // 36: eventrecorderpb.Silence.ends_at:type_name -> google.protobuf.Timestamp + 25, // 37: eventrecorderpb.Silence.updated_at:type_name -> google.protobuf.Timestamp + 18, // 38: eventrecorderpb.Silence.matcher_sets:type_name -> eventrecorderpb.MatcherSet + 1, // 39: eventrecorderpb.Matcher.type:type_name -> eventrecorderpb.Matcher.Type + 17, // 40: eventrecorderpb.MatcherSet.matchers:type_name -> eventrecorderpb.Matcher + 16, // 41: eventrecorderpb.SilenceCreatedEvent.silence:type_name -> eventrecorderpb.Silence + 16, // 42: eventrecorderpb.SilenceUpdatedEvent.silence:type_name -> eventrecorderpb.Silence + 7, // 43: eventrecorderpb.MutedAlert.labels:type_name -> eventrecorderpb.LabelSet + 16, // 44: eventrecorderpb.SilenceMutedAlertEvent.silence:type_name -> eventrecorderpb.Silence + 21, // 45: eventrecorderpb.SilenceMutedAlertEvent.muted_alert:type_name -> eventrecorderpb.MutedAlert + 17, // 46: eventrecorderpb.InhibitRule.source_matchers:type_name -> eventrecorderpb.Matcher + 17, // 47: eventrecorderpb.InhibitRule.target_matchers:type_name -> eventrecorderpb.Matcher + 23, // 48: eventrecorderpb.InhibitionMutedAlertEvent.inhibit_rules:type_name -> eventrecorderpb.InhibitRule + 21, // 49: eventrecorderpb.InhibitionMutedAlertEvent.muted_alert:type_name -> eventrecorderpb.MutedAlert + 50, // [50:50] is the sub-list for method output_type + 50, // [50:50] is the sub-list for method input_type + 50, // [50:50] is the sub-list for extension type_name + 50, // [50:50] is the sub-list for extension extendee + 0, // [0:50] is the sub-list for field type_name +} + +func init() { file_eventrecorder_proto_init() } +func file_eventrecorder_proto_init() { + if File_eventrecorder_proto != nil { + return + } + file_eventrecorder_proto_msgTypes[1].OneofWrappers = []any{ + (*EventData_AlertmanagerStartupEvent)(nil), + (*EventData_AlertmanagerShutdownEvent)(nil), + (*EventData_AlertCreated)(nil), + (*EventData_AlertResolved)(nil), + (*EventData_AlertGrouped)(nil), + (*EventData_Notification)(nil), + (*EventData_SilenceCreated)(nil), + (*EventData_SilenceUpdated)(nil), + (*EventData_SilenceMutedAlert)(nil), + (*EventData_InhibitionMutedAlert)(nil), + } + file_eventrecorder_proto_msgTypes[7].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_eventrecorder_proto_rawDesc), len(file_eventrecorder_proto_rawDesc)), + NumEnums: 2, + NumMessages: 23, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_eventrecorder_proto_goTypes, + DependencyIndexes: file_eventrecorder_proto_depIdxs, + EnumInfos: file_eventrecorder_proto_enumTypes, + MessageInfos: file_eventrecorder_proto_msgTypes, + }.Build() + File_eventrecorder_proto = out.File + file_eventrecorder_proto_goTypes = nil + file_eventrecorder_proto_depIdxs = nil +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.proto b/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.proto new file mode 100644 index 00000000000..3004b5e55cf --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb/eventrecorder.proto @@ -0,0 +1,389 @@ +syntax = "proto3"; + +package eventrecorderpb; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb"; + +// Event is the top-level envelope written to each event recorder output. +// It wraps the specific event data with metadata about when and where +// the event was produced. +message Event { + // The wall-clock time at which the event was recorded. + google.protobuf.Timestamp timestamp = 1 [json_name = "@timestamp"]; + + // The hostname or address of the Alertmanager instance that produced + // the event. + string instance = 2; + + // The event payload. Exactly one of the oneof fields inside EventData + // will be set. + EventData data = 3; + + // The ordinal position of this instance among its HA cluster peers. + // Zero when clustering is disabled. + uint32 cluster_position = 4; +} + +// EventData carries the payload for a single event recorder entry. +// Exactly one of the oneof fields will be populated. +message EventData { + oneof event_type { + // Recorded when the Alertmanager process starts. + AlertmanagerStartupEvent alertmanager_startup_event = 1; + + // Recorded when the Alertmanager process shuts down gracefully. + AlertmanagerShutdownEvent alertmanager_shutdown_event = 2; + + // Recorded when a new alert is first inserted into the alert store. + AlertCreatedEvent alert_created = 3; + + // Recorded when an alert transitions to the resolved state and is + // removed from its aggregation group after successful notification. + AlertResolvedEvent alert_resolved = 4; + + // Recorded when an alert is inserted into an aggregation group for + // the first time. + AlertGroupedEvent alert_grouped = 5; + + // Recorded after a notification is successfully delivered to an + // integration (e.g., webhook, email, PagerDuty). + NotificationEvent notification = 6; + + // Recorded when a new silence is created. + SilenceCreatedEvent silence_created = 7; + + // Recorded when an existing silence is updated (e.g., extended or + // re-commented). + SilenceUpdatedEvent silence_updated = 8; + + // Recorded each time a silence actively suppresses an alert during + // the muting evaluation pass. + SilenceMutedAlertEvent silence_muted_alert = 9; + + // Recorded each time one or more inhibition rules suppress an alert + // during the muting evaluation pass. + InhibitionMutedAlertEvent inhibition_muted_alert = 10; + } +} + +// AlertmanagerStartupEvent is emitted once when the process starts. +message AlertmanagerStartupEvent { + // The semantic version of the Alertmanager binary (e.g., "0.28.0"). + string version = 1; + + // Free-form build metadata such as Go version, branch, and revision. + string build_context = 2; +} + +// AlertmanagerShutdownEvent is emitted when the process shuts down +// gracefully. It carries no additional data. +message AlertmanagerShutdownEvent {} + +// LabelPair is a single key-value label. +message LabelPair { + // The label name (e.g., "alertname"). + string key = 1; + + // The label value (e.g., "HighMemoryUsage"). + string value = 2; +} + +// LabelSet is an ordered collection of label pairs. +message LabelSet { + // The label pairs that make up this set. + repeated LabelPair labels = 1; +} + +// Alert represents a snapshot of an alert at the time the event was +// recorded. +message Alert { + // The unique fingerprint derived from the alert's label set. + uint64 fingerprint = 1; + + // The value of the "alertname" label, provided for convenience. + string name = 2; + + // The full label set that identifies this alert. + LabelSet labels = 3; + + // Informational annotations attached to the alert (e.g., summary, + // description). + LabelSet annotations = 4; + + // The time at which the alert started firing. + google.protobuf.Timestamp starts_at = 5; + + // The time at which the alert is considered resolved. For firing + // alerts this is typically set to a time in the future. + google.protobuf.Timestamp ends_at = 6; + + // Whether the alert was resolved at the time the event was recorded. + bool resolved = 7; +} + +// GroupedAlert is a reference to an alert within an aggregation group. +// It always carries the content hash; the full alert details are +// included when available. +message GroupedAlert { + // A hash of the alert's label set, used for deduplication within the + // notification pipeline. + uint64 hash = 1; + + // The full alert details. May be absent when only the hash is needed + // (e.g., in firing/resolved lists on NotificationEvent). + optional Alert details = 2; +} + +// AlertGroupInfo describes the aggregation group context in which an +// alert is being processed. +message AlertGroupInfo { + // The composite key that uniquely identifies this aggregation group + // (encodes route and group label values). + string group_key = 1; + + // The label set used to group alerts together within this route. + LabelSet group_labels = 2; + + // A stable, shortened identifier derived from the group key (SHA-256 + // hex). + string group_id = 3; + + // The name of the receiver that this group routes to. + string receiver_name = 4; + + // The set of matchers defined on the route that matched these alerts. + repeated Matcher matchers = 5; + + // A UUID that uniquely identifies this aggregation group instance. + string group_uuid = 6; +} + +// AlertCreatedEvent is emitted when a brand-new alert is inserted into +// the in-memory alert store. +message AlertCreatedEvent { + // The newly created alert. + Alert alert = 1; +} + +// AlertResolvedEvent is emitted when an alert is removed from its +// aggregation group after a successful notification that included the +// resolution. +message AlertResolvedEvent { + // The resolved alert, including its hash and full details. + GroupedAlert alert = 1; + + // The aggregation group from which the alert was resolved. + AlertGroupInfo group_info = 2; +} + +// AlertGroupedEvent is emitted the first time an alert is inserted into +// an aggregation group. +message AlertGroupedEvent { + // The alert being grouped, including its hash and full details. + GroupedAlert alert = 1; + + // The aggregation group the alert was added to. + AlertGroupInfo group_info = 2; +} + +// NotifyReason describes why a notification was sent for an aggregation +// group. +enum NotifyReason { + // Default / unknown reason. + NOTIFY_REASON_UNSPECIFIED = 0; + + // The group has never been notified before and contains at least one + // firing alert. + NOTIFY_REASON_FIRST_NOTIFICATION = 1; + + // New firing alerts have been added to the group since the last + // notification. + NOTIFY_REASON_NEW_ALERTS_IN_GROUP = 2; + + // Some alerts in the group have resolved since the last notification. + NOTIFY_REASON_NEW_RESOLVED_ALERTS = 3; + + // All alerts in the group have resolved. + NOTIFY_REASON_ALL_ALERTS_RESOLVED = 4; + + // The configured repeat interval has elapsed since the last + // notification. + NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED = 5; +} + +// Integration identifies a specific notification integration (e.g., +// the second PagerDuty receiver in a receiver definition). +message Integration { + // The type of the integration (e.g., "webhook", "pagerduty"). + string name = 1; + + // The zero-based index of this integration within its receiver. + int64 index = 2; +} + +// NotificationEvent is emitted after a notification is successfully +// delivered to an integration. +message NotificationEvent { + // All alerts that were included in the notification. + repeated GroupedAlert alerts = 1; + + // The subset of alerts that are currently firing. + repeated GroupedAlert firing_alerts = 2; + + // The subset of alerts that are resolved. + repeated GroupedAlert resolved_alerts = 3; + + // Alerts that were muted (silenced or inhibited) during this flush + // cycle. + repeated GroupedAlert muted_alerts = 4; + + // The aggregation group context for this notification. + AlertGroupInfo group_info = 5; + + // The configured repeat interval for the aggregation group's route. + google.protobuf.Duration repeat_interval = 6; + + // The reason the notification was triggered. + NotifyReason reason = 7; + + // A monotonically increasing identifier for each flush cycle of the + // aggregation group. + uint64 flush_id = 8; + + // The integration that delivered the notification. + Integration integration = 9; +} + +// Silence is a snapshot of a silence definition at the time the event +// was recorded. +message Silence { + // The globally unique silence identifier (UUID). + string id = 1; + + // The matchers that define which alerts this silence suppresses. + // For silences with multiple matcher sets, this is the first set. + repeated Matcher matchers = 2; + + // Optional structured annotations on the silence (key-value pairs). + LabelSet annotations = 3; + + // The time at which the silence becomes active. + google.protobuf.Timestamp starts_at = 4; + + // The time at which the silence expires. + google.protobuf.Timestamp ends_at = 5; + + // The last time the silence was created or updated. + google.protobuf.Timestamp updated_at = 6; + + // The author who created the silence. + string created_by = 7; + + // A human-readable comment explaining the silence. + string comment = 8; + + // Additional matcher sets evaluated with OR logic. At least one + // matcher set must match for the silence to apply. + repeated MatcherSet matcher_sets = 9; +} + +// Matcher defines a single label matching rule. +message Matcher { + // Type enumerates the supported matching operators. + enum Type { + // Unspecified / unknown match type. + TYPE_UNSPECIFIED = 0; + + // Exact string equality (=). + TYPE_EQUAL = 1; + + // Regular expression match (=~). + TYPE_REGEXP = 2; + + // Negated exact string equality (!=). + TYPE_NOT_EQUAL = 3; + + // Negated regular expression match (!~). + TYPE_NOT_REGEXP = 4; + } + + // The matching operator to apply. + Type type = 1; + + // The label name to match against. + string name = 2; + + // The value or pattern to match, interpreted according to type. + string pattern = 3; + + // Human-readable string representation (e.g., "env=~prod.*"). + string rendered = 4; +} + +// MatcherSet is a conjunction of matchers: all matchers in the set must +// match for the set to match. +message MatcherSet { + // The matchers that make up this set (evaluated with AND logic). + repeated Matcher matchers = 1; +} + +// SilenceCreatedEvent is emitted when a new silence is created. +message SilenceCreatedEvent { + // The newly created silence. + Silence silence = 1; +} + +// SilenceUpdatedEvent is emitted when an existing silence is modified. +message SilenceUpdatedEvent { + // The silence after the update. + Silence silence = 1; +} + +// MutedAlert identifies an alert that was suppressed by a silence or +// inhibition rule. +message MutedAlert { + // The label set of the muted alert. + LabelSet labels = 1; + + // The fingerprint of the muted alert. + uint64 fingerprint = 2; +} + +// SilenceMutedAlertEvent is emitted each time a silence suppresses an +// alert during the muting evaluation pass. +message SilenceMutedAlertEvent { + // The silence that suppressed the alert. + Silence silence = 1; + + // The alert that was suppressed. + MutedAlert muted_alert = 2; +} + +// InhibitRule is a snapshot of an inhibition rule definition. +message InhibitRule { + // Matchers that identify source alerts (those that do the inhibiting). + repeated Matcher source_matchers = 1; + + // Matchers that identify target alerts (those that get inhibited). + repeated Matcher target_matchers = 2; + + // Label names whose values must be equal between source and target + // alerts for the inhibition to take effect. + repeated string equal_labels = 3; +} + +// InhibitionMutedAlertEvent is emitted when one or more inhibition +// rules suppress an alert. +message InhibitionMutedAlertEvent { + // The inhibition rules that matched. + repeated InhibitRule inhibit_rules = 1; + + // The alert that was suppressed. + MutedAlert muted_alert = 2; + + // The fingerprints of the source alerts that caused the inhibition. + repeated uint64 inhibiting_fingerprints = 3; +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/events.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/events.go new file mode 100644 index 00000000000..1eccba9eff9 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/events.go @@ -0,0 +1,291 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains pure-functional helpers that convert internal +// Alertmanager types into eventrecorderpb messages, plus convenience +// constructors for the EventData oneof variants. None of these +// functions touch the Recorder; they are imported and called by the +// dispatch, silence, inhibit, and provider packages to build event +// payloads that are then handed to Recorder.RecordEvent. + +package eventrecorder + +import ( + "slices" + "strings" + + "github.com/prometheus/common/model" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" + "github.com/prometheus/alertmanager/pkg/labels" + silencepb "github.com/prometheus/alertmanager/silence/silencepb" + "github.com/prometheus/alertmanager/types" +) + +// LabelSetAsProto converts a model.LabelSet to an eventrecorderpb.LabelSet. +// Labels are sorted by name for deterministic output. +func LabelSetAsProto(ls model.LabelSet) *eventrecorderpb.LabelSet { + names := make([]model.LabelName, 0, len(ls)) + for k := range ls { + names = append(names, k) + } + slices.SortFunc(names, func(a, b model.LabelName) int { + return strings.Compare(string(a), string(b)) + }) + pairs := make([]*eventrecorderpb.LabelPair, 0, len(ls)) + for _, k := range names { + pairs = append(pairs, &eventrecorderpb.LabelPair{Key: string(k), Value: string(ls[k])}) + } + return &eventrecorderpb.LabelSet{Labels: pairs} +} + +// AlertAsProto converts a types.Alert to an eventrecorderpb.Alert. +func AlertAsProto(alert *types.Alert) *eventrecorderpb.Alert { + return &eventrecorderpb.Alert{ + Fingerprint: uint64(alert.Fingerprint()), + Name: alert.Name(), + Labels: LabelSetAsProto(alert.Labels), + Annotations: LabelSetAsProto(alert.Annotations), + StartsAt: timestamppb.New(alert.StartsAt), + EndsAt: timestamppb.New(alert.EndsAt), + Resolved: alert.Resolved(), + } +} + +// MatcherAsProto converts a single *labels.Matcher to its protobuf +// representation. +func MatcherAsProto(m *labels.Matcher) *eventrecorderpb.Matcher { + var matcherType eventrecorderpb.Matcher_Type + switch m.Type { + case labels.MatchEqual: + matcherType = eventrecorderpb.Matcher_TYPE_EQUAL + case labels.MatchNotEqual: + matcherType = eventrecorderpb.Matcher_TYPE_NOT_EQUAL + case labels.MatchRegexp: + matcherType = eventrecorderpb.Matcher_TYPE_REGEXP + case labels.MatchNotRegexp: + matcherType = eventrecorderpb.Matcher_TYPE_NOT_REGEXP + default: + matcherType = eventrecorderpb.Matcher_TYPE_UNSPECIFIED + } + return &eventrecorderpb.Matcher{ + Type: matcherType, + Name: m.Name, + Pattern: m.Value, + Rendered: m.String(), + } +} + +// MatchersAsProto converts a slice of matchers to their protobuf +// representations. +func MatchersAsProto(matchers labels.Matchers) []*eventrecorderpb.Matcher { + result := make([]*eventrecorderpb.Matcher, len(matchers)) + for i, m := range matchers { + result[i] = MatcherAsProto(m) + } + return result +} + +// SilenceMatcherAsProto converts a silencepb.Matcher to an +// eventrecorderpb.Matcher. +func SilenceMatcherAsProto(m *silencepb.Matcher) *eventrecorderpb.Matcher { + var matcherType eventrecorderpb.Matcher_Type + switch m.Type { + case silencepb.Matcher_EQUAL: + matcherType = eventrecorderpb.Matcher_TYPE_EQUAL + case silencepb.Matcher_REGEXP: + matcherType = eventrecorderpb.Matcher_TYPE_REGEXP + case silencepb.Matcher_NOT_EQUAL: + matcherType = eventrecorderpb.Matcher_TYPE_NOT_EQUAL + case silencepb.Matcher_NOT_REGEXP: + matcherType = eventrecorderpb.Matcher_TYPE_NOT_REGEXP + default: + matcherType = eventrecorderpb.Matcher_TYPE_UNSPECIFIED + } + + var rendered string + var matchType labels.MatchType + switch m.Type { + case silencepb.Matcher_EQUAL: + matchType = labels.MatchEqual + case silencepb.Matcher_NOT_EQUAL: + matchType = labels.MatchNotEqual + case silencepb.Matcher_REGEXP: + matchType = labels.MatchRegexp + case silencepb.Matcher_NOT_REGEXP: + matchType = labels.MatchNotRegexp + default: + matchType = labels.MatchEqual + } + if lm, err := labels.NewMatcher(matchType, m.Name, m.Pattern); err == nil { + rendered = lm.String() + } + + return &eventrecorderpb.Matcher{ + Type: matcherType, + Name: m.Name, + Pattern: m.Pattern, + Rendered: rendered, + } +} + +// SilenceAsProto converts a silencepb.Silence to an +// eventrecorderpb.Silence. +func SilenceAsProto(sil *silencepb.Silence) *eventrecorderpb.Silence { + matcherSets := make([]*eventrecorderpb.MatcherSet, len(sil.MatcherSets)) + for i, ms := range sil.MatcherSets { + matcherSet := &eventrecorderpb.MatcherSet{ + Matchers: make([]*eventrecorderpb.Matcher, len(ms.Matchers)), + } + for j, m := range ms.Matchers { + matcherSet.Matchers[j] = SilenceMatcherAsProto(m) + } + matcherSets[i] = matcherSet + } + + var matchers []*eventrecorderpb.Matcher + if len(matcherSets) > 0 { + matchers = matcherSets[0].Matchers + } + + return &eventrecorderpb.Silence{ + Id: sil.Id, + Matchers: matchers, + MatcherSets: matcherSets, + StartsAt: sil.StartsAt, + EndsAt: sil.EndsAt, + UpdatedAt: sil.UpdatedAt, + CreatedBy: sil.CreatedBy, + Comment: sil.Comment, + } +} + +// InhibitRuleAsProto converts inhibit rule fields to an +// eventrecorderpb.InhibitRule. It accepts the individual fields rather +// than the InhibitRule struct to avoid an import cycle. +func InhibitRuleAsProto(sourceMatchers, targetMatchers labels.Matchers, equal map[model.LabelName]struct{}) *eventrecorderpb.InhibitRule { + equalLabels := make([]string, 0, len(equal)) + for label := range equal { + equalLabels = append(equalLabels, string(label)) + } + slices.Sort(equalLabels) + return &eventrecorderpb.InhibitRule{ + SourceMatchers: MatchersAsProto(sourceMatchers), + TargetMatchers: MatchersAsProto(targetMatchers), + EqualLabels: equalLabels, + } +} + +// NewAlertCreatedEvent constructs an AlertCreated event. +func NewAlertCreatedEvent(alert *types.Alert) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_AlertCreated{ + AlertCreated: &eventrecorderpb.AlertCreatedEvent{ + Alert: AlertAsProto(alert), + }, + }, + } +} + +// NewSilenceMutedAlertEvent constructs a SilenceMutedAlert event. +func NewSilenceMutedAlertEvent(silence *eventrecorderpb.Silence, fp model.Fingerprint, lset model.LabelSet) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_SilenceMutedAlert{ + SilenceMutedAlert: &eventrecorderpb.SilenceMutedAlertEvent{ + Silence: silence, + MutedAlert: &eventrecorderpb.MutedAlert{ + Fingerprint: uint64(fp), + Labels: LabelSetAsProto(lset), + }, + }, + }, + } +} + +// NewSilenceCreatedEvent constructs a SilenceCreated event. +func NewSilenceCreatedEvent(silence *eventrecorderpb.Silence) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_SilenceCreated{ + SilenceCreated: &eventrecorderpb.SilenceCreatedEvent{ + Silence: silence, + }, + }, + } +} + +// NewSilenceUpdatedEvent constructs a SilenceUpdated event. +func NewSilenceUpdatedEvent(silence *eventrecorderpb.Silence) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_SilenceUpdated{ + SilenceUpdated: &eventrecorderpb.SilenceUpdatedEvent{ + Silence: silence, + }, + }, + } +} + +// NewInhibitionMutedAlertEvent constructs an InhibitionMutedAlert event. +func NewInhibitionMutedAlertEvent(rules []*eventrecorderpb.InhibitRule, fp model.Fingerprint, lset model.LabelSet, inhibitingFPs []model.Fingerprint) *eventrecorderpb.EventData { + fps := make([]uint64, len(inhibitingFPs)) + for i, f := range inhibitingFPs { + fps[i] = uint64(f) + } + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_InhibitionMutedAlert{ + InhibitionMutedAlert: &eventrecorderpb.InhibitionMutedAlertEvent{ + InhibitRules: rules, + MutedAlert: &eventrecorderpb.MutedAlert{ + Fingerprint: uint64(fp), + Labels: LabelSetAsProto(lset), + }, + InhibitingFingerprints: fps, + }, + }, + } +} + +// extractEventType returns the proto oneof field name for the event +// type (e.g. "alert_created", "notification"). It uses a type switch +// on the generated oneof wrapper types, avoiding proto reflection. +// A nil input is reported as "unknown" so logging and metric paths +// stay panic-free. +func extractEventType(event *eventrecorderpb.EventData) string { + if event == nil { + return "unknown" + } + switch event.EventType.(type) { + case *eventrecorderpb.EventData_AlertmanagerStartupEvent: + return "alertmanager_startup_event" + case *eventrecorderpb.EventData_AlertmanagerShutdownEvent: + return "alertmanager_shutdown_event" + case *eventrecorderpb.EventData_AlertCreated: + return "alert_created" + case *eventrecorderpb.EventData_AlertResolved: + return "alert_resolved" + case *eventrecorderpb.EventData_AlertGrouped: + return "alert_grouped" + case *eventrecorderpb.EventData_Notification: + return "notification" + case *eventrecorderpb.EventData_SilenceCreated: + return "silence_created" + case *eventrecorderpb.EventData_SilenceUpdated: + return "silence_updated" + case *eventrecorderpb.EventData_SilenceMutedAlert: + return "silence_muted_alert" + case *eventrecorderpb.EventData_InhibitionMutedAlert: + return "inhibition_muted_alert" + default: + return "unknown" + } +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/file.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/file.go new file mode 100644 index 00000000000..d29572de66e --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/file.go @@ -0,0 +1,198 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventrecorder + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" + "google.golang.org/protobuf/encoding/protojson" + + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" +) + +// FileOutputConfig configures a JSONL file event recorder output. +type FileOutputConfig struct { + // Path is the JSONL file to append events to. Created if absent. + Path string `yaml:"path" json:"path"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface, validating +// the file output configuration. +func (c *FileOutputConfig) UnmarshalYAML(unmarshal func(any) error) error { + type plain FileOutputConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.Path == "" { + return errors.New("event_recorder file output requires a path") + } + return nil +} + +// equal reports whether two file output configs are semantically equal. +func (c FileOutputConfig) equal(o FileOutputConfig) bool { + return c.Path == o.Path +} + +// FileOutput writes pre-serialized JSON event bytes to a JSONL file. +// The file is reopened when fsnotify detects a rename or remove (e.g. +// logrotate). +type FileOutput struct { + path string + mu sync.Mutex + f *os.File + closed bool + logger *slog.Logger + done chan struct{} + wg sync.WaitGroup +} + +// Name returns a stable identifier for this output. +func (fo *FileOutput) Name() string { + return fmt.Sprintf("file:%s", fo.path) +} + +// NewFileOutput creates a new file-based event recorder output at the given +// path. The file is watched with fsnotify so that external log +// rotation tools (e.g., logrotate) trigger an immediate reopen. +func NewFileOutput(path string, logger *slog.Logger) (*FileOutput, error) { + f, err := openAppend(path) + if err != nil { + return nil, err + } + + fo := &FileOutput{ + path: path, + f: f, + logger: logger, + done: make(chan struct{}), + } + + ready := make(chan error, 1) + fo.wg.Add(1) + go fo.watchLoop(ready) + if err := <-ready; err != nil { + f.Close() + return nil, fmt.Errorf("starting file watcher: %w", err) + } + return fo, nil +} + +func openAppend(path string) (*os.File, error) { + return os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) +} + +func (fo *FileOutput) reopen() { + fo.mu.Lock() + defer fo.mu.Unlock() + + if fo.closed { + return + } + if fo.f != nil { + fo.f.Close() + } + f, err := openAppend(fo.path) + if err != nil { + fo.logger.Error("Failed to reopen event recorder file", "path", fo.path, "err", err) + return + } + fo.f = f +} + +func (fo *FileOutput) watchLoop(ready chan<- error) { + defer fo.wg.Done() + + watcher, err := fsnotify.NewWatcher() + if err != nil { + ready <- fmt.Errorf("creating fsnotify watcher: %w", err) + return + } + defer watcher.Close() + + // Watch the parent directory rather than the file itself. + // When logrotate renames the file, an inode-level watch is + // lost with the old inode. A directory watch reliably + // delivers Rename/Remove/Create events for the target path. + dir := filepath.Dir(fo.path) + if err := watcher.Add(dir); err != nil { + ready <- fmt.Errorf("watching directory %s: %w", dir, err) + return + } + + absPath, err := filepath.Abs(fo.path) + if err != nil { + ready <- fmt.Errorf("resolving absolute path for %s: %w", fo.path, err) + return + } + + ready <- nil + + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + // Only react to events on our target file. + evPath, _ := filepath.Abs(event.Name) + if evPath != absPath { + continue + } + if event.Op&(fsnotify.Rename|fsnotify.Remove|fsnotify.Create) != 0 { + fo.reopen() + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + fo.logger.Error("fsnotify error on event recorder directory", "err", err) + case <-fo.done: + return + } + } +} + +// SendEvent serializes the event as a JSON line and appends it to the +// file. It returns the number of bytes written (including the trailing +// newline) for the bytes-written metric. +func (fo *FileOutput) SendEvent(event *eventrecorderpb.Event) (int, error) { + data, err := protojson.Marshal(event) + if err != nil { + return 0, &serializeError{err: err} + } + data = append(data, '\n') + + fo.mu.Lock() + defer fo.mu.Unlock() + n, err := fo.f.Write(data) + return n, err +} + +// Close stops the watcher goroutine, waits for it to exit, and closes +// the file. +func (fo *FileOutput) Close() error { + close(fo.done) + fo.wg.Wait() + fo.mu.Lock() + defer fo.mu.Unlock() + fo.closed = true + return fo.f.Close() +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/kafka.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/kafka.go new file mode 100644 index 00000000000..d60f7082b77 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/kafka.go @@ -0,0 +1,364 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventrecorder + +import ( + "context" + "errors" + "fmt" + "log/slog" + "reflect" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + commoncfg "github.com/prometheus/common/config" + "github.com/twmb/franz-go/pkg/kgo" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" + "github.com/prometheus/alertmanager/kafka" +) + +const defaultKafkaBufferSize = 1024 + +// KafkaOutputConfig configures a Kafka event recorder output. +type KafkaOutputConfig struct { + // Brokers is the list of Kafka seed brokers in host:port form. + Brokers []string `yaml:"brokers" json:"brokers"` + // Topic is the Kafka topic to produce events to. + Topic string `yaml:"topic" json:"topic"` + // ClientID is reported to the Kafka brokers (default "alertmanager"). + ClientID string `yaml:"client_id,omitempty" json:"client_id,omitempty"` + // Format selects the on-the-wire encoding of each event value: + // "json" (default, JSON via protojson) or "protobuf" (binary proto). + Format kafka.Format `yaml:"format,omitempty" json:"format,omitempty"` + // Acks controls the producer acknowledgement level: + // "none", "leader" (default), or "all". + Acks kafka.Acks `yaml:"acks,omitempty" json:"acks,omitempty"` + // Compression selects the producer compression codec: + // "" (default, no compression), "none", "gzip", "snappy", "lz4", or "zstd". + Compression kafka.Compression `yaml:"compression,omitempty" json:"compression,omitempty"` + // BufferSize is the capacity of the local channel between the event + // recorder dispatcher and the franz-go producer (default 1024). + BufferSize int `yaml:"buffer_size,omitempty" json:"buffer_size,omitempty"` + // TLSConfig configures TLS for the Kafka broker connection. If unset, + // PLAINTEXT is used. + TLSConfig *commoncfg.TLSConfig `yaml:"tls_config,omitempty" json:"tls_config,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface, validating +// and normalising the Kafka output configuration. Transport-layer +// validation (brokers, acks, compression, TLS) is delegated to the +// shared kafka package so this code and a future Kafka receiver share a +// single source of truth. +func (c *KafkaOutputConfig) UnmarshalYAML(unmarshal func(any) error) error { + type plain KafkaOutputConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if err := c.clientOptions().Validate(); err != nil { + // The shared validator's messages already say "kafka: ..."; we + // prefix with the event_recorder context for user clarity. + return fmt.Errorf("event_recorder %w", err) + } + if c.Topic == "" { + return errors.New("event_recorder kafka output requires a topic") + } + if c.Format == "" { + c.Format = kafka.FormatJSON + } + if err := kafka.ValidateFormat(c.Format); err != nil { + return fmt.Errorf("event_recorder %w", err) + } + return nil +} + +// clientOptions copies the config into a kafka.ClientOptions value +// suitable for passing to kafka.BuildOpts. +func (c KafkaOutputConfig) clientOptions() kafka.ClientOptions { + return kafka.ClientOptions{ + Brokers: c.Brokers, + Topic: c.Topic, + ClientID: c.ClientID, + Acks: c.Acks, + Compression: c.Compression, + TLSConfig: c.TLSConfig, + } +} + +// equal reports whether two kafka output configs are semantically +// equal. Broker lists are compared order-independently because +// reordering brokers in YAML is semantically a no-op. +func (c KafkaOutputConfig) equal(o KafkaOutputConfig) bool { + if !kafka.BrokerListsEqual(c.Brokers, o.Brokers) { + return false + } + if c.Topic != o.Topic { + return false + } + if c.ClientID != o.ClientID { + return false + } + if c.Format != o.Format { + return false + } + if c.Acks != o.Acks { + return false + } + if c.Compression != o.Compression { + return false + } + if c.BufferSize != o.BufferSize { + return false + } + return reflect.DeepEqual(c.TLSConfig, o.TLSConfig) +} + +// KafkaOutput delivers serialized events to a Kafka topic via franz-go. +// Events are buffered in a bounded local channel and produced by a +// single dispatcher goroutine; franz-go handles batching, compression, +// and retries internally. +// +// When the local buffer is full, events are dropped (with a log +// message and a metric increment) so that a slow or unreachable +// broker cannot block the upstream event recorder pipeline. +type KafkaOutput struct { + mu sync.RWMutex + closed bool + client *kgo.Client + topic string + instance string // used as the message key + format kafka.Format + name string // "kafka:/" + logger *slog.Logger + drops prometheus.Counter + produceErrs *prometheus.CounterVec + work chan []byte + done chan struct{} + wg sync.WaitGroup + flushBudget time.Duration +} + +// NewKafkaOutput constructs a KafkaOutput from the supplied configuration. +// A failure to reach the brokers at startup is logged at warn level but +// does not fail construction; franz-go retries connections in the +// background and records are buffered until delivery becomes possible. +func NewKafkaOutput( + cfg KafkaOutputConfig, + instance string, + dropsCounter *prometheus.CounterVec, + produceErrors *prometheus.CounterVec, + logger *slog.Logger, +) (*KafkaOutput, error) { + if cfg.Topic == "" { + return nil, errors.New("kafka output requires a topic") + } + format := cfg.Format + if format == "" { + format = kafka.FormatJSON + } + if err := kafka.ValidateFormat(format); err != nil { + return nil, err + } + + // Shared validation + franz-go option construction lives in the + // kafka package so a future Kafka receiver can reuse it. + kopts, err := kafka.BuildOpts(cfg.clientOptions(), logger) + if err != nil { + return nil, err + } + + client, err := kgo.NewClient(kopts...) + if err != nil { + return nil, fmt.Errorf("kafka output: creating client: %w", err) + } + + bufferSize := cfg.BufferSize + if bufferSize <= 0 { + bufferSize = defaultKafkaBufferSize + } + + name := fmt.Sprintf("kafka:%s/%s", kafka.BrokerList(cfg.Brokers), cfg.Topic) + + ko := &KafkaOutput{ + client: client, + topic: cfg.Topic, + instance: instance, + format: format, + name: name, + logger: logger, + drops: dropsCounter.WithLabelValues(name), + produceErrs: produceErrors, + work: make(chan []byte, bufferSize), + done: make(chan struct{}), + flushBudget: kafka.DefaultFlushBudget, + } + + // Best-effort connectivity check runs in the background so that + // alertmanager startup (and event_recorder hot reload) is never + // blocked by an unreachable broker. + kafka.PingInBackground(client, logger) + + ko.wg.Add(1) + go ko.dispatch() + + return ko, nil +} + +// Name returns the stable identifier for this output. +func (ko *KafkaOutput) Name() string { return ko.name } + +// SendEvent serializes the event in the configured format (JSON or +// protobuf) and queues it for asynchronous delivery. It returns the +// serialized size (for the bytes-written metric). +func (ko *KafkaOutput) SendEvent(event *eventrecorderpb.Event) (int, error) { + var ( + data []byte + err error + ) + if ko.format == kafka.FormatProtobuf { + data, err = proto.Marshal(event) + } else { + data, err = protojson.Marshal(event) + } + if err != nil { + return 0, &serializeError{err: err} + } + if err := ko.enqueue(data); err != nil { + return 0, err + } + return len(data), nil +} + +// enqueue places the value on the local buffer. Returns an error if +// the output is already closed (so a SendEvent racing with Close cannot +// land a record on a channel that no dispatcher will drain). If the +// buffer is full the event is dropped; the upstream metric records the +// drop directly so callers do not need to inspect this error to count +// drops. +func (ko *KafkaOutput) enqueue(value []byte) error { + ko.mu.RLock() + defer ko.mu.RUnlock() + if ko.closed { + return errors.New("kafka output: closed") + } + select { + case ko.work <- value: + return nil + default: + ko.drops.Inc() + ko.logger.Warn("Kafka event recorder buffer full, dropping event", "output", ko.name) + return nil + } +} + +// dispatch reads queued payloads and hands them to franz-go for +// asynchronous production. A single goroutine is sufficient because +// produce uses kgo.Client.TryProduce, which never blocks (see produce). +func (ko *KafkaOutput) dispatch() { + defer ko.wg.Done() + for { + select { + case value := <-ko.work: + ko.produce(value) + case <-ko.done: + // Drain whatever is left in the local channel into + // franz-go's producer before returning. The actual + // flush to brokers happens in Close. + ko.drainWork() + return + } + } +} + +// drainWork non-blockingly drains every queued value into the +// producer. Called by dispatch on shutdown. +func (ko *KafkaOutput) drainWork() { + for { + select { + case value := <-ko.work: + ko.produce(value) + default: + return + } + } +} + +// produce hands a single record to franz-go. The promise callback +// updates per-output metrics; it must be quick (franz-go calls all +// promises serially from a single goroutine). +// +// TryProduce is used instead of Produce so the dispatcher goroutine +// never blocks: kgo.Client.Produce blocks once the producer's internal +// buffer (MaxBufferedRecords / MaxBufferedBytes) is full, which an +// unreachable broker can trigger. A blocked dispatcher would stall +// shutdown, since Close waits for it to drain. TryProduce instead +// fails the record immediately with kgo.ErrMaxBuffered, which we count +// as a drop — consistent with the drop-on-full behaviour of our own +// local buffer. +func (ko *KafkaOutput) produce(value []byte) { + rec := &kgo.Record{ + Key: []byte(ko.instance), + Value: value, + Topic: ko.topic, + } + ko.client.TryProduce(context.Background(), rec, func(_ *kgo.Record, err error) { + switch { + case err == nil: + return + case errors.Is(err, kgo.ErrMaxBuffered): + ko.drops.Inc() + ko.logger.Warn("Kafka producer buffer full, dropping event", "output", ko.name) + default: + ko.produceErrs.WithLabelValues(ko.name, string(kafka.ClassifyError(err))).Inc() + ko.logger.Warn("Kafka event recorder produce failed", "output", ko.name, "err", err) + } + }) +} + +// Close stops the dispatcher, drains any remaining buffered records +// into franz-go's producer, flushes the producer (up to flushBudget), +// and then closes the underlying client. Pending records that do not +// flush before the budget expires are dropped on client close. +// +// Close is safe to call multiple times; subsequent calls are no-ops. +// +// Note: franz-go's Client.Close has documented blocking behaviour +// around leaving consumer groups, but this client is configured as a +// producer only (no ConsumeTopics / ConsumePartitions / InstanceID), +// so the leave-group path is a no-op and Close will not block on it. +// The only bounded wait here is the explicit Flush above. +func (ko *KafkaOutput) Close() error { + ko.mu.Lock() + if ko.closed { + ko.mu.Unlock() + return nil + } + ko.closed = true + close(ko.done) + ko.mu.Unlock() + + ko.wg.Wait() + + ctx, cancel := context.WithTimeout(context.Background(), ko.flushBudget) + defer cancel() + if err := ko.client.Flush(ctx); err != nil { + ko.logger.Warn("Kafka event recorder flush did not complete within budget; remaining records will be dropped", + "output", ko.name, "err", err) + } + ko.client.Close() + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/metrics.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/metrics.go new file mode 100644 index 00000000000..254aecbb65a --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/metrics.go @@ -0,0 +1,82 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventrecorder + +import "github.com/prometheus/client_golang/prometheus" + +// metrics holds Prometheus metrics for the event recorder. The struct +// is internal — individual counters are passed to output constructors +// that need them (e.g. outputDrops to webhook and kafka, kafkaProduceErrors +// to kafka). +type metrics struct { + eventsRecorded *prometheus.CounterVec + eventRecorderBytesWritten *prometheus.CounterVec + eventsDropped *prometheus.CounterVec + eventSerializeErrors *prometheus.CounterVec + outputDrops *prometheus.CounterVec + kafkaProduceErrors *prometheus.CounterVec +} + +// newMetrics builds and (if r is non-nil) registers every metric the +// event recorder exposes. Pass nil for r in tests to obtain a metric +// set that is not registered against a global registry. +func newMetrics(r prometheus.Registerer) *metrics { + eventsRecorded := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_events_recorded_total", + Help: "The total number of events recorded by the event recorder.", + }, []string{"event_type", "output", "result"}) + + eventRecorderBytesWritten := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_event_recorder_bytes_written_total", + Help: "The total number of bytes written to the event recorder.", + }, []string{"event_type", "output"}) + + eventsDropped := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_events_dropped_total", + Help: "The total number of events dropped due to a full queue.", + }, []string{"event_type"}) + + eventSerializeErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_event_serialize_errors_total", + Help: "The total number of events that failed to serialize.", + }, []string{"event_type"}) + + // outputDrops is incremented when an output's local delivery buffer + // is full and an event has to be dropped before reaching the wire. + // Replaces the legacy alertmanager_event_webhook_drops_total metric + // and is shared by the webhook and kafka outputs. + outputDrops := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_event_output_drops_total", + Help: "The total number of events dropped by an output due to a full local buffer.", + }, []string{"output"}) + + kafkaProduceErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alertmanager_event_kafka_produce_errors_total", + Help: "The total number of Kafka produce attempts that failed.", + }, []string{"output", "error_type"}) + + if r != nil { + r.MustRegister(eventsRecorded, eventRecorderBytesWritten, eventsDropped, + eventSerializeErrors, outputDrops, kafkaProduceErrors) + } + + return &metrics{ + eventsRecorded: eventsRecorded, + eventRecorderBytesWritten: eventRecorderBytesWritten, + eventsDropped: eventsDropped, + eventSerializeErrors: eventSerializeErrors, + outputDrops: outputDrops, + kafkaProduceErrors: kafkaProduceErrors, + } +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/recorder.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/recorder.go new file mode 100644 index 00000000000..49f978cddfe --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/recorder.go @@ -0,0 +1,371 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package eventrecorder provides a structured event recorder for +// significant Alertmanager events. Events are serialized as JSON and +// fanned out to one or more configured destinations (JSONL file, +// webhook, kafka). +// +// RecordEvent never blocks the caller: events are serialized and +// placed on a bounded in-memory queue. A background goroutine +// drains the queue and sends to destinations. If the queue is full, +// events are dropped and a metric is incremented. +// +// Package layout: +// +// - recorder.go Recorder core: types, write loop, fan-out. +// - metrics.go Prometheus metric definitions. +// - events.go Pure proto-conversion helpers and event constructors. +// - config.go Top-level Config: per-type output lists + equality. +// - file.go File output and its config. +// - webhook.go Webhook output and its config. +// - kafka.go Kafka output and its config. +package eventrecorder + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/alertmanager/cluster" + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" +) + +const ( + // Maximum number of events buffered before new events are dropped. + // At ~500 bytes per event this caps memory usage at roughly 4 MB. + eventQueueSize = 8192 +) + +type recordingEnabledContextKey struct{} + +// WithEventRecording returns a context that enables event recording. +// By default, event recording is disabled; callers must opt in by +// decorating their context with this function. +func WithEventRecording(ctx context.Context) context.Context { + return context.WithValue(ctx, recordingEnabledContextKey{}, true) +} + +// EventRecordingEnabled reports whether event recording has been +// enabled in the given context via WithEventRecording. +func EventRecordingEnabled(ctx context.Context) bool { + v, _ := ctx.Value(recordingEnabledContextKey{}).(bool) + return v +} + +// Recorder is a concrete, non-nil-able handle to an event recorder. +// Because it is a struct (not an interface), passing nil where a +// Recorder is expected is a compile-time error. +// +// The zero value (Recorder{}) is safe to use and silently discards all +// events, but prefer NopRecorder() for clarity. +type Recorder struct { + core *sharedRecorder +} + +// writeRequest is a single event queued for background serialization +// and writing. It carries the proto message so that the expensive +// protojson.Marshal call happens in the write-loop goroutine, not on +// the caller's hot path. +type writeRequest struct { + event *eventrecorderpb.Event + eventType string +} + +// sharedRecorder holds the mutable state shared by all copies of a +// Recorder value. Mutable state (outputs, currentCfg) is owned +// exclusively by the writeLoop goroutine and updated via the +// cfgUpdate channel, eliminating the need for a mutex. +type sharedRecorder struct { + instance string + logger *slog.Logger + metrics *metrics + peer atomic.Pointer[cluster.Peer] + + // Async write queue. nil for NopRecorder, non-nil for active. + events chan writeRequest + cfgUpdate chan cfgUpdateMsg + done chan struct{} + closeOnce sync.Once + wg sync.WaitGroup +} + +// cfgUpdateMsg is sent to writeLoop to hot-reload the configuration. +// The sender blocks until the writeLoop acknowledges by closing done. +type cfgUpdateMsg struct { + cfg Config + done chan struct{} +} + +// Destination is a single event destination. Each implementation +// owns its own serialization: it receives the structured event and is +// responsible for encoding it (e.g. JSON or protobuf) and delivering it. +// +// Owning serialization per destination — rather than handing every +// destination a pre-encoded JSON blob — avoids the footgun of, say, a +// protobuf-configured Kafka output silently shipping a JSON payload. +type Destination interface { + // Name returns a stable identifier for this destination, suitable + // for use as a Prometheus label value (e.g. "file:/var/log/events.jsonl" + // or "webhook:https://example.com/hook"). + Name() string + // SendEvent encodes and delivers the event. It returns the number + // of payload bytes written (for the bytes-written metric) and any + // delivery error. A serialization failure should be returned + // wrapped in *serializeError so the recorder can attribute it to + // the serialize-errors metric. + SendEvent(event *eventrecorderpb.Event) (size int, err error) + io.Closer +} + +// serializeError marks a failure to encode an event (as opposed to a +// delivery failure) so marshalAndSend can attribute it to the +// serialize-errors metric. Destinations wrap encoding failures in it. +type serializeError struct{ err error } + +func (e *serializeError) Error() string { return e.err.Error() } +func (e *serializeError) Unwrap() error { return e.err } + +// NopRecorder returns a Recorder that silently discards all events. +// Use this in tests or when the event recorder is not configured. +func NopRecorder() Recorder { + return Recorder{core: &sharedRecorder{}} +} + +// NewRecorderFromConfig builds a Recorder from the given configuration. +// A background goroutine is started to drain the event queue; call +// Close to stop it. +func NewRecorderFromConfig(cfg Config, instance string, logger *slog.Logger, r prometheus.Registerer) Recorder { + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + core := &sharedRecorder{ + instance: instance, + logger: logger, + metrics: newMetrics(r), + events: make(chan writeRequest, eventQueueSize), + cfgUpdate: make(chan cfgUpdateMsg), + done: make(chan struct{}), + } + initialOutputs := buildOutputs(cfg, instance, core.metrics, logger) + + if r != nil { + r.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "alertmanager_event_recorder_queue_length", + Help: "Current number of events waiting in the event recorder write queue.", + }, func() float64 { + return float64(len(core.events)) + })) + } + + core.wg.Add(1) + go core.writeLoop(initialOutputs, cfg) + + return Recorder{core: core} +} + +// buildOutputs creates Destination implementations from the given config. +func buildOutputs(cfg Config, instance string, m *metrics, logger *slog.Logger) []Destination { + var outputs []Destination + for _, fc := range cfg.FileOutputs { + fo, err := NewFileOutput(fc.Path, logger) + if err != nil { + logger.Error("Failed to create file event recorder output", "path", fc.Path, "err", err) + continue + } + outputs = append(outputs, fo) + } + for _, wc := range cfg.WebhookOutputs { + wo, err := NewWebhookOutput(wc, m.outputDrops, logger) + if err != nil { + logger.Error("Failed to create webhook event recorder output", "url", wc.URL, "err", err) + continue + } + outputs = append(outputs, wo) + } + for _, kc := range cfg.KafkaOutputs { + ko, err := NewKafkaOutput(kc, instance, m.outputDrops, m.kafkaProduceErrors, logger) + if err != nil { + logger.Error("Failed to create kafka event recorder output", "brokers", kc.Brokers, "topic", kc.Topic, "err", err) + continue + } + outputs = append(outputs, ko) + } + return outputs +} + +// writeLoop drains the event queue, serializes events, and writes to +// outputs. It owns the outputs and currentCfg exclusively — all +// mutations arrive via the cfgUpdate channel, so no mutex is needed. +// +// The protojson.Marshal runs here (not in the caller goroutine) so that +// the serialization cost is off the alert-processing hot path. +// +// It runs until the done channel is closed, then drains remaining +// events and closes all outputs before returning. +func (c *sharedRecorder) writeLoop(outputs []Destination, currentCfg Config) { + defer c.wg.Done() + defer func() { + for _, out := range outputs { + if err := out.Close(); err != nil && c.logger != nil { + c.logger.Error("Failed to close event recorder output", "err", err) + } + } + }() + + for { + select { + case req := <-c.events: + c.marshalAndSend(req, outputs) + case update := <-c.cfgUpdate: + if !configEqual(update.cfg, currentCfg) { + newOutputs := buildOutputs(update.cfg, c.instance, c.metrics, c.logger) + if len(newOutputs) != update.cfg.totalOutputs() { + // Some outputs failed to initialize. Keep the existing + // (known-good) set rather than risking partial coverage. + c.logger.Error("Failed to reload event recorder outputs; keeping existing outputs") + for _, out := range newOutputs { + if err := out.Close(); err != nil { + c.logger.Error("Failed to close partially-built event recorder output", "err", err) + } + } + close(update.done) + continue + } + oldOutputs := outputs + outputs = newOutputs + currentCfg = update.cfg + for _, out := range oldOutputs { + if err := out.Close(); err != nil { + c.logger.Error("Failed to close old event recorder output", "err", err) + } + } + c.logger.Info("Event recorder configuration reloaded", "outputs", len(outputs)) + } + close(update.done) + case <-c.done: + // Drain remaining events and any pending config updates. + for { + select { + case req := <-c.events: + c.marshalAndSend(req, outputs) + case update := <-c.cfgUpdate: + close(update.done) + default: + return + } + } + } + } +} + +// marshalAndSend fans the queued event out to all outputs. Each +// destination owns its own serialization, so the recorder hands every +// output the structured event and records per-output result and +// bytes-written metrics from the returned size/error. +func (c *sharedRecorder) marshalAndSend(req writeRequest, outputs []Destination) { + for _, out := range outputs { + name := out.Name() + size, err := out.SendEvent(req.event) + if err != nil { + var se *serializeError + if errors.As(err, &se) { + c.metrics.eventSerializeErrors.WithLabelValues(req.eventType).Inc() + } + c.metrics.eventsRecorded.WithLabelValues(req.eventType, name, "error").Inc() + c.logger.Error("Failed to write event", "event_type", req.eventType, "output", name, "err", err) + continue + } + c.metrics.eventsRecorded.WithLabelValues(req.eventType, name, "success").Inc() + c.metrics.eventRecorderBytesWritten.WithLabelValues(req.eventType, name).Add(float64(size)) + } +} + +// RecordEvent wraps the event and places it on a bounded queue for +// background serialization and delivery. If the queue is full the +// event is dropped (never blocks the caller). Recording only occurs +// when the context has been decorated with WithEventRecording. +// +// The expensive protojson.Marshal call is deferred to the write-loop +// goroutine so that the caller's hot path only pays for the proto +// wrapping and a channel send. +func (r Recorder) RecordEvent(ctx context.Context, event *eventrecorderpb.EventData) { + if r.core == nil || r.core.events == nil { + return + } + if !EventRecordingEnabled(ctx) { + return + } + + eventType := extractEventType(event) + + wrappedEvent := &eventrecorderpb.Event{ + Timestamp: timestamppb.Now(), + Instance: r.core.instance, + Data: event, + } + + if peer := r.core.peer.Load(); peer != nil { + wrappedEvent.ClusterPosition = uint32(peer.Position()) + } + + select { + case r.core.events <- writeRequest{event: wrappedEvent, eventType: eventType}: + default: + // Queue full; drop event to avoid blocking alertmanager. + r.core.metrics.eventsDropped.WithLabelValues(eventType).Inc() + } +} + +// SetClusterPeer sets the cluster peer for HA position tracking. +func (r Recorder) SetClusterPeer(peer *cluster.Peer) { + if r.core == nil { + return + } + r.core.peer.Store(peer) +} + +// ApplyConfig hot-reloads the event recorder configuration. The update is +// sent to the writeLoop goroutine, which owns the outputs; this method +// blocks until the writeLoop has acknowledged the update. +func (r Recorder) ApplyConfig(cfg Config) { + if r.core == nil || r.core.cfgUpdate == nil { + return + } + ack := make(chan struct{}) + select { + case r.core.cfgUpdate <- cfgUpdateMsg{cfg: cfg, done: ack}: + <-ack + case <-r.core.done: + // Shutting down; ignore config update. + } +} + +// Close signals the background goroutine to drain remaining events +// and stop. The writeLoop closes all outputs before returning. +func (r Recorder) Close() error { + if r.core == nil || r.core.done == nil { + return nil + } + r.core.closeOnce.Do(func() { + close(r.core.done) + }) + r.core.wg.Wait() + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/eventrecorder/webhook.go b/vendor/github.com/prometheus/alertmanager/eventrecorder/webhook.go new file mode 100644 index 00000000000..b2d4bc548a6 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/eventrecorder/webhook.go @@ -0,0 +1,302 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventrecorder + +import ( + "bytes" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "reflect" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + commoncfg "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "google.golang.org/protobuf/encoding/protojson" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" +) + +// WebhookOutputConfig configures an HTTP webhook event recorder output. +type WebhookOutputConfig struct { + // URL is the endpoint to POST each event to. + URL *amcommoncfg.SecretURL `yaml:"url" json:"url"` + // HTTPConfig configures the HTTP client used for webhook delivery. + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + // Timeout for webhook HTTP requests (default 10s). + Timeout model.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` + // Workers is the number of concurrent delivery goroutines (default 4). + Workers int `yaml:"workers,omitempty" json:"workers,omitempty"` + // MaxRetries is the maximum number of delivery attempts per event + // (default 3). + MaxRetries int `yaml:"max_retries,omitempty" json:"max_retries,omitempty"` + // RetryBackoff is the base backoff between retry attempts (default + // 500ms). Successive attempts use exponential backoff (base * + // 2^attempt). + RetryBackoff model.Duration `yaml:"retry_backoff,omitempty" json:"retry_backoff,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface, validating +// the webhook output configuration. +// +// Note: SecretURL.UnmarshalYAML delegates to ParseURL, which already +// enforces a non-empty host and an http(s) scheme. The only way an +// otherwise-valid config reaches this function with a degenerate URL is +// via the "" placeholder shortcut in SecretURL.UnmarshalYAML, +// which sets URL to an empty url.URL{}. We catch that case here. +func (c *WebhookOutputConfig) UnmarshalYAML(unmarshal func(any) error) error { + type plain WebhookOutputConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.URL == nil || c.URL.URL == nil { + return errors.New("event_recorder webhook output requires a url") + } + if c.URL.Scheme == "" || c.URL.Host == "" { + return errors.New("event_recorder webhook output requires an absolute http(s) url") + } + return nil +} + +// equal reports whether two webhook output configs are semantically +// equal. +func (c WebhookOutputConfig) equal(o WebhookOutputConfig) bool { + aURL, bURL := "", "" + if c.URL != nil { + aURL = c.URL.String() + } + if o.URL != nil { + bURL = o.URL.String() + } + if aURL != bURL { + return false + } + if c.Timeout != o.Timeout { + return false + } + if c.Workers != o.Workers { + return false + } + if c.MaxRetries != o.MaxRetries { + return false + } + if c.RetryBackoff != o.RetryBackoff { + return false + } + return reflect.DeepEqual(c.HTTPConfig, o.HTTPConfig) +} + +const ( + defaultWebhookTimeout = 10 * time.Second + defaultWebhookWorkers = 4 + defaultWebhookMaxRetries = 3 + defaultWebhookRetryBackoff = 500 * time.Millisecond + defaultWebhookMaxBackoff = 30 * time.Second + webhookQueueSize = 1024 +) + +// WebhookOutput POSTs each event as a JSON body to a configured URL. +// Events are processed by a bounded worker pool so that a slow or +// temporarily unavailable webhook does not block the event recorder queue. +// Events are dropped (with a log message) when the internal queue is +// full. +type WebhookOutput struct { + client *http.Client + url string + name string + maxRetries int + retryBackoff time.Duration + maxBackoff time.Duration + logger *slog.Logger + drops prometheus.Counter + work chan []byte + done chan struct{} + cancel chan struct{} // closed after drain to abort remaining retries + wg sync.WaitGroup +} + +// NewWebhookOutput creates a new webhook-based event recorder output. +func NewWebhookOutput(cfg WebhookOutputConfig, dropsCounter *prometheus.CounterVec, logger *slog.Logger) (*WebhookOutput, error) { + httpCfg := commoncfg.DefaultHTTPClientConfig + if cfg.HTTPConfig != nil { + httpCfg = *cfg.HTTPConfig + } + + client, err := commoncfg.NewClientFromConfig(httpCfg, "eventrecorder") + if err != nil { + return nil, fmt.Errorf("creating HTTP client for event recorder webhook: %w", err) + } + + timeout := defaultWebhookTimeout + if cfg.Timeout > 0 { + timeout = time.Duration(cfg.Timeout) + } + client.Timeout = timeout + + workers := defaultWebhookWorkers + if cfg.Workers > 0 { + workers = cfg.Workers + } + + maxRetries := defaultWebhookMaxRetries + if cfg.MaxRetries > 0 { + maxRetries = cfg.MaxRetries + } + + retryBackoff := defaultWebhookRetryBackoff + if cfg.RetryBackoff > 0 { + retryBackoff = time.Duration(cfg.RetryBackoff) + } + + urlStr := cfg.URL.String() + wo := &WebhookOutput{ + client: client, + url: urlStr, + name: fmt.Sprintf("webhook:%s", sanitizeURL(urlStr)), + maxRetries: maxRetries, + retryBackoff: retryBackoff, + maxBackoff: defaultWebhookMaxBackoff, + logger: logger, + drops: dropsCounter.WithLabelValues(fmt.Sprintf("webhook:%s", sanitizeURL(urlStr))), + work: make(chan []byte, webhookQueueSize), + done: make(chan struct{}), + cancel: make(chan struct{}), + } + + for range workers { + wo.wg.Add(1) + go wo.worker() + } + + return wo, nil +} + +// sanitizeURL strips userinfo and query parameters from a URL string, +// returning only scheme://host/path. This prevents credentials from +// leaking into metrics labels and log messages. +func sanitizeURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + u.User = nil + u.RawQuery = "" + u.Fragment = "" + return u.String() +} + +// Name returns a stable identifier for this output. The URL is +// sanitized to avoid leaking credentials. +func (wo *WebhookOutput) Name() string { + return wo.name +} + +// SendEvent serializes the event as JSON and queues it for delivery by +// a worker. It returns the serialized size (for the bytes-written +// metric). If the internal queue is full the event is dropped and +// counted via the output-drops metric. +func (wo *WebhookOutput) SendEvent(event *eventrecorderpb.Event) (int, error) { + data, err := protojson.Marshal(event) + if err != nil { + return 0, &serializeError{err: err} + } + select { + case wo.work <- data: + default: + wo.drops.Inc() + wo.logger.Warn("Event recorder webhook queue full, dropping event", "output", wo.name) + } + return len(data), nil +} + +func (wo *WebhookOutput) worker() { + defer wo.wg.Done() + for { + select { + case data := <-wo.work: + wo.postWithRetry(data) + case <-wo.done: + // Drain remaining items. + for { + select { + case data := <-wo.work: + wo.postWithRetry(data) + default: + return + } + } + } + } +} + +func (wo *WebhookOutput) postWithRetry(data []byte) { + for attempt := range wo.maxRetries { + err := wo.post(data) + if err == nil { + return + } + wo.logger.Warn("Event recorder webhook POST failed", "output", wo.name, "attempt", attempt+1, "err", err) + if attempt < wo.maxRetries-1 { + backoff := min(wo.retryBackoff</") compose around this +// helper. +func BrokerList(brokers []string) string { + sorted := append([]string(nil), brokers...) + sort.Strings(sorted) + return strings.Join(sorted, ",") +} + +// BrokerListsEqual reports whether two broker lists are +// content-equal, ignoring order. Useful for hot-reload diffing where +// reordering a YAML broker list is semantically a no-op. +func BrokerListsEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + aa := append([]string(nil), a...) + bb := append([]string(nil), b...) + sort.Strings(aa) + sort.Strings(bb) + return slices.Equal(aa, bb) +} diff --git a/vendor/github.com/prometheus/alertmanager/kafka/client.go b/vendor/github.com/prometheus/alertmanager/kafka/client.go new file mode 100644 index 00000000000..2a8fc5dca4a --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/kafka/client.go @@ -0,0 +1,156 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "fmt" + "log/slog" + + commoncfg "github.com/prometheus/common/config" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/plugin/kslog" +) + +// BuildOpts converts the high-level ClientOptions into a slice of +// franz-go options. Callers append role-specific options (e.g. +// kgo.ConsumeTopics for consumers, or extra producer tuning) before +// passing the slice to kgo.NewClient. +// +// The returned option set: +// +// - Seeds the supplied brokers and sets ClientID (defaulting to +// DefaultClientID). +// - Sets DefaultProduceTopic when Topic is non-empty. +// - Applies ProducerLinger(DefaultLinger) — harmless for consumers. +// - Applies RequiredAcks based on opts.Acks (defaults to LeaderAck). +// - Disables idempotent writes unless the caller explicitly opted +// into AcksAll; franz-go's idempotent producer mandates acks=all +// and our default is acks=leader for low latency. +// - Applies ProducerBatchCompression when opts.Compression is set. +// - Applies DialTLSConfig when opts.TLSConfig is non-nil. +// - Wires a kslog adapter so franz-go logs through the caller's +// slog.Logger. A nil logger is replaced with a discard logger. +// +// Validate is called internally; callers do not need to call it +// separately. +func BuildOpts(opts ClientOptions, logger *slog.Logger) ([]kgo.Opt, error) { + if err := opts.Validate(); err != nil { + return nil, err + } + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + + clientID := opts.ClientID + if clientID == "" { + clientID = DefaultClientID + } + + kopts := []kgo.Opt{ + kgo.SeedBrokers(opts.Brokers...), + kgo.ClientID(clientID), + kgo.ProducerLinger(DefaultLinger), + kgo.WithLogger(kslog.New(logger)), + } + if opts.Topic != "" { + kopts = append(kopts, kgo.DefaultProduceTopic(opts.Topic)) + } + + acks, err := acksOpt(opts.Acks) + if err != nil { + return nil, err + } + kopts = append(kopts, acks) + + // franz-go's idempotent producer requires acks=all; disable it + // when the caller chose a weaker (or unspecified, leader) ack + // level to keep the default low-latency path working. + if opts.Acks != AcksAll { + kopts = append(kopts, kgo.DisableIdempotentWrite()) + } + + if opts.Compression != "" { + codec, err := compressionCodec(opts.Compression) + if err != nil { + return nil, err + } + kopts = append(kopts, kgo.ProducerBatchCompression(codec)) + } + + if opts.TLSConfig != nil { + tlsCfg, err := commoncfg.NewTLSConfig(opts.TLSConfig) + if err != nil { + return nil, fmt.Errorf("kafka: building TLS config: %w", err) + } + kopts = append(kopts, kgo.DialTLSConfig(tlsCfg)) + } + + return kopts, nil +} + +// PingInBackground performs a best-effort connectivity check against +// the supplied client without blocking the caller. Failure is logged +// at warn level; franz-go retries connections internally so subsequent +// produce/fetch calls will succeed once a broker becomes reachable. +// +// The goroutine exits after DefaultPingTimeout or when the client is +// closed (which cancels any in-flight broker dial). +func PingInBackground(client *kgo.Client, logger *slog.Logger) { + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), DefaultPingTimeout) + defer cancel() + if err := client.Ping(ctx); err != nil { + logger.Warn("Kafka client could not reach brokers at startup; will retry in background", + "err", err) + } + }() +} + +// acksOpt translates the user-facing acks value into a franz-go +// option. An empty value defaults to LeaderAck. +func acksOpt(s Acks) (kgo.Opt, error) { + switch s { + case "", AcksLeader: + return kgo.RequiredAcks(kgo.LeaderAck()), nil + case AcksNone: + return kgo.RequiredAcks(kgo.NoAck()), nil + case AcksAll: + return kgo.RequiredAcks(kgo.AllISRAcks()), nil + default: + return nil, fmt.Errorf("kafka: unknown acks %q", s) + } +} + +// compressionCodec translates the user-facing compression value into +// a franz-go codec. +func compressionCodec(s Compression) (kgo.CompressionCodec, error) { + switch s { + case CompressionNone: + return kgo.NoCompression(), nil + case CompressionGzip: + return kgo.GzipCompression(), nil + case CompressionSnappy: + return kgo.SnappyCompression(), nil + case CompressionLZ4: + return kgo.Lz4Compression(), nil + case CompressionZstd: + return kgo.ZstdCompression(), nil + default: + return kgo.NoCompression(), fmt.Errorf("kafka: unknown compression %q", s) + } +} diff --git a/vendor/github.com/prometheus/alertmanager/kafka/errors.go b/vendor/github.com/prometheus/alertmanager/kafka/errors.go new file mode 100644 index 00000000000..871013ea55b --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/kafka/errors.go @@ -0,0 +1,62 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "errors" + "net" + "strings" +) + +// ErrorCategory is a coarse bucket for a Kafka error, suitable for use +// as a bounded-cardinality Prometheus label value. +type ErrorCategory string + +// Error categories returned by ClassifyError. +const ( + ErrorCategoryNone ErrorCategory = "none" + ErrorCategoryTimeout ErrorCategory = "timeout" + ErrorCategoryNetwork ErrorCategory = "network" + ErrorCategoryBroker ErrorCategory = "broker" + ErrorCategoryUnknown ErrorCategory = "unknown" +) + +// ClassifyError buckets a franz-go error into one of the +// ErrorCategory* constants. It keeps Prometheus metric label +// cardinality bounded regardless of the specific error string franz-go +// surfaces. +func ClassifyError(err error) ErrorCategory { + if err == nil { + return ErrorCategoryNone + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return ErrorCategoryTimeout + } + var netErr net.Error + if errors.As(err, &netErr) { + if netErr.Timeout() { + return ErrorCategoryTimeout + } + return ErrorCategoryNetwork + } + // franz-go surfaces broker-side errors typed as kerr.Error; we + // detect them via the error string so this package doesn't have + // to depend on kerr just for the type assertion. + msg := err.Error() + if strings.Contains(msg, "broker") || strings.Contains(msg, "kafka:") { + return ErrorCategoryBroker + } + return ErrorCategoryUnknown +} diff --git a/vendor/github.com/prometheus/alertmanager/kafka/kafka.go b/vendor/github.com/prometheus/alertmanager/kafka/kafka.go new file mode 100644 index 00000000000..e49d36e5ed1 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/kafka/kafka.go @@ -0,0 +1,147 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kafka provides the transport-layer building blocks for +// Alertmanager components that talk to Apache Kafka via franz-go. +// +// It deliberately stays narrow: configuration shared by producers and +// consumers, franz-go option construction, an error classifier, and a +// broker-list naming helper. Higher-level concerns (bounded +// producer buffers, consumer group orchestration, message +// serialisation) live in the calling package. +// +// At the moment the only consumer of this package is the event +// recorder's Kafka output (eventrecorder/kafka.go). A future Kafka +// receiver (see github.com/prometheus/alertmanager/issues/1996) is the +// other intended user. +package kafka + +import ( + "errors" + "fmt" + "slices" + "time" + + commoncfg "github.com/prometheus/common/config" +) + +// Format is the wire encoding of a Kafka message value. +type Format string + +// Wire format identifiers used by producers and consumers. +const ( + FormatJSON Format = "json" + FormatProtobuf Format = "protobuf" +) + +// Acks is the producer acknowledgement level. +type Acks string + +// Producer acknowledgement levels. +const ( + AcksNone Acks = "none" + AcksLeader Acks = "leader" + AcksAll Acks = "all" +) + +// Compression is the producer batch compression codec. +type Compression string + +// Compression codecs supported on the producer wire. +const ( + CompressionNone Compression = "none" + CompressionGzip Compression = "gzip" + CompressionSnappy Compression = "snappy" + CompressionLZ4 Compression = "lz4" + CompressionZstd Compression = "zstd" +) + +// Default values applied by BuildOpts when ClientOptions leaves a +// field zero. +const ( + DefaultClientID = "alertmanager" + DefaultPingTimeout = 5 * time.Second + DefaultLinger = 5 * time.Millisecond + DefaultFlushBudget = 30 * time.Second +) + +// ClientOptions is the configuration shared between Kafka producers +// and consumers used in Alertmanager. It is the input to BuildOpts. +// +// All fields are optional except Brokers; consumers ignore producer-only +// knobs (Acks, Compression) and vice versa. Topic doubles as the +// franz-go DefaultProduceTopic for producers and may be ignored or +// repurposed (e.g. as a subscription target) by consumers. +type ClientOptions struct { + // Brokers is the list of Kafka seed brokers in host:port form. + // At least one entry is required. + Brokers []string + + // Topic is the default produce topic. Optional for consumers. + Topic string + + // ClientID is reported to the brokers. Defaults to "alertmanager". + ClientID string + + // Acks is the producer acknowledgement level: "", AcksNone, + // AcksLeader (default), or AcksAll. Producer-only. + Acks Acks + + // Compression is the producer compression codec: "" (default, + // no compression), CompressionNone, CompressionGzip, + // CompressionSnappy, CompressionLZ4, or CompressionZstd. + // Producer-only. + Compression Compression + + // TLSConfig configures TLS for the broker connection. If nil, + // PLAINTEXT is used. + TLSConfig *commoncfg.TLSConfig +} + +// Validate checks ClientOptions for obvious problems. It does not +// contact the brokers and does not mutate the receiver. +func (o ClientOptions) Validate() error { + if len(o.Brokers) == 0 { + return errors.New("kafka: at least one broker is required") + } + if slices.Contains(o.Brokers, "") { + return errors.New("kafka: broker entries must be non-empty") + } + switch o.Acks { + case "", AcksNone, AcksLeader, AcksAll: + default: + return fmt.Errorf("kafka: unknown acks %q, must be %q, %q, or %q", + o.Acks, AcksNone, AcksLeader, AcksAll) + } + switch o.Compression { + case "", CompressionNone, CompressionGzip, + CompressionSnappy, CompressionLZ4, CompressionZstd: + default: + return fmt.Errorf("kafka: unknown compression %q", o.Compression) + } + return nil +} + +// ValidateFormat checks that a wire-format value is recognised. It is +// provided as a free helper because some callers (e.g. the event +// recorder) carry the format on a per-output struct rather than on the +// shared ClientOptions. +func ValidateFormat(format Format) error { + switch format { + case FormatJSON, FormatProtobuf: + return nil + default: + return fmt.Errorf("kafka: unknown format %q, must be %q or %q", + format, FormatJSON, FormatProtobuf) + } +} diff --git a/vendor/github.com/prometheus/alertmanager/marker/alert.go b/vendor/github.com/prometheus/alertmanager/marker/alert.go new file mode 100644 index 00000000000..57cbcf2837b --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/marker/alert.go @@ -0,0 +1,98 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marker + +import ( + "slices" + "sync" + + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/alert" +) + +// NewAlertMarker returns a new AlertMarker backed by an in-memory map. +func NewAlertMarker() AlertMarker { + return &alertMarker{ + status: map[model.Fingerprint]*alertStatus{}, + } +} + +// alertMarker is an in-memory implementation of AlertMarker. +type alertMarker struct { + status map[model.Fingerprint]*alertStatus + mtx sync.RWMutex +} + +// SetSilenced implements AlertMarker. +func (m *alertMarker) SetSilenced(fp model.Fingerprint, silencedBy []string) { + m.mtx.Lock() + defer m.mtx.Unlock() + + s, found := m.status[fp] + if !found { + s = &alertStatus{} + m.status[fp] = s + } + s.SilencedBy = slices.Clone(silencedBy) +} + +// SetInhibited implements AlertMarker. +func (m *alertMarker) SetInhibited(fp model.Fingerprint, inhibitedBy []string) { + m.mtx.Lock() + defer m.mtx.Unlock() + + s, found := m.status[fp] + if !found { + s = &alertStatus{} + m.status[fp] = s + } + s.InhibitedBy = slices.Clone(inhibitedBy) +} + +// Status implements AlertMarker. +func (m *alertMarker) Status(fp model.Fingerprint) alert.AlertStatus { + m.mtx.RLock() + defer m.mtx.RUnlock() + + status := alert.AlertStatus{ + State: alert.AlertStateUnprocessed, + SilencedBy: []string{}, + InhibitedBy: []string{}, + } + + s, found := m.status[fp] + if !found { + return status + } + + status.State = s.state() + if s.SilencedBy != nil { + status.SilencedBy = slices.Clone(s.SilencedBy) + } + if s.InhibitedBy != nil { + status.InhibitedBy = slices.Clone(s.InhibitedBy) + } + return status +} + +// Delete implements AlertMarker. +func (m *alertMarker) Delete(alerts ...model.Fingerprint) { + m.mtx.Lock() + defer m.mtx.Unlock() + + for _, alert := range alerts { + delete(m.status, alert) + } +} diff --git a/vendor/github.com/prometheus/alertmanager/marker/context.go b/vendor/github.com/prometheus/alertmanager/marker/context.go new file mode 100644 index 00000000000..f099bf8d254 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/marker/context.go @@ -0,0 +1,40 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marker + +import ( + "context" +) + +// markerKey defines a custom type with which a context is populated to +// avoid accidental collisions. +type markerKey int + +const ( + keyAlertMarker markerKey = iota +) + +// WithContext returns a copy of ctx carrying the given +// AlertMarker. Inhibitor and Silencer extract it from the context to +// write per-group alert status. +func WithContext(ctx context.Context, m AlertMarker) context.Context { + return context.WithValue(ctx, keyAlertMarker, m) +} + +// FromContext extracts the AlertMarker from ctx. +// If no marker is present, it returns (nil, false). +func FromContext(ctx context.Context) (AlertMarker, bool) { + m, ok := ctx.Value(keyAlertMarker).(AlertMarker) + return m, ok +} diff --git a/vendor/github.com/prometheus/alertmanager/marker/group.go b/vendor/github.com/prometheus/alertmanager/marker/group.go new file mode 100644 index 00000000000..b132e4794ee --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/marker/group.go @@ -0,0 +1,79 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marker + +import "sync" + +// groupStatus stores the state of the group, and, as applicable, the names +// of all active and mute time intervals that are muting it. +type groupStatus struct { + // mutedBy contains the names of all active and mute time intervals that + // are muting it. + mutedBy []string +} + +// NewGroupMarker returns an instance of a GroupMarker implementation. +func NewGroupMarker() GroupMarker { + return &groupMarker{ + groups: map[groupMarkerKey]*groupStatus{}, + } +} + +type groupMarkerKey struct { + routeID string + groupKey string +} + +func newGroupMarkerKey(routeID, groupKey string) groupMarkerKey { + return groupMarkerKey{ + routeID: routeID, + groupKey: groupKey, + } +} + +type groupMarker struct { + groups map[groupMarkerKey]*groupStatus + + mtx sync.RWMutex +} + +// Muted implements GroupMarker. +func (m *groupMarker) Muted(routeID, groupKey string) ([]string, bool) { + m.mtx.RLock() + defer m.mtx.RUnlock() + status, ok := m.groups[newGroupMarkerKey(routeID, groupKey)] + if !ok { + return nil, false + } + return status.mutedBy, len(status.mutedBy) > 0 +} + +// SetMuted implements GroupMarker. +func (m *groupMarker) SetMuted(routeID, groupKey string, timeIntervalNames []string) { + key := newGroupMarkerKey(routeID, groupKey) + m.mtx.Lock() + defer m.mtx.Unlock() + status, ok := m.groups[key] + if !ok { + status = &groupStatus{} + m.groups[key] = status + } + status.mutedBy = timeIntervalNames +} + +func (m *groupMarker) DeleteByGroupKey(routeID, groupKey string) { + m.mtx.Lock() + defer m.mtx.Unlock() + delete(m.groups, newGroupMarkerKey(routeID, groupKey)) +} diff --git a/vendor/github.com/prometheus/alertmanager/marker/marker.go b/vendor/github.com/prometheus/alertmanager/marker/marker.go new file mode 100644 index 00000000000..90024cbec3b --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/marker/marker.go @@ -0,0 +1,61 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marker + +import ( + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/alert" +) + +// AlertMarker tracks per-alert silenced/inhibited status within a single +// aggregation group. Each aggregation group owns its own instance. +// All methods are goroutine-safe. +type AlertMarker interface { + // SetInhibited sets the inhibitedBy for the given fingerprint. + // If inhibitedBy is empty, it clears the inhibitedBy. + SetInhibited(fingerprint model.Fingerprint, inhibitedBy []string) + + // SetSilenced sets the silencedBy for the given fingerprint. + // If silencedBy is empty, it clears the silencedBy. + SetSilenced(fingerprint model.Fingerprint, silencedBy []string) + + // Status returns the AlertStatus for the given fingerprint. + // If the fingerprint is not found, it returns an unknown status. + Status(fingerprint model.Fingerprint) alert.AlertStatus + + // Delete removes markers for the given fingerprints. + Delete(fingerprints ...model.Fingerprint) +} + +// GroupMarker helps to mark groups as active or muted. +// All methods are goroutine-safe. +// +// TODO(grobinson): routeID is used in Muted and SetMuted because groupKey +// is not unique (see #3817). Once groupKey uniqueness is fixed routeID can +// be removed from the GroupMarker interface. +type GroupMarker interface { + // Muted returns true if the group is muted, otherwise false. If the group + // is muted then it also returns the names of the time intervals that muted + // it. + Muted(routeID, groupKey string) ([]string, bool) + + // SetMuted marks the group as muted, and sets the names of the time + // intervals that mute it. If the list of names is nil or the empty slice + // then the muted marker is removed. + SetMuted(routeID, groupKey string, timeIntervalNames []string) + + // DeleteByGroupKey removes all markers for the GroupKey. + DeleteByGroupKey(routeID, groupKey string) +} diff --git a/vendor/github.com/prometheus/alertmanager/marker/status.go b/vendor/github.com/prometheus/alertmanager/marker/status.go new file mode 100644 index 00000000000..00835808d17 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/marker/status.go @@ -0,0 +1,29 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marker + +import "github.com/prometheus/alertmanager/alert" + +type alertStatus struct { + InhibitedBy []string + SilencedBy []string +} + +// state calculates the alert state based on inhibition and silence status. +func (s *alertStatus) state() alert.AlertState { + if len(s.InhibitedBy) > 0 || len(s.SilencedBy) > 0 { + return alert.AlertStateSuppressed + } + return alert.AlertStateActive +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/cluster_stages.go b/vendor/github.com/prometheus/alertmanager/notify/cluster_stages.go new file mode 100644 index 00000000000..98b7179116a --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/cluster_stages.go @@ -0,0 +1,63 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "context" + "log/slog" + "time" + + "github.com/prometheus/alertmanager/alert" +) + +// ClusterGossipSettleStage waits until the Gossip has settled to forward alerts. +type ClusterGossipSettleStage struct { + peer Peer +} + +// NewClusterGossipSettleStage returns a new ClusterGossipSettleStage. +func NewClusterGossipSettleStage(p Peer) *ClusterGossipSettleStage { + return &ClusterGossipSettleStage{peer: p} +} + +func (n *ClusterGossipSettleStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + if n.peer != nil { + if err := n.peer.WaitReady(ctx); err != nil { + return ctx, nil, err + } + } + return ctx, alerts, nil +} + +// ClusterWaitStage waits for a certain amount of time before continuing or until the +// context is done. +type ClusterWaitStage struct { + wait func() time.Duration +} + +// NewClusterWaitStage returns a new ClusterWaitStage. +func NewClusterWaitStage(wait func() time.Duration) *ClusterWaitStage { + return &ClusterWaitStage{ + wait: wait, + } +} + +// Exec implements the Stage interface. +func (ws *ClusterWaitStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + select { + case <-time.After(ws.wait()): + case <-ctx.Done(): + return ctx, nil, ctx.Err() + } + return ctx, alerts, nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/context.go b/vendor/github.com/prometheus/alertmanager/notify/context.go new file mode 100644 index 00000000000..6b7c3eeabc8 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/context.go @@ -0,0 +1,233 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "context" + "time" + + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/nflog" + "github.com/prometheus/alertmanager/pkg/labels" +) + +// notifyKey defines a custom type with which a context is populated to +// avoid accidental collisions. +type notifyKey int + +const ( + keyReceiverName notifyKey = iota + keyRepeatInterval + keyGroupLabels + keyGroupKey + keyFiringAlerts + keyResolvedAlerts + keyNow + keyMuteTimeIntervals + keyActiveTimeIntervals + keyRouteID + keyNflogStore + keyNotificationReason + keyMutedAlerts + keyAggrGroupID + keyFlushID + keyGroupMatchers +) + +// WithReceiverName populates a context with a receiver name. +func WithReceiverName(ctx context.Context, rcv string) context.Context { + return context.WithValue(ctx, keyReceiverName, rcv) +} + +// WithGroupKey populates a context with a group key. +func WithGroupKey(ctx context.Context, s string) context.Context { + return context.WithValue(ctx, keyGroupKey, s) +} + +// WithFiringAlerts populates a context with a slice of firing alerts. +func WithFiringAlerts(ctx context.Context, alerts []uint64) context.Context { + return context.WithValue(ctx, keyFiringAlerts, alerts) +} + +// WithResolvedAlerts populates a context with a slice of resolved alerts. +func WithResolvedAlerts(ctx context.Context, alerts []uint64) context.Context { + return context.WithValue(ctx, keyResolvedAlerts, alerts) +} + +// WithGroupLabels populates a context with grouping labels. +func WithGroupLabels(ctx context.Context, lset model.LabelSet) context.Context { + return context.WithValue(ctx, keyGroupLabels, lset) +} + +// WithNow populates a context with a now timestamp. +func WithNow(ctx context.Context, t time.Time) context.Context { + return context.WithValue(ctx, keyNow, t) +} + +// WithRepeatInterval populates a context with a repeat interval. +func WithRepeatInterval(ctx context.Context, t time.Duration) context.Context { + return context.WithValue(ctx, keyRepeatInterval, t) +} + +// WithMuteTimeIntervals populates a context with a slice of mute time names. +func WithMuteTimeIntervals(ctx context.Context, mt []string) context.Context { + return context.WithValue(ctx, keyMuteTimeIntervals, mt) +} + +// WithActiveTimeIntervals populates a context with a slice of active time names. +func WithActiveTimeIntervals(ctx context.Context, at []string) context.Context { + return context.WithValue(ctx, keyActiveTimeIntervals, at) +} + +// WithRouteID populates a context with the route ID. +func WithRouteID(ctx context.Context, routeID string) context.Context { + return context.WithValue(ctx, keyRouteID, routeID) +} + +// WithNotificationReason populates a context with a NotifyReason. +func WithNotificationReason(ctx context.Context, reason NotifyReason) context.Context { + return context.WithValue(ctx, keyNotificationReason, reason) +} + +// RepeatInterval extracts a repeat interval from the context. Iff none exists, the +// second argument is false. +func RepeatInterval(ctx context.Context) (time.Duration, bool) { + v, ok := ctx.Value(keyRepeatInterval).(time.Duration) + return v, ok +} + +// ReceiverName extracts a receiver name from the context. Iff none exists, the +// second argument is false. +func ReceiverName(ctx context.Context) (string, bool) { + v, ok := ctx.Value(keyReceiverName).(string) + return v, ok +} + +// GroupKey extracts a group key from the context. Iff none exists, the +// second argument is false. +func GroupKey(ctx context.Context) (string, bool) { + v, ok := ctx.Value(keyGroupKey).(string) + return v, ok +} + +// GroupLabels extracts grouping label set from the context. Iff none exists, the +// second argument is false. +func GroupLabels(ctx context.Context) (model.LabelSet, bool) { + v, ok := ctx.Value(keyGroupLabels).(model.LabelSet) + return v, ok +} + +// Now extracts a now timestamp from the context. Iff none exists, the +// second argument is false. +func Now(ctx context.Context) (time.Time, bool) { + v, ok := ctx.Value(keyNow).(time.Time) + return v, ok +} + +// FiringAlerts extracts a slice of firing alerts from the context. +// Iff none exists, the second argument is false. +func FiringAlerts(ctx context.Context) ([]uint64, bool) { + v, ok := ctx.Value(keyFiringAlerts).([]uint64) + return v, ok +} + +// ResolvedAlerts extracts a slice of resolved alerts from the context. +// Iff none exists, the second argument is false. +func ResolvedAlerts(ctx context.Context) ([]uint64, bool) { + v, ok := ctx.Value(keyResolvedAlerts).([]uint64) + return v, ok +} + +// MuteTimeIntervalNames extracts a slice of mute time names from the context. If and only if none exists, the +// second argument is false. +func MuteTimeIntervalNames(ctx context.Context) ([]string, bool) { + v, ok := ctx.Value(keyMuteTimeIntervals).([]string) + return v, ok +} + +// ActiveTimeIntervalNames extracts a slice of active time names from the context. If none exists, the +// second argument is false. +func ActiveTimeIntervalNames(ctx context.Context) ([]string, bool) { + v, ok := ctx.Value(keyActiveTimeIntervals).([]string) + return v, ok +} + +// RouteID extracts a RouteID from the context. Iff none exists, the +// // second argument is false. +func RouteID(ctx context.Context) (string, bool) { + v, ok := ctx.Value(keyRouteID).(string) + return v, ok +} + +// NotificationReason extracts a NotifyReason from the context. +func NotificationReason(ctx context.Context) (NotifyReason, bool) { + v, ok := ctx.Value(keyNotificationReason).(NotifyReason) + return v, ok +} + +// WithMutedAlerts populates a context with a set of muted alert hashes. +func WithMutedAlerts(ctx context.Context, alerts map[uint64]struct{}) context.Context { + return context.WithValue(ctx, keyMutedAlerts, alerts) +} + +// MutedAlerts extracts a set of muted alert hashes from the context. +func MutedAlerts(ctx context.Context) (map[uint64]struct{}, bool) { + v, ok := ctx.Value(keyMutedAlerts).(map[uint64]struct{}) + return v, ok +} + +// WithAggrGroupID populates a context with an aggregation group UUID. +func WithAggrGroupID(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, keyAggrGroupID, id) +} + +// AggrGroupID extracts an aggregation group UUID from the context. +func AggrGroupID(ctx context.Context) (string, bool) { + v, ok := ctx.Value(keyAggrGroupID).(string) + return v, ok +} + +// WithFlushID populates a context with a flush identifier. +func WithFlushID(ctx context.Context, id uint64) context.Context { + return context.WithValue(ctx, keyFlushID, id) +} + +// FlushID extracts a flush identifier from the context. +func FlushID(ctx context.Context) (uint64, bool) { + v, ok := ctx.Value(keyFlushID).(uint64) + return v, ok +} + +// WithGroupMatchers populates a context with the route's matchers. +func WithGroupMatchers(ctx context.Context, matchers labels.Matchers) context.Context { + return context.WithValue(ctx, keyGroupMatchers, matchers) +} + +// GroupMatchers extracts the route's matchers from the context. +func GroupMatchers(ctx context.Context) (labels.Matchers, bool) { + v, ok := ctx.Value(keyGroupMatchers).(labels.Matchers) + return v, ok +} + +// WithNflogStore populates a context with a reference to an nflog.Store. +func WithNflogStore(ctx context.Context, store *nflog.Store) context.Context { + return context.WithValue(ctx, keyNflogStore, store) +} + +// NflogStore extracts the nflog.Store from the context. The returned +// NflogStore is a pointer to a mutable store which remains in the context. +func NflogStore(ctx context.Context) (*nflog.Store, bool) { + v, ok := ctx.Value(keyNflogStore).(*nflog.Store) + return v, ok +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/dedup_stage.go b/vendor/github.com/prometheus/alertmanager/notify/dedup_stage.go new file mode 100644 index 00000000000..a066d682f56 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/dedup_stage.go @@ -0,0 +1,174 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/nflog" + "github.com/prometheus/alertmanager/nflog/nflogpb" +) + +// DedupStage filters alerts. +// Filtering happens based on a notification log. +type DedupStage struct { + rs ResolvedSender + nflog NotificationLog + recv *nflogpb.Receiver + + now func() time.Time + hash func(*alert.Alert) uint64 +} + +// NewDedupStage wraps a DedupStage that runs against the given notification log. +func NewDedupStage(rs ResolvedSender, l NotificationLog, recv *nflogpb.Receiver) *DedupStage { + return &DedupStage{ + rs: rs, + nflog: l, + recv: recv, + now: utcNow, + hash: hashAlert, + } +} + +func (n *DedupStage) needsUpdate(entry *nflogpb.Entry, firing, resolved map[uint64]struct{}, repeat time.Duration, now time.Time) NotifyReason { + // If we haven't notified about the alert group before, notify right away + // unless we only have resolved alerts. + if entry == nil { + if len(firing) > 0 { + return ReasonFirstNotification + } + return ReasonDoNotNotify + } + + // new alerts in the group + if !entry.IsFiringSubset(firing) { + // If the previous entry has no firing alerts, it was a resolution and we + // should treat this as the first notification for the group. + if len(entry.FiringAlerts) == 0 { + return ReasonFirstNotification + } + return ReasonNewAlertsInGroup + } + + // Notify about all alerts being resolved. + // This is done irrespective of the send_resolved flag to make sure that + // the firing alerts are cleared from the notification log. + if len(firing) == 0 { + // If the current alert group and last notification contain no firing + // alert, it means that some alerts have been fired and resolved during the + // last interval. In this case, there is no need to notify the receiver + // since it doesn't know about them. + if len(entry.FiringAlerts) > 0 { + return ReasonAllAlertsResolved + } + return ReasonDoNotNotify + } + + if n.rs.SendResolved() && !entry.IsResolvedSubset(resolved) { + return ReasonNewResolvedAlerts + } + + // Nothing changed, only notify if the repeat interval has passed. + isRepeatIntervalElapsed := entry.Timestamp.AsTime().Before(now.Add(-repeat)) + if isRepeatIntervalElapsed { + return ReasonRepeatIntervalElapsed + } + return ReasonDoNotNotify +} + +// partitionAlertsByState separates alerts into firing and resolved, returning both slices and sets. +func partitionAlertsByState(alerts []*alert.Alert, hashFn func(*alert.Alert) uint64) (firing, resolved []uint64, firingSet, resolvedSet map[uint64]struct{}) { + firingSet = make(map[uint64]struct{}, len(alerts)) + resolvedSet = make(map[uint64]struct{}, len(alerts)) + firing = make([]uint64, 0, len(alerts)) + resolved = make([]uint64, 0, len(alerts)) + + for _, a := range alerts { + hash := hashFn(a) + if a.Resolved() { + resolved = append(resolved, hash) + resolvedSet[hash] = struct{}{} + } else { + firing = append(firing, hash) + firingSet[hash] = struct{}{} + } + } + return firing, resolved, firingSet, resolvedSet +} + +// Exec implements the Stage interface. +func (n *DedupStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + gkey, ok := GroupKey(ctx) + if !ok { + return ctx, nil, errors.New("group key missing") + } + + ctx, span := tracer.Start(ctx, "notify.DedupStage.Exec", + trace.WithAttributes(attribute.String("alerting.group.key", gkey)), + trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), + trace.WithSpanKind(trace.SpanKindInternal), + ) + defer span.End() + + repeatInterval, ok := RepeatInterval(ctx) + if !ok { + return ctx, nil, errors.New("repeat interval missing") + } + + firing, resolved, firingSet, resolvedSet := partitionAlertsByState(alerts, n.hash) + + ctx = WithFiringAlerts(ctx, firing) + ctx = WithResolvedAlerts(ctx, resolved) + + entries, err := n.nflog.Query(nflog.QGroupKey(gkey), nflog.QReceiver(n.recv)) + if err != nil && !errors.Is(err, nflog.ErrNotFound) { + return ctx, nil, err + } + + var entry *nflogpb.Entry + switch len(entries) { + case 0: + case 1: + entry = entries[0] + default: + return ctx, nil, fmt.Errorf("unexpected entry result size %d", len(entries)) + } + + now := n.now() + if ctxNow, ok := Now(ctx); ok { + now = ctxNow + } + updateReason := n.needsUpdate(entry, firingSet, resolvedSet, repeatInterval, now) + ctx = WithNotificationReason(ctx, updateReason) + + if updateReason == ReasonFirstNotification { + ctx = WithNflogStore(ctx, nflog.NewStore(nil)) + } else { + ctx = WithNflogStore(ctx, nflog.NewStore(entry)) + } + + if updateReason.shouldNotify() { + span.AddEvent("notify.DedupStage.Exec nflog needs update") + return ctx, alerts, nil + } + return ctx, nil, nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/discord/config.go b/vendor/github.com/prometheus/alertmanager/notify/discord/config.go new file mode 100644 index 00000000000..6133c831b8f --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/discord/config.go @@ -0,0 +1,65 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discord + +import ( + "errors" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" + + commoncfg "github.com/prometheus/common/config" +) + +// defaultDiscordConfig defines default values for Discord configurations. +var defaultDiscordConfig = DiscordConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, + Title: `{{ template "discord.default.title" . }}`, + Message: `{{ template "discord.default.message" . }}`, +} + +// DiscordConfig configures notifications via Discord. +type DiscordConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` + WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` + + Content string `yaml:"content,omitempty" json:"content,omitempty"` + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Message string `yaml:"message,omitempty" json:"message,omitempty"` + Username string `yaml:"username,omitempty" json:"username,omitempty"` + AvatarURL string `yaml:"avatar_url,omitempty" json:"avatar_url,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *DiscordConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = defaultDiscordConfig + type plain DiscordConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + + if c.WebhookURL == nil && c.WebhookURLFile == "" { + return errors.New("one of webhook_url or webhook_url_file must be configured") + } + + if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { + return errors.New("at most one of webhook_url & webhook_url_file must be configured") + } + + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/discord/discord.go b/vendor/github.com/prometheus/alertmanager/notify/discord/discord.go index 8941996a6ce..485838cf89f 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/discord/discord.go +++ b/vendor/github.com/prometheus/alertmanager/notify/discord/discord.go @@ -29,7 +29,6 @@ import ( amcommoncfg "github.com/prometheus/alertmanager/config/common" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -52,7 +51,7 @@ const ( // Notifier implements a Notifier for Discord notifications. type Notifier struct { - conf *config.DiscordConfig + conf *DiscordConfig tmpl *template.Template logger *slog.Logger client *http.Client @@ -61,7 +60,7 @@ type Notifier struct { } // New returns a new Discord notifier. -func New(c *config.DiscordConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(c *DiscordConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "discord", httpOpts...) if err != nil { return nil, err diff --git a/vendor/github.com/prometheus/alertmanager/notify/event.go b/vendor/github.com/prometheus/alertmanager/notify/event.go new file mode 100644 index 00000000000..740cd0e77e1 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/event.go @@ -0,0 +1,143 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notify + +// This file contains helpers for constructing event recorder protobuf messages +// from the notification pipeline context. It lives in the notify package +// because it accesses unexported context keys (keyFiringAlerts, etc.). + +import ( + "context" + + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/prometheus/alertmanager/eventrecorder" + "github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb" + "github.com/prometheus/alertmanager/types" +) + +func groupedAlertAsProto(alert *types.Alert) *eventrecorderpb.GroupedAlert { + return &eventrecorderpb.GroupedAlert{ + Hash: hashAlert(alert), + Details: eventrecorder.AlertAsProto(alert), + } +} + +func extractAlertGroupInfo(ctx context.Context) *eventrecorderpb.AlertGroupInfo { + groupKey, _ := ExtractGroupKey(ctx) + receiverName, _ := ReceiverName(ctx) + groupLabels, _ := GroupLabels(ctx) + groupMatchers, _ := GroupMatchers(ctx) + aggrGroupID, _ := AggrGroupID(ctx) + + return &eventrecorderpb.AlertGroupInfo{ + GroupKey: groupKey.String(), + GroupLabels: eventrecorder.LabelSetAsProto(groupLabels), + GroupId: groupKey.Hash(), + ReceiverName: receiverName, + Matchers: eventrecorder.MatchersAsProto(groupMatchers), + GroupUuid: aggrGroupID, + } +} + +func extractGroupedAlerts(ctx context.Context, key notifyKey) []*eventrecorderpb.GroupedAlert { + var result []*eventrecorderpb.GroupedAlert + if list, ok := ctx.Value(key).([]uint64); ok { + for _, hash := range list { + result = append(result, &eventrecorderpb.GroupedAlert{Hash: hash}) + } + } + return result +} + +func extractMutedGroupedAlerts(ctx context.Context) []*eventrecorderpb.GroupedAlert { + var result []*eventrecorderpb.GroupedAlert + if muted, ok := MutedAlerts(ctx); ok { + for hash := range muted { + result = append(result, &eventrecorderpb.GroupedAlert{Hash: hash}) + } + } + return result +} + +func notifyReasonToProto(reason NotifyReason) eventrecorderpb.NotifyReason { + switch reason { + case ReasonFirstNotification: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_FIRST_NOTIFICATION + case ReasonNewAlertsInGroup: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_NEW_ALERTS_IN_GROUP + case ReasonAllAlertsResolved: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_ALL_ALERTS_RESOLVED + case ReasonNewResolvedAlerts: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_NEW_RESOLVED_ALERTS + case ReasonRepeatIntervalElapsed: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_REPEAT_INTERVAL_ELAPSED + default: + return eventrecorderpb.NotifyReason_NOTIFY_REASON_UNSPECIFIED + } +} + +// NewNotificationEvent constructs a NotificationEvent from the pipeline +// context after a successful notification delivery. +func NewNotificationEvent(ctx context.Context, alerts []*types.Alert, integration Integration) *eventrecorderpb.EventData { + groupedAlerts := make([]*eventrecorderpb.GroupedAlert, 0, len(alerts)) + for _, alert := range alerts { + groupedAlerts = append(groupedAlerts, groupedAlertAsProto(alert)) + } + + notifyReason, _ := NotificationReason(ctx) + repeatInterval, _ := RepeatInterval(ctx) + flushID, _ := FlushID(ctx) + + notification := &eventrecorderpb.NotificationEvent{ + Alerts: groupedAlerts, + FiringAlerts: extractGroupedAlerts(ctx, keyFiringAlerts), + ResolvedAlerts: extractGroupedAlerts(ctx, keyResolvedAlerts), + MutedAlerts: extractMutedGroupedAlerts(ctx), + GroupInfo: extractAlertGroupInfo(ctx), + RepeatInterval: durationpb.New(repeatInterval), + Reason: notifyReasonToProto(notifyReason), + FlushId: flushID, + Integration: &eventrecorderpb.Integration{ + Name: integration.Name(), + Index: int64(integration.Index()), + }, + } + + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_Notification{Notification: notification}, + } +} + +func NewAlertResolvedEvent(groupInfo *eventrecorderpb.AlertGroupInfo, alert *types.Alert) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_AlertResolved{ + AlertResolved: &eventrecorderpb.AlertResolvedEvent{ + Alert: groupedAlertAsProto(alert), + GroupInfo: groupInfo, + }, + }, + } +} + +func NewAlertGroupedEvent(groupInfo *eventrecorderpb.AlertGroupInfo, alert *types.Alert) *eventrecorderpb.EventData { + return &eventrecorderpb.EventData{ + EventType: &eventrecorderpb.EventData_AlertGrouped{ + AlertGrouped: &eventrecorderpb.AlertGroupedEvent{ + Alert: groupedAlertAsProto(alert), + GroupInfo: groupInfo, + }, + }, + } +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/incidentio/config.go b/vendor/github.com/prometheus/alertmanager/notify/incidentio/config.go new file mode 100644 index 00000000000..86b8e9f9cb0 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/incidentio/config.go @@ -0,0 +1,82 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package incidentio + +import ( + "errors" + "time" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" + + commoncfg "github.com/prometheus/common/config" +) + +// defaultIncidentioConfig defines default values for Incident.io configurations. +var defaultIncidentioConfig = IncidentioConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, +} + +// IncidentioConfig configures notifications via incident.io. +type IncidentioConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + + // URL to send POST request to. + URL *amcommoncfg.URL `yaml:"url" json:"url"` + URLFile string `yaml:"url_file" json:"url_file"` + + // AlertSourceToken is the key used to authenticate with the alert source in incident.io. + AlertSourceToken commoncfg.Secret `yaml:"alert_source_token,omitempty" json:"alert_source_token,omitempty"` + AlertSourceTokenFile string `yaml:"alert_source_token_file,omitempty" json:"alert_source_token_file,omitempty"` + + // MaxAlerts is the maximum number of alerts to be sent per incident.io message. + // Alerts exceeding this threshold will be truncated. Setting this to 0 + // allows an unlimited number of alerts. Note that if the payload exceeds + // incident.io's size limits, you will receive a 429 response and alerts + // will not be ingested. + MaxAlerts uint64 `yaml:"max_alerts" json:"max_alerts"` + + // Timeout is the maximum time allowed to invoke incident.io. Setting this to 0 + // does not impose a timeout. + Timeout time.Duration `yaml:"timeout" json:"timeout"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *IncidentioConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = defaultIncidentioConfig + type plain IncidentioConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.URL == nil && c.URLFile == "" { + return errors.New("one of url or url_file must be configured") + } + if c.URL != nil && c.URLFile != "" { + return errors.New("at most one of url & url_file must be configured") + } + if c.AlertSourceToken != "" && c.AlertSourceTokenFile != "" { + return errors.New("at most one of alert_source_token & alert_source_token_file must be configured") + } + if c.HTTPConfig != nil && c.HTTPConfig.Authorization != nil && (c.AlertSourceToken != "" || c.AlertSourceTokenFile != "") { + return errors.New("cannot specify alert_source_token or alert_source_token_file when using http_config.authorization") + } + + if (c.HTTPConfig != nil && c.HTTPConfig.Authorization == nil) && c.AlertSourceToken == "" && c.AlertSourceTokenFile == "" { + return errors.New("at least one of alert_source_token, alert_source_token_file or http_config.authorization must be configured") + } + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/incidentio/incidentio.go b/vendor/github.com/prometheus/alertmanager/notify/incidentio/incidentio.go index b985853eaba..a93c05ff47b 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/incidentio/incidentio.go +++ b/vendor/github.com/prometheus/alertmanager/notify/incidentio/incidentio.go @@ -26,7 +26,6 @@ import ( commoncfg "github.com/prometheus/common/config" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -39,7 +38,7 @@ const ( // Notifier implements a Notifier for incident.io. type Notifier struct { - conf *config.IncidentioConfig + conf *IncidentioConfig tmpl *template.Template logger *slog.Logger client *http.Client @@ -47,7 +46,7 @@ type Notifier struct { } // New returns a new incident.io notifier. -func New(conf *config.IncidentioConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(conf *IncidentioConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { // conf.HTTPConfig is likely to be the global shared HTTPConfig, so we take a // copy to avoid modifying it. httpConfig := *conf.HTTPConfig diff --git a/vendor/github.com/prometheus/alertmanager/notify/jira/config.go b/vendor/github.com/prometheus/alertmanager/notify/jira/config.go new file mode 100644 index 00000000000..3af2d1af6f8 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/jira/config.go @@ -0,0 +1,109 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jira + +import ( + "errors" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" + + commoncfg "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +var DefaultJiraConfig = JiraConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, + APIType: "auto", + Summary: JiraFieldConfig{ + Template: `{{ template "jira.default.summary" . }}`, + }, + Description: JiraFieldConfig{ + Template: `{{ template "jira.default.description" . }}`, + }, + Priority: `{{ template "jira.default.priority" . }}`, +} + +type JiraFieldConfig struct { + // Template is the template string used to render the field. + Template string `yaml:"template,omitempty" json:"template,omitempty"` + // EnableUpdate indicates whether this field should be omitted when updating an existing issue. + EnableUpdate *bool `yaml:"enable_update,omitempty" json:"enable_update,omitempty"` +} + +type JiraConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + + APIURL *amcommoncfg.URL `yaml:"api_url,omitempty" json:"api_url,omitempty"` + APIType string `yaml:"api_type,omitempty" json:"api_type,omitempty"` + + Project string `yaml:"project,omitempty" json:"project,omitempty"` + Summary JiraFieldConfig `yaml:"summary,omitempty" json:"summary,omitempty"` + Description JiraFieldConfig `yaml:"description,omitempty" json:"description,omitempty"` + Labels []string `yaml:"labels,omitempty" json:"labels,omitempty"` + Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` + IssueType string `yaml:"issue_type,omitempty" json:"issue_type,omitempty"` + + ReopenTransition string `yaml:"reopen_transition,omitempty" json:"reopen_transition,omitempty"` + ResolveTransition string `yaml:"resolve_transition,omitempty" json:"resolve_transition,omitempty"` + WontFixResolution string `yaml:"wont_fix_resolution,omitempty" json:"wont_fix_resolution,omitempty"` + ReopenDuration model.Duration `yaml:"reopen_duration,omitempty" json:"reopen_duration,omitempty"` + + Fields map[string]any `yaml:"fields,omitempty" json:"custom_fields,omitempty"` +} + +func (f *JiraFieldConfig) EnableUpdateValue() bool { + if f.EnableUpdate == nil { + return true + } + return *f.EnableUpdate +} + +// Supports both the legacy string and the new object form. +func (f *JiraFieldConfig) UnmarshalYAML(unmarshal func(any) error) error { + // Try simple string first (backward compatibility). + var s string + if err := unmarshal(&s); err == nil { + f.Template = s + // DisableUpdate stays false by default. + return nil + } + + // Fallback to full object form. + type plain JiraFieldConfig + return unmarshal((*plain)(f)) +} + +func (c *JiraConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultJiraConfig + type plain JiraConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + + if c.Project == "" { + return errors.New("missing project in jira_config") + } + if c.IssueType == "" { + return errors.New("missing issue_type in jira_config") + } + if c.APIType != "auto" && + c.APIType != "cloud" && + c.APIType != "datacenter" { + return errors.New("unknown api_type on jira_config, must be auto, cloud or datacenter") + } + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/jira/jira.go b/vendor/github.com/prometheus/alertmanager/notify/jira/jira.go index 401ae1df6fe..428ac89a1d5 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/jira/jira.go +++ b/vendor/github.com/prometheus/alertmanager/notify/jira/jira.go @@ -28,7 +28,6 @@ import ( commoncfg "github.com/prometheus/common/config" "github.com/prometheus/common/model" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -41,14 +40,14 @@ const ( // Notifier implements a Notifier for JIRA notifications. type Notifier struct { - conf *config.JiraConfig + conf *JiraConfig tmpl *template.Template logger *slog.Logger client *http.Client retrier *notify.Retrier } -func New(c *config.JiraConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(c *JiraConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "jira", httpOpts...) if err != nil { return nil, err @@ -219,7 +218,11 @@ func (n *Notifier) searchExistingIssue(ctx context.Context, logger *slog.Logger, jql := strings.Builder{} if n.conf.WontFixResolution != "" { - fmt.Fprintf(&jql, `resolution != %q and `, n.conf.WontFixResolution) + // JQL's != on resolution silently excludes issues whose resolution + // is EMPTY (unresolved). Use (resolution is EMPTY or resolution != X) + // so open issues remain in the candidate set and only the won't-fix + // resolved ones are filtered out. See prometheus/alertmanager#4295. + fmt.Fprintf(&jql, `(resolution is EMPTY or resolution != %q) and `, n.conf.WontFixResolution) } // If the group is firing, search for open issues. If a reopen transition is @@ -346,6 +349,10 @@ func (n *Notifier) transitionIssue(ctx context.Context, logger *slog.Logger, i * transition = n.conf.ResolveTransition } + if transition == "" { + return false, nil + } + transitionID, shouldRetry, err := n.getIssueTransitionByName(ctx, i.Key, transition) if err != nil { return shouldRetry, err diff --git a/vendor/github.com/prometheus/alertmanager/notify/mattermost/config.go b/vendor/github.com/prometheus/alertmanager/notify/mattermost/config.go new file mode 100644 index 00000000000..f77724186be --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/mattermost/config.go @@ -0,0 +1,138 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mattermost + +import ( + "errors" + + commoncfg "github.com/prometheus/common/config" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" +) + +var DefaultMattermostConfig = MattermostConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, + Username: `{{ template "mattermost.default.username" . }}`, + Color: `{{ template "mattermost.default.color" . }}`, + Text: `{{ template "mattermost.default.text" . }}`, + Title: `{{ template "mattermost.default.title" . }}`, + TitleLink: `{{ template "mattermost.default.titlelink" . }}`, + Fallback: `{{ template "mattermost.default.fallback" . }}`, +} + +// MattermostPriority defines the priority for a mattermost notification. +type MattermostPriority struct { + Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` + RequestedAck bool `yaml:"requested_ack,omitempty" json:"requested_ack,omitempty"` + PersistentNotifications bool `yaml:"persistent_notifications,omitempty" json:"persistent_notifications,omitempty"` +} + +// MattermostProps defines additional properties for a mattermost notification. +// Only 'card' property takes effect now. +type MattermostProps struct { + Card string `yaml:"card,omitempty" json:"card,omitempty"` +} + +// MattermostField configures a single Mattermost field for Slack compatibility. +// See https://developers.mattermost.com/integrate/reference/message-attachments/#fields for more information. +type MattermostField struct { + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Short *bool `yaml:"short,omitempty" json:"short,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for MattermostField. +func (c *MattermostField) UnmarshalYAML(unmarshal func(any) error) error { + type plain MattermostField + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.Title == "" { + return errors.New("missing title in Mattermost field configuration") + } + if c.Value == "" { + return errors.New("missing value in Mattermost field configuration") + } + return nil +} + +// MattermostAttachment defines an attachment for a Mattermost notification. +// See https://developers.mattermost.com/integrate/reference/message-attachments/#fields for more information. +type MattermostAttachment struct { + Fallback string `yaml:"fallback,omitempty" json:"fallback,omitempty"` + Color string `yaml:"color,omitempty" json:"color,omitempty"` + Pretext string `yaml:"pretext,omitempty" json:"pretext,omitempty"` + Text string `yaml:"text,omitempty" json:"text,omitempty"` + AuthorName string `yaml:"author_name,omitempty" json:"author_name,omitempty"` + AuthorLink string `yaml:"author_link,omitempty" json:"author_link,omitempty"` + AuthorIcon string `yaml:"author_icon,omitempty" json:"author_icon,omitempty"` + Title string `yaml:"title,omitempty" json:"title,omitempty"` + TitleLink string `yaml:"title_link,omitempty" json:"title_link,omitempty"` + Fields []*MattermostField `yaml:"fields,omitempty" json:"fields,omitempty"` + ThumbURL string `yaml:"thumb_url,omitempty" json:"thumb_url,omitempty"` + Footer string `yaml:"footer,omitempty" json:"footer,omitempty"` + FooterIcon string `yaml:"footer_icon,omitempty" json:"footer_icon,omitempty"` + ImageURL string `yaml:"image_url,omitempty" json:"image_url,omitempty"` +} + +// MattermostConfig configures notifications via Mattermost. +// See https://developers.mattermost.com/integrate/webhooks/incoming/ for more information. +type MattermostConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` + WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` + + Channel string `yaml:"channel,omitempty" json:"channel,omitempty"` + Username string `yaml:"username,omitempty" json:"username,omitempty"` + + Text string `yaml:"text,omitempty" json:"text,omitempty"` + Fallback string `yaml:"fallback,omitempty" json:"fallback,omitempty"` + Color string `yaml:"color,omitempty" json:"color,omitempty"` + Pretext string `yaml:"pretext,omitempty" json:"pretext,omitempty"` + AuthorName string `yaml:"author_name,omitempty" json:"author_name,omitempty"` + AuthorLink string `yaml:"author_link,omitempty" json:"author_link,omitempty"` + AuthorIcon string `yaml:"author_icon,omitempty" json:"author_icon,omitempty"` + Title string `yaml:"title,omitempty" json:"title,omitempty"` + TitleLink string `yaml:"title_link,omitempty" json:"title_link,omitempty"` + Fields []*MattermostField `yaml:"fields,omitempty" json:"fields,omitempty"` + ThumbURL string `yaml:"thumb_url,omitempty" json:"thumb_url,omitempty"` + Footer string `yaml:"footer,omitempty" json:"footer,omitempty"` + FooterIcon string `yaml:"footer_icon,omitempty" json:"footer_icon,omitempty"` + ImageURL string `yaml:"image_url,omitempty" json:"image_url,omitempty"` + IconURL string `yaml:"icon_url,omitempty" json:"icon_url,omitempty"` + IconEmoji string `yaml:"icon_emoji,omitempty" json:"icon_emoji,omitempty"` + Attachments []*MattermostAttachment `yaml:"attachments,omitempty" json:"attachments,omitempty"` + Type string `yaml:"type,omitempty" json:"type,omitempty"` + Props *MattermostProps `yaml:"props,omitempty" json:"props,omitempty"` + Priority *MattermostPriority `yaml:"priority,omitempty" json:"priority,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *MattermostConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultMattermostConfig + type plain MattermostConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + + if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { + return errors.New("at most one of webhook_url & webhook_url_file must be configured") + } + + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/mattermost/mattermost.go b/vendor/github.com/prometheus/alertmanager/notify/mattermost/mattermost.go index 625aaa7aaa2..9522763ff75 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/mattermost/mattermost.go +++ b/vendor/github.com/prometheus/alertmanager/notify/mattermost/mattermost.go @@ -27,7 +27,6 @@ import ( commoncfg "github.com/prometheus/common/config" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -39,7 +38,7 @@ const maxTextLenRunes = 16383 // Notifier implements a Notifier for Mattermost notifications. type Notifier struct { - conf *config.MattermostConfig + conf *MattermostConfig tmpl *template.Template logger *slog.Logger client *http.Client @@ -49,7 +48,7 @@ type Notifier struct { } // New returns a new Mattermost notifier. -func New(c *config.MattermostConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(c *MattermostConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "mattermost", httpOpts...) if err != nil { return nil, err @@ -68,34 +67,34 @@ func New(c *config.MattermostConfig, t *template.Template, l *slog.Logger, httpO // request is the request for sending a Mattermost notification. // https://developers.mattermost.com/integrate/webhooks/incoming/#parameters type request struct { - Text string `json:"text,omitempty"` - Channel string `json:"channel,omitempty"` - Username string `json:"username,omitempty"` - IconURL string `json:"icon_url,omitempty"` - IconEmoji string `json:"icon_emoji,omitempty"` - Attachments []attachment `json:"attachments,omitempty"` - Type string `json:"type,omitempty"` - Props *config.MattermostProps `json:"props,omitempty"` - Priority *config.MattermostPriority `json:"priority,omitempty"` + Text string `json:"text,omitempty"` + Channel string `json:"channel,omitempty"` + Username string `json:"username,omitempty"` + IconURL string `json:"icon_url,omitempty"` + IconEmoji string `json:"icon_emoji,omitempty"` + Attachments []attachment `json:"attachments,omitempty"` + Type string `json:"type,omitempty"` + Props *MattermostProps `json:"props,omitempty"` + Priority *MattermostPriority `json:"priority,omitempty"` } // attachment is used to display a richly-formatted message block for compatibility with Slack. // https://developers.mattermost.com/integrate/reference/message-attachments/ type attachment struct { - Fallback string `json:"fallback,omitempty"` - Color string `json:"color,omitempty"` - Pretext string `json:"pretext,omitempty"` - Text string `json:"text,omitempty"` - AuthorName string `json:"author_name,omitempty"` - AuthorLink string `json:"author_link,omitempty"` - AuthorIcon string `json:"author_icon,omitempty"` - Title string `json:"title,omitempty"` - TitleLink string `json:"title_link,omitempty"` - Fields []config.MattermostField `json:"fields,omitempty"` - ThumbURL string `json:"thumb_url,omitempty"` - Footer string `json:"footer,omitempty"` - FooterIcon string `json:"footer_icon,omitempty"` - ImageURL string `json:"image_url,omitempty"` + Fallback string `json:"fallback,omitempty"` + Color string `json:"color,omitempty"` + Pretext string `json:"pretext,omitempty"` + Text string `json:"text,omitempty"` + AuthorName string `json:"author_name,omitempty"` + AuthorLink string `json:"author_link,omitempty"` + AuthorIcon string `json:"author_icon,omitempty"` + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Fields []MattermostField `json:"fields,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + Footer string `json:"footer,omitempty"` + FooterIcon string `json:"footer_icon,omitempty"` + ImageURL string `json:"image_url,omitempty"` } // Notify implements the Notifier interface. @@ -163,7 +162,7 @@ func (n *Notifier) createRequest(tmpl func(string) string) *request { } if n.conf.Priority != nil && n.conf.Priority.Priority != "" { - req.Priority = &config.MattermostPriority{ + req.Priority = &MattermostPriority{ Priority: tmpl(n.conf.Priority.Priority), RequestedAck: n.conf.Priority.RequestedAck, PersistentNotifications: n.conf.Priority.PersistentNotifications, @@ -171,7 +170,7 @@ func (n *Notifier) createRequest(tmpl func(string) string) *request { } if n.conf.Props != nil && n.conf.Props.Card != "" { - req.Props = &config.MattermostProps{ + req.Props = &MattermostProps{ Card: tmpl(n.conf.Props.Card), } } @@ -198,9 +197,9 @@ func (n *Notifier) createRequest(tmpl func(string) string) *request { lenFields := len(cfgAtt.Fields) if lenFields > 0 { - att.Fields = make([]config.MattermostField, lenFields) + att.Fields = make([]MattermostField, lenFields) for idxField, field := range cfgAtt.Fields { - att.Fields[idxField] = config.MattermostField{ + att.Fields[idxField] = MattermostField{ Title: tmpl(field.Title), Value: tmpl(field.Value), Short: field.Short, @@ -231,9 +230,9 @@ func (n *Notifier) createRequest(tmpl func(string) string) *request { lenFields := len(n.conf.Fields) if lenFields > 0 { - att.Fields = make([]config.MattermostField, lenFields) + att.Fields = make([]MattermostField, lenFields) for idxField, field := range n.conf.Fields { - att.Fields[idxField] = config.MattermostField{ + att.Fields[idxField] = MattermostField{ Title: tmpl(field.Title), Value: tmpl(field.Value), Short: field.Short, diff --git a/vendor/github.com/prometheus/alertmanager/notify/metrics.go b/vendor/github.com/prometheus/alertmanager/notify/metrics.go new file mode 100644 index 00000000000..33a58cf4645 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/metrics.go @@ -0,0 +1,140 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/prometheus/alertmanager/featurecontrol" +) + +type Metrics struct { + numNotifications *prometheus.CounterVec + numTotalFailedNotifications *prometheus.CounterVec + numNotificationRequestsTotal *prometheus.CounterVec + numNotificationRequestsFailedTotal *prometheus.CounterVec + numNotificationSuppressedTotal *prometheus.CounterVec + notificationLatencySeconds *prometheus.HistogramVec + + ff featurecontrol.Flagger +} + +func NewMetrics(r prometheus.Registerer, ff featurecontrol.Flagger) *Metrics { + labels := []string{"integration"} + + if ff.EnableReceiverNamesInMetrics() { + labels = append(labels, "receiver_name") + } + + m := &Metrics{ + numNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "alertmanager", + Name: "notifications_total", + Help: "The total number of attempted notifications.", + }, labels), + numTotalFailedNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "alertmanager", + Name: "notifications_failed_total", + Help: "The total number of failed notifications.", + }, append(labels, "reason")), + numNotificationRequestsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "alertmanager", + Name: "notification_requests_total", + Help: "The total number of attempted notification requests.", + }, labels), + numNotificationRequestsFailedTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "alertmanager", + Name: "notification_requests_failed_total", + Help: "The total number of failed notification requests.", + }, labels), + numNotificationSuppressedTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: "alertmanager", + Name: "notifications_suppressed_total", + Help: "The total number of notifications suppressed for being silenced, inhibited, outside of active time intervals or within muted time intervals.", + }, []string{"reason"}), + notificationLatencySeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "alertmanager", + Name: "notification_latency_seconds", + Help: "The latency of notifications in seconds.", + Buckets: []float64{1, 5, 10, 15, 20}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, labels), + ff: ff, + } + + return m +} + +func (m *Metrics) InitializeFor(receiver map[string][]Integration) { + if m.ff.EnableReceiverNamesInMetrics() { + + // Reset the vectors to take into account receiver names changing after hot reloads. + m.numNotifications.Reset() + m.numNotificationRequestsTotal.Reset() + m.numNotificationRequestsFailedTotal.Reset() + m.notificationLatencySeconds.Reset() + m.numTotalFailedNotifications.Reset() + + for name, integrations := range receiver { + for _, integration := range integrations { + + m.numNotifications.WithLabelValues(integration.Name(), name) + m.numNotificationRequestsTotal.WithLabelValues(integration.Name(), name) + m.numNotificationRequestsFailedTotal.WithLabelValues(integration.Name(), name) + m.notificationLatencySeconds.WithLabelValues(integration.Name(), name) + + for _, reason := range possibleFailureReasonCategory { + m.numTotalFailedNotifications.WithLabelValues(integration.Name(), name, reason) + } + } + } + + return + } + + // When the feature flag is not enabled, we just carry on registering _all_ the integrations. + for _, integration := range []string{ + "email", + "pagerduty", + "wechat", + "pushover", + "slack", + "opsgenie", + "webhook", + "victorops", + "sns", + "telegram", + "discord", + "webex", + "msteams", + "msteamsv2", + "incidentio", + "jira", + "rocketchat", + "mattermost", + } { + m.numNotifications.WithLabelValues(integration) + m.numNotificationRequestsTotal.WithLabelValues(integration) + m.numNotificationRequestsFailedTotal.WithLabelValues(integration) + m.notificationLatencySeconds.WithLabelValues(integration) + + for _, reason := range possibleFailureReasonCategory { + m.numTotalFailedNotifications.WithLabelValues(integration, reason) + } + } +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/msteams/config.go b/vendor/github.com/prometheus/alertmanager/notify/msteams/config.go new file mode 100644 index 00000000000..e2624fb1a4e --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/msteams/config.go @@ -0,0 +1,60 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msteams + +import ( + "errors" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" + + commoncfg "github.com/prometheus/common/config" +) + +var DefaultMSTeamsConfig = MSTeamsConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, + Title: `{{ template "msteams.default.title" . }}`, + Summary: `{{ template "msteams.default.summary" . }}`, + Text: `{{ template "msteams.default.text" . }}`, +} + +type MSTeamsConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + WebhookURL *amcommoncfg.SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"` + WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"` + + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Summary string `yaml:"summary,omitempty" json:"summary,omitempty"` + Text string `yaml:"text,omitempty" json:"text,omitempty"` +} + +func (c *MSTeamsConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = DefaultMSTeamsConfig + type plain MSTeamsConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + + if c.WebhookURL == nil && c.WebhookURLFile == "" { + return errors.New("one of webhook_url or webhook_url_file must be configured") + } + + if c.WebhookURL != nil && len(c.WebhookURLFile) > 0 { + return errors.New("at most one of webhook_url & webhook_url_file must be configured") + } + + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/msteams/msteams.go b/vendor/github.com/prometheus/alertmanager/notify/msteams/msteams.go index 1f606479c51..cf7748e27f6 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/msteams/msteams.go +++ b/vendor/github.com/prometheus/alertmanager/notify/msteams/msteams.go @@ -29,7 +29,6 @@ import ( amcommoncfg "github.com/prometheus/alertmanager/config/common" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -42,7 +41,7 @@ const ( ) type Notifier struct { - conf *config.MSTeamsConfig + conf *MSTeamsConfig tmpl *template.Template logger *slog.Logger client *http.Client @@ -62,7 +61,7 @@ type teamsMessage struct { } // New returns a new notifier that uses the Microsoft Teams Webhook API. -func New(c *config.MSTeamsConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(c *MSTeamsConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "msteams", httpOpts...) if err != nil { return nil, err diff --git a/vendor/github.com/prometheus/alertmanager/notify/mute.go b/vendor/github.com/prometheus/alertmanager/notify/mute.go index f095bfd012e..12abc0ccbbc 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/mute.go +++ b/vendor/github.com/prometheus/alertmanager/notify/mute.go @@ -25,9 +25,17 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + "github.com/prometheus/alertmanager/alert" "github.com/prometheus/alertmanager/inhibit" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/silence" - "github.com/prometheus/alertmanager/types" +) + +const ( + SuppressedReasonSilence = "silence" + SuppressedReasonInhibition = "inhibition" + SuppressedReasonMuteTimeInterval = "mute_time_interval" + SuppressedReasonActiveTimeInterval = "active_time_interval" ) // A Muter determines whether a given label set is muted. Implementers that @@ -55,7 +63,7 @@ func NewMuteStage(m Muter, metrics *Metrics) *MuteStage { } // Exec implements the Stage interface. -func (n *MuteStage) Exec(ctx context.Context, logger *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (n *MuteStage) Exec(ctx context.Context, logger *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { ctx, span := tracer.Start(ctx, "notify.MuteStage.Exec", trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), trace.WithSpanKind(trace.SpanKindInternal), @@ -63,8 +71,8 @@ func (n *MuteStage) Exec(ctx context.Context, logger *slog.Logger, alerts ...*ty defer span.End() var ( - filtered []*types.Alert - muted []*types.Alert + filtered []*alert.Alert + muted []*alert.Alert ) for _, a := range alerts { // TODO(fabxc): increment total alerts counter. @@ -93,6 +101,17 @@ func (n *MuteStage) Exec(ctx context.Context, logger *slog.Logger, alerts ...*ty ) n.metrics.numNotificationSuppressedTotal.WithLabelValues(reason).Add(float64(len(muted))) logger.Debug("Notifications will not be sent for muted alerts", "alerts", fmt.Sprintf("%v", muted), "reason", reason) + + // Record muted alert hashes in the context so downstream stages + // (e.g., the event recorder) can observe which alerts were muted. + mutedHashes, _ := MutedAlerts(ctx) + if mutedHashes == nil { + mutedHashes = make(map[uint64]struct{}, len(muted)) + } + for _, a := range muted { + mutedHashes[hashAlert(a)] = struct{}{} + } + ctx = WithMutedAlerts(ctx, mutedHashes) } return ctx, filtered, nil @@ -107,19 +126,19 @@ type TimeMuter interface { type timeStage struct { muter TimeMuter - marker types.GroupMarker + marker marker.GroupMarker metrics *Metrics } type TimeMuteStage timeStage -func NewTimeMuteStage(muter TimeMuter, marker types.GroupMarker, metrics *Metrics) *TimeMuteStage { +func NewTimeMuteStage(muter TimeMuter, marker marker.GroupMarker, metrics *Metrics) *TimeMuteStage { return &TimeMuteStage{muter, marker, metrics} } // Exec implements the stage interface for TimeMuteStage. // TimeMuteStage is responsible for muting alerts whose route is not in an active time. -func (tms TimeMuteStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (tms TimeMuteStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { ctx, span := tracer.Start(ctx, "notify.TimeMuteStage.Exec", trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), trace.WithSpanKind(trace.SpanKindInternal), @@ -143,15 +162,18 @@ func (tms TimeMuteStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*ty muteTimeIntervalNames, ok := MuteTimeIntervalNames(ctx) if !ok { + tms.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, nil } now, ok := Now(ctx) if !ok { + tms.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, errors.New("missing now timestamp") } // Skip this stage if there are no mute timings. if len(muteTimeIntervalNames) == 0 { + tms.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, nil } @@ -177,13 +199,13 @@ func (tms TimeMuteStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*ty type TimeActiveStage timeStage -func NewTimeActiveStage(muter TimeMuter, marker types.GroupMarker, metrics *Metrics) *TimeActiveStage { +func NewTimeActiveStage(muter TimeMuter, marker marker.GroupMarker, metrics *Metrics) *TimeActiveStage { return &TimeActiveStage{muter, marker, metrics} } // Exec implements the stage interface for TimeActiveStage. // TimeActiveStage is responsible for muting alerts whose route is not in an active time. -func (tas TimeActiveStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (tas TimeActiveStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { routeID, ok := RouteID(ctx) if !ok { return ctx, nil, errors.New("route ID missing") @@ -203,16 +225,19 @@ func (tas TimeActiveStage) Exec(ctx context.Context, l *slog.Logger, alerts ...* activeTimeIntervalNames, ok := ActiveTimeIntervalNames(ctx) if !ok { + tas.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, nil } // if we don't have active time intervals at all it is always active. if len(activeTimeIntervalNames) == 0 { + tas.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, nil } now, ok := Now(ctx) if !ok { + tas.marker.SetMuted(routeID, gkey, nil) return ctx, alerts, errors.New("missing now timestamp") } diff --git a/vendor/github.com/prometheus/alertmanager/notify/notify.go b/vendor/github.com/prometheus/alertmanager/notify/notify.go index 8a51dcfd304..0bb2dc6e622 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/notify.go +++ b/vendor/github.com/prometheus/alertmanager/notify/notify.go @@ -22,24 +22,23 @@ import ( "sync" "time" - "github.com/cenkalti/backoff/v4" "github.com/cespare/xxhash/v2" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/eventrecorder" "github.com/prometheus/alertmanager/featurecontrol" "github.com/prometheus/alertmanager/inhibit" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/nflog" "github.com/prometheus/alertmanager/nflog/nflogpb" "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/timeinterval" "github.com/prometheus/alertmanager/tracing" - "github.com/prometheus/alertmanager/types" ) var tracer = tracing.NewTracer("github.com/prometheus/alertmanager/notify") @@ -63,7 +62,7 @@ const MinTimeout = 10 * time.Second // returns an error if unsuccessful and a flag whether the error is // recoverable. This information is useful for a retry logic. type Notifier interface { - Notify(context.Context, ...*types.Alert) (bool, error) + Notify(context.Context, ...*alert.Alert) (bool, error) } // Integration wraps a notifier and its configuration to be uniquely identified @@ -88,7 +87,7 @@ func NewIntegration(notifier Notifier, rs ResolvedSender, name string, idx int, } // Notify implements the Notifier interface. -func (i *Integration) Notify(ctx context.Context, alerts ...*types.Alert) (recoverable bool, err error) { +func (i *Integration) Notify(ctx context.Context, alerts ...*alert.Alert) (recoverable bool, err error) { ctx, span := tracer.Start(ctx, "notify.Integration.Notify", trace.WithAttributes(attribute.String("alerting.notify.integration.name", i.name)), trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), @@ -128,171 +127,16 @@ func (i *Integration) String() string { return fmt.Sprintf("%s[%d]", i.name, i.idx) } -// notifyKey defines a custom type with which a context is populated to -// avoid accidental collisions. -type notifyKey int - -const ( - keyReceiverName notifyKey = iota - keyRepeatInterval - keyGroupLabels - keyGroupKey - keyFiringAlerts - keyResolvedAlerts - keyNow - keyMuteTimeIntervals - keyActiveTimeIntervals - keyRouteID - keyNflogStore - keyNotificationReason -) - -// WithReceiverName populates a context with a receiver name. -func WithReceiverName(ctx context.Context, rcv string) context.Context { - return context.WithValue(ctx, keyReceiverName, rcv) -} - -// WithGroupKey populates a context with a group key. -func WithGroupKey(ctx context.Context, s string) context.Context { - return context.WithValue(ctx, keyGroupKey, s) -} - -// WithFiringAlerts populates a context with a slice of firing alerts. -func WithFiringAlerts(ctx context.Context, alerts []uint64) context.Context { - return context.WithValue(ctx, keyFiringAlerts, alerts) -} - -// WithResolvedAlerts populates a context with a slice of resolved alerts. -func WithResolvedAlerts(ctx context.Context, alerts []uint64) context.Context { - return context.WithValue(ctx, keyResolvedAlerts, alerts) -} - -// WithGroupLabels populates a context with grouping labels. -func WithGroupLabels(ctx context.Context, lset model.LabelSet) context.Context { - return context.WithValue(ctx, keyGroupLabels, lset) -} - -// WithNow populates a context with a now timestamp. -func WithNow(ctx context.Context, t time.Time) context.Context { - return context.WithValue(ctx, keyNow, t) -} - -// WithRepeatInterval populates a context with a repeat interval. -func WithRepeatInterval(ctx context.Context, t time.Duration) context.Context { - return context.WithValue(ctx, keyRepeatInterval, t) -} - -// WithMuteTimeIntervals populates a context with a slice of mute time names. -func WithMuteTimeIntervals(ctx context.Context, mt []string) context.Context { - return context.WithValue(ctx, keyMuteTimeIntervals, mt) -} - -func WithActiveTimeIntervals(ctx context.Context, at []string) context.Context { - return context.WithValue(ctx, keyActiveTimeIntervals, at) -} - -func WithRouteID(ctx context.Context, routeID string) context.Context { - return context.WithValue(ctx, keyRouteID, routeID) -} - -func WithNotificationReason(ctx context.Context, reason NotifyReason) context.Context { - return context.WithValue(ctx, keyNotificationReason, reason) -} - -// RepeatInterval extracts a repeat interval from the context. Iff none exists, the -// second argument is false. -func RepeatInterval(ctx context.Context) (time.Duration, bool) { - v, ok := ctx.Value(keyRepeatInterval).(time.Duration) - return v, ok -} - -// ReceiverName extracts a receiver name from the context. Iff none exists, the -// second argument is false. -func ReceiverName(ctx context.Context) (string, bool) { - v, ok := ctx.Value(keyReceiverName).(string) - return v, ok -} - -// GroupKey extracts a group key from the context. Iff none exists, the -// second argument is false. -func GroupKey(ctx context.Context) (string, bool) { - v, ok := ctx.Value(keyGroupKey).(string) - return v, ok -} - -// GroupLabels extracts grouping label set from the context. Iff none exists, the -// second argument is false. -func GroupLabels(ctx context.Context) (model.LabelSet, bool) { - v, ok := ctx.Value(keyGroupLabels).(model.LabelSet) - return v, ok -} - -// Now extracts a now timestamp from the context. Iff none exists, the -// second argument is false. -func Now(ctx context.Context) (time.Time, bool) { - v, ok := ctx.Value(keyNow).(time.Time) - return v, ok -} - -// FiringAlerts extracts a slice of firing alerts from the context. -// Iff none exists, the second argument is false. -func FiringAlerts(ctx context.Context) ([]uint64, bool) { - v, ok := ctx.Value(keyFiringAlerts).([]uint64) - return v, ok -} - -// ResolvedAlerts extracts a slice of firing alerts from the context. -// Iff none exists, the second argument is false. -func ResolvedAlerts(ctx context.Context) ([]uint64, bool) { - v, ok := ctx.Value(keyResolvedAlerts).([]uint64) - return v, ok -} - -// MuteTimeIntervalNames extracts a slice of mute time names from the context. If and only if none exists, the -// second argument is false. -func MuteTimeIntervalNames(ctx context.Context) ([]string, bool) { - v, ok := ctx.Value(keyMuteTimeIntervals).([]string) - return v, ok -} - -// ActiveTimeIntervalNames extracts a slice of active time names from the context. If none exists, the -// second argument is false. -func ActiveTimeIntervalNames(ctx context.Context) ([]string, bool) { - v, ok := ctx.Value(keyActiveTimeIntervals).([]string) - return v, ok -} - -// RouteID extracts a RouteID from the context. Iff none exists, the -// // second argument is false. -func RouteID(ctx context.Context) (string, bool) { - v, ok := ctx.Value(keyRouteID).(string) - return v, ok -} - -func NotificationReason(ctx context.Context) (NotifyReason, bool) { - v, ok := ctx.Value(keyNotificationReason).(NotifyReason) - return v, ok -} - -func WithNflogStore(ctx context.Context, store *nflog.Store) context.Context { - return context.WithValue(ctx, keyNflogStore, store) -} - -func NflogStore(ctx context.Context) (*nflog.Store, bool) { - v, ok := ctx.Value(keyNflogStore).(*nflog.Store) - return v, ok -} - // A Stage processes alerts under the constraints of the given context. type Stage interface { - Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) + Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) } // StageFunc wraps a function to represent a Stage. -type StageFunc func(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) +type StageFunc func(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) // Exec implements Stage interface. -func (f StageFunc) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (f StageFunc) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { return f(ctx, l, alerts...) } @@ -301,133 +145,17 @@ type NotificationLog interface { Query(params ...nflog.QueryParam) ([]*nflogpb.Entry, error) } -type Metrics struct { - numNotifications *prometheus.CounterVec - numTotalFailedNotifications *prometheus.CounterVec - numNotificationRequestsTotal *prometheus.CounterVec - numNotificationRequestsFailedTotal *prometheus.CounterVec - numNotificationSuppressedTotal *prometheus.CounterVec - notificationLatencySeconds *prometheus.HistogramVec - - ff featurecontrol.Flagger -} - -func NewMetrics(r prometheus.Registerer, ff featurecontrol.Flagger) *Metrics { - labels := []string{"integration"} - - if ff.EnableReceiverNamesInMetrics() { - labels = append(labels, "receiver_name") - } - - m := &Metrics{ - numNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "alertmanager", - Name: "notifications_total", - Help: "The total number of attempted notifications.", - }, labels), - numTotalFailedNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "alertmanager", - Name: "notifications_failed_total", - Help: "The total number of failed notifications.", - }, append(labels, "reason")), - numNotificationRequestsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "alertmanager", - Name: "notification_requests_total", - Help: "The total number of attempted notification requests.", - }, labels), - numNotificationRequestsFailedTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "alertmanager", - Name: "notification_requests_failed_total", - Help: "The total number of failed notification requests.", - }, labels), - numNotificationSuppressedTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "alertmanager", - Name: "notifications_suppressed_total", - Help: "The total number of notifications suppressed for being silenced, inhibited, outside of active time intervals or within muted time intervals.", - }, []string{"reason"}), - notificationLatencySeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "alertmanager", - Name: "notification_latency_seconds", - Help: "The latency of notifications in seconds.", - Buckets: []float64{1, 5, 10, 15, 20}, - NativeHistogramBucketFactor: 1.1, - NativeHistogramMaxBucketNumber: 100, - NativeHistogramMinResetDuration: 1 * time.Hour, - }, labels), - ff: ff, - } - - return m -} - -func (m *Metrics) InitializeFor(receiver map[string][]Integration) { - if m.ff.EnableReceiverNamesInMetrics() { - - // Reset the vectors to take into account receiver names changing after hot reloads. - m.numNotifications.Reset() - m.numNotificationRequestsTotal.Reset() - m.numNotificationRequestsFailedTotal.Reset() - m.notificationLatencySeconds.Reset() - m.numTotalFailedNotifications.Reset() - - for name, integrations := range receiver { - for _, integration := range integrations { - - m.numNotifications.WithLabelValues(integration.Name(), name) - m.numNotificationRequestsTotal.WithLabelValues(integration.Name(), name) - m.numNotificationRequestsFailedTotal.WithLabelValues(integration.Name(), name) - m.notificationLatencySeconds.WithLabelValues(integration.Name(), name) - - for _, reason := range possibleFailureReasonCategory { - m.numTotalFailedNotifications.WithLabelValues(integration.Name(), name, reason) - } - } - } - - return - } - - // When the feature flag is not enabled, we just carry on registering _all_ the integrations. - for _, integration := range []string{ - "email", - "pagerduty", - "wechat", - "pushover", - "slack", - "opsgenie", - "webhook", - "victorops", - "sns", - "telegram", - "discord", - "webex", - "msteams", - "msteamsv2", - "incidentio", - "jira", - "rocketchat", - "mattermost", - } { - m.numNotifications.WithLabelValues(integration) - m.numNotificationRequestsTotal.WithLabelValues(integration) - m.numNotificationRequestsFailedTotal.WithLabelValues(integration) - m.notificationLatencySeconds.WithLabelValues(integration) - - for _, reason := range possibleFailureReasonCategory { - m.numTotalFailedNotifications.WithLabelValues(integration, reason) - } - } -} - type PipelineBuilder struct { - metrics *Metrics - ff featurecontrol.Flagger + metrics *Metrics + ff featurecontrol.Flagger + recorder eventrecorder.Recorder } -func NewPipelineBuilder(r prometheus.Registerer, ff featurecontrol.Flagger) *PipelineBuilder { +func NewPipelineBuilder(r prometheus.Registerer, ff featurecontrol.Flagger, recorder eventrecorder.Recorder) *PipelineBuilder { return &PipelineBuilder{ - metrics: NewMetrics(r, ff), - ff: ff, + metrics: NewMetrics(r, ff), + ff: ff, + recorder: recorder, } } @@ -438,20 +166,20 @@ func (pb *PipelineBuilder) New( inhibitor *inhibit.Inhibitor, silencer *silence.Silencer, intervener *timeinterval.Intervener, - marker types.GroupMarker, + marker marker.GroupMarker, notificationLog NotificationLog, peer Peer, ) RoutingStage { rs := make(RoutingStage, len(receivers)) - ms := NewGossipSettleStage(peer) + ms := NewClusterGossipSettleStage(peer) is := NewMuteStage(inhibitor, pb.metrics) tas := NewTimeActiveStage(intervener, marker, pb.metrics) tms := NewTimeMuteStage(intervener, marker, pb.metrics) ss := NewMuteStage(silencer, pb.metrics) for name := range receivers { - st := createReceiverStage(name, receivers[name], wait, notificationLog, pb.metrics) + st := createReceiverStage(name, receivers[name], wait, notificationLog, pb.metrics, pb.recorder) rs[name] = MultiStage{ms, is, tas, tms, ss, st} } @@ -467,6 +195,7 @@ func createReceiverStage( wait func() time.Duration, notificationLog NotificationLog, metrics *Metrics, + recorder eventrecorder.Recorder, ) Stage { var fs FanoutStage for i := range integrations { @@ -476,9 +205,9 @@ func createReceiverStage( Idx: uint32(integrations[i].Index()), } var s MultiStage - s = append(s, NewWaitStage(wait)) + s = append(s, NewClusterWaitStage(wait)) s = append(s, NewDedupStage(&integrations[i], notificationLog, recv)) - s = append(s, NewRetryStage(integrations[i], name, metrics)) + s = append(s, NewRetryStage(integrations[i], name, metrics, recorder)) s = append(s, NewSetNotifiesStage(notificationLog, recv)) fs = append(fs, s) @@ -491,7 +220,7 @@ func createReceiverStage( type RoutingStage map[string]Stage // Exec implements the Stage interface. -func (rs RoutingStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (rs RoutingStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { receiver, ok := ReceiverName(ctx) if !ok { return ctx, nil, errors.New("receiver missing") @@ -518,7 +247,7 @@ func (rs RoutingStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*type type MultiStage []Stage // Exec implements the Stage interface. -func (ms MultiStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (ms MultiStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { var err error for _, s := range ms { if len(alerts) == 0 { @@ -538,7 +267,7 @@ type FanoutStage []Stage // Exec attempts to execute all stages concurrently and discards the results. // It returns its input alerts and an error if one or more stages fail. -func (fs FanoutStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { +func (fs FanoutStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { var ( wg sync.WaitGroup mtx sync.Mutex @@ -561,116 +290,6 @@ func (fs FanoutStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types return ctx, alerts, errs } -// GossipSettleStage waits until the Gossip has settled to forward alerts. -type GossipSettleStage struct { - peer Peer -} - -// NewGossipSettleStage returns a new GossipSettleStage. -func NewGossipSettleStage(p Peer) *GossipSettleStage { - return &GossipSettleStage{peer: p} -} - -func (n *GossipSettleStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - if n.peer != nil { - if err := n.peer.WaitReady(ctx); err != nil { - return ctx, nil, err - } - } - return ctx, alerts, nil -} - -const ( - SuppressedReasonSilence = "silence" - SuppressedReasonInhibition = "inhibition" - SuppressedReasonMuteTimeInterval = "mute_time_interval" - SuppressedReasonActiveTimeInterval = "active_time_interval" -) - -// WaitStage waits for a certain amount of time before continuing or until the -// context is done. -type WaitStage struct { - wait func() time.Duration -} - -// NewWaitStage returns a new WaitStage. -func NewWaitStage(wait func() time.Duration) *WaitStage { - return &WaitStage{ - wait: wait, - } -} - -// Exec implements the Stage interface. -func (ws *WaitStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - select { - case <-time.After(ws.wait()): - case <-ctx.Done(): - return ctx, nil, ctx.Err() - } - return ctx, alerts, nil -} - -// DedupStage filters alerts. -// Filtering happens based on a notification log. -type DedupStage struct { - rs ResolvedSender - nflog NotificationLog - recv *nflogpb.Receiver - - now func() time.Time - hash func(*types.Alert) uint64 -} - -// NewDedupStage wraps a DedupStage that runs against the given notification log. -func NewDedupStage(rs ResolvedSender, l NotificationLog, recv *nflogpb.Receiver) *DedupStage { - return &DedupStage{ - rs: rs, - nflog: l, - recv: recv, - now: utcNow, - hash: hashAlert, - } -} - -func utcNow() time.Time { - return time.Now().UTC() -} - -// Wrap a slice in a struct so we can store a pointer in sync.Pool. -type hashBuffer struct { - buf []byte -} - -var hashBuffers = sync.Pool{ - New: func() any { return &hashBuffer{buf: make([]byte, 0, 1024)} }, -} - -func hashAlert(a *types.Alert) uint64 { - const sep = '\xff' - - hb := hashBuffers.Get().(*hashBuffer) - defer hashBuffers.Put(hb) - b := hb.buf[:0] - - names := make(model.LabelNames, 0, len(a.Labels)) - - for ln := range a.Labels { - names = append(names, ln) - } - sort.Sort(names) - - for _, ln := range names { - b = append(b, string(ln)...) - b = append(b, sep) - b = append(b, string(a.Labels[ln])...) - b = append(b, sep) - } - - hash := xxhash.Sum64(b) - - return hash -} - type NotifyReason int const ( @@ -706,335 +325,41 @@ func (r NotifyReason) String() string { } } -func (n *DedupStage) needsUpdate(entry *nflogpb.Entry, firing, resolved map[uint64]struct{}, repeat time.Duration, now time.Time) NotifyReason { - // If we haven't notified about the alert group before, notify right away - // unless we only have resolved alerts. - if entry == nil { - if len(firing) > 0 { - return ReasonFirstNotification - } - return ReasonDoNotNotify - } - - // new alerts in the group - if !entry.IsFiringSubset(firing) { - // If the previous entry has no firing alerts, it was a resolution and we - // should treat this as the first notification for the group. - if len(entry.FiringAlerts) == 0 { - return ReasonFirstNotification - } - return ReasonNewAlertsInGroup - } - - // Notify about all alerts being resolved. - // This is done irrespective of the send_resolved flag to make sure that - // the firing alerts are cleared from the notification log. - if len(firing) == 0 { - // If the current alert group and last notification contain no firing - // alert, it means that some alerts have been fired and resolved during the - // last interval. In this case, there is no need to notify the receiver - // since it doesn't know about them. - if len(entry.FiringAlerts) > 0 { - return ReasonAllAlertsResolved - } - return ReasonDoNotNotify - } - - if n.rs.SendResolved() && !entry.IsResolvedSubset(resolved) { - return ReasonNewResolvedAlerts - } - - // Nothing changed, only notify if the repeat interval has passed. - isRepeatIntervalElapsed := entry.Timestamp.AsTime().Before(now.Add(-repeat)) - if isRepeatIntervalElapsed { - return ReasonRepeatIntervalElapsed - } - return ReasonDoNotNotify -} - -// partitionAlertsByState separates alerts into firing and resolved, returning both slices and sets. -func partitionAlertsByState(alerts []*types.Alert, hashFn func(*types.Alert) uint64) (firing, resolved []uint64, firingSet, resolvedSet map[uint64]struct{}) { - firingSet = make(map[uint64]struct{}, len(alerts)) - resolvedSet = make(map[uint64]struct{}, len(alerts)) - firing = make([]uint64, 0, len(alerts)) - resolved = make([]uint64, 0, len(alerts)) - - for _, a := range alerts { - hash := hashFn(a) - if a.Resolved() { - resolved = append(resolved, hash) - resolvedSet[hash] = struct{}{} - } else { - firing = append(firing, hash) - firingSet[hash] = struct{}{} - } - } - return firing, resolved, firingSet, resolvedSet -} - -// Exec implements the Stage interface. -func (n *DedupStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - gkey, ok := GroupKey(ctx) - if !ok { - return ctx, nil, errors.New("group key missing") - } - - ctx, span := tracer.Start(ctx, "notify.DedupStage.Exec", - trace.WithAttributes(attribute.String("alerting.group.key", gkey)), - trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), - trace.WithSpanKind(trace.SpanKindInternal), - ) - defer span.End() - - repeatInterval, ok := RepeatInterval(ctx) - if !ok { - return ctx, nil, errors.New("repeat interval missing") - } - - firing, resolved, firingSet, resolvedSet := partitionAlertsByState(alerts, n.hash) - - ctx = WithFiringAlerts(ctx, firing) - ctx = WithResolvedAlerts(ctx, resolved) - - entries, err := n.nflog.Query(nflog.QGroupKey(gkey), nflog.QReceiver(n.recv)) - if err != nil && !errors.Is(err, nflog.ErrNotFound) { - return ctx, nil, err - } - - var entry *nflogpb.Entry - switch len(entries) { - case 0: - case 1: - entry = entries[0] - default: - return ctx, nil, fmt.Errorf("unexpected entry result size %d", len(entries)) - } - - now := n.now() - if ctxNow, ok := Now(ctx); ok { - now = ctxNow - } - updateReason := n.needsUpdate(entry, firingSet, resolvedSet, repeatInterval, now) - ctx = WithNotificationReason(ctx, updateReason) - - if updateReason == ReasonFirstNotification { - ctx = WithNflogStore(ctx, nflog.NewStore(nil)) - } else { - ctx = WithNflogStore(ctx, nflog.NewStore(entry)) - } - - if updateReason.shouldNotify() { - span.AddEvent("notify.DedupStage.Exec nflog needs update") - return ctx, alerts, nil - } - return ctx, nil, nil -} - -// RetryStage notifies via passed integration with exponential backoff until it -// succeeds. It aborts if the context is canceled or timed out. -type RetryStage struct { - integration Integration - groupName string - metrics *Metrics - labelValues []string -} - -// NewRetryStage returns a new instance of a RetryStage. -func NewRetryStage(i Integration, groupName string, metrics *Metrics) *RetryStage { - labelValues := []string{i.Name()} - - if metrics.ff.EnableReceiverNamesInMetrics() { - labelValues = append(labelValues, i.receiverName) - } - - return &RetryStage{ - integration: i, - groupName: groupName, - metrics: metrics, - labelValues: labelValues, - } -} - -func (r RetryStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - r.metrics.numNotifications.WithLabelValues(r.labelValues...).Inc() - - ctx, span := tracer.Start(ctx, "notify.RetryStage.Exec", - trace.WithAttributes(attribute.String("alerting.group.name", r.groupName)), - trace.WithAttributes(attribute.String("alerting.integration.name", r.integration.name)), - trace.WithAttributes(attribute.StringSlice("alerting.label.values", r.labelValues)), - trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), - trace.WithSpanKind(trace.SpanKindInternal), - ) - defer span.End() - - ctx, alerts, err := r.exec(ctx, l, alerts...) - - failureReason := DefaultReason.String() - if err != nil { - span.SetStatus(codes.Error, err.Error()) - span.RecordError(err) - - var e *ErrorWithReason - if errors.As(err, &e) { - failureReason = e.Reason.String() - } - r.metrics.numTotalFailedNotifications.WithLabelValues(append(r.labelValues, failureReason)...).Inc() - } - return ctx, alerts, err -} - -func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - var sent alert.AlertSlice - - // If we shouldn't send notifications for resolved alerts, but there are only - // resolved alerts, report them all as successfully notified (we still want the - // notification log to log them for the next run of DedupStage). - if !r.integration.SendResolved() { - firing, ok := FiringAlerts(ctx) - if !ok { - return ctx, nil, errors.New("firing alerts missing") - } - if len(firing) == 0 { - return ctx, alerts, nil - } - for _, a := range alerts { - if a.Status() != model.AlertResolved { - sent = append(sent, a) - } - } - } else { - sent = alerts - } - - b := backoff.NewExponentialBackOff() - b.MaxElapsedTime = 0 // Always retry. - - tick := backoff.NewTicker(b) - defer tick.Stop() - - var ( - i = 0 - iErr error - ) - - l = l.With("receiver", r.groupName, "integration", r.integration.String()) - if groupKey, ok := GroupKey(ctx); ok { - l = l.With("aggrGroup", groupKey) - } - - for { - - // Always check the context first to not notify again. - select { - case <-ctx.Done(): - if iErr == nil { - iErr = ctx.Err() - if errors.Is(iErr, context.Canceled) { - iErr = NewErrorWithReason(ContextCanceledReason, iErr) - } else if errors.Is(iErr, context.DeadlineExceeded) { - iErr = NewErrorWithReason(ContextDeadlineExceededReason, iErr) - } - } - - if iErr != nil { - return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr) - } - return ctx, nil, nil - default: - } - - select { - case <-tick.C: - now := time.Now() - retry, err := r.integration.Notify(ctx, sent...) - i++ - dur := time.Since(now) - r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(dur.Seconds()) - r.metrics.numNotificationRequestsTotal.WithLabelValues(r.labelValues...).Inc() - if err != nil { - r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc() - if !retry { - return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err) - } - if ctx.Err() == nil { - if iErr == nil || err.Error() != iErr.Error() { - // Log the error if the context isn't done and the error isn't the same as before. - l.Warn("Notify attempt failed, will retry later", "attempts", i, "err", err) - } - // Save this error to be able to return the last seen error by an - // integration upon context timeout. - iErr = err - } - } else { - l := l.With( - "attempts", i, - "duration", dur, - "numAlerts", len(sent), - ) - if i <= 1 { - l.Debug("Notify success", "alerts", sent) - } else { - l.Info("Notify success") - } - - return ctx, alerts, nil - } - case <-ctx.Done(): - } - } +func utcNow() time.Time { + return time.Now().UTC() } -// SetNotifiesStage sets the notification information about passed alerts. The -// passed alerts should have already been sent to the receivers. -type SetNotifiesStage struct { - nflog NotificationLog - recv *nflogpb.Receiver +// Wrap a slice in a struct so we can store a pointer in sync.Pool. +type hashBuffer struct { + buf []byte } -// NewSetNotifiesStage returns a new instance of a SetNotifiesStage. -func NewSetNotifiesStage(l NotificationLog, recv *nflogpb.Receiver) *SetNotifiesStage { - return &SetNotifiesStage{ - nflog: l, - recv: recv, - } +var hashBuffers = sync.Pool{ + New: func() any { return &hashBuffer{buf: make([]byte, 0, 1024)} }, } -// Exec implements the Stage interface. -func (n SetNotifiesStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { - gkey, ok := GroupKey(ctx) - if !ok { - return ctx, nil, errors.New("group key missing") - } +func hashAlert(a *alert.Alert) uint64 { + const sep = '\xff' - ctx, span := tracer.Start(ctx, "notify.SetNotifiesStage.Exec", - trace.WithAttributes(attribute.String("alerting.group.key", gkey)), - trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), - trace.WithSpanKind(trace.SpanKindInternal), - ) - defer span.End() + hb := hashBuffers.Get().(*hashBuffer) + defer hashBuffers.Put(hb) + b := hb.buf[:0] - firing, ok := FiringAlerts(ctx) - if !ok { - return ctx, nil, errors.New("firing alerts missing") - } + names := make(model.LabelNames, 0, len(a.Labels)) - resolved, ok := ResolvedAlerts(ctx) - if !ok { - return ctx, nil, errors.New("resolved alerts missing") + for ln := range a.Labels { + names = append(names, ln) } + sort.Sort(names) - repeat, ok := RepeatInterval(ctx) - if !ok { - return ctx, nil, errors.New("repeat interval missing") + for _, ln := range names { + b = append(b, string(ln)...) + b = append(b, sep) + b = append(b, string(a.Labels[ln])...) + b = append(b, sep) } - expiry := 2 * repeat - span.SetAttributes( - attribute.Int("alerting.alerts.firing.count", len(firing)), - attribute.Int("alerting.alerts.resolved.count", len(resolved)), - ) + hash := xxhash.Sum64(b) - // Extract receiver data from context if present (it's ok for it to be nil). - store, _ := NflogStore(ctx) - return ctx, alerts, n.nflog.Log(n.recv, gkey, firing, resolved, store, expiry) + return hash } diff --git a/vendor/github.com/prometheus/alertmanager/notify/retry_stage.go b/vendor/github.com/prometheus/alertmanager/notify/retry_stage.go new file mode 100644 index 00000000000..7734ffe4f10 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/retry_stage.go @@ -0,0 +1,187 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/prometheus/common/model" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/eventrecorder" +) + +// RetryStage notifies via passed integration with exponential backoff until it +// succeeds. It aborts if the context is canceled or timed out. +type RetryStage struct { + integration Integration + groupName string + metrics *Metrics + labelValues []string + recorder eventrecorder.Recorder +} + +// NewRetryStage returns a new instance of a RetryStage. +func NewRetryStage(i Integration, groupName string, metrics *Metrics, recorder eventrecorder.Recorder) *RetryStage { + labelValues := []string{i.Name()} + + if metrics.ff.EnableReceiverNamesInMetrics() { + labelValues = append(labelValues, i.receiverName) + } + + return &RetryStage{ + integration: i, + groupName: groupName, + metrics: metrics, + labelValues: labelValues, + recorder: recorder, + } +} + +func (r RetryStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + r.metrics.numNotifications.WithLabelValues(r.labelValues...).Inc() + + ctx, span := tracer.Start(ctx, "notify.RetryStage.Exec", + trace.WithAttributes(attribute.String("alerting.group.name", r.groupName)), + trace.WithAttributes(attribute.String("alerting.integration.name", r.integration.name)), + trace.WithAttributes(attribute.StringSlice("alerting.label.values", r.labelValues)), + trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), + trace.WithSpanKind(trace.SpanKindInternal), + ) + defer span.End() + + ctx, alerts, err := r.exec(ctx, l, alerts...) + + failureReason := DefaultReason.String() + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + var e *ErrorWithReason + if errors.As(err, &e) { + failureReason = e.Reason.String() + } + r.metrics.numTotalFailedNotifications.WithLabelValues(append(r.labelValues, failureReason)...).Inc() + } + return ctx, alerts, err +} + +func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + var sent alert.AlertSlice + + // If we shouldn't send notifications for resolved alerts, but there are only + // resolved alerts, report them all as successfully notified (we still want the + // notification log to log them for the next run of DedupStage). + if !r.integration.SendResolved() { + firing, ok := FiringAlerts(ctx) + if !ok { + return ctx, nil, errors.New("firing alerts missing") + } + if len(firing) == 0 { + return ctx, alerts, nil + } + for _, a := range alerts { + if a.Status() != model.AlertResolved { + sent = append(sent, a) + } + } + } else { + sent = alerts + } + + b := backoff.NewExponentialBackOff() + b.MaxElapsedTime = 0 // Always retry. + + tick := backoff.NewTicker(b) + defer tick.Stop() + + var ( + i = 0 + iErr error + ) + + l = l.With("receiver", r.groupName, "integration", r.integration.String()) + if groupKey, ok := GroupKey(ctx); ok { + l = l.With("aggrGroup", groupKey) + } + + for { + + // Always check the context first to not notify again. + select { + case <-ctx.Done(): + if iErr == nil { + iErr = ctx.Err() + if errors.Is(iErr, context.Canceled) { + iErr = NewErrorWithReason(ContextCanceledReason, iErr) + } else if errors.Is(iErr, context.DeadlineExceeded) { + iErr = NewErrorWithReason(ContextDeadlineExceededReason, iErr) + } + } + + if iErr != nil { + return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr) + } + return ctx, nil, nil + default: + } + + select { + case <-tick.C: + now := time.Now() + retry, err := r.integration.Notify(ctx, sent...) + i++ + dur := time.Since(now) + r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(dur.Seconds()) + r.metrics.numNotificationRequestsTotal.WithLabelValues(r.labelValues...).Inc() + if err != nil { + r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc() + if !retry { + return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err) + } + if ctx.Err() == nil { + if iErr == nil || err.Error() != iErr.Error() { + // Log the error if the context isn't done and the error isn't the same as before. + l.Warn("Notify attempt failed, will retry later", "attempts", i, "err", err) + } + // Save this error to be able to return the last seen error by an + // integration upon context timeout. + iErr = err + } + } else { + l := l.With( + "attempts", i, + "duration", dur, + "numAlerts", len(sent), + ) + if i <= 1 { + l.Debug("Notify success", "alerts", sent) + } else { + l.Info("Notify success") + } + + r.recorder.RecordEvent(ctx, NewNotificationEvent(ctx, sent, r.integration)) + return ctx, alerts, nil + } + case <-ctx.Done(): + } + } +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/set_notifies_stage.go b/vendor/github.com/prometheus/alertmanager/notify/set_notifies_stage.go new file mode 100644 index 00000000000..57d501dee79 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/set_notifies_stage.go @@ -0,0 +1,80 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package notify + +import ( + "context" + "errors" + "log/slog" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/prometheus/alertmanager/alert" + "github.com/prometheus/alertmanager/nflog/nflogpb" +) + +// SetNotifiesStage sets the notification information about passed alerts. The +// passed alerts should have already been sent to the receivers. +type SetNotifiesStage struct { + nflog NotificationLog + recv *nflogpb.Receiver +} + +// NewSetNotifiesStage returns a new instance of a SetNotifiesStage. +func NewSetNotifiesStage(l NotificationLog, recv *nflogpb.Receiver) *SetNotifiesStage { + return &SetNotifiesStage{ + nflog: l, + recv: recv, + } +} + +// Exec implements the Stage interface. +func (n SetNotifiesStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) { + gkey, ok := GroupKey(ctx) + if !ok { + return ctx, nil, errors.New("group key missing") + } + + ctx, span := tracer.Start(ctx, "notify.SetNotifiesStage.Exec", + trace.WithAttributes(attribute.String("alerting.group.key", gkey)), + trace.WithAttributes(attribute.Int("alerting.alerts.count", len(alerts))), + trace.WithSpanKind(trace.SpanKindInternal), + ) + defer span.End() + + firing, ok := FiringAlerts(ctx) + if !ok { + return ctx, nil, errors.New("firing alerts missing") + } + + resolved, ok := ResolvedAlerts(ctx) + if !ok { + return ctx, nil, errors.New("resolved alerts missing") + } + + repeat, ok := RepeatInterval(ctx) + if !ok { + return ctx, nil, errors.New("repeat interval missing") + } + expiry := 2 * repeat + + span.SetAttributes( + attribute.Int("alerting.alerts.firing.count", len(firing)), + attribute.Int("alerting.alerts.resolved.count", len(resolved)), + ) + + // Extract receiver data from context if present (it's ok for it to be nil). + store, _ := NflogStore(ctx) + return ctx, alerts, n.nflog.Log(n.recv, gkey, firing, resolved, store, expiry) +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/slack/slack.go b/vendor/github.com/prometheus/alertmanager/notify/slack/slack.go index dbcea63dd7f..f96efe61fad 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/slack/slack.go +++ b/vendor/github.com/prometheus/alertmanager/notify/slack/slack.go @@ -36,17 +36,6 @@ import ( // https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters. const maxTitleLenRunes = 1024 -// Notifier implements a Notifier for Slack notifications. -type Notifier struct { - conf *config.SlackConfig - tmpl *template.Template - logger *slog.Logger - client *http.Client - retrier *notify.Retrier - - postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) -} - // New returns a new Slack notification handler. func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "slack", httpOpts...) @@ -64,43 +53,6 @@ func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts . }, nil } -// request is the request for sending a slack notification. -type request struct { - Channel string `json:"channel,omitempty"` - Timestamp string `json:"ts,omitempty"` - Username string `json:"username,omitempty"` - IconEmoji string `json:"icon_emoji,omitempty"` - IconURL string `json:"icon_url,omitempty"` - LinkNames bool `json:"link_names,omitempty"` - Text string `json:"text,omitempty"` - Attachments []attachment `json:"attachments"` -} - -// attachment is used to display a richly-formatted message block. -type attachment struct { - Title string `json:"title,omitempty"` - TitleLink string `json:"title_link,omitempty"` - Pretext string `json:"pretext,omitempty"` - Text string `json:"text"` - Fallback string `json:"fallback"` - CallbackID string `json:"callback_id"` - Fields []config.SlackField `json:"fields,omitempty"` - Actions []config.SlackAction `json:"actions,omitempty"` - ImageURL string `json:"image_url,omitempty"` - ThumbURL string `json:"thumb_url,omitempty"` - Footer string `json:"footer"` - Color string `json:"color,omitempty"` - MrkdwnIn []string `json:"mrkdwn_in,omitempty"` -} - -// slackResponse represents the response from Slack API. -type slackResponse struct { - OK bool `json:"ok"` - Error string `json:"error,omitempty"` - Channel string `json:"channel,omitempty"` - Timestamp string `json:"ts,omitempty"` -} - // Notify implements the Notifier interface. func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { var err error diff --git a/vendor/github.com/prometheus/alertmanager/notify/slack/types.go b/vendor/github.com/prometheus/alertmanager/notify/slack/types.go new file mode 100644 index 00000000000..a85734e1ad2 --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/slack/types.go @@ -0,0 +1,73 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package slack + +import ( + "context" + "io" + "log/slog" + "net/http" + + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/template" +) + +// Notifier implements a Notifier for Slack notifications. +type Notifier struct { + conf *config.SlackConfig + tmpl *template.Template + logger *slog.Logger + client *http.Client + retrier *notify.Retrier + + postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) +} + +// request is the request for sending a Slack notification. +type request struct { + Channel string `json:"channel,omitempty"` + Timestamp string `json:"ts,omitempty"` + Username string `json:"username,omitempty"` + IconEmoji string `json:"icon_emoji,omitempty"` + IconURL string `json:"icon_url,omitempty"` + LinkNames bool `json:"link_names,omitempty"` + Text string `json:"text,omitempty"` + Attachments []attachment `json:"attachments"` +} + +// attachment is used to display a richly formatted message block. +type attachment struct { + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Pretext string `json:"pretext,omitempty"` + Text string `json:"text"` + Fallback string `json:"fallback"` + CallbackID string `json:"callback_id"` + Fields []config.SlackField `json:"fields,omitempty"` + Actions []config.SlackAction `json:"actions,omitempty"` + ImageURL string `json:"image_url,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + Footer string `json:"footer"` + Color string `json:"color,omitempty"` + MrkdwnIn []string `json:"mrkdwn_in,omitempty"` +} + +// slackResponse represents the response from Slack API. +type slackResponse struct { + OK bool `json:"ok"` + Error string `json:"error,omitempty"` + Channel string `json:"channel,omitempty"` + Timestamp string `json:"ts,omitempty"` +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/sns/sns.go b/vendor/github.com/prometheus/alertmanager/notify/sns/sns.go index 80b039b8b1e..873e3328d1a 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/sns/sns.go +++ b/vendor/github.com/prometheus/alertmanager/notify/sns/sns.go @@ -19,6 +19,7 @@ import ( "fmt" "log/slog" "net/http" + "os" "strings" "unicode/utf8" @@ -32,6 +33,7 @@ import ( "github.com/aws/smithy-go" smithyhttp "github.com/aws/smithy-go/transport/http" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" commoncfg "github.com/prometheus/common/config" "github.com/prometheus/alertmanager/config" @@ -45,13 +47,13 @@ type Notifier struct { conf *config.SNSConfig tmpl *template.Template logger *slog.Logger - client *http.Client + client aws.HTTPClient retrier *notify.Retrier } // New returns a new SNS notification handler. func New(c *config.SNSConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { - client, err := notify.NewClientWithTracing(*c.HTTPConfig, "sns", httpOpts...) + client, err := newSNSHTTPClient(c, httpOpts...) if err != nil { return nil, err } @@ -64,6 +66,40 @@ func New(c *config.SNSConfig, t *template.Template, l *slog.Logger, httpOpts ... }, nil } +// newSNSHTTPClient picks the right HTTP client for the AWS SDK. The SDK's +// custom-CA-bundle path requires a *awshttp.BuildableClient (concrete type +// assertion in resolveCustomCABundle), so we use one when AWS_CA_BUNDLE is set +// or the user opts in. Otherwise we use the standard tracing-wrapped client. +// +// AWS_CA_BUNDLE is handled by the aws sdk's resolveCustomCABundle function, +// so we don't need to handle it manually when building the client. +func newSNSHTTPClient(c *config.SNSConfig, httpOpts ...commoncfg.HTTPClientOption) (aws.HTTPClient, error) { + if c.UseAWSHTTPClient || os.Getenv("AWS_CA_BUNDLE") != "" { + return newAWSBuildableClient(c) + } + return notify.NewClientWithTracing(*c.HTTPConfig, "sns", httpOpts...) +} + +// newAWSBuildableClient builds a BuildableClient pre-configured with the +// user's TLSConfig and proxy settings from HTTPClientConfig. Other +// HTTPClientConfig knobs (BasicAuth, OAuth2, FollowRedirects, EnableHTTP2, +// HTTPHeaders, etc.) are intentionally not propagated: AWS uses sigv4 so +// auth knobs are irrelevant, and the rest are managed by the SDK itself. +func newAWSBuildableClient(c *config.SNSConfig) (*awshttp.BuildableClient, error) { + tlsConfig, err := commoncfg.NewTLSConfig(&c.HTTPConfig.TLSConfig) + if err != nil { + return nil, fmt.Errorf("build SNS TLS config: %w", err) + } + proxyFn := c.HTTPConfig.Proxy() + return awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { + tr.TLSClientConfig = tlsConfig + if proxyFn != nil { + tr.Proxy = proxyFn + tr.ProxyConnectHeader = c.HTTPConfig.GetProxyConnectHeader() + } + }), nil +} + func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, error) { var ( tmplErr error diff --git a/vendor/github.com/prometheus/alertmanager/notify/webhook/config.go b/vendor/github.com/prometheus/alertmanager/notify/webhook/config.go new file mode 100644 index 00000000000..4a61042920c --- /dev/null +++ b/vendor/github.com/prometheus/alertmanager/notify/webhook/config.go @@ -0,0 +1,67 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "errors" + "time" + + commoncfg "github.com/prometheus/common/config" + + amcommoncfg "github.com/prometheus/alertmanager/config/common" +) + +// defaultWebhookConfig defines default values for Webhook configurations. +var defaultWebhookConfig = WebhookConfig{ + NotifierConfig: amcommoncfg.NotifierConfig{ + VSendResolved: true, + }, +} + +// WebhookConfig configures notifications via a generic webhook. +type WebhookConfig struct { + amcommoncfg.NotifierConfig `yaml:",inline" json:",inline"` + + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + + // URL to send POST request to. + URL amcommoncfg.SecretTemplateURL `yaml:"url,omitempty" json:"url,omitempty"` + URLFile string `yaml:"url_file" json:"url_file"` + + // MaxAlerts is the maximum number of alerts to be sent per webhook message. + // Alerts exceeding this threshold will be truncated. Setting this to 0 + // allows an unlimited number of alerts. + MaxAlerts uint64 `yaml:"max_alerts" json:"max_alerts"` + + // Timeout is the maximum time allowed to invoke the webhook. Setting this to 0 + // does not impose a timeout. + Timeout time.Duration `yaml:"timeout" json:"timeout"` + Payload any `yaml:"payload,omitempty" json:"payload,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *WebhookConfig) UnmarshalYAML(unmarshal func(any) error) error { + *c = defaultWebhookConfig + type plain WebhookConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + if c.URL == "" && c.URLFile == "" { + return errors.New("one of url or url_file must be configured") + } + if c.URL != "" && c.URLFile != "" { + return errors.New("at most one of url & url_file must be configured") + } + return nil +} diff --git a/vendor/github.com/prometheus/alertmanager/notify/webhook/webhook.go b/vendor/github.com/prometheus/alertmanager/notify/webhook/webhook.go index 39677bb193a..b0f4913ca75 100644 --- a/vendor/github.com/prometheus/alertmanager/notify/webhook/webhook.go +++ b/vendor/github.com/prometheus/alertmanager/notify/webhook/webhook.go @@ -26,7 +26,6 @@ import ( commoncfg "github.com/prometheus/common/config" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -34,7 +33,7 @@ import ( // Notifier implements a Notifier for generic webhooks. type Notifier struct { - conf *config.WebhookConfig + conf *WebhookConfig tmpl *template.Template logger *slog.Logger client *http.Client @@ -42,7 +41,7 @@ type Notifier struct { } // New returns a new Webhook. -func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { +func New(conf *WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*conf.HTTPConfig, "webhook", httpOpts...) if err != nil { return nil, err diff --git a/vendor/github.com/prometheus/alertmanager/provider/mem/mem.go b/vendor/github.com/prometheus/alertmanager/provider/mem/mem.go index ce26657900f..d4e662804b1 100644 --- a/vendor/github.com/prometheus/alertmanager/provider/mem/mem.go +++ b/vendor/github.com/prometheus/alertmanager/provider/mem/mem.go @@ -28,6 +28,7 @@ import ( "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" + "github.com/prometheus/alertmanager/eventrecorder" "github.com/prometheus/alertmanager/featurecontrol" "github.com/prometheus/alertmanager/provider" "github.com/prometheus/alertmanager/store" @@ -46,7 +47,6 @@ type Alerts struct { mtx sync.Mutex alerts *store.Alerts - marker types.AlertMarker listeners map[int]listeningAlerts next int @@ -55,6 +55,7 @@ type Alerts struct { logger *slog.Logger propagator propagation.TextMapPropagator + recorder eventrecorder.Recorder flagger featurecontrol.Flagger alertsLimit prometheus.Gauge @@ -86,8 +87,6 @@ type listeningAlerts struct { } func (a *Alerts) registerMetrics(r prometheus.Registerer) { - r.MustRegister(&alertsCollector{alerts: a}) - a.alertsLimit = promauto.With(r).NewGauge(prometheus.GaugeOpts{ Name: "alertmanager_alerts_per_alert_limit", Help: "Current limit on number of alerts per alert name", @@ -117,11 +116,11 @@ func (a *Alerts) registerMetrics(r prometheus.Registerer) { // NewAlerts returns a new alert provider. func NewAlerts( ctx context.Context, - m types.AlertMarker, intervalGC time.Duration, perAlertNameLimit int, alertCallback AlertStoreCallback, l *slog.Logger, + recorder eventrecorder.Recorder, r prometheus.Registerer, flagger featurecontrol.Flagger, ) (*Alerts, error) { @@ -139,13 +138,13 @@ func NewAlerts( ctx, cancel := context.WithCancel(ctx) a := &Alerts{ - marker: m, alerts: store.NewAlerts().WithPerAlertLimit(perAlertNameLimit), cancel: cancel, listeners: map[int]listeningAlerts{}, next: 0, logger: l.With("component", "provider"), propagator: otel.GetTextMapPropagator(), + recorder: recorder, callback: alertCallback, flagger: flagger, } @@ -186,13 +185,11 @@ func (a *Alerts) gc() { return } - // Delete markers for deleted alerts. ff := make(model.Fingerprints, len(deleted)) for i, alert := range deleted { ff[i] = alert.Fingerprint() a.callback.PostDelete(alert) } - a.marker.Delete(ff...) a.callback.PostGC(ff) } @@ -349,6 +346,10 @@ func (a *Alerts) Put(ctx context.Context, alerts ...*types.Alert) error { a.callback.PostStore(alert, existing) + if !existing { + a.recorder.RecordEvent(ctx, eventrecorder.NewAlertCreatedEvent(alert)) + } + metadata := map[string]string{} a.propagator.Inject(ctx, propagation.MapCarrier(metadata)) msg := &provider.Alert{ @@ -368,48 +369,6 @@ func (a *Alerts) Put(ctx context.Context, alerts ...*types.Alert) error { return nil } -// countByState returns the number of non-resolved alerts by state. -func (a *Alerts) countByState() (active, suppressed, unprocessed int) { - for _, alert := range a.alerts.List() { - if alert.Resolved() { - continue - } - - switch a.marker.Status(alert.Fingerprint()).State { - case types.AlertStateActive: - active++ - case types.AlertStateSuppressed: - suppressed++ - case types.AlertStateUnprocessed: - unprocessed++ - } - } - return active, suppressed, unprocessed -} - -// alertsCollector implements prometheus.Collector to collect all alert count metrics in a single pass. -type alertsCollector struct { - alerts *Alerts -} - -var alertsDesc = prometheus.NewDesc( - "alertmanager_alerts", - "How many alerts by state.", - []string{"state"}, nil, -) - -func (c *alertsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- alertsDesc -} - -func (c *alertsCollector) Collect(ch chan<- prometheus.Metric) { - active, suppressed, unprocessed := c.alerts.countByState() - - ch <- prometheus.MustNewConstMetric(alertsDesc, prometheus.GaugeValue, float64(active), string(types.AlertStateActive)) - ch <- prometheus.MustNewConstMetric(alertsDesc, prometheus.GaugeValue, float64(suppressed), string(types.AlertStateSuppressed)) - ch <- prometheus.MustNewConstMetric(alertsDesc, prometheus.GaugeValue, float64(unprocessed), string(types.AlertStateUnprocessed)) -} - type noopCallback struct{} func (n noopCallback) PreStore(_ *types.Alert, _ bool) error { return nil } diff --git a/vendor/github.com/prometheus/alertmanager/silence/silence.go b/vendor/github.com/prometheus/alertmanager/silence/silence.go index 8c6d07897b8..d2fd4f10eed 100644 --- a/vendor/github.com/prometheus/alertmanager/silence/silence.go +++ b/vendor/github.com/prometheus/alertmanager/silence/silence.go @@ -45,12 +45,14 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/prometheus/alertmanager/alert" "github.com/prometheus/alertmanager/cluster" + "github.com/prometheus/alertmanager/eventrecorder" + "github.com/prometheus/alertmanager/marker" "github.com/prometheus/alertmanager/matcher/compat" "github.com/prometheus/alertmanager/pkg/labels" pb "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/alertmanager/tracing" - "github.com/prometheus/alertmanager/types" ) var tracer = tracing.NewTracer("github.com/prometheus/alertmanager/silence") @@ -141,22 +143,21 @@ func (s versionIndex) findVersionGreaterThan(version int) (index int, found bool return startIdx, startIdx < len(s) } -// Silencer binds together a AlertMarker and a Silences to implement the Muter -// interface. +// Silencer holds Silences and implements the Muter interface. type Silencer struct { silences *Silences cache *cache - marker types.AlertMarker logger *slog.Logger + recorder eventrecorder.Recorder } // NewSilencer returns a new Silencer. -func NewSilencer(silences *Silences, marker types.AlertMarker, logger *slog.Logger) *Silencer { +func NewSilencer(silences *Silences, logger *slog.Logger, recorder eventrecorder.Recorder) *Silencer { return &Silencer{ silences: silences, cache: &cache{entries: map[model.Fingerprint]*cacheEntry{}}, - marker: marker, logger: logger, + recorder: recorder, } } @@ -171,6 +172,16 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { ) defer span.End() + // Track the silences to set on marker + var markedSilences []string + defer func() { + // Get the marker from context and set the silences on it if any. + m, ok := marker.FromContext(ctx) + if ok { + m.SetSilenced(fp, markedSilences) + } + }() + // Get the cached entry for this fingerprint. cachedEntry := s.cache.get(fp) @@ -243,7 +254,6 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { if totalSilences == 0 { // Easy case, neither active nor pending silences anymore. s.cache.set(fp, newCacheEntry(newVersion)) - s.marker.SetActiveOrSilenced(fp, nil) span.AddEvent("No silences to match", trace.WithAttributes( attribute.Int("alerting.silences.count", totalSilences), )) @@ -274,6 +284,10 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { case SilenceStateActive: activeIDs = append(activeIDs, sil.Id) allIDs = append(allIDs, sil.Id) + + s.recorder.RecordEvent(ctx, eventrecorder.NewSilenceMutedAlertEvent( + eventrecorder.SilenceAsProto(sil), fp, lset, + )) default: // Do nothing, silence has expired in the meantime. } @@ -286,11 +300,8 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { "active", len(activeIDs), "pending", len(allIDs)-len(activeIDs), ) - // TODO: remove this sort once the marker is removed. - sort.Strings(activeIDs) s.cache.set(fp, newCacheEntry(newVersion, allIDs...)) - s.marker.SetActiveOrSilenced(fp, activeIDs) t := trace.WithAttributes( attribute.Int("alerting.silences.active.count", len(activeIDs)), @@ -300,6 +311,7 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { mutes := len(activeIDs) > 0 if mutes { + markedSilences = activeIDs span.AddEvent("Silencer mutes alert", t) } else { span.AddEvent("Silencer does not mute alert", t) @@ -308,9 +320,9 @@ func (s *Silencer) Mutes(ctx context.Context, lset model.LabelSet) bool { } // The following methods implement mem.AlertStoreCallback. -func (s *Silencer) PreStore(_ *types.Alert, _ bool) error { return nil } -func (s *Silencer) PostStore(_ *types.Alert, _ bool) {} -func (s *Silencer) PostDelete(alert *types.Alert) {} +func (s *Silencer) PreStore(_ *alert.Alert, _ bool) error { return nil } +func (s *Silencer) PostStore(_ *alert.Alert, _ bool) {} +func (s *Silencer) PostDelete(alert *alert.Alert) {} func (s *Silencer) PostGC(ff model.Fingerprints) { for _, fp := range ff { s.cache.delete(fp) @@ -333,6 +345,7 @@ type Silences struct { broadcast func([]byte) mi matcherIndex vi versionIndex + recorder eventrecorder.Recorder } // Limits contains the limits for silences. @@ -494,6 +507,9 @@ type Options struct { // A logger used by background processing. Logger *slog.Logger Metrics prometheus.Registerer + + // EventRecorder records silence-related events to the event recorder. + EventRecorder eventrecorder.Recorder } func (o *Options) validate() error { @@ -519,6 +535,7 @@ func New(o Options) (*Silences, error) { logging: o.Logging, broadcast: func([]byte) {}, st: state{}, + recorder: o.EventRecorder, } if o.Metrics == nil { return nil, errors.New("Options.Metrics is nil") @@ -807,18 +824,20 @@ func (s *Silences) toMeshSilence(sil *pb.Silence) *pb.MeshSilence { } } -func (s *Silences) setSilence(msil *pb.MeshSilence, now time.Time) error { +func (s *Silences) setSilence(msil *pb.MeshSilence, now time.Time) (changed, added bool, err error) { b, err := marshalMeshSilence(msil) if err != nil { - return err + return false, false, err } - _, added := s.st.merge(msil, now) + changed, added = s.st.merge(msil, now) if added { s.indexSilence(msil.Silence) s.updateSizeMetrics() } - s.broadcast(b) - return nil + if changed { + s.broadcast(b) + } + return changed, added, nil } // Set the specified silence. If a silence with the ID already exists and the modification @@ -853,7 +872,16 @@ func (s *Silences) Set(ctx context.Context, sil *pb.Silence) error { if s.logging { s.logSilence("update silence", sil) } - return s.setSilence(msil, now) + changed, _, err := s.setSilence(msil, now) + if err != nil { + return err + } + if changed { + s.recorder.RecordEvent(ctx, eventrecorder.NewSilenceUpdatedEvent( + eventrecorder.SilenceAsProto(sil), + )) + } + return nil } // If we got here it's either a new silence or a replacing one (which would @@ -892,7 +920,16 @@ func (s *Silences) Set(ctx context.Context, sil *pb.Silence) error { if s.logging { s.logSilence("create silence", sil) } - return s.setSilence(msil, now) + _, added, err := s.setSilence(msil, now) + if err != nil { + return err + } + if added { + s.recorder.RecordEvent(ctx, eventrecorder.NewSilenceCreatedEvent( + eventrecorder.SilenceAsProto(sil), + )) + } + return nil } // canUpdate returns true if silence a can be updated to b without @@ -962,7 +999,8 @@ func (s *Silences) expire(id string) error { if s.logging { s.logSilence("expire silence", sil) } - return s.setSilence(s.toMeshSilence(sil), now) + _, _, err := s.setSilence(s.toMeshSilence(sil), now) + return err } // QueryParam expresses parameters along which silences are queried. diff --git a/vendor/github.com/prometheus/alertmanager/template/template.go b/vendor/github.com/prometheus/alertmanager/template/template.go index 587c3bbb469..73f1ac526ff 100644 --- a/vendor/github.com/prometheus/alertmanager/template/template.go +++ b/vendor/github.com/prometheus/alertmanager/template/template.go @@ -215,6 +215,7 @@ var DefaultFuncs = FuncMap{ } return t.In(loc), nil }, + "now": time.Now, "since": time.Since, "humanizeDuration": commonTemplates.HumanizeDuration, "toJson": func(v any) (string, error) { diff --git a/vendor/github.com/prometheus/alertmanager/tracing/tracing.go b/vendor/github.com/prometheus/alertmanager/tracing/tracing.go index 6c1fd0a7a97..77a86842671 100644 --- a/vendor/github.com/prometheus/alertmanager/tracing/tracing.go +++ b/vendor/github.com/prometheus/alertmanager/tracing/tracing.go @@ -31,7 +31,7 @@ import ( "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.39.0" + semconv "go.opentelemetry.io/otel/semconv/v1.40.0" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" "google.golang.org/grpc/credentials" diff --git a/vendor/github.com/prometheus/alertmanager/types/types.go b/vendor/github.com/prometheus/alertmanager/types/types.go index b03be09c025..cd1f87f70a9 100644 --- a/vendor/github.com/prometheus/alertmanager/types/types.go +++ b/vendor/github.com/prometheus/alertmanager/types/types.go @@ -14,11 +14,6 @@ package types import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/alertmanager/alert" ) @@ -30,270 +25,3 @@ type AlertSlice = alert.AlertSlice // Deprecated: Use alert.Alerts directly. var Alerts = alert.Alerts - -// Deprecated: Use alert.AlertState constants directly. -type AlertState = alert.AlertState - -// Deprecated: Use alert.AlertStateActive directly. -const AlertStateActive AlertState = alert.AlertStateActive - -// Deprecated: Use alert.AlertStateSuppressed directly. -const AlertStateSuppressed AlertState = alert.AlertStateSuppressed - -// Deprecated: Use alert.AlertStateUnprocessed directly. -const AlertStateUnprocessed AlertState = alert.AlertStateUnprocessed - -// Deprecated: Use alert.AlertStatus directly. -type AlertStatus = alert.AlertStatus - -// groupStatus stores the state of the group, and, as applicable, the names -// of all active and mute time intervals that are muting it. -type groupStatus struct { - // mutedBy contains the names of all active and mute time intervals that - // are muting it. - mutedBy []string -} - -// AlertMarker helps to mark alerts as silenced and/or inhibited. -// All methods are goroutine-safe. -type AlertMarker interface { - // SetActiveOrSilenced replaces the previous SilencedBy by the provided IDs of - // active silences. The set of provided IDs is supposed to represent the - // complete set of relevant silences. If no active silence IDs are provided and - // InhibitedBy is already empty, it sets the provided alert to AlertStateActive. - // Otherwise, it sets the provided alert to AlertStateSuppressed. - SetActiveOrSilenced(alert model.Fingerprint, activeSilenceIDs []string) - // SetInhibited replaces the previous InhibitedBy by the provided IDs of - // alerts. In contrast to SetActiveOrSilenced, the set of provided IDs is not - // expected to represent the complete set of inhibiting alerts. (In - // practice, this method is only called with one or zero IDs. However, - // this expectation might change in the future. If no IDs are provided - // and InhibitedBy is already empty, it sets the provided alert to - // AlertStateActive. Otherwise, it sets the provided alert to - // AlertStateSuppressed. - SetInhibited(alert model.Fingerprint, alertIDs ...string) - - // Count alerts of the given state(s). With no state provided, count all - // alerts. - Count(...AlertState) int - - // Status of the given alert. - Status(model.Fingerprint) AlertStatus - // Delete the given alert. - Delete(...model.Fingerprint) - - // Various methods to inquire if the given alert is in a certain - // AlertState. Silenced also returns all the active silences, - // while Inhibited may return only a subset of inhibiting alerts. - Unprocessed(model.Fingerprint) bool - Active(model.Fingerprint) bool - Silenced(model.Fingerprint) (activeIDs []string, silenced bool) - Inhibited(model.Fingerprint) ([]string, bool) -} - -// GroupMarker helps to mark groups as active or muted. -// All methods are goroutine-safe. -// -// TODO(grobinson): routeID is used in Muted and SetMuted because groupKey -// is not unique (see #3817). Once groupKey uniqueness is fixed routeID can -// be removed from the GroupMarker interface. -type GroupMarker interface { - // Muted returns true if the group is muted, otherwise false. If the group - // is muted then it also returns the names of the time intervals that muted - // it. - Muted(routeID, groupKey string) ([]string, bool) - - // SetMuted marks the group as muted, and sets the names of the time - // intervals that mute it. If the list of names is nil or the empty slice - // then the muted marker is removed. - SetMuted(routeID, groupKey string, timeIntervalNames []string) - - // DeleteByGroupKey removes all markers for the GroupKey. - DeleteByGroupKey(routeID, groupKey string) -} - -// NewMarker returns an instance of a AlertMarker implementation. -func NewMarker(r prometheus.Registerer) *MemMarker { - m := &MemMarker{ - alerts: map[model.Fingerprint]*AlertStatus{}, - groups: map[string]*groupStatus{}, - } - m.registerMetrics(r) - return m -} - -type MemMarker struct { - alerts map[model.Fingerprint]*AlertStatus - groups map[string]*groupStatus - - mtx sync.RWMutex -} - -// Muted implements GroupMarker. -func (m *MemMarker) Muted(routeID, groupKey string) ([]string, bool) { - m.mtx.Lock() - defer m.mtx.Unlock() - status, ok := m.groups[routeID+groupKey] - if !ok { - return nil, false - } - return status.mutedBy, len(status.mutedBy) > 0 -} - -// SetMuted implements GroupMarker. -func (m *MemMarker) SetMuted(routeID, groupKey string, timeIntervalNames []string) { - m.mtx.Lock() - defer m.mtx.Unlock() - status, ok := m.groups[routeID+groupKey] - if !ok { - status = &groupStatus{} - m.groups[routeID+groupKey] = status - } - status.mutedBy = timeIntervalNames -} - -func (m *MemMarker) DeleteByGroupKey(routeID, groupKey string) { - m.mtx.Lock() - defer m.mtx.Unlock() - delete(m.groups, routeID+groupKey) -} - -func (m *MemMarker) registerMetrics(r prometheus.Registerer) { - newMarkedAlertMetricByState := func(st AlertState) prometheus.GaugeFunc { - return prometheus.NewGaugeFunc( - prometheus.GaugeOpts{ - Name: "alertmanager_marked_alerts", - Help: "How many alerts by state are currently marked in the Alertmanager regardless of their expiry.", - ConstLabels: prometheus.Labels{"state": string(st)}, - }, - func() float64 { - return float64(m.Count(st)) - }, - ) - } - - alertsActive := newMarkedAlertMetricByState(AlertStateActive) - alertsSuppressed := newMarkedAlertMetricByState(AlertStateSuppressed) - alertStateUnprocessed := newMarkedAlertMetricByState(AlertStateUnprocessed) - - r.MustRegister(alertsActive) - r.MustRegister(alertsSuppressed) - r.MustRegister(alertStateUnprocessed) -} - -// Count implements AlertMarker. -func (m *MemMarker) Count(states ...AlertState) int { - m.mtx.RLock() - defer m.mtx.RUnlock() - - if len(states) == 0 { - return len(m.alerts) - } - - var count int - for _, status := range m.alerts { - for _, state := range states { - if status.State == state { - count++ - } - } - } - return count -} - -// SetActiveOrSilenced implements AlertMarker. -func (m *MemMarker) SetActiveOrSilenced(alert model.Fingerprint, activeIDs []string) { - m.mtx.Lock() - defer m.mtx.Unlock() - - s, found := m.alerts[alert] - if !found { - s = &AlertStatus{} - m.alerts[alert] = s - } - s.SilencedBy = activeIDs - - // If there are any silence or alert IDs associated with the - // fingerprint, it is suppressed. Otherwise, set it to - // AlertStateActive. - if len(activeIDs) == 0 && len(s.InhibitedBy) == 0 { - s.State = AlertStateActive - return - } - - s.State = AlertStateSuppressed -} - -// SetInhibited implements AlertMarker. -func (m *MemMarker) SetInhibited(alert model.Fingerprint, ids ...string) { - m.mtx.Lock() - defer m.mtx.Unlock() - - s, found := m.alerts[alert] - if !found { - s = &AlertStatus{} - m.alerts[alert] = s - } - s.InhibitedBy = ids - - // If there are any silence or alert IDs associated with the - // fingerprint, it is suppressed. Otherwise, set it to - // AlertStateActive. - if len(ids) == 0 && len(s.SilencedBy) == 0 { - s.State = AlertStateActive - return - } - - s.State = AlertStateSuppressed -} - -// Status implements AlertMarker. -func (m *MemMarker) Status(alert model.Fingerprint) AlertStatus { - m.mtx.RLock() - defer m.mtx.RUnlock() - - if s, found := m.alerts[alert]; found { - return *s - } - return AlertStatus{ - State: AlertStateUnprocessed, - SilencedBy: []string{}, - InhibitedBy: []string{}, - } -} - -// Delete implements AlertMarker. -func (m *MemMarker) Delete(alerts ...model.Fingerprint) { - m.mtx.Lock() - defer m.mtx.Unlock() - - for _, alert := range alerts { - delete(m.alerts, alert) - } -} - -// Unprocessed implements AlertMarker. -func (m *MemMarker) Unprocessed(alert model.Fingerprint) bool { - return m.Status(alert).State == AlertStateUnprocessed -} - -// Active implements AlertMarker. -func (m *MemMarker) Active(alert model.Fingerprint) bool { - return m.Status(alert).State == AlertStateActive -} - -// Inhibited implements AlertMarker. -func (m *MemMarker) Inhibited(alert model.Fingerprint) ([]string, bool) { - s := m.Status(alert) - return s.InhibitedBy, - s.State == AlertStateSuppressed && len(s.InhibitedBy) > 0 -} - -// Silenced returns whether the alert for the given Fingerprint is in the -// Silenced state, any associated silence IDs, and the silences state version -// the result is based on. -func (m *MemMarker) Silenced(alert model.Fingerprint) (activeIDs []string, silenced bool) { - s := m.Status(alert) - return s.SilencedBy, - s.State == AlertStateSuppressed && len(s.SilencedBy) > 0 -} diff --git a/vendor/github.com/twmb/franz-go/LICENSE b/vendor/github.com/twmb/franz-go/LICENSE new file mode 100644 index 00000000000..36e18034325 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/LICENSE @@ -0,0 +1,24 @@ +Copyright 2020, Travis Bischel. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/twmb/franz-go/pkg/kbin/primitives.go b/vendor/github.com/twmb/franz-go/pkg/kbin/primitives.go new file mode 100644 index 00000000000..487e7f6c2a3 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kbin/primitives.go @@ -0,0 +1,856 @@ +// Package kbin contains Kafka primitive reading and writing functions. +package kbin + +import ( + "encoding/binary" + "errors" + "math" + "math/bits" + "reflect" + "unsafe" +) + +// This file contains primitive type encoding and decoding. +// +// The Reader helper can be used even when content runs out +// or an error is hit; all other number requests will return +// zero so a decode will basically no-op. + +// ErrNotEnoughData is returned when a type could not fully decode +// from a slice because the slice did not have enough data. +var ErrNotEnoughData = errors.New("response did not contain enough data to be valid") + +// AppendBool appends 1 for true or 0 for false to dst. +func AppendBool(dst []byte, v bool) []byte { + if v { + return append(dst, 1) + } + return append(dst, 0) +} + +// AppendInt8 appends an int8 to dst. +func AppendInt8(dst []byte, i int8) []byte { + return append(dst, byte(i)) +} + +// AppendInt16 appends a big endian int16 to dst. +func AppendInt16(dst []byte, i int16) []byte { + return AppendUint16(dst, uint16(i)) +} + +// AppendUint16 appends a big endian uint16 to dst. +func AppendUint16(dst []byte, u uint16) []byte { + return append(dst, byte(u>>8), byte(u)) +} + +// AppendInt32 appends a big endian int32 to dst. +func AppendInt32(dst []byte, i int32) []byte { + return AppendUint32(dst, uint32(i)) +} + +// AppendInt64 appends a big endian int64 to dst. +func AppendInt64(dst []byte, i int64) []byte { + return appendUint64(dst, uint64(i)) +} + +// AppendFloat64 appends a big endian float64 to dst. +func AppendFloat64(dst []byte, f float64) []byte { + return appendUint64(dst, math.Float64bits(f)) +} + +// AppendUuid appends the 16 uuid bytes to dst. +func AppendUuid(dst []byte, uuid [16]byte) []byte { + return append(dst, uuid[:]...) +} + +func appendUint64(dst []byte, u uint64) []byte { + return append(dst, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), + byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +// AppendUint32 appends a big endian uint32 to dst. +func AppendUint32(dst []byte, u uint32) []byte { + return append(dst, byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +// uvarintLens could only be length 65, but using 256 allows bounds check +// elimination on lookup. +const uvarintLens = "\x01\x01\x01\x01\x01\x01\x01\x01\x02\x02\x02\x02\x02\x02\x02\x03\x03\x03\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04\x04\x05\x05\x05\x05\x05\x05\x05\x06\x06\x06\x06\x06\x06\x06\x07\x07\x07\x07\x07\x07\x07\x08\x08\x08\x08\x08\x08\x08\x09\x09\x09\x09\x09\x09\x09\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +// VarintLen returns how long i would be if it were varint encoded. +func VarintLen(i int32) int { + u := uint32(i)<<1 ^ uint32(i>>31) + return UvarintLen(u) +} + +// UvarintLen returns how long u would be if it were uvarint encoded. +func UvarintLen(u uint32) int { + return int(uvarintLens[byte(bits.Len32(u))]) +} + +// VarlongLen returns how long i would be if it were varlong encoded. +func VarlongLen(i int64) int { + u := uint64(i)<<1 ^ uint64(i>>63) + return uvarlongLen(u) +} + +func uvarlongLen(u uint64) int { + return int(uvarintLens[byte(bits.Len64(u))]) +} + +// Varint is a loop unrolled 32 bit varint decoder. The return semantics +// are the same as binary.Varint, with the added benefit that overflows +// in 5 byte encodings are handled rather than left to the user. +func Varint(in []byte) (int32, int) { + x, n := Uvarint(in) + return int32((x >> 1) ^ -(x & 1)), n +} + +// Uvarint is a loop unrolled 32 bit uvarint decoder. The return semantics +// are the same as binary.Uvarint, with the added benefit that overflows +// in 5 byte encodings are handled rather than left to the user. +func Uvarint(in []byte) (uint32, int) { + var x uint32 + var overflow int + + if len(in) < 1 { + goto fail + } + + x = uint32(in[0] & 0x7f) + if in[0]&0x80 == 0 { + return x, 1 + } else if len(in) < 2 { + goto fail + } + + x |= uint32(in[1]&0x7f) << 7 + if in[1]&0x80 == 0 { + return x, 2 + } else if len(in) < 3 { + goto fail + } + + x |= uint32(in[2]&0x7f) << 14 + if in[2]&0x80 == 0 { + return x, 3 + } else if len(in) < 4 { + goto fail + } + + x |= uint32(in[3]&0x7f) << 21 + if in[3]&0x80 == 0 { + return x, 4 + } else if len(in) < 5 { + goto fail + } + + x |= uint32(in[4]) << 28 + if in[4] <= 0x0f { + return x, 5 + } + + overflow = -5 + +fail: + return 0, overflow +} + +// Varlong is a loop unrolled 64 bit varint decoder. The return semantics +// are the same as binary.Varint, with the added benefit that overflows +// in 10 byte encodings are handled rather than left to the user. +func Varlong(in []byte) (int64, int) { + x, n := uvarlong(in) + return int64((x >> 1) ^ -(x & 1)), n +} + +func uvarlong(in []byte) (uint64, int) { + var x uint64 + var overflow int + + if len(in) < 1 { + goto fail + } + + x = uint64(in[0] & 0x7f) + if in[0]&0x80 == 0 { + return x, 1 + } else if len(in) < 2 { + goto fail + } + + x |= uint64(in[1]&0x7f) << 7 + if in[1]&0x80 == 0 { + return x, 2 + } else if len(in) < 3 { + goto fail + } + + x |= uint64(in[2]&0x7f) << 14 + if in[2]&0x80 == 0 { + return x, 3 + } else if len(in) < 4 { + goto fail + } + + x |= uint64(in[3]&0x7f) << 21 + if in[3]&0x80 == 0 { + return x, 4 + } else if len(in) < 5 { + goto fail + } + + x |= uint64(in[4]&0x7f) << 28 + if in[4]&0x80 == 0 { + return x, 5 + } else if len(in) < 6 { + goto fail + } + + x |= uint64(in[5]&0x7f) << 35 + if in[5]&0x80 == 0 { + return x, 6 + } else if len(in) < 7 { + goto fail + } + + x |= uint64(in[6]&0x7f) << 42 + if in[6]&0x80 == 0 { + return x, 7 + } else if len(in) < 8 { + goto fail + } + + x |= uint64(in[7]&0x7f) << 49 + if in[7]&0x80 == 0 { + return x, 8 + } else if len(in) < 9 { + goto fail + } + + x |= uint64(in[8]&0x7f) << 56 + if in[8]&0x80 == 0 { + return x, 9 + } else if len(in) < 10 { + goto fail + } + + x |= uint64(in[9]) << 63 + if in[9] <= 0x01 { + return x, 10 + } + + overflow = -10 + +fail: + return 0, overflow +} + +// AppendVarint appends a varint encoded i to dst. +func AppendVarint(dst []byte, i int32) []byte { + return AppendUvarint(dst, uint32(i)<<1^uint32(i>>31)) +} + +// AppendUvarint appends a uvarint encoded u to dst. +func AppendUvarint(dst []byte, u uint32) []byte { + switch UvarintLen(u) { + case 5: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte(u>>28)) + case 4: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte(u>>21)) + case 3: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte(u>>14)) + case 2: + return append(dst, + byte(u&0x7f|0x80), + byte(u>>7)) + case 1: + return append(dst, byte(u)) + } + return dst +} + +// AppendVarlong appends a varint encoded i to dst. +func AppendVarlong(dst []byte, i int64) []byte { + return appendUvarlong(dst, uint64(i)<<1^uint64(i>>63)) +} + +func appendUvarlong(dst []byte, u uint64) []byte { + switch uvarlongLen(u) { + case 10: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte((u>>49)&0x7f|0x80), + byte((u>>56)&0x7f|0x80), + byte(u>>63)) + case 9: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte((u>>49)&0x7f|0x80), + byte(u>>56)) + case 8: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte(u>>49)) + case 7: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte(u>>42)) + case 6: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte(u>>35)) + case 5: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte(u>>28)) + case 4: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte(u>>21)) + case 3: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte(u>>14)) + case 2: + return append(dst, + byte(u&0x7f|0x80), + byte(u>>7)) + case 1: + return append(dst, byte(u)) + } + return dst +} + +// AppendString appends a string to dst prefixed with its int16 length. +func AppendString(dst []byte, s string) []byte { + dst = AppendInt16(dst, int16(len(s))) + return append(dst, s...) +} + +// AppendCompactString appends a string to dst prefixed with its uvarint length +// starting at 1; 0 is reserved for null, which compact strings are not +// (nullable compact ones are!). Thus, the length is the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactString(dst []byte, s string) []byte { + dst = AppendUvarint(dst, 1+uint32(len(s))) + return append(dst, s...) +} + +// AppendNullableString appends potentially nil string to dst prefixed with its +// int16 length or int16(-1) if nil. +func AppendNullableString(dst []byte, s *string) []byte { + if s == nil { + return AppendInt16(dst, -1) + } + return AppendString(dst, *s) +} + +// AppendCompactNullableString appends a potentially nil string to dst with its +// uvarint length starting at 1, with 0 indicating null. Thus, the length is +// the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactNullableString(dst []byte, s *string) []byte { + if s == nil { + return AppendUvarint(dst, 0) + } + return AppendCompactString(dst, *s) +} + +// AppendBytes appends bytes to dst prefixed with its int32 length. +func AppendBytes(dst, b []byte) []byte { + dst = AppendInt32(dst, int32(len(b))) + return append(dst, b...) +} + +// AppendCompactBytes appends bytes to dst prefixed with a its uvarint length +// starting at 1; 0 is reserved for null, which compact bytes are not (nullable +// compact ones are!). Thus, the length is the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactBytes(dst, b []byte) []byte { + dst = AppendUvarint(dst, 1+uint32(len(b))) + return append(dst, b...) +} + +// AppendNullableBytes appends a potentially nil slice to dst prefixed with its +// int32 length or int32(-1) if nil. +func AppendNullableBytes(dst, b []byte) []byte { + if b == nil { + return AppendInt32(dst, -1) + } + return AppendBytes(dst, b) +} + +// AppendCompactNullableBytes appends a potentially nil slice to dst with its +// uvarint length starting at 1, with 0 indicating null. Thus, the length is +// the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactNullableBytes(dst, b []byte) []byte { + if b == nil { + return AppendUvarint(dst, 0) + } + return AppendCompactBytes(dst, b) +} + +// AppendVarintString appends a string to dst prefixed with its length encoded +// as a varint. +func AppendVarintString(dst []byte, s string) []byte { + dst = AppendVarint(dst, int32(len(s))) + return append(dst, s...) +} + +// AppendVarintBytes appends a slice to dst prefixed with its length encoded as +// a varint. +func AppendVarintBytes(dst, b []byte) []byte { + if b == nil { + return AppendVarint(dst, -1) + } + dst = AppendVarint(dst, int32(len(b))) + return append(dst, b...) +} + +// AppendArrayLen appends the length of an array as an int32 to dst. +func AppendArrayLen(dst []byte, l int) []byte { + return AppendInt32(dst, int32(l)) +} + +// AppendCompactArrayLen appends the length of an array as a uvarint to dst +// as the length + 1. +// +// For KIP-482. +func AppendCompactArrayLen(dst []byte, l int) []byte { + return AppendUvarint(dst, 1+uint32(l)) +} + +// AppendNullableArrayLen appends the length of an array as an int32 to dst, +// or -1 if isNil is true. +func AppendNullableArrayLen(dst []byte, l int, isNil bool) []byte { + if isNil { + return AppendInt32(dst, -1) + } + return AppendInt32(dst, int32(l)) +} + +// AppendCompactNullableArrayLen appends the length of an array as a uvarint to +// dst as the length + 1; if isNil is true, this appends 0 as a uvarint. +// +// For KIP-482. +func AppendCompactNullableArrayLen(dst []byte, l int, isNil bool) []byte { + if isNil { + return AppendUvarint(dst, 0) + } + return AppendUvarint(dst, 1+uint32(l)) +} + +// Reader is used to decode Kafka messages. +// +// For all functions on Reader, if the reader has been invalidated, functions +// return defaults (false, 0, nil, ""). Use Complete to detect if the reader +// was invalidated or if the reader has remaining data. +type Reader struct { + Src []byte + bad bool +} + +// Bool returns a bool from the reader. +func (b *Reader) Bool() bool { + if len(b.Src) < 1 { + b.bad = true + b.Src = nil + return false + } + t := b.Src[0] != 0 // if '0', false + b.Src = b.Src[1:] + return t +} + +// Int8 returns an int8 from the reader. +func (b *Reader) Int8() int8 { + if len(b.Src) < 1 { + b.bad = true + b.Src = nil + return 0 + } + r := b.Src[0] + b.Src = b.Src[1:] + return int8(r) +} + +// Int16 returns an int16 from the reader. +func (b *Reader) Int16() int16 { + if len(b.Src) < 2 { + b.bad = true + b.Src = nil + return 0 + } + r := int16(binary.BigEndian.Uint16(b.Src)) + b.Src = b.Src[2:] + return r +} + +// Uint16 returns an uint16 from the reader. +func (b *Reader) Uint16() uint16 { + if len(b.Src) < 2 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint16(b.Src) + b.Src = b.Src[2:] + return r +} + +// Int32 returns an int32 from the reader. +func (b *Reader) Int32() int32 { + if len(b.Src) < 4 { + b.bad = true + b.Src = nil + return 0 + } + r := int32(binary.BigEndian.Uint32(b.Src)) + b.Src = b.Src[4:] + return r +} + +// Int64 returns an int64 from the reader. +func (b *Reader) Int64() int64 { + return int64(b.readUint64()) +} + +// Uuid returns a uuid from the reader. +func (b *Reader) Uuid() [16]byte { + var r [16]byte + copy(r[:], b.Span(16)) + return r +} + +// Float64 returns a float64 from the reader. +func (b *Reader) Float64() float64 { + return math.Float64frombits(b.readUint64()) +} + +func (b *Reader) readUint64() uint64 { + if len(b.Src) < 8 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint64(b.Src) + b.Src = b.Src[8:] + return r +} + +// Uint32 returns a uint32 from the reader. +func (b *Reader) Uint32() uint32 { + if len(b.Src) < 4 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint32(b.Src) + b.Src = b.Src[4:] + return r +} + +// Varint returns a varint int32 from the reader. +func (b *Reader) Varint() int32 { + val, n := Varint(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Varlong returns a varlong int64 from the reader. +func (b *Reader) Varlong() int64 { + val, n := Varlong(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Uvarint returns a uvarint encoded uint32 from the reader. +func (b *Reader) Uvarint() uint32 { + val, n := Uvarint(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Span returns l bytes from the reader. +func (b *Reader) Span(l int) []byte { + if len(b.Src) < l || l < 0 { + b.bad = true + b.Src = nil + return nil + } + r := b.Src[:l:l] + b.Src = b.Src[l:] + return r +} + +// UnsafeString returns a Kafka string from the reader without allocating using +// the unsafe package. This must be used with care; note the string holds a +// reference to the original slice. +func (b *Reader) UnsafeString() string { + l := b.Int16() + return UnsafeString(b.Span(int(l))) +} + +// String returns a Kafka string from the reader. +func (b *Reader) String() string { + l := b.Int16() + return string(b.Span(int(l))) +} + +// UnsafeCompactString returns a Kafka compact string from the reader without +// allocating using the unsafe package. This must be used with care; note the +// string holds a reference to the original slice. +func (b *Reader) UnsafeCompactString() string { + l := int(b.Uvarint()) - 1 + return UnsafeString(b.Span(l)) +} + +// CompactString returns a Kafka compact string from the reader. +func (b *Reader) CompactString() string { + l := int(b.Uvarint()) - 1 + return string(b.Span(l)) +} + +// UnsafeNullableString returns a Kafka nullable string from the reader without +// allocating using the unsafe package. This must be used with care; note the +// string holds a reference to the original slice. +func (b *Reader) UnsafeNullableString() *string { + l := b.Int16() + if l < 0 { + return nil + } + s := UnsafeString(b.Span(int(l))) + return &s +} + +// NullableString returns a Kafka nullable string from the reader. +func (b *Reader) NullableString() *string { + l := b.Int16() + if l < 0 { + return nil + } + s := string(b.Span(int(l))) + return &s +} + +// UnsafeCompactNullableString returns a Kafka compact nullable string from the +// reader without allocating using the unsafe package. This must be used with +// care; note the string holds a reference to the original slice. +func (b *Reader) UnsafeCompactNullableString() *string { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + s := UnsafeString(b.Span(l)) + return &s +} + +// CompactNullableString returns a Kafka compact nullable string from the +// reader. +func (b *Reader) CompactNullableString() *string { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + s := string(b.Span(l)) + return &s +} + +// Bytes returns a Kafka byte array from the reader. +// +// This never returns nil. +func (b *Reader) Bytes() []byte { + l := b.Int32() + // This is not to spec, but it is not clearly documented and Microsoft + // EventHubs fails here. -1 means null, which should throw an + // exception. EventHubs uses -1 to mean "does not exist" on some + // non-nullable fields. + // + // Until EventHubs is fixed, we return an empty byte slice for null. + if l == -1 { + return []byte{} + } + return b.Span(int(l)) +} + +// CompactBytes returns a Kafka compact byte array from the reader. +// +// This never returns nil. +func (b *Reader) CompactBytes() []byte { + l := int(b.Uvarint()) - 1 + if l == -1 { // same as above: -1 should not be allowed here + return []byte{} + } + return b.Span(l) +} + +// NullableBytes returns a Kafka nullable byte array from the reader, returning +// nil as appropriate. +func (b *Reader) NullableBytes() []byte { + l := b.Int32() + if l < 0 { + return nil + } + r := b.Span(int(l)) + return r +} + +// CompactNullableBytes returns a Kafka compact nullable byte array from the +// reader, returning nil as appropriate. +func (b *Reader) CompactNullableBytes() []byte { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + r := b.Span(l) + return r +} + +// ArrayLen returns a Kafka array length from the reader. +func (b *Reader) ArrayLen() int32 { + r := b.Int32() + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// VarintArrayLen returns a Kafka array length from the reader. +func (b *Reader) VarintArrayLen() int32 { + r := b.Varint() + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// CompactArrayLen returns a Kafka compact array length from the reader. +func (b *Reader) CompactArrayLen() int32 { + r := int32(b.Uvarint()) - 1 + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// VarintBytes returns a Kafka encoded varint array from the reader, returning +// nil as appropriate. +func (b *Reader) VarintBytes() []byte { + l := b.Varint() + if l < 0 { + return nil + } + return b.Span(int(l)) +} + +// UnsafeVarintString returns a Kafka encoded varint string from the reader +// without allocating using the unsafe package. This must be used with care; +// note the string holds a reference to the original slice. +func (b *Reader) UnsafeVarintString() string { + return UnsafeString(b.VarintBytes()) +} + +// VarintString returns a Kafka encoded varint string from the reader. +func (b *Reader) VarintString() string { + return string(b.VarintBytes()) +} + +// Complete returns ErrNotEnoughData if the source ran out while decoding. +func (b *Reader) Complete() error { + if b.bad { + return ErrNotEnoughData + } + return nil +} + +// Ok returns true if the reader is still ok. +func (b *Reader) Ok() bool { + return !b.bad +} + +// UnsafeString returns the slice as a string using unsafe rule (6). +func UnsafeString(slice []byte) string { + var str string + strhdr := (*reflect.StringHeader)(unsafe.Pointer(&str)) //nolint:gosec // known way to convert slice to string + strhdr.Data = ((*reflect.SliceHeader)(unsafe.Pointer(&slice))).Data //nolint:gosec // known way to convert slice to string + strhdr.Len = len(slice) + return str +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kerr/kerr.go b/vendor/github.com/twmb/franz-go/pkg/kerr/kerr.go new file mode 100644 index 00000000000..035531a2612 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kerr/kerr.go @@ -0,0 +1,348 @@ +// Package kerr contains Kafka errors. +// +// The errors are undocumented to avoid duplicating the official descriptions +// that can be found at https://kafka.apache.org/protocol.html#protocol_error_codes (although, +// this code does duplicate the descriptions into the errors themselves, so the +// descriptions can be seen as the documentation). +// +// Since this package is dedicated to errors and the package is named "kerr", +// all errors elide the standard "Err" prefix. +package kerr + +import ( + "errors" + "fmt" +) + +// Error is a Kafka error. +type Error struct { + // Message is the string form of a Kafka error code + // (UNKNOWN_SERVER_ERROR, etc). + Message string + // Code is a Kafka error code. + Code int16 + // Retriable is whether the error is considered retriable by Kafka. + Retriable bool + // Description is a succinct description of what this error means. + Description string +} + +func (e *Error) Error() string { + return fmt.Sprintf("%s: %s", e.Message, e.Description) +} + +// ErrorForCode returns the error corresponding to the given error code. +// +// If the code is unknown, this returns UnknownServerError. +// If the code is 0, this returns nil. +func ErrorForCode(code int16) error { + err, exists := code2err[code] + if !exists { + return UnknownServerError + } + return err +} + +// TypedErrorForCode returns the kerr.Error corresponding to the given error +// code. +// +// If the code is unknown, this returns UnknownServerError. +// If the code is 0, this returns nil. +// +// Note that this function is provided as a simplicity function for code that +// needs to work with the *Error only, but this function comes with caveats. +// Because this can return a typed nil, passing the return of this to a +// function that accepts an error (the Go error interface), the return from +// this will never be considered a nil error. Instead, it will be an error with +// a nil internal value. +func TypedErrorForCode(code int16) *Error { + err, exists := code2err[code] + if !exists { + return UnknownServerError + } + if err == nil { + return nil + } + return err.(*Error) +} + +// IsRetriable returns whether a Kafka error is considered retriable. +func IsRetriable(err error) bool { + var kerr *Error + return errors.As(err, &kerr) && kerr.Retriable +} + +var ( + UnknownServerError = &Error{"UNKNOWN_SERVER_ERROR", -1, false, "The server experienced an unexpected error when processing the request."} + OffsetOutOfRange = &Error{"OFFSET_OUT_OF_RANGE", 1, false, "The requested offset is not within the range of offsets maintained by the server."} + CorruptMessage = &Error{"CORRUPT_MESSAGE", 2, true, "This message has failed its CRC checksum, exceeds the valid size, has a null key for a compacted topic, or is otherwise corrupt."} + UnknownTopicOrPartition = &Error{"UNKNOWN_TOPIC_OR_PARTITION", 3, true, "This server does not host this topic-partition."} + InvalidFetchSize = &Error{"INVALID_FETCH_SIZE", 4, false, "The requested fetch size is invalid."} + LeaderNotAvailable = &Error{"LEADER_NOT_AVAILABLE", 5, true, "There is no leader for this topic-partition as we are in the middle of a leadership election."} + NotLeaderForPartition = &Error{"NOT_LEADER_FOR_PARTITION", 6, true, "This server is not the leader for that topic-partition."} + RequestTimedOut = &Error{"REQUEST_TIMED_OUT", 7, true, "The request timed out."} + BrokerNotAvailable = &Error{"BROKER_NOT_AVAILABLE", 8, true, "The broker is not available."} + ReplicaNotAvailable = &Error{"REPLICA_NOT_AVAILABLE", 9, true, "The replica is not available for the requested topic-partition."} + MessageTooLarge = &Error{"MESSAGE_TOO_LARGE", 10, false, "The request included a message larger than the max message size the server will accept"} /* error description doesn't have period at the end as more info would be appended to it upstream */ + StaleControllerEpoch = &Error{"STALE_CONTROLLER_EPOCH", 11, false, "The controller moved to another broker."} + OffsetMetadataTooLarge = &Error{"OFFSET_METADATA_TOO_LARGE", 12, false, "The metadata field of the offset request was too large."} + NetworkException = &Error{"NETWORK_EXCEPTION", 13, true, "The server disconnected before a response was received."} + CoordinatorLoadInProgress = &Error{"COORDINATOR_LOAD_IN_PROGRESS", 14, true, "The coordinator is loading and hence can't process requests."} + CoordinatorNotAvailable = &Error{"COORDINATOR_NOT_AVAILABLE", 15, true, "The coordinator is not available."} + NotCoordinator = &Error{"NOT_COORDINATOR", 16, true, "This is not the correct coordinator."} + InvalidTopicException = &Error{"INVALID_TOPIC_EXCEPTION", 17, false, "The request attempted to perform an operation on an invalid topic."} + RecordListTooLarge = &Error{"RECORD_LIST_TOO_LARGE", 18, false, "The request included message batch larger than the configured segment size on the server."} + NotEnoughReplicas = &Error{"NOT_ENOUGH_REPLICAS", 19, true, "Messages are rejected since there are fewer in-sync replicas than required."} + NotEnoughReplicasAfterAppend = &Error{"NOT_ENOUGH_REPLICAS_AFTER_APPEND", 20, true, "Messages are written to the log, but to fewer in-sync replicas than required."} + InvalidRequiredAcks = &Error{"INVALID_REQUIRED_ACKS", 21, false, "Produce request specified an invalid value for required acks."} + IllegalGeneration = &Error{"ILLEGAL_GENERATION", 22, false, "Specified group generation id is not valid."} + InconsistentGroupProtocol = &Error{"INCONSISTENT_GROUP_PROTOCOL", 23, false, "The group member's supported protocols are incompatible with those of existing members or first group member tried to join with empty protocol type or empty protocol list."} + InvalidGroupID = &Error{"INVALID_GROUP_ID", 24, false, "The configured groupID is invalid."} + UnknownMemberID = &Error{"UNKNOWN_MEMBER_ID", 25, false, "The coordinator is not aware of this member."} + InvalidSessionTimeout = &Error{"INVALID_SESSION_TIMEOUT", 26, false, "The session timeout is not within the range allowed by the broker (as configured by group.min.session.timeout.ms and group.max.session.timeout.ms)."} + RebalanceInProgress = &Error{"REBALANCE_IN_PROGRESS", 27, false, "The group is rebalancing, so a rejoin is needed."} + InvalidCommitOffsetSize = &Error{"INVALID_COMMIT_OFFSET_SIZE", 28, false, "The committing offset data size is not valid."} + TopicAuthorizationFailed = &Error{"TOPIC_AUTHORIZATION_FAILED", 29, false, "Not authorized to access topics: [Topic authorization failed.]"} + GroupAuthorizationFailed = &Error{"GROUP_AUTHORIZATION_FAILED", 30, false, "Not authorized to access group: Group authorization failed."} + ClusterAuthorizationFailed = &Error{"CLUSTER_AUTHORIZATION_FAILED", 31, false, "Cluster authorization failed."} + InvalidTimestamp = &Error{"INVALID_TIMESTAMP", 32, false, "The timestamp of the message is out of acceptable range."} + UnsupportedSaslMechanism = &Error{"UNSUPPORTED_SASL_MECHANISM", 33, false, "The broker does not support the requested SASL mechanism."} + IllegalSaslState = &Error{"ILLEGAL_SASL_STATE", 34, false, "Request is not valid given the current SASL state."} + UnsupportedVersion = &Error{"UNSUPPORTED_VERSION", 35, false, "The version of API is not supported."} + TopicAlreadyExists = &Error{"TOPIC_ALREADY_EXISTS", 36, false, "Topic with this name already exists."} + InvalidPartitions = &Error{"INVALID_PARTITIONS", 37, false, "Number of partitions is below 1."} + InvalidReplicationFactor = &Error{"INVALID_REPLICATION_FACTOR", 38, false, "Replication factor is below 1 or larger than the number of available brokers."} + InvalidReplicaAssignment = &Error{"INVALID_REPLICA_ASSIGNMENT", 39, false, "Replica assignment is invalid."} + InvalidConfig = &Error{"INVALID_CONFIG", 40, false, "Configuration is invalid."} + NotController = &Error{"NOT_CONTROLLER", 41, true, "This is not the correct controller for this cluster."} + InvalidRequest = &Error{"INVALID_REQUEST", 42, false, "This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details."} + UnsupportedForMessageFormat = &Error{"UNSUPPORTED_FOR_MESSAGE_FORMAT", 43, false, "The message format version on the broker does not support the request."} + PolicyViolation = &Error{"POLICY_VIOLATION", 44, false, "Request parameters do not satisfy the configured policy."} + OutOfOrderSequenceNumber = &Error{"OUT_OF_ORDER_SEQUENCE_NUMBER", 45, false, "The broker received an out of order sequence number."} + DuplicateSequenceNumber = &Error{"DUPLICATE_SEQUENCE_NUMBER", 46, false, "The broker received a duplicate sequence number."} + InvalidProducerEpoch = &Error{"INVALID_PRODUCER_EPOCH", 47, false, "Producer attempted an operation with an old epoch."} + InvalidTxnState = &Error{"INVALID_TXN_STATE", 48, false, "The producer attempted a transactional operation in an invalid state."} + InvalidProducerIDMapping = &Error{"INVALID_PRODUCER_ID_MAPPING", 49, false, "The producer attempted to use a producer id which is not currently assigned to its transactional id."} + InvalidTransactionTimeout = &Error{"INVALID_TRANSACTION_TIMEOUT", 50, false, "The transaction timeout is larger than the maximum value allowed by the broker (as configured by transaction.max.timeout.ms)."} + ConcurrentTransactions = &Error{"CONCURRENT_TRANSACTIONS", 51, false, "The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing."} + TransactionCoordinatorFenced = &Error{"TRANSACTION_COORDINATOR_FENCED", 52, false, "Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer."} + TransactionalIDAuthorizationFailed = &Error{"TRANSACTIONAL_ID_AUTHORIZATION_FAILED", 53, false, "Transactional ID authorization failed."} + SecurityDisabled = &Error{"SECURITY_DISABLED", 54, false, "Security features are disabled."} + OperationNotAttempted = &Error{"OPERATION_NOT_ATTEMPTED", 55, false, "The broker did not attempt to execute this operation. This may happen for batched RPCs where some operations in the batch failed, causing the broker to respond without trying the rest."} + KafkaStorageError = &Error{"KAFKA_STORAGE_ERROR", 56, true, "Disk error when trying to access log file on the disk."} + LogDirNotFound = &Error{"LOG_DIR_NOT_FOUND", 57, false, "The user-specified log directory is not found in the broker config."} + SaslAuthenticationFailed = &Error{"SASL_AUTHENTICATION_FAILED", 58, false, "SASL Authentication failed."} + UnknownProducerID = &Error{"UNKNOWN_PRODUCER_ID", 59, false, "This exception is raised by the broker if it could not locate the producer metadata associated with the producerID in question. This could happen if, for instance, the producer's records were deleted because their retention time had elapsed. Once the last records of the producerID are removed, the producer's metadata is removed from the broker, and future appends by the producer will return this exception."} + ReassignmentInProgress = &Error{"REASSIGNMENT_IN_PROGRESS", 60, false, "A partition reassignment is in progress."} + DelegationTokenAuthDisabled = &Error{"DELEGATION_TOKEN_AUTH_DISABLED", 61, false, "Delegation Token feature is not enabled."} + DelegationTokenNotFound = &Error{"DELEGATION_TOKEN_NOT_FOUND", 62, false, "Delegation Token is not found on server."} + DelegationTokenOwnerMismatch = &Error{"DELEGATION_TOKEN_OWNER_MISMATCH", 63, false, "Specified Principal is not valid Owner/Renewer."} + DelegationTokenRequestNotAllowed = &Error{"DELEGATION_TOKEN_REQUEST_NOT_ALLOWED", 64, false, "Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels."} + DelegationTokenAuthorizationFailed = &Error{"DELEGATION_TOKEN_AUTHORIZATION_FAILED", 65, false, "Delegation Token authorization failed."} + DelegationTokenExpired = &Error{"DELEGATION_TOKEN_EXPIRED", 66, false, "Delegation Token is expired."} + InvalidPrincipalType = &Error{"INVALID_PRINCIPAL_TYPE", 67, false, "Supplied principalType is not supported."} + NonEmptyGroup = &Error{"NON_EMPTY_GROUP", 68, false, "The group is not empty."} + GroupIDNotFound = &Error{"GROUP_ID_NOT_FOUND", 69, false, "The group id does not exist."} + FetchSessionIDNotFound = &Error{"FETCH_SESSION_ID_NOT_FOUND", 70, true, "The fetch session ID was not found."} + InvalidFetchSessionEpoch = &Error{"INVALID_FETCH_SESSION_EPOCH", 71, true, "The fetch session epoch is invalid."} + ListenerNotFound = &Error{"LISTENER_NOT_FOUND", 72, true, "There is no listener on the leader broker that matches the listener on which metadata request was processed."} + TopicDeletionDisabled = &Error{"TOPIC_DELETION_DISABLED", 73, false, "Topic deletion is disabled."} + FencedLeaderEpoch = &Error{"FENCED_LEADER_EPOCH", 74, true, "The leader epoch in the request is older than the epoch on the broker"} + UnknownLeaderEpoch = &Error{"UNKNOWN_LEADER_EPOCH", 75, true, "The leader epoch in the request is newer than the epoch on the broker"} + UnsupportedCompressionType = &Error{"UNSUPPORTED_COMPRESSION_TYPE", 76, false, "The requesting client does not support the compression type of given partition."} + StaleBrokerEpoch = &Error{"STALE_BROKER_EPOCH", 77, false, "Broker epoch has changed"} + OffsetNotAvailable = &Error{"OFFSET_NOT_AVAILABLE", 78, true, "The leader high watermark has not caught up from a recent leader election so the offsets cannot be guaranteed to be monotonically increasing"} + MemberIDRequired = &Error{"MEMBER_ID_REQUIRED", 79, false, "The group member needs to have a valid member id before actually entering a consumer group"} + PreferredLeaderNotAvailable = &Error{"PREFERRED_LEADER_NOT_AVAILABLE", 80, true, "The preferred leader was not available"} + GroupMaxSizeReached = &Error{"GROUP_MAX_SIZE_REACHED", 81, false, "The consumer group has reached its max size"} + FencedInstanceID = &Error{"FENCED_INSTANCE_ID", 82, false, "The broker rejected this static consumer since another consumer with the same group.instance.id has registered with a different member.id."} + EligibleLeadersNotAvailable = &Error{"ELIGIBLE_LEADERS_NOT_AVAILABLE", 83, true, "Eligible topic partition leaders are not available"} + ElectionNotNeeded = &Error{"ELECTION_NOT_NEEDED", 84, true, "Leader election not needed for topic partition"} + NoReassignmentInProgress = &Error{"NO_REASSIGNMENT_IN_PROGRESS", 85, false, "No partition reassignment is in progress."} + GroupSubscribedToTopic = &Error{"GROUP_SUBSCRIBED_TO_TOPIC", 86, false, "Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it."} + InvalidRecord = &Error{"INVALID_RECORD", 87, false, "This record has failed the validation on the broker and hence been rejected."} + UnstableOffsetCommit = &Error{"UNSTABLE_OFFSET_COMMIT", 88, true, "There are unstable offsets that need to be cleared."} + ThrottlingQuotaExceeded = &Error{"THROTTLING_QUOTA_EXCEEDED", 89, true, "The throttling quota has been exceeded."} + ProducerFenced = &Error{"PRODUCER_FENCED", 90, false, "There is a newer producer with the same transactionalId which fences the current one."} + ResourceNotFound = &Error{"RESOURCE_NOT_FOUND", 91, false, "A request illegally referred to a resource that does not exist."} + DuplicateResource = &Error{"DUPLICATE_RESOURCE", 92, false, "A request illegally referred to the same resource twice."} + UnacceptableCredential = &Error{"UNACCEPTABLE_CREDENTIAL", 93, false, "Requested credential would not meet criteria for acceptability."} + InconsistentVoterSet = &Error{"INCONSISTENT_VOTER_SET", 94, false, "Indicates that either the sender or recipient of a voter-only request is not one of the expected voters."} + InvalidUpdateVersion = &Error{"INVALID_UPDATE_VERSION", 95, false, "The given update version was invalid."} + FeatureUpdateFailed = &Error{"FEATURE_UPDATE_FAILED", 96, false, "Unable to update finalized features due to an unexpected server error."} + PrincipalDeserializationFailure = &Error{"PRINCIPAL_DESERIALIZATION_FAILURE", 97, false, "Request principal deserialization failed during forwarding. This indicates an internal error on the broker cluster security setup."} + SnapshotNotFound = &Error{"SNAPSHOT_NOT_FOUND", 98, false, "Requested snapshot was not found."} + PositionOutOfRange = &Error{"POSITION_OUT_OF_RANGE", 99, false, "Requested position is not greater than or equal to zero, and less than the size of the snapshot."} + UnknownTopicID = &Error{"UNKNOWN_TOPIC_ID", 100, true, "This server does not host this topic ID."} + DuplicateBrokerRegistration = &Error{"DUPLICATE_BROKER_REGISTRATION", 101, false, "This broker ID is already in use."} + BrokerIDNotRegistered = &Error{"BROKER_ID_NOT_REGISTERED", 102, false, "The given broker ID was not registered."} + InconsistentTopicID = &Error{"INCONSISTENT_TOPIC_ID", 103, true, "The log's topic ID did not match the topic ID in the request."} + InconsistentClusterID = &Error{"INCONSISTENT_CLUSTER_ID", 104, false, "The clusterId in the request does not match that found on the server."} + TransactionalIDNotFound = &Error{"TRANSACTIONAL_ID_NOT_FOUND", 105, false, "The transactionalId could not be found."} + FetchSessionTopicIDError = &Error{"FETCH_SESSION_TOPIC_ID_ERROR", 106, true, "The fetch session encountered inconsistent topic ID usage."} + IneligibleReplica = &Error{"INELIGIBLE_REPLICA", 107, false, "The new ISR contains at least one ineligible replica."} + NewLeaderElected = &Error{"NEW_LEADER_ELECTED", 108, false, "The AlterPartition request successfully updated the partition state but the leader has changed."} + OffsetMovedToTieredStorage = &Error{"OFFSET_MOVED_TO_TIERED_STORAGE", 109, false, "The requested offset is moved to tiered storage."} + FencedMemberEpoch = &Error{"FENCED_MEMBER_EPOCH", 110, false, "The member epoch is fenced by the group coordinator. The member must abandon all its partitions and rejoin."} + UnreleasedInstanceID = &Error{"UNRELEASED_INSTANCE_ID", 111, false, "The instance ID is still used by another member in the consumer group. That member must leave first."} + UnsupportedAssignor = &Error{"UNSUPPORTED_ASSIGNOR", 112, false, "The assignor or its version range is not supported by the consumer group."} + StaleMemberEpoch = &Error{"STALE_MEMBER_EPOCH", 113, false, "The member epoch is stale. The member must retry after receiving its updated member epoch via the ConsumerGroupHeartbeat API."} + MismatchedEndpointType = &Error{"MISMATCHED_ENDPOINT_TYPE", 114, false, "The request was sent to an endpoint of the wrong type."} + UnsupportedEndpointType = &Error{"UNSUPPORTED_ENDPOINT_TYPE", 115, false, "This endpoint type is not supported yet."} + UnknownControllerID = &Error{"UNKNOWN_CONTROLLER_ID", 116, false, "This controller ID is not known"} + UnknownSubscriptionID = &Error{"UNKNOWN_SUBSCRIPTION_ID", 117, false, "Client sent a push telemetry request with an invalid or outdated subscription ID."} + TelemetryTooLarge = &Error{"TELEMETRY_TOO_LARGE", 118, false, "Client sent a push telemetry request larger than the maximum size the broker will accept."} + InvalidRegistration = &Error{"INVALID_REGISTRATION", 119, false, "The controller has considered the broker registration to be invalid."} + TransactionAbortable = &Error{"TRANSACTION_ABORTABLE", 120, false, "The server encountered an error with the transaction. The client can abort the transaction to continue using this transactional ID."} + InvalidRecordState = &Error{"INVALID_RECORD_STATE", 121, false, "The record state is invalid. The acknowledgement of delivery could not be completed."} + ShareSessionNotFound = &Error{"SHARE_SESSION_NOT_FOUND", 122, false, "The share session was not found."} + InvalidShareSessionEpoch = &Error{"INVALID_SHARE_SESSION_EPOCH", 123, false, "The share session epoch is invalid."} + FencedStateEpoch = &Error{"FENCED_STATE_EPOCH", 124, false, "The share coordinator rejected the request because the share-group state epoch did not match."} + InvalidVoterKey = &Error{"INVALID_VOTER_KEY", 125, false, "The voter key doesn't match the receiving replica's key."} + DuplicateVoter = &Error{"DUPLICATE_VOTER", 126, false, "The voter is already part of the set of voters."} + VoterNotFound = &Error{"VOTER_NOT_FOUND", 127, false, "The voter is not part of the set of voters."} + InvalidRegularExpression = &Error{"INVALID_REGULAR_EXPRESSION", 128, false, "The regular expression is not valid."} + RebootstrapRequired = &Error{"REBOOTSTRAP_REQUIRED", 129, false, "Client metadata is stale. The client should rebootstrap to obtain new metadata."} + StreamsInvalidTopology = &Error{"STREAMS_INVALID_TOPOLOGY", 130, false, "The supplied topology is invalid."} + StreamsInvalidTopologyEpoch = &Error{"STREAMS_INVALID_TOPOLOGY_EPOCH", 131, false, "The supplied topology epoch is invalid."} + StreamsTopologyFenced = &Error{"STREAMS_TOPOLOGY_FENCED", 132, false, "The supplied topology epoch is outdated."} + ShareSessionLimitReached = &Error{"SHARE_SESSION_LIMIT_REACHED", 133, true, "The limit of share sessions has been reached."} +) + +var code2err = map[int16]error{ + -1: UnknownServerError, + 0: nil, + 1: OffsetOutOfRange, + 2: CorruptMessage, + 3: UnknownTopicOrPartition, + 4: InvalidFetchSize, + 5: LeaderNotAvailable, + 6: NotLeaderForPartition, + 7: RequestTimedOut, + 8: BrokerNotAvailable, + 9: ReplicaNotAvailable, + 10: MessageTooLarge, + 11: StaleControllerEpoch, + 12: OffsetMetadataTooLarge, + 13: NetworkException, + 14: CoordinatorLoadInProgress, + 15: CoordinatorNotAvailable, + 16: NotCoordinator, + 17: InvalidTopicException, + 18: RecordListTooLarge, + 19: NotEnoughReplicas, + 20: NotEnoughReplicasAfterAppend, + 21: InvalidRequiredAcks, + 22: IllegalGeneration, + 23: InconsistentGroupProtocol, + 24: InvalidGroupID, + 25: UnknownMemberID, + 26: InvalidSessionTimeout, + 27: RebalanceInProgress, + 28: InvalidCommitOffsetSize, + 29: TopicAuthorizationFailed, + 30: GroupAuthorizationFailed, + 31: ClusterAuthorizationFailed, + 32: InvalidTimestamp, + 33: UnsupportedSaslMechanism, + 34: IllegalSaslState, + 35: UnsupportedVersion, + 36: TopicAlreadyExists, + 37: InvalidPartitions, + 38: InvalidReplicationFactor, + 39: InvalidReplicaAssignment, + 40: InvalidConfig, + 41: NotController, + 42: InvalidRequest, + 43: UnsupportedForMessageFormat, + 44: PolicyViolation, + 45: OutOfOrderSequenceNumber, + 46: DuplicateSequenceNumber, + 47: InvalidProducerEpoch, + 48: InvalidTxnState, + 49: InvalidProducerIDMapping, + 50: InvalidTransactionTimeout, + 51: ConcurrentTransactions, + 52: TransactionCoordinatorFenced, + 53: TransactionalIDAuthorizationFailed, + 54: SecurityDisabled, + 55: OperationNotAttempted, + 56: KafkaStorageError, + 57: LogDirNotFound, + 58: SaslAuthenticationFailed, + 59: UnknownProducerID, + 60: ReassignmentInProgress, + 61: DelegationTokenAuthDisabled, + 62: DelegationTokenNotFound, + 63: DelegationTokenOwnerMismatch, + 64: DelegationTokenRequestNotAllowed, + 65: DelegationTokenAuthorizationFailed, + 66: DelegationTokenExpired, + 67: InvalidPrincipalType, + 68: NonEmptyGroup, + 69: GroupIDNotFound, + 70: FetchSessionIDNotFound, + 71: InvalidFetchSessionEpoch, + 72: ListenerNotFound, + 73: TopicDeletionDisabled, + 74: FencedLeaderEpoch, + 75: UnknownLeaderEpoch, + 76: UnsupportedCompressionType, + 77: StaleBrokerEpoch, + 78: OffsetNotAvailable, + 79: MemberIDRequired, + 80: PreferredLeaderNotAvailable, + 81: GroupMaxSizeReached, + 82: FencedInstanceID, + 83: EligibleLeadersNotAvailable, + 84: ElectionNotNeeded, + 85: NoReassignmentInProgress, + 86: GroupSubscribedToTopic, + 87: InvalidRecord, + 88: UnstableOffsetCommit, + 89: ThrottlingQuotaExceeded, + 90: ProducerFenced, + 91: ResourceNotFound, + 92: DuplicateResource, + 93: UnacceptableCredential, + 94: InconsistentVoterSet, + 95: InvalidUpdateVersion, + 96: FeatureUpdateFailed, + 97: PrincipalDeserializationFailure, + 98: SnapshotNotFound, + 99: PositionOutOfRange, + 100: UnknownTopicID, + 101: DuplicateBrokerRegistration, + 102: BrokerIDNotRegistered, + 103: InconsistentTopicID, + 104: InconsistentClusterID, + 105: TransactionalIDNotFound, + 106: FetchSessionTopicIDError, + 107: IneligibleReplica, + 108: NewLeaderElected, + 109: OffsetMovedToTieredStorage, // KIP-405, v3.5 + 110: FencedMemberEpoch, // KIP-848, released unstable in v3.6, stable in 3.7 + 111: UnreleasedInstanceID, // "" + 112: UnsupportedAssignor, // "" + 113: StaleMemberEpoch, // "" + 114: MismatchedEndpointType, // KIP-919, v3.7 + 115: UnsupportedEndpointType, // "" + 116: UnknownControllerID, // "" + 117: UnknownSubscriptionID, // KIP-714 f1819f448 KAFKA-15778 & KAFKA-15779 + 118: TelemetryTooLarge, // "" + 119: InvalidRegistration, // KIP-858 f467f6bb4 KAFKA-15361 + 120: TransactionAbortable, // KIP-890 2e8d69b78 KAFKA-16314 + 121: InvalidRecordState, + 122: ShareSessionNotFound, + 123: InvalidShareSessionEpoch, + 124: FencedStateEpoch, + 125: InvalidVoterKey, + 126: DuplicateVoter, + 127: VoterNotFound, + 128: InvalidRegularExpression, + 129: RebootstrapRequired, + 130: StreamsInvalidTopology, + 131: StreamsInvalidTopologyEpoch, + 132: StreamsTopologyFenced, + 133: ShareSessionLimitReached, +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/CLAUDE.md b/vendor/github.com/twmb/franz-go/pkg/kgo/CLAUDE.md new file mode 100644 index 00000000000..9a38226d09a --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/CLAUDE.md @@ -0,0 +1,127 @@ +# CLAUDE.md + +## Package Overview + +franz-go is a pure Go Kafka client library. The `kgo` package is the main client package. + +## Style + +- Never use non-ASCII characters in code or comments, use simple characters instead: dashes, => for arrows, etc. +- Always run `gofmt` before committing +- Internal comments should explain WHY we are doing something, not just WHAT we are doing. WHAT comments are almost never useful, unless the block that follows is complex. +- Comments around subtle race conditions (logic or data) should contain a walkthrough of how the race is encountered. Don't JUST say what the race is, but also how a sequence of events can encounter the race. + +## Context keys: NEVER use empty struct as a key + +Per the Go spec, two distinct zero-size variables may share the same address. That means `type myKey struct{}` is unsafe as a `context.WithValue` key - it can collide with any other package using the same pattern. Use the project's pointer-to-string idiom instead: + +```go +var myKey = func() *string { s := "my_key"; return &s }() +``` + +The string body is for debugging; the pointer's identity is what makes the key unique. Examples in this package: `ctxPinReq` (broker.go), `noShardRetryCtx` (client.go), `commitContextFn` / `txnCommitContextFn` (consumer_group.go), `ctxRecRecycle` (pools.go). + +## Commit Style + +- Format: `kgo: ` (lowercase, no period) +- Body should explain the "why" not just the "what" +- Reference KIPs (Kafka Improvement Proposals) for protocol changes +- Include `Closes #` when fixing GitHub issues + +## Protocol Behavior + +The Java broker at `~/src/apache/kafka/` is the protocol's ground +truth. The Java reference client at +`clients/src/main/java/org/apache/kafka/clients/consumer/internals/` is +the ground truth for how to talk to it. KIPs are a tertiary source: +https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Improvement+Proposals + +When evaluating whether the broker will accept X, trace the full state +lifecycle behind X, not just the entry point: creation, destruction +(leader change `onBecomingFollower`, disconnect `onDisconnect`, member +fence, acquisition-lock timeout, session replace, cache eviction), and +rehydration (including what defaults the reloaded state uses for +transient fields). Missing a destruction or rehydration path is the +usual failure mode. + +Existing conservative kgo guards are presumed correct. Removal requires +both (a) a broker trace covering creation/destruction/rehydration that +shows the broker accepts what kgo drops, and (b) the Java reference +client not having an equivalent guard. + +If the user pushes back on a conclusion ("are you sure?", "isn't this +big?"), retrace the broker from a path you didn't read -- don't +restate the previous reasoning. + +## Approach + +Before implementing a fix for any bug, first verify whether existing code +already handles the scenario. Analyze the current codebase's safety mechanisms +(e.g., prerevoke logic, error handlers, retry paths) before writing new code. + +### Audit vs. implementation + +Audit requests ("review this", "find bugs", "propose a refactor") report +findings in prose with file:line citations; they do NOT call Edit/Write +on the audited code. Edits only happen on an explicit imperative for a +specific change ("apply this", "go ahead") or a direct yes to "want me +to apply this?". Passing remarks like "you do it" inside an analysis +request do not count. If a session will land many edits to one file, +ask for a standing per-file grant scoped to that file and session. + +### Refactoring threshold (DRY) + +DRY is about logic, not line counts. Reject helpers that exist to +differentiate callers with a `bool` flag (that's two operations with +shared infrastructure, not one operation with a knob) or that save +fewer than ~5 lines per site. Accept helpers that name a genuinely +duplicated operation, have one job, and leave the call site reading +like the operation it is performing. + +## Tool use + +Use `Read`, `Grep`, and `Glob` for code exploration; do not reach for +`Bash` with `cat`/`head`/`tail`/`find`/`ls`/`awk`/`sed`/`grep`. Reserve +`Bash` for tests, `gofmt`/`go vet`/`go build`, git, and gh CLI. + +## Key Files + +- `broker.go` - Connection handling, SASL, request/response. +- `config.go` - Client configuration options. +- `client.go` - Main client logic. +- `source.go` - Fetches from one broker. A source owns many cursors; + each cursor tracks consume progress for one partition. +- `sink.go` - Produces to one broker. A sink owns many recBufs (one + per partition); a recBuf owns many recBatch. +- `consumer.go` - Consumer abstraction over fetching. Validates cursor + offsets (OffsetForLeaderEpoch for data loss, ListOffsets for + epoch/leader). Owns sources. +- `producer.go` - Producer abstraction over producing. Picks the sink + for each record, finishes promises, and runs cross-sink operations. +- `consumer_group.go` - Group consumer: decides which partitions to + consume and feeds the subscription into consumer. Manages membership + and offset commits. +- `consumer_direct.go` - Direct consumer: user-assigned partitions. + Mostly metadata-driven topic/regex resolution. +- `txn.go` - GroupTransactSession. Bundles group logic with transaction + logic; paranoid about aborting to prevent duplicates. +- `metadata.go` - Periodic metadata refresh; feeds producer/consumer. + +## Testing + +- `go test ./...` needs a local broker on port 9092. `../kfake/` is an + in-process fake broker for unit-level tests; use a `go.work` in + `pkg/kfake/` to test against local kgo changes. Always run + `go test -race`. +- Unit tests should be fast. Tests >2s are suspicious even when + passing. Only TestGroupETL / TestTxnETL should take real time + (~2min without -race, ~3-4min with). +- When working in `pkg/kgo`, don't speculatively run kfake tests -- a + parallel session may have kfake in a broken state. Verify with + `go build` / `go vet`; run kfake tests only when the task is inside + `pkg/kfake/` or explicitly asked. + +## Design / concurrency. + +- Refer to ../../DESIGN.md for a high level design of all the operations that can happen. +- Update the design file as necessary / when making significant changes. diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/atomic_maybe_work.go b/vendor/github.com/twmb/franz-go/pkg/kgo/atomic_maybe_work.go new file mode 100644 index 00000000000..1cf93ad68ed --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/atomic_maybe_work.go @@ -0,0 +1,76 @@ +package kgo + +import "sync/atomic" + +const ( + stateUnstarted = iota + stateWorking + stateContinueWorking +) + +type workLoop struct{ state atomic.Uint32 } + +// maybeBegin returns whether a work loop should begin. +func (l *workLoop) maybeBegin() bool { + var state uint32 + var done bool + for !done { + switch state = l.state.Load(); state { + case stateUnstarted: + done = l.state.CompareAndSwap(state, stateWorking) + state = stateWorking + case stateWorking: + done = l.state.CompareAndSwap(state, stateContinueWorking) + state = stateContinueWorking + case stateContinueWorking: + done = true + } + } + + return state == stateWorking +} + +// maybeFinish demotes loop's internal state and returns whether work should +// keep going. This function should be called before looping to continue +// work. +// +// If again is true, this will avoid demoting from working to not +// working. Again would be true if the loop knows it should continue working; +// calling this function is necessary even in this case to update loop's +// internal state. +// +// This function is a no-op if the loop is already finished, but generally, +// since the loop itself calls MaybeFinish after it has been started, this +// should never be called if the loop is unstarted. +func (l *workLoop) maybeFinish(again bool) bool { + switch state := l.state.Load(); state { + // Working: + // If again, we know we should continue; keep our state. + // If not again, we try to downgrade state and stop. + // If we cannot, then something slipped in to say keep going. + case stateWorking: + if !again { + again = !l.state.CompareAndSwap(state, stateUnstarted) + } + // Continue: demote ourself and run again no matter what. + case stateContinueWorking: + l.state.Store(stateWorking) + again = true + } + + return again +} + +func (l *workLoop) hardFinish() { + l.state.Store(stateUnstarted) +} + +// lazyI32 is used in a few places where we want atomics _sometimes_. Some +// uses do not need to be atomic (notably, setup), and we do not want the +// noCopy guard. +// +// Specifically, this is used for a few int32 settings in the config. +type lazyI32 int32 + +func (v *lazyI32) store(s int32) { atomic.StoreInt32((*int32)(v), s) } +func (v *lazyI32) load() int32 { return atomic.LoadInt32((*int32)(v)) } diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/broker.go b/vendor/github.com/twmb/franz-go/pkg/kgo/broker.go new file mode 100644 index 00000000000..2386eb24ba6 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/broker.go @@ -0,0 +1,1651 @@ +package kgo + +import ( + "context" + "crypto/tls" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "math/rand" + "net" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/twmb/franz-go/pkg/kbin" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" + "github.com/twmb/franz-go/pkg/sasl" +) + +type pinReq struct { + min int16 + max int16 + pinMin bool + pinMax bool +} + +var ctxPinReq = func() *string { v := "pin_req"; return &v }() + +type forceOpenReq struct{ kmsg.Request } + +type promisedReq struct { + ctx context.Context + req kmsg.Request + promise func(kmsg.Response, error) + enqueue time.Time // used to calculate writeWait +} + +type promisedResp struct { + ctx context.Context + + corrID int32 + // With flexible headers, we skip tags at the end of the response + // header for now because they're currently unused. However, the + // ApiVersions response uses v0 response header (no tags) even if the + // response body has flexible versions. This is done in support of the + // v0 fallback logic that allows for indexing into an exact offset. + // Thus, for ApiVersions specifically, this is false even if the + // request is flexible. + // + // As a side note, this note was not mentioned in KIP-482 which + // introduced flexible versions, and was mentioned in passing in + // KIP-511 which made ApiVersion flexible, so discovering what was + // wrong was not too fun ("Note that ApiVersionsResponse is flexible + // version but the response header is not flexible" is *it* in the + // entire KIP.) + // + // To see the version pinning, look at the code generator function + // generateHeaderVersion in + // generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java + flexibleHeader bool + + resp kmsg.Response + promise func(kmsg.Response, error) + readTimeout time.Duration + + // The following block is used for the read / e2e hooks. + bytesWritten int + writeWait time.Duration + timeToWrite time.Duration + readEnqueue time.Time +} + +// NodeName returns the name of a node, given the kgo internal node ID. +// +// Internally, seed brokers are stored with very negative node IDs, and these +// node IDs are visible in the BrokerMetadata struct. You can use NodeName to +// convert the negative node ID into "seed_#". Brokers discovered through +// metadata responses have standard non-negative numbers and this function just +// returns the number as a string. +func NodeName(nodeID int32) string { + return logID(nodeID) +} + +func logID(id int32) string { + if id >= -10 { + return strconv.FormatInt(int64(id), 10) + } + return "seed_" + strconv.FormatInt(int64(id)-math.MinInt32, 10) +} + +// BrokerMetadata is metadata for a broker. +// +// This struct mirrors kmsg.MetadataResponseBroker. +type BrokerMetadata struct { + // NodeID is the broker node ID. + // + // Seed brokers will have very negative IDs; kgo does not try to map + // seed brokers to loaded brokers. You can use NodeName to convert + // the seed node ID into a formatted string. + NodeID int32 + + // Port is the port of the broker. + Port int32 + + // Host is the hostname of the broker. + Host string + + // Rack is an optional rack of the broker. It is invalid to modify this + // field. + // + // Seed brokers will not have a rack. + Rack *string + + _ struct{} // allow us to add fields later +} + +func (me BrokerMetadata) equals(other kmsg.MetadataResponseBroker) bool { + return me.NodeID == other.NodeID && + me.Port == other.Port && + me.Host == other.Host && + (me.Rack == nil && other.Rack == nil || + me.Rack != nil && other.Rack != nil && *me.Rack == *other.Rack) +} + +// broker manages the concept how a client would interact with a broker. +type broker struct { + cl *Client + + addr string // net.JoinHostPort(meta.Host, meta.Port) + meta BrokerMetadata + + // versions tracks the first load of an ApiVersions. We store this + // after the first connect, which helps speed things up on future + // reconnects (across any of the three broker connections) because we + // will never look up API versions for this broker again. + versions atomic.Value // *brokerVersions + + // The cxn fields each manage a single tcp connection to one broker. + // Each field is managed serially in handleReqs. This means that only + // one write can happen at a time, regardless of which connection the + // write goes to, but the write is expected to be fast whereas the wait + // for the response is expected to be slow. + // + // Produce requests go to cxnProduce, fetch to cxnFetch, join/sync go + // to cxnGroup, anything with TimeoutMillis goes to cxnSlow, and + // everything else goes to cxnNormal. + cxnNormal *brokerCxn + cxnProduce *brokerCxn + cxnFetch *brokerCxn + cxnGroup *brokerCxn + cxnSlow *brokerCxn + + reapMu xsync.Mutex // held when modifying a brokerCxn + + // reqs manages incoming message requests. + reqs ring[promisedReq] + // dead is an atomic so a backed up reqs cannot block broker stoppage. + dead atomic.Bool +} + +// brokerVersions is loaded once (and potentially a few times concurrently if +// multiple connections are opening at once) and then forever stored for a +// broker. +type brokerVersions struct { + maxVers map[int16]int16 + minVers map[int16]int16 + features map[string]int16 +} + +func (v *brokerVersions) maxVersion(key int16) int16 { + if version, ok := v.maxVers[key]; ok { + return version + } + return -1 +} + +func (v *brokerVersions) minVersion(key int16) int16 { + if version, ok := v.minVers[key]; ok { + return version + } + return -1 +} + +func newBrokerVersions(capacity int) *brokerVersions { + v := &brokerVersions{ + maxVers: make(map[int16]int16, capacity), + minVers: make(map[int16]int16, capacity), + features: make(map[string]int16), + } + return v +} + +func (b *broker) loadVersions() *brokerVersions { + loaded := b.versions.Load() + if loaded == nil { + return nil + } + return loaded.(*brokerVersions) +} + +func (b *broker) storeVersions(v *brokerVersions) { b.versions.Store(v) } + +const unknownControllerID = -1 + +var unknownBrokerMetadata = BrokerMetadata{ + NodeID: -1, +} + +// broker IDs are all positive, but Kafka uses -1 to signify unknown +// controllers. To avoid issues where a client broker ID map knows of +// a -1 ID controller, we start unknown seeds at MinInt32. +func unknownSeedID(seedNum int) int32 { + return int32(math.MinInt32 + seedNum) +} + +func (cl *Client) newBroker(nodeID int32, host string, port int32, rack *string) *broker { + return &broker{ + cl: cl, + + addr: net.JoinHostPort(host, strconv.Itoa(int(port))), + meta: BrokerMetadata{ + NodeID: nodeID, + Host: host, + Port: port, + Rack: rack, + }, + } +} + +// stopForever permanently disables this broker. +func (b *broker) stopForever() { + if b.dead.Swap(true) { + return + } + + b.reqs.die() // no more pushing + + b.reapMu.Lock() + defer b.reapMu.Unlock() + + b.cxnNormal.die() + b.cxnProduce.die() + b.cxnFetch.die() + b.cxnGroup.die() + b.cxnSlow.die() +} + +// do issues a request to the broker, eventually calling the response +// once a the request either fails or is responded to (with failure or not). +// +// The promise will block broker processing. +func (b *broker) do( + ctx context.Context, + req kmsg.Request, + promise func(kmsg.Response, error), +) { + pr := promisedReq{ctx, req, promise, time.Now()} + + first, dead := b.reqs.push(pr) + + if dead { + promise(nil, errChosenBrokerDead) + } else if first { + go b.handleReqs(pr) + } +} + +// waitResp runs a req, waits for the resp and returns the resp and err. +func (b *broker) waitResp(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { + var resp kmsg.Response + var err error + done := make(chan struct{}) + wait := func(kresp kmsg.Response, kerr error) { + resp, err = kresp, kerr + close(done) + } + b.do(ctx, req, wait) + <-done + return resp, err +} + +func (b *broker) handleReqs(pr promisedReq) { + var more, dead bool +start: + if dead { + pr.promise(nil, errChosenBrokerDead) + } else { + b.handleReq(pr) + } + + pr, more, dead = b.reqs.dropPeek() + if more { + goto start + } +} + +func (b *broker) handleReq(pr promisedReq) { + req := pr.req + var cxn *brokerCxn + var retriedOnNewConnection bool +start: + { + var err error + if cxn, err = b.loadConnection(pr.ctx, req); err != nil { + // It is rare, but it is possible that the broker has + // an immediate issue on a new connection. We retry + // once. + if isRetryableBrokerErr(err) && !retriedOnNewConnection { + retriedOnNewConnection = true + goto start + } + pr.promise(nil, err) + return + } + } + + v := b.loadVersions() + + if b.cl.cfg.maxVersions != nil && !b.cl.cfg.maxVersions.HasKey(req.Key()) { + pr.promise(nil, errUnknownRequestKey) + return + } + + // If v.maxVersion(0) is non-negative, then we loaded API + // versions. If the version for this request is negative, we + // know the broker cannot handle this request. + if v.maxVersion(0) >= 0 && v.maxVersion(req.Key()) < 0 { + pr.promise(nil, errBrokerTooOld) + return + } + + ourMax := req.MaxVersion() + ourMin := int16(-1) + if pr, ok := pr.ctx.Value(ctxPinReq).(*pinReq); ok { + if pr.pinMax && pr.max < ourMax { + ourMax = pr.max + } + if pr.pinMin { + ourMin = pr.min + } + } + + // If we have no broker versions, we are pinned pre 0.10.0 and did not + // issue ApiVersions. + if brokerMax := v.maxVersion(req.Key()); brokerMax >= 0 && brokerMax < ourMax { + ourMax = brokerMax + } + if brokerMin := v.minVersion(req.Key()); brokerMin >= 0 && brokerMin > ourMin { + ourMin = brokerMin + } + + if b.cl.cfg.maxVersions != nil { + userMax, _ := b.cl.cfg.maxVersions.LookupMaxKeyVersion(req.Key()) // we validated HasKey above + if userMax < ourMax { + ourMax = userMax + } + } + if b.cl.cfg.minVersions != nil { + userMin, _ := b.cl.cfg.minVersions.LookupMaxKeyVersion(req.Key()) + if userMin > ourMax { + pr.promise(nil, fmt.Errorf("request key %d version returned has max version %d below the user defined min of %d", req.Key(), ourMax, userMin)) + return + } + if userMin > ourMin { + ourMin = userMin + } + } + + // * If we pinned the min high, higher than the broker's max version or + // the user max version, we fail early with errBrokerTooOld. Resharding + // relies on this error. + // + // * If the user set min too high, we already handle that and return a + // targeted non-retryable error just above. We only pin max if we + // cannot reissue with a higher version, so no matter what, if the + // user min is higher than pinned max or broker max, we cannot retry. + // + // * Pinned max is lower than broker's min: We should not get here from + // sharded requests because those *start* high and then downgrade, + // except for AddPartitionsToTxn which outright should not be issued on + // newer brokers (and newer brokers support older requests). Worst + // case, after three spins, the request will stop retrying and fail. + // + // * If the user pinned max lower than the broker min AND there is no + // pinning, this error should be "errBrokerTooNew". This is quite + // unlikely and requires more logic above to differentiate pinned maxes + // vs broker maxes. Technically the broker isn't too old, but we keep + // it simple here. + if ourMin > -1 && ourMin > ourMax { + pr.promise(nil, errBrokerTooOld) // this error is relied on for sharding + return + } + req.SetVersion(ourMax) + + if !cxn.expiry.IsZero() && time.Now().After(cxn.expiry) { + // If we are after the reauth time, try to reauth. We + // can only have an expiry if we went the authenticate + // flow, so we know we are authenticating again. + // + // Some implementations (AWS) occasionally fail for + // unclear reasons (principals change, somehow). If + // we receive SASL_AUTHENTICATION_FAILED, we retry + // once on a new connection. See #249. + // + // For KIP-368. + cxn.cl.cfg.logger.Log(LogLevelDebug, "sasl expiry limit reached, reauthenticating", "broker", logID(cxn.b.meta.NodeID)) + if err := cxn.sasl(); err != nil { + cxn.die() + if errors.Is(err, kerr.SaslAuthenticationFailed) && !retriedOnNewConnection { + cxn.cl.cfg.logger.Log(LogLevelDebug, "sasl reauth failed, retrying once on new connection", "broker", logID(cxn.b.meta.NodeID), "err", err) + retriedOnNewConnection = true + goto start + } + pr.promise(nil, err) + return + } + } + + // Juuuust before we issue the request, we check if it was + // canceled. We could have previously tried this request, which + // then failed and retried. + // + // Checking the context was canceled here ensures we do not + // loop. We could be more precise with error tracking, though. + select { + case <-pr.ctx.Done(): + pr.promise(nil, pr.ctx.Err()) + return + default: + } + + if _, isForceOpen := req.(*forceOpenReq); isForceOpen { + // We issue ApiVersions with v0; we could try to bound the + // version by going to the start above, but it really does + // not matter much. + kreq := kmsg.NewPtrApiVersionsRequest() + kreq.ClientSoftwareName = b.cl.cfg.softwareName + kreq.ClientSoftwareVersion = b.cl.cfg.softwareVersion + req = kreq + } + + // Produce requests (and only produce requests) can be written + // without receiving a reply. If we see required acks is 0, + // then we immediately call the promise with no response. + // + // We provide a non-nil *kmsg.ProduceResponse for + // *kmsg.ProduceRequest just to ensure we do not return with no + // error and no kmsg.Response, per the client contract. + // + // As documented on the client's Request function, if this is a + // *kmsg.ProduceRequest, we rewrite the acks to match the + // client configured acks, and we rewrite the timeout millis if + // acks is 0. We do this to ensure that our discard goroutine + // is used correctly, and so that we do not write a request + // with 0 acks and then send it to handleResps where it will + // not get a response. + var isNoResp bool + var noResp *kmsg.ProduceResponse + switch r := req.(type) { + case *produceRequest: + isNoResp = r.acks == 0 + case *kmsg.ProduceRequest: + r.Acks = b.cl.cfg.acks.val + if r.Acks == 0 { + isNoResp = true + r.TimeoutMillis = int32(b.cl.cfg.produceTimeout.Milliseconds()) + } + noResp = kmsg.NewPtrProduceResponse() + noResp.Version = req.GetVersion() + } + + corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr := cxn.writeRequest(pr.ctx, pr.enqueue, req) + + if writeErr != nil { + cxn.die() + cxn.hookWriteE2E(req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + // If we wrote 0 bytes, the broker never saw the request. + // Safe to retry once on a new connection, same as the + // loadConnection retry above. Does not count against the + // client's retry budget. + if bytesWritten == 0 && !retriedOnNewConnection { + retriedOnNewConnection = true + goto start + } + pr.promise(nil, writeErr) + return + } + + if isNoResp { + pr.promise(noResp, nil) + cxn.hookWriteE2E(req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + return + } + + rt, _ := cxn.cl.connTimeouter.timeouts(req) + + cxn.waitResp(promisedResp{ + pr.ctx, + corrID, + req.IsFlexible() && req.Key() != 18, // response header not flexible if ApiVersions; see promisedResp doc + req.ResponseKind(), + pr.promise, + rt, + bytesWritten, + writeWait, + timeToWrite, + readEnqueue, + }) +} + +func (cxn *brokerCxn) hookWriteE2E(key int16, bytesWritten int, writeWait, timeToWrite time.Duration, writeErr error) { + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerE2E); ok { + h.OnBrokerE2E(cxn.b.meta, key, BrokerE2E{ + BytesWritten: bytesWritten, + WriteWait: writeWait, + TimeToWrite: timeToWrite, + WriteErr: writeErr, + }) + } + }) +} + +// bufPool is used to reuse issued-request buffers across writes to brokers. +type bufPool struct{ p *sync.Pool } + +func newBufPool() bufPool { + return bufPool{ + p: &sync.Pool{New: func() any { r := make([]byte, 1<<10); return &r }}, + } +} + +func (p bufPool) get() []byte { return (*p.p.Get().(*[]byte))[:0] } +func (p bufPool) put(b []byte) { p.p.Put(&b) } + +// loadConnection returns the broker's connection, creating it if necessary +// and returning an error of if that fails. +func (b *broker) loadConnection(ctx context.Context, req kmsg.Request) (*brokerCxn, error) { + var ( + pcxn = &b.cxnNormal + isProduceCxn bool + isFetchCxn bool + reqKey = req.Key() + _, isTimeout = req.(kmsg.TimeoutRequest) + reuse = true + ) + switch { + case reqKey == 0: + pcxn = &b.cxnProduce + isProduceCxn = true + case reqKey == 1 || reqKey == 78: // Fetch or ShareFetch (both long-poll) + pcxn = &b.cxnFetch + isFetchCxn = true + case reqKey == 11 || reqKey == 14: // join || sync + pcxn = &b.cxnGroup + case isTimeout: + pcxn = &b.cxnSlow + } + + // Do not reuse a connection that has been idle for longer than idle timeout. + // Kill it instead. + if *pcxn != nil && !(*pcxn).dead.Load() && (*pcxn).isIdleTimeout(b.cl.cfg.connIdleTimeout) { + // die() in a goroutine to avoid blocking + go (*pcxn).die() + reuse = false + } + + if reuse && *pcxn != nil && !(*pcxn).dead.Load() { + return *pcxn, nil + } + + var tries int + start := time.Now() +doConnect: + tries++ + conn, err := b.connect(ctx) + defer func() { + since := time.Since(start) + b.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerConnect); ok { + h.OnBrokerConnect(b.meta, since, conn, err) + } + }) + }() + if err != nil { + return nil, err + } + + cxn := &brokerCxn{ + cl: b.cl, + b: b, + + addr: b.addr, + conn: conn, + deadCh: make(chan struct{}), + } + if err = cxn.init(isProduceCxn, tries); err != nil { + // EventHubs does not handle v4 and resets the connection. We + // retry twice. On the first and second attempt, we try our max + // version possible (as should be allowed). On the third try, + // we downgrade to v0. + if er := (*errApiVersionsReset)(nil); errors.As(err, &er) { + if tries < 3 { + tries++ + goto doConnect + } + } + b.cl.cfg.logger.Log(LogLevelDebug, "connection initialization failed", "addr", b.addr, "broker", logID(b.meta.NodeID), "err", err) + cxn.closeConn() + return nil, err + } + b.cl.cfg.logger.Log(LogLevelDebug, "connection initialized successfully", "addr", b.addr, "broker", logID(b.meta.NodeID)) + + if isProduceCxn { + b.cl.metrics.observeRate(&b.cl.metrics.pConnCreation) + } else if isFetchCxn { + b.cl.metrics.observeRate(&b.cl.metrics.cConnCreation) + } + + b.reapMu.Lock() + defer b.reapMu.Unlock() + + // If stopForever ran while we were connecting, the broker is + // dead and we must not store the connection. stopForever kills + // cxnProduce/etc under reapMu, but if the connection was nil at + // that time (we were mid-connect), stopForever's die() was a + // no-op. Without this check, the connection escapes destruction + // and a produce request succeeds on a connection that will never + // be reused, which -- combined with other connections from other + // broker objects for the same nodeID -- breaks the single- + // connection-per-broker ordering guarantee that Kafka requires + // for idempotent produce. + if b.dead.Load() { + cxn.closeConn() + return nil, errChosenBrokerDead + } + + *pcxn = cxn + return cxn, nil +} + +func (cl *Client) reapConnectionsLoop() { + idleTimeout := cl.cfg.connIdleTimeout + if idleTimeout < 0 { // impossible due to cfg.validate, but just in case + return + } + + ticker := time.NewTicker(idleTimeout) + defer ticker.Stop() + last := time.Now() + for { + select { + case <-cl.ctx.Done(): + return + case tick := <-ticker.C: + start := time.Now() + reaped := cl.reapConnections(idleTimeout) + dur := time.Since(start) + if reaped > 0 { + cl.cfg.logger.Log(LogLevelDebug, "reaped connections", "time_since_last_reap", tick.Sub(last), "reap_dur", dur, "num_reaped", reaped) + } + last = tick + } + } +} + +func (cl *Client) reapConnections(idleTimeout time.Duration) (total int) { + cl.brokersMu.Lock() + seeds := cl.loadSeeds() + brokers := make([]*broker, 0, len(cl.brokers)+len(seeds)) + brokers = append(brokers, cl.brokers...) + brokers = append(brokers, seeds...) + cl.brokersMu.Unlock() + + for _, broker := range brokers { + total += broker.reapConnections(idleTimeout) + } + return total +} + +func (b *broker) reapConnections(idleTimeout time.Duration) (total int) { + b.reapMu.Lock() + defer b.reapMu.Unlock() + + for _, cxn := range []*brokerCxn{ + b.cxnNormal, + b.cxnProduce, + b.cxnFetch, + b.cxnGroup, + b.cxnSlow, + } { + if cxn == nil || cxn.dead.Load() { + continue + } + + // If we have not written nor read in a long time, the + // connection can be reaped. If only one is idle, the other may + // be busy (or may not happen): + // + // - produce can write but never read + // - fetch can hang for a while reading (infrequent writes) + if cxn.isIdleTimeout(idleTimeout) { + cxn.die() + total++ + } + } + return total +} + +// connect connects to the broker's addr, returning the new connection. +func (b *broker) connect(ctx context.Context) (net.Conn, error) { + b.cl.cfg.logger.Log(LogLevelDebug, "opening connection to broker", "addr", b.addr, "broker", logID(b.meta.NodeID)) + conn, err := b.cl.cfg.dialFn(ctx, "tcp", b.addr) + if err != nil { + if !errors.Is(err, ErrClientClosed) && !errors.Is(err, context.Canceled) && !strings.Contains(err.Error(), "operation was canceled") { + if errors.Is(err, io.EOF) { + b.cl.cfg.logger.Log(LogLevelWarn, "unable to open connection to broker due to an immediate EOF, which often means the client is using TLS when the broker is not expecting it (is TLS misconfigured?)", "addr", b.addr, "broker", logID(b.meta.NodeID), "err", err) + return nil, &ErrFirstReadEOF{kind: firstReadDial, err: err, retry: b.cl.cfg.alwaysRetryEOF} + } + b.cl.cfg.logger.Log(LogLevelWarn, "unable to open connection to broker", "addr", b.addr, "broker", logID(b.meta.NodeID), "err", err) + } + return nil, fmt.Errorf("unable to dial: %w", err) + } + b.cl.cfg.logger.Log(LogLevelDebug, "connection opened to broker", "addr", b.addr, "broker", logID(b.meta.NodeID)) + return conn, nil +} + +// brokerCxn manages an actual connection to a Kafka broker. This is separate +// the broker struct to allow lazy connection (re)creation. +type brokerCxn struct { + throttleUntil atomic.Int64 // atomic nanosec + + conn net.Conn + + cl *Client + b *broker + + addr string + + mechanism sasl.Mechanism + expiry time.Time + + corrID int32 + + // The following four fields are used for connection reaping. + // Write is only updated in one location; read is updated in three + // due to readConn, readConnAsync, and discard. + lastWrite atomic.Int64 + lastRead atomic.Int64 + writing atomic.Bool + reading atomic.Bool + + successes uint64 + + sizeBuf [4]byte // reused in readConn, tiny win + + // resps manages reading kafka responses. + resps ring[promisedResp] + // dead is an atomic so that a backed up resps cannot block cxn death. + dead atomic.Bool + // closed in cloneConn; allows throttle waiting to quit + deadCh chan struct{} +} + +func (cxn *brokerCxn) init(isProduceCxn bool, tries int) error { + // We always send ApiVersions on every new connection, even if we have + // already cached the broker's versions from a previous connection. + // ApiVersions is how we advertise our ClientSoftwareName/Version to the + // broker for the lifetime of this connection (KIP-714 client metrics + // match on these). If we skip it on a reused-broker connection, that + // connection registers as "unknown" software on the broker side, and + // any broker-side metric subscriptions scoped to our software name + // silently miss it. See twmb/franz-go#1296. + if cxn.b.cl.cfg.maxVersions == nil || cxn.b.cl.cfg.maxVersions.HasKey(18) { + if err := cxn.requestAPIVersions(tries); err != nil { + if !errors.Is(err, ErrClientClosed) && !isRetryableBrokerErr(err) { + cxn.cl.cfg.logger.Log(LogLevelError, "unable to request api versions", "broker", logID(cxn.b.meta.NodeID), "err", err) + } + return err + } + } else if cxn.b.loadVersions() == nil { + // We have a max versions, and it indicates no support for + // ApiVersions. We just store a default empty map (once). + cxn.b.storeVersions(newBrokerVersions(0)) + } + + if err := cxn.sasl(); err != nil { + if !errors.Is(err, ErrClientClosed) && !isRetryableBrokerErr(err) { + cxn.cl.cfg.logger.Log(LogLevelError, "unable to initialize sasl", "broker", logID(cxn.b.meta.NodeID), "err", err) + } + return err + } + + if isProduceCxn && cxn.cl.cfg.acks.val == 0 { + go cxn.discard() // see docs on discard for why we do this + } + return nil +} + +func (cxn *brokerCxn) requestAPIVersions(tries int) error { + maxVersion := int16(4) + if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization + maxVersion = 0 + } else if cxn.cl.cfg.maxVersions != nil { + // If the user configured a max versions, we check that the key exists + // before entering this function. Thus, we expect exists to be true, + // but we still doubly check it for sanity (as well as userMax, which + // can only be non-negative based off of LookupMaxKeyVersion's API). + userMax, exists := cxn.cl.cfg.maxVersions.LookupMaxKeyVersion(18) // 18 == api versions + if exists && userMax >= 0 { + maxVersion = userMax + } + } + +start: + req := kmsg.NewPtrApiVersionsRequest() + req.Version = maxVersion + req.ClientSoftwareName = cxn.cl.cfg.softwareName + req.ClientSoftwareVersion = cxn.cl.cfg.softwareVersion + cxn.cl.cfg.logger.Log(LogLevelDebug, "issuing api versions request", "broker", logID(cxn.b.meta.NodeID), "version", maxVersion) + corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr := cxn.writeRequest(nil, time.Now(), req) + if writeErr != nil { + cxn.hookWriteE2E(req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + return writeErr + } + + rt, _ := cxn.cl.connTimeouter.timeouts(req) + // api versions does *not* use flexible response headers; see comment in promisedResp + rawResp, err := cxn.readResponse(nil, req.Key(), req.GetVersion(), corrID, false, rt, bytesWritten, writeWait, timeToWrite, readEnqueue) + if err != nil { + var errno syscall.Errno + if errors.As(err, &errno) && isConnReset(errno) { + return &errApiVersionsReset{err} + } else if errors.Is(err, io.EOF) { + cxn.b.cl.cfg.logger.Log(LogLevelWarn, "read from broker received EOF during api versions discovery, which often happens when the broker requires TLS and the client is not using it (is TLS misconfigured?)", "addr", cxn.b.addr, "broker", logID(cxn.b.meta.NodeID), "err", err) + err = &ErrFirstReadEOF{kind: firstReadTLS, err: err, retry: cxn.b.cl.cfg.alwaysRetryEOF} + } + return err + } + if len(rawResp) < 2 { + return fmt.Errorf("invalid length %d short response from ApiVersions request", len(rawResp)) + } + + resp := req.ResponseKind().(*kmsg.ApiVersionsResponse) + + // If we used a version larger than Kafka supports, Kafka replies with + // Version 0 and an UNSUPPORTED_VERSION error. + // + // Pre Kafka 2.4, we have to retry the request with version 0. + // Post, Kafka replies with the version we should retry with (KIP-511). + if rawResp[1] == 35 { + if maxVersion == 0 { + return errors.New("broker replied with UNSUPPORTED_VERSION to an ApiVersions request of version 0") + } + + resp.Version = 0 + if err = resp.ReadFrom(rawResp); err != nil { + return fmt.Errorf("unable to read ApiVersions response: %w", err) + } + switch { + case len(resp.ApiKeys) == 0: + maxVersion = 0 + cxn.cl.cfg.logger.Log(LogLevelDebug, "broker does not know our ApiVersions version, downgrading to version 0 and retrying", "broker", logID(cxn.b.meta.NodeID)) + goto start + case len(resp.ApiKeys) == 1 && resp.ApiKeys[0].ApiKey == 18: + maxVersion = resp.ApiKeys[0].MaxVersion + cxn.cl.cfg.logger.Log(LogLevelDebug, fmt.Sprintf("broker does not know our ApiVersions version but replied version %[1]d, downgrading to version %[1]d and retrying", maxVersion), "broker", logID(cxn.b.meta.NodeID)) + goto start + default: + // Should not hit this case, but we hope the broker replied with all keys + } + resp = req.ResponseKind().(*kmsg.ApiVersionsResponse) + resp.Version = 0 + } + + if err = resp.ReadFrom(rawResp); err != nil { + return fmt.Errorf("unable to read ApiVersions response: %w", err) + } + if len(resp.ApiKeys) == 0 { + return errors.New("ApiVersions response invalidly contained no ApiKeys") + } + + v := newBrokerVersions(len(resp.ApiKeys)) + for _, key := range resp.ApiKeys { + v.maxVers[key.ApiKey] = key.MaxVersion + v.minVers[key.ApiKey] = key.MinVersion + } + if resp.FinalizedFeaturesEpoch != -1 { + for _, feat := range resp.FinalizedFeatures { + v.features[feat.Name] = feat.MaxVersionLevel + } + } + + cxn.b.storeVersions(v) + return nil +} + +func (cxn *brokerCxn) sasl() error { + if len(cxn.cl.cfg.sasls) == 0 { + return nil + } + mechanism := cxn.cl.cfg.sasls[0] + retried := false + authenticate := false + + v := cxn.b.loadVersions() + req := kmsg.NewPtrSASLHandshakeRequest() + +start: + // KIP-152 establishes the modern SASL flow: ApiVersions, then + // SaslHandshake, then SaslAuthenticate for all mechanisms. The + // legacy raw GSSAPI flow (where GSSAPI clients could skip the + // handshake) only worked when GSSAPI bytes were the *first* packet + // on the connection. Since we send ApiVersions first, we must send + // SaslHandshake for all mechanisms including GSSAPI. KIP-896 removed + // support for the legacy raw GSSAPI protocol entirely in Kafka 4.0. + if v.maxVersion(req.Key()) >= 0 { + req.Mechanism = mechanism.Name() + req.Version = v.maxVersion(req.Key()) + cxn.cl.cfg.logger.Log(LogLevelDebug, "issuing SASLHandshakeRequest", "broker", logID(cxn.b.meta.NodeID)) + corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr := cxn.writeRequest(nil, time.Now(), req) + if writeErr != nil { + cxn.hookWriteE2E(req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + return writeErr + } + + rt, _ := cxn.cl.connTimeouter.timeouts(req) + rawResp, err := cxn.readResponse(nil, req.Key(), req.GetVersion(), corrID, req.IsFlexible(), rt, bytesWritten, writeWait, timeToWrite, readEnqueue) + if err != nil { + return err + } + resp := req.ResponseKind().(*kmsg.SASLHandshakeResponse) + if err = resp.ReadFrom(rawResp); err != nil { + return err + } + + err = kerr.ErrorForCode(resp.ErrorCode) + if err != nil { + if !retried && err == kerr.UnsupportedSaslMechanism { + for _, ours := range cxn.cl.cfg.sasls[1:] { + for _, supported := range resp.SupportedMechanisms { + if supported == ours.Name() { + mechanism = ours + retried = true + goto start + } + } + } + } + return err + } + authenticate = req.Version == 1 + } + cxn.cl.cfg.logger.Log(LogLevelDebug, "beginning sasl authentication", "broker", logID(cxn.b.meta.NodeID), "addr", cxn.addr, "mechanism", mechanism.Name(), "authenticate", authenticate) + cxn.mechanism = mechanism + return cxn.doSasl(authenticate) +} + +func (cxn *brokerCxn) doSasl(authenticate bool) error { + session, clientWrite, err := cxn.mechanism.Authenticate(cxn.cl.ctx, cxn.addr) + if err != nil { + return err + } + if len(clientWrite) == 0 { + return fmt.Errorf("unexpected server-write sasl with mechanism %s", cxn.mechanism.Name()) + } + + prereq := time.Now() // used below for sasl lifetime calculation + var lifetimeMillis int64 + + // Even if we do not wrap our reads/writes in SASLAuthenticate, we + // still use the SASLAuthenticate timeouts. + rt, wt := cxn.cl.connTimeouter.timeouts(kmsg.NewPtrSASLAuthenticateRequest()) + + // We continue writing until both the challenging is done AND the + // responses are done. We can have an additional response once we + // are done with challenges. + step := -1 + for done := false; !done || len(clientWrite) > 0; { + step++ + var challenge []byte + + if !authenticate { + buf := cxn.cl.bufPool.get() + + buf = append(buf[:0], 0, 0, 0, 0) + binary.BigEndian.PutUint32(buf, uint32(len(clientWrite))) + buf = append(buf, clientWrite...) + + cxn.cl.cfg.logger.Log(LogLevelDebug, "issuing raw sasl authenticate", "broker", logID(cxn.b.meta.NodeID), "addr", cxn.addr, "step", step) + _, _, _, _, err = cxn.writeConn(context.Background(), buf, wt, time.Now()) + + cxn.cl.bufPool.put(buf) + + if err != nil { + return err + } + if !done { + if _, challenge, _, _, err = cxn.readConn(context.Background(), rt, time.Now()); err != nil { + return err + } + } + } else { + req := kmsg.NewPtrSASLAuthenticateRequest() + req.SASLAuthBytes = clientWrite + req.Version = cxn.b.loadVersions().maxVersion(req.Key()) + cxn.cl.cfg.logger.Log(LogLevelDebug, "issuing SASLAuthenticate", "broker", logID(cxn.b.meta.NodeID), "version", req.Version, "step", step) + + // Lifetime: we take the timestamp before we write our + // request; see usage below for why. + prereq = time.Now() + corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr := cxn.writeRequest(nil, time.Now(), req) + if writeErr != nil { + cxn.hookWriteE2E(req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + return writeErr + } + + // Unlike the raw SASL path, SaslAuthenticate always has + // a response for every request. Per KIP-152, "server + // always sends a response to each SASL_AUTHENTICATE + // request". We must read this response even when the + // SASL session is complete to avoid leaving data in the + // connection buffer. + rawResp, err := cxn.readResponse(nil, req.Key(), req.GetVersion(), corrID, req.IsFlexible(), rt, bytesWritten, writeWait, timeToWrite, readEnqueue) + if err != nil { + return err + } + resp := req.ResponseKind().(*kmsg.SASLAuthenticateResponse) + if err = resp.ReadFrom(rawResp); err != nil { + return err + } + + if err := errCodeMessage(resp.ErrorCode, resp.ErrorMessage); err != nil { + return err + } + challenge = resp.SASLAuthBytes + lifetimeMillis = resp.SessionLifetimeMillis + } + + clientWrite = nil + + if !done { + if done, clientWrite, err = session.Challenge(challenge); err != nil { + return err + } + } + } + + if lifetimeMillis > 0 { + // Lifetime is problematic. We need to be a bit pessimistic. + // + // We want a lowerbound: we use 1s (arbitrary), but if 1.1x our + // e2e sasl latency is more than 1s, we use the latency. + // + // We do not want to reauthenticate too close to the lifetime + // especially for larger lifetimes due to clock issues (#205). + // We take 95% to 98% of the lifetime. + minPessimismMillis := float64(time.Second.Milliseconds()) + latencyMillis := 1.1 * float64(time.Since(prereq).Milliseconds()) + if latencyMillis > minPessimismMillis { + minPessimismMillis = latencyMillis + } + var random float64 + cxn.b.cl.rng(func(r *rand.Rand) { random = r.Float64() }) + maxPessimismMillis := float64(lifetimeMillis) * (0.05 - 0.03*random) // 95 to 98% of lifetime (pessimism 2% to 5%) + + // Our minimum lifetime is always 1s (or latency, if larger). + // When our max pessimism becomes more than min pessimism, + // every second after, we add between 0.05s or 0.08s to our + // backoff. At 12hr, we reauth ~24 to 28min before the + // lifetime. + usePessimismMillis := max(minPessimismMillis, maxPessimismMillis) + useLifetimeMillis := lifetimeMillis - int64(usePessimismMillis) + + // Subtracting our min pessimism may result in our connection + // immediately expiring. We always accept this one reauth to + // issue our one request, and our next request will again + // reauth. Brokers should give us longer lifetimes, but that + // may not always happen (see #136, #249). + now := time.Now() + cxn.expiry = now.Add(time.Duration(useLifetimeMillis) * time.Millisecond) + cxn.cl.cfg.logger.Log(LogLevelDebug, "sasl has a limited lifetime", + "broker", logID(cxn.b.meta.NodeID), + "session_lifetime", time.Duration(lifetimeMillis)*time.Millisecond, + "lifetime_pessimism", time.Duration(usePessimismMillis)*time.Millisecond, + "reauthenticate_in", cxn.expiry.Sub(now), + ) + } + return nil +} + +// Some internal requests use the client context to issue requests, so if the +// client is closed, this select case can be selected. We want to return the +// proper error. +// +// This function is used in this file anywhere the client context can cause +// ErrClientClosed. +func maybeUpdateCtxErr(clientCtx, reqCtx context.Context, err *error) { + if clientCtx == reqCtx { + *err = ErrClientClosed + } +} + +// writeRequest writes a message request to the broker connection, bumping the +// connection's correlation ID as appropriate for the next write. +func (cxn *brokerCxn) writeRequest(ctx context.Context, enqueuedForWritingAt time.Time, req kmsg.Request) (corrID int32, bytesWritten int, writeWait, timeToWrite time.Duration, readEnqueue time.Time, writeErr error) { + // A nil ctx means we cannot be throttled. + if ctx != nil { + throttleUntil := time.Unix(0, cxn.throttleUntil.Load()) + if sleep := time.Until(throttleUntil); sleep > 0 { + after := time.NewTimer(sleep) + select { + case <-after.C: + case <-ctx.Done(): + writeErr = ctx.Err() + maybeUpdateCtxErr(cxn.cl.ctx, ctx, &writeErr) + case <-cxn.cl.ctx.Done(): + writeErr = ErrClientClosed + case <-cxn.deadCh: + writeErr = errChosenBrokerDead + } + if writeErr != nil { + after.Stop() + writeWait = time.Since(enqueuedForWritingAt) + return corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr + } + } + } + + buf := cxn.cl.reqFormatter.AppendRequest( + cxn.cl.bufPool.get()[:0], + req, + cxn.corrID, + ) + + _, wt := cxn.cl.connTimeouter.timeouts(req) + bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr = cxn.writeConn(ctx, buf, wt, enqueuedForWritingAt) + + cxn.cl.bufPool.put(buf) + + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerWrite); ok { + h.OnBrokerWrite(cxn.b.meta, req.Key(), bytesWritten, writeWait, timeToWrite, writeErr) + } + }) + if logger := cxn.cl.cfg.logger; logger.Level() >= LogLevelDebug { + logger.Log(LogLevelDebug, fmt.Sprintf("wrote %s v%d", kmsg.NameForKey(req.Key()), req.GetVersion()), "broker", logID(cxn.b.meta.NodeID), "bytes_written", bytesWritten, "write_wait", writeWait, "time_to_write", timeToWrite, "err", writeErr) + } + + if writeErr != nil { + return corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr + } + corrID = cxn.corrID + cxn.corrID++ + if cxn.corrID < 0 { + cxn.corrID = 0 + } + return corrID, bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr +} + +func (cxn *brokerCxn) writeConn( + ctx context.Context, + buf []byte, + timeout time.Duration, + enqueuedForWritingAt time.Time, +) (bytesWritten int, writeWait, timeToWrite time.Duration, readEnqueue time.Time, writeErr error) { + cxn.writing.Store(true) + defer func() { + cxn.lastWrite.Store(time.Now().UnixNano()) + cxn.writing.Store(false) + }() + + if ctx == nil { + ctx = context.Background() + } + if timeout > 0 { + cxn.conn.SetWriteDeadline(time.Now().Add(timeout)) + } + defer cxn.conn.SetWriteDeadline(time.Time{}) + writeDone := make(chan struct{}) + go func() { + defer close(writeDone) + writeStart := time.Now() + bytesWritten, writeErr = cxn.conn.Write(buf) + // As soon as we are done writing, we track that we have now + // enqueued this request for reading. + readEnqueue = time.Now() + writeWait = writeStart.Sub(enqueuedForWritingAt) + timeToWrite = readEnqueue.Sub(writeStart) + }() + select { + case <-writeDone: + case <-cxn.cl.ctx.Done(): + cxn.conn.SetWriteDeadline(time.Now()) + <-writeDone + if writeErr != nil { + writeErr = ErrClientClosed + } + case <-ctx.Done(): + cxn.conn.SetWriteDeadline(time.Now()) + <-writeDone + if writeErr != nil && ctx.Err() != nil { + writeErr = ctx.Err() + maybeUpdateCtxErr(cxn.cl.ctx, ctx, &writeErr) + } + } + return bytesWritten, writeWait, timeToWrite, readEnqueue, writeErr +} + +func (cxn *brokerCxn) readConn( + ctx context.Context, + timeout time.Duration, + enqueuedForReadingAt time.Time, +) (nread int, buf []byte, readWait, timeToRead time.Duration, err error) { + cxn.reading.Store(true) + defer func() { + cxn.lastRead.Store(time.Now().UnixNano()) + cxn.reading.Store(false) + }() + + if ctx == nil { + ctx = context.Background() + } + if timeout > 0 { + cxn.conn.SetReadDeadline(time.Now().Add(timeout)) + } + defer cxn.conn.SetReadDeadline(time.Time{}) + readDone := make(chan struct{}) + go func() { + defer close(readDone) + readStart := time.Now() + defer func() { + timeToRead = time.Since(readStart) + readWait = readStart.Sub(enqueuedForReadingAt) + }() + if nread, err = io.ReadFull(cxn.conn, cxn.sizeBuf[:]); err != nil { + return + } + var size int32 + if size, err = cxn.parseReadSize(cxn.sizeBuf[:]); err != nil { + return + } + buf = make([]byte, size) + var nread2 int + nread2, err = io.ReadFull(cxn.conn, buf) + nread += nread2 + buf = buf[:nread2] + if err != nil { + return + } + }() + select { + case <-readDone: + case <-cxn.cl.ctx.Done(): + cxn.conn.SetReadDeadline(time.Now()) + <-readDone + if err != nil { + err = ErrClientClosed + } + case <-ctx.Done(): + cxn.conn.SetReadDeadline(time.Now()) + <-readDone + if err != nil && ctx.Err() != nil { + err = ctx.Err() + maybeUpdateCtxErr(cxn.cl.ctx, ctx, &err) + } + } + return nread, buf, readWait, timeToRead, err +} + +// Parses a length 4 slice and enforces the min / max read size based off the +// client configuration. +func (cxn *brokerCxn) parseReadSize(sizeBuf []byte) (int32, error) { + size := int32(binary.BigEndian.Uint32(sizeBuf)) + if size < 0 { + return 0, fmt.Errorf("invalid negative response size %d", size) + } + if maxSize := cxn.b.cl.cfg.maxBrokerReadBytes; size > maxSize { + if size == 0x48545450 { // "HTTP" + return 0, fmt.Errorf("invalid large response size %d > limit %d; the four size bytes are 'HTTP' in ascii, the beginning of an HTTP response; is your broker port correct?", size, maxSize) + } + // A TLS alert is 21, and a TLS alert has the version + // following, where all major versions are 03xx. We + // look for an alert and major version byte to suspect + // if this we received a TLS alert. + tlsVersion := uint16(sizeBuf[1])<<8 | uint16(sizeBuf[2]) + if sizeBuf[0] == 21 && tlsVersion&0x0300 != 0 { + versionGuess := fmt.Sprintf("unknown TLS version (hex %x)", tlsVersion) + for _, guess := range []struct { + num uint16 + text string + }{ + {tls.VersionSSL30, "SSL v3"}, + {tls.VersionTLS10, "TLS v1.0"}, + {tls.VersionTLS11, "TLS v1.1"}, + {tls.VersionTLS12, "TLS v1.2"}, + {tls.VersionTLS13, "TLS v1.3"}, + } { + if tlsVersion == guess.num { + versionGuess = guess.text + } + } + return 0, fmt.Errorf("invalid large response size %d > limit %d; the first three bytes received appear to be a tls alert record for %s; is this a plaintext connection speaking to a tls endpoint?", size, maxSize, versionGuess) + } + return 0, fmt.Errorf("invalid large response size %d > limit %d", size, maxSize) + } + return size, nil +} + +// readResponse reads a response from conn, ensures the correlation ID is +// correct, and returns a newly allocated slice on success. +// +// This takes a bunch of extra arguments in support of HookBrokerE2E, overall +// this function takes 11 bytes in arguments. +func (cxn *brokerCxn) readResponse( + ctx context.Context, + key int16, + version int16, + corrID int32, + flexibleHeader bool, + timeout time.Duration, + bytesWritten int, + writeWait time.Duration, + timeToWrite time.Duration, + readEnqueue time.Time, +) ([]byte, error) { + bytesRead, buf, readWait, timeToRead, readErr := cxn.readConn(ctx, timeout, readEnqueue) + + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerRead); ok { + h.OnBrokerRead(cxn.b.meta, key, bytesRead, readWait, timeToRead, readErr) + } + if h, ok := h.(HookBrokerE2E); ok { + h.OnBrokerE2E(cxn.b.meta, key, BrokerE2E{ + BytesWritten: bytesWritten, + BytesRead: bytesRead, + WriteWait: writeWait, + TimeToWrite: timeToWrite, + ReadWait: readWait, + TimeToRead: timeToRead, + ReadErr: readErr, + }) + } + }) + + if readErr == nil { + latencyMillis := (writeWait + timeToWrite + readWait + timeToRead).Milliseconds() + if key == 0 { //nolint:staticcheck // tagged switch not needed for one case... + cxn.b.cl.metrics.observeNodeTime(cxn.b.meta.NodeID, &cxn.b.cl.metrics.pReqLatency, latencyMillis) + } else if key == 1 { + cxn.b.cl.metrics.observeNodeTime(cxn.b.meta.NodeID, &cxn.b.cl.metrics.cReqLatency, latencyMillis) + } + } + + if logger := cxn.cl.cfg.logger; logger.Level() >= LogLevelDebug { + logger.Log(LogLevelDebug, fmt.Sprintf("read %s v%d", kmsg.NameForKey(key), version), "broker", logID(cxn.b.meta.NodeID), "bytes_read", bytesRead, "read_wait", readWait, "time_to_read", timeToRead, "err", readErr) + } + + if readErr != nil { + return nil, readErr + } + if len(buf) < 4 { + return nil, kbin.ErrNotEnoughData + } + gotID := int32(binary.BigEndian.Uint32(buf)) + if gotID != corrID { + return nil, errCorrelationIDMismatch + } + // If the response header is flexible, we skip the tags at the end of + // it. They are currently unused. + if flexibleHeader { + b := kbin.Reader{Src: buf[4:]} + kmsg.SkipTags(&b) + return b.Src, b.Complete() + } + return buf[4:], nil +} + +// closeConn is the one place we close broker connections. This is always done +// in either die, which is called when handleResps returns, or if init fails, +// which means we did not succeed enough to start handleResps. +func (cxn *brokerCxn) closeConn() { + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerDisconnect); ok { + h.OnBrokerDisconnect(cxn.b.meta, cxn.conn) + } + }) + cxn.conn.Close() + close(cxn.deadCh) +} + +// die kills a broker connection (which could be dead already) and replies to +// all requests awaiting responses appropriately. +func (cxn *brokerCxn) die() { + if cxn == nil || cxn.dead.Swap(true) { + return + } + cxn.closeConn() + cxn.resps.die() +} + +// waitResp, called serially by a broker's handleReqs, manages handling a +// message requests's response. +func (cxn *brokerCxn) waitResp(pr promisedResp) { + first, dead := cxn.resps.push(pr) + if first { + go cxn.handleResps(pr) + } else if dead { + pr.promise(nil, errChosenBrokerDead) + cxn.hookWriteE2E(pr.resp.Key(), pr.bytesWritten, pr.writeWait, pr.timeToWrite, errChosenBrokerDead) + } +} + +// If acks are zero, then a real Kafka installation never replies to produce +// requests. Unfortunately, Microsoft EventHubs rolled their own implementation +// and _does_ reply to ack-0 produce requests. We need to process these +// responses, because otherwise kernel buffers will fill up, Microsoft will be +// unable to reply, and then they will stop taking our produce requests. +// +// Thus, we just simply discard everything. +// +// Since we still want to support hooks, we still read the size of a response +// and then read that entire size before calling a hook. There are a few +// differences: +// +// (1) we do not know what version we produced, so we cannot validate the read, +// we just have to trust that the size is valid (and the data follows +// correctly). +// +// (2) rather than creating a slice for the response, we discard the entire +// response into a reusable small slice. The small size is because produce +// responses are relatively small to begin with, so we expect only a few reads +// per response. +// +// (3) we have no time for when the read was enqueued, so we miss that in the +// hook. +// +// (4) we start the time-to-read duration *after* the size bytes are read, +// since we have no idea when a read actually should start, since we should not +// receive responses to begin with. +// +// (5) we set a read deadline *after* the size bytes are read, and only if the +// client has not yet closed. +func (cxn *brokerCxn) discard() { + var firstTimeout bool + defer func() { + if !firstTimeout { // see below + cxn.die() + } else { + cxn.b.cl.cfg.logger.Log(LogLevelDebug, "produce acks==0 discard goroutine exiting; this broker looks to correctly not reply to ack==0 produce requests", "addr", cxn.b.addr, "broker", logID(cxn.b.meta.NodeID)) + } + }() + + discardBuf := make([]byte, 256) + for i := 0; ; i++ { + var ( + nread int + err error + timeToRead time.Duration + + deadlineMu xsync.Mutex + deadlineSet bool + + readDone = make(chan struct{}) + ) + + // On all but the first request, we use no deadline. We could + // be hanging reading while we wait for more produce requests. + // We know we are talking to azure when i > 0 and we should not + // quit this goroutine. + // + // However, on the *first* produce request, we know that we are + // writing *right now*. We can deadline our read side with + // ample overhead, and if this first read hits the deadline, + // then we can quit this discard / read goroutine with no + // problems. + // + // We choose 3x our timeouts: + // - first we cover the write, connTimeoutOverhead + produceTimeout + // - then we cover the read, connTimeoutOverhead + // - then we throw in another connTimeoutOverhead just to be sure + // + deadline := time.Time{} + if i == 0 { + deadline = time.Now().Add(3*cxn.cl.cfg.requestTimeoutOverhead + cxn.cl.cfg.produceTimeout) + } + cxn.conn.SetReadDeadline(deadline) + + go func() { + defer close(readDone) + if nread, err = io.ReadFull(cxn.conn, discardBuf[:4]); err != nil { + if i == 0 && errors.Is(err, os.ErrDeadlineExceeded) { + firstTimeout = true + } + return + } + deadlineMu.Lock() + if !deadlineSet { + cxn.conn.SetReadDeadline(time.Now().Add(cxn.cl.cfg.produceTimeout)) + } + deadlineMu.Unlock() + + cxn.reading.Store(true) + defer func() { + cxn.lastRead.Store(time.Now().UnixNano()) + cxn.reading.Store(false) + }() + + readStart := time.Now() + defer func() { timeToRead = time.Since(readStart) }() + var size int32 + if size, err = cxn.parseReadSize(discardBuf[:4]); err != nil { + return + } + + var nread2 int + for size > 0 && err == nil { + discard := discardBuf + if int(size) < len(discard) { + discard = discard[:size] + } + nread2, err = cxn.conn.Read(discard) + nread += nread2 + size -= int32(nread2) // nread2 max is 128 + } + }() + + select { + case <-readDone: + case <-cxn.cl.ctx.Done(): + deadlineMu.Lock() + deadlineSet = true + deadlineMu.Unlock() + cxn.conn.SetReadDeadline(time.Now()) + <-readDone + return + } + + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerRead); ok { + h.OnBrokerRead(cxn.b.meta, 0, nread, 0, timeToRead, err) + } + }) + if err != nil { + return + } + } +} + +// handleResps serially handles all broker responses for an single connection. +func (cxn *brokerCxn) handleResps(pr promisedResp) { + var more, dead bool +start: + if dead { + pr.promise(nil, errChosenBrokerDead) + cxn.hookWriteE2E(pr.resp.Key(), pr.bytesWritten, pr.writeWait, pr.timeToWrite, errChosenBrokerDead) + } else { + cxn.handleResp(pr) + } + + pr, more, dead = cxn.resps.dropPeek() + if more { + goto start + } +} + +func (cxn *brokerCxn) handleResp(pr promisedResp) { + rawResp, err := cxn.readResponse( + pr.ctx, + pr.resp.Key(), + pr.resp.GetVersion(), + pr.corrID, + pr.flexibleHeader, + pr.readTimeout, + pr.bytesWritten, + pr.writeWait, + pr.timeToWrite, + pr.readEnqueue, + ) + if err != nil { + if !errors.Is(err, ErrClientClosed) && !errors.Is(err, context.Canceled) { + if cxn.successes > 0 || len(cxn.b.cl.cfg.sasls) > 0 { + cxn.b.cl.cfg.logger.Log(LogLevelDebug, "read from broker errored, killing connection", "req", kmsg.Key(pr.resp.Key()).Name(), "addr", cxn.b.addr, "broker", logID(cxn.b.meta.NodeID), "successful_reads", cxn.successes, "err", err) + } else { + cxn.b.cl.cfg.logger.Log(LogLevelWarn, "read from broker errored, killing connection after 0 successful responses (is SASL missing?)", "req", kmsg.Key(pr.resp.Key()).Name(), "addr", cxn.b.addr, "broker", logID(cxn.b.meta.NodeID), "err", err) + if err == io.EOF { // specifically avoid checking errors.Is to ensure this is not already wrapped + err = &ErrFirstReadEOF{kind: firstReadSASL, err: err, retry: cxn.b.cl.cfg.alwaysRetryEOF} + } + } + } else { + cxn.b.cl.cfg.logger.Log(LogLevelDebug, "read from broker canceled, closing connection and killing any other in-flight requests on this connection", "req", kmsg.Key(pr.resp.Key()).Name(), "addr", cxn.b.addr, "broker", logID(cxn.b.meta.NodeID), "err", err) + } + pr.promise(nil, err) + cxn.die() + return + } + + if pr.resp.Key() == 18 && len(rawResp) > 2 && rawResp[1] == 35 { + cxn.b.cl.cfg.logger.Log(LogLevelDebug, "ApiVersions response returned with UNSUPPORTED_VERSION error at byte 1, downgraded to v0 to deserialize response") + pr.resp.SetVersion(0) + } + + cxn.successes++ + readErr := pr.resp.ReadFrom(rawResp) + + // If we had no error, we read the response successfully. + // + // Any response that can cause throttling satisfies the + // kmsg.ThrottleResponse interface. We check that here. + if readErr == nil { + if throttleResponse, ok := pr.resp.(kmsg.ThrottleResponse); ok { + millis, throttlesAfterResp := throttleResponse.Throttle() + if millis > 0 { + if pr.resp.Key() == 0 { + cxn.b.cl.metrics.observeTime(&cxn.b.cl.metrics.pThrottle, int64(millis)) + } + cxn.b.cl.cfg.logger.Log(LogLevelInfo, "broker is throttling us in response", "broker", logID(cxn.b.meta.NodeID), "req", kmsg.Key(pr.resp.Key()).Name(), "throttle_millis", millis, "throttles_after_resp", throttlesAfterResp) + if throttlesAfterResp { + throttleUntil := time.Now().Add(time.Millisecond * time.Duration(millis)).UnixNano() + if throttleUntil > cxn.throttleUntil.Load() { + cxn.throttleUntil.Store(throttleUntil) + } + } + cxn.cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookBrokerThrottle); ok { + h.OnBrokerThrottle(cxn.b.meta, time.Duration(millis)*time.Millisecond, throttlesAfterResp) + } + }) + } + } + } + + pr.promise(pr.resp, readErr) +} + +func (cxn *brokerCxn) isIdleTimeout(idleTimeout time.Duration) bool { + lastWrite := time.Unix(0, cxn.lastWrite.Load()) + lastRead := time.Unix(0, cxn.lastRead.Load()) + + writeIdle := time.Since(lastWrite) > idleTimeout && !cxn.writing.Load() + readIdle := time.Since(lastRead) > idleTimeout && !cxn.reading.Load() + return writeIdle && readIdle +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/client.go b/vendor/github.com/twmb/franz-go/pkg/kgo/client.go new file mode 100644 index 00000000000..f82e59838b4 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/client.go @@ -0,0 +1,5527 @@ +// Package kgo provides a pure Go efficient Kafka client for Kafka 0.8+ with +// support for transactions, regex topic consuming, the latest partition +// strategies, and more. This client supports all client related KIPs. +// +// This client aims to be simple to use while still interacting with Kafka in a +// near ideal way. For more overview of the entire client itself, please see +// the README on the project's Github page. +package kgo + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "hash/crc32" + "maps" + "math" + "math/rand" + "net" + "reflect" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" + "github.com/twmb/franz-go/pkg/sasl" +) + +var crc32c = crc32.MakeTable(crc32.Castagnoli) // record crc's use Castagnoli table; for consuming/producing + +// Client issues requests and handles responses to a Kafka cluster. +type Client struct { + cfg cfg + opts []Opt + + ctx context.Context + ctxCancel func() + + rng func(func(*rand.Rand)) + + brokersMu xsync.RWMutex + brokers []*broker // ordered by broker ID + seeds atomic.Value // []*broker, seed brokers, also ordered by ID + anyBrokerOrd []int32 // shuffled brokers, for random ordering + anySeedIdx int32 + stopBrokers bool // set to true on close to stop updateBrokers + + // A sink and a source is created once per node ID and persists + // forever. We expect the list to be small. + // + // The mutex only exists to allow consumer session stopping to read + // sources to notify when starting a session; all writes happen in the + // metadata loop. + sinksAndSourcesMu xsync.Mutex + sinksAndSources map[int32]sinkAndSource + + reqFormatter *kmsg.RequestFormatter + connTimeouter connTimeouter + + bufPool bufPool // for to brokers to share underlying reusable request buffers + prsPool prsPool // for sinks to reuse []promisedNumberedRecord + + controllerIDMu xsync.Mutex + controllerID int32 + clusterID *string // we piggy back updating clusterID + + // The following two ensure that we only have one fetchBrokerMetadata + // at once. This avoids unnecessary broker metadata requests and + // metadata trampling. + fetchingBrokersMu xsync.Mutex + fetchingBrokers *struct { + done chan struct{} + err error + } + + producer producer + consumer consumer + id2t atomic.Value // map[[16]byte]string + + metrics metrics + + coordinatorsMu xsync.Mutex + coordinators map[coordinatorKey]*coordinatorLoad + + updateMetadataCh chan string + updateMetadataNowCh chan string // like above, but with high priority + blockingMetadataFnCh chan func() + metawait metawait + metadone chan struct{} + + metaCache struct { + mu xsync.Mutex + topics map[string]cachedMetaTopic + byID map[[16]byte]string // TopicID => topic name + allAt time.Time // when last all-topics fetch completed + anyAt time.Time // when any cached metadata was last stored + } +} + +func (cl *Client) idempotent() bool { return !cl.cfg.disableIdempotency } + +type sinkAndSource struct { + sink *sink + source *source +} + +func (cl *Client) allSinksAndSources(fn func(sns sinkAndSource)) { + cl.sinksAndSourcesMu.Lock() + defer cl.sinksAndSourcesMu.Unlock() + + for _, sns := range cl.sinksAndSources { + fn(sns) + } +} + +func (cl *Client) allSources(fn func(s *source)) { + cl.sinksAndSourcesMu.Lock() + srcs := make([]*source, 0, len(cl.sinksAndSources)) + for _, sns := range cl.sinksAndSources { + if sns.source != nil { + srcs = append(srcs, sns.source) + } + } + cl.sinksAndSourcesMu.Unlock() + for _, s := range srcs { + fn(s) + } +} + +type hostport struct { + host string + port int32 +} + +// ValidateOpts returns an error if the options are invalid. +func ValidateOpts(opts ...Opt) error { + _, _, err := validateCfg(opts...) + return err +} + +func parseSeeds(addrs []string) ([]hostport, error) { + seeds := make([]hostport, 0, len(addrs)) + for _, seedBroker := range addrs { + hp, err := parseBrokerAddr(seedBroker) + if err != nil { + return nil, err + } + seeds = append(seeds, hp) + } + return seeds, nil +} + +// This function validates the configuration and returns a few things that we +// initialize while validating. The difference between this and NewClient +// initialization is all NewClient initialization is infallible. +func validateCfg(opts ...Opt) (cfg, []hostport, error) { + cfg := defaultCfg() + for _, opt := range opts { + opt.apply(&cfg) + } + if err := cfg.validate(); err != nil { + return cfg, nil, err + } + seeds, err := parseSeeds(cfg.seedBrokers) + if err != nil { + return cfg, nil, err + } + if cfg.compressor == nil { + cfg.compressor, err = DefaultCompressor(cfg.compression...) + if err != nil { + return cfg, nil, err + } + } + if cfg.decompressor == nil { + cfg.decompressor = DefaultDecompressor(cfg.pools...) + } + + return cfg, seeds, nil +} + +func namefn(fn any) string { + v := reflect.ValueOf(fn) + if v.Type().Kind() != reflect.Func { + return "" + } + name := runtime.FuncForPC(v.Pointer()).Name() + dot := strings.LastIndexByte(name, '.') + if dot >= 0 { + return name[dot+1:] + } + return name +} + +// OptValue returns the value for the given configuration option. If the +// given option does not exist, this returns nil. This function takes either a +// raw Opt, or an Opt function name. +// +// If a configuration option has multiple inputs, this function returns only +// the first input. If the function is a boolean function (such as +// BlockRebalanceOnPoll), this function returns the value of the internal bool. +// Variadic option inputs are returned as a single slice. Options that are +// internally stored as a pointer (ClientID, TransactionalID, and InstanceID) +// are returned as their string input; you can see if the option is internally +// nil by looking at the second value returned from OptValues. +// +// var ( +// cl, _ := NewClient( +// InstanceID("foo"), +// ConsumeTopics("foo", "bar"), +// ) +// iid = cl.OptValue(InstanceID) // iid is "foo" +// gid = cl.OptValue(ConsumerGroup) // gid is "" since groups are not used +// topics = cl.OptValue("ConsumeTopics") // topics is []string{"foo", "bar"}; string lookup for the option works +// bpoll = cl.OptValue(BlockRebalanceOnPoll) // bpoll is false +// t = cl.OptValue(SessionTimeout) // t is 45s, the internal default +// td = t.(time.Duration) // safe conversion since SessionTimeout's input is a time.Duration +// unk = cl.OptValue("Unknown"), // unk is nil +// ) +func (cl *Client) OptValue(opt any) any { + vs := cl.OptValues(opt) + if len(vs) > 0 { + return vs[0] + } + return nil +} + +// OptValues returns all values for options. This method is useful for +// options that have multiple inputs (notably, SoftwareNameAndVersion). This is +// also useful for options that are internally stored as a pointer (ClientID, +// TransactionalID, and InstanceID) -- this function will return the string +// value of the option but also whether the option is non-nil. Boolean options +// are returned as a single-element slice with the bool value. Variadic inputs +// are returned as a single slice. If the input option does not exist, this +// returns nil. +// +// var ( +// cl, _ = NewClient( +// InstanceID("foo"), +// ConsumeTopics("foo", "bar"), +// ) +// idValues = cl.OptValues(InstanceID) // idValues is []any{"foo", true} +// tValues = cl.OptValues(SessionTimeout) // tValues is []any{45 * time.Second} +// topics = cl.OptValues(ConsumeTopics) // topics is []any{[]string{"foo", "bar"} +// bpoll = cl.OptValues(BlockRebalanceOnPoll) // bpoll is []any{false} +// unknown = cl.OptValues("Unknown") // unknown is nil +// ) +func (cl *Client) OptValues(opt any) []any { + name := namefn(opt) + if s, ok := opt.(string); ok { + name = s + } + cfg := &cl.cfg + + switch name { + case namefn(ClientID): + if cfg.id != nil { + return []any{*cfg.id, true} + } + return []any{"", false} + case namefn(SoftwareNameAndVersion): + return []any{cfg.softwareName, cfg.softwareVersion} + case namefn(WithLogger): + if _, wrapped := cfg.logger.(*wrappedLogger); wrapped { + return []any{cfg.logger.(*wrappedLogger).inner} + } + return []any{nil} + case namefn(RequestTimeoutOverhead): + return []any{cfg.requestTimeoutOverhead} + case namefn(ConnIdleTimeout): + return []any{cfg.connIdleTimeout} + case namefn(Dialer): + return []any{cfg.dialFn} + case namefn(DialTLSConfig): + return []any{cfg.dialTLS} + case namefn(DialTLS): + return []any{cfg.dialTLS != nil} + case namefn(DialTimeout): + return []any{cfg.dialTimeout} + case namefn(SeedBrokers): + return []any{cfg.seedBrokers} + case namefn(MaxVersions): + return []any{cfg.maxVersions} + case namefn(MinVersions): + return []any{cfg.minVersions} + case namefn(RetryBackoffFn): + return []any{cfg.retryBackoff} + case namefn(RequestRetries): + return []any{cfg.retries} + case namefn(RetryTimeout): + return []any{cfg.retryTimeout(0)} + case namefn(RetryTimeoutFn): + return []any{cfg.retryTimeout} + case namefn(AllowAutoTopicCreation): + return []any{cfg.allowAutoTopicCreation} + case namefn(BrokerMaxWriteBytes): + return []any{cfg.maxBrokerWriteBytes} + case namefn(BrokerMaxReadBytes): + return []any{cfg.maxBrokerReadBytes} + case namefn(MetadataMaxAge): + return []any{cfg.metadataMaxAge} + case namefn(MetadataMinAge): + return []any{cfg.metadataMinAge} + case namefn(SASL): + return []any{cfg.sasls} + case namefn(WithHooks): + return []any{cfg.hooks} + case namefn(WithPools): + return []any{cfg.pools} + case namefn(ConcurrentTransactionsBackoff): + return []any{cfg.txnBackoff} + case namefn(ConsiderMissingTopicDeletedAfter): + return []any{cfg.missingTopicDelete} + case namefn(OnRebootstrapRequired): + return []any{cfg.onRebootstrapRequired} + case namefn(WithContext): + return []any{cfg.ctx} + case namefn(DisableClientMetrics): + return []any{cfg.disableClientMetrics} + case namefn(UserMetricsFn): + return []any{cfg.userMetrics} + case namefn(AlwaysRetryEOF): + return []any{cfg.alwaysRetryEOF} + + case namefn(DefaultProduceTopic): + return []any{cfg.defaultProduceTopic} + case namefn(DefaultProduceTopicAlways): + return []any{cfg.defaultProduceTopicAlways} + case namefn(RequiredAcks): + return []any{cfg.acks} + case namefn(DisableIdempotentWrite): + return []any{cfg.disableIdempotency} + case namefn(AllowIdempotentProduceCancellation): + return []any{cfg.allowIdempotentProduceCancellation} + case namefn(MaxProduceRequestsInflightPerBroker): + return []any{cfg.maxProduceInflight} + case namefn(ProducerBatchCompression): + return []any{cfg.compression} + case namefn(WithCompressor): + return []any{cfg.compressor} + case namefn(ProducerBatchMaxBytes): + return []any{cfg.maxRecordBatchBytes("")} + case namefn(ProducerBatchMaxBytesFn): + return []any{cfg.maxRecordBatchBytes} + case namefn(MaxBufferedRecords): + return []any{cfg.maxBufferedRecords} + case namefn(MaxBufferedBytes): + return []any{cfg.maxBufferedBytes} + case namefn(RecordPartitioner): + return []any{cfg.partitioner} + case namefn(ProduceRequestTimeout): + return []any{cfg.produceTimeout} + case namefn(RecordRetries): + return []any{cfg.recordRetries} + case namefn(UnknownTopicRetries): + return []any{cfg.maxUnknownFailures} + case namefn(StopProducerOnDataLossDetected): + return []any{cfg.stopOnDataLoss} + case namefn(ProducerOnDataLossDetected): + return []any{cfg.onDataLoss} + case namefn(ProducerLinger): + return []any{cfg.linger} + case namefn(ManualFlushing): + return []any{cfg.manualFlushing} + case namefn(RecordDeliveryTimeout): + return []any{cfg.recordTimeout} + case namefn(TransactionalID): + if cfg.txnID != nil { + return []any{cfg.txnID, true} + } + return []any{"", false} + case namefn(TransactionTimeout): + return []any{cfg.txnTimeout} + + case namefn(ConsumePartitions): + return []any{cfg.partitions} + case namefn(ConsumePreferringLagFn): + return []any{cfg.preferLagFn} + case namefn(WithDecompressor): + return []any{cfg.decompressor} + case namefn(ConsumeRegex): + return []any{cfg.regex} + case namefn(ConsumeExcludeTopics): + return []any{slices.Collect(maps.Keys(cfg.excludeTopics))} + case namefn(ConsumeStartOffset): + return []any{cfg.startOffset} + case namefn(ConsumeResetOffset): + return []any{cfg.resetOffset} + case namefn(ConsumeTopics): + return []any{cfg.topics} + case namefn(DisableFetchSessions): + return []any{cfg.disableFetchSessions} + case namefn(FetchIsolationLevel): + return []any{cfg.isolationLevel} + case namefn(FetchMaxBytes): + return []any{int32(cfg.maxBytes)} + case namefn(FetchMaxPartitionBytes): + return []any{int32(cfg.maxPartBytes)} + case namefn(FetchMaxWait): + return []any{time.Duration(cfg.maxWait) * time.Millisecond} + case namefn(FetchMinBytes): + return []any{cfg.minBytes} + case namefn(KeepControlRecords): + return []any{cfg.keepControl} + case namefn(MaxConcurrentFetches): + return []any{cfg.maxConcurrentFetches} + case namefn(Rack): + return []any{cfg.rack} + case namefn(KeepRetryableFetchErrors): + return []any{cfg.keepRetryableFetchErrors} + case namefn(DisableFetchCRCValidation): + return []any{cfg.disableFetchCRCValidation} + case namefn(RecheckPreferredReplicaInterval): + return []any{cfg.recheckPreferredReplicaInterval} + + case namefn(AdjustFetchOffsetsFn): + return []any{cfg.adjustOffsetsBeforeAssign} + case namefn(AutoCommitCallback): + return []any{cfg.commitCallback} + case namefn(AutoCommitInterval): + return []any{cfg.autocommitInterval} + case namefn(AutoCommitMarks): + return []any{cfg.autocommitMarks} + case namefn(Balancers): + return []any{cfg.balancers} + case namefn(BlockRebalanceOnPoll): + return []any{cfg.blockRebalanceOnPoll} + case namefn(ConsumerGroup): + return []any{cfg.group} + case namefn(DisableAutoCommit): + return []any{cfg.autocommitDisable} + case namefn(GreedyAutoCommit): + return []any{cfg.autocommitGreedy} + case namefn(GroupProtocol): + return []any{cfg.protocol} + case namefn(HeartbeatInterval): + return []any{cfg.heartbeatInterval} + case namefn(InstanceID): + if cfg.instanceID != nil { + return []any{*cfg.instanceID, true} + } + return []any{"", false} + case namefn(OnOffsetsFetched): + return []any{cfg.onFetched} + case namefn(OnPartitionsAssigned): + return []any{cfg.onAssigned} + case namefn(OnPartitionsLost): + return []any{cfg.onLost} + case namefn(OnPartitionsRevoked): + return []any{cfg.onRevoked} + case namefn(OnPartitionsCallbackBlocked): + return []any{cfg.onBlocked} + case namefn(RebalanceTimeout): + return []any{cfg.rebalanceTimeout} + case namefn(RequireStableFetchOffsets): + return []any{true} + case namefn(SessionTimeout): + return []any{cfg.sessionTimeout} + + default: + return nil + } +} + +// NewClient returns a new Kafka client with the given options or an error if +// the options are invalid. Connections to brokers are lazily created only when +// requests are written to them. +// +// By default, the client uses the latest stable request versions when talking +// to Kafka. If you use a broker older than 0.10.0, then you need to manually +// set a MaxVersions option. Otherwise, there is usually no harm in defaulting +// to the latest API versions, although occasionally Kafka introduces new +// required parameters that do not have zero value defaults. +// +// NewClient also launches a goroutine which periodically updates the cached +// topic metadata. +func NewClient(opts ...Opt) (*Client, error) { + cfg, seeds, err := validateCfg(opts...) + if err != nil { + return nil, err + } + + if cfg.retryTimeout == nil { + cfg.retryTimeout = func(key int16) time.Duration { + switch key { + case ((*kmsg.JoinGroupRequest)(nil)).Key(), + ((*kmsg.SyncGroupRequest)(nil)).Key(), + ((*kmsg.HeartbeatRequest)(nil)).Key(), + ((*kmsg.ConsumerGroupHeartbeatRequest)(nil)).Key(): + return cfg.sessionTimeout + } + return 30 * time.Second + } + } + + if cfg.dialFn == nil { + dialer := &net.Dialer{Timeout: cfg.dialTimeout} + cfg.dialFn = dialer.DialContext + if cfg.dialTLS != nil { + cfg.dialFn = func(ctx context.Context, network, host string) (net.Conn, error) { + c := cfg.dialTLS.Clone() + if c.ServerName == "" { + server, _, err := net.SplitHostPort(host) + if err != nil { + return nil, fmt.Errorf("unable to split host:port for dialing: %w", err) + } + c.ServerName = server + } + return (&tls.Dialer{ + NetDialer: dialer, + Config: c, + }).DialContext(ctx, network, host) + } + } + } + + if cfg.setResetOffset && !cfg.setStartOffset { + cfg.startOffset = cfg.resetOffset + } else if cfg.setStartOffset && !cfg.setResetOffset { + cfg.resetOffset = cfg.startOffset + } // else they are both set (keep) or both unset (defaults) + + ctx := context.Background() + + if cfg.ctx != nil { + ctx = cfg.ctx + } + + ctx, cancel := context.WithCancel(ctx) + + cl := &Client{ + cfg: cfg, + opts: opts, + ctx: ctx, + ctxCancel: cancel, + + rng: func() func(func(*rand.Rand)) { + var mu xsync.Mutex + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + return func(fn func(*rand.Rand)) { + mu.Lock() + defer mu.Unlock() + fn(rng) + } + }(), + + controllerID: unknownControllerID, + + sinksAndSources: make(map[int32]sinkAndSource), + + reqFormatter: kmsg.NewRequestFormatter(), + connTimeouter: connTimeouter{def: cfg.requestTimeoutOverhead}, + + bufPool: newBufPool(), + prsPool: newPrsPool(), + + coordinators: make(map[coordinatorKey]*coordinatorLoad), + + updateMetadataCh: make(chan string, 1), + updateMetadataNowCh: make(chan string, 1), + blockingMetadataFnCh: make(chan func()), + metadone: make(chan struct{}), + } + + // Before we start any goroutines below, we must notify any interested + // hooks of our existence. + cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookNewClient); ok { + h.OnNewClient(cl) + } + }) + + cl.producer.init(cl) + cl.consumer.init(cl) + cl.metawait.init() + cl.metrics.init(cl) + + if cfg.id != nil { + cl.reqFormatter = kmsg.NewRequestFormatter(kmsg.FormatterClientID(*cfg.id)) + } + + seedBrokers := make([]*broker, 0, len(seeds)) + for i, seed := range seeds { + b := cl.newBroker(unknownSeedID(i), seed.host, seed.port, nil) + seedBrokers = append(seedBrokers, b) + } + cl.seeds.Store(seedBrokers) + go cl.updateMetadataLoop() + go cl.reapConnectionsLoop() + go cl.pushMetrics() + + return cl, nil +} + +// Opts returns the options that were used to create this client. This can be +// as a base to generate a new client, where you can add override options to +// the end of the original input list. If you want to know a specific option +// value, you can use OptValue or OptValues. +func (cl *Client) Opts() []Opt { + return cl.opts +} + +// Context returns the internal context used wherever possible in the client. +// By default this is context.WithCancel(context.Background()). You may +// override the background context with your own via [WithContext]. +// The context is occasionally wrapped further internally in client subsystems. +func (cl *Client) Context() context.Context { + return cl.ctx +} + +func (cl *Client) loadSeeds() []*broker { + return cl.seeds.Load().([]*broker) +} + +// Ping returns whether any broker is reachable and that the client can +// communicate with it, iterating over any discovered broker or seed broker +// until one returns a successful response to a broker-only Metadata request. +// No discovered broker nor seed broker is attempted more than once. If all +// requests fail, this returns final error. +func (cl *Client) Ping(ctx context.Context) error { + req := kmsg.NewPtrMetadataRequest() + req.Topics = []kmsg.MetadataRequestTopic{} + + cl.brokersMu.RLock() + brokers := slices.Clone(cl.brokers) + cl.brokersMu.RUnlock() + + var lastErr error + for _, brs := range [2][]*broker{ + brokers, + cl.loadSeeds(), + } { + for _, br := range brs { + resp, err := br.waitResp(ctx, req) + if lastErr = err; lastErr == nil { + cl.updateMetadataBrokers(resp.(*kmsg.MetadataResponse)) + return nil + } else if isContextErr(lastErr) && ctx.Err() != nil { + // No point in trying the next broker if context is done + // as it will create noise in OnBrokerConnect hook. + // Check both lastErr and ctx.Err() to avoid race condition + // where context error happens immediately after waitResp. + return lastErr + } + } + } + return lastErr +} + +// PurgeTopicsFromClient internally removes all internal information about the +// input topics. If you want to purge information for only consuming or +// only producing, see the related functions [PurgeTopicsFromConsuming] and +// [PurgeTopicsFromProducing]. +// +// For producing, this clears all knowledge that these topics have ever been +// produced to. Producing to the topic again may result in out of order +// sequence number errors, or, if idempotency is disabled and the sequence +// numbers align, may result in invisibly discarded records at the broker. +// Purging a topic that was previously produced to may be useful to free up +// resources if you are producing to many disparate and short lived topic in +// the lifetime of this client and you do not plan to produce to the topic +// anymore. You may want to flush buffered records before purging if records +// for a topic you are purging are currently in flight. +// +// For consuming, this removes all concept of the topic from being consumed. +// This is different from PauseFetchTopics, which literally pauses the fetching +// of topics but keeps the topic information around for resuming fetching +// later. Purging a topic that was being consumed can be useful if you know the +// topic no longer exists, or if you are consuming via regex and know that some +// previously consumed topics no longer exist, or if you simply do not want to +// ever consume from a topic again. If you are group consuming, this function +// will likely cause a rebalance. If you are consuming via regex and the topic +// still exists on the broker, this function will at most only temporarily +// remove the topic from the client and the topic will be re-discovered. +// +// For admin requests, this deletes the topic from the cached metadata map for +// sharded requests. Metadata for sharded admin requests is only cached for +// MetadataMinAge anyway, but the map is not cleaned up one the metadata +// expires. This function ensures the map is purged. +func (cl *Client) PurgeTopicsFromClient(topics ...string) { + if len(topics) == 0 { + return + } + sort.Strings(topics) // for logging in the functions + cl.blockingMetadataFn(func() { // make reasoning about concurrency easier + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + cl.producer.purgeTopics(topics) + }() + go func() { + defer wg.Done() + cl.consumer.purgeTopics(topics) + }() + wg.Wait() + + cl.metaCache.mu.Lock() + var purgedIDs [][16]byte + for _, t := range topics { + if ct, ok := cl.metaCache.topics[t]; ok { + var zeroID [16]byte + if ct.id != zeroID { + delete(cl.metaCache.byID, ct.id) + purgedIDs = append(purgedIDs, ct.id) + } + delete(cl.metaCache.topics, t) + } + } + cl.metaCache.mu.Unlock() + + if len(purgedIDs) > 0 { + old := cl.id2tMap() + merged := make(map[[16]byte]string, len(old)) + maps.Copy(merged, old) + for _, id := range purgedIDs { + delete(merged, id) + } + cl.id2t.Store(merged) + } + }) +} + +// PurgeTopicsFromProducing internally removes all internal information for +// producing about the input topics. This runs the producer bit of logic that +// is documented in [PurgeTopicsFromClient]; see that function for more +// details. +func (cl *Client) PurgeTopicsFromProducing(topics ...string) { + if len(topics) == 0 { + return + } + sort.Strings(topics) + cl.blockingMetadataFn(func() { + cl.producer.purgeTopics(topics) + }) +} + +// PurgeTopicsFromConsuming internally removes all internal information for +// consuming about the input topics. This runs the consumer bit of logic that +// is documented in [PurgeTopicsFromClient]; see that function for more +// details. +func (cl *Client) PurgeTopicsFromConsuming(topics ...string) { + if len(topics) == 0 { + return + } + sort.Strings(topics) + cl.blockingMetadataFn(func() { + cl.consumer.purgeTopics(topics) + }) +} + +// Parse broker IP/host and port from a string, using the default Kafka port if +// unspecified. Supported address formats: +// +// - IPv4 host/IP without port: "127.0.0.1", "localhost" +// - IPv4 host/IP with port: "127.0.0.1:1234", "localhost:1234" +// - IPv6 IP without port: "[2001:1000:2000::1]", "::1" +// - IPv6 IP with port: "[2001:1000:2000::1]:1234" +func parseBrokerAddr(addr string) (hostport, error) { + const defaultKafkaPort = 9092 + + // Bracketed IPv6 + if strings.IndexByte(addr, '[') == 0 { + parts := strings.Split(addr[1:], "]") + if len(parts) != 2 { + return hostport{}, fmt.Errorf("invalid addr: %s", addr) + } + // No port specified -> use default + if len(parts[1]) == 0 { + return hostport{parts[0], defaultKafkaPort}, nil + } + port, err := strconv.ParseInt(parts[1][1:], 10, 32) + if err != nil { + return hostport{}, fmt.Errorf("unable to parse port from addr: %w", err) + } + return hostport{parts[0], int32(port)}, nil + } + + // IPv4 with no port + if strings.IndexByte(addr, ':') == -1 { + return hostport{addr, defaultKafkaPort}, nil + } + + // Either a IPv6 literal ("::1"), IP:port or host:port + // Try to parse as IP:port or host:port + h, p, err := net.SplitHostPort(addr) + if err != nil { + return hostport{addr, defaultKafkaPort}, nil //nolint:nilerr // ipv6 literal -- use default kafka port + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + return hostport{}, fmt.Errorf("unable to parse port from addr: %w", err) + } + return hostport{h, int32(port)}, nil +} + +type connTimeouter struct { + def time.Duration + joinMu xsync.Mutex + lastRebalanceTimeout time.Duration +} + +func (c *connTimeouter) timeouts(req kmsg.Request) (r, w time.Duration) { + def := c.def + millis := func(m int32) time.Duration { return time.Duration(m) * time.Millisecond } + switch t := req.(type) { + default: + if timeoutRequest, ok := req.(kmsg.TimeoutRequest); ok { + timeoutMillis := timeoutRequest.Timeout() + return def + millis(timeoutMillis), def + } + return def, def + + case *produceRequest: + return def + millis(t.timeout), def + case *fetchRequest: + return def + millis(t.maxWait), def + case *kmsg.FetchRequest: + return def + millis(t.MaxWaitMillis), def + + // Join and sync can take a long time. Sync has no notion of + // timeouts, but since the flow of requests should be first + // join, then sync, we can stash the timeout from the join. + + case *kmsg.JoinGroupRequest: + c.joinMu.Lock() + c.lastRebalanceTimeout = millis(t.RebalanceTimeoutMillis) + c.joinMu.Unlock() + + return def + millis(t.RebalanceTimeoutMillis), def + case *kmsg.SyncGroupRequest: + read := def + c.joinMu.Lock() + if c.lastRebalanceTimeout != 0 { + read = c.lastRebalanceTimeout + } + c.joinMu.Unlock() + + return read, def + } +} + +func (cl *Client) reinitAnyBrokerOrd() { + cl.anyBrokerOrd = append(cl.anyBrokerOrd[:0], make([]int32, len(cl.brokers))...) + for i := range cl.anyBrokerOrd { + cl.anyBrokerOrd[i] = int32(i) + } + cl.rng(func(r *rand.Rand) { + r.Shuffle(len(cl.anyBrokerOrd), func(i, j int) { + cl.anyBrokerOrd[i], cl.anyBrokerOrd[j] = cl.anyBrokerOrd[j], cl.anyBrokerOrd[i] + }) + }) +} + +// broker returns a random broker from all brokers ever known. +func (cl *Client) broker() *broker { + cl.brokersMu.Lock() + defer cl.brokersMu.Unlock() + + // Every time we loop through all discovered brokers, we issue one + // request to the next seed. This ensures that if all discovered + // brokers are down, we will *eventually* loop through seeds and + // hopefully have a reachable seed. + var b *broker + + if len(cl.anyBrokerOrd) > 0 { + b = cl.brokers[cl.anyBrokerOrd[0]] + cl.anyBrokerOrd = cl.anyBrokerOrd[1:] + return b + } + + seeds := cl.loadSeeds() + cl.anySeedIdx %= int32(len(seeds)) + b = seeds[cl.anySeedIdx] + cl.anySeedIdx++ + + // If we have brokers, we ranged past discovered brokers. + // We now reset the anyBrokerOrd to begin ranging through + // discovered brokers again. If there are still no brokers, + // this reinit will do nothing and we will keep looping seeds. + cl.reinitAnyBrokerOrd() + return b +} + +func (cl *Client) waitTries(ctx context.Context, backoff time.Duration) bool { + after := time.NewTimer(backoff) + defer after.Stop() + select { + case <-ctx.Done(): + return false + case <-cl.ctx.Done(): + return false + case <-after.C: + return true + } +} + +// A broker may sometimes indicate it supports offset for leader epoch v2+ when +// it does not. We need to catch that and avoid issuing offset for leader +// epoch, because we will just loop continuously failing. We do not catch every +// case, such as when a person explicitly assigns offsets with epochs, but we +// catch a few areas that would be returned from a broker itself. +// +// This function is always used *after* at least one request has been issued. +// +// NOTE: This is a weak check; we check if any broker in the cluster supports +// the request. We use this function in three locations: +// +// 1. When using the LeaderEpoch returned in a metadata response. This guards +// against buggy brokers that return 0 rather than -1 even if they do not +// support OffsetForLeaderEpoch. If any support, the cluster is in the +// middle of an upgrade and we can start using the epoch. +// 2. When deciding whether to keep LeaderEpoch from fetched offsets. +// Realistically, clients should only commit epochs if the cluster supports +// them. +// 3. When receiving OffsetOutOfRange when follower fetching and we fetched +// past the end. +// +// In any of these cases, if we OffsetForLeaderEpoch against a broker that does +// not support (even though one in the cluster does), we will loop fail until +// the rest of the cluster is upgraded and supports the request. +func (cl *Client) supportsOffsetForLeaderEpoch() bool { + return cl.supportsKeyVersion(int16(kmsg.OffsetForLeaderEpoch), 2) +} + +// Called after the first metadata request, before we go into either +// (*groupConsumer).manage or (*groupConsumer).manage848. +// +// v1 introduces support for regex and requires the client to generate +// the member ID, and fully stabilizes KIP-848. +func (cl *Client) supportsKIP848v1() bool { + return cl.supportsKeyVersion(int16(kmsg.ConsumerGroupHeartbeat), 1) +} + +// Called after the first metric observed, which is always after a response. +func (cl *Client) supportsClientMetrics() bool { + return cl.supportsKeyVersion(int16(kmsg.GetTelemetrySubscriptions), 0) && + cl.supportsKeyVersion(int16(kmsg.PushTelemetry), 0) +} + +// A broker may not support some requests we want to make. This function checks +// support. This should only be used *after* at least one successful response. +func (cl *Client) supportsKeyVersion(key, version int16) bool { + cl.brokersMu.RLock() + defer cl.brokersMu.RUnlock() + + for _, brokers := range [][]*broker{ + cl.brokers, + cl.loadSeeds(), + } { + for _, b := range brokers { + if v := b.loadVersions(); v != nil && v.maxVersion(key) >= version { + return true + } + } + } + return false +} + +func (cl *Client) supportsKIP890p2() bool { + return cl.supportsFeature("transaction.version", 2) +} + +// Same as above. A cluster returns a max version for a feature only once the +// entire cluster supports the feature. This should only be used *after* at +// least one successful response. +func (cl *Client) supportsFeature(name string, version int16) bool { + cl.brokersMu.RLock() + defer cl.brokersMu.RUnlock() + + for _, brokers := range [][]*broker{ + cl.brokers, + cl.loadSeeds(), + } { + for _, b := range brokers { + if v := b.loadVersions(); v != nil && v.features[name] >= version { + return true + } + } + } + return false +} + +// fetchBrokerMetadata issues a metadata request solely for broker information. +func (cl *Client) fetchBrokerMetadata(ctx context.Context) error { + cl.fetchingBrokersMu.Lock() + wait := cl.fetchingBrokers + if wait != nil { + cl.fetchingBrokersMu.Unlock() + <-wait.done + return wait.err + } + wait = &struct { + done chan struct{} + err error + }{done: make(chan struct{})} + cl.fetchingBrokers = wait + cl.fetchingBrokersMu.Unlock() + + defer func() { + cl.fetchingBrokersMu.Lock() + defer cl.fetchingBrokersMu.Unlock() + cl.fetchingBrokers = nil + close(wait.done) + }() + + req := kmsg.NewPtrMetadataRequest() + req.Topics = []kmsg.MetadataRequestTopic{} + _, _, wait.err = cl.fetchMetadata(ctx, req, true, nil) + return wait.err +} + +func (cl *Client) fetchMetadataByName(ctx context.Context, all bool, topics []string, results map[string]cachedMetaTopic) (*broker, *kmsg.MetadataResponse, error) { + req := kmsg.NewPtrMetadataRequest() + req.AllowAutoTopicCreation = cl.cfg.allowAutoTopicCreation + if all { + req.Topics = nil + } else if len(topics) == 0 { + req.Topics = []kmsg.MetadataRequestTopic{} + } else { + for _, topic := range topics { + reqTopic := kmsg.NewMetadataRequestTopic() + reqTopic.Topic = kmsg.StringPtr(topic) + req.Topics = append(req.Topics, reqTopic) + } + } + return cl.fetchMetadata(ctx, req, true, results) +} + +// resolveTopicMetaByID fetches metadata by TopicID and caches the results. +// This is used when we only have topic IDs (e.g. v10+ OffsetFetch) and +// need to resolve them to names before processing a response. +func (cl *Client) resolveTopicMetaByID(ctx context.Context, ids [][16]byte) (map[string]cachedMetaTopic, error) { + // Check cache first. + cl.metaCache.mu.Lock() + var missing [][16]byte + for _, id := range ids { + if _, ok := cl.metaCache.byID[id]; !ok { + missing = append(missing, id) + } + } + cl.metaCache.mu.Unlock() + if len(missing) == 0 { + return nil, nil + } + + req := kmsg.NewPtrMetadataRequest() + for _, id := range missing { + reqTopic := kmsg.NewMetadataRequestTopic() + reqTopic.TopicID = id + req.Topics = append(req.Topics, reqTopic) + } + results := make(map[string]cachedMetaTopic) + _, _, err := cl.fetchMetadata(ctx, req, true, results) + return results, err +} + +func (cl *Client) fetchMetadata(ctx context.Context, req *kmsg.MetadataRequest, limitRetries bool, results map[string]cachedMetaTopic) (*broker, *kmsg.MetadataResponse, error) { + r := cl.retryable() + + var rebootstrapped bool +start: + // We limit retries for internal metadata refreshes, because these do + // not need to retry forever and are usually blocking *other* requests. + // e.g., producing bumps load errors when metadata returns, so 3 + // failures here will correspond to 1 bumped error count. To make the + // number more accurate, we should *never* retry here, but this is + // pretty intolerant of immediately-temporary network issues. Rather, + // we use a small count of 3 retries, which with the default backoff, + // will be <2s of retrying. This is still intolerant of temporary + // failures, but it does allow recovery from a dns issue / bad path. + if limitRetries { + r.limitRetries = 3 + } + + meta, err := req.RequestWith(ctx, r) + if err == nil { + if err = kerr.ErrorForCode(meta.ErrorCode); !rebootstrapped && errors.Is(err, kerr.RebootstrapRequired) && cl.cfg.onRebootstrapRequired != nil { + var seeds []string + seeds, err = cl.cfg.onRebootstrapRequired() + if err == nil && len(seeds) > 0 { + err = cl.UpdateSeedBrokers(seeds...) + if err == nil { + cl.updateBrokers(nil) + rebootstrapped = true + goto start + } + } + } + cl.updateMetadataBrokers(meta) + + // Cache the metadata, and potentially store each topic in the results. + cl.storeCachedMeta(meta, req.Topics == nil, results) + } + return r.last, meta, err +} + +func (cl *Client) updateMetadataBrokers(resp *kmsg.MetadataResponse) { + cl.controllerIDMu.Lock() + if resp.ControllerID >= 0 { + cl.controllerID = resp.ControllerID + } + cl.clusterID = resp.ClusterID + cl.controllerIDMu.Unlock() + cl.updateBrokers(resp.Brokers) +} + +// updateBrokers is called with the broker portion of every metadata response. +// All metadata responses contain all known live brokers, so we can always +// use the response. +func (cl *Client) updateBrokers(brokers []kmsg.MetadataResponseBroker) { + sort.Slice(brokers, func(i, j int) bool { return brokers[i].NodeID < brokers[j].NodeID }) + newBrokers := make([]*broker, 0, len(brokers)) + + cl.brokersMu.Lock() + defer cl.brokersMu.Unlock() + + if cl.stopBrokers { + return + } + + for len(brokers) > 0 && len(cl.brokers) > 0 { + ob := cl.brokers[0] + nb := brokers[0] + + switch { + case ob.meta.NodeID < nb.NodeID: + ob.stopForever() + cl.brokers = cl.brokers[1:] + + case ob.meta.NodeID == nb.NodeID: + if !ob.meta.equals(nb) { + ob.stopForever() + ob = cl.newBroker(nb.NodeID, nb.Host, nb.Port, nb.Rack) + } + newBrokers = append(newBrokers, ob) + cl.brokers = cl.brokers[1:] + brokers = brokers[1:] + + case ob.meta.NodeID > nb.NodeID: + newBrokers = append(newBrokers, cl.newBroker(nb.NodeID, nb.Host, nb.Port, nb.Rack)) + brokers = brokers[1:] + } + } + + for len(cl.brokers) > 0 { + ob := cl.brokers[0] + ob.stopForever() + cl.brokers = cl.brokers[1:] + } + + for len(brokers) > 0 { + nb := brokers[0] + newBrokers = append(newBrokers, cl.newBroker(nb.NodeID, nb.Host, nb.Port, nb.Rack)) + brokers = brokers[1:] + } + + cl.brokers = newBrokers + cl.reinitAnyBrokerOrd() +} + +// brokerRacks returns a map of broker node ID to rack for all known brokers +// that have a rack configured. +func (cl *Client) brokerRacks() map[int32]string { + cl.brokersMu.Lock() + defer cl.brokersMu.Unlock() + racks := make(map[int32]string, len(cl.brokers)) + for _, b := range cl.brokers { + if b.meta.Rack != nil { + racks[b.meta.NodeID] = *b.meta.Rack + } + } + return racks +} + +// CloseAllowingRebalance allows rebalances, leaves any group, and closes all +// connections and goroutines. This function is only useful if you are using +// the BlockRebalanceOnPoll option. Close itself does not allow rebalances and +// will hang if you polled, did not allow rebalances, and want to close. Close +// does not automatically allow rebalances because leaving a group causes a +// revoke, and the client does not assume that the final revoke is concurrency +// safe. The CloseAllowingRebalance function exists a shortcut to opt into +// allowing rebalance while closing. +// +// If you are using static membership, CloseAllowingRebalance will NOT send a +// leave group request. See InstanceID for more details. +func (cl *Client) CloseAllowingRebalance() { + cl.AllowRebalance() + cl.Close() +} + +// Close leaves any group and closes all connections and goroutines. This +// function waits for the group to be left. If you want to force leave a group +// immediately and ensure a speedy shutdown you can use LeaveGroupContext first +// (and then Close will be immediate). +// +// If you are group consuming and have overridden the default +// OnPartitionsRevoked, you must manually commit offsets before closing the +// client. +// +// If you are using the BlockRebalanceOnPoll option and have polled, this +// function does not automatically allow rebalancing. You must AllowRebalance +// before calling this function. Internally, this function leaves the group, +// and leaving a group causes a rebalance so that you can get one final +// notification of revoked partitions. If you want to automatically allow +// rebalancing, use CloseAllowingRebalance. +// +// If you are using static membership, Close will NOT send a leave group +// request. See InstanceID for more details. +func (cl *Client) Close() { + cl.close(cl.ctx) +} + +func (cl *Client) close(ctx context.Context) (rerr error) { + defer cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookClientClosed); ok { + h.OnClientClosed(cl) + } + }) + + c := &cl.consumer + c.kill.Store(true) + if c.g != nil || c.s != nil { + rerr = cl.LeaveGroupContext(ctx) + if c.g != nil { + <-c.g.left + } else { + <-c.s.left + } + } else if c.d != nil { + c.mu.Lock() // lock for assign + c.assignPartitions(nil, assignInvalidateAll, nil, "") // we do not use a log message when not in a group + c.mu.Unlock() + } + + // After the above, consumers cannot consume anymore. + // + // LeaveGroupContext may return early if ctx is already canceled + // (e.g. the user provided a parent context via WithContext that + // was canceled). Waiting on c.g.left or c.s.left ensures the + // goroutine spawned by LeaveGroupContext has fully completed + // the leave sequence before we proceed. For direct consumers, + // assignPartitions above is synchronous. + + sessCloseCtx, sessCloseCancel := context.WithTimeout(ctx, time.Second) + var wg sync.WaitGroup + cl.allSinksAndSources(func(sns sinkAndSource) { + if sns.source.session.id != 0 { + sns := sns + wg.Add(1) + go func() { + defer wg.Done() + sns.source.killSessionOnClose(sessCloseCtx) + }() + } + }) + wg.Wait() + sessCloseCancel() + + // Now we kill the client context and all brokers, ensuring all + // requests fail. This will finish all producer callbacks and + // stop the metadata loop and metrics loop. + cl.ctxCancel() + + // Before killing brokers, give metrics 1s to push any final + // terminating message. The client context cancelation awakens + // the push-period-wait loop. + after := time.NewTimer(time.Second) + select { + case <-cl.metrics.ctx.Done(): + case <-after.C: + cl.metrics.ctxCancel() + case <-ctx.Done(): + cl.metrics.ctxCancel() + } + + cl.brokersMu.Lock() + cl.stopBrokers = true + for _, broker := range cl.brokers { + broker.stopForever() + } + cl.brokersMu.Unlock() + for _, broker := range cl.loadSeeds() { + broker.stopForever() + } + + // Wait for metadata to quit so we know no more erroring topic + // partitions will be created. After metadata has quit, we can + // safely stop sinks and sources, as no more will be made. + <-cl.metadone + + // We do not need a lock in `sink` and `source` access because, + // with the metadata loop done, partition migration will not occur. + for _, sns := range cl.sinksAndSources { + sns.sink.maybeDrain() // awaken anything in backoff + sns.source.maybeConsume() // same + } + + cl.failBufferedRecords(ErrClientClosed) + + // We need one final poll: if any sources buffered a fetch, then the + // manageFetchConcurrency loop only exits when all fetches have been + // drained, because draining a fetch is what decrements an "active" + // fetch. PollFetches with `nil` is instant. + cl.PollFetches(nil) + + for _, s := range cl.cfg.sasls { + if closing, ok := s.(sasl.ClosingMechanism); ok { + closing.Close() + } + } + + return rerr +} + +// Request issues a request to Kafka, waiting for and returning the response. +// If a retryable network error occurs, or if a retryable group / transaction +// coordinator error occurs, the request is retried. All other errors are +// returned. +// +// If the request is an admin request, this will issue it to the Kafka +// controller. If the controller ID is unknown, this will attempt to fetch it. +// If the fetch errors, this will return an unknown controller error. +// +// If the request is a group or transaction coordinator request, this will +// issue the request to the appropriate group or transaction coordinator. +// +// For transaction requests, the request is issued to the transaction +// coordinator. However, if the request is an init producer ID request and the +// request has no transactional ID, the request goes to any broker. +// +// Some requests need to be split and sent to many brokers. For these requests, +// it is *highly* recommended to use RequestSharded. Not all responses from +// many brokers can be cleanly merged. However, for the requests that are +// split, this does attempt to merge them in a sane way. +// +// The following requests are split: +// +// ListOffsets +// OffsetFetch (if using v8+ for Kafka 3.0+) +// FindCoordinator (if using v4+ for Kafka 3.0+) +// DescribeGroups +// ListGroups +// DeleteRecords +// OffsetForLeaderEpoch +// DescribeConfigs +// AlterConfigs +// AlterReplicaLogDirs +// DescribeLogDirs +// DeleteGroups +// IncrementalAlterConfigs +// DescribeProducers +// DescribeTransactions +// ListTransactions +// ConsumerGroupDescribe +// ShareGroupDescribe +// DescribeShareGroupOffsets +// +// Kafka 3.0 introduced batch OffsetFetch and batch FindCoordinator requests. +// This function is forward and backward compatible: old requests will be +// batched as necessary, and batched requests will be split as necessary. It is +// recommended to always use batch requests for simplicity. +// +// In short, this method tries to do the correct thing depending on what type +// of request is being issued. +// +// The passed context can be used to cancel a request and return early. Note +// that if the request was written to Kafka but the context canceled before a +// response is received, Kafka may still operate on the received request. +// +// If using this function to issue kmsg.ProduceRequest's, you must configure +// the client with the same RequiredAcks option that you use in the request. +// If you are issuing produce requests with 0 acks, you must configure the +// client with the same timeout you use in the request. The client will +// internally rewrite the incoming request's acks to match the client's +// configuration, and it will rewrite the timeout millis if the acks is 0. It +// is strongly recommended to not issue raw kmsg.ProduceRequest's. +func (cl *Client) Request(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { + resps, merge := cl.shardedRequest(ctx, req) + // If there is no merge function, only one request was issued directly + // to a broker. Return the resp and err directly. + if merge == nil { + return resps[0].Resp, resps[0].Err + } + return merge(resps) +} + +// RequestCachedMetadata returns a metadata response, using any cached topic +// data possible. Any topic with data cached longer than 'limit' has its +// metadata updated before being returned. If limit is zero or less, +// MetadataMinAge is used (default 5s). +// +// This function is useful if you run a lot of functions that internally +// fetch metadata to execute. As an example, many functions in the kadm +// package all require metadata to run; those functions use cached metadata +// as much as possible. +// +// This function does *not* return authorized operations, even if the request +// has IncludeClusterAuthorizedOperations or IncludeTopicAuthorizedOperations +// set to true. +func (cl *Client) RequestCachedMetadata(ctx context.Context, req *kmsg.MetadataRequest, limit time.Duration) (*kmsg.MetadataResponse, error) { + var topics []string + if req.Topics != nil { + topics = make([]string, 0, len(req.Topics)) + } + + // Phase 1: classify request topics into named and ID-only. + // Resolve IDs from the cache and the main metadata loop's id2t + // map under one lock, then fall back to a broker fetch for any + // truly unresolved IDs. + var zeroID [16]byte + var unresolvedIDs [][16]byte + if len(req.Topics) > 0 { + id2t := cl.id2tMap() + cl.metaCache.mu.Lock() + for _, t := range req.Topics { + if t.Topic != nil && *t.Topic != "" { + topics = append(topics, *t.Topic) + continue + } + if t.TopicID == zeroID { + cl.metaCache.mu.Unlock() + return nil, errors.New("unable to request cached metadata with a missing topic name and zero topic ID") + } + if name, ok := cl.metaCache.byID[t.TopicID]; ok { + topics = append(topics, name) + } else if name := id2t[t.TopicID]; name != "" { + topics = append(topics, name) + } else { + unresolvedIDs = append(unresolvedIDs, t.TopicID) + } + } + cl.metaCache.mu.Unlock() + } + + // Phase 2: for truly unresolved IDs, fetch by ID to learn the + // name. This also caches the results in metaCache via + // storeCachedMeta (called inside fetchMetadata). + var idErrTopics []kmsg.MetadataResponseTopic + if len(unresolvedIDs) > 0 { + idReq := kmsg.NewPtrMetadataRequest() + for _, id := range unresolvedIDs { + rt := kmsg.NewMetadataRequestTopic() + rt.TopicID = id + idReq.Topics = append(idReq.Topics, rt) + } + _, meta, err := cl.fetchMetadata(ctx, idReq, true, nil) + if err != nil { + return nil, err + } + for _, t := range meta.Topics { + if t.Topic != nil && *t.Topic != "" { + topics = append(topics, *t.Topic) + } else { + idErrTopics = append(idErrTopics, t) + } + } + } + + // Phase 3: fetch all resolved topic names, using the cache. + cached, err := cl.resolveTopicMeta(ctx, topics, true, limit) + if err != nil { + return nil, err + } + + // Phase 4: build the response. We deeply clone all cached data so + // that the end user cannot modify internal data. + dups := func(s *string) *string { + if s == nil { + return nil + } + s2 := *s + return &s2 + } + dupp := func(p kmsg.MetadataResponseTopicPartition) kmsg.MetadataResponseTopicPartition { + p2 := p + p2.Replicas = slices.Clone(p2.Replicas) + p2.ISR = slices.Clone(p2.ISR) + p2.OfflineReplicas = slices.Clone(p2.OfflineReplicas) + p2.UnknownTags = kmsg.Tags{} + return p2 + } + dupt := func(t kmsg.MetadataResponseTopic) kmsg.MetadataResponseTopic { + t2 := t + t2.Topic = dups(t2.Topic) + t2.Partitions = make([]kmsg.MetadataResponseTopicPartition, 0, len(t2.Partitions)) + for _, p := range t.Partitions { + t2.Partitions = append(t2.Partitions, dupp(p)) + } + // We do not request or return authorized operations. + // Populating them can be noticeably slow on some brokers, + // and always requesting them would penalize the common + // case. If the caller needs auth ops, they should issue a + // standard metadata request directly. + t2.AuthorizedOperations = math.MinInt32 + t2.UnknownTags = kmsg.Tags{} + return t2 + } + + resp := kmsg.NewPtrMetadataResponse() + + cl.brokersMu.RLock() + for _, b := range cl.brokers { + resp.Brokers = append(resp.Brokers, kmsg.MetadataResponseBroker{ + NodeID: b.meta.NodeID, + Host: b.meta.Host, + Port: b.meta.Port, + Rack: b.meta.Rack, + }) + } + cl.brokersMu.RUnlock() + + cl.controllerIDMu.Lock() + resp.ClusterID = dups(cl.clusterID) + resp.ControllerID = cl.controllerID + cl.controllerIDMu.Unlock() + + for _, t := range cached { + resp.Topics = append(resp.Topics, dupt(t.t)) + } + for _, t := range idErrTopics { + resp.Topics = append(resp.Topics, dupt(t)) + } + + resp.AuthorizedOperations = math.MinInt32 // see comment in dupt + + return resp, nil +} + +func (cl *Client) retryable() *retryable { + return cl.retryableBrokerFn(func() (*broker, error) { return cl.broker(), nil }) +} + +func (cl *Client) retryableBrokerFn(fn func() (*broker, error)) *retryable { + return &retryable{cl: cl, br: fn} +} + +func (cl *Client) shouldRetry(tries int, err error) bool { + return (kerr.IsRetriable(err) || isRetryableBrokerErr(err)) && int64(tries) <= cl.cfg.retries +} + +func (cl *Client) shouldRetryNext(tries int, err error) bool { + return isSkippableBrokerErr(err) && int64(tries) <= cl.cfg.retries +} + +type retryable struct { + cl *Client + br func() (*broker, error) + last *broker + + // If non-zero, limitRetries may specify a smaller # of retries than + // the client RequestRetries number. This is used for internal requests + // that can fail / do not need to retry forever. + limitRetries int + + // parseRetryErr, if non-nil, can delete stale cached brokers. We do + // *not* return the error from this function to the caller, but we do + // use it to potentially retry. It is not necessary, but also not + // harmful, to return the input error. + parseRetryErr func(kmsg.Response, error) error +} + +type failDial struct { + clearFn func() +} + +// handleDialErr converts transient dial errors into errChosenBrokerDead so +// that the retryable loop retries with backoff. isRetryableBrokerErr excludes +// dial errors by design, and shouldRetryNext cannot help because pinned broker +// functions return the same broker every time. +// +// On every dial failure we call clearFn to drop cached controller and +// coordinator entries pointing at the dead broker. We call it every time, not just +// once, because between retries the cache may be repopulated with a different +// broker that also fails. The clearFns are idempotent. +// +// Permanent dial errors (NXDOMAIN, EACCES, EPERM) still clear the cache but +// are not retried. Retries are bounded by retryTimeout / ctx / RequestRetries. +func (d *failDial) handleDialErr(err error) error { + if !isAnyDialErr(err) { + return err + } + d.clearFn() + if isPermanentDialErr(err) { + return err + } + return errChosenBrokerDead +} + +func (r *retryable) Request(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { + tries := 0 + tryStart := time.Now() + retryTimeout := r.cl.cfg.retryTimeout(req.Key()) + + next, nextErr := r.br() +start: + tries++ + br, err := next, nextErr + r.last = br + var resp kmsg.Response + var retryErr error + if err == nil { + resp, err = r.last.waitResp(ctx, req) + if r.parseRetryErr != nil { + retryErr = r.parseRetryErr(resp, err) + } + } + + log := func(backoff time.Duration) { + r.cl.cfg.logger.Log(LogLevelDebug, "retrying request", + "request", kmsg.NameForKey(req.Key()), + "tries", tries, + "backoff", backoff, + "time_since_start", time.Since(tryStart), + "request_error", err, + "response_error", retryErr, + ) + } + + if err != nil || retryErr != nil { + if r.limitRetries == 0 || tries <= r.limitRetries { + backoff := r.cl.cfg.retryBackoff(tries) + if retryTimeout == 0 || time.Now().Add(backoff).Sub(tryStart) <= retryTimeout { + // If this broker / request had a retryable error, we can + // just retry now. If the error is *not* retryable but + // is a broker-specific network error, and the next + // broker is different than the current, we also retry. + if r.cl.shouldRetry(tries, err) || r.cl.shouldRetry(tries, retryErr) { + log(backoff) + if r.cl.waitTries(ctx, backoff) { + next, nextErr = r.br() + goto start + } + } else if r.cl.shouldRetryNext(tries, err) { + log(backoff) + next, nextErr = r.br() + if next != br && r.cl.waitTries(ctx, backoff) { + goto start + } + } + } + } + } + return resp, err +} + +// ResponseShard ties together a request with either the response it received +// or an error that prevented a response from being received. +type ResponseShard struct { + // Meta contains the broker that this request was issued to, or an + // unknown (node ID -1) metadata if the request could not be issued. + // + // Requests can fail to even be issued if an appropriate broker cannot + // be loaded of if the client cannot understand the request. + Meta BrokerMetadata + + // Req is the request that was issued to this broker. + Req kmsg.Request + + // Resp is the response received from the broker, if any. + Resp kmsg.Response + + // Err, if non-nil, is the error that prevented a response from being + // received or the request from being issued. + Err error +} + +// RequestSharded performs the same logic as Request, but returns all responses +// from any broker that the request was split to. This always returns at least +// one shard. If the request does not need to be issued (describing no groups), +// this issues the request to a random broker just to ensure that one shard +// exists. +// +// There are only a few requests that are strongly recommended to explicitly +// use RequestSharded; the rest can by default use Request. These few requests +// are mentioned in the documentation for Request. +// +// If, in the process of splitting a request, some topics or partitions are +// found to not exist, or Kafka replies that a request should go to a broker +// that does not exist, all those non-existent pieces are grouped into one +// request to the first seed broker. This will show up as a seed broker node ID +// (min int32) and the response will likely contain purely errors. +// +// The response shards are ordered by broker metadata. +func (cl *Client) RequestSharded(ctx context.Context, req kmsg.Request) []ResponseShard { + resps, _ := cl.shardedRequest(ctx, req) + sort.Slice(resps, func(i, j int) bool { + l := &resps[i].Meta + r := &resps[j].Meta + + if l.NodeID < r.NodeID { + return true + } + if r.NodeID < l.NodeID { + return false + } + if l.Host < r.Host { + return true + } + if r.Host < l.Host { + return false + } + if l.Port < r.Port { + return true + } + if r.Port < l.Port { + return false + } + if l.Rack == nil { + return true + } + if r.Rack == nil { + return false + } + return *l.Rack < *r.Rack + }) + return resps +} + +type shardMerge func([]ResponseShard) (kmsg.Response, error) + +func (cl *Client) shardedRequest(ctx context.Context, req kmsg.Request) ([]ResponseShard, shardMerge) { + ctx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + defer close(done) + go func() { + defer cancel() + select { + case <-done: + case <-ctx.Done(): + case <-cl.ctx.Done(): + } + }() + + // First, handle any sharded request. This comes before the conditional + // below because this handles two group requests, which we do not want + // to fall into the handleCoordinatorReq logic. + switch t := req.(type) { + case *kmsg.ListOffsetsRequest, // key 2 + *kmsg.OffsetFetchRequest, // key 9 + *kmsg.FindCoordinatorRequest, // key 10 + *kmsg.DescribeGroupsRequest, // key 15 + *kmsg.ListGroupsRequest, // key 16 + *kmsg.DeleteRecordsRequest, // key 21 + *kmsg.OffsetForLeaderEpochRequest, // key 23 + *kmsg.AddPartitionsToTxnRequest, // key 24 + *kmsg.WriteTxnMarkersRequest, // key 27 + *kmsg.DescribeConfigsRequest, // key 32 + *kmsg.AlterConfigsRequest, // key 33 + *kmsg.AlterReplicaLogDirsRequest, // key 34 + *kmsg.DescribeLogDirsRequest, // key 35 + *kmsg.DeleteGroupsRequest, // key 42 + *kmsg.IncrementalAlterConfigsRequest, // key 44 + *kmsg.DescribeProducersRequest, // key 61 + *kmsg.DescribeTransactionsRequest, // key 65 + *kmsg.ListTransactionsRequest, // key 66 + *kmsg.ConsumerGroupDescribeRequest, // key 69 + *kmsg.ShareGroupDescribeRequest, // key 77 + *kmsg.DescribeShareGroupOffsetsRequest: // key 90 + return cl.handleShardedReq(ctx, req) + + case *kmsg.MetadataRequest: + // We hijack any metadata request so as to populate our + // own brokers and controller ID. + br, resp, err := cl.fetchMetadata(ctx, t, false, nil) + return shards(shard(br, req, resp, err)), nil + + case kmsg.AdminRequest: + return shards(cl.handleAdminReq(ctx, t)), nil + + case kmsg.GroupCoordinatorRequest, + kmsg.TxnCoordinatorRequest: + return shards(cl.handleCoordinatorReq(ctx, t)), nil + + case *kmsg.ApiVersionsRequest: + // As of v3, software name and version are required. + // If they are missing, we use the config options. + if t.ClientSoftwareName == "" && t.ClientSoftwareVersion == "" { + dup := *t + dup.ClientSoftwareName = cl.cfg.softwareName + dup.ClientSoftwareVersion = cl.cfg.softwareVersion + req = &dup + } + } + + // All other requests not handled above can be issued to any broker + // with the default retryable logic. + r := cl.retryable() + resp, err := r.Request(ctx, req) + return shards(shard(r.last, req, resp, err)), nil +} + +func shard(br *broker, req kmsg.Request, resp kmsg.Response, err error) ResponseShard { + if br == nil { // the broker could be nil if loading the broker failed. + return ResponseShard{unknownBrokerMetadata, req, resp, err} + } + return ResponseShard{br.meta, req, resp, err} +} + +func shards(shard ...ResponseShard) []ResponseShard { + return shard +} + +func findBroker(candidates []*broker, node int32) *broker { + n := sort.Search(len(candidates), func(n int) bool { return candidates[n].meta.NodeID >= node }) + var b *broker + if n < len(candidates) { + c := candidates[n] + if c.meta.NodeID == node { + b = c + } + } + return b +} + +// brokerOrErr returns the broker for ID or the error if the broker does not +// exist. +// +// If tryLoad is true and the broker does not exist, this attempts a broker +// metadata load once before failing. If the metadata load fails, this returns +// that error. +func (cl *Client) brokerOrErr(ctx context.Context, id int32, err error) (*broker, error) { + if id < 0 { + return nil, err + } + + tryLoad := ctx != nil + tries := 0 +start: + var broker *broker + if id < 0 { + broker = findBroker(cl.loadSeeds(), id) + } else { + cl.brokersMu.RLock() + broker = findBroker(cl.brokers, id) + cl.brokersMu.RUnlock() + } + + if broker == nil { + if tryLoad { + if loadErr := cl.fetchBrokerMetadata(ctx); loadErr != nil { + return nil, loadErr + } + // We will retry loading up to two times, if we load broker + // metadata twice successfully but neither load has the broker + // we are looking for, then we say our broker does not exist. + tries++ + if tries < 2 { + goto start + } + } + return nil, err + } + return broker, nil +} + +// controller returns the controller broker, forcing a broker load if +// necessary. +func (cl *Client) controller(ctx context.Context) (b *broker, err error) { + get := func() int32 { + cl.controllerIDMu.Lock() + defer cl.controllerIDMu.Unlock() + return cl.controllerID + } + + defer func() { + if ec := (*errUnknownController)(nil); errors.As(err, &ec) { + cl.forgetControllerID(ec.id) + } + }() + + var id int32 + if id = get(); id < 0 { + if err := cl.fetchBrokerMetadata(ctx); err != nil { + return nil, err + } + if id = get(); id < 0 { + return nil, &errUnknownController{id} + } + } + + return cl.brokerOrErr(nil, id, &errUnknownController{id}) +} + +// forgetControllerID is called once an admin requests sees NOT_CONTROLLER. +func (cl *Client) forgetControllerID(id int32) { + cl.controllerIDMu.Lock() + defer cl.controllerIDMu.Unlock() + if cl.controllerID == id { + cl.controllerID = unknownControllerID + } +} + +const ( + coordinatorTypeGroup int8 = 0 + coordinatorTypeTxn int8 = 1 + coordinatorTypeShare int8 = 2 +) + +type coordinatorKey struct { + name string + typ int8 +} + +type coordinatorLoad struct { + loadWait chan struct{} + node int32 + err error +} + +func (cl *Client) loadCoordinator(ctx context.Context, typ int8, key string) (*broker, error) { + berr := cl.loadCoordinators(ctx, typ, key)[key] + return berr.b, berr.err +} + +func (cl *Client) loadCoordinators(ctx context.Context, typ int8, keys ...string) map[string]brokerOrErr { + mch := make(chan map[string]brokerOrErr, 1) + go func() { mch <- cl.doLoadCoordinators(ctx, typ, keys...) }() + select { + case m := <-mch: + return m + case <-ctx.Done(): + m := make(map[string]brokerOrErr, len(keys)) + for _, k := range keys { + m[k] = brokerOrErr{nil, ctx.Err()} + } + return m + } +} + +// doLoadCoordinators uses the caller context to cancel loading metadata +// (brokerOrErr), but we use the client context to actually issue the request. +// There should be only one direct call to doLoadCoordinators, just above in +// loadCoordinator. It is possible for two requests to be loading the same +// coordinator (in fact, that's the point of this function -- collapse these +// requests). We do not want the first request canceling it's context to cause +// errors for the second request. +// +// It is ok to leave FindCoordinator running even if the caller quits. Worst +// case, we just cache things for some time in the future; yay. +func (cl *Client) doLoadCoordinators(ctx context.Context, typ int8, keys ...string) map[string]brokerOrErr { + m := make(map[string]brokerOrErr, len(keys)) + if len(keys) == 0 { + return m + } + + toRequest := make(map[string]bool, len(keys)) // true == bypass the cache + for _, key := range keys { + toRequest[key] = false + } + + // For each of these keys, we have two cases: + // + // 1) The key is cached. It is either loading or loaded. We do not + // request the key ourselves; we wait for the load to finish. + // + // 2) The key is not cached, and we request it. + // + // If a key is cached but the coordinator no longer exists for us, we + // re-request to refresh the coordinator by setting toRequest[key] to + // true (bypass cache). + // + // If we ever request a key ourselves, we do not request it again. We + // ensure this by deleting from toRequest. We also delete if the key + // was cached with no error. + // + // We could have some keys cached and some that need to be requested. + // We issue a request but do not request what is cached. + // + // Lastly, we only ever trigger one metadata update, which happens if + // we have an unknown coordinator after we load coordinators. + var hasLoadedBrokers bool + for len(toRequest) > 0 { + var loadWait chan struct{} + load2key := make(map[*coordinatorLoad][]string) + + cl.coordinatorsMu.Lock() + for key, bypassCache := range toRequest { + c, ok := cl.coordinators[coordinatorKey{key, typ}] + if !ok || bypassCache { + if loadWait == nil { + loadWait = make(chan struct{}) + } + c = &coordinatorLoad{ + loadWait: loadWait, + err: errors.New("coordinator was not returned in broker response"), + } + cl.coordinators[coordinatorKey{key, typ}] = c + } + load2key[c] = append(load2key[c], key) + } + cl.coordinatorsMu.Unlock() + + if loadWait == nil { // all coordinators were cached + hasLoadedBrokers = cl.waitCoordinatorLoad(ctx, typ, load2key, !hasLoadedBrokers, toRequest, m) + continue + } + + key2load := make(map[string]*coordinatorLoad) + req := kmsg.NewPtrFindCoordinatorRequest() + req.CoordinatorType = typ + for c, keys := range load2key { + if c.loadWait == loadWait { // if this is our wait, this is ours to request + req.CoordinatorKeys = append(req.CoordinatorKeys, keys...) + for _, key := range keys { + key2load[key] = c + delete(toRequest, key) + } + } + } + + cl.cfg.logger.Log(LogLevelDebug, "prepared to issue find coordinator request", + "coordinator_type", typ, + "coordinator_keys", req.CoordinatorKeys, + ) + + ctx := context.WithValue(cl.ctx, noShardRetryCtx, true) + shards := cl.RequestSharded(ctx, req) + + for _, shard := range shards { + if shard.Err != nil { + req := shard.Req.(*kmsg.FindCoordinatorRequest) + for _, key := range req.CoordinatorKeys { + c, ok := key2load[key] + if ok { + c.err = shard.Err + } + } + } else { + resp := shard.Resp.(*kmsg.FindCoordinatorResponse) + for _, rc := range resp.Coordinators { + c, ok := key2load[rc.Key] + if ok { + c.err = kerr.ErrorForCode(rc.ErrorCode) + c.node = rc.NodeID + } + } + } + } + + // For anything we loaded, if it has a load failure (including + // not being replied to), we remove the key from the cache. We + // do not want to cache erroring values. + // + // We range key2load, which contains only coordinators we are + // responsible for loading. + cl.coordinatorsMu.Lock() + for key, c := range key2load { + if c.err != nil { + ck := coordinatorKey{key, typ} + if loading, ok := cl.coordinators[ck]; ok && loading == c { + delete(cl.coordinators, ck) + } + } + } + cl.coordinatorsMu.Unlock() + + close(loadWait) + hasLoadedBrokers = cl.waitCoordinatorLoad(ctx, typ, load2key, !hasLoadedBrokers, toRequest, m) + } + return m +} + +// After some prep work, we wait for coordinators to load. We update toRequest +// values with true if the caller should bypass cache and re-load these +// coordinators. +// +// This returns if we load brokers, and populates m with results. +func (cl *Client) waitCoordinatorLoad(ctx context.Context, typ int8, load2key map[*coordinatorLoad][]string, shouldLoadBrokers bool, toRequest map[string]bool, m map[string]brokerOrErr) bool { + var loadedBrokers bool + for c, keys := range load2key { + <-c.loadWait + for _, key := range keys { + if c.err != nil { + delete(toRequest, key) + m[key] = brokerOrErr{nil, c.err} + continue + } + + var brokerCtx context.Context + if shouldLoadBrokers && !loadedBrokers { + brokerCtx = ctx + loadedBrokers = true + } + + b, err := cl.brokerOrErr(brokerCtx, c.node, &errUnknownCoordinator{c.node, coordinatorKey{key, typ}}) + if err != nil { + if _, exists := toRequest[key]; exists { + toRequest[key] = true + continue + } + // If the key does not exist, we just loaded this + // coordinator and also the brokers. We do not + // re-request. + } + delete(toRequest, key) + m[key] = brokerOrErr{b, err} + } + } + return loadedBrokers +} + +func (cl *Client) maybeDeleteStaleCoordinator(name string, typ int8, err error) bool { + switch { + case errors.Is(err, kerr.CoordinatorNotAvailable), + errors.Is(err, kerr.CoordinatorLoadInProgress), + errors.Is(err, kerr.NotCoordinator): + cl.deleteStaleCoordinator(name, typ) + return true + } + return false +} + +func (cl *Client) deleteStaleCoordinator(name string, typ int8) { + cl.coordinatorsMu.Lock() + defer cl.coordinatorsMu.Unlock() + k := coordinatorKey{name, typ} + v := cl.coordinators[k] + if v == nil { + return + } + select { + case <-v.loadWait: + delete(cl.coordinators, k) + default: + // We are actively reloading this coordinator. + } +} + +// deleteStaleCoordinatorsByNode removes all cached coordinator entries +// resolved to the given broker. Entries that are actively loading are +// skipped. +func (cl *Client) deleteStaleCoordinatorsByNode(node int32) { + cl.coordinatorsMu.Lock() + defer cl.coordinatorsMu.Unlock() + for k, v := range cl.coordinators { + if v == nil || v.node != node { + continue + } + select { + case <-v.loadWait: + delete(cl.coordinators, k) + default: + } + } +} + +type brokerOrErr struct { + b *broker + err error +} + +func (cl *Client) handleAdminReq(ctx context.Context, req kmsg.Request) ResponseShard { + // Loading a controller can perform some wait; we accept that and do + // not account for the retries or the time to load the controller as + // part of the retries / time to issue the req. + r := cl.retryableBrokerFn(func() (*broker, error) { + return cl.controller(ctx) + }) + + // The only request that can break cached metadata is CreatePartitions, + // because our mapping will still be "valid" but behind the scenes, + // more partitions exist. If CreatePartitions is going through this + // client, we preemptively delete any mapping for these topics. + if t, ok := req.(*kmsg.CreatePartitionsRequest); ok { + var topics []string + for i := range t.Topics { + topics = append(topics, t.Topics[i].Topic) + } + cl.maybeDeleteCachedMeta(false, topics...) + } + + d := failDial{clearFn: func() { + cl.forgetControllerID(r.last.meta.NodeID) + cl.deleteStaleCoordinatorsByNode(r.last.meta.NodeID) + }} + r.parseRetryErr = func(resp kmsg.Response, err error) error { + if err != nil { + return d.handleDialErr(err) + } + var code int16 + switch t := resp.(type) { + case *kmsg.CreateTopicsResponse: + if len(t.Topics) > 0 { + code = t.Topics[0].ErrorCode + } + case *kmsg.DeleteTopicsResponse: + if len(t.Topics) > 0 { + code = t.Topics[0].ErrorCode + } + case *kmsg.CreatePartitionsResponse: + if len(t.Topics) > 0 { + code = t.Topics[0].ErrorCode + } + case *kmsg.ElectLeadersResponse: + if len(t.Topics) > 0 && len(t.Topics[0].Partitions) > 0 { + code = t.Topics[0].Partitions[0].ErrorCode + } + case *kmsg.AlterPartitionAssignmentsResponse: + code = t.ErrorCode + case *kmsg.ListPartitionReassignmentsResponse: + code = t.ErrorCode + case *kmsg.AlterUserSCRAMCredentialsResponse: + if len(t.Results) > 0 { + code = t.Results[0].ErrorCode + } + case *kmsg.VoteResponse: + code = t.ErrorCode + case *kmsg.BeginQuorumEpochResponse: + code = t.ErrorCode + case *kmsg.EndQuorumEpochResponse: + code = t.ErrorCode + case *kmsg.DescribeQuorumResponse: + code = t.ErrorCode + case *kmsg.AlterPartitionResponse: + code = t.ErrorCode + case *kmsg.UpdateFeaturesResponse: + code = t.ErrorCode + case *kmsg.EnvelopeResponse: + code = t.ErrorCode + } + if err := kerr.ErrorForCode(code); errors.Is(err, kerr.NotController) { + // There must be a last broker if we were able to issue + // the request and get a response. + cl.forgetControllerID(r.last.meta.NodeID) + return err + } + return nil + } + + resp, err := r.Request(ctx, req) + return shard(r.last, req, resp, err) +} + +// handleCoordinatorReq issues simple (non-shardable) group or txn requests. +func (cl *Client) handleCoordinatorReq(ctx context.Context, req kmsg.Request) ResponseShard { + switch t := req.(type) { + default: + // All group requests should be listed below, so if it isn't, + // then we do not know what this request is. + return shard(nil, req, nil, errors.New("client is too old; this client does not know what to do with this request")) + + ///////// + // TXN // -- all txn reqs are simple + ///////// + + case *kmsg.InitProducerIDRequest: + if t.TransactionalID != nil { + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeTxn, *t.TransactionalID, req) + } + // InitProducerID can go to any broker if the transactional ID + // is nil. By using handleReqWithCoordinator, we get the + // retryable-error parsing, even though we are not actually + // using a defined txn coordinator. This is fine; by passing no + // names, we delete no coordinator. + coordinator, resp, err := cl.handleReqWithCoordinator(ctx, func() (*broker, error) { return cl.broker(), nil }, coordinatorTypeTxn, "", req) + return shard(coordinator, req, resp, err) + case *kmsg.AddOffsetsToTxnRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeTxn, t.TransactionalID, req) + case *kmsg.EndTxnRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeTxn, t.TransactionalID, req) + + /////////// + // GROUP // -- most group reqs are simple + /////////// + + case *kmsg.OffsetCommitRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.TxnOffsetCommitRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.JoinGroupRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.HeartbeatRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.LeaveGroupRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.SyncGroupRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + case *kmsg.OffsetDeleteRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeGroup, t.Group, req) + + // ConsumerGroupHeartbeat cannot be retried at all + case *kmsg.ConsumerGroupHeartbeatRequest: + br, err := cl.loadCoordinator(ctx, coordinatorTypeGroup, t.Group) + var resp kmsg.Response + if err == nil { + resp, err = br.waitResp(ctx, req) + } + return shard(br, req, resp, err) + + /////////// + // SHARE // + /////////// + + // ShareGroupHeartbeat cannot be retried (like ConsumerGroupHeartbeat). + // Like ConsumerGroupHeartbeat, share membership is managed by the + // GROUP coordinator, not the share-state coordinator. + case *kmsg.ShareGroupHeartbeatRequest: + br, err := cl.loadCoordinator(ctx, coordinatorTypeGroup, t.GroupID) + var resp kmsg.Response + if err == nil { + resp, err = br.waitResp(ctx, req) + } + return shard(br, req, resp, err) + case *kmsg.AlterShareGroupOffsetsRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeShare, t.GroupID, req) + case *kmsg.DeleteShareGroupOffsetsRequest: + return cl.handleCoordinatorReqSimple(ctx, coordinatorTypeShare, t.GroupID, req) + } +} + +// handleCoordinatorReqSimple issues a request that contains a single group or +// txn to its coordinator. +// +// The error is inspected to see if it is a retryable error and, if so, the +// coordinator is deleted. +func (cl *Client) handleCoordinatorReqSimple(ctx context.Context, typ int8, name string, req kmsg.Request) ResponseShard { + coordinator, resp, err := cl.handleReqWithCoordinator(ctx, func() (*broker, error) { + return cl.loadCoordinator(ctx, typ, name) + }, typ, name, req) + return shard(coordinator, req, resp, err) +} + +// handleReqWithCoordinator actually issues a request to a coordinator and +// does retry handling. +// +// This avoids retries on the two group requests that need to be sharded. +func (cl *Client) handleReqWithCoordinator( + ctx context.Context, + coordinator func() (*broker, error), + typ int8, + name string, // group ID or the transactional id + req kmsg.Request, +) (*broker, kmsg.Response, error) { + r := cl.retryableBrokerFn(coordinator) + d := failDial{clearFn: func() { + cl.forgetControllerID(r.last.meta.NodeID) + cl.deleteStaleCoordinatorsByNode(r.last.meta.NodeID) + }} + r.parseRetryErr = func(resp kmsg.Response, err error) error { + if err != nil { + return d.handleDialErr(err) + } + var code int16 + switch t := resp.(type) { + // TXN + case *kmsg.InitProducerIDResponse: + code = t.ErrorCode + case *kmsg.AddOffsetsToTxnResponse: + code = t.ErrorCode + case *kmsg.EndTxnResponse: + code = t.ErrorCode + + // GROUP + case *kmsg.OffsetCommitResponse: + if len(t.Topics) > 0 && len(t.Topics[0].Partitions) > 0 { + code = t.Topics[0].Partitions[0].ErrorCode + } + case *kmsg.TxnOffsetCommitResponse: + if len(t.Topics) > 0 && len(t.Topics[0].Partitions) > 0 { + code = t.Topics[0].Partitions[0].ErrorCode + } + case *kmsg.JoinGroupResponse: + code = t.ErrorCode + case *kmsg.HeartbeatResponse: + code = t.ErrorCode + case *kmsg.LeaveGroupResponse: + code = t.ErrorCode + case *kmsg.SyncGroupResponse: + code = t.ErrorCode + case *kmsg.ConsumerGroupHeartbeatResponse: + code = t.ErrorCode + } + + // ListGroups, OffsetFetch, DeleteGroups, DescribeGroups, and + // DescribeTransactions handled in sharding. + + if err := kerr.ErrorForCode(code); cl.maybeDeleteStaleCoordinator(name, typ, err) { + return err + } + return nil + } + + resp, err := r.Request(ctx, req) + return r.last, resp, err +} + +// Broker returns a handle to a specific broker to directly issue requests to. +// Note that there is no guarantee that this broker exists; if it does not, +// requests will fail with an unknown broker error. +func (cl *Client) Broker(id int) *Broker { + return &Broker{ + id: int32(id), + cl: cl, + } +} + +// DiscoveredBrokers returns all brokers that were discovered from prior +// metadata responses. This does not actually issue a metadata request to load +// brokers; if you wish to ensure this returns all brokers, be sure to manually +// issue a metadata request before this. This also does not include seed +// brokers, which are internally saved under special internal broker IDs (but, +// it does include those brokers under their normal IDs as returned from a +// metadata response). +func (cl *Client) DiscoveredBrokers() []*Broker { + cl.brokersMu.RLock() + defer cl.brokersMu.RUnlock() + + var bs []*Broker + for _, broker := range cl.brokers { + bs = append(bs, &Broker{id: broker.meta.NodeID, cl: cl}) + } + return bs +} + +// SeedBrokers returns the all seed brokers. +func (cl *Client) SeedBrokers() []*Broker { + var bs []*Broker + for _, broker := range cl.loadSeeds() { + bs = append(bs, &Broker{id: broker.meta.NodeID, cl: cl}) + } + return bs +} + +// UpdateSeedBrokers updates the client's list of seed brokers. Over the course +// of a long period of time, your might replace all brokers that you originally +// specified as seeds. This command allows you to replace the client's list of +// seeds. +// +// This returns an error if any of the input addrs is not a host:port. If the +// input list is empty, the function returns without replacing the seeds. +func (cl *Client) UpdateSeedBrokers(addrs ...string) error { + if len(addrs) == 0 { + return nil + } + seeds, err := parseSeeds(addrs) + if err != nil { + return err + } + + seedBrokers := make([]*broker, 0, len(seeds)) + for i, seed := range seeds { + b := cl.newBroker(unknownSeedID(i), seed.host, seed.port, nil) + seedBrokers = append(seedBrokers, b) + } + + // We lock to guard against concurrently updating seeds; we do not need + // the lock for what this usually guards. + cl.brokersMu.Lock() + old := cl.loadSeeds() + cl.seeds.Store(seedBrokers) + cl.brokersMu.Unlock() + + for _, b := range old { + b.stopForever() + } + + return nil +} + +// Broker pairs a broker ID with a client to directly issue requests to a +// specific broker. +type Broker struct { + id int32 + cl *Client +} + +// Request issues a request to a broker. If the broker does not exist in the +// client, this returns an unknown broker error. Requests are not retried. +// +// The passed context can be used to cancel a request and return early. +// Note that if the request is not canceled before it is written to Kafka, +// you may just end up canceling and not receiving the response to what Kafka +// inevitably does. +// +// It is more beneficial to always use RetriableRequest. +func (b *Broker) Request(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { + return b.request(ctx, false, req) +} + +// RetriableRequest issues a request to a broker the same as Broker, but +// retries in the face of retryable broker connection errors. This does not +// retry on response internal errors. +func (b *Broker) RetriableRequest(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { + return b.request(ctx, true, req) +} + +func (b *Broker) request(ctx context.Context, retry bool, req kmsg.Request) (kmsg.Response, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var resp kmsg.Response + var err error + done := make(chan struct{}) + + go func() { + defer close(done) + + if !retry { + var br *broker + br, err = b.cl.brokerOrErr(ctx, b.id, errUnknownBroker) + if err == nil { + resp, err = br.waitResp(ctx, req) + } + } else { + r := b.cl.retryableBrokerFn(func() (*broker, error) { + return b.cl.brokerOrErr(ctx, b.id, errUnknownBroker) + }) + // Pinned by ID: on dial failure, retry with backoff to give + // the broker time to come back (e.g. rolling restart). + // Also clear cached controller/coordinator entries pointing + // at this broker so other code paths re-resolve. + d := failDial{clearFn: func() { + b.cl.forgetControllerID(b.id) + b.cl.deleteStaleCoordinatorsByNode(b.id) + }} + r.parseRetryErr = func(_ kmsg.Response, err error) error { + if err != nil { + return d.handleDialErr(err) + } + return nil + } + resp, err = r.Request(ctx, req) + } + }() + + select { + case <-done: + return resp, err + case <-ctx.Done(): + return nil, ctx.Err() + case <-b.cl.ctx.Done(): + return nil, b.cl.ctx.Err() + } +} + +////////////////////// +// REQUEST SHARDING // +////////////////////// + +// Below here lies all logic to handle requests that need to be split and sent +// to many brokers. A lot of the logic for each sharding function is very +// similar, but each sharding function uses slightly different types. + +// issueShard is a request that has been split and is ready to be sent to the +// given broker ID. +type issueShard struct { + req kmsg.Request + pin *pinReq + broker int32 + any bool + + // if non-nil, we could not map this request shard to any broker, and + // this error is the reason. + err error +} + +// sharder splits a request. +type sharder interface { + // shard splits a request and returns the requests to issue tied to the + // brokers to issue the requests to. This can return an error if there + // is some pre-loading that needs to happen. If an error is returned, + // the request that was intended for splitting is failed wholesale. + // + // Due to sharded requests not being retryable if a response is + // received, to avoid stale coordinator errors, this function should + // not use any previously cached metadata. + // + // This takes the last error if the request is being retried, which is + // currently only useful for errBrokerTooOld. + shard(context.Context, kmsg.Request, error) ([]issueShard, bool, error) + + // onResp is called on a successful response to investigate the + // response and potentially perform cleanup, and potentially returns an + // error signifying to retry. See onShardRespErr below for more + // details. + onResp(kmsg.Request, kmsg.Response) error + + // merge is a function that can be used to merge sharded responses into + // one response. This is used by the client.Request method. + merge([]ResponseShard) (kmsg.Response, error) +} + +var noShardRetryCtx = func() *string { s := "no_shard_retry"; return &s }() + +// handleShardedReq splits and issues requests to brokers, recursively +// splitting as necessary if requests fail and need remapping. +func (cl *Client) handleShardedReq(ctx context.Context, req kmsg.Request) ([]ResponseShard, shardMerge) { + // First, determine our sharder. + var sharder sharder + switch req.(type) { + case *kmsg.ListOffsetsRequest: + sharder = &listOffsetsSharder{cl} + case *kmsg.OffsetFetchRequest: + sharder = &offsetFetchSharder{cl} + case *kmsg.FindCoordinatorRequest: + sharder = &findCoordinatorSharder{cl} + case *kmsg.DescribeGroupsRequest: + sharder = &describeGroupsSharder{cl} + case *kmsg.ListGroupsRequest: + sharder = &listGroupsSharder{cl} + case *kmsg.DeleteRecordsRequest: + sharder = &deleteRecordsSharder{cl} + case *kmsg.OffsetForLeaderEpochRequest: + sharder = &offsetForLeaderEpochSharder{cl} + case *kmsg.AddPartitionsToTxnRequest: + sharder = &addPartitionsToTxnSharder{cl} + case *kmsg.WriteTxnMarkersRequest: + sharder = &writeTxnMarkersSharder{cl} + case *kmsg.DescribeConfigsRequest: + sharder = &describeConfigsSharder{cl} + case *kmsg.AlterConfigsRequest: + sharder = &alterConfigsSharder{cl} + case *kmsg.AlterReplicaLogDirsRequest: + sharder = &alterReplicaLogDirsSharder{cl} + case *kmsg.DescribeLogDirsRequest: + sharder = &describeLogDirsSharder{cl} + case *kmsg.DeleteGroupsRequest: + sharder = &deleteGroupsSharder{cl} + case *kmsg.IncrementalAlterConfigsRequest: + sharder = &incrementalAlterConfigsSharder{cl} + case *kmsg.DescribeProducersRequest: + sharder = &describeProducersSharder{cl} + case *kmsg.DescribeTransactionsRequest: + sharder = &describeTransactionsSharder{cl} + case *kmsg.ListTransactionsRequest: + sharder = &listTransactionsSharder{cl} + case *kmsg.ConsumerGroupDescribeRequest: + sharder = &consumerGroupDescribeSharder{cl} + case *kmsg.ShareGroupDescribeRequest: + sharder = &shareGroupDescribeSharder{cl} + case *kmsg.DescribeShareGroupOffsetsRequest: + sharder = &describeShareGroupOffsetsSharder{cl} + } + + // If a request fails, we re-shard it (in case it needs to be split + // again). reqTry tracks how many total tries a request piece has had; + // we quit at either the max configured tries or max configured time. + type reqTry struct { + tries int + req kmsg.Request + lastErr error + } + + var ( + shardsMu xsync.Mutex + shards []ResponseShard + + addShard = func(shard ResponseShard) { + shardsMu.Lock() + defer shardsMu.Unlock() + shards = append(shards, shard) + } + + start = time.Now() + retryTimeout = cl.cfg.retryTimeout(req.Key()) + + wg sync.WaitGroup + issue func(reqTry, int32) + ) + + l := cl.cfg.logger + debug := l.Level() >= LogLevelDebug + noRetries := ctx != nil && ctx.Value(noShardRetryCtx) != nil + + // issue is called to progressively split and issue requests. + // + // This recursively calls itself if a request fails and can be retried. + // We avoid stack problems because this calls itself in a goroutine. + issue = func(try reqTry, avoidBroker int32) { + issues, reshardable, err := sharder.shard(ctx, try.req, try.lastErr) + if err != nil { + l.Log(LogLevelDebug, "unable to shard request", "req", kmsg.Key(try.req.Key()).Name(), "previous_tries", try.tries, "err", err) + addShard(shard(nil, try.req, nil, err)) // failure to shard means data loading failed; this request is failed + return + } + + // If the request actually does not need to be issued, we issue + // it to a random broker. There is no benefit to this, but at + // least we will return one shard. + if len(issues) == 0 { + issues = []issueShard{{ + req: try.req, + any: true, + }} + reshardable = true + } + + if debug { + var key int16 + var brokerAnys []string + for _, issue := range issues { + key = issue.req.Key() + if issue.err != nil { + brokerAnys = append(brokerAnys, "err") + } else if issue.any { + brokerAnys = append(brokerAnys, "any") + } else { + brokerAnys = append(brokerAnys, strconv.Itoa(int(issue.broker))) + } + } + l.Log(LogLevelDebug, "sharded request", "req", kmsg.Key(key).Name(), "destinations", brokerAnys) + } + + for i := range issues { + var ( + myIssue = issues[i] + isPinned bool + ctx = ctx // loop local context, in case we override by pinning + avoidBroker = avoidBroker // same + tries = try.tries // same + ) + if isPinned = myIssue.pin != nil; isPinned { + ctx = context.WithValue(ctx, ctxPinReq, myIssue.pin) + } + + if myIssue.err != nil { + addShard(shard(nil, myIssue.req, nil, myIssue.err)) + continue + } + + wg.Add(1) + go func() { + defer wg.Done() + start: + tries++ + + br := cl.broker() + var err error + if !myIssue.any { + br, err = cl.brokerOrErr(ctx, myIssue.broker, errUnknownBroker) + } else if avoidBroker != -1 { + for i := 0; i < 3 && br.meta.NodeID == avoidBroker; i++ { + br = cl.broker() + } + } + if err != nil { + addShard(shard(nil, myIssue.req, nil, err)) // failure to load a broker is a failure to issue a request + return + } + + resp, err := br.waitResp(ctx, myIssue.req) + var errIsFromResp bool + if err == nil { + err = sharder.onResp(myIssue.req, resp) // perform some potential cleanup, and potentially receive an error to retry + if ke := (*kerr.Error)(nil); errors.As(err, &ke) { + errIsFromResp = true + } + } + + // If we failed to issue the request, we *maybe* will retry. + // We could have failed to even issue the request or receive + // a response, which is retryable. + // + // If a pinned req fails with errBrokerTooOld, we always retry + // immediately. The request was not even issued. However, as a + // safety, we only do this 3 times to avoid some super weird + // pathological spin loop. + // + // We do retry on pinnedOld even if noRetries==true because + // the request was not issued; the sharder may handle + // errBrokerTooOld by pinning / splitting differently next try. + var ( + backoff = cl.cfg.retryBackoff(tries) + pinnedOld = reshardable && isPinned && errors.Is(err, errBrokerTooOld) && tries <= 3 + notTimedOut = retryTimeout == 0 || time.Now().Add(backoff).Sub(start) <= retryTimeout + shouldRetry = cl.shouldRetry(tries, err) + shouldRetryNext = myIssue.any && cl.shouldRetryNext(tries, err) + ) + + // If we retried on a "next" broker, but we randomly chose + // that same broker 3x, then we avoid retrying again on a + // "next" broker. + // + // If we retry at all, we need to clear `avoidBroker` in + // case it's already set. however, if we *do* need to retry + // on a different broker, then we set it. + if avoidBroker != -1 && br.meta.NodeID == avoidBroker { + shouldRetryNext = false + } + avoidBroker = -1 + if shouldRetryNext { + avoidBroker = br.meta.NodeID + } + + if err != nil && + (pinnedOld || + !noRetries && notTimedOut && (shouldRetry || shouldRetryNext) && cl.waitTries(ctx, backoff)) { + // Non-reshardable re-requests just jump back to the + // top where the broker is loaded. This is the case on + // requests where the original request is split to + // dedicated brokers; we do not want to re-shard that. + if !reshardable { + l.Log(LogLevelDebug, "sharded request failed, reissuing without resharding", "req", kmsg.Key(myIssue.req.Key()).Name(), "time_since_start", time.Since(start), "tries", tries, "err", err) + goto start + } + l.Log(LogLevelDebug, "sharded request failed, resharding and reissuing", "req", kmsg.Key(myIssue.req.Key()).Name(), "time_since_start", time.Since(start), "tries", tries, "err", err) + issue(reqTry{tries, myIssue.req, err}, avoidBroker) + return + } + + // If we pulled an error out of the response body in an attempt + // to possibly retry, the request was NOT an error that we want + // to bubble as a shard error. The request was successful, we + // have a response. Before we add the shard, strip the error. + // The end user can parse the response ErrorCode. + if errIsFromResp { + err = nil + } + addShard(shard(br, myIssue.req, resp, err)) // the error was not retryable + }() + } + } + + issue(reqTry{0, req, nil}, -1) + wg.Wait() + + return shards, sharder.merge +} + +// For sharded errors, we prefer to keep retryable errors rather than +// non-retryable errors. We keep the non-retryable if everything is +// non-retryable. +// +// We favor retryable because retryable means we used a stale cache value; we +// clear the stale entries on failure and the retry uses fresh data. The +// request will be split and remapped, and the non-retryable errors will be +// encountered again. +func onRespShardErr(err *error, newKerr error) { + if newKerr == nil || *err != nil && kerr.IsRetriable(*err) { + return + } + *err = newKerr +} + +// a convenience function for when a request needs to be issued identically to +// all brokers. +// +// If the request returns objects that are owned by a coordinator (groups, +// transactional IDs, etc.), an object that is mid-migration can transiently +// appear in BOTH the old and new coordinator's response. The sharder's should +// dedupe. +func (cl *Client) allBrokersShardedReq(ctx context.Context, fn func() kmsg.Request) ([]issueShard, bool, error) { + if err := cl.fetchBrokerMetadata(ctx); err != nil { + return nil, false, err + } + + var issues []issueShard + cl.brokersMu.RLock() + for _, broker := range cl.brokers { + issues = append(issues, issueShard{ + req: fn(), + broker: broker.meta.NodeID, + }) + } + cl.brokersMu.RUnlock() + + return issues, false, nil // we do NOT re-shard these requests request +} + +// a convenience function for saving the first ResponseShard error. +func firstErrMerger(sresps []ResponseShard, merge func(kresp kmsg.Response)) error { + var firstErr error + for _, sresp := range sresps { + if sresp.Err != nil { + if firstErr == nil { + firstErr = sresp.Err + } + continue + } + merge(sresp.Resp) + } + return firstErr +} + +type cachedMetaTopic struct { + id [16]byte + t kmsg.MetadataResponseTopic + ps map[int32]kmsg.MetadataResponseTopicPartition + when time.Time +} + +// For NOT_LEADER_FOR_PARTITION: +// We always delete stale metadata. It's possible that a leader rebalance +// happened immediately after we requested metadata; we should not pin to +// the stale metadata for 1s. +// +// For UNKNOWN_TOPIC_OR_PARTITION: +// We only delete stale metadata if it is older than the min age or 1s, +// whichever is smaller. We use 1s even if min age is larger, because we want +// to encourage larger min age for caching purposes. More obvious would be to +// *always* evict the cache here, but if we *just* requested metadata, then +// evicting the cache would cause churn for a topic that genuinely does not +// exist. +func (cl *Client) maybeDeleteCachedMeta(unknownTopic bool, ts ...string) (shouldRetry bool) { + if len(ts) == 0 { + return shouldRetry + } + + var min time.Duration + if unknownTopic { + min = time.Second + if cl.cfg.metadataMinAge < min { + min = cl.cfg.metadataMinAge + } + } + + now := time.Now() + cl.metaCache.mu.Lock() + defer cl.metaCache.mu.Unlock() + var zeroID [16]byte + for _, t := range ts { + ct, exists := cl.metaCache.topics[t] + if exists && (min == 0 || now.Sub(ct.when) > min) { + shouldRetry = true + delete(cl.metaCache.topics, t) + if ct.id != zeroID { + delete(cl.metaCache.byID, ct.id) + } + } + } + return shouldRetry +} + +// resolveTopicMeta provides a convenience type of working with metadata; +// this is garbage heavy, so it is only used in one off requests in this +// package. +// +// We only cache for metadata min age. We could theoretically cache forever, +// but an out of band CreatePartitions can result in our metadata being stale +// and us never knowing. So, we choose metadata min age. There are only a few +// requests that are sharded and use metadata, and the one this benefits most +// is ListOffsets. Likely, ListOffsets for the same topic will be issued back +// to back, so not caching for so long is ok. +func (cl *Client) resolveTopicMeta(ctx context.Context, topics []string, useCache bool, limit time.Duration) (map[string]cachedMetaTopic, error) { + if limit <= 0 { + limit = cl.cfg.metadataMinAge + } + + all := topics == nil + + // All-topics: return a copy of the cache if fresh. + if all && useCache { + cl.metaCache.mu.Lock() + if cl.metaCache.topics != nil && time.Since(cl.metaCache.allAt) < limit { + cached := make(map[string]cachedMetaTopic, len(cl.metaCache.topics)) + for k, v := range cl.metaCache.topics { + cached[k] = v + } + cl.metaCache.mu.Unlock() + return cached, nil + } + cl.metaCache.mu.Unlock() + } + + // No-topics: just need brokers/controller. The main metadata loop + // maintains both; we only fetch if we have no broker info yet. + if !all && len(topics) == 0 { + cl.brokersMu.RLock() + hasBrokers := len(cl.brokers) > 0 + cl.brokersMu.RUnlock() + if hasBrokers { + return make(map[string]cachedMetaTopic), nil + } + } + + // Specific topics: check cache for individual topics. + var results map[string]cachedMetaTopic + needed := topics + if len(topics) > 0 && useCache { + cl.metaCache.mu.Lock() + if len(cl.metaCache.topics) > 0 { + results = make(map[string]cachedMetaTopic) + needed = topics[:0] + for _, t := range topics { + tcached, exists := cl.metaCache.topics[t] + if exists && time.Since(tcached.when) < limit { + results[t] = tcached + } else { + needed = append(needed, t) + } + } + } + cl.metaCache.mu.Unlock() + if results != nil && len(needed) == 0 { + return results, nil + } + } + if results == nil { + results = make(map[string]cachedMetaTopic) + } + _, _, err := cl.fetchMetadataByName(ctx, all, needed, results) + return results, err +} + +// storeCachedMeta caches the fetched metadata in the Client, and +// optionally stores each topic in results. If all is true, this was an +// all-topics fetch and stale entries not in the response are evicted. +// +// Topics with a nil name are skipped. Per the Kafka protocol, the broker +// always populates the topic name for successfully resolved topics, even +// for TopicID-only requests. A nil name only occurs for error responses +// (e.g. UnknownTopicID), which cannot be meaningfully cached by name. +func (cl *Client) storeCachedMeta(meta *kmsg.MetadataResponse, all bool, results map[string]cachedMetaTopic) { + cl.metaCache.mu.Lock() + defer cl.metaCache.mu.Unlock() + if cl.metaCache.topics == nil { + cl.metaCache.topics = make(map[string]cachedMetaTopic) + } + if cl.metaCache.byID == nil { + cl.metaCache.byID = make(map[[16]byte]string) + } + when := time.Now() + cl.metaCache.anyAt = when + var zeroID [16]byte + var stored int + for _, topic := range meta.Topics { + if topic.Topic == nil { + // ID-only responses with no resolved name (e.g. + // UnknownTopicID errors) cannot be cached by name. + continue + } + stored++ + t := cachedMetaTopic{ + id: topic.TopicID, + t: topic, + ps: make(map[int32]kmsg.MetadataResponseTopicPartition), + when: when, + } + cl.metaCache.topics[*topic.Topic] = t + for _, partition := range topic.Partitions { + t.ps[partition.Partition] = partition + } + if topic.TopicID != zeroID { + cl.metaCache.byID[topic.TopicID] = *topic.Topic + } + + if results != nil { + results[*t.t.Topic] = t + } + } + + if all { + cl.metaCache.allAt = when + } + + // Prune entries older than metadataMinAge. If the number of + // topics we just stored equals the map size, every entry is + // fresh and there is nothing to prune. + if stored < len(cl.metaCache.topics) { + for topic, ct := range cl.metaCache.topics { + if ct.when.Equal(when) { + continue + } + if when.Sub(ct.when) > cl.cfg.metadataMinAge { + delete(cl.metaCache.topics, topic) + if ct.id != zeroID { + delete(cl.metaCache.byID, ct.id) + } + } + } + } +} + +func unknownOrCode(exists bool, code int16) error { + if !exists { + return kerr.UnknownTopicOrPartition + } + return kerr.ErrorForCode(code) +} + +func noLeader(l int32) error { + if l < 0 { + return kerr.LeaderNotAvailable + } + return nil +} + +// This is a helper for the sharded requests below; if mapping metadata fails +// to load topics or partitions, we group the failures by error. +// +// We use a lot of reflect magic to make the actual usage much nicer. +type unknownErrShards struct { + // load err => topic => mystery slice type + // + // The mystery type is basically just []Partition, where Partition can + // be any kmsg type. + mapped map[error]map[string]reflect.Value +} + +// err stores a new failing partition with its failing error. +// +// partition's type is equal to the arg1 type of l.fn. +func (l *unknownErrShards) err(err error, topic string, partition any) { + if l.mapped == nil { + l.mapped = make(map[error]map[string]reflect.Value) + } + t := l.mapped[err] + if t == nil { + t = make(map[string]reflect.Value) + l.mapped[err] = t + } + slice, ok := t[topic] + if !ok { + // We make a slice of the input partition type. + slice = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(partition)), 0, 1) + } + + t[topic] = reflect.Append(slice, reflect.ValueOf(partition)) +} + +// errs takes an input slice of partitions and stores each with its failing +// error. +// +// partitions is a slice where each element has type of arg1 of l.fn. +func (l *unknownErrShards) errs(err error, topic string, partitions any) { + v := reflect.ValueOf(partitions) + for i := range v.Len() { + l.err(err, topic, v.Index(i).Interface()) + } +} + +// Returns issueShards for each error stored in l. +// +// This takes a factory function: the first return is a new kmsg.Request, the +// second is a function that adds a topic and its partitions to that request. +// +// Thus, fn is of type func() (kmsg.Request, func(string, []P)) +func (l *unknownErrShards) collect(mkreq, mergeParts any) []issueShard { + if len(l.mapped) == 0 { + return nil + } + + var shards []issueShard + + factory := reflect.ValueOf(mkreq) + perTopic := reflect.ValueOf(mergeParts) + for err, topics := range l.mapped { + req := factory.Call(nil)[0] + + var ntopics, npartitions int + for topic, partitions := range topics { + ntopics++ + npartitions += partitions.Len() + perTopic.Call([]reflect.Value{req, reflect.ValueOf(topic), partitions}) + } + + shards = append(shards, issueShard{ + req: req.Interface().(kmsg.Request), + err: err, + }) + } + + return shards +} + +// handles sharding ListOffsetsRequest +type listOffsetsSharder struct{ *Client } + +func (cl *listOffsetsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.ListOffsetsRequest) + + // For listing offsets, we need the broker leader for each partition we + // are listing. Thus, we first load metadata for the topics. + // + // Metadata loading performs retries; if we fail here, the we do not + // issue sharded requests. + var need []string + for _, topic := range req.Topics { + need = append(need, topic.Topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, true, 0) + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string][]kmsg.ListOffsetsRequestTopicPartition) + var unknowns unknownErrShards + + // For any topic or partition that had an error load, we blindly issue + // a load to the first seed broker. We expect the list to fail, but it + // is the best we could do. + for _, topic := range req.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + unknowns.errs(err, t, topic.Partitions) + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition.Partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + unknowns.err(err, t, partition) + continue + } + if err := noLeader(p.Leader); err != nil { + unknowns.err(err, t, partition) + continue + } + + brokerReq := brokerReqs[p.Leader] + if brokerReq == nil { + brokerReq = make(map[string][]kmsg.ListOffsetsRequestTopicPartition) + brokerReqs[p.Leader] = brokerReq + } + brokerReq[t] = append(brokerReq[t], partition) + } + } + + mkreq := func() *kmsg.ListOffsetsRequest { + r := kmsg.NewPtrListOffsetsRequest() + r.ReplicaID = req.ReplicaID + r.IsolationLevel = req.IsolationLevel + return r + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for topic, parts := range brokerReq { + reqTopic := kmsg.NewListOffsetsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + req.Topics = append(req.Topics, reqTopic) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + return append(issues, unknowns.collect(mkreq, func(r *kmsg.ListOffsetsRequest, topic string, parts []kmsg.ListOffsetsRequestTopicPartition) { + reqTopic := kmsg.NewListOffsetsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + r.Topics = append(r.Topics, reqTopic) + })...), true, nil // this is reshardable +} + +func (cl *listOffsetsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + var ( + resp = kresp.(*kmsg.ListOffsetsResponse) + del []string + retErr error + unknownTopic bool + ) + + for i := range resp.Topics { + t := &resp.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + err := kerr.ErrorForCode(p.ErrorCode) + if err == kerr.UnknownTopicOrPartition || err == kerr.NotLeaderForPartition { + del = append(del, t.Topic) + unknownTopic = unknownTopic || err == kerr.UnknownTopicOrPartition + } + onRespShardErr(&retErr, err) + } + } + if cl.maybeDeleteCachedMeta(unknownTopic, del...) { + return retErr + } + return nil +} + +func (*listOffsetsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrListOffsetsResponse() + topics := make(map[string][]kmsg.ListOffsetsResponseTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.ListOffsetsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, topic := range resp.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + }) + for topic, partitions := range topics { + respTopic := kmsg.NewListOffsetsResponseTopic() + respTopic.Topic = topic + respTopic.Partitions = partitions + merged.Topics = append(merged.Topics, respTopic) + } + return merged, firstErr +} + +// handles sharding OffsetFetchRequest +type offsetFetchSharder struct{ *Client } + +func offsetFetchReqToGroup(req *kmsg.OffsetFetchRequest) kmsg.OffsetFetchRequestGroup { + g := kmsg.NewOffsetFetchRequestGroup() + g.Group = req.Group + for _, topic := range req.Topics { + reqTopic := kmsg.NewOffsetFetchRequestGroupTopic() + reqTopic.Topic = topic.Topic + reqTopic.Partitions = topic.Partitions + g.Topics = append(g.Topics, reqTopic) + } + return g +} + +func offsetFetchGroupToReq(requireStable bool, group kmsg.OffsetFetchRequestGroup) *kmsg.OffsetFetchRequest { + req := kmsg.NewPtrOffsetFetchRequest() + req.RequireStable = requireStable + req.Group = group.Group + for _, topic := range group.Topics { + reqTopic := kmsg.NewOffsetFetchRequestTopic() + reqTopic.Topic = topic.Topic + reqTopic.Partitions = topic.Partitions + req.Topics = append(req.Topics, reqTopic) + } + return req +} + +func offsetFetchRespToGroup(req *kmsg.OffsetFetchRequest, resp *kmsg.OffsetFetchResponse) kmsg.OffsetFetchResponseGroup { + g := kmsg.NewOffsetFetchResponseGroup() + g.Group = req.Group + g.ErrorCode = resp.ErrorCode + for _, topic := range resp.Topics { + t := kmsg.NewOffsetFetchResponseGroupTopic() + t.Topic = topic.Topic + for _, partition := range topic.Partitions { + p := kmsg.NewOffsetFetchResponseGroupTopicPartition() + p.Partition = partition.Partition + p.Offset = partition.Offset + p.LeaderEpoch = partition.LeaderEpoch + p.Metadata = partition.Metadata + p.ErrorCode = partition.ErrorCode + t.Partitions = append(t.Partitions, p) + } + g.Topics = append(g.Topics, t) + } + return g +} + +func offsetFetchRespGroupIntoResp(g kmsg.OffsetFetchResponseGroup, into *kmsg.OffsetFetchResponse) { + into.ErrorCode = g.ErrorCode + into.Topics = into.Topics[:0] + for _, topic := range g.Topics { + t := kmsg.NewOffsetFetchResponseTopic() + t.Topic = topic.Topic + for _, partition := range topic.Partitions { + p := kmsg.NewOffsetFetchResponseTopicPartition() + p.Partition = partition.Partition + p.Offset = partition.Offset + p.LeaderEpoch = partition.LeaderEpoch + p.Metadata = partition.Metadata + p.ErrorCode = partition.ErrorCode + t.Partitions = append(t.Partitions, p) + } + into.Topics = append(into.Topics, t) + } +} + +func (cl *offsetFetchSharder) shard(ctx context.Context, kreq kmsg.Request, lastErr error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.OffsetFetchRequest) + + // We always try batching and only split at the end if lastErr + // indicates too old. We convert to batching immediately. + dup := *req + req = &dup + + if len(req.Groups) == 0 { + req.Groups = append(req.Groups, offsetFetchReqToGroup(req)) + } + + // Fill in both Topic and TopicID on each request topic so that + // the request works regardless of the broker version (v10+ uses + // TopicID; v0-v9 uses Topic). This also means onResp can use + // the client's cached metadata to fill in response fields + // without any additional metadata fetches. + var ( + unresolvedNames []string + unresolvedIDs [][16]byte + ) + var resolving bool + for _, g := range req.Groups { + for _, t := range g.Topics { + if t.Topic == "" && t.TopicID != ([16]byte{}) { + unresolvedIDs = append(unresolvedIDs, t.TopicID) + resolving = true + } + if t.Topic != "" && t.TopicID == ([16]byte{}) { + unresolvedNames = append(unresolvedNames, t.Topic) + resolving = true + } + } + } + if len(unresolvedIDs) > 0 { + cl.resolveTopicMetaByID(ctx, unresolvedIDs) + } + var nameMeta map[string]cachedMetaTopic + if len(unresolvedNames) > 0 { + nameMeta, _ = cl.resolveTopicMeta(ctx, unresolvedNames, true, 0) + } + if resolving { + id2t := cl.id2tMap() + for i := range req.Groups { + g := &req.Groups[i] + for j := range g.Topics { + t := &g.Topics[j] + if t.Topic == "" && t.TopicID != ([16]byte{}) { + t.Topic = id2t[t.TopicID] + } + if t.TopicID == ([16]byte{}) && t.Topic != "" { + if ct, ok := nameMeta[t.Topic]; ok { + t.TopicID = ct.id + } + } + } + } + } + + groups := make([]string, 0, len(req.Groups)) + for i := range req.Groups { + groups = append(groups, req.Groups[i].Group) + } + + coordinators := cl.loadCoordinators(ctx, coordinatorTypeGroup, groups...) + + // Loading coordinators can have each group fail with its unique error, + // or with a kerr.Error that can be merged. Unique errors get their own + // failure shard, while kerr.Error's get merged. + type unkerr struct { + err error + group kmsg.OffsetFetchRequestGroup + } + var ( + brokerReqs = make(map[int32]*kmsg.OffsetFetchRequest) + kerrs = make(map[*kerr.Error][]kmsg.OffsetFetchRequestGroup) + unkerrs []unkerr + ) + + newReq := func(groups ...kmsg.OffsetFetchRequestGroup) *kmsg.OffsetFetchRequest { + newReq := kmsg.NewPtrOffsetFetchRequest() + newReq.RequireStable = req.RequireStable + newReq.Groups = groups + return newReq + } + + for _, group := range req.Groups { + berr := coordinators[group.Group] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.Groups = append(brokerReq.Groups, group) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], group) + default: + unkerrs = append(unkerrs, unkerr{berr.err, group}) + } + } + + splitReq := errors.Is(lastErr, errBrokerTooOld) + + var issues []issueShard + for id, req := range brokerReqs { + if splitReq { + for _, group := range req.Groups { + req := offsetFetchGroupToReq(req.RequireStable, group) + issues = append(issues, issueShard{ + req: req, + pin: &pinReq{pinMax: true, max: 7}, + broker: id, + }) + } + } else if len(req.Groups) <= 1 { + single := offsetFetchGroupToReq(req.RequireStable, req.Groups[0]) + single.Groups = req.Groups + issues = append(issues, issueShard{ + req: single, + broker: id, + }) + } else { + issues = append(issues, issueShard{ + req: req, + pin: &pinReq{pinMin: true, min: 8}, + broker: id, + }) + } + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.group), + err: unkerr.err, + }) + } + for kerr, groups := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groups...), + err: kerr, + }) + } + + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *offsetFetchSharder) onResp(kreq kmsg.Request, kresp kmsg.Response) error { + req := kreq.(*kmsg.OffsetFetchRequest) + resp := kresp.(*kmsg.OffsetFetchResponse) + + // All-topics fetches could leave topics nil, in which case we DONT + // bi-directionally resolve the name in shard. Thus, we have to handle + // here. + // + // We always run the resolution from cache (it lets clients use the + // "by ID" APIs against v9 brokers via name -> ID fallback). What we + // gate on resp.Version >= 10 is the safety-net error injection: + // below v10, TopicID is structurally absent from the wire, so a + // zero TopicID after cache lookup is expected, not an error. At v10+ + // the broker should have sent the missing side; failing to resolve + // is a real surprise. See #1312 for the EH case where a v8/v9 OffsetFetch + // against a sub-v10 Metadata broker would synthesize a bogus + // UNKNOWN_TOPIC_OR_PARTITION on every partition. + var unresolvedIDs [][16]byte + var unresolvedNames []string + for i := range resp.Groups { + for j := range resp.Groups[i].Topics { + t := &resp.Groups[i].Topics[j] + if t.Topic == "" && t.TopicID != ([16]byte{}) { + unresolvedIDs = append(unresolvedIDs, t.TopicID) + } + if t.TopicID == ([16]byte{}) && t.Topic != "" { + unresolvedNames = append(unresolvedNames, t.Topic) + } + } + } + + // If anything is unresolved, resolve both, then do the walk again. + if len(unresolvedIDs) > 0 { + cl.resolveTopicMetaByID(cl.ctx, unresolvedIDs) + } + if len(unresolvedNames) > 0 { + cl.resolveTopicMeta(cl.ctx, unresolvedNames, false, 0) + } + if len(unresolvedIDs) > 0 || len(unresolvedNames) > 0 { + // Use metaCache which was just populated by the resolve + // calls above. If we cannot map topic ID or topic name, + // we inject an error - but only at v10+, where TopicID is + // expected on the wire. Below v10 the absence is structural, + // not a broker error. + cl.metaCache.mu.Lock() + for i := range resp.Groups { + for j := range resp.Groups[i].Topics { + t := &resp.Groups[i].Topics[j] + if t.Topic == "" && t.TopicID != ([16]byte{}) { + t.Topic = cl.metaCache.byID[t.TopicID] + if t.Topic == "" && resp.Version >= 10 { + for k := range t.Partitions { + if t.Partitions[k].ErrorCode == 0 { + t.Partitions[k].ErrorCode = kerr.UnknownTopicID.Code + } + } + } + } + if t.TopicID == ([16]byte{}) && t.Topic != "" { + if ct, ok := cl.metaCache.topics[t.Topic]; ok { + t.TopicID = ct.id + } + if t.TopicID == ([16]byte{}) && resp.Version >= 10 { + for k := range t.Partitions { + if t.Partitions[k].ErrorCode == 0 { + t.Partitions[k].ErrorCode = kerr.UnknownTopicOrPartition.Code + } + } + } + } + } + } + cl.metaCache.mu.Unlock() + } + + switch len(resp.Groups) { + case 0: + // Requested no groups: move top level into batch for v0-v7 to + // v8 forward compat. + resp.Groups = append(resp.Groups, offsetFetchRespToGroup(req, resp)) + case 1: + // Requested 1 group v8+: set top level for v0-v7 back-compat. + offsetFetchRespGroupIntoResp(resp.Groups[0], resp) + default: + } + + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.Group, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + + // For a final bit of extra fun, v0 and v1 do not have a top level + // error code but instead a per-partition error code. If the + // coordinator is loading &c, then all per-partition error codes are + // the same so we only need to look at the first partition. + if resp.Version < 2 && len(resp.Topics) > 0 && len(resp.Topics[0].Partitions) > 0 { + code := resp.Topics[0].Partitions[0].ErrorCode + err := kerr.ErrorForCode(code) + cl.maybeDeleteStaleCoordinator(req.Group, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + + return retErr +} + +func (*offsetFetchSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrOffsetFetchResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.OffsetFetchResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + + // Old requests only support one group; *either* the commit + // used multiple groups and they are expecting the batch + // response, *or* the commit used one group and we always merge + // that one group into the old format. + if len(resp.Groups) == 1 { + offsetFetchRespGroupIntoResp(resp.Groups[0], merged) + } + }) +} + +// handles sharding FindCoordinatorRequest +type findCoordinatorSharder struct{ *Client } + +func findCoordinatorRespCoordinatorIntoResp(c kmsg.FindCoordinatorResponseCoordinator, into *kmsg.FindCoordinatorResponse) { + into.NodeID = c.NodeID + into.Host = c.Host + into.Port = c.Port + into.ErrorCode = c.ErrorCode + into.ErrorMessage = c.ErrorMessage +} + +func (*findCoordinatorSharder) shard(_ context.Context, kreq kmsg.Request, lastErr error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.FindCoordinatorRequest) + + // We always try batching and only split at the end if lastErr + // indicates too old. We convert to batching immediately. + dup := *req + req = &dup + + uniq := make(map[string]struct{}, len(req.CoordinatorKeys)) + if len(req.CoordinatorKeys) == 0 { + uniq[req.CoordinatorKey] = struct{}{} + } else { + for _, key := range req.CoordinatorKeys { + uniq[key] = struct{}{} + } + } + req.CoordinatorKeys = req.CoordinatorKeys[:0] + for key := range uniq { + req.CoordinatorKeys = append(req.CoordinatorKeys, key) + } + if len(req.CoordinatorKeys) == 1 { + req.CoordinatorKey = req.CoordinatorKeys[0] + } + + splitReq := errors.Is(lastErr, errBrokerTooOld) + if !splitReq { + // With only one key, we do not need to split nor pin this. + if len(req.CoordinatorKeys) <= 1 { + return []issueShard{{req: req, any: true}}, false, nil + } + return []issueShard{{ + req: req, + pin: &pinReq{pinMin: true, min: 4}, + any: true, + }}, true, nil // this is "reshardable", in that we will split the request next + } + + var issues []issueShard + for _, key := range req.CoordinatorKeys { + sreq := kmsg.NewPtrFindCoordinatorRequest() + sreq.CoordinatorType = req.CoordinatorType + sreq.CoordinatorKey = key + issues = append(issues, issueShard{ + req: sreq, + pin: &pinReq{pinMax: true, max: 3}, + any: true, + }) + } + return issues, false, nil // not reshardable +} + +func (*findCoordinatorSharder) onResp(kreq kmsg.Request, kresp kmsg.Response) error { + req := kreq.(*kmsg.FindCoordinatorRequest) + resp := kresp.(*kmsg.FindCoordinatorResponse) + + switch len(resp.Coordinators) { + case 0: + // Convert v3 and prior to v4+ + rc := kmsg.NewFindCoordinatorResponseCoordinator() + rc.Key = req.CoordinatorKey + rc.NodeID = resp.NodeID + rc.Host = resp.Host + rc.Port = resp.Port + rc.ErrorCode = resp.ErrorCode + rc.ErrorMessage = resp.ErrorMessage + resp.Coordinators = append(resp.Coordinators, rc) + case 1: + // Convert v4 to v3 and prior + findCoordinatorRespCoordinatorIntoResp(resp.Coordinators[0], resp) + } + + return nil +} + +func (*findCoordinatorSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrFindCoordinatorResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.FindCoordinatorResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Coordinators = append(merged.Coordinators, resp.Coordinators...) + + if len(resp.Coordinators) == 1 { + findCoordinatorRespCoordinatorIntoResp(resp.Coordinators[0], merged) + } + }) +} + +// handles sharding DescribeGroupsRequest +type describeGroupsSharder struct{ *Client } + +func (cl *describeGroupsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeGroupsRequest) + + coordinators := cl.loadCoordinators(ctx, coordinatorTypeGroup, req.Groups...) + type unkerr struct { + err error + group string + } + var ( + brokerReqs = make(map[int32]*kmsg.DescribeGroupsRequest) + kerrs = make(map[*kerr.Error][]string) + unkerrs []unkerr + ) + + newReq := func(groups ...string) *kmsg.DescribeGroupsRequest { + newReq := kmsg.NewPtrDescribeGroupsRequest() + newReq.IncludeAuthorizedOperations = req.IncludeAuthorizedOperations + newReq.Groups = groups + return newReq + } + + for _, group := range req.Groups { + berr := coordinators[group] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.Groups = append(brokerReq.Groups, group) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], group) + default: + unkerrs = append(unkerrs, unkerr{berr.err, group}) + } + } + + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.group), + err: unkerr.err, + }) + } + for kerr, groups := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groups...), + err: kerr, + }) + } + + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *describeGroupsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { // cleanup any stale groups + resp := kresp.(*kmsg.DescribeGroupsResponse) + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.Group, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*describeGroupsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeGroupsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeGroupsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + }) +} + +// handles sharding ListGroupsRequest +type listGroupsSharder struct{ *Client } + +func (cl *listGroupsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.ListGroupsRequest) + return cl.allBrokersShardedReq(ctx, func() kmsg.Request { + dup := *req + return &dup + }) +} + +func (*listGroupsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.ListGroupsResponse) + return kerr.ErrorForCode(resp.ErrorCode) +} + +func (*listGroupsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrListGroupsResponse() + // During a coordinator migration, a group can transiently appear in + // both the old and new coordinator's ListGroups response. We dedupe + // by group name so callers do not see duplicates; the first shard + // response wins. + seen := make(map[string]struct{}) + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.ListGroupsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + if merged.ErrorCode == 0 { + merged.ErrorCode = resp.ErrorCode + } + for _, g := range resp.Groups { + if _, ok := seen[g.Group]; ok { + continue + } + seen[g.Group] = struct{}{} + merged.Groups = append(merged.Groups, g) + } + }) +} + +// handle sharding DeleteRecordsRequest +type deleteRecordsSharder struct{ *Client } + +func (cl *deleteRecordsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DeleteRecordsRequest) + + var need []string + for _, topic := range req.Topics { + need = append(need, topic.Topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, true, 0) + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string][]kmsg.DeleteRecordsRequestTopicPartition) + var unknowns unknownErrShards + + for _, topic := range req.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + unknowns.errs(err, t, topic.Partitions) + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition.Partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + unknowns.err(err, t, partition) + continue + } + if err := noLeader(p.Leader); err != nil { + unknowns.err(err, t, partition) + continue + } + + brokerReq := brokerReqs[p.Leader] + if brokerReq == nil { + brokerReq = make(map[string][]kmsg.DeleteRecordsRequestTopicPartition) + brokerReqs[p.Leader] = brokerReq + } + brokerReq[t] = append(brokerReq[t], partition) + } + } + + mkreq := func() *kmsg.DeleteRecordsRequest { + r := kmsg.NewPtrDeleteRecordsRequest() + r.TimeoutMillis = req.TimeoutMillis + return r + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for topic, parts := range brokerReq { + reqTopic := kmsg.NewDeleteRecordsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + req.Topics = append(req.Topics, reqTopic) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + return append(issues, unknowns.collect(mkreq, func(r *kmsg.DeleteRecordsRequest, topic string, parts []kmsg.DeleteRecordsRequestTopicPartition) { + reqTopic := kmsg.NewDeleteRecordsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + r.Topics = append(r.Topics, reqTopic) + })...), true, nil // this is reshardable +} + +func (cl *deleteRecordsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + var ( + resp = kresp.(*kmsg.DeleteRecordsResponse) + del []string + retErr error + unknownTopic bool + ) + for i := range resp.Topics { + t := &resp.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + err := kerr.ErrorForCode(p.ErrorCode) + if err == kerr.UnknownTopicOrPartition || err == kerr.NotLeaderForPartition { + del = append(del, t.Topic) + unknownTopic = unknownTopic || err == kerr.UnknownTopicOrPartition + } + onRespShardErr(&retErr, err) + } + } + if cl.maybeDeleteCachedMeta(unknownTopic, del...) { + return retErr + } + return nil +} + +func (*deleteRecordsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDeleteRecordsResponse() + topics := make(map[string][]kmsg.DeleteRecordsResponseTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DeleteRecordsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, topic := range resp.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + }) + for topic, partitions := range topics { + respTopic := kmsg.NewDeleteRecordsResponseTopic() + respTopic.Topic = topic + respTopic.Partitions = partitions + merged.Topics = append(merged.Topics, respTopic) + } + return merged, firstErr +} + +// handle sharding OffsetForLeaderEpochRequest +type offsetForLeaderEpochSharder struct{ *Client } + +func (cl *offsetForLeaderEpochSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.OffsetForLeaderEpochRequest) + + var need []string + for _, topic := range req.Topics { + need = append(need, topic.Topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, true, 0) + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string][]kmsg.OffsetForLeaderEpochRequestTopicPartition) + var unknowns unknownErrShards + + for _, topic := range req.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + unknowns.errs(err, t, topic.Partitions) + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition.Partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + unknowns.err(err, t, partition) + continue + } + if err := noLeader(p.Leader); err != nil { + unknowns.err(err, t, partition) + continue + } + + brokerReq := brokerReqs[p.Leader] + if brokerReq == nil { + brokerReq = make(map[string][]kmsg.OffsetForLeaderEpochRequestTopicPartition) + brokerReqs[p.Leader] = brokerReq + } + brokerReq[topic.Topic] = append(brokerReq[topic.Topic], partition) + } + } + + mkreq := func() *kmsg.OffsetForLeaderEpochRequest { + r := kmsg.NewPtrOffsetForLeaderEpochRequest() + r.ReplicaID = req.ReplicaID + return r + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for topic, parts := range brokerReq { + reqTopic := kmsg.NewOffsetForLeaderEpochRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + req.Topics = append(req.Topics, reqTopic) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + return append(issues, unknowns.collect(mkreq, func(r *kmsg.OffsetForLeaderEpochRequest, topic string, parts []kmsg.OffsetForLeaderEpochRequestTopicPartition) { + reqTopic := kmsg.NewOffsetForLeaderEpochRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + r.Topics = append(r.Topics, reqTopic) + })...), true, nil // this is reshardable +} + +func (cl *offsetForLeaderEpochSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + var ( + resp = kresp.(*kmsg.OffsetForLeaderEpochResponse) + del []string + retErr error + unknownTopic bool + ) + for i := range resp.Topics { + t := &resp.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + err := kerr.ErrorForCode(p.ErrorCode) + if err == kerr.UnknownTopicOrPartition || err == kerr.NotLeaderForPartition { + del = append(del, t.Topic) + unknownTopic = unknownTopic || err == kerr.UnknownTopicOrPartition + } + onRespShardErr(&retErr, err) + } + } + if cl.maybeDeleteCachedMeta(unknownTopic, del...) { + return retErr + } + return nil +} + +func (*offsetForLeaderEpochSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrOffsetForLeaderEpochResponse() + topics := make(map[string][]kmsg.OffsetForLeaderEpochResponseTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.OffsetForLeaderEpochResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, topic := range resp.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + }) + for topic, partitions := range topics { + respTopic := kmsg.NewOffsetForLeaderEpochResponseTopic() + respTopic.Topic = topic + respTopic.Partitions = partitions + merged.Topics = append(merged.Topics, respTopic) + } + return merged, firstErr +} + +// handle sharding AddPartitionsToTXn, where v4+ switched to batch requests +type addPartitionsToTxnSharder struct{ *Client } + +func addPartitionsReqToTxn(req *kmsg.AddPartitionsToTxnRequest) { + t := kmsg.NewAddPartitionsToTxnRequestTransaction() + t.TransactionalID = req.TransactionalID + t.ProducerID = req.ProducerID + t.ProducerEpoch = req.ProducerEpoch + for i := range req.Topics { + rt := &req.Topics[i] + tt := kmsg.NewAddPartitionsToTxnRequestTransactionTopic() + tt.Topic = rt.Topic + tt.Partitions = rt.Partitions + t.Topics = append(t.Topics, tt) + } + req.Transactions = append(req.Transactions, t) +} + +func addPartitionsTxnToReq(req *kmsg.AddPartitionsToTxnRequest) { + if len(req.Transactions) != 1 { + return + } + t0 := &req.Transactions[0] + req.TransactionalID = t0.TransactionalID + req.ProducerID = t0.ProducerID + req.ProducerEpoch = t0.ProducerEpoch + for _, tt := range t0.Topics { + rt := kmsg.NewAddPartitionsToTxnRequestTopic() + rt.Topic = tt.Topic + rt.Partitions = tt.Partitions + req.Topics = append(req.Topics, rt) + } +} + +func addPartitionsTxnToResp(resp *kmsg.AddPartitionsToTxnResponse) { + if len(resp.Transactions) == 0 { + return + } + t0 := &resp.Transactions[0] + for _, tt := range t0.Topics { + rt := kmsg.NewAddPartitionsToTxnResponseTopic() + rt.Topic = tt.Topic + for _, tp := range tt.Partitions { + rp := kmsg.NewAddPartitionsToTxnResponseTopicPartition() + rp.Partition = tp.Partition + rp.ErrorCode = tp.ErrorCode + rt.Partitions = append(rt.Partitions, rp) + } + resp.Topics = append(resp.Topics, rt) + } +} + +func (cl *addPartitionsToTxnSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.AddPartitionsToTxnRequest) + + if len(req.Transactions) == 0 { + addPartitionsReqToTxn(req) + } + txnIDs := make([]string, 0, len(req.Transactions)) + for i := range req.Transactions { + txnIDs = append(txnIDs, req.Transactions[i].TransactionalID) + } + coordinators := cl.loadCoordinators(ctx, coordinatorTypeTxn, txnIDs...) + + type unkerr struct { + err error + txn kmsg.AddPartitionsToTxnRequestTransaction + } + var ( + brokerReqs = make(map[int32]*kmsg.AddPartitionsToTxnRequest) + kerrs = make(map[*kerr.Error][]kmsg.AddPartitionsToTxnRequestTransaction) + unkerrs []unkerr + ) + + newReq := func(txns ...kmsg.AddPartitionsToTxnRequestTransaction) *kmsg.AddPartitionsToTxnRequest { + req := kmsg.NewPtrAddPartitionsToTxnRequest() + req.Transactions = txns + addPartitionsTxnToReq(req) + return req + } + + for _, txn := range req.Transactions { + berr := coordinators[txn.TransactionalID] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq(txn) + brokerReqs[berr.b.meta.NodeID] = brokerReq + } else { + brokerReq.Transactions = append(brokerReq.Transactions, txn) + } + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], txn) + default: + unkerrs = append(unkerrs, unkerr{berr.err, txn}) + } + } + + var issues []issueShard + for id, req := range brokerReqs { + if len(req.Transactions) <= 1 || len(req.Transactions) == 1 && !req.Transactions[0].VerifyOnly { + issues = append(issues, issueShard{ + req: req, + pin: &pinReq{pinMax: true, max: 3}, + broker: id, + }) + } else { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.txn), + err: unkerr.err, + }) + } + for kerr, txns := range kerrs { + issues = append(issues, issueShard{ + req: newReq(txns...), + err: kerr, + }) + } + + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *addPartitionsToTxnSharder) onResp(kreq kmsg.Request, kresp kmsg.Response) error { + req := kreq.(*kmsg.AddPartitionsToTxnRequest) + resp := kresp.(*kmsg.AddPartitionsToTxnResponse) + + // We default to the top level error, which is used in v4+. For v3 + // (case 0), we use the per-partition error, which is the same for + // every partition on not_coordinator errors. + code := resp.ErrorCode + if code == 0 && len(resp.Transactions) == 0 { + // Convert v3 and prior to v4+ + resptxn := kmsg.NewAddPartitionsToTxnResponseTransaction() + resptxn.TransactionalID = req.TransactionalID + for _, rt := range resp.Topics { + respt := kmsg.NewAddPartitionsToTxnResponseTransactionTopic() + respt.Topic = rt.Topic + for _, rp := range rt.Partitions { + respp := kmsg.NewAddPartitionsToTxnResponseTransactionTopicPartition() + respp.Partition = rp.Partition + respp.ErrorCode = rp.ErrorCode + code = rp.ErrorCode // v3 and prior has per-partition errors, not top level + respt.Partitions = append(respt.Partitions, respp) + } + resptxn.Topics = append(resptxn.Topics, respt) + } + resp.Transactions = append(resp.Transactions, resptxn) + } else { + // Convert v4 to v3 and prior: either we have a top level error + // code or we have at least one transaction. + // + // If the code is non-zero, we convert it to per-partition error + // codes; v3 does not have a top level err. + addPartitionsTxnToResp(resp) + if code != 0 { + for _, reqt := range req.Topics { + respt := kmsg.NewAddPartitionsToTxnResponseTopic() + respt.Topic = reqt.Topic + for _, reqp := range reqt.Partitions { + respp := kmsg.NewAddPartitionsToTxnResponseTopicPartition() + respp.Partition = reqp + respp.ErrorCode = resp.ErrorCode + respt.Partitions = append(respt.Partitions, respp) + } + resp.Topics = append(resp.Topics, respt) + } + } + } + if err := kerr.ErrorForCode(code); cl.maybeDeleteStaleCoordinator(req.TransactionalID, coordinatorTypeTxn, err) { + return err + } + return nil +} + +func (*addPartitionsToTxnSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrAddPartitionsToTxnResponse() + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.AddPartitionsToTxnResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.ErrorCode = resp.ErrorCode + merged.Transactions = append(merged.Transactions, resp.Transactions...) + }) + addPartitionsTxnToResp(merged) + return merged, firstErr +} + +// handle sharding WriteTxnMarkersRequest +type writeTxnMarkersSharder struct{ *Client } + +func (cl *writeTxnMarkersSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.WriteTxnMarkersRequest) + + var need []string + for _, marker := range req.Markers { + for _, topic := range marker.Topics { + need = append(need, topic.Topic) + } + } + mapping, err := cl.resolveTopicMeta(ctx, need, true, 0) + if err != nil { + return nil, false, err + } + + type pidEpochCommit struct { + pid int64 + epoch int16 + commit bool + txnVersion int8 + } + + brokerReqs := make(map[int32]map[pidEpochCommit]map[string][]int32) + unknown := make(map[error]map[pidEpochCommit]map[string][]int32) // err => pec => topic => partitions + + addreq := func(b int32, pec pidEpochCommit, t string, p int32) { + pecs := brokerReqs[b] + if pecs == nil { + pecs = make(map[pidEpochCommit]map[string][]int32) + brokerReqs[b] = pecs + } + ts := pecs[pec] + if ts == nil { + ts = make(map[string][]int32) + pecs[pec] = ts + } + ts[t] = append(ts[t], p) + } + addunk := func(err error, pec pidEpochCommit, t string, p int32) { + pecs := unknown[err] + if pecs == nil { + pecs = make(map[pidEpochCommit]map[string][]int32) + unknown[err] = pecs + } + ts := pecs[pec] + if ts == nil { + ts = make(map[string][]int32) + pecs[pec] = ts + } + ts[t] = append(ts[t], p) + } + + for _, marker := range req.Markers { + pec := pidEpochCommit{ + marker.ProducerID, + marker.ProducerEpoch, + marker.Committed, + marker.TransactionVersion, + } + for _, topic := range marker.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + for _, partition := range topic.Partitions { + addunk(err, pec, t, partition) + } + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + addunk(err, pec, t, partition) + continue + } + if err := noLeader(p.Leader); err != nil { + addunk(err, pec, t, partition) + continue + } + addreq(p.Leader, pec, t, partition) + } + } + } + + mkreq := kmsg.NewPtrWriteTxnMarkersRequest + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for pec, topics := range brokerReq { + rm := kmsg.NewWriteTxnMarkersRequestMarker() + rm.ProducerID = pec.pid + rm.ProducerEpoch = pec.epoch + rm.Committed = pec.commit + rm.TransactionVersion = pec.txnVersion + for topic, parts := range topics { + rt := kmsg.NewWriteTxnMarkersRequestMarkerTopic() + rt.Topic = topic + rt.Partitions = parts + rm.Topics = append(rm.Topics, rt) + } + req.Markers = append(req.Markers, rm) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + for err, errReq := range unknown { + req := mkreq() + for pec, topics := range errReq { + rm := kmsg.NewWriteTxnMarkersRequestMarker() + rm.ProducerID = pec.pid + rm.ProducerEpoch = pec.epoch + rm.Committed = pec.commit + rm.TransactionVersion = pec.txnVersion + for topic, parts := range topics { + rt := kmsg.NewWriteTxnMarkersRequestMarkerTopic() + rt.Topic = topic + rt.Partitions = parts + rm.Topics = append(rm.Topics, rt) + } + req.Markers = append(req.Markers, rm) + } + issues = append(issues, issueShard{ + req: req, + err: err, + }) + } + return issues, true, nil // this is reshardable +} + +func (cl *writeTxnMarkersSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + var ( + resp = kresp.(*kmsg.WriteTxnMarkersResponse) + del []string + retErr error + unknownTopic bool + ) + for i := range resp.Markers { + m := &resp.Markers[i] + for j := range m.Topics { + t := &m.Topics[j] + for k := range t.Partitions { + p := &t.Partitions[k] + err := kerr.ErrorForCode(p.ErrorCode) + if err == kerr.UnknownTopicOrPartition || err == kerr.NotLeaderForPartition { + del = append(del, t.Topic) + unknownTopic = unknownTopic || err == kerr.UnknownTopicOrPartition + } + onRespShardErr(&retErr, err) + } + } + } + if cl.maybeDeleteCachedMeta(unknownTopic, del...) { + return retErr + } + return nil +} + +func (*writeTxnMarkersSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrWriteTxnMarkersResponse() + markers := make(map[int64]map[string][]kmsg.WriteTxnMarkersResponseMarkerTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.WriteTxnMarkersResponse) + merged.Version = resp.Version + for _, marker := range resp.Markers { + topics := markers[marker.ProducerID] + if topics == nil { + topics = make(map[string][]kmsg.WriteTxnMarkersResponseMarkerTopicPartition) + markers[marker.ProducerID] = topics + } + for _, topic := range marker.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + } + }) + for pid, topics := range markers { + respMarker := kmsg.NewWriteTxnMarkersResponseMarker() + respMarker.ProducerID = pid + for topic, partitions := range topics { + respTopic := kmsg.NewWriteTxnMarkersResponseMarkerTopic() + respTopic.Topic = topic + respTopic.Partitions = append(respTopic.Partitions, partitions...) + respMarker.Topics = append(respMarker.Topics, respTopic) + } + merged.Markers = append(merged.Markers, respMarker) + } + return merged, firstErr +} + +// handle sharding DescribeConfigsRequest +type describeConfigsSharder struct{ *Client } + +func (*describeConfigsSharder) shard(_ context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeConfigsRequest) + + brokerReqs := make(map[int32][]kmsg.DescribeConfigsRequestResource) + var any []kmsg.DescribeConfigsRequestResource + + for i := range req.Resources { + resource := req.Resources[i] + switch resource.ResourceType { + case kmsg.ConfigResourceTypeBroker: + case kmsg.ConfigResourceTypeBrokerLogger: + default: + any = append(any, resource) + continue + } + id, err := strconv.ParseInt(resource.ResourceName, 10, 32) + if err != nil || id < 0 { + any = append(any, resource) + continue + } + brokerReqs[int32(id)] = append(brokerReqs[int32(id)], resource) + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + newReq := kmsg.NewPtrDescribeConfigsRequest() + newReq.Resources = brokerReq + newReq.IncludeSynonyms = req.IncludeSynonyms + newReq.IncludeDocumentation = req.IncludeDocumentation + + issues = append(issues, issueShard{ + req: newReq, + broker: brokerID, + }) + } + + if len(any) > 0 { + newReq := kmsg.NewPtrDescribeConfigsRequest() + newReq.Resources = any + newReq.IncludeSynonyms = req.IncludeSynonyms + newReq.IncludeDocumentation = req.IncludeDocumentation + issues = append(issues, issueShard{ + req: newReq, + any: true, + }) + } + + return issues, false, nil // this is not reshardable, but the any block can go anywhere +} + +func (*describeConfigsSharder) onResp(kmsg.Request, kmsg.Response) error { return nil } // configs: topics not mapped, nothing retryable + +func (*describeConfigsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeConfigsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeConfigsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Resources = append(merged.Resources, resp.Resources...) + }) +} + +// handle sharding AlterConfigsRequest +type alterConfigsSharder struct{ *Client } + +func (*alterConfigsSharder) shard(_ context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.AlterConfigsRequest) + + brokerReqs := make(map[int32][]kmsg.AlterConfigsRequestResource) + var any []kmsg.AlterConfigsRequestResource + + for i := range req.Resources { + resource := req.Resources[i] + switch resource.ResourceType { + case kmsg.ConfigResourceTypeBroker: + case kmsg.ConfigResourceTypeBrokerLogger: + default: + any = append(any, resource) + continue + } + id, err := strconv.ParseInt(resource.ResourceName, 10, 32) + if err != nil || id < 0 { + any = append(any, resource) + continue + } + brokerReqs[int32(id)] = append(brokerReqs[int32(id)], resource) + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + newReq := kmsg.NewPtrAlterConfigsRequest() + newReq.Resources = brokerReq + newReq.ValidateOnly = req.ValidateOnly + + issues = append(issues, issueShard{ + req: newReq, + broker: brokerID, + }) + } + + if len(any) > 0 { + newReq := kmsg.NewPtrAlterConfigsRequest() + newReq.Resources = any + newReq.ValidateOnly = req.ValidateOnly + issues = append(issues, issueShard{ + req: newReq, + any: true, + }) + } + + return issues, false, nil // this is not reshardable, but the any block can go anywhere +} + +func (*alterConfigsSharder) onResp(kmsg.Request, kmsg.Response) error { return nil } // configs: topics not mapped, nothing retryable + +func (*alterConfigsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrAlterConfigsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.AlterConfigsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Resources = append(merged.Resources, resp.Resources...) + }) +} + +// handles sharding AlterReplicaLogDirsRequest +type alterReplicaLogDirsSharder struct{ *Client } + +func (cl *alterReplicaLogDirsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.AlterReplicaLogDirsRequest) + + needMap := make(map[string]struct{}) + for _, dir := range req.Dirs { + for _, topic := range dir.Topics { + needMap[topic.Topic] = struct{}{} + } + } + var need []string + for topic := range needMap { + need = append(need, topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, false, 0) // bypass cache, tricky to manage response + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string]map[string][]int32) // broker => dir => topic => partitions + unknowns := make(map[error]map[string]map[string][]int32) // err => dir => topic => partitions + + addBroker := func(broker int32, dir, topic string, partition int32) { + brokerDirs := brokerReqs[broker] + if brokerDirs == nil { + brokerDirs = make(map[string]map[string][]int32) + brokerReqs[broker] = brokerDirs + } + dirTopics := brokerDirs[dir] + if dirTopics == nil { + dirTopics = make(map[string][]int32) + brokerDirs[dir] = dirTopics + } + dirTopics[topic] = append(dirTopics[topic], partition) + } + + addUnknown := func(err error, dir, topic string, partition int32) { + dirs := unknowns[err] + if dirs == nil { + dirs = make(map[string]map[string][]int32) + unknowns[err] = dirs + } + dirTopics := dirs[dir] + if dirTopics == nil { + dirTopics = make(map[string][]int32) + dirs[dir] = dirTopics + } + dirTopics[topic] = append(dirTopics[topic], partition) + } + + for _, dir := range req.Dirs { + for _, topic := range dir.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + for _, partition := range topic.Partitions { + addUnknown(err, dir.Dir, t, partition) + } + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + addUnknown(err, dir.Dir, t, partition) + continue + } + + for _, replica := range p.Replicas { + addBroker(replica, dir.Dir, t, partition) + } + } + } + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := kmsg.NewPtrAlterReplicaLogDirsRequest() + for dir, topics := range brokerReq { + rd := kmsg.NewAlterReplicaLogDirsRequestDir() + rd.Dir = dir + for topic, partitions := range topics { + rdTopic := kmsg.NewAlterReplicaLogDirsRequestDirTopic() + rdTopic.Topic = topic + rdTopic.Partitions = partitions + rd.Topics = append(rd.Topics, rdTopic) + } + req.Dirs = append(req.Dirs, rd) + } + + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + for err, dirs := range unknowns { + req := kmsg.NewPtrAlterReplicaLogDirsRequest() + for dir, topics := range dirs { + rd := kmsg.NewAlterReplicaLogDirsRequestDir() + rd.Dir = dir + for topic, partitions := range topics { + rdTopic := kmsg.NewAlterReplicaLogDirsRequestDirTopic() + rdTopic.Topic = topic + rdTopic.Partitions = partitions + rd.Topics = append(rd.Topics, rdTopic) + } + req.Dirs = append(req.Dirs, rd) + } + + issues = append(issues, issueShard{ + req: req, + err: err, + }) + } + + return issues, true, nil // this is reshardable +} + +func (*alterReplicaLogDirsSharder) onResp(kmsg.Request, kmsg.Response) error { return nil } // topic / partitions: not retried + +// merge does not make sense for this function, but we provide a one anyway. +func (*alterReplicaLogDirsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrAlterReplicaLogDirsResponse() + topics := make(map[string][]kmsg.AlterReplicaLogDirsResponseTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.AlterReplicaLogDirsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, topic := range resp.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + }) + for topic, partitions := range topics { + respTopic := kmsg.NewAlterReplicaLogDirsResponseTopic() + respTopic.Topic = topic + respTopic.Partitions = partitions + merged.Topics = append(merged.Topics, respTopic) + } + return merged, firstErr +} + +// handles sharding DescribeLogDirsRequest +type describeLogDirsSharder struct{ *Client } + +func (cl *describeLogDirsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeLogDirsRequest) + + // If req.Topics is nil, the request is to describe all logdirs. Thus, + // we will issue the request to all brokers (similar to ListGroups). + if req.Topics == nil { + return cl.allBrokersShardedReq(ctx, func() kmsg.Request { + dup := *req + return &dup + }) + } + + var need []string + for _, topic := range req.Topics { + need = append(need, topic.Topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, false, 0) // bypass cache, tricky to manage response + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string][]int32) + var unknowns unknownErrShards + + for _, topic := range req.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + unknowns.errs(err, t, topic.Partitions) + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + unknowns.err(err, t, partition) + continue + } + + for _, replica := range p.Replicas { + brokerReq := brokerReqs[replica] + if brokerReq == nil { + brokerReq = make(map[string][]int32) + brokerReqs[replica] = brokerReq + } + brokerReq[topic.Topic] = append(brokerReq[topic.Topic], partition) + } + } + } + + mkreq := kmsg.NewPtrDescribeLogDirsRequest + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for topic, parts := range brokerReq { + reqTopic := kmsg.NewDescribeLogDirsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + req.Topics = append(req.Topics, reqTopic) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + return append(issues, unknowns.collect(mkreq, func(r *kmsg.DescribeLogDirsRequest, topic string, parts []int32) { + reqTopic := kmsg.NewDescribeLogDirsRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + r.Topics = append(r.Topics, reqTopic) + })...), true, nil // this is reshardable +} + +func (*describeLogDirsSharder) onResp(kmsg.Request, kmsg.Response) error { return nil } // topic / configs: not retried + +// merge does not make sense for this function, but we provide one anyway. +// We lose the error code for directories. +func (*describeLogDirsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeLogDirsResponse() + dirs := make(map[string]map[string][]kmsg.DescribeLogDirsResponseDirTopicPartition) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeLogDirsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, dir := range resp.Dirs { + mergeDir := dirs[dir.Dir] + if mergeDir == nil { + mergeDir = make(map[string][]kmsg.DescribeLogDirsResponseDirTopicPartition) + dirs[dir.Dir] = mergeDir + } + for _, topic := range dir.Topics { + mergeDir[topic.Topic] = append(mergeDir[topic.Topic], topic.Partitions...) + } + } + }) + for dir, topics := range dirs { + md := kmsg.NewDescribeLogDirsResponseDir() + md.Dir = dir + for topic, partitions := range topics { + mdTopic := kmsg.NewDescribeLogDirsResponseDirTopic() + mdTopic.Topic = topic + mdTopic.Partitions = partitions + md.Topics = append(md.Topics, mdTopic) + } + merged.Dirs = append(merged.Dirs, md) + } + return merged, firstErr +} + +// handles sharding DeleteGroupsRequest +type deleteGroupsSharder struct{ *Client } + +func (cl *deleteGroupsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DeleteGroupsRequest) + + coordinators := cl.loadCoordinators(ctx, coordinatorTypeGroup, req.Groups...) + type unkerr struct { + err error + group string + } + var ( + brokerReqs = make(map[int32]*kmsg.DeleteGroupsRequest) + kerrs = make(map[*kerr.Error][]string) + unkerrs []unkerr + ) + + newReq := func(groups ...string) *kmsg.DeleteGroupsRequest { + newReq := kmsg.NewPtrDeleteGroupsRequest() + newReq.Groups = groups + return newReq + } + + for _, group := range req.Groups { + berr := coordinators[group] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.Groups = append(brokerReq.Groups, group) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], group) + default: + unkerrs = append(unkerrs, unkerr{berr.err, group}) + } + } + + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.group), + err: unkerr.err, + }) + } + for kerr, groups := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groups...), + err: kerr, + }) + } + + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *deleteGroupsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.DeleteGroupsResponse) + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.Group, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*deleteGroupsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDeleteGroupsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DeleteGroupsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + }) +} + +// handle sharding IncrementalAlterConfigsRequest +type incrementalAlterConfigsSharder struct{ *Client } + +func (*incrementalAlterConfigsSharder) shard(_ context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.IncrementalAlterConfigsRequest) + + brokerReqs := make(map[int32][]kmsg.IncrementalAlterConfigsRequestResource) + var any []kmsg.IncrementalAlterConfigsRequestResource + + for i := range req.Resources { + resource := req.Resources[i] + switch resource.ResourceType { + case kmsg.ConfigResourceTypeBroker: + case kmsg.ConfigResourceTypeBrokerLogger: + default: + any = append(any, resource) + continue + } + id, err := strconv.ParseInt(resource.ResourceName, 10, 32) + if err != nil || id < 0 { + any = append(any, resource) + continue + } + brokerReqs[int32(id)] = append(brokerReqs[int32(id)], resource) + } + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + newReq := kmsg.NewPtrIncrementalAlterConfigsRequest() + newReq.Resources = brokerReq + newReq.ValidateOnly = req.ValidateOnly + + issues = append(issues, issueShard{ + req: newReq, + broker: brokerID, + }) + } + + if len(any) > 0 { + newReq := kmsg.NewPtrIncrementalAlterConfigsRequest() + newReq.Resources = any + newReq.ValidateOnly = req.ValidateOnly + issues = append(issues, issueShard{ + req: newReq, + any: true, + }) + } + + return issues, false, nil // this is not reshardable, but the any block can go anywhere +} + +func (*incrementalAlterConfigsSharder) onResp(kmsg.Request, kmsg.Response) error { return nil } // configs: topics not mapped, nothing retryable + +func (*incrementalAlterConfigsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrIncrementalAlterConfigsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.IncrementalAlterConfigsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Resources = append(merged.Resources, resp.Resources...) + }) +} + +// handle sharding DescribeProducersRequest +type describeProducersSharder struct{ *Client } + +func (cl *describeProducersSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeProducersRequest) + + var need []string + for _, topic := range req.Topics { + need = append(need, topic.Topic) + } + mapping, err := cl.resolveTopicMeta(ctx, need, true, 0) + if err != nil { + return nil, false, err + } + + brokerReqs := make(map[int32]map[string][]int32) // broker => topic => partitions + var unknowns unknownErrShards + + for _, topic := range req.Topics { + t := topic.Topic + tmapping, exists := mapping[t] + if err := unknownOrCode(exists, tmapping.t.ErrorCode); err != nil { + unknowns.errs(err, t, topic.Partitions) + continue + } + for _, partition := range topic.Partitions { + p, exists := tmapping.ps[partition] + if err := unknownOrCode(exists, p.ErrorCode); err != nil { + unknowns.err(err, t, partition) + continue + } + + brokerReq := brokerReqs[p.Leader] + if brokerReq == nil { + brokerReq = make(map[string][]int32) + brokerReqs[p.Leader] = brokerReq + } + brokerReq[topic.Topic] = append(brokerReq[topic.Topic], partition) + } + } + + mkreq := kmsg.NewPtrDescribeProducersRequest + + var issues []issueShard + for brokerID, brokerReq := range brokerReqs { + req := mkreq() + for topic, parts := range brokerReq { + reqTopic := kmsg.NewDescribeProducersRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + req.Topics = append(req.Topics, reqTopic) + } + issues = append(issues, issueShard{ + req: req, + broker: brokerID, + }) + } + + return append(issues, unknowns.collect(mkreq, func(r *kmsg.DescribeProducersRequest, topic string, parts []int32) { + reqTopic := kmsg.NewDescribeProducersRequestTopic() + reqTopic.Topic = topic + reqTopic.Partitions = parts + r.Topics = append(r.Topics, reqTopic) + })...), true, nil // this is reshardable +} + +func (cl *describeProducersSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + var ( + resp = kresp.(*kmsg.DescribeProducersResponse) + del []string + retErr error + unknownTopic bool + ) + for i := range resp.Topics { + t := &resp.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + err := kerr.ErrorForCode(p.ErrorCode) + if err == kerr.UnknownTopicOrPartition || err == kerr.NotLeaderForPartition { + del = append(del, t.Topic) + unknownTopic = unknownTopic || err == kerr.UnknownTopicOrPartition + } + onRespShardErr(&retErr, err) + } + } + if cl.maybeDeleteCachedMeta(unknownTopic, del...) { + return retErr + } + return nil +} + +func (*describeProducersSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeProducersResponse() + topics := make(map[string][]kmsg.DescribeProducersResponseTopicPartition) + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeProducersResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + + for _, topic := range resp.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + }) + for topic, partitions := range topics { + respTopic := kmsg.NewDescribeProducersResponseTopic() + respTopic.Topic = topic + respTopic.Partitions = partitions + merged.Topics = append(merged.Topics, respTopic) + } + return merged, firstErr +} + +// handles sharding DescribeTransactionsRequest +type describeTransactionsSharder struct{ *Client } + +func (cl *describeTransactionsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeTransactionsRequest) + + coordinators := cl.loadCoordinators(ctx, coordinatorTypeTxn, req.TransactionalIDs...) + type unkerr struct { + err error + txnID string + } + var ( + brokerReqs = make(map[int32]*kmsg.DescribeTransactionsRequest) + kerrs = make(map[*kerr.Error][]string) + unkerrs []unkerr + ) + + newReq := func(txnIDs ...string) *kmsg.DescribeTransactionsRequest { + r := kmsg.NewPtrDescribeTransactionsRequest() + r.TransactionalIDs = txnIDs + return r + } + + for _, txnID := range req.TransactionalIDs { + berr := coordinators[txnID] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.TransactionalIDs = append(brokerReq.TransactionalIDs, txnID) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], txnID) + default: + unkerrs = append(unkerrs, unkerr{berr.err, txnID}) + } + } + + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.txnID), + err: unkerr.err, + }) + } + for kerr, txnIDs := range kerrs { + issues = append(issues, issueShard{ + req: newReq(txnIDs...), + err: kerr, + }) + } + + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *describeTransactionsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { // cleanup any stale coordinators + resp := kresp.(*kmsg.DescribeTransactionsResponse) + var retErr error + for i := range resp.TransactionStates { + txnState := &resp.TransactionStates[i] + err := kerr.ErrorForCode(txnState.ErrorCode) + cl.maybeDeleteStaleCoordinator(txnState.TransactionalID, coordinatorTypeTxn, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*describeTransactionsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeTransactionsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeTransactionsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.TransactionStates = append(merged.TransactionStates, resp.TransactionStates...) + }) +} + +// handles sharding ListTransactionsRequest +type listTransactionsSharder struct{ *Client } + +func (cl *listTransactionsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.ListTransactionsRequest) + return cl.allBrokersShardedReq(ctx, func() kmsg.Request { + dup := *req + return &dup + }) +} + +func (*listTransactionsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.ListTransactionsResponse) + return kerr.ErrorForCode(resp.ErrorCode) +} + +func (*listTransactionsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrListTransactionsResponse() + + unknownStates := make(map[string]struct{}) + // During a txn coordinator migration, a transactional ID can + // transiently appear in both the old and new coordinator's + // ListTransactions response. Dedupe by transactional ID; the first + // shard response wins. + seen := make(map[string]struct{}) + + firstErr := firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.ListTransactionsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + if merged.ErrorCode == 0 { + merged.ErrorCode = resp.ErrorCode + } + for _, state := range resp.UnknownStateFilters { + unknownStates[state] = struct{}{} + } + for _, s := range resp.TransactionStates { + if _, ok := seen[s.TransactionalID]; ok { + continue + } + seen[s.TransactionalID] = struct{}{} + merged.TransactionStates = append(merged.TransactionStates, s) + } + }) + for unknownState := range unknownStates { + merged.UnknownStateFilters = append(merged.UnknownStateFilters, unknownState) + } + + return merged, firstErr +} + +// handles sharding ConsumerGroupDescribeRequest +type consumerGroupDescribeSharder struct{ *Client } + +func (cl *consumerGroupDescribeSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.ConsumerGroupDescribeRequest) + coordinators := cl.loadCoordinators(ctx, coordinatorTypeGroup, req.Groups...) + type unkerr struct { + err error + group string + } + var ( + brokerReqs = make(map[int32]*kmsg.ConsumerGroupDescribeRequest) + kerrs = make(map[*kerr.Error][]string) + unkerrs []unkerr + ) + newReq := func(groups ...string) *kmsg.ConsumerGroupDescribeRequest { + newReq := kmsg.NewPtrConsumerGroupDescribeRequest() + newReq.IncludeAuthorizedOperations = req.IncludeAuthorizedOperations + newReq.Groups = groups + return newReq + } + for _, group := range req.Groups { + berr := coordinators[group] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.Groups = append(brokerReq.Groups, group) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], group) + default: + unkerrs = append(unkerrs, unkerr{berr.err, group}) + } + } + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.group), + err: unkerr.err, + }) + } + for kerr, groups := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groups...), + err: kerr, + }) + } + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *consumerGroupDescribeSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.ConsumerGroupDescribeResponse) + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.Group, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*consumerGroupDescribeSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrConsumerGroupDescribeResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.ConsumerGroupDescribeResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + }) +} + +// handles sharding ShareGroupDescribeRequest +type shareGroupDescribeSharder struct{ *Client } + +func (cl *shareGroupDescribeSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.ShareGroupDescribeRequest) + coordinators := cl.loadCoordinators(ctx, coordinatorTypeGroup, req.GroupIDs...) + type unkerr struct { + err error + groupID string + } + var ( + brokerReqs = make(map[int32]*kmsg.ShareGroupDescribeRequest) + kerrs = make(map[*kerr.Error][]string) + unkerrs []unkerr + ) + newReq := func(groupIDs ...string) *kmsg.ShareGroupDescribeRequest { + newReq := kmsg.NewPtrShareGroupDescribeRequest() + newReq.IncludeAuthorizedOperations = req.IncludeAuthorizedOperations + newReq.GroupIDs = groupIDs + return newReq + } + for _, groupID := range req.GroupIDs { + berr := coordinators[groupID] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.GroupIDs = append(brokerReq.GroupIDs, groupID) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], groupID) + default: + unkerrs = append(unkerrs, unkerr{berr.err, groupID}) + } + } + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(unkerr.groupID), + err: unkerr.err, + }) + } + for kerr, groupIDs := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groupIDs...), + err: kerr, + }) + } + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *shareGroupDescribeSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.ShareGroupDescribeResponse) + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.GroupID, coordinatorTypeGroup, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*shareGroupDescribeSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrShareGroupDescribeResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.ShareGroupDescribeResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + }) +} + +// handles sharding DescribeShareGroupOffsetsRequest +type describeShareGroupOffsetsSharder struct{ *Client } + +func (cl *describeShareGroupOffsetsSharder) shard(ctx context.Context, kreq kmsg.Request, _ error) ([]issueShard, bool, error) { + req := kreq.(*kmsg.DescribeShareGroupOffsetsRequest) + groupIDs := make([]string, 0, len(req.Groups)) + for _, g := range req.Groups { + groupIDs = append(groupIDs, g.GroupID) + } + coordinators := cl.loadCoordinators(ctx, coordinatorTypeShare, groupIDs...) + type unkerr struct { + err error + groupID string + } + var ( + brokerReqs = make(map[int32]*kmsg.DescribeShareGroupOffsetsRequest) + kerrs = make(map[*kerr.Error][]kmsg.DescribeShareGroupOffsetsRequestGroup) + unkerrs []unkerr + ) + newReq := func(groups ...kmsg.DescribeShareGroupOffsetsRequestGroup) *kmsg.DescribeShareGroupOffsetsRequest { + newReq := kmsg.NewPtrDescribeShareGroupOffsetsRequest() + newReq.Groups = groups + return newReq + } + for _, g := range req.Groups { + berr := coordinators[g.GroupID] + var ke *kerr.Error + switch { + case berr.err == nil: + brokerReq := brokerReqs[berr.b.meta.NodeID] + if brokerReq == nil { + brokerReq = newReq() + brokerReqs[berr.b.meta.NodeID] = brokerReq + } + brokerReq.Groups = append(brokerReq.Groups, g) + case errors.As(berr.err, &ke): + kerrs[ke] = append(kerrs[ke], g) + default: + unkerrs = append(unkerrs, unkerr{berr.err, g.GroupID}) + } + } + var issues []issueShard + for id, req := range brokerReqs { + issues = append(issues, issueShard{ + req: req, + broker: id, + }) + } + for _, unkerr := range unkerrs { + issues = append(issues, issueShard{ + req: newReq(kmsg.DescribeShareGroupOffsetsRequestGroup{GroupID: unkerr.groupID}), + err: unkerr.err, + }) + } + for kerr, groups := range kerrs { + issues = append(issues, issueShard{ + req: newReq(groups...), + err: kerr, + }) + } + return issues, true, nil // reshardable to load correct coordinators +} + +func (cl *describeShareGroupOffsetsSharder) onResp(_ kmsg.Request, kresp kmsg.Response) error { + resp := kresp.(*kmsg.DescribeShareGroupOffsetsResponse) + var retErr error + for i := range resp.Groups { + group := &resp.Groups[i] + err := kerr.ErrorForCode(group.ErrorCode) + cl.maybeDeleteStaleCoordinator(group.GroupID, coordinatorTypeShare, err) + onRespShardErr(&retErr, err) + } + return retErr +} + +func (*describeShareGroupOffsetsSharder) merge(sresps []ResponseShard) (kmsg.Response, error) { + merged := kmsg.NewPtrDescribeShareGroupOffsetsResponse() + return merged, firstErrMerger(sresps, func(kresp kmsg.Response) { + resp := kresp.(*kmsg.DescribeShareGroupOffsetsResponse) + merged.Version = resp.Version + merged.ThrottleMillis = resp.ThrottleMillis + merged.Groups = append(merged.Groups, resp.Groups...) + }) +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/compression.go b/vendor/github.com/twmb/franz-go/pkg/kgo/compression.go new file mode 100644 index 00000000000..ff114664a9b --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/compression.go @@ -0,0 +1,453 @@ +package kgo + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "errors" + "io" + "runtime" + "slices" + "sync" + + "github.com/klauspost/compress/s2" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" +) + +var byteBuffers = sync.Pool{New: func() any { return bytes.NewBuffer(make([]byte, 8<<10)) }} + +// CompressionCodecType is a bitfield specifying a Kafka-defined compression +// codec. Per spec, only four compression codecs are supported. However, if +// you control both the producer and consumer, you can technically override the +// codec to anything. +type CompressionCodecType int8 + +const ( + // CodecNone is a compression codec signifying no compression is used. + CodecNone CompressionCodecType = iota + // CodecGzip is a compression codec signifying gzip compression. + CodecGzip + // CodecSnappy is a compression codec signifying snappy compression. + CodecSnappy + // CodecLz4 is a compression codec signifying lz4 compression. + CodecLz4 + // CodecZstd is a compression codec signifying zstd compression. + CodecZstd + + // CodecError is returned from compressing or decompressing if an error + // occurred. + CodecError = -1 +) + +// CompressionCodec configures how records are compressed before being sent. +// +// Records are compressed within individual topics and partitions, inside of a +// RecordBatch. All records in a RecordBatch are compressed into one record +// for that batch. +type CompressionCodec struct { + codec CompressionCodecType + level int +} + +// NoCompression is a compression option that avoids compression. This can +// always be used as a fallback compression. +func NoCompression() CompressionCodec { return CompressionCodec{CodecNone, 0} } + +// GzipCompression enables gzip compression with the default compression level. +func GzipCompression() CompressionCodec { return CompressionCodec{CodecGzip, gzip.DefaultCompression} } + +// SnappyCompression enables snappy compression. +func SnappyCompression() CompressionCodec { return CompressionCodec{CodecSnappy, 0} } + +// Lz4Compression enables lz4 compression with the fastest compression level. +func Lz4Compression() CompressionCodec { return CompressionCodec{CodecLz4, 0} } + +// ZstdCompression enables zstd compression with the default compression level. +func ZstdCompression() CompressionCodec { return CompressionCodec{CodecZstd, 0} } + +// CompressFlag is a flag to instruct the compressor. +type CompressFlag uint16 + +const ( + // CompressDisableZstd instructs the compressor that zstd should not be + // used, even if the compressor supports it. This is used when + // producing to an old broker (pre Kafka v2.1) that does not yet + // support zstd compression. If you are confident you will only produce + // to new brokers, you can ignore this flag. + CompressDisableZstd CompressFlag = 1 + iota +) + +func mkCompressFlags(produceRequestVersion int16) []CompressFlag { + if produceRequestVersion < 7 { + return []CompressFlag{CompressDisableZstd} + } + return nil +} + +// Compressor is an interface that defines how produce batches are compressed. +// You can override the default client internal compressor for more control +// over what compressors to use, level, and memory reuse. +type Compressor interface { + // Compress compresses src and returns the compressed data as well as + // the codec type that was used. The 'dst' [bytes.Buffer] argument is + // pooled within the client and reused across calls to Compress. You + // can use 'dst' to save memory and return 'dst.Bytes()'. The returned + // slice is fully used *before* 'dst' is put back into the internal + // pool. As an example, you can look at the franz-go internal + // implementation of the default compressor in compression.go. + // + // Flags may optionally be provided to direct the compressor to enable + // or disable features. New backwards compatible flags may be + // introduced. If you add features to your compressor, be sure to + // evaluate if new flags exist to opt into or out of features. + Compress(dst *bytes.Buffer, src []byte, flags ...CompressFlag) ([]byte, CompressionCodecType) +} + +// Decompressor is an interface that defines how fetch batches are +// decompressed. You can override the default client internal decompressor for +// more control over what decompressors to use and memory reuse. +type Decompressor interface { + // Decompress decompresses src, which is compressed with codecType, + // and returns the decompressed data or an error. + // + // If the decompression codec type is CodecNone, this should return + // the input slice. + Decompress(src []byte, codecType CompressionCodecType) ([]byte, error) +} + +// WithLevel changes the compression codec's "level", effectively allowing for +// higher or lower compression ratios at the expense of CPU speed. +// +// For the zstd package, the level is a typed int; simply convert the type back +// to an int for this function. +// +// If the level is invalid, compressors just use a default level. +func (c CompressionCodec) WithLevel(level int) CompressionCodec { + c.level = level + return c +} + +type compressor struct { + options []CompressionCodecType + gzPool sync.Pool + lz4Pool sync.Pool + zstdPool sync.Pool +} + +// DefaultCompressor returns the default client compressor. The returned +// compressor will compress produce batches in preference-order of the +// specified codecs. Usually, you only need to specify one codec. If you are +// speaking to an old broker that may not support zstd, you may need to specify +// a second compressor as fallback (old Kafka did not support zstd). If no +// codecs are specified, or the specified codec is CodecNone, this returns +// 'nil, nil'. A compressor is only used within the client if it is non-nil. +func DefaultCompressor(codecs ...CompressionCodec) (Compressor, error) { + if len(codecs) == 0 { + return nil, nil + } + + used := make(map[CompressionCodecType]bool) // we keep one type of codec per CompressionCodec + var keepIdx int + for _, codec := range codecs { + if _, exists := used[codec.codec]; exists { + continue + } + used[codec.codec] = true + codecs[keepIdx] = codec + keepIdx++ + } + codecs = codecs[:keepIdx] + + for _, codec := range codecs { + if codec.codec < 0 || codec.codec > 4 { + return nil, errors.New("unknown compression codec") + } + } + + c := new(compressor) + +out: + for _, codec := range codecs { + c.options = append(c.options, codec.codec) + switch codec.codec { + case CodecNone: + break out + case CodecGzip: + level := gzip.DefaultCompression + if codec.level != 0 { + if _, err := gzip.NewWriterLevel(nil, codec.level); err != nil { + level = codec.level + } + } + c.gzPool = sync.Pool{New: func() any { c, _ := gzip.NewWriterLevel(nil, level); return c }} + case CodecSnappy: // (no pool needed for snappy) + case CodecLz4: + level := max(codec.level, 0) + fn := func() any { return lz4.NewWriter(new(bytes.Buffer)) } + w := lz4.NewWriter(new(bytes.Buffer)) + if err := w.Apply(lz4.CompressionLevelOption(lz4.CompressionLevel(level))); err == nil { + fn = func() any { + w := lz4.NewWriter(new(bytes.Buffer)) + w.Apply(lz4.CompressionLevelOption(lz4.CompressionLevel(level))) + return w + } + } + w.Close() + c.lz4Pool = sync.Pool{New: fn} + case CodecZstd: + opts := []zstd.EOption{ + zstd.WithWindowSize(64 << 10), + zstd.WithEncoderConcurrency(1), + zstd.WithZeroFrames(true), + } + fn := func() any { + zstdEnc, _ := zstd.NewWriter(nil, opts...) + r := &zstdEncoder{zstdEnc} + runtime.SetFinalizer(r, func(r *zstdEncoder) { r.inner.Close() }) + return r + } + zstdEnc, err := zstd.NewWriter(nil, append(opts, zstd.WithEncoderLevel(zstd.EncoderLevel(codec.level)))...) + if err == nil { + zstdEnc.Close() + opts = append(opts, zstd.WithEncoderLevel(zstd.EncoderLevel(codec.level))) + } + c.zstdPool = sync.Pool{New: fn} + } + } + + if c.options[0] == CodecNone { + return nil, nil // first codec was passthrough + } + + return c, nil +} + +type zstdEncoder struct { + inner *zstd.Encoder +} + +func (c *compressor) Compress(dst *bytes.Buffer, src []byte, flags ...CompressFlag) ([]byte, CompressionCodecType) { + var disableZstd bool + for _, flag := range flags { + if flag == CompressDisableZstd { + disableZstd = true + } + } + + var use CompressionCodecType + for _, option := range c.options { + if option == CodecZstd && disableZstd { + continue + } + use = option + break + } + + var out []byte + switch use { + case CodecNone: + return src, 0 + case CodecGzip: + gz := c.gzPool.Get().(*gzip.Writer) + defer c.gzPool.Put(gz) + gz.Reset(dst) + if _, err := gz.Write(src); err != nil { + return nil, CodecError + } + if err := gz.Close(); err != nil { + return nil, CodecError + } + out = dst.Bytes() + case CodecLz4: + lz := c.lz4Pool.Get().(*lz4.Writer) + defer c.lz4Pool.Put(lz) + lz.Reset(dst) + if _, err := lz.Write(src); err != nil { + return nil, CodecError + } + if err := lz.Close(); err != nil { + return nil, CodecError + } + out = dst.Bytes() + case CodecSnappy: + // Because the Snappy and Zstd codecs do not accept an io.Writer interface + // and directly take a []byte slice, here, the underlying []byte slice (`dst`) + // obtained from the bytes.Buffer{} from the pool is passed. + // As the `Write()` method on the buffer isn't used, its internal + // book-keeping goes out of sync, making the buffer unusable for further + // reading and writing via it's (eg: accessing via `Byte()`). For subsequent + // reads, the underlying slice has to be used directly. + // + // In this particular context, it is acceptable as there are no subsequent + // operations performed on the buffer and it is immediately returned to the + // pool and `Reset()` the next time it is obtained and used where `compress()` + // is called. + if l := s2.MaxEncodedLen(len(src)); l > dst.Cap() { + dst.Grow(l) + } + out = s2.EncodeSnappy(dst.Bytes(), src) + case CodecZstd: + zstdEnc := c.zstdPool.Get().(*zstdEncoder) + defer c.zstdPool.Put(zstdEnc) + if l := zstdEnc.inner.MaxEncodedSize(len(src)); l > dst.Cap() { + dst.Grow(l) + } + out = zstdEnc.inner.EncodeAll(src, dst.Bytes()) + } + + return out, use +} + +type decompressor struct { + ungzPool sync.Pool + unlz4Pool sync.Pool + unzstdPool sync.Pool + pools pools +} + +// DefaultDecompressor returns the default decompressor used by clients. +// The first pool provided that implements PoolDecompressBytes will be +// used where possible. +func DefaultDecompressor(pools ...Pool) Decompressor { + d := &decompressor{ + ungzPool: sync.Pool{ + New: func() any { return new(gzip.Reader) }, + }, + unlz4Pool: sync.Pool{ + New: func() any { return lz4.NewReader(nil) }, + }, + unzstdPool: sync.Pool{ + New: func() any { + zstdDec, _ := zstd.NewReader(nil, + zstd.WithDecoderLowmem(true), + zstd.WithDecoderConcurrency(1), + ) + r := &zstdDecoder{zstdDec} + runtime.SetFinalizer(r, func(r *zstdDecoder) { + r.inner.Close() + }) + return r + }, + }, + pools: pools, + } + return d +} + +type zstdDecoder struct { + inner *zstd.Decoder +} + +func (d *decompressor) Decompress(src []byte, codecType CompressionCodecType) ([]byte, error) { + if codecType == CodecNone { + return src, nil + } + + var ( + out *bytes.Buffer + rfn func() []byte + userPooled bool + ) + d.pools.each(func(p Pool) bool { + if pdecompressBytes, ok := p.(PoolDecompressBytes); ok { + s := pdecompressBytes.GetDecompressBytes(src, codecType) + out = bytes.NewBuffer(s) + rfn = out.Bytes + userPooled = true + return true + } + return false + }) + if out == nil { + out = byteBuffers.Get().(*bytes.Buffer) + out.Reset() + defer byteBuffers.Put(out) + // We clone out.Bytes since we are pooling out ourselves; we + // need to clone before return since we immediately put into + // the pool. + // + // For user provided slices, we put back into the pool only + // after the user calls Recycle on every record that has a + // reference to the slice. Thus, we can return the original + // slice from the user-provided pool: it is only recycled + // at the end when the user says they are done. + rfn = func() []byte { return slices.Clone(out.Bytes()) } + } + + switch codecType { + case CodecGzip: + ungz := d.ungzPool.Get().(*gzip.Reader) + defer d.ungzPool.Put(ungz) + if err := ungz.Reset(bytes.NewReader(src)); err != nil { + return nil, err + } + if _, err := io.Copy(out, ungz); err != nil { + return nil, err + } + return rfn(), nil + case CodecSnappy: + if len(src) > 16 && bytes.HasPrefix(src, xerialPfx) { + return xerialDecode(src) + } + decoded, err := s2.Decode(out.Bytes(), src) + if err != nil { + return nil, err + } + if userPooled { + return decoded, nil + } + return slices.Clone(decoded), nil + case CodecLz4: + unlz4 := d.unlz4Pool.Get().(*lz4.Reader) + defer d.unlz4Pool.Put(unlz4) + unlz4.Reset(bytes.NewReader(src)) + if _, err := io.Copy(out, unlz4); err != nil { + return nil, err + } + return rfn(), nil + case CodecZstd: + unzstd := d.unzstdPool.Get().(*zstdDecoder) + defer d.unzstdPool.Put(unzstd) + decoded, err := unzstd.inner.DecodeAll(src, out.Bytes()) + if err != nil { + return nil, err + } + if userPooled { + return decoded, nil + } + return slices.Clone(decoded), nil + default: + return nil, errors.New("unknown compression codec") + } +} + +var xerialPfx = []byte{130, 83, 78, 65, 80, 80, 89, 0} + +var errMalformedXerial = errors.New("malformed xerial framing") + +func xerialDecode(src []byte) ([]byte, error) { + // bytes 0-8: xerial header + // bytes 8-16: xerial version + // everything after: uint32 chunk size, snappy chunk + // we come into this function knowing src is at least 16 + src = src[16:] + var dst, chunk []byte + var err error + for len(src) > 0 { + if len(src) < 4 { + return nil, errMalformedXerial + } + size := int32(binary.BigEndian.Uint32(src)) + src = src[4:] + if size < 0 || len(src) < int(size) { + return nil, errMalformedXerial + } + if chunk, err = s2.Decode(chunk[:cap(chunk)], src[:size]); err != nil { + return nil, err + } + src = src[size:] + dst = append(dst, chunk...) + } + return dst, nil +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/config.go b/vendor/github.com/twmb/franz-go/pkg/kgo/config.go new file mode 100644 index 00000000000..ee361a1c05f --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/config.go @@ -0,0 +1,2203 @@ +package kgo + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "iter" + "math" + "math/rand" + "net" + "regexp" + "runtime/debug" + "time" + + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" + "github.com/twmb/franz-go/pkg/kversion" + "github.com/twmb/franz-go/pkg/sasl" +) + +//////////////////////////////////////////////////////// +// NOTE: // +// NOTE: Make sure new configs are added to OptValues // +// NOTE: // +//////////////////////////////////////////////////////// + +// Opt is an option to configure a client. +type Opt interface { + apply(*cfg) +} + +// ProducerOpt is a producer specific option to configure a client. +// This is simply a namespaced Opt. +type ProducerOpt interface { + Opt + producerOpt() +} + +// ConsumerOpt is a consumer specific option to configure a client. +// This is simply a namespaced Opt. +type ConsumerOpt interface { + Opt + consumerOpt() +} + +// GroupOpt is a consumer group specific option to configure a client. +// This is simply a namespaced Opt. +type GroupOpt interface { + Opt + groupOpt() +} + +type ( + clientOpt struct{ fn func(*cfg) } + producerOpt struct{ fn func(*cfg) } + consumerOpt struct{ fn func(*cfg) } + groupOpt struct{ fn func(*cfg) } +) + +func (opt clientOpt) apply(cfg *cfg) { opt.fn(cfg) } +func (opt producerOpt) apply(cfg *cfg) { opt.fn(cfg) } +func (opt consumerOpt) apply(cfg *cfg) { opt.fn(cfg) } +func (opt groupOpt) apply(cfg *cfg) { opt.fn(cfg) } +func (producerOpt) producerOpt() {} +func (consumerOpt) consumerOpt() {} +func (groupOpt) groupOpt() {} + +// A cfg can be written to while initializing a client, and after that it is +// (mostly) only ever read from. Some areas can continue to be modified -- +// particularly reconfiguring what to consume from -- but most areas are +// static. +type cfg struct { + ///////////////////// + // GENERAL SECTION // + ///////////////////// + + id *string // client ID + ctx context.Context + dialFn func(context.Context, string, string) (net.Conn, error) + dialTimeout time.Duration + dialTLS *tls.Config + requestTimeoutOverhead time.Duration + connIdleTimeout time.Duration + + softwareName string // KIP-511 + softwareVersion string // KIP-511 + + logger Logger + + seedBrokers []string + maxVersions *kversion.Versions + minVersions *kversion.Versions + + onRebootstrapRequired func() ([]string, error) + + retryBackoff func(int) time.Duration + retries int64 + retryTimeout func(int16) time.Duration + + maxBrokerWriteBytes int32 + maxBrokerReadBytes int32 + + metadataMaxAge time.Duration + metadataMinAge time.Duration + + sasls []sasl.Mechanism + + alwaysRetryEOF bool + allowAutoTopicCreation bool + disableClientMetrics bool + userMetrics func() iter.Seq[Metric] + + hooks hooks + pools pools + + ////////////////////// + // PRODUCER SECTION // + ////////////////////// + + txnID *string + txnTimeout time.Duration + acks Acks + disableIdempotency bool + allowIdempotentProduceCancellation bool + maxProduceInflight int // if idempotency is disabled, we allow a configurable max inflight + compression []CompressionCodec // order of preference + + defaultProduceTopic string + defaultProduceTopicAlways bool + maxRecordBatchBytes func(string) int32 + maxBufferedRecords int64 + maxBufferedBytes int64 + produceTimeout time.Duration + recordRetries int64 + maxUnknownFailures int64 + linger time.Duration + recordTimeout time.Duration + manualFlushing bool + txnBackoff time.Duration + missingTopicDelete time.Duration + + partitioner Partitioner + compressor Compressor + + stopOnDataLoss bool + onDataLoss func(string, int32) + + ////////////////////// + // CONSUMER SECTION // + ////////////////////// + + // maxWait holds the user-configured FetchMaxWait in milliseconds. + // math.MinInt32 is a sigil meaning "unset": validate replaces it with + // the default for the current mode (500ms for share groups, 5s + // otherwise). + maxWait int32 + minBytes int32 + maxBytes lazyI32 + maxPartBytes lazyI32 + startOffset Offset + resetOffset Offset + setStartOffset bool + setResetOffset bool + isolationLevel int8 + keepControl bool + rack string + preferLagFn PreferLagFn + decompressor Decompressor + + maxConcurrentFetches int + disableFetchSessions bool + keepRetryableFetchErrors bool + disableFetchCRCValidation bool + + recheckPreferredReplicaInterval time.Duration + + topics map[string]*regexp.Regexp // topics to consume; if regex is true, values are compiled regular expressions + excludeTopics map[string]*regexp.Regexp // topics to exclude; only used if regex is true, values are compiled regular expressions + partitions map[string]map[int32]Offset // partitions to directly consume from + regex bool + + //////////////////////////// + // CONSUMER GROUP SECTION // + //////////////////////////// + + group string // group we are in + shareGroup string // share group we are in + shareMaxRecords int32 // MaxRecords and BatchSize for ShareFetch (KIP-1206) + shareMaxRecordsStrict bool // if true, ShareAcquireMode=1 (record-limit) per KIP-1206 + shareAckCallback func(*Client, ShareAckResults) + instanceID *string // optional group instance ID + balancers []GroupBalancer // balancers we can use + protocol string // "consumer" by default, expected to never be overridden + + sessionTimeout time.Duration + rebalanceTimeout time.Duration + heartbeatInterval time.Duration + + onAssigned func(context.Context, *Client, map[string][]int32) + onRevoked func(context.Context, *Client, map[string][]int32) + onLost func(context.Context, *Client, map[string][]int32) + onBlocked func(context.Context, *Client) + onFetched func(context.Context, *Client, *kmsg.OffsetFetchResponse) error + userHasOnAssign bool // captured before initGroup wraps onAssigned non-nil + + adjustOffsetsBeforeAssign func(ctx context.Context, offsets map[string]map[int32]Offset) (map[string]map[int32]Offset, error) + + blockRebalanceOnPoll bool + + autocommitDisable bool // true if autocommit was disabled or we are transactional + autocommitGreedy bool + autocommitMarks bool + autocommitInterval time.Duration + commitCallback func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error) + + disableNextGenBalancer bool +} + +func (cfg *cfg) validate() error { + if len(cfg.seedBrokers) == 0 { + return errors.New("config erroneously has no seed brokers") + } + + if cfg.maxWait == math.MinInt32 { + if cfg.shareGroup != "" { + cfg.maxWait = 500 + } else { + cfg.maxWait = 5000 + } + } + + // We clamp maxPartBytes to maxBytes because some fake Kafka endpoints + // (Oracle) cannot handle the mismatch correctly. + if cfg.maxPartBytes > cfg.maxBytes { + cfg.maxPartBytes = cfg.maxBytes + } + + if cfg.allowIdempotentProduceCancellation && cfg.txnID != nil { + return errors.New("cannot allow idempotent produce cancellation and use transactional IDs") + } + if cfg.disableIdempotency { + if cfg.txnID != nil { + return errors.New("cannot both disable idempotent writes and use transactional IDs") + } + if cfg.maxProduceInflight <= 0 { + return fmt.Errorf("invalid max produce inflight %d with idempotency disabled", cfg.maxProduceInflight) + } + } else { + if cfg.acks.val != -1 { + return errors.New("idempotency requires acks=all") + } + if cfg.maxProduceInflight != 1 { + return fmt.Errorf("invalid usage of MaxProduceRequestsInflightPerBroker with idempotency enabled") + } + } + + for _, limit := range []struct { + name string + sp **string // if field is a *string, we take addr to it + s string + allowed int + }{ + // A 256 byte ID / software name & version is good enough and + // fits with our max broker write byte min of 1K. + {name: "client id", sp: &cfg.id, allowed: 256}, + {name: "software name", s: cfg.softwareName, allowed: 256}, + {name: "software version", s: cfg.softwareVersion, allowed: 256}, + + // The following is the limit transitioning from two byte + // prefix for flexible stuff to three bytes; as with above, it + // is more than reasonable. + {name: "transactional id", sp: &cfg.txnID, allowed: 16382}, + + {name: "rack", s: cfg.rack, allowed: 512}, + {name: "share group", s: cfg.shareGroup, allowed: 16382}, + } { + s := limit.s + if limit.sp != nil && *limit.sp != nil { + s = **limit.sp + } + if len(s) > limit.allowed { + return fmt.Errorf("%s length %d is larger than max allowed %d", limit.name, len(s), limit.allowed) + } + } + + i64lt := func(l, r int64) (bool, string) { return l < r, "less" } + i64gt := func(l, r int64) (bool, string) { return l > r, "larger" } + for _, limit := range []struct { + name string + v int64 + allowed int64 + badcmp func(int64, int64) (bool, string) + + fmt string + durs bool + }{ + // Min write of 1K and max of 1G is reasonable. + {name: "max broker write bytes", v: int64(cfg.maxBrokerWriteBytes), allowed: 1 << 10, badcmp: i64lt}, + {name: "max broker write bytes", v: int64(cfg.maxBrokerWriteBytes), allowed: 1 << 30, badcmp: i64gt}, + + // Same for read bytes. + {name: "max broker read bytes", v: int64(cfg.maxBrokerReadBytes), allowed: 1 << 10, badcmp: i64lt}, + {name: "max broker read bytes", v: int64(cfg.maxBrokerReadBytes), allowed: 1 << 30, badcmp: i64gt}, + + // For batches, we want at least 512 (reasonable), and the + // upper limit is the max num when a uvarint transitions from 4 + // to 5 bytes. The upper limit is also more than reasonable (1G). + {name: "max record batch bytes", v: int64(cfg.maxRecordBatchBytes("")), allowed: 512, badcmp: i64lt}, + {name: "max record batch bytes", v: int64(cfg.maxRecordBatchBytes("")), allowed: 1 << 30, badcmp: i64gt}, + + // We do not want the broker write bytes to be less than the + // record batch bytes, nor the read bytes to be less than what + // we indicate to fetch. + // + // We cannot enforce if a single batch is larger than the max + // fetch bytes limit, but hopefully we do not run into that. + {v: int64(cfg.maxBrokerWriteBytes), allowed: int64(cfg.maxRecordBatchBytes("")), badcmp: i64lt, fmt: "max broker write bytes %v is erroneously less than max record batch bytes %v"}, + {v: int64(cfg.maxBrokerReadBytes), allowed: int64(cfg.maxBytes), badcmp: i64lt, fmt: "max broker read bytes %v is erroneously less than max fetch bytes %v"}, + + // -1 <= allowed concurrency (-1 is unbounded) + {name: "max concurrent fetches", v: int64(cfg.maxConcurrentFetches), allowed: -1, badcmp: i64lt}, + + // 100ms <= request timeout overhead <= 15m + {name: "request timeout max overhead", v: int64(cfg.requestTimeoutOverhead), allowed: int64(15 * time.Minute), badcmp: i64gt, durs: true}, + {name: "request timeout min overhead", v: int64(cfg.requestTimeoutOverhead), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + + // 100ms <= conn idle <= 15m + {name: "conn min idle timeout", v: int64(cfg.connIdleTimeout), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + {name: "conn max idle timeout", v: int64(cfg.connIdleTimeout), allowed: int64(15 * time.Minute), badcmp: i64gt, durs: true}, + + // 10ms <= metadata <= 1hr + {name: "metadata max age", v: int64(cfg.metadataMaxAge), allowed: int64(time.Hour), badcmp: i64gt, durs: true}, + {name: "metadata min age", v: int64(cfg.metadataMinAge), allowed: int64(10 * time.Millisecond), badcmp: i64lt, durs: true}, + {v: int64(cfg.metadataMaxAge), allowed: int64(cfg.metadataMinAge), badcmp: i64lt, fmt: "metadata max age %v is erroneously less than metadata min age %v", durs: true}, + + // 10ms <= preferred recheck interval <= 7d + {name: "recheck preferred replica interval", v: int64(cfg.recheckPreferredReplicaInterval), allowed: int64(10 * time.Millisecond), badcmp: i64lt, durs: true}, + {name: "recheck preferred replica interval", v: int64(cfg.recheckPreferredReplicaInterval), allowed: int64(7 * 24 * time.Hour), badcmp: i64gt, durs: true}, + + // Some random producer settings. + {name: "max buffered records", v: cfg.maxBufferedRecords, allowed: 1, badcmp: i64lt}, + {name: "max buffered bytes", v: cfg.maxBufferedBytes, allowed: 0, badcmp: i64lt}, + {name: "linger", v: int64(cfg.linger), allowed: int64(time.Minute), badcmp: i64gt, durs: true}, + {name: "produce timeout", v: int64(cfg.produceTimeout), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + {name: "record timeout", v: int64(cfg.recordTimeout), allowed: int64(time.Second), badcmp: func(l, r int64) (bool, string) { + if l == 0 { + return false, "" // we print nothing when things are good + } + return l < r, "less" + }, durs: true}, + + // Consumer settings. maxWait is stored as int32 milliseconds, + // but we want the error message to be in the nice + // time.Duration string format. + {name: "max fetch wait", v: int64(cfg.maxWait) * int64(time.Millisecond), allowed: int64(10 * time.Millisecond), badcmp: i64lt, durs: true}, + + // Group settings. + {name: "number of balancers", v: int64(len(cfg.balancers)), allowed: 1, badcmp: i64lt}, + {name: "consumer protocol length", v: int64(len(cfg.protocol)), allowed: 1, badcmp: i64lt}, + + {name: "session timeout", v: int64(cfg.sessionTimeout), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + {name: "rebalance timeout", v: int64(cfg.rebalanceTimeout), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + {name: "autocommit interval", v: int64(cfg.autocommitInterval), allowed: int64(100 * time.Millisecond), badcmp: i64lt, durs: true}, + + {v: int64(cfg.heartbeatInterval), allowed: int64(cfg.rebalanceTimeout) * int64(time.Millisecond), badcmp: i64gt, durs: true, fmt: "heartbeat interval %v is erroneously larger than the session timeout %v"}, + } { + bad, cmp := limit.badcmp(limit.v, limit.allowed) + if bad { + if limit.fmt != "" { + if limit.durs { + return fmt.Errorf(limit.fmt, time.Duration(limit.v), time.Duration(limit.allowed)) + } + return fmt.Errorf(limit.fmt, limit.v, limit.allowed) + } + if limit.durs { + return fmt.Errorf("%s %v is %s than allowed %v", limit.name, time.Duration(limit.v), cmp, time.Duration(limit.allowed)) + } + return fmt.Errorf("%s %v is %s than allowed %v", limit.name, limit.v, cmp, limit.allowed) + } + } + + if cfg.defaultProduceTopicAlways && cfg.defaultProduceTopic == "" { + return errors.New("invalid empty DefaultProduceTopic when using DefaultProduceTopicAlways") + } + + if cfg.dialFn != nil { + if cfg.dialTLS != nil { + return errors.New("cannot set both Dialer and DialTLSConfig") + } + } + + if len(cfg.group) > 0 { + if len(cfg.partitions) != 0 { + return errors.New("invalid direct-partition consuming option when consuming as a group") + } + } + + if len(cfg.shareGroup) > 0 { + if len(cfg.group) > 0 { + return errors.New("cannot use both ConsumerGroup and ShareGroup") + } + if len(cfg.partitions) != 0 { + return errors.New("invalid direct-partition consuming option when consuming as a share group") + } + if cfg.autocommitGreedy || cfg.autocommitMarks || cfg.autocommitDisable || cfg.commitCallback != nil { + return errors.New("autocommit options are not applicable to share groups") + } + if cfg.onLost != nil || cfg.onRevoked != nil || cfg.onAssigned != nil { + return errors.New("partition lifecycle callbacks are not supported with share groups") + } + if cfg.shareMaxRecords < -1 || cfg.shareMaxRecords == 0 { + return errors.New("ShareMaxRecords must be positive") + } + // In record-limit mode (ShareMaxRecordsStrict), restrict to + // a single broker per poll round to respect the record limit. + if cfg.shareMaxRecordsStrict && cfg.maxConcurrentFetches == -1 { + cfg.maxConcurrentFetches = 0 + } + } else if cfg.shareMaxRecords != -1 || cfg.shareMaxRecordsStrict || cfg.shareAckCallback != nil { + return errors.New("ShareMaxRecords, ShareMaxRecordsStrict, and ShareAckCallback only apply when using ShareGroup") + } + + if cfg.regex { + if len(cfg.partitions) != 0 { + return errors.New("invalid direct-partition consuming option when consuming as regex") + } + for re := range cfg.topics { + compiled, err := regexp.Compile(re) + if err != nil { + return fmt.Errorf("invalid regular expression %q", re) + } + cfg.topics[re] = compiled + } + for re := range cfg.excludeTopics { + compiled, err := regexp.Compile(re) + if err != nil { + return fmt.Errorf("invalid regular expression %q", re) + } + cfg.excludeTopics[re] = compiled + } + } else if len(cfg.excludeTopics) > 0 { + return errors.New("invalid use of ConsumeExcludeTopics when not using ConsumeRegex") + } + + if cfg.topics != nil && cfg.partitions != nil { + for topic := range cfg.partitions { + if _, exists := cfg.topics[topic]; exists { + return fmt.Errorf("topic %q seen in both ConsumePartitions and ConsumeTopics; these options are a union, it is invalid to specify specific partitions for a topic while also consuming the entire topic", topic) + } + } + } + + if cfg.autocommitDisable && cfg.autocommitGreedy { + return errors.New("cannot both disable autocommitting and enable greedy autocommitting") + } + if cfg.autocommitDisable && cfg.autocommitMarks { + return errors.New("cannot both disable autocommitting and enable marked autocommitting") + } + if cfg.autocommitGreedy && cfg.autocommitMarks { + return errors.New("cannot enable both greedy autocommitting and marked autocommitting") + } + if (cfg.autocommitGreedy || cfg.autocommitDisable || cfg.autocommitMarks || cfg.commitCallback != nil) && len(cfg.group) == 0 && len(cfg.shareGroup) == 0 { + return errors.New("invalid autocommit options specified when a group was not specified") + } + if (cfg.onLost != nil || cfg.onRevoked != nil || cfg.onAssigned != nil) && len(cfg.group) == 0 && len(cfg.shareGroup) == 0 { + return errors.New("invalid group partition assigned/revoked/lost functions set when a group was not specified") + } + + processedHooks, err := processHooks(cfg.hooks) + if err != nil { + return err + } + cfg.hooks = processedHooks + + processedPools, err := processPools(cfg.pools) + if err != nil { + return err + } + cfg.pools = processedPools + + return nil +} + +// processHooks will inspect and recursively unpack slices of hooks stopping +// if the instance implements any hook interface. It will return an error on +// the first instance that implements no hook interface +func processHooks(hooks []Hook) ([]Hook, error) { + var processedHooks []Hook + for _, hook := range hooks { + if implementsAnyHook(hook) { + processedHooks = append(processedHooks, hook) + } else if moreHooks, ok := hook.([]Hook); ok { + more, err := processHooks(moreHooks) + if err != nil { + return nil, err + } + processedHooks = append(processedHooks, more...) + } else { + return nil, errors.New("found an argument that implements no hook interfaces") + } + } + return processedHooks, nil +} + +// Same as the above, but for pools. +func processPools(pools []Pool) ([]Pool, error) { + var processedPools []Pool + for _, pool := range pools { + if implementsAnyPool(pool) { + processedPools = append(processedPools, pool) + } else if morePools, ok := pool.([]Pool); ok { + more, err := processPools(morePools) + if err != nil { + return nil, err + } + processedPools = append(processedPools, more...) + } else { + return nil, errors.New("found an argument that implements no pool interfaces") + } + } + return processedPools, nil +} + +var reVersion = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9.-]*[a-zA-Z0-9])?$`) + +func softwareVersion() string { + info, ok := debug.ReadBuildInfo() + if ok { + for _, dep := range info.Deps { + if dep.Path == "github.com/twmb/franz-go" { + if reVersion.MatchString(dep.Version) { + return dep.Version + } + } + } + } + return "unknown" +} + +func defaultCfg() cfg { + defaultID := "kgo" + return cfg{ + ///////////// + // general // + ///////////// + id: &defaultID, + + dialTimeout: 10 * time.Second, + requestTimeoutOverhead: 10 * time.Second, + connIdleTimeout: 30 * time.Second, + + softwareName: "kgo", + softwareVersion: softwareVersion(), + + logger: new(nopLogger), + + seedBrokers: []string{"127.0.0.1"}, + maxVersions: kversion.Stable(), // kversion bumps what is returned from Stable on the same release we add support for new features to kgo + + retryBackoff: func() func(int) time.Duration { + var rngMu xsync.Mutex + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + return func(fails int) time.Duration { + const ( + min = 250 * time.Millisecond + max = 5 * time.Second + ) + if fails <= 0 { + return min + } + if fails > 10 { + return max + } + + backoff := min * time.Duration(1<<(fails-1)) + + rngMu.Lock() + jitter := 0.8 + 0.4*rng.Float64() + rngMu.Unlock() + + backoff = time.Duration(float64(backoff) * jitter) + + if backoff > max { + return max + } + return backoff + } + }(), + retries: 20, + + maxBrokerWriteBytes: 100 << 20, // Kafka socket.request.max.bytes default is 100<<20 + maxBrokerReadBytes: 100 << 20, + + metadataMaxAge: 5 * time.Minute, + metadataMinAge: 5 * time.Second, + missingTopicDelete: 15 * time.Second, + + ////////////// + // producer // + ////////////// + + txnTimeout: 40 * time.Second, + acks: AllISRAcks(), + maxProduceInflight: 1, + compression: []CompressionCodec{SnappyCompression(), NoCompression()}, + maxRecordBatchBytes: func(string) int32 { return 1000012 }, // Kafka max.message.bytes default is 1000012 + maxBufferedRecords: 10000, + produceTimeout: 10 * time.Second, + recordRetries: math.MaxInt64, // effectively unbounded + maxUnknownFailures: 4, + linger: 10 * time.Millisecond, + partitioner: UniformBytesPartitioner(64<<10, true, true, nil), + txnBackoff: 20 * time.Millisecond, + + ////////////// + // consumer // + ////////////// + + maxWait: math.MinInt32, // sigil: resolved by validate based on shareGroup + minBytes: 1, + maxBytes: 50 << 20, + maxPartBytes: 1 << 20, + startOffset: NewOffset().AtStart(), + resetOffset: NewOffset().AtStart(), + isolationLevel: 0, + + maxConcurrentFetches: -1, // unbounded default + + recheckPreferredReplicaInterval: 30 * time.Minute, + + /////////// + // group // + /////////// + + balancers: []GroupBalancer{ + CooperativeStickyBalancer(), + }, + protocol: "consumer", + + sessionTimeout: 45000 * time.Millisecond, + rebalanceTimeout: 60000 * time.Millisecond, + heartbeatInterval: 3000 * time.Millisecond, + + autocommitInterval: 5 * time.Second, + + shareMaxRecords: -1, // sigil: not set, ShareFetch sends 500 + } +} + +////////////////////////// +// CLIENT CONFIGURATION // +////////////////////////// + +// ClientID uses id for all requests sent to Kafka brokers, overriding the +// default "kgo". +func ClientID(id string) Opt { + return clientOpt{func(cfg *cfg) { cfg.id = &id }} +} + +// SoftwareNameAndVersion sets the client software name and version that will +// be sent to Kafka as part of the ApiVersions request as of Kafka 2.4, +// overriding the default "kgo" and internal version number. +// +// Kafka exposes this through metrics to help operators understand the impact +// of clients. +// +// It is generally not recommended to set this. As well, if you do, the name +// and version must match the following regular expression: +// +// [a-zA-Z0-9](?:[a-zA-Z0-9\.-]*[a-zA-Z0-9])? +// +// Note this means neither the name nor version can be empty. +func SoftwareNameAndVersion(name, version string) Opt { + return clientOpt{func(cfg *cfg) { cfg.softwareName = name; cfg.softwareVersion = version }} +} + +// WithLogger sets the client to use the given logger, overriding the default +// to not use a logger. +// +// It is invalid to use a nil logger; doing so will cause panics. +func WithLogger(l Logger) Opt { + return clientOpt{func(cfg *cfg) { cfg.logger = &wrappedLogger{l} }} +} + +// WithContext sets the client to use a custom context. +// +// By default, the client uses context.Background. +func WithContext(ctx context.Context) Opt { + return clientOpt{func(cfg *cfg) { cfg.ctx = ctx }} +} + +// RequestTimeoutOverhead uses the given time as overhead while deadlining +// requests, overriding the default overhead of 10s. +// +// For most requests, the timeout is set to the overhead. However, for +// any request with a TimeoutMillis field, the overhead is added on top of the +// request's TimeoutMillis. This ensures that we give Kafka enough time to +// actually process the request given the timeout, while still having a +// deadline on the connection as a whole to ensure it does not hang. +// +// For writes, the timeout is always the overhead. We buffer writes in our +// client before one quick flush, so we always expect the write to be fast. +// +// Note that hitting the timeout kills a connection, which will fail any other +// active writes or reads on the connection. +// +// This option is roughly equivalent to request.timeout.ms, but grants +// additional time to requests that have timeout fields. +func RequestTimeoutOverhead(overhead time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.requestTimeoutOverhead = overhead }} +} + +// ConnIdleTimeout sets the amount of time after which an idle connection will +// not be reused, overriding the default 30s. The connection may be closed after +// this timeout. +// +// In the worst case, a connection can be allowed to idle for up to 2x this +// time before being closed. +// +// It is possible that a connection can be reaped just as it is about to be +// written to, but the client internally retries in these cases. +// +// Connections are not reaped if they are actively being written to or read +// from; thus, a request can take a really long time itself and not be reaped +// (however, this may lead to the RequestTimeoutOverhead). +func ConnIdleTimeout(timeout time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.connIdleTimeout = timeout }} +} + +// Dialer uses fn to dial addresses, overriding the default dialer that uses a +// 10s dial timeout and no TLS. +// +// The context passed to the dial function is the context used in the request +// that caused the dial. If the request is a client-internal request, the +// context is the context on the client itself (which is canceled when the +// client is closed). +// +// This function has the same signature as net.Dialer's DialContext and +// tls.Dialer's DialContext, meaning you can use this function like so: +// +// kgo.Dialer((&net.Dialer{Timeout: 10*time.Second}).DialContext) +// +// or +// +// kgo.Dialer((&tls.Dialer{...}).DialContext) +func Dialer(fn func(ctx context.Context, network, host string) (net.Conn, error)) Opt { + return clientOpt{func(cfg *cfg) { cfg.dialFn = fn }} +} + +// DialTimeout sets the dial timeout, overriding the default of 10s. This +// option is useful if you do not want to set a custom dialer, and is useful in +// tandem with DialTLSConfig. +func DialTimeout(timeout time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.dialTimeout = timeout }} +} + +// DialTLSConfig opts into dialing brokers with the given TLS config with a +// 10s dial timeout. This is a shortcut for manually specifying a tls dialer +// using the Dialer option. You can also change the default 10s timeout with +// DialTimeout. +// +// Every dial, the input config is cloned. If the config's ServerName is not +// specified, this function uses net.SplitHostPort to extract the host from the +// broker being dialed and sets the ServerName. In short, it is not necessary +// to set the ServerName. +func DialTLSConfig(c *tls.Config) Opt { + return clientOpt{func(cfg *cfg) { cfg.dialTLS = c }} +} + +// DialTLS opts into dialing brokers with TLS. This is a shortcut for +// DialTLSConfig with an empty config. See DialTLSConfig for more details. +func DialTLS() Opt { + return DialTLSConfig(new(tls.Config)) +} + +// SeedBrokers sets the seed brokers for the client to use, overriding the +// default 127.0.0.1:9092. +// +// Any seeds that are missing a port use the default Kafka port 9092. +func SeedBrokers(seeds ...string) Opt { + return clientOpt{func(cfg *cfg) { cfg.seedBrokers = append(cfg.seedBrokers[:0], seeds...) }} +} + +// MaxVersions sets the maximum Kafka version to try, overriding the +// internal unbounded (latest stable) versions. +// +// Note that specific max version pinning is required if trying to interact +// with versions pre 0.10.0. Otherwise, unless using more complicated requests +// that this client itself does not natively use, it is generally safe to opt +// for the latest version. If using the kmsg package directly to issue +// requests, it is recommended to pin versions so that new fields on requests +// do not get invalid default zero values before you update your usage. +func MaxVersions(versions *kversion.Versions) Opt { + return clientOpt{func(cfg *cfg) { cfg.maxVersions = versions }} +} + +// MinVersions sets the minimum Kafka version a request can be downgraded to, +// overriding the default of the lowest version. +// +// This option is useful if you are issuing requests that you absolutely do not +// want to be downgraded; that is, if you are relying on features in newer +// requests, and you are not sure if your brokers can handle those features. +// By setting a min version, if the client detects it needs to downgrade past +// the version, it will instead avoid issuing the request. +// +// Unlike MaxVersions, if a request is issued that is unknown to the min +// versions, the request is allowed. It is assumed that there is no lower bound +// for that request. +func MinVersions(versions *kversion.Versions) Opt { + return clientOpt{func(cfg *cfg) { cfg.minVersions = versions }} +} + +// RetryBackoffFn sets the backoff strategy for how long to backoff for a given +// amount of retries, overriding the default jittery exponential backoff that +// ranges from 250ms min to 2.5s max. +// +// This (roughly) corresponds to Kafka's retry.backoff.ms setting and +// retry.backoff.max.ms (which is being introduced with KIP-500). +func RetryBackoffFn(backoff func(int) time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.retryBackoff = backoff }} +} + +// RequestRetries sets the number of tries that retryable requests are allowed, +// overriding the default of 20. +// +// This option does not apply to produce requests; to limit produce request +// retries / record retries, see RecordRetries. +func RequestRetries(n int) Opt { + return clientOpt{func(cfg *cfg) { cfg.retries = int64(n) }} +} + +// RetryTimeout sets the upper limit on how long we allow a request to be +// issued and then reissued on failure. That is, this control the total +// end-to-end maximum time we allow for trying a request, This overrides the +// default of: +// +// JoinGroup: cfg.SessionTimeout (default 45s) +// SyncGroup: cfg.SessionTimeout (default 45s) +// Heartbeat: cfg.SessionTimeout (default 45s) +// others: 30s +// +// This timeout applies to any request issued through a client's Request +// function. It does not apply to fetches nor produces. +// +// A value of zero indicates no request timeout. +// +// The timeout is evaluated after a request errors. If the time since the start +// of the first request plus any backoff for the latest failure is less than +// the retry timeout, the request will be issued again. +func RetryTimeout(t time.Duration) Opt { + return RetryTimeoutFn(func(int16) time.Duration { return t }) +} + +// RetryTimeoutFn sets the upper limit on how long we allow a request to be +// issued and then reissued on failure. That is, this control the total +// end-to-end maximum time we allow for trying a request, This overrides the +// default of: +// +// JoinGroup: cfg.SessionTimeout (default 45s) +// SyncGroup: cfg.SessionTimeout (default 45s) +// Heartbeat: cfg.SessionTimeout (default 45s) +// others: 30s +// +// This timeout applies to any request issued through a client's Request +// function. It does not apply to fetches nor produces. +// +// The function is called with the request key that is being retried. While it +// is not expected that the request key will be used, including it gives users +// the opportunity to have different retry timeouts for different keys. +// +// If the function returns zero, there is no retry timeout. +// +// The timeout is evaluated after a request errors. If the time since the start +// of the first request plus any backoff for the latest failure is less than +// the retry timeout, the request will be issued again. +func RetryTimeoutFn(t func(int16) time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.retryTimeout = t }} +} + +// AllowAutoTopicCreation enables topics to be auto created if they do +// not exist when fetching their metadata. +func AllowAutoTopicCreation() Opt { + return clientOpt{func(cfg *cfg) { cfg.allowAutoTopicCreation = true }} +} + +// BrokerMaxWriteBytes upper bounds the number of bytes written to a broker +// connection in a single write, overriding the default 100MiB. +// +// This number corresponds to the broker's socket.request.max.bytes, which +// defaults to 100MiB. +// +// The only Kafka request that could come reasonable close to hitting this +// limit should be produce requests, and thus this limit is only enforced for +// produce requests. +func BrokerMaxWriteBytes(v int32) Opt { + return clientOpt{func(cfg *cfg) { cfg.maxBrokerWriteBytes = v }} +} + +// BrokerMaxReadBytes sets the maximum response size that can be read from +// Kafka, overriding the default 100MiB. +// +// This is a safety measure to avoid OOMing on invalid responses. This is +// slightly double FetchMaxBytes; if bumping that, consider bump this. No other +// response should run the risk of hitting this limit. +func BrokerMaxReadBytes(v int32) Opt { + return clientOpt{func(cfg *cfg) { cfg.maxBrokerReadBytes = v }} +} + +// MetadataMaxAge sets the maximum age for the client's cached metadata, +// overriding the default 5m, to allow detection of new topics, partitions, +// etc. +// +// This corresponds to Kafka's metadata.max.age.ms. +func MetadataMaxAge(age time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.metadataMaxAge = age }} +} + +// MetadataMinAge sets the minimum time between metadata queries, overriding +// the default 5s. You may want to raise or lower this to reduce the number of +// metadata queries the client will make. Notably, if metadata detects an error +// in any topic or partition, it triggers itself to update as soon as allowed. +func MetadataMinAge(age time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.metadataMinAge = age }} +} + +// SASL appends sasl authentication options to use for all connections. +// +// SASL is tried in order; if the broker supports the first mechanism, all +// connections will use that mechanism. If the first mechanism fails, the +// client will pick the first supported mechanism. If the broker does not +// support any client mechanisms, connections will fail. +func SASL(sasls ...sasl.Mechanism) Opt { + return clientOpt{func(cfg *cfg) { cfg.sasls = append(cfg.sasls, sasls...) }} +} + +// WithHooks sets hooks to call whenever relevant. +// +// Hooks can be used to layer in metrics (such as Prometheus hooks) or anything +// else. The client will call all hooks in order. See the Hooks interface for +// more information, as well as any interface that contains "Hook" in the name +// to know the available hooks. A single hook can implement any or all hook +// interfaces, and only the hooks that it implements will be called. +func WithHooks(hooks ...Hook) Opt { + return clientOpt{func(cfg *cfg) { cfg.hooks = append(cfg.hooks, hooks...) }} +} + +// WithPools sets memory pools to use wherever relevant. +// +// Pools can be used to optimize memory usage for data that is frequently +// thrown away after a short usage. For a list of all supported pools, look at +// the documentation for any interface that begins with "Pool". Multiple pools +// may be used; the first pool that is received from is the first pull put back +// into. A single pool can implement any or all pool interfaces. +// +// If you use pools for fetching, the record Context field will be populated. +// This field is used for recycling the underlying memory once Recycle is +// called; do not clear the field. +func WithPools(pools ...Pool) Opt { + return clientOpt{func(cfg *cfg) { cfg.pools = append(cfg.pools, pools...) }} +} + +// ConcurrentTransactionsBackoff sets the backoff interval to use during +// transactional requests in case we encounter CONCURRENT_TRANSACTIONS error, +// overriding the default 20ms. +// +// Sometimes, when a client begins a transaction quickly enough after finishing +// a previous one, Kafka will return a CONCURRENT_TRANSACTIONS error. Clients +// are expected to backoff slightly and retry the operation. Lower backoffs may +// increase load on the brokers, while higher backoffs may increase transaction +// latency in clients. +// +// Note that if brokers are hanging in this concurrent transactions state for +// too long, the client progressively increases the backoff. +func ConcurrentTransactionsBackoff(backoff time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.txnBackoff = backoff }} +} + +// ConsiderMissingTopicDeletedAfter sets the amount of time a topic can be +// missing from metadata responses _after_ loading it at least once before it +// is considered deleted, overriding the default of 15s. Note that for newer +// versions of Kafka, it may take a bit of time (~15s) for the cluster to fully +// recognize a newly created topic. If this option is set too low, there is +// some risk that the client will internally purge and re-see a topic a few +// times until the cluster fully broadcasts the topic creation. +func ConsiderMissingTopicDeletedAfter(t time.Duration) Opt { + return clientOpt{func(cfg *cfg) { cfg.missingTopicDelete = t }} +} + +// OnRebootstrapRequired sets the function to call when a metadata response has +// the REBOOTSTRAP_REQUIRED errored. The function should return new seed +// brokers for the client to use, or an error. Internally, the client will then +// call UpdateSeedBrokers with the seeds you return. All other live connections +// to brokers are stopped and active requests are failed. +// +// The REBOOTSTRAP_REQUIRED error was introduced in Kafka 4.0, as a way for +// Kafka to tell the client that the client needs to stop all non seed broker +// connections to stop and for the client to query the seed brokers again. +// Franz-go by default already periodically sends a request to a seed broker +// to prevent a scenario where all previously discovered brokers are down +// or unavailable, so this client does not have as much of a need for +// REBOOTSTRAP_REQUIRED. That said, this function can be useful if Kafka knows +// the client should specifically talk to seed brokers next, and this function +// allows you a chance to update your seed brokers at the same time. If you +// do not want to update your seed brokers, you can just return the same value +// that you use in your [SeedBrokers] configuration option. +// +// You can read KIP-1102 for more info about this option. +func OnRebootstrapRequired(fn func() ([]string, error)) Opt { + return clientOpt{func(cfg *cfg) { cfg.onRebootstrapRequired = fn }} +} + +// DisableClientMetrics opts out of collecting and sending client metrics to +// the broker (if the broker supports receiving client metrics). By default, +// clients are recommended to gather a small set of metrics to help cluster +// operators debug client issues (rather than relying on clients which may not +// be instrumented at all). +// +// For more details on client metrics, see KIP-714. +func DisableClientMetrics() Opt { + return clientOpt{func(cfg *cfg) { cfg.disableClientMetrics = true }} +} + +// UserMetricsFn sets the function to call to add user metrics when rolling up +// client metrics to send to the broker. Every metric rollup, fn is called and +// returns an iterator. All metrics returned from the iterator are included in +// the client metric aggregation and are sent to the broker. It is your +// responsibility to ensure the metric name is formatted correctly (namespaced +// and following OpenTelemetry format), and you need to ensure your Sum metrics +// are monotonically increasing. See the documentation on [Metric] for more +// details. +// +// For more details about the client sending metrics, see KIP-714. For more +// details about enhancing client metrics with user metrics, see KIP-1076. +func UserMetricsFn(fn func() iter.Seq[Metric]) Opt { + return clientOpt{func(cfg *cfg) { cfg.userMetrics = fn }} +} + +// AlwaysRetryEOF switches the client to *always* retry EOF. +// +// By default, if an EOF is experienced on the FIRST request being written to +// or read from a connection, the client does not retry on the error. EOFs are +// encountered for many reasons, and the client has no information available as +// to what exact reason the EOF was encountered. If your configuration is +// correct, then EOF is usually experienced during timeouts, or when you have +// some high load systems and connections are being cut for some reason, etc. +// If your configuration is incorrect (on the client or on the broker), EOF can +// be experienced due to some TLS settings mismatch or missing SASL +// credentials, and it's very hard to debug EOF in this case. Thus, after much +// feedback, the client was changed to assume an EOF experienced immediately +// means invalid configuration. If you *know* your configuration is correct, +// this option opts into always retrying EOF, allowing requests to retry and +// succeed as they normally should on very busy systems. +func AlwaysRetryEOF() Opt { + return clientOpt{func(cfg *cfg) { cfg.alwaysRetryEOF = true }} +} + +//////////////////////////// +// PRODUCER CONFIGURATION // +//////////////////////////// + +// DefaultProduceTopic sets the default topic to produce to if the topic field +// is empty in a Record. +// +// If this option is not used, if a record has an empty topic, the record +// cannot be produced and will be failed immediately. +func DefaultProduceTopic(t string) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.defaultProduceTopic = t }} +} + +// DefaultProduceTopicAlways sets the client to ALWAYS produce to the +// [DefaultProduceTopic], overriding any Topic field that may be present +// in the Record when producing. +func DefaultProduceTopicAlways() ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.defaultProduceTopicAlways = true }} +} + +// Acks represents the number of acks a broker leader must have before +// a produce request is considered complete. +// +// This controls the durability of written records and corresponds to "acks" in +// Kafka's Producer Configuration documentation. +// +// The default is LeaderAck. +type Acks struct { + val int16 +} + +// NoAck considers records sent as soon as they are written on the wire. +// The leader does not reply to records. +func NoAck() Acks { return Acks{0} } + +// LeaderAck causes Kafka to reply that a record is written after only +// the leader has written a message. The leader does not wait for in-sync +// replica replies. +func LeaderAck() Acks { return Acks{1} } + +// AllISRAcks ensures that all in-sync replicas have acknowledged they +// wrote a record before the leader replies success. +func AllISRAcks() Acks { return Acks{-1} } + +// RequiredAcks sets the required acks for produced records, +// overriding the default RequireAllISRAcks. +func RequiredAcks(acks Acks) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.acks = acks }} +} + +// DisableIdempotentWrite disables idempotent produce requests, opting out of +// Kafka server-side deduplication in the face of reissued requests due to +// transient network problems. Disabling idempotent write by default +// upper-bounds the number of in-flight produce requests per broker to 1, vs. +// the default of 5 when using idempotency. +// +// Idempotent production is strictly a win, but does require the +// IDEMPOTENT_WRITE permission on CLUSTER (pre Kafka 3.0), and not all clients +// can have that permission. +// +// If the goal is to allow cancellation of in-flight records while keeping +// idempotent deduplication, see [AllowIdempotentProduceCancellation] instead. +// +// This option is incompatible with specifying a transactional id. +func DisableIdempotentWrite() ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.disableIdempotency = true }} +} + +// AllowIdempotentProduceCancellation permits cancellation of in-flight +// idempotent records, at the cost of breaking idempotency's +// duplicate-avoidance guarantee. +// +// When a record is in-flight, the client cannot tell "the broker never +// received it" from "the broker wrote it but the reply was lost". +// Cancelling at this point leaves the client's idempotent sequence +// window inconsistent with the broker: once cancelled records are +// failed to the user, the next produce either silently gap-accepts (if +// the broker wrote them) or hits OUT_OF_ORDER_SEQUENCE and forces the +// client to reload its producer ID (new epoch, reset sequence). A +// subsequent application-level retry of the cancelled record races +// against what the broker may already have stored - the broker cannot +// dedupe it, and you can get duplicates. By default, the client refuses +// to cancel in this state and instead waits for the record's outcome +// so idempotency holds. +// +// With this option, context cancellation, RecordDeliveryTimeout, and +// RecordRetries exhaustion are allowed to fail in-flight records. +// Idempotent dedupe continues to protect the client's own internal +// retry path, but any cancelled record that the application re-produces +// may land on the broker twice. +// +// Use this when time-bounded delivery matters more than +// duplicate-avoidance. +// +// This option is incompatible with specifying a transactional id. +func AllowIdempotentProduceCancellation() ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.allowIdempotentProduceCancellation = true }} +} + +// MaxProduceRequestsInflightPerBroker changes the number of allowed produce +// requests in flight per broker if you disable idempotency, overriding the +// default of 1. If using idempotency, this option has no effect: the maximum +// in flight for Kafka v0.11 is 1, and from v1 onward is 5. +// +// Using more than 1 may result in out of order records and may result in +// duplicates if there are connection issues. +func MaxProduceRequestsInflightPerBroker(n int) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.maxProduceInflight = n }} +} + +// ProducerBatchCompression sets the compression codec to use for producing +// records. +// +// Compression is chosen in the order preferred based on broker support. For +// example, zstd compression was introduced in Kafka 2.1, so the preference can +// be first zstd, fallback snappy, fallback none. +// +// The default preference is [snappy, none], which should be fine for all old +// consumers since snappy compression has existed since Kafka 0.8.0. To use +// zstd, your brokers must be at least 2.1 and all consumers must be upgraded +// to support decoding zstd records. +// +// Alternatively, if you want finer control over compression you can use +// [WithCompressor] for complete control. +func ProducerBatchCompression(preference ...CompressionCodec) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.compression = preference }} +} + +// WithCompressor allows you to completely control how produce batches are +// compressed, allowing you to use alternative libraries than what franz-go +// supports, allowing you to have more control over memory & pooling, and +// other benefits. It is recommended to just use [ProducerBatchCompression] +// for simplicity (or specify nothing, which opts into snappy by default). +// The client default compressor is the [DefaultCompressor]. +func WithCompressor(compressor Compressor) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.compressor = compressor }} +} + +// ProducerBatchMaxBytes upper bounds the size of a record batch, overriding +// the default 1,000,012 bytes. This mirrors Kafka's max.message.bytes. +// +// Record batches are independent of a ProduceRequest: a record batch is +// specific to a topic and partition, whereas the produce request can contain +// many record batches for many topics. +// +// If a single record encodes larger than this number (before compression), it +// will not be written and a callback will have the appropriate error. +// +// Note that this is the maximum size of a record batch before compression. If +// a batch compresses poorly and actually grows the batch, the uncompressed +// form will be used. +// +// For per-topic control, see [ProducerBatchMaxBytesFn]. +func ProducerBatchMaxBytes(v int32) ProducerOpt { + return ProducerBatchMaxBytesFn(func(string) int32 { return v }) +} + +// ProducerBatchMaxBytesFn is the functional form of [ProducerBatchMaxBytes]: +// it returns the per-batch byte cap for the given topic, overriding the +// default 1,000,012 bytes. This is useful when different topics have +// different max.message.bytes configurations on the broker. +// +// The function is called once when a partition is first discovered and the +// result is cached on the per-partition record buffer - it is NOT consulted +// per record or per batch, and topic-config changes at runtime are not +// picked up until partition state is rebuilt. Return a value in +// [512, 1<<30]; returning a value outside that range for a specific topic +// will cause produces to that topic to fail (config validation only checks +// the return for the empty-topic at client construction). +// +// For a static limit applied to all topics, see [ProducerBatchMaxBytes]. +func ProducerBatchMaxBytesFn(fn func(string) int32) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.maxRecordBatchBytes = fn }} +} + +// MaxBufferedRecords sets the max amount of records the client will buffer, +// blocking produces until records are finished if this limit is reached. +// This overrides the default of 10,000. +func MaxBufferedRecords(n int) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.maxBufferedRecords = int64(n) }} +} + +// MaxBufferedBytes sets the max amount of bytes that the client will buffer +// while producing, blocking produces until records are finished if this limit +// is reached. This overrides the unlimited default. +// +// Note that this option does _not_ apply for consuming: the client cannot +// limit bytes buffered for consuming because of decompression. You can roughly +// control consuming memory by using [MaxConcurrentFetches], [FetchMaxBytes], +// and [FetchMaxPartitionBytes]. +// +// If you produce a record that is larger than n, the record is immediately +// failed with kerr.MessageTooLarge. +// +// Note that this limit applies after [MaxBufferedRecords]. +func MaxBufferedBytes(n int) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.maxBufferedBytes = int64(n) }} +} + +// RecordPartitioner uses the given partitioner to partition records, overriding +// the default UniformBytesPartitioner(64KiB, true, true, nil). +func RecordPartitioner(partitioner Partitioner) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.partitioner = partitioner }} +} + +// ProduceRequestTimeout sets how long Kafka broker's are allowed to respond to +// produce requests, overriding the default 10s. If a broker exceeds this +// duration, it will reply with a request timeout error. +// +// This somewhat corresponds to Kafka's request.timeout.ms setting, but only +// applies to produce requests. This settings sets the TimeoutMillis field in +// the produce request itself. The RequestTimeoutOverhead is applied as a write +// limit and read limit in addition to this. +func ProduceRequestTimeout(limit time.Duration) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.produceTimeout = limit }} +} + +// RecordRetries sets the number of tries for producing records, overriding the +// unlimited default. +// +// If idempotency is enabled (as it is by default), this option is only +// enforced if it is safe to do so without creating invalid sequence numbers. +// It is safe to enforce if a record was never issued in a request to Kafka, or +// if it was requested and received a response. +// +// If a record fails due to retries, all records buffered in the same partition +// are failed as well. This ensures gapless ordering: the client will not fail +// one record only to produce a later one successfully. This also allows for +// easier sequence number ordering internally. +// +// If a topic repeatedly fails to load with UNKNOWN_TOPIC_OR_PARTITION, it has +// a different limit (the UnknownTopicRetries option). All records for a topic +// that repeatedly cannot be loaded are failed when that limit is hit. +// +// This option is different from RequestRetries to allow finer grained control +// of when to fail when producing records. +func RecordRetries(n int) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.recordRetries = int64(n) }} +} + +// UnknownTopicRetries sets the number of times a record can fail with +// UNKNOWN_TOPIC_OR_PARTITION, overriding the default 4. +// +// This is a separate limit from RecordRetries because unknown topic or +// partition errors should only happen if the topic does not exist. It is +// pointless for the client to continue producing to a topic that does not +// exist, and if we repeatedly see that the topic does not exist across +// multiple metadata queries (which are going to different brokers), then we +// may as well stop trying and fail the records. +// +// If this is -1, the client never fails records with this error. +func UnknownTopicRetries(n int) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.maxUnknownFailures = int64(n) }} +} + +// StopProducerOnDataLossDetected sets the client to stop producing if data +// loss is detected, overriding the default false. +// +// Note that if using this option, it is strongly recommended to not have a +// retry limit. Doing so may lead to errors where the client fails a batch on a +// recoverable error, which internally bumps the idempotent sequence number +// used for producing, which may then later cause an inadvertent out of order +// sequence number and false "data loss" detection. +func StopProducerOnDataLossDetected() ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.stopOnDataLoss = true }} +} + +// ProducerOnDataLossDetected sets a function to call if data loss is detected +// when producing records if the client is configured to continue on data loss. +// Thus, this option is mutually exclusive with StopProducerOnDataLossDetected. +// +// The passed function will be called with the topic and partition that data +// loss was detected on. +func ProducerOnDataLossDetected(fn func(string, int32)) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.onDataLoss = fn }} +} + +// ProducerLinger sets how long individual topic partitions will linger waiting +// for more records before triggering a request to be built. +// +// If a produce request is triggered by any topic partition, all partitions +// with a possible batch to be sent are used and all lingers are reset. +func ProducerLinger(linger time.Duration) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.linger = linger }} +} + +// ManualFlushing disables auto-flushing when producing. While you can still +// set lingering, it would be useless to do so. +// +// With manual flushing, producing while MaxBufferedRecords or MaxBufferedBytes +// have already been produced and not flushed will return ErrMaxBuffered. +func ManualFlushing() ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.manualFlushing = true }} +} + +// RecordDeliveryTimeout sets a rough time of how long a record can sit around +// in a batch before timing out, overriding the unlimited default. +// +// If idempotency is enabled (as it is by default), this option is only +// enforced if it is safe to do so without creating invalid sequence numbers. +// It is safe to enforce if a record was never issued in a request to Kafka, or +// if it was requested and received a response. +// +// The timeout for all records in a batch inherit the timeout of the first +// record in that batch. That is, once the first record's timeout expires, all +// records in the batch are expired. This generally is a non-issue unless using +// this option with lingering. In that case, simply add the linger to the +// record timeout to avoid problems. +// +// If a record times out, all records buffered in the same partition are failed +// as well. This ensures gapless ordering: the client will not fail one record +// only to produce a later one successfully. This also allows for easier +// sequence number ordering internally. +// +// The timeout is only evaluated before writing a request or after a +// produce response. Thus, a sink backoff may delay record timeout slightly. +// +// This option is roughly equivalent to delivery.timeout.ms. +func RecordDeliveryTimeout(timeout time.Duration) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.recordTimeout = timeout }} +} + +// TransactionalID sets a transactional ID for the client, ensuring that +// records are produced transactionally under this ID (exactly once semantics). +// +// For Kafka-to-Kafka transactions, the transactional ID is only one half of +// the equation. You must also assign a group to consume from. +// +// To produce transactionally, you first [BeginTransaction], then produce records +// consumed from a group, then you [EndTransaction]. All records produced outside +// of a transaction will fail immediately with an error. +// +// After producing a batch, you must commit what you consumed. Auto committing +// offsets is disabled during transactional consuming / producing. +// +// Note that unless using Kafka 2.5, a consumer group rebalance may be +// problematic. Production should finish and be committed before the client +// rejoins the group. It may be safer to use an eager group balancer and just +// abort the transaction. Alternatively, any time a partition is revoked, you +// could abort the transaction and reset offsets being consumed. +// +// If the client detects an unrecoverable error, all records produced +// thereafter will fail. +// +// Lastly, the default read level is READ_UNCOMMITTED. Be sure to use the +// ReadIsolationLevel option if you want to only read committed. +func TransactionalID(id string) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.txnID = &id }} +} + +// TransactionTimeout sets the allowed for a transaction, overriding the +// default 40s. It is a good idea to keep this less than a group's session +// timeout, so that a group member will always be alive for the duration of a +// transaction even if connectivity dies. This helps prevent a transaction +// finishing after a rebalance, which is problematic pre-Kafka 2.5. If you +// are on Kafka 2.5+, then you can use the RequireStableFetchOffsets option +// when assigning the group, and you can set this to whatever you would like. +// +// Transaction timeouts begin when the first record is produced within a +// transaction, not when a transaction begins. +func TransactionTimeout(timeout time.Duration) ProducerOpt { + return producerOpt{func(cfg *cfg) { cfg.txnTimeout = timeout }} +} + +//////////////////////////// +// CONSUMER CONFIGURATION // +//////////////////////////// + +// FetchMaxWait sets the maximum amount of time a broker will wait for a +// fetch response to hit the minimum number of required bytes before returning, +// overriding the default 5s (or 500ms when [ShareGroup] is set). +// +// This corresponds to the Java fetch.max.wait.ms setting. +// +// For share consumers, values above 500ms are not recommended: while a +// ShareFetch is in flight, no acks can be sent to the broker, so a long +// MaxWait stalls ack delivery for that entire window. +func FetchMaxWait(wait time.Duration) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.maxWait = int32(wait.Milliseconds()) }} +} + +// FetchMaxBytes sets the maximum amount of bytes a broker will try to send +// during a fetch, overriding the default 50MiB. Note that brokers may not obey +// this limit if it has records larger than this limit. Also note that this +// client sends a fetch to each broker concurrently, meaning the client will +// buffer up to worth of memory. +// +// This corresponds to the Java fetch.max.bytes setting. +// +// If bumping this, consider bumping BrokerMaxReadBytes. +// +// If what you are consuming is compressed, and compressed well, it is strongly +// recommended to set this option so that decompression does not eat all of +// your RAM. +func FetchMaxBytes(b int32) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.maxBytes = lazyI32(b) }} +} + +// FetchMinBytes sets the minimum amount of bytes a broker will try to send +// during a fetch, overriding the default 1 byte. +// +// With the default of 1, data is sent as soon as it is available. By bumping +// this, the broker will try to wait for more data, which may improve server +// throughput at the expense of added latency. +// +// This corresponds to the Java fetch.min.bytes setting. +func FetchMinBytes(b int32) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.minBytes = b }} +} + +// FetchMaxPartitionBytes sets the maximum amount of bytes that will be +// consumed for a single partition in a fetch request, overriding the default +// 1MiB. Note that if a single batch is larger than this number, that batch +// will still be returned so the client can make progress. +// +// This corresponds to the Java max.partition.fetch.bytes setting. +func FetchMaxPartitionBytes(b int32) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.maxPartBytes = lazyI32(b) }} +} + +// MaxConcurrentFetches sets the maximum number of fetch requests to allow in +// flight or buffered at once, overriding the unbounded (i.e. number of +// brokers) default. +// +// This setting, paired with FetchMaxBytes, can upper bound the maximum amount +// of memory that the client can use for consuming. +// +// Requests are issued to brokers in a FIFO order: once the client is ready to +// issue a request to a broker, it registers that request and issues it in +// order with other registrations. +// +// If Kafka replies with any data, the client does not track the fetch as +// completed until the user has polled the buffered fetch. Thus, a concurrent +// fetch is not considered complete until all data from it is done being +// processed and out of the client itself. +// +// Note that brokers are allowed to hang for up to FetchMaxWait before replying +// to a request, so if this option is too constrained and you are consuming a +// low throughput topic, the client may take a long time before requesting a +// broker that has new data. For high throughput topics, or if the allowed +// concurrent fetches is large enough, this should not be a concern. +// +// Negative values imply unlimited concurrent fetches (bounded by the number of +// brokers in the cluster). A value of 0 means that a single fetch is allowed +// ONLY when you poll - there is no fetch buffering. +func MaxConcurrentFetches(n int) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.maxConcurrentFetches = n }} +} + +// ConsumeStartOffset sets the offset to start consuming from when consuming a +// partition for the first time. If you do not set [ConsumeResetOffset], this +// is also the offset to reset to if the client sees an OffsetOutOfRange error +// while consuming a partition. The default is NewOffset().AtStart(), i.e., +// start processing a partition from the earliest offset. If using this option, +// it is strongly recommended to also set ConsumeResetOffset. +// +// If you use an exact or relative offsets and the offset ends up out of range, +// the client chooses the nearest of either the log start offset or the log end +// offset. For example, using At(3) when the partition starts at 8 results in +// the partition being consumed from offset 8. +// +// For group consuming, you can use [Offset.AtCommitted] to prevent starting +// consuming a partition in a group if the partition has no prior commits. +// +// The following determines the offset for when a partition is seen for the +// first time: +// +// at start? => start at the log start offset +// at end? => start at the log end offset +// at exact? => start at an exact offset (3 means offset 3) +// relative? => start at the above, + / - the relative amount +// exact/relative are out of bounds? => start at the nearest boundary (start or end) +// after millisec? => start at first offset after millisec if one exists, else log end offset +// +// To match Kafka's auto.offset.reset which is used for both the start offset +// and the reset offset, +// +// NewOffset().AtStart() == auto.offset.reset "earliest" +// NewOffset().AtEnd() == auto.offset.reset "latest" +// NewOffset().AtCommitted() == auto.offset.reset "none" +// +// Be sure to check the documentation for [ConsumeResetOffset], especially if +// you rely on this option as the reset offset as well. +func ConsumeStartOffset(offset Offset) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.startOffset, cfg.setStartOffset = offset, true }} +} + +// ConsumeResetOffset sets the offset to reset to if the client ever sees +// OffsetOutOfRange while fetching. If you do not set [ConsumeStartOffset], +// this is also the offset to start consuming from when consuming a partition +// for the first time. The default is NewOffset().AtStart(), i.e., reset to the +// earliest offset. If using this option, it is strongly recommended to also +// set ConsumeStartOffset. +// +// This option is *only* used if a consumer seeds OffsetOutOfRange on the +// *first* fetch of a partition. If the consumer has consumed the partition at +// all and sees the error, it will automatically reset to the first offset +// after the timestamp of the last successfully consumed offset. If data loss +// occurred such that even the last successfully consumed offset is lost, the +// client automatically resets to the new current end offset. If you want to +// disable offset resetting entirely, you can use [NoResetOffset]. +// +// If you use an exact or relative offsets and the offset ends up out of range, +// the client chooses the nearest of either the log start offset or the log end +// offset. For example, using At(3) when the partition starts at 8 results in +// the partition being consumed from offset 8. +// +// The following determines the offset for when a partition is seen for the +// first time, or reset while fetching: +// +// at start? => reset to the log start offset +// at end? => reset to the log end offset +// at exact? => reset to an exact offset (3 means offset 3) +// relative? => reset to the above, + / - the relative amount +// exact/relative are out of bounds? => reset to the nearest boundary (start or end) +// after millisec? => reset to the first offset after millisec if one exists, else the log end offset +// +// To match Kafka's auto.offset.reset, +// +// NewOffset().AtStart() == auto.offset.reset "earliest" +// NewOffset().AtEnd() == auto.offset.reset "latest" +// NewOffset().AtCommitted() == auto.offset.reset "none" +// +// With the above, make sure to use [NoResetOffset] if you want to stop +// consuming when you encounter OffsetOutOfRange. It is highly recommended +// to read the docs for all Offset methods. +// +// Be sure to check the documentation for [ConsumeStartOffset], especially if +// you rely on this option as the start offset as well. +func ConsumeResetOffset(offset Offset) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.resetOffset, cfg.setResetOffset = offset, true }} +} + +// Rack specifies where the client is physically located and changes fetch +// requests to consume from the closest replica as opposed to the leader +// replica. +// +// Consuming from a preferred replica can increase latency but can decrease +// cross datacenter costs. See KIP-392 for more information. +func Rack(rack string) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.rack = rack }} +} + +// IsolationLevel controls whether uncommitted or only committed records are +// returned from fetch requests. +type IsolationLevel struct { + level int8 +} + +// ReadUncommitted (the default) is an isolation level that returns the latest +// produced records, be they committed or not. +func ReadUncommitted() IsolationLevel { return IsolationLevel{0} } + +// ReadCommitted is an isolation level to only fetch committed records. +func ReadCommitted() IsolationLevel { return IsolationLevel{1} } + +// FetchIsolationLevel sets the "isolation level" used for fetching +// records, overriding the default ReadUncommitted. +func FetchIsolationLevel(level IsolationLevel) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.isolationLevel = level.level }} +} + +// KeepControlRecords sets the client to keep control messages and return +// them with fetches, overriding the default that discards them. +// +// Generally, control messages are not useful. +func KeepControlRecords() ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.keepControl = true }} +} + +// ConsumeTopics adds topics to use for consuming. +// +// By default, consuming will start at the beginning of partitions. To change +// this, use the ConsumeResetOffset option. +func ConsumeTopics(topics ...string) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { + cfg.topics = make(map[string]*regexp.Regexp, len(topics)) + for _, topic := range topics { + cfg.topics[topic] = nil + } + }} +} + +// ConsumePartitions sets partitions to consume from directly and the offsets +// to start consuming those partitions from. +// +// This option is basically a way to explicitly consume from subsets of +// partitions in topics, or to consume at exact offsets. Offsets from this +// option have higher precedence than the ConsumeResetOffset. +// +// This option is not compatible with group consuming and regex consuming. If +// you want to assign partitions directly, but still use Kafka to commit +// offsets, check out the kadm package's FetchOffsets and CommitOffsets +// methods. These will allow you to commit as a group outside the context of a +// Kafka group. +func ConsumePartitions(partitions map[string]map[int32]Offset) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.partitions = partitions }} +} + +// ConsumeRegex sets the client to parse all topics passed to ConsumeTopics as +// regular expressions. You can further use ConsumeExcludeTopics to exclude +// topics that would match any ConsumeTopics regex. +// +// When consuming via regex, every metadata request loads *all* topics, so that +// all topics can be passed to any regular expressions. Every topic is +// evaluated only once ever across all regular expressions; either it +// permanently is known to match, or is permanently known to not match. +func ConsumeRegex() ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.regex = true }} +} + +// ConsumeExcludeTopics sets topics to exclude when using regex consumption. +// This option only has effect when ConsumeRegex is enabled. +// +// Topics matching any of the provided regular expressions will be excluded from +// consumption, even if they match patterns provided to ConsumeTopics. +// +// When using the next-gen consumer group protocol (not yet enabled by default), +// the heartbeat's SubscribedTopicRegex field is include-only with no exclude +// counterpart. If exclude topics are configured, the client resolves regex +// matching locally and sends explicit topic names instead. This means new +// topics matching the include regex are discovered on the next metadata +// refresh rather than immediately by the broker. +func ConsumeExcludeTopics(topics ...string) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { + if cfg.excludeTopics == nil { + cfg.excludeTopics = make(map[string]*regexp.Regexp, len(topics)) + } + for _, topic := range topics { + cfg.excludeTopics[topic] = nil + } + }} +} + +// DisableFetchSessions sets the client to not use fetch sessions (Kafka 1.0+). +// +// A "fetch session" is a way to reduce bandwidth for fetch requests & +// responses, and to potentially reduce the amount of work that brokers have to +// do to handle fetch requests. A fetch session opts into the broker tracking +// some state of what the client is interested in. For example, say that you +// are interested in thousands of topics, and most of these topics are +// receiving data only rarely. A fetch session allows the client to register +// that it is interested in those thousands of topics on the first request. On +// future requests, if the offsets for these topics have not changed, those +// topics will be elided from the request. The broker knows to reply with the +// extra topics if any new data is available, otherwise the topics are also +// elided from the response. This massively reduces the amount of information +// that needs to be included in requests or responses. +// +// Using fetch sessions means more state is stored on brokers. Maintaining this +// state eats some memory. If you have thousands of consumers, you may not want +// fetch sessions to be used for everything. Brokers intelligently handle this +// by not creating sessions if they are at their configured limit, but you may +// consider disabling sessions if they are generally not useful to you. Brokers +// have metrics for the number of fetch sessions active, so you can monitor +// that to determine whether enabling or disabling sessions is beneficial or +// not. +// +// For more details on fetch sessions, see KIP-227. +func DisableFetchSessions() ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.disableFetchSessions = true }} +} + +// ConsumePreferringLagFn allows you to re-order partitions before they are +// fetched, given each partition's current lag. +// +// By default, the client rotates partitions fetched by one after every fetch +// request. Kafka answers fetch requests in the order that partitions are +// requested, filling the fetch response until FetchMaxBytes and +// FetchMaxPartitionBytes are hit. All partitions eventually rotate to the +// front, ensuring no partition is starved. +// +// With this option, you can return topic order and per-topic partition +// ordering. These orders will sort to the front (first by topic, then by +// partition). Any topic or partitions that you do not return are added to the +// end, preserving their original ordering. +// +// For a simple lag preference that sorts the laggiest topics and partitions +// first, use `kgo.ConsumePreferringLagFn(kgo.PreferLagAt(50))` (or some other +// similar lag number). +func ConsumePreferringLagFn(fn PreferLagFn) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.preferLagFn = fn }} +} + +// WithDecompressor allows you to completely control how fetch batches are +// decompressed, allowing you to use alternative libraries than what franz-go +// supports, allowing you to have more control over memory & pooling, and other +// benefits. The client default compressor is the [DefaultDecompressor]. +func WithDecompressor(decompressor Decompressor) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.decompressor = decompressor }} +} + +// KeepRetryableFetchErrors switches the client to always return any retryable +// broker error when fetching, rather than stripping them. By default, the +// client strips retryable errors from fetch responses; these are usually +// signals that a client needs to update its metadata to learn of where a +// partition has moved to (from one broker to another), or they are signals +// that one broker is temporarily unhealthy (broker not available). You can opt +// into keeping these errors if you want to specifically react to certain +// events. For example, if you want to react to you yourself deleting a topic, +// you can watch for either UNKNOWN_TOPIC_OR_PARTITION or UNKNOWN_TOPIC_ID +// errors being returned in fetches (and ignore the other errors). +func KeepRetryableFetchErrors() ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.keepRetryableFetchErrors = true }} +} + +// DisableFetchCRCValidation disables crc32 checksum validation when fetching. +// This should only be used if you are working with a broker that does not +// properly support CRCs in record batches. +func DisableFetchCRCValidation() ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.disableFetchCRCValidation = true }} +} + +// RecheckPreferredReplicaInterval configures how long the consumer should +// fetch from a preferred replica before switching back to the leader. +// Periodically switching back to the leader allows the leader to re-choose a +// perhaps better preferred replica (say you added a new cluster, or added +// nodes to an existing cluster, or something else changed). For implementation +// simplicity, the interval is checked after fetch responses, meaning one more +// request can be issued after the interval has elapsed. +// +// The default interval is 30 minutes. +func RecheckPreferredReplicaInterval(interval time.Duration) ConsumerOpt { + return consumerOpt{func(cfg *cfg) { cfg.recheckPreferredReplicaInterval = interval }} +} + +////////////////////////////////// +// CONSUMER GROUP CONFIGURATION // +////////////////////////////////// + +// ConsumerGroup sets the consumer group for the client to join and consume in. +// This option is required if using any other group options. +// +// Note that when group consuming, the default is to autocommit every 5s. To be +// safe, autocommitting only commits what is *previously* polled. If you poll +// once, nothing will be committed. If you poll again, the first poll is +// available to be committed. This ensures at-least-once processing, but does +// mean there is likely some duplicate processing during rebalances. When your +// client shuts down, you should issue one final synchronous commit before +// leaving the group (because you will not be polling again, and you are not +// waiting for an autocommit). +func ConsumerGroup(group string) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.group = group }} +} + +// ShareGroup sets the share group for the client to join and consume in. +// Share groups (KIP-932) provide queue-like semantics: the broker controls +// record delivery and offset management. +// +// This is mutually exclusive with ConsumerGroup and ConsumePartitions. +// +// For existing topics to be consumable via a share group, the group-level +// configuration share.auto.offset.reset must be set using +// IncrementalAlterConfigs with resource type GROUP. Without this, share +// groups default to "latest" and only records produced after the group +// begins consuming are delivered. +func ShareGroup(group string) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.shareGroup = group }} +} + +// ShareMaxRecords sets the MaxRecords and BatchSize fields in ShareFetch +// requests, overriding the default of 500. +// +// This option only applies when using [ShareGroup]. +func ShareMaxRecords(n int32) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.shareMaxRecords = n }} +} + +// ShareMaxRecordsStrict opts into strict record-count limiting for share +// fetch requests. By default, the broker may return more records than +// [ShareMaxRecords] to avoid splitting record batches. With this option +// the broker returns at most [ShareMaxRecords] records per fetch, +// splitting batches if necessary. +// +// As a secondary effect, this option also flips the [MaxConcurrentFetches] +// default from unbounded to 0 (one fetch per poll). Setting +// [MaxConcurrentFetches] to a positive value re-enables pre-fetching; +// note that the per-fetch limit still holds but in-flight concurrent +// fetches can cumulatively return more. +// +// This option only applies when using [ShareGroup]. +func ShareMaxRecordsStrict() GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.shareMaxRecordsStrict = true }} +} + +// ShareAckCallback sets a callback invoked with the results of share-group +// acknowledgements. The broker reports per-partition outcomes for every +// ack the client sends. +// +// Without a callback, ack errors are invisible: retryable errors are +// retried internally and non-retryable errors are dropped. The callback +// lets you observe all ack outcomes. The client has already handled +// retries where possible, so errors surfaced here are mostly +// informational. +// +// This option only applies when using [ShareGroup]. +func ShareAckCallback(fn func(*Client, ShareAckResults)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.shareAckCallback = fn }} +} + +// Balancers sets the group balancers to use for dividing topic partitions +// among group members, overriding the current default [cooperative-sticky]. +// This option is equivalent to Kafka's partition.assignment.strategies option. +// +// For balancing, Kafka chooses the first protocol that all group members agree +// to support. +// +// Note that if you opt into cooperative-sticky rebalancing, cooperative group +// balancing is incompatible with eager (classical) rebalancing and requires a +// careful rollout strategy (see KIP-429). +func Balancers(balancers ...GroupBalancer) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.balancers = balancers }} +} + +// SessionTimeout sets how long a member in the group can go between +// heartbeats, overriding the default 45,000ms. If a member does not heartbeat +// in this timeout, the broker will remove the member from the group and +// initiate a rebalance. +// +// If you are using a [GroupTransactSession] for EOS, wish to lower this, and are +// talking to a Kafka cluster pre 2.5, consider lowering the +// TransactionTimeout. If you do not, you risk a transaction finishing after a +// group has rebalanced, which could lead to duplicate processing. If you are +// talking to a Kafka 2.5+ cluster, you can safely use the +// RequireStableFetchOffsets group option and prevent any problems. +// +// This option corresponds to Kafka's session.timeout.ms setting and must be +// within the broker's group.min.session.timeout.ms and +// group.max.session.timeout.ms. +func SessionTimeout(timeout time.Duration) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.sessionTimeout = timeout }} +} + +// RebalanceTimeout sets how long group members are allowed to take when a a +// rebalance has begun, overriding the default 60,000ms. This timeout is how +// long all members are allowed to complete work and commit offsets, minus the +// time it took to detect the rebalance (from a heartbeat). +// +// Kafka uses the largest rebalance timeout of all members in the group. If a +// member does not rejoin within this timeout, Kafka will kick that member from +// the group. +// +// This corresponds to Kafka's rebalance.timeout.ms. +func RebalanceTimeout(timeout time.Duration) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.rebalanceTimeout = timeout }} +} + +// HeartbeatInterval sets how long a group member goes between heartbeats to +// Kafka, overriding the default 3,000ms. +// +// Kafka uses heartbeats to ensure that a group member's session stays active. +// This value can be any value lower than the session timeout, but should be no +// higher than 1/3rd the session timeout. +// +// This corresponds to Kafka's heartbeat.interval.ms. +func HeartbeatInterval(interval time.Duration) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.heartbeatInterval = interval }} +} + +// RequireStableFetchOffsets previously set the group consumer to require +// "stable" fetch offsets before consuming from the group. +// +// Deprecated: RequireStable is now permanently enabled for all group +// consumers. This function is a no-op. +func RequireStableFetchOffsets() GroupOpt { + return groupOpt{func(*cfg) {}} +} + +// BlockRebalanceOnPoll switches the client to block rebalances whenever you +// receive a non-empty result from polling until you explicitly call +// AllowRebalance. This option also ensures that any +// OnPartitions{Assigned,Revoked,Lost} callbacks are only called when you allow +// rebalances; they cannot be called if you have polled and are processing +// records. +// +// By default, a consumer group is managed completely independently of +// consuming. A rebalance may occur at any moment. If you poll records, and +// then a rebalance happens, and then you commit, you may be committing to +// partitions you no longer own. This will result in duplicates. In the worst +// case, you could rewind commits that a different member has already made +// (risking duplicates if another rebalance were to happen before that other +// member commits again). +// +// By blocking rebalancing after you poll until you call AllowRebalances, you +// can be sure that you commit records that your member currently owns. +// However, the big tradeoff is that by blocking rebalances, you put your group +// member at risk of waiting so long that the group member is kicked from the +// group because it exceeded the rebalance timeout. To compare clients, Sarama +// takes the default choice of blocking rebalancing; this option makes kgo more +// similar to Sarama. +// +// If you use this option, you should ensure that you always process records +// quickly, and that your OnPartitions{Assigned,Revoked,Lost} callbacks are +// fast. If your record processing may be slow, it is recommended you also use +// PollRecords rather than PollFetches so that you can bound how many records +// you process at once. You must always AllowRebalances when you are done +// processing the records you received. +// +// Polls and rebalance callbacks gate each other so that you cannot commit +// offsets across a rebalance: polls block while a callback is running, and +// callbacks wait for in-flight polls to release via AllowRebalance before +// running. The assign phase is gated only when you registered +// [OnPartitionsAssigned]; otherwise it runs concurrent with poll because +// adding partitions cannot cause a wrong-ownership commit. +// +// You can use [OnPartitionsCallbackBlocked] as a signal that a rebalance WANTS +// to happen, but you are currently blocking it, and that you need to either +// finish processing or abort processing to allow the rebalance to continue. +func BlockRebalanceOnPoll() GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.blockRebalanceOnPoll = true }} +} + +// AdjustFetchOffsetsFn sets the function to be called when a group is joined +// after offsets are fetched so that a user can adjust offsets before +// consumption begins. +// +// This function should not exceed the rebalance interval. It is possible +// for the group, immediately after finishing a balance, to re-enter a new balancing +// session. This function is passed a context that is canceled if the current group +// session finishes (i.e., after revoking). +// +// If you are resetting the position of the offset, you may want to clear any existing +// "epoch" with WithEpoch(-1). If the epoch is non-negative, the client performs +// data loss detection, which may result in errors and unexpected behavior. +// +// This function is called after OnPartitionsAssigned and may be called before +// or after OnPartitionsRevoked. +func AdjustFetchOffsetsFn(adjustOffsetsBeforeAssign func(context.Context, map[string]map[int32]Offset) (map[string]map[int32]Offset, error)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.adjustOffsetsBeforeAssign = adjustOffsetsBeforeAssign }} +} + +// OnPartitionsAssigned sets the function to be called when a group is joined +// after partitions are assigned before fetches for those partitions begin. +// +// This function, combined with OnPartitionsRevoked, should not exceed the +// rebalance interval. It is possible for the group to re-enter a new balancing +// session immediately after finishing a balance. +// +// This function is passed the client's context, which is only canceled if the +// client is closed. +// +// This function is not called concurrent with any other OnPartitions callback, +// and this function is given a new map that the user is free to modify. +// +// This function can be called at any time you are polling or processing +// records. If you want to ensure this function is called serially with +// processing, consider the BlockRebalanceOnPoll option. +func OnPartitionsAssigned(onAssigned func(context.Context, *Client, map[string][]int32)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.onAssigned = onAssigned }} +} + +// OnPartitionsRevoked sets the function to be called once this group member +// has partitions revoked. +// +// This function, combined with [OnPartitionsAssigned], should not exceed the +// rebalance interval. It is possible for the group to re-enter a new balancing +// session immediately after finishing a balance. +// +// If autocommit is enabled, the default OnPartitionsRevoked is a blocking +// commit of all non-dirty offsets (where "dirty" is the most recent poll). +// +// The OnPartitionsRevoked function is passed the client's context, which is +// only canceled if the client is closed. OnPartitionsRevoked function is +// called at the end of a group session even if there are no partitions being +// revoked. If you are committing offsets manually (have disabled +// autocommitting), it is highly recommended to do a proper blocking commit in +// OnPartitionsRevoked. +// +// This function is not called concurrent with any other OnPartitions callback, +// and this function is given a new map that the user is free to modify. +// +// This function can be called at any time you are polling or processing +// records. If you want to ensure this function is called serially with +// processing, consider the BlockRebalanceOnPoll option. It is guaranteed +// that once the callback has completed any subsequent polls will not return +// records for the revoked partitions. +// +// This function is called if a "fatal" group error is encountered and you have +// not set [OnPartitionsLost]. See OnPartitionsLost for more details. +func OnPartitionsRevoked(onRevoked func(context.Context, *Client, map[string][]int32)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.onRevoked = onRevoked }} +} + +// OnPartitionsLost sets the function to be called on "fatal" group errors, +// such as IllegalGeneration, UnknownMemberID, and authentication failures. +// This function differs from [OnPartitionsRevoked] in that it is unlikely that +// commits will succeed when partitions are outright lost, whereas commits +// likely will succeed when revoking partitions. +// +// Because this function is called on any fatal group error, it is possible for +// this function to be called without the group ever being joined. +// +// This function is not called concurrent with any other OnPartitions callback, +// and this function is given a new map that the user is free to modify. +// +// This function can be called at any time you are polling or processing +// records. If you want to ensure this function is called serially with +// processing, consider the BlockRebalanceOnPoll option. +func OnPartitionsLost(onLost func(context.Context, *Client, map[string][]int32)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.onLost = onLost }} +} + +// OnPartitionsCallbackBlocked sets a function to be called just before any +// [OnPartitionsAssigned], [OnPartitionsRevoked], or [OnPartitionsLost] +// callbacks are blocked from [BlockRebalanceOnPoll]. You can use this as a +// signal in your processing function to hurry up and unblock rebalancing +// before your group member is kicked from the group at the session timeout. +// +// Since this is meant to be a signal to blocking operations, and to ensure +// nothing can block this signal, this callback is called in a goroutine. +func OnPartitionsCallbackBlocked(fn func(context.Context, *Client)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.onBlocked = fn }} +} + +// OnOffsetsFetched sets a function to be called after offsets have been +// fetched after a group has been balanced. This function is meant to allow +// users to inspect offset commit metadata. An error can be returned to exit +// this group session and exit back to join group. +// +// This function should not exceed the rebalance interval. It is possible for +// the group, immediately after finishing a balance, to re-enter a new +// balancing session. This function is passed a context that is canceled if the +// current group session finishes (i.e., after revoking). +// +// This function is called after OnPartitionsAssigned and may be called before +// or after OnPartitionsRevoked. +func OnOffsetsFetched(onFetched func(context.Context, *Client, *kmsg.OffsetFetchResponse) error) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.onFetched = onFetched }} +} + +// DisableAutoCommit disable auto committing. +// +// If you disable autocommitting, you may want to use a custom +// OnPartitionsRevoked, otherwise you may end up doubly processing records +// (which is fine, just leads to duplicate processing). Consider the scenario: +// you, member A, are processing partition 0, and previously committed offset 4 +// and have now locally processed through offset 30. A rebalance happens, and +// partition 0 moves to member B. If you use OnPartitionsRevoked, you can +// detect that you are losing this partition and commit your work through +// offset 30, so that member B can start processing at offset 30. If you do not +// commit (i.e. you do not use a custom OnPartitionsRevoked), the other member +// will start processing at offset 4. It may process through offset 50, leading +// to double processing of offsets 4 through 29. Worse, you, member A, can +// rewind member B's commit, because member B may commit offset 50 and you may +// finally eventually commit offset 30. If a rebalance happens, then even more +// duplicate processing will occur of offsets 30 through 49. +// +// Again, OnPartitionsRevoked is not necessary, and not using it just means +// double processing, which for most workloads is fine since a simple group +// consumer is not EOS / transactional, only at-least-once. But, this is +// something to be aware of. +func DisableAutoCommit() GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.autocommitDisable = true }} +} + +// GreedyAutoCommit opts into committing everything that has been polled when +// autocommitting (the dirty offsets), rather than committing what has +// previously been polled. This option may result in message loss if your +// application crashes. +func GreedyAutoCommit() GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.autocommitGreedy = true }} +} + +// AutoCommitInterval sets how long to go between autocommits, overriding the +// default 5s. +func AutoCommitInterval(interval time.Duration) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.autocommitInterval = interval }} +} + +// AutoCommitMarks switches the autocommitting behavior to only commit "marked" +// records, which can be done with the MarkCommitRecords method. +// +// This option is basically a halfway point between autocommitting and manually +// committing. If you have slow batch processing of polls, then you can +// manually mark records to be autocommitted before you poll again. This way, +// if you usually take a long time between polls, your partial work can still +// be automatically checkpointed through autocommitting. +func AutoCommitMarks() GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.autocommitMarks = true }} +} + +// InstanceID sets the group consumer's instance ID, switching the group member +// from "dynamic" to "static". +// +// Prior to Kafka 2.3, joining a group gave a group member a new member ID. +// The group leader could not tell if this was a rejoining member. Thus, any +// join caused the group to rebalance. +// +// Kafka 2.3 introduced the concept of an instance ID, which can persist across +// restarts. This allows for avoiding many costly rebalances and allows for +// stickier rebalancing for rejoining members (since the ID for balancing stays +// the same). The main downsides are that you, the user of a client, have to +// manage instance IDs properly, and that it may take longer to rebalance in +// the event that a client legitimately dies. +// +// When using an instance ID, the client does NOT send a leave group request +// when closing. This allows for the client to restart with the same instance +// ID and rejoin the group to avoid a rebalance. It is strongly recommended to +// increase the session timeout enough to allow time for the restart (remember +// that the default session timeout is 45s). +// +// To actually leave the group, you must use an external admin command that +// issues a leave group request on behalf of this instance ID (see kcl), or you +// can manually use the kmsg package with a proper LeaveGroupRequest. +// +// NOTE: Leaving a group with an instance ID is only supported in Kafka 2.4+. +// +// NOTE: If you restart a consumer group leader that is using an instance ID, +// it will not cause a rebalance even if you change which topics the leader is +// consuming. If your cluster is 3.2+, this client internally works around this +// limitation and you do not need to trigger a rebalance manually. +func InstanceID(id string) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.instanceID = &id }} +} + +// GroupProtocol sets the group's join protocol, overriding the default value +// "consumer". The only reason to override this is if you are implementing +// custom join and sync group logic. +func GroupProtocol(protocol string) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.protocol = protocol }} +} + +// AutoCommitCallback sets the callback to use if autocommitting is enabled. +// This overrides the default callback that logs errors and continues. +func AutoCommitCallback(fn func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error)) GroupOpt { + return groupOpt{func(cfg *cfg) { cfg.commitCallback = fn }} +} + +// !!! Only uncomment once we trust the broker implementation! +// !!! And add this option to Opt! +// +// DisableNextGenRebalancer opts out of the "next gen" rebalancer that is +// the default as of Kafka 4.0+. The client opts in to the next gen rebalancer +// automatically if the broker supports it AND if you are using either the +// [RangeBalancer] or [StickyBalancer] or [CooperativeStickyBalancer]. If you +// use your own rebalancer or use the [RoundRobinBalancer] or are talking to +// a broker that does not support the next gen balancer, the client uses the +// old client-driven group balancing behavior. +// +// You may want to use this function if you notice a regression or run into +// a broker or client bug, or if you prefer the performance of the old +// client driven rebalancers. +// func DisableNextGenRebalancer() GroupOpt { +// return groupOpt{func(cfg *cfg) { cfg.disableNextGenBalancer = true }} +// } +// diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_unix.go b/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_unix.go new file mode 100644 index 00000000000..3ce4868c0ca --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_unix.go @@ -0,0 +1,11 @@ +//go:build !windows + +package kgo + +import ( + "syscall" +) + +func isConnReset(errno syscall.Errno) bool { + return errno == syscall.ECONNRESET +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_windows.go b/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_windows.go new file mode 100644 index 00000000000..f06c04e7c71 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/connreset_windows.go @@ -0,0 +1,9 @@ +package kgo + +import ( + "syscall" +) + +func isConnReset(errno syscall.Errno) bool { + return errno == syscall.WSAECONNRESET +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-bugs-prompt.md b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-bugs-prompt.md new file mode 100644 index 00000000000..f405e837978 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-bugs-prompt.md @@ -0,0 +1,71 @@ +# franz-go consumer: bugs & races + +You are analyzing the franz-go Kafka client library (pkg/kgo). Find correctness +bugs and race conditions in the CONSUMER codepath. Do not flag style, naming, +missing tests, or refactors. + +Files in scope: +- source.go per-broker fetch loop; owns cursors (one per partition) +- consumer.go consumer abstraction, source/cursor management +- consumer_group.go group consumer: join/sync/heartbeat, commits, + KIP-848 manage loop, static membership +- consumer_direct.go user-assigned consumer; metadata-driven resolution +- txn.go consume side of GroupTransactSession; read_committed +- metadata.go cursor migration on partition reassignment +- client.go close, broker selection, retry + +Invariants (assume these hold): +- cursor.useState is atomic.Bool: Swap(true) -> fetchable; + Store(false) -> in-flight / not fetchable. +- Cursor.source MUST be read BEFORE useState.Swap(true): after Swap, a + concurrent fetch can complete and move() can overwrite c.source. +- move() is safe because it removes the cursor from the old source's list + BEFORE Swap; no concurrent pickup until addCursor on the new source. +- Within a partition, fetched offsets must be monotonic; rewinds only via + OffsetForLeaderEpoch / ListOffsets validation. +- read_committed drops aborted records using LSO + abortedTransactions. +- Lock ordering: c.mu -> g.mu. g.uncommitted protected by g.mu; + usingCursors protected by c.mu. +- GroupTransactSession: users must not Poll concurrently with End(). +- KIP-848 manage loop treats errChosenBrokerDead as retriable; not fatal. + +Known intentional behavior - DO NOT flag: +- "load offsets, then validate via OffsetForLeaderEpoch" two-step on + assignment. +- Cursor state-machine transitions between sources. +- ctxRecRecycle context value for Fetches pooling. +- Sharder fan-out for cross-broker requests. + +Find only: +1. Data races, especially around cursor migration, source replacement, + group state transitions, fetch session state. +2. Offset corruption: unjustified rewind; offsets advancing past records + never yielded; double-yielded records. +3. Commit safety: committing offsets for a partition we no longer own; + committing offsets for records the user hasn't acknowledged + (autocommit modes); missing commits on close. +4. Aborted txn handling under read_committed: dropping records that should + be yielded, or yielding records that should be dropped (LSO / aborted + list edge cases). +5. Rebalance correctness: + - eager: revoke fires before new assignment is used. + - cooperative-sticky: only revoked partitions stop; retained partitions + have no gap. + - KIP-848: target reconciliation, member epoch bumps, lost-partition + detection, fence handling. +6. Fetch session desync (KIP-227): client/broker disagree on session state. +7. Goroutine leaks past Close. +8. Channel close races (double-close, send on closed). +9. Static membership (KIP-345): instance ID handling during reconnect / + fenced rejoin. + +Output per finding: +- Severity: critical (data loss / dup / corruption) | high (hang / leak) + | medium (rare race, recoverable) | low +- File:line +- What: one sentence +- How: numbered walkthrough of goroutines/events that triggers it +- Fix: one-paragraph sketch (not full code) + +If a category yields nothing, say "none found" - don't pad. If you cannot +trace a finding to a concrete sequence of events, omit it. diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-efficiency-prompt.md b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-efficiency-prompt.md new file mode 100644 index 00000000000..270dda8a552 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer-efficiency-prompt.md @@ -0,0 +1,57 @@ +# franz-go consumer: efficiency + +Find efficiency improvements in the CONSUMER codepath of franz-go (pkg/kgo). + +Files in scope: +- source.go per-broker fetch loop; owns cursors (one per partition) +- consumer.go consumer abstraction, source/cursor management +- consumer_group.go group consumer: join/sync/heartbeat, commits, + KIP-848 manage loop, static membership +- consumer_direct.go user-assigned consumer; metadata-driven resolution +- txn.go consume side of GroupTransactSession; read_committed +- metadata.go cursor migration on partition reassignment +- client.go close, broker selection, retry + +Hot paths in priority order: +1. PER-FETCH-RESPONSE: parse, decompress, decode records, yield via PollFetches. +2. PER-RECORD: any handling that scales with record count - key/value/header + allocation, header parsing, time conversion. +3. PER-FETCH: pick cursors, build Fetch request, send. +4. PER-COMMIT: build OffsetCommit, send (usually low rate). +5. PER-HEARTBEAT / PER-REBALANCE: low rate but latency-sensitive. + +Already tuned - do not suggest: +- Fetch buffer pooling. +- Record / Fetches recycling via ctxRecRecycle. +- cursor.useState atomic state. +- Fetch sessions (KIP-227). + +Find only: +1. Allocations during fetch response parsing scaling with record count: + key/value/header byte slices, RecordHeader slice growth, string headers, + time conversions per record. +2. Decompression: are decompressors and output buffers pooled across + fetches? Are we decompressing whole batches when read_committed will + drop most records? +3. Lock contention on consumer / group mutexes during fetch yield; + broader locks than necessary. +4. Cursor list iteration: linear scans where indexed access works. +5. Group: redundant metadata refreshes, redundant coordinator lookups, + redundant SyncGroup payloads. +6. Heartbeat / manage loop overhead scaling with assignment size that + could be cached. +7. Aborted txn handling under read_committed: per-record work that could + be amortized to per-batch using LSO + aborted list. +8. Goroutine reuse: per-fetch goroutines that could be reused per-broker. + +Output per finding: +- Cost class: per-record | per-batch | per-response | per-flush | startup +- File:line +- What: one sentence +- Cost: e.g. "allocates a 64-byte slice per record; 1M rec/s -> 60MB/s garbage" +- Fix: sketch + +Skip: micro-optimizations on cold paths (config parsing, client init, +error paths). Skip "consider sync.Pool" unless you've identified the +specific allocation hot spot it would address. Skip anything where you +can't name the cost class. diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer.go b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer.go new file mode 100644 index 00000000000..b5e5443123e --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer.go @@ -0,0 +1,2574 @@ +package kgo + +import ( + "context" + "errors" + "fmt" + "math" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +// Offset is a message offset in a partition. +type Offset struct { + at int64 + relative int64 + epoch int32 + + currentEpoch int32 // set by us when mapping offsets to brokers + + noReset bool + afterMilli bool +} + +// Random negative, only significant within this package. +const atCommitted = -999 + +// MarshalJSON implements json.Marshaler. +func (o Offset) MarshalJSON() ([]byte, error) { + if o.relative == 0 { + return fmt.Appendf(nil, `{"At":%d,"Epoch":%d,"CurrentEpoch":%d}`, o.at, o.epoch, o.currentEpoch), nil + } + return fmt.Appendf(nil, `{"At":%d,"Relative":%d,"Epoch":%d,"CurrentEpoch":%d}`, o.at, o.relative, o.epoch, o.currentEpoch), nil +} + +// String returns the offset as a string; the purpose of this is for logs. +func (o Offset) String() string { + if o.relative == 0 { + return fmt.Sprintf("{%d e%d ce%d}", o.at, o.epoch, o.currentEpoch) + } else if o.relative > 0 { + return fmt.Sprintf("{%d+%d e%d ce%d}", o.at, o.relative, o.epoch, o.currentEpoch) + } + return fmt.Sprintf("{%d-%d e%d ce%d}", o.at, -o.relative, o.epoch, o.currentEpoch) +} + +// EpochOffset returns this offset as an EpochOffset, allowing visibility into +// what this offset actually currently is. +func (o Offset) EpochOffset() EpochOffset { + return EpochOffset{ + Epoch: o.epoch, + Offset: o.at, + } +} + +// NewOffset creates and returns an offset to use in [ConsumePartitions] or +// [ConsumeResetOffset]. +// +// The default offset begins at the end. +func NewOffset() Offset { + return Offset{ + at: -1, + epoch: -1, + } +} + +// NoResetOffset returns an offset that can be used as a "none" option for the +// [ConsumeResetOffset] option. By default, NoResetOffset starts consuming from +// the beginning of partitions (similar to NewOffset().AtStart()). This can be +// changed with AtEnd, Relative, etc. +// +// Using this offset will make it such that if OffsetOutOfRange is ever +// encountered while consuming, rather than trying to recover, the client will +// return the error to the user and enter a fatal state (for the affected +// partition). +func NoResetOffset() Offset { + return Offset{ + at: -1, + epoch: -1, + noReset: true, + } +} + +// AfterMilli returns an offset that consumes from the first offset after a +// given timestamp. This option is *not* compatible with any At options (nor +// Relative nor WithEpoch); using any of those will clear the special +// millisecond state. +// +// This option can be used to consume at the end of existing partitions, but at +// the start of any new partitions that are created later: +// +// AfterMilli(time.Now().UnixMilli()) +// +// By default when using this offset, if consuming encounters an +// OffsetOutOfRange error, consuming will reset to the first offset after this +// timestamp. You can use NoResetOffset().AfterMilli(...) to instead switch the +// client to a fatal state (for the affected partition). +func (o Offset) AfterMilli(millisec int64) Offset { + o.at = millisec + o.relative = 0 + o.epoch = -1 + o.afterMilli = true + return o +} + +// AtStart copies 'o' and returns an offset starting at the beginning of a +// partition. +func (o Offset) AtStart() Offset { + o.afterMilli = false + o.at = -2 + return o +} + +// AtEnd copies 'o' and returns an offset starting at the end of a partition. +// If you want to consume at the end of the topic as it exists right now, but +// at the beginning of new partitions as they are added to the topic later, +// check out AfterMilli. +func (o Offset) AtEnd() Offset { + o.afterMilli = false + o.at = -1 + return o +} + +// AtCommitted copies 'o' and returns an offset that is used *only if* +// there is an existing commit. This is only useful for group consumers. +// If a partition being consumed does not have a commit, the partition will +// enter a fatal state and return an error from PollFetches. +// +// Using this function automatically opts into [NoResetOffset]. +func (o Offset) AtCommitted() Offset { + o.noReset = true + o.afterMilli = false + o.at = atCommitted + return o +} + +// Relative copies 'o' and returns an offset that starts 'n' relative to what +// 'o' currently is. If 'o' is at the end (from [AtEnd]), Relative(-100) will +// begin 100 before the end. +func (o Offset) Relative(n int64) Offset { + o.afterMilli = false + o.relative = n + return o +} + +// WithEpoch copies 'o' and returns an offset with the given epoch. This epoch +// is used for truncation detection; the default of -1 implies no truncation +// detection. +func (o Offset) WithEpoch(e int32) Offset { + o.afterMilli = false + if e < 0 { + e = -1 + } + o.epoch = e + return o +} + +// At returns a copy of the calling offset, changing the returned offset to +// begin at exactly the requested offset. +// +// There are two potential special offsets to use: -2 allows for consuming at +// the start, and -1 allows for consuming at the end. These two offsets are +// equivalent to calling AtStart or AtEnd. +// +// If the offset is less than -2, the client bounds it to -2 to consume at the +// start. +func (o Offset) At(at int64) Offset { + o.afterMilli = false + if at < -2 { + at = -2 + } + o.at = at + return o +} + +type consumer struct { + bufferedRecords atomic.Int64 + bufferedBytes atomic.Int64 + + cl *Client + + pausedMu xsync.Mutex // grabbed when updating paused + paused atomic.Value // loaded when issuing fetches + + // mu is grabbed when + // - polling fetches, for quickly draining sources / updating group uncommitted + // - calling assignPartitions (group / direct updates) + mu xsync.Mutex + d *directConsumer // if non-nil, we are consuming partitions directly + g *groupConsumer // if non-nil, we are consuming as a group member + s *shareConsumer // if non-nil, we are consuming as a share group member + + // On metadata update, if the consumer is set (direct or group), the + // client begins a goroutine that updates the consumer kind's + // assignments. + // + // This is done in a goroutine to not block the metadata loop, because + // the update **could** wait on a group consumer leaving if a + // concurrent LeaveGroup is called, or if restarting a session takes + // just a little bit of time. + // + // The update realistically should be instantaneous, but if it is slow, + // some metadata updates could pile up. We loop with our atomic work + // loop, which collapses repeated updates into one extra update, so we + // loop as little as necessary. + outstandingMetadataUpdates workLoop + + // pollActive / pollWake implement strict-pull gating when + // MaxConcurrentFetches == 0. See manageFetchConcurrency for the full + // description of the protocol; this pair is the input side. + pollActive atomic.Bool + pollWake chan struct{} // buffered(1); nil when maxConcurrentFetches != 0 + + // sessionChangeMu is grabbed when a session is stopped and held through + // when a session can be started again. The sole purpose is to block an + // assignment change running concurrently with a metadata update. + sessionChangeMu xsync.Mutex + + session atomic.Value // *consumerSession + kill atomic.Bool + + usingCursors usedCursors + + sourcesReadyMu xsync.Mutex + sourcesReadyCond *sync.Cond + sourcesReadyForDraining []*source + fakeReadyForDraining []Fetch + + pollWaitMu xsync.Mutex + pollWaitC *sync.Cond + pollWaitState uint64 // 0 == nothing, low 32 bits: # pollers, high 32: # waiting rebalances +} + +func (c *consumer) loadPaused() pausedTopics { return c.paused.Load().(pausedTopics) } +func (c *consumer) clonePaused() pausedTopics { return c.paused.Load().(pausedTopics).clone() } +func (c *consumer) storePaused(p pausedTopics) { c.paused.Store(p) } + +func (c *consumer) waitAndAddPoller() { + if !c.cl.cfg.blockRebalanceOnPoll { + return + } + c.pollWaitMu.Lock() + defer c.pollWaitMu.Unlock() + if c.pollWaitState&math.MaxUint32 == 0 { + for c.pollWaitState>>32 != 0 { + c.pollWaitC.Wait() + } + } + // Rebalance always takes priority, but if there are no active + // rebalances, our poll blocks rebalances. + c.pollWaitState++ +} + +func (c *consumer) unaddPoller() { + if !c.cl.cfg.blockRebalanceOnPoll { + return + } + c.pollWaitMu.Lock() + defer c.pollWaitMu.Unlock() + c.pollWaitState-- + c.pollWaitC.Broadcast() +} + +func (c *consumer) allowRebalance() { + if !c.cl.cfg.blockRebalanceOnPoll { + return + } + c.pollWaitMu.Lock() + defer c.pollWaitMu.Unlock() + // When allowing rebalances, the user is explicitly saying all pollers + // are done. We mask them out. + c.pollWaitState &= math.MaxUint32 << 32 + c.pollWaitC.Broadcast() +} + +func (c *consumer) waitAndAddRebalance() { + c.waitAndAddRebalanceMaybeSignal(true) +} + +// waitAndAddRebalanceSilent is waitAndAddRebalance without the +// OnPartitionsCallbackBlocked signal. Used by LeaveGroup: the gate guards +// assignPartitions invalidation rather than a user callback, so signaling +// "your callback is blocked" would be misleading. +func (c *consumer) waitAndAddRebalanceSilent() { + c.waitAndAddRebalanceMaybeSignal(false) +} + +func (c *consumer) waitAndAddRebalanceMaybeSignal(signal bool) { + if !c.cl.cfg.blockRebalanceOnPoll { + return + } + var blockedCalled bool + c.pollWaitMu.Lock() + defer c.pollWaitMu.Unlock() + c.pollWaitState += 1 << 32 + for c.pollWaitState&math.MaxUint32 != 0 { + if signal && !blockedCalled { + if c.cl.cfg.onBlocked != nil { + go c.cl.cfg.onBlocked(c.cl.ctx, c.cl) + } + blockedCalled = true + } + c.pollWaitC.Wait() + } +} + +func (c *consumer) unaddRebalance() { + if !c.cl.cfg.blockRebalanceOnPoll { + return + } + c.pollWaitMu.Lock() + defer c.pollWaitMu.Unlock() + c.pollWaitState -= 1 << 32 + c.pollWaitC.Broadcast() +} + +// BufferedFetchRecords returns the number of records currently buffered from +// fetching within the client. +// +// This can be used as a gauge to determine how behind your application is for +// processing records the client has fetched. Note that it is perfectly normal +// to see a spike of buffered records, which would correspond to a fetch +// response being processed just before a call to this function. It is only +// problematic if for you if this function is consistently returning large +// values. +func (cl *Client) BufferedFetchRecords() int64 { + return cl.consumer.bufferedRecords.Load() +} + +// BufferedFetchBytes returns the number of bytes currently buffered from +// fetching within the client. This is the sum of all keys, values, and header +// keys/values. See the related [BufferedFetchRecords] for more information. +func (cl *Client) BufferedFetchBytes() int64 { + return cl.consumer.bufferedBytes.Load() +} + +type usedCursors map[*cursor]struct{} + +func (u *usedCursors) use(c *cursor) { + if *u == nil { + *u = make(map[*cursor]struct{}) + } + (*u)[c] = struct{}{} +} + +func (c *consumer) init(cl *Client) { + c.cl = cl + c.paused.Store(make(pausedTopics)) + c.sourcesReadyCond = sync.NewCond(&c.sourcesReadyMu) + c.pollWaitC = sync.NewCond(&c.pollWaitMu) + if cl.cfg.maxConcurrentFetches == 0 { + c.pollWake = make(chan struct{}, 1) + } + + if len(cl.cfg.topics) > 0 || len(cl.cfg.partitions) > 0 { + defer cl.triggerUpdateMetadataNow("querying metadata for consumer initialization") // we definitely want to trigger a metadata update + } + + if len(cl.cfg.shareGroup) > 0 { + c.initShare() + } else if len(cl.cfg.group) == 0 { + c.initDirect() + } else { + c.initGroup() + } +} + +func (c *consumer) consuming() bool { + return c.g != nil || c.d != nil || c.s != nil +} + +// addSourceReadyForDraining tracks that a source needs its buffered fetch +// consumed. +func (c *consumer) addSourceReadyForDraining(source *source) { + c.sourcesReadyMu.Lock() + c.sourcesReadyForDraining = append(c.sourcesReadyForDraining, source) + c.sourcesReadyMu.Unlock() + c.sourcesReadyCond.Broadcast() +} + +// addFakeReadyForDraining saves a fake fetch that has important partition +// errors--data loss or auth failures. +func (c *consumer) addFakeReadyForDraining(topic string, partition int32, err error, why string) { + c.cl.cfg.logger.Log(LogLevelInfo, "injecting fake fetch with an error", "err", err, "why", why) + c.sourcesReadyMu.Lock() + c.fakeReadyForDraining = append(c.fakeReadyForDraining, Fetch{Topics: []FetchTopic{{ + Topic: topic, + Partitions: []FetchPartition{{ + Partition: partition, + Err: err, + }}, + }}}) + c.sourcesReadyMu.Unlock() + c.sourcesReadyCond.Broadcast() +} + +// NewErrFetch returns a fake fetch containing a single empty topic with a +// single zero partition with the given error. +func NewErrFetch(err error) Fetches { + return []Fetch{{ + Topics: []FetchTopic{{ + Topic: "", + Partitions: []FetchPartition{{ + Partition: -1, + Err: err, + }}, + }}, + }} +} + +// PollFetches waits for fetches to be available, returning as soon as any +// broker returns a fetch. If the context is nil, this function will return +// immediately with any currently buffered records. It is functionally +// equivalent to calling PollRecords(ctx, 0). +// +// If the client is closed, a fake fetch will be injected that has no topic, a +// partition of 0, and a partition error of ErrClientClosed. If the context is +// canceled, a fake fetch will be injected with ctx.Err. These injected errors +// can be used to break out of a poll loop. +// +// It is important to check all partition errors in the returned fetches. If +// any partition has a fatal error and actually had no records, fake fetch will +// be injected with the error. +// +// If you are group consuming, a rebalance can happen under the hood while you +// process the returned fetches. This can result in duplicate work, and you may +// accidentally commit to partitions that you no longer own. You can prevent +// this by using BlockRebalanceOnPoll, but this comes with different tradeoffs. +// See the documentation on BlockRebalanceOnPoll for more information. +func (cl *Client) PollFetches(ctx context.Context) Fetches { + return cl.PollRecords(ctx, 0) +} + +// PollRecords waits for fetches to be available, returning as soon as any +// broker returns a fetch. If the context is nil, this function will return +// immediately with any currently buffered fetches. +// +// If the client is closed, a fake fetch will be injected that has no topic, a +// partition of -1, and a partition error of ErrClientClosed. If the context is +// canceled, a fake fetch will be injected with ctx.Err. These injected errors +// can be used to break out of a poll loop. +// +// This returns a maximum of maxPollRecords total across all fetches, or +// returns all buffered records if maxPollRecords is <= 0. +// +// It is important to check all partition errors in the returned fetches. If +// any partition has a fatal error and actually had no records, fake fetch will +// be injected with the error. +// +// If you are group consuming, a rebalance can happen under the hood while you +// process the returned fetches. This can result in duplicate work, and you may +// accidentally commit to partitions that you no longer own. You can prevent +// this by using BlockRebalanceOnPoll, but this comes with different tradeoffs. +// See the documentation on BlockRebalanceOnPoll for more information. +func (cl *Client) PollRecords(ctx context.Context, maxPollRecords int) Fetches { + cl.cfg.hooks.each(func(h Hook) { + if hh, ok := h.(HookPollStart); ok { + hh.OnPollStart(ctx) + } + }) + + if maxPollRecords == 0 { + maxPollRecords = -1 + } + c := &cl.consumer + + if c.pollWake != nil { + // Store before sending on pollWake: a successful send + // synchronizes-before the receive, so the reader's subsequent + // pollActive.Load observes this Store. If the send hits the + // default (buffer already full), ordering is irrelevant -- + // manageFetchConcurrency reloads pollActive on its next event. + c.pollActive.Store(true) + select { + case c.pollWake <- struct{}{}: + default: + } + defer func() { + c.pollActive.Store(false) + select { + case c.pollWake <- struct{}{}: + default: + } + }() + } + + if c.s != nil { + return c.s.poll(ctx, maxPollRecords) + } + + c.g.undirtyUncommitted() + + // If the user gave us a canceled context, we bail immediately after + // un-dirty-ing marked records. We still need to add a poller to block + // rebalances if configured, since we are returning a fetch. + if ctx != nil { + select { + case <-ctx.Done(): + c.waitAndAddPoller() + return NewErrFetch(ctx.Err()) + default: + } + } + + var fetches Fetches + fill := func() { + if c.cl.cfg.blockRebalanceOnPoll { + c.waitAndAddPoller() + defer func() { + if len(fetches) == 0 { + c.unaddPoller() + } + }() + } + + paused := c.loadPaused() + + // A group can grab the consumer lock then the group mu and + // assign partitions. The group mu is grabbed to update its + // uncommitted map. Assigning partitions clears sources ready + // for draining. + // + // We need to grab the consumer mu to ensure proper lock + // ordering and prevent lock inversion. Polling fetches also + // updates the group's uncommitted map; if we do not grab the + // consumer mu at the top, we have a problem: without the lock, + // we could have grabbed some sources, then a group assigned, + // and after the assign, we update uncommitted with fetches + // from the old assignment + c.mu.Lock() + defer c.mu.Unlock() + + c.sourcesReadyMu.Lock() + if maxPollRecords < 0 { + for _, ready := range c.sourcesReadyForDraining { + fetches = append(fetches, ready.takeBuffered(paused)) + } + c.sourcesReadyForDraining = nil + } else { + for len(c.sourcesReadyForDraining) > 0 && maxPollRecords > 0 { + source := c.sourcesReadyForDraining[0] + fetch, taken, drained := source.takeNBuffered(paused, maxPollRecords) + if drained { + c.sourcesReadyForDraining = c.sourcesReadyForDraining[1:] + } + maxPollRecords -= taken + fetches = append(fetches, fetch) + } + } + + realFetches := fetches + + fetches = append(fetches, c.fakeReadyForDraining...) + c.fakeReadyForDraining = nil + + c.sourcesReadyMu.Unlock() + + if len(realFetches) == 0 { + return + } + + // Before returning, we want to update our uncommitted. If we + // updated after, then we could end up with weird interactions + // with group invalidations where we return a stale fetch after + // committing in onRevoke. + // + // A blocking onRevoke commit, on finish, allows a new group + // session to start. If we returned stale fetches that did not + // have their uncommitted offset tracked, then we would allow + // duplicates. + if c.g != nil { + c.g.updateUncommitted(realFetches) + } + } + + // We try filling fetches once before waiting. If we have no context, + // we guarantee that we just drain anything available and return. + fill() + if len(fetches) > 0 || ctx == nil { + return fetches + } + + done := make(chan struct{}) + quit := false + go func() { + c.sourcesReadyMu.Lock() + defer c.sourcesReadyMu.Unlock() + defer close(done) + + for !quit && len(c.sourcesReadyForDraining) == 0 && len(c.fakeReadyForDraining) == 0 { + c.sourcesReadyCond.Wait() + } + }() + + exit := func() { + c.sourcesReadyMu.Lock() + quit = true + c.sourcesReadyMu.Unlock() + c.sourcesReadyCond.Broadcast() + } + + select { + case <-cl.ctx.Done(): + exit() + c.waitAndAddPoller() + return NewErrFetch(ErrClientClosed) + case <-ctx.Done(): + exit() + c.waitAndAddPoller() + return NewErrFetch(ctx.Err()) + case <-done: + } + + fill() + return fetches +} + +// AllowRebalance allows a consumer group to rebalance if it was blocked by you +// polling records in tandem with the BlockRebalanceOnPoll option. +// +// You can poll many times before calling this function; this function +// internally resets the poll count and allows any blocked rebalances to +// continue. Rebalances take priority: if a rebalance is blocked, and you allow +// rebalances and then immediately poll, your poll will be blocked until the +// rebalance completes. Internally, this function simply waits for lost +// partitions to stop being fetched before allowing you to poll again. +func (cl *Client) AllowRebalance() { + cl.consumer.allowRebalance() +} + +// UpdateFetchMaxBytes updates the max bytes that a fetch request will ask for +// and the max partition bytes that a fetch request will ask for each +// partition. +func (cl *Client) UpdateFetchMaxBytes(maxBytes, maxPartBytes int32) { + cl.cfg.maxBytes.store(maxBytes) + cl.cfg.maxPartBytes.store(maxPartBytes) +} + +// PauseFetchTopics sets the client to no longer fetch the given topics and +// returns all currently paused topics. Paused topics persist until resumed. +// You can call this function with no topics to simply receive the list of +// currently paused topics. +// +// Pausing topics is independent from pausing individual partitions with the +// PauseFetchPartitions method. If you pause partitions for a topic with +// PauseFetchPartitions, and then pause that same topic with PauseFetchTopics, +// the individually paused partitions will not be unpaused if you only call +// ResumeFetchTopics. +func (cl *Client) PauseFetchTopics(topics ...string) []string { + c := &cl.consumer + if len(topics) == 0 { + return c.loadPaused().pausedTopics() + } + c.pausedMu.Lock() + defer c.pausedMu.Unlock() + paused := c.clonePaused() + paused.addTopics(topics...) + c.storePaused(paused) + return paused.pausedTopics() +} + +// PauseFetchPartitions sets the client to no longer fetch the given partitions +// and returns all currently paused partitions. Paused partitions persist until +// resumed. You can call this function with no partitions to simply receive the +// list of currently paused partitions. +// +// Pausing individual partitions is independent from pausing topics with the +// PauseFetchTopics method. If you pause partitions for a topic with +// PauseFetchPartitions, and then pause that same topic with PauseFetchTopics, +// the individually paused partitions will not be unpaused if you only call +// ResumeFetchTopics. +func (cl *Client) PauseFetchPartitions(topicPartitions map[string][]int32) map[string][]int32 { + c := &cl.consumer + if len(topicPartitions) == 0 { + return c.loadPaused().pausedPartitions() + } + c.pausedMu.Lock() + defer c.pausedMu.Unlock() + paused := c.clonePaused() + paused.addPartitions(topicPartitions) + c.storePaused(paused) + return paused.pausedPartitions() +} + +// ResumeFetchTopics resumes fetching the input topics if they were previously +// paused. Resuming topics that are not currently paused is a per-topic no-op. +// See the documentation on PauseFetchTopics for more details. +func (cl *Client) ResumeFetchTopics(topics ...string) { + defer cl.allSources(func(s *source) { + s.maybeConsume() + }) + + c := &cl.consumer + c.pausedMu.Lock() + defer c.pausedMu.Unlock() + + paused := c.clonePaused() + paused.delTopics(topics...) + c.storePaused(paused) +} + +// ResumeFetchPartitions resumes fetching the input partitions if they were +// previously paused. Resuming partitions that are not currently paused is a +// per-topic no-op. See the documentation on PauseFetchPartitions for more +// details. +func (cl *Client) ResumeFetchPartitions(topicPartitions map[string][]int32) { + defer cl.allSources(func(s *source) { + s.maybeConsume() + }) + + c := &cl.consumer + c.pausedMu.Lock() + defer c.pausedMu.Unlock() + + paused := c.clonePaused() + paused.delPartitions(topicPartitions) + c.storePaused(paused) +} + +// SetOffsets sets any matching offsets in setOffsets to the given +// epoch/offset. Partitions that are not specified are not set. It is invalid +// to set topics that were not yet returned from a PollFetches: this function +// sets only partitions that were previously consumed, any extra partitions are +// skipped. +// +// If directly consuming, this function operates as expected given the caveats +// of the prior paragraph. +// +// If using transactions, it is advised to just use a GroupTransactSession and +// avoid this function entirely. +// +// If using group consuming, it is strongly recommended to use this function +// outside of the context of a PollFetches loop and only when you know the +// group is not revoked (i.e., block any concurrent revoke while issuing this +// call) and to not use this concurrent with committing. Any other usage is +// prone to odd interactions. +func (cl *Client) SetOffsets(setOffsets map[string]map[int32]EpochOffset) { + cl.setOffsets(setOffsets, true) +} + +func (cl *Client) setOffsets(setOffsets map[string]map[int32]EpochOffset, log bool) { + if len(setOffsets) == 0 { + return + } + + // We assignPartitions before returning, so we grab the consumer lock + // first to preserve consumer mu => group mu ordering, or to ensure + // no concurrent metadata assign for direct consuming. + c := &cl.consumer + c.mu.Lock() + defer c.mu.Unlock() + + var assigns map[string]map[int32]Offset + var tps *topicsPartitions + switch { + case c.d != nil: + assigns = c.d.applySetOffsets(setOffsets) + tps = c.d.tps + case c.g != nil: + assigns = c.g.applySetOffsets(setOffsets) + tps = c.g.tps + } + if len(assigns) == 0 { + return + } + if log { + c.assignPartitions(assigns, assignSetMatching, tps, "from manual SetOffsets") + } else { + c.assignPartitions(assigns, assignSetMatching, tps, "") + } +} + +// This is guaranteed to be called in a blocking metadata fn, which ensures +// that metadata does not load the tps we are changing. Basically, we ensure +// everything w.r.t. consuming is at a stand still. +func (c *consumer) purgeTopics(topics []string) { + if c.g == nil && c.d == nil && c.s == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.s != nil { + c.s.purgeTopics(topics) // save a useless map alloc by doing this early, not in the block below + return + } + + purgeAssignments := make(map[string]map[int32]Offset, len(topics)) + for _, topic := range topics { + purgeAssignments[topic] = nil + } + + // assignPartitions removes the topics from 'tps', which removes them + // from FUTURE metadata requests meaning they will not be repopulated + // in the future. Any loaded tps is fine; metadata updates the topic + // pointers underneath -- metadata does not re-store the tps itself + // with any new topics that we just purged (except in the case of regex, + // which is impossible to permanently purge anyway). + // + // We are guarded from adding back to 'using' via the consumer mu; + // this cannot run concurrent with findNewAssignments. Thus, we first + // purge tps, then clear using, and once the lock releases, findNewAssignments + // will use the now-purged tps and will not add back to using. + if c.g != nil { + c.g.mu.Lock() // required when updating using + defer c.g.mu.Unlock() + c.assignPartitions(purgeAssignments, assignPurgeMatching, c.g.tps, fmt.Sprintf("purge of %v requested", topics)) + for _, topic := range topics { + delete(c.g.using, topic) + delete(c.g.reSeen, topic) + } + c.g.rejoin("rejoin from PurgeFetchTopics") + } else { + c.assignPartitions(purgeAssignments, assignPurgeMatching, c.d.tps, fmt.Sprintf("purge of %v requested", topics)) + for _, topic := range topics { + delete(c.d.using, topic) + delete(c.d.reSeen, topic) + delete(c.d.m, topic) + delete(c.d.ps, topic) + } + } +} + +// AddConsumeTopics adds new topics to be consumed. This function is a no-op if +// the client is configured to consume via regex. +// +// Note that if you are directly consuming and specified ConsumePartitions, +// this function will not add the rest of the partitions for a topic unless the +// topic has been previously purged. That is, if you directly consumed only one +// of five partitions originally, this will not add the other four until the +// entire topic is purged. +func (cl *Client) AddConsumeTopics(topics ...string) { + c := &cl.consumer + if len(topics) == 0 || c.g == nil && c.d == nil || cl.cfg.regex { + return + } + + // We can do this outside of the metadata loop because we are strictly + // adding new topics and forbid regex consuming. + c.mu.Lock() + defer c.mu.Unlock() + + if c.g != nil { + c.g.tps.storeTopics(topics) + } else if c.s != nil { + c.s.tps.storeTopics(topics) + } else { + c.d.tps.storeTopics(topics) + for _, topic := range topics { + c.d.m.addt(topic) + } + } + cl.triggerUpdateMetadataNow("from AddConsumeTopics") +} + +// GetConsumeTopics retrieves a list of current topics being consumed. +func (cl *Client) GetConsumeTopics() []string { + c := &cl.consumer + if c.g == nil && c.d == nil { + return nil + } + var m map[string]*topicPartitions + var ok bool + if c.g != nil { + m, ok = c.g.tps.v.Load().(topicsPartitionsData) + } else if c.s != nil { + m, ok = c.s.tps.v.Load().(topicsPartitionsData) + } else { + m, ok = c.d.tps.v.Load().(topicsPartitionsData) + } + if !ok { + return nil + } + topics := make([]string, 0, len(m)) + for k := range m { + topics = append(topics, k) + } + return topics +} + +// AddConsumePartitions adds new partitions to be consumed at the given +// offsets. This function works only for direct, non-regex consumers. +func (cl *Client) AddConsumePartitions(partitions map[string]map[int32]Offset) { + c := &cl.consumer + if c.d == nil || cl.cfg.regex { + return + } + var topics []string + for t, ps := range partitions { + if len(ps) == 0 { + delete(partitions, t) + continue + } + topics = append(topics, t) + } + if len(partitions) == 0 { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.d.tps.storeTopics(topics) + for t, ps := range partitions { + if c.d.ps[t] == nil { + c.d.ps[t] = make(map[int32]Offset) + } + for p, o := range ps { + c.d.m.add(t, p) + c.d.ps[t][p] = o + } + } + cl.triggerUpdateMetadataNow("from AddConsumePartitions") +} + +// RemoveConsumePartitions removes partitions from being consumed. This +// function works only for direct, non-regex consumers. +// +// This method does not purge the concept of any topics from the client -- if +// you remove all partitions from a topic that was being consumed, metadata +// fetches will still occur for the topic. If you want to remove the topic +// entirely, use PurgeTopicsFromClient. +// +// If you specified ConsumeTopics and this function removes all partitions for +// a topic, the topic will no longer be consumed. +func (cl *Client) RemoveConsumePartitions(partitions map[string][]int32) { + c := &cl.consumer + if c.d == nil || cl.cfg.regex { + return + } + for t, ps := range partitions { + if len(ps) == 0 { + delete(partitions, t) + continue + } + } + if len(partitions) == 0 { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + removeOffsets := make(map[string]map[int32]Offset, len(partitions)) + for t, ps := range partitions { + removePartitionOffsets := make(map[int32]Offset, len(ps)) + for _, p := range ps { + removePartitionOffsets[p] = Offset{} + } + removeOffsets[t] = removePartitionOffsets + } + + c.assignPartitions(removeOffsets, assignInvalidateMatching, c.d.tps, fmt.Sprintf("remove of %v requested", partitions)) + for t, ps := range partitions { + for _, p := range ps { + c.d.using.remove(t, p) + c.d.m.remove(t, p) + delete(c.d.ps[t], p) + } + if len(c.d.ps[t]) == 0 { + delete(c.d.ps, t) + } + } +} + +// assignHow controls how assignPartitions operates. +type assignHow int8 + +const ( + // This option simply assigns new offsets, doing nothing with existing + // offsets / active fetches / buffered fetches. + assignWithoutInvalidating assignHow = iota + + // This option invalidates active fetches so they will not buffer and + // drops all buffered fetches, and then continues to assign the new + // assignments. + assignInvalidateAll + + // This option does not assign, but instead invalidates any active + // fetches for "assigned" (actually lost) partitions. This additionally + // drops all buffered fetches, because they could contain partitions we + // lost. Thus, with this option, the actual offset in the map is + // meaningless / a dummy offset. + assignInvalidateMatching + + assignPurgeMatching + + // The counterpart to assignInvalidateMatching, assignSetMatching + // resets all matching partitions to the specified offset / epoch. + assignSetMatching +) + +func (h assignHow) String() string { + switch h { + case assignWithoutInvalidating: + return "assigning everything new, keeping current assignment" + case assignInvalidateAll: + return "unassigning everything" + case assignInvalidateMatching: + return "unassigning any currently assigned matching partition that is in the input" + case assignPurgeMatching: + return "unassigning and purging any partition matching the input topics" + case assignSetMatching: + return "reassigning any currently assigned matching partition to the input" + } + return "" +} + +type fmtAssignment map[string]map[int32]Offset + +func (f fmtAssignment) String() string { + var sb strings.Builder + + var topicsWritten int + for topic, partitions := range f { + topicsWritten++ + sb.WriteString(topic) + sb.WriteString("[") + + var partitionsWritten int + for partition, offset := range partitions { + fmt.Fprintf(&sb, "%d%s", partition, offset) + partitionsWritten++ + if partitionsWritten < len(partitions) { + sb.WriteString(" ") + } + } + + sb.WriteString("]") + if topicsWritten < len(f) { + sb.WriteString(", ") + } + } + + return sb.String() +} + +// assignPartitions, called under the consumer's mu, is used to set new cursors +// or add to the existing cursors. +// +// We do not need to pass tps when we are bumping the session or when we are +// invalidating all. All other cases, we want the tps -- the logic below does +// not fully differentiate needing to start a new session vs. just reusing the +// old (third if case below) +func (c *consumer) assignPartitions(assignments map[string]map[int32]Offset, how assignHow, tps *topicsPartitions, why string) { + if c.mu.TryLock() { + c.mu.Unlock() + panic("assignPartitions called without holding the consumer lock, this is a bug in franz-go, please open an issue at github.com/twmb/franz-go") + } + + // The internal code can avoid giving an assign reason in cases where + // the caller logs itself immediately before assigning. We only log if + // there is a reason. + if len(why) > 0 && c.cl.cfg.logger.Level() >= LogLevelInfo { + c.cl.cfg.logger.Log(LogLevelInfo, "assigning partitions", + "why", why, + "how", how, + "input", fmtAssignment(assignments), + ) + } + var session *consumerSession + var loadOffsets listOrEpochLoads + + defer func() { + if session == nil { // if nil, we stopped the session + session = c.startNewSession(tps) + } else { // else we guarded it + c.unguardSessionChange(session) + } + loadOffsets.loadWithSession(session, "loading offsets in new session from assign") // odds are this assign came from a metadata update, so no reason to force a refresh with loadWithSessionNow + + // If we started a new session or if we unguarded, we have one + // worker. This one worker allowed us to safely add our load + // offsets before the session could be concurrently stopped + // again. Now that we have added the load offsets, we allow the + // session to be stopped. + session.decWorker() + }() + + if how == assignWithoutInvalidating { + // Guarding a session change can actually create a new session + // if we had no session before, which is why we need to pass in + // our topicPartitions. + session = c.guardSessionChange(tps) + } else { + loadOffsets, _ = c.stopSession() + + // First, over all cursors currently in use, we unset them or set them + // directly as appropriate. Anything we do not unset, we keep. + + var keep usedCursors + for usedCursor := range c.usingCursors { + shouldKeep := true + if how == assignInvalidateAll { + usedCursor.unset() + shouldKeep = false + } else { // invalidateMatching or setMatching + if assignTopic, ok := assignments[usedCursor.topic]; ok { + if how == assignPurgeMatching { // topic level + usedCursor.source.removeCursor(usedCursor) + shouldKeep = false + } else if assignPart, ok := assignTopic[usedCursor.partition]; ok { + if how == assignInvalidateMatching { + usedCursor.unset() + shouldKeep = false + } else { // how == assignSetMatching + usedCursor.setOffset(cursorOffset{ + offset: assignPart.at, + lastConsumedEpoch: assignPart.epoch, + }) + } + } + } + } + if shouldKeep { + keep.use(usedCursor) + } + } + c.usingCursors = keep + + // For any partition that was listing offsets or loading + // epochs, we want to ensure that if we are keeping those + // partitions, we re-start the list/load. + // + // Note that we do not need to unset cursors here; anything + // that actually resulted in a cursor is forever tracked in + // usedCursors. We only do not have used cursors if an + // assignment went straight to listing / epoch loading, and + // that list/epoch never finished. + switch how { + case assignWithoutInvalidating: + // Nothing to do -- this is handled above. + case assignInvalidateAll: + loadOffsets = listOrEpochLoads{} + case assignSetMatching: + // We had not yet loaded this partition, so there is + // nothing to set, and we keep everything. + case assignInvalidateMatching: + loadOffsets.keepFilter(func(t string, p int32) bool { + if assignTopic, ok := assignments[t]; ok { + if _, ok := assignTopic[p]; ok { + return false + } + } + return true + }) + case assignPurgeMatching: + // This is slightly different than invalidate in that + // we invalidate whole topics. + loadOffsets.keepFilter(func(t string, _ int32) bool { + _, ok := assignments[t] + return !ok // assignments are topics to purge -- do NOT keep the topic if it is being purged + }) + // We have to purge from tps _after_ the session is + // stopped. If we purge early while the session is + // ongoing, then another goroutine could be loading and + // using tps and expecting topics not yet removed from + // assignPartitions to still be there. Specifically, + // mapLoadsToBrokers could be expecting topic foo to be + // there (from the session!), so if we purge foo before + // stopping the session, we will panic. + topics := make([]string, 0, len(assignments)) + for t := range assignments { + topics = append(topics, t) + } + tps.purgeTopics(topics) + } + } + + // This assignment could contain nothing (for the purposes of + // invalidating active fetches), so we only do this if needed. + if len(assignments) == 0 || how != assignWithoutInvalidating { + return + } + + c.cl.cfg.logger.Log(LogLevelDebug, "assign requires loading offsets") + + // We could have a prior handleListOrEpochResults finishing concurrent + // with a new assignment. We need to guard below, because list/epoch + // results can set offsets for loaded cursors. + // + // Example: I was just cooperatively assigned p1o10e3, and I issued an + // epoch request to validate data loss. I was then assigned p2o4e-1: + // there is no epoch to validate, so we immediately begin consuming + // at the offset. The epoch request returns and tries also using the + // cursor concurrently. We need to guard it. + session.listOrEpochMu.Lock() + defer session.listOrEpochMu.Unlock() + + topics := tps.load() + for topic, partitions := range assignments { + topicPartitions := topics.loadTopic(topic) // should be non-nil + if topicPartitions == nil { + c.cl.cfg.logger.Log(LogLevelError, "BUG! consumer was assigned topic that we did not ask for in ConsumeTopics nor ConsumePartitions, skipping!", "topic", topic) + continue + } + + for partition, offset := range partitions { + // If we are loading the first record after a millisec, + // we go directly to listing offsets. Epoch validation + // does not ever set afterMilli. + if offset.afterMilli { + loadOffsets.addLoad(topic, partition, loadTypeList, offsetLoad{ + replica: -1, + Offset: offset, + }) + continue + } + + // First, if the request is exact, get rid of the relative + // portion. We are modifying a copy of the offset, i.e. we + // are appropriately not modifying 'assignments' itself. + if offset.at >= 0 { + offset.at += offset.relative + if offset.at < 0 { + offset.at = 0 + } + offset.relative = 0 + } + + // If we are requesting an exact offset with an epoch, + // we do truncation detection and then use the offset. + // + // Otherwise, an epoch is specified without an exact + // request which is useless for us, or a request is + // specified without a known epoch. + // + // The client ensures the epoch is non-negative from + // fetch offsets only if the broker supports KIP-320, + // but we do not override the user manually specifying + // an epoch. + if offset.at >= 0 && offset.epoch >= 0 { + loadOffsets.addLoad(topic, partition, loadTypeEpoch, offsetLoad{ + replica: -1, + Offset: offset, + }) + continue + } + + // If an exact offset is specified and we have loaded + // the partition, we use it. We have to use epoch -1 + // rather than the latest loaded epoch on the partition + // because the offset being requested to use could be + // from an epoch after OUR loaded epoch. Otherwise, we + // could update the metadata, see the later epoch, + // request the end offset for our prior epoch, and then + // think data loss occurred. + // + // If an offset is unspecified or we have not loaded + // the partition, we list offsets to find out what to + // use. + if offset.at >= 0 && partition >= 0 && partition < int32(len(topicPartitions.partitions)) { + part := topicPartitions.partitions[partition] + cursor := part.cursor + cursor.setOffset(cursorOffset{ + offset: offset.at, + lastConsumedEpoch: -1, + }) + cursor.allowUsable() + c.usingCursors.use(cursor) + continue + } + + // If the offset is atCommitted, then no offset was + // loaded from FetchOffsets. We inject an error and + // avoid using this partition. + if offset.at == atCommitted { + c.addFakeReadyForDraining(topic, partition, errNoCommittedOffset, "notification of uncommitted partition") + continue + } + + loadOffsets.addLoad(topic, partition, loadTypeList, offsetLoad{ + replica: -1, + Offset: offset, + }) + } + } +} + +// filterMetadataAllTopics, called BEFORE doOnMetadataUpdate, evaluates +// all topics received against the user provided regex. +func (c *consumer) filterMetadataAllTopics(topics []string) []string { + c.mu.Lock() + defer c.mu.Unlock() + + var rns reNews + defer rns.log(&c.cl.cfg) + + var reSeen map[string]bool + if c.g != nil { + reSeen = c.g.reSeen + } else if c.s != nil { + reSeen = c.s.reSeen + } else { + reSeen = c.d.reSeen + } + + keep := topics[:0] + for _, topic := range topics { + want, seen := reSeen[topic] + if !seen { + for rawRe, re := range c.cl.cfg.topics { + if want = re.MatchString(topic); want { + rns.add(rawRe, topic) + break + } + } + if want { + for _, re := range c.cl.cfg.excludeTopics { + if re.MatchString(topic) { + want = false + break + } + } + } + if !want { + rns.skip(topic) + } + reSeen[topic] = want + } + if want { + keep = append(keep, topic) + } + } + return keep +} + +func (c *consumer) doOnMetadataUpdate() { + if !c.consuming() { + return + } + + // See the comment on the outstandingMetadataUpdates field for why this + // block below. + if c.outstandingMetadataUpdates.maybeBegin() { + doUpdate := func() { + // We forbid reassignments while we do a quick check for + // new assignments--for the direct consumer particularly, + // this prevents TOCTOU, and guards against a concurrent + // assignment from SetOffsets. + c.mu.Lock() + defer c.mu.Unlock() + + switch { + case c.d != nil: + if new := c.d.findNewAssignments(); len(new) > 0 { + c.assignPartitions(new, assignWithoutInvalidating, c.d.tps, "new assignments from direct consumer") + } + case c.g != nil: + c.g.findNewAssignments() + case c.s != nil: + c.s.maybeStartManage() + } + + go c.loadSession().doOnMetadataUpdate() + } + + go func() { + again := true + for again { + doUpdate() + again = c.outstandingMetadataUpdates.maybeFinish(false) + } + }() + } +} + +func (s *consumerSession) doOnMetadataUpdate() { + if s == nil || s == noConsumerSession { // no session started yet + return + } + + s.listOrEpochMu.Lock() + defer s.listOrEpochMu.Unlock() + + if s.listOrEpochMetaCh == nil { + return // nothing waiting to load epochs / offsets + } + select { + case s.listOrEpochMetaCh <- struct{}{}: + default: + } +} + +type offsetLoadMap map[string]map[int32]offsetLoad + +// offsetLoad is effectively an Offset, but also includes a potential replica +// to directly use if a cursor had a preferred replica. +type offsetLoad struct { + replica int32 // -1 means leader + Offset +} + +func (o offsetLoad) MarshalJSON() ([]byte, error) { + if o.replica == -1 { + return o.Offset.MarshalJSON() + } + if o.relative == 0 { + return fmt.Appendf(nil, `{"Replica":%d,"At":%d,"Epoch":%d,"CurrentEpoch":%d}`, o.replica, o.at, o.epoch, o.currentEpoch), nil + } + return fmt.Appendf(nil, `{"Replica":%d,"At":%d,"Relative":%d,"Epoch":%d,"CurrentEpoch":%d}`, o.replica, o.at, o.relative, o.epoch, o.currentEpoch), nil +} + +func (o offsetLoadMap) errToLoaded(err error) []loadedOffset { + var loaded []loadedOffset + for t, ps := range o { + for p, o := range ps { + loaded = append(loaded, loadedOffset{ + topic: t, + partition: p, + err: err, + request: o, + }) + } + } + return loaded +} + +// Combines list and epoch loads into one type for simplicity. +type listOrEpochLoads struct { + // List and Epoch are public so that anything marshaling through + // reflect (i.e. json) can see the fields. + List offsetLoadMap + Epoch offsetLoadMap +} + +type listOrEpochLoadType uint8 + +const ( + loadTypeList listOrEpochLoadType = iota + loadTypeEpoch +) + +func (l listOrEpochLoadType) String() string { + switch l { + case loadTypeList: + return "list" + default: + return "epoch" + } +} + +// adds an offset to be loaded, ensuring it exists only in the final loadType. +func (l *listOrEpochLoads) addLoad(t string, p int32, loadType listOrEpochLoadType, load offsetLoad) { + l.removeLoad(t, p) + dst := &l.List + if loadType == loadTypeEpoch { + dst = &l.Epoch + } + + if *dst == nil { + *dst = make(offsetLoadMap) + } + ps := (*dst)[t] + if ps == nil { + ps = make(map[int32]offsetLoad) + (*dst)[t] = ps + } + ps[p] = load +} + +func (l *listOrEpochLoads) removeLoad(t string, p int32) { + for _, m := range []offsetLoadMap{ + l.List, + l.Epoch, + } { + if m == nil { + continue + } + ps := m[t] + if ps == nil { + continue + } + delete(ps, p) + if len(ps) == 0 { + delete(m, t) + } + } +} + +func (l listOrEpochLoads) each(fn func(string, int32)) { + for _, m := range []offsetLoadMap{ + l.List, + l.Epoch, + } { + for topic, partitions := range m { + for partition := range partitions { + fn(topic, partition) + } + } + } +} + +func (l *listOrEpochLoads) keepFilter(keep func(string, int32) bool) { + for _, m := range []offsetLoadMap{ + l.List, + l.Epoch, + } { + for t, ps := range m { + for p := range ps { + if !keep(t, p) { + delete(ps, p) + if len(ps) == 0 { + delete(m, t) + } + } + } + } + } +} + +// Merges loads into the caller; used to coalesce loads while a metadata update +// is happening (see the only use below). +func (l *listOrEpochLoads) mergeFrom(src listOrEpochLoads) { + for _, srcs := range []struct { + m offsetLoadMap + loadType listOrEpochLoadType + }{ + {src.List, loadTypeList}, + {src.Epoch, loadTypeEpoch}, + } { + for t, ps := range srcs.m { + for p, load := range ps { + l.addLoad(t, p, srcs.loadType, load) + } + } + } +} + +func (l listOrEpochLoads) isEmpty() bool { return len(l.List) == 0 && len(l.Epoch) == 0 } + +func (l listOrEpochLoads) loadWithSession(s *consumerSession, why string) { + if !l.isEmpty() { + s.incWorker() + go s.listOrEpoch(l, false, why) + } +} + +func (l listOrEpochLoads) loadWithSessionNow(s *consumerSession, why string) bool { + if !l.isEmpty() { + s.incWorker() + go s.listOrEpoch(l, true, why) + return true + } + return false +} + +// fetchManager controls fetch concurrency. Sources register their desire to +// fetch, and the manager grants permission up to the configured limit. This +// type is shared by consumerSession (regular consumers) and shareConsumer +// (share groups). +type fetchManager struct { + ctx context.Context + cancel func() + + // pollActive + pollWake are the consumer's strict-pull-mode state + // (MaxConcurrentFetches == 0). Both are nil otherwise. The atomic + // is the ground truth ("a poll is in progress"); the channel kicks + // the select loop to re-read it. See consumer.pollActive comment. + pollActive *atomic.Bool + pollWake chan struct{} + + // We receive desires from sources, we reply when they can fetch, and + // they send back when they are done. Thus, three level chan. + desireFetchCh chan chan chan bool + cancelFetchCh chan chan chan bool + allowedFetches int + fetchManagerStarted atomic.Bool // atomic, once true, we start the fetch manager +} + +func newFetchManager(ctx context.Context, cancel func(), pollActive *atomic.Bool, pollWake chan struct{}, maxConcurrentFetches int) fetchManager { + return fetchManager{ + ctx: ctx, + cancel: cancel, + + pollActive: pollActive, + pollWake: pollWake, + + // NOTE: This channel must be unbuffered. If it is buffered, + // then we can exit manageFetchConcurrency when we should not + // and have a deadlock: + // + // * source sends to desireFetchCh, is buffered + // * source seeds context canceled, tries sending to cancelFetchCh + // * session concurrently sees context canceled + // * session has not drained desireFetchCh, sees activeFetches is 0 + // * session exits + // * source permanently hangs sending to desireFetchCh + // + // By having desireFetchCh unbuffered, we *ensure* that if the + // source indicates it wants a fetch, the session knows it and + // tracks it in wantFetch. + // + // See #198. + desireFetchCh: make(chan chan chan bool), + + cancelFetchCh: make(chan chan chan bool, 4), + allowedFetches: maxConcurrentFetches, + } +} + +func (fm *fetchManager) desireFetch() chan chan chan bool { + if !fm.fetchManagerStarted.Swap(true) { + go fm.manageFetchConcurrency() + } + return fm.desireFetchCh +} + +func (fm *fetchManager) manageFetchConcurrency() { + var ( + activeFetches int + doneFetch = make(chan bool, 20) + wantFetch []chan chan bool + pollAllowed bool + ctxCh = fm.ctx.Done() + wantQuit bool + ) + + for { + select { + case register := <-fm.desireFetchCh: + wantFetch = append(wantFetch, register) + case cancel := <-fm.cancelFetchCh: + var found bool + for i, want := range wantFetch { + if want == cancel { + wantFetch = slices.Delete(wantFetch, i, i+1) + found = true + break + } + } + // If we did not find the channel, then we have already + // sent to it, removed it from our wantFetch list, and + // bumped activeFetches. + if !found { + activeFetches-- + } + + case <-doneFetch: + activeFetches-- + case <-ctxCh: + wantQuit = true + ctxCh = nil + case <-fm.pollWake: + // Wake only; the post-select Load below picks up + // the current pollActive value. + } + + // pollActive is the ground truth; re-read after every event, + // not only on the pollWake case. Two reasons the Load cannot + // move into the kick case: + // 1. newFetchManager may be constructed after PollRecords has + // already run (pollActive==true, wake already consumed or + // silently dropped into a full buffer). The first + // iteration needs to observe pollActive=true without a + // wake to unblock first. + // 2. pollActive can flip true => false between the wake's + // buffer send and our Load. Re-reading on every event + // (desireFetch, cancel, doneFetch, ctx) ensures we never + // treat a stale "true" as authoritative for gating the + // next fetch. + // Missed wakes (buffer-full default) are harmless because the + // next event triggers another Load. + if fm.pollActive != nil { + pollAllowed = fm.pollActive.Load() + } + + if len(wantFetch) > 0 && (activeFetches < fm.allowedFetches || pollAllowed && activeFetches == 0 || fm.allowedFetches < 0) { // negative means unbounded + wantFetch[0] <- doneFetch + wantFetch = wantFetch[1:] + activeFetches++ + continue + } + + if wantQuit && activeFetches == 0 { + return + } + } +} + +// A consumer session is responsible for an era of fetching records for a set +// of cursors. The set can be added to without killing an active session, but +// it cannot be removed from. Removing any cursor from being consumed kills the +// current consumer session and begins a new one. +type consumerSession struct { + c *consumer + + // tps tracks the topics that were assigned in this session. We use + // this field to build and handle list offset / load epoch requests. + tps *topicsPartitions + + fetchManager + + // Workers signify the number of fetch and list / epoch goroutines that + // are currently running within the context of this consumer session. + // Stopping a session only returns once workers hits zero. + workersMu xsync.Mutex + workersCond *sync.Cond + workers int + + // listOrEpochMu largely guards the below. It is a sub-mutex of the + // consumer mutex to guard one concurrent data access (see below in + // assignPartitions). + listOrEpochMu xsync.Mutex + listOrEpochLoadsWaiting listOrEpochLoads + listOrEpochMetaCh chan struct{} // non-nil if Loads is non-nil, signalled on meta update + listOrEpochLoadsLoading listOrEpochLoads +} + +func (c *consumer) newConsumerSession(tps *topicsPartitions) *consumerSession { + if tps == nil || len(tps.load()) == 0 { + return noConsumerSession + } + ctx, cancel := context.WithCancel(c.cl.ctx) + session := &consumerSession{ + c: c, + tps: tps, + fetchManager: newFetchManager(ctx, cancel, &c.pollActive, c.pollWake, c.cl.cfg.maxConcurrentFetches), + } + session.workersCond = sync.NewCond(&session.workersMu) + return session +} + +func (s *consumerSession) incWorker() { + if s == noConsumerSession { // from startNewSession + return + } + s.workersMu.Lock() + defer s.workersMu.Unlock() + s.workers++ +} + +func (s *consumerSession) decWorker() { + if s == noConsumerSession { // from followup to startNewSession + return + } + s.workersMu.Lock() + defer s.workersMu.Unlock() + s.workers-- + if s.workers == 0 { + s.workersCond.Broadcast() + } +} + +// noConsumerSession exists because we cannot store nil into an atomic.Value. +var noConsumerSession = new(consumerSession) + +func (c *consumer) loadSession() *consumerSession { + if session := c.session.Load(); session != nil { + return session.(*consumerSession) + } + return noConsumerSession +} + +// Guards against a session being stopped, and must be paired with an unguard. +// This returns a new session if there was no session. +// +// The purpose of this function is when performing additive-only changes to an +// existing session, because additive-only changes can avoid killing a running +// session. +func (c *consumer) guardSessionChange(tps *topicsPartitions) *consumerSession { + c.sessionChangeMu.Lock() + + session := c.loadSession() + if session == noConsumerSession { + // If there is no session, we simply store one. This is fine; + // sources will be able to begin a fetch loop, but they will + // have no cursors to consume yet. + session = c.newConsumerSession(tps) + c.session.Store(session) + } + + return session +} + +// For the same reason below as in startNewSession, we inc a worker before +// unguarding. This allows the unguarding to execute a bit of logic if +// necessary before the session can be stopped. +func (c *consumer) unguardSessionChange(session *consumerSession) { + session.incWorker() + c.sessionChangeMu.Unlock() +} + +// Stops an active consumer session if there is one, and does not return until +// all fetching, listing, offset for leader epoching is complete. This +// invalidates any buffered fetches for the previous session and returns any +// partitions that were listing offsets or loading epochs. +func (c *consumer) stopSession() (listOrEpochLoads, *topicsPartitions) { + c.sessionChangeMu.Lock() + + session := c.loadSession() + + if session == noConsumerSession { + return listOrEpochLoads{}, nil // we had no session + } + + // Before storing noConsumerSession, cancel our old. This pairs + // with the reverse ordering in source, which checks noConsumerSession + // then checks the session context. + session.cancel() + + // At this point, any in progress fetches, offset lists, or epoch loads + // will quickly die. + + c.session.Store(noConsumerSession) + + // At this point, no source can be started, because the session is + // noConsumerSession. + + session.workersMu.Lock() + for session.workers > 0 { + session.workersCond.Wait() + } + session.workersMu.Unlock() + + // At this point, all fetches, lists, and loads are dead. We can close + // our num-fetches manager without worrying about a source trying to + // register itself. + + c.cl.allSources(func(s *source) { + s.session.reset() + }) + + // At this point, if we begin fetching anew, then the sources will not + // be using stale fetch sessions. + + c.sourcesReadyMu.Lock() + defer c.sourcesReadyMu.Unlock() + for _, ready := range c.sourcesReadyForDraining { + ready.discardBuffered() + } + c.sourcesReadyForDraining = nil + + // At this point, we have invalidated any buffered data from the prior + // session. We deliberately leave c.fakeReadyForDraining so the user can + // still observe errors that happened in the dying session (data loss, + // list/epoch failures, no-committed-offset notices, etc.). A consequence + // is that the user's next poll may surface an error for a partition the + // new assignment does not include; callers that key off fake-fetch errors + // for committing must check current ownership before acting. The session + // is dead. + + session.listOrEpochLoadsWaiting.mergeFrom(session.listOrEpochLoadsLoading) + return session.listOrEpochLoadsWaiting, session.tps +} + +// Starts a new consumer session, allowing fetches to happen. +// +// If there are no topic partitions to start with, this returns noConsumerSession. +// +// This is returned with 1 worker; decWorker must be called after return. The +// 1 worker allows for initialization work to prevent the session from being +// immediately stopped. +func (c *consumer) startNewSession(tps *topicsPartitions) *consumerSession { + if c.kill.Load() { + tps = nil + } + session := c.newConsumerSession(tps) + c.session.Store(session) + + // Ensure that this session is usable before being stopped immediately. + // The caller must dec workers. + session.incWorker() + + // At this point, sources can start consuming. + + c.sessionChangeMu.Unlock() + + c.cl.allSources(func(s *source) { + s.maybeConsume() + }) + + // At this point, any source that was not consuming because it saw the + // session was stopped has been notified to potentially start consuming + // again. The session is alive. + + return session +} + +// This function is responsible for issuing ListOffsets or +// OffsetForLeaderEpoch. These requests's responses are only handled within +// the context of a consumer session. +func (s *consumerSession) listOrEpoch(waiting listOrEpochLoads, immediate bool, why string) { + defer s.decWorker() + + // It is possible for a metadata update to try to migrate partition + // loads if the update moves partitions between brokers. If we are + // closing the client, the consumer session could already be stopped, + // but this stops before the metadata goroutine is killed. So, if we + // are in this function but actually have no session, we return. + if s == noConsumerSession { + return + } + + // We must set up listOrEpochMetaCh BEFORE triggering metadata. If we + // trigger first and set the channel second, there is a race where the + // metadata update completes and doOnMetadataUpdate checks for the + // channel before we create it, causing the signal to be lost. With the + // channel created first, doOnMetadataUpdate will always find the + // channel and signal it. + // + // Race without this ordering: + // 1. listOrEpoch: triggerUpdateMetadata -> sends trigger + // 2. listOrEpoch: goroutine descheduled under load + // 3. metadata loop: processes trigger, runs doOnMetadataUpdate + // 4. doOnMetadataUpdate: listOrEpochMetaCh is nil -> returns (signal lost) + // 5. listOrEpoch: resumes, creates listOrEpochMetaCh, waits forever + s.listOrEpochMu.Lock() // collapse any listOrEpochs that occur during meta update into one + if !s.listOrEpochLoadsWaiting.isEmpty() { + s.listOrEpochLoadsWaiting.mergeFrom(waiting) + s.listOrEpochMu.Unlock() + return + } + s.listOrEpochLoadsWaiting = waiting + s.listOrEpochMetaCh = make(chan struct{}, 1) + s.listOrEpochMu.Unlock() + + wait := true + if immediate { + s.c.cl.triggerUpdateMetadataNow(why) + } else { + wait = s.c.cl.triggerUpdateMetadata(false, why) // avoid trigger if within refresh interval + } + + if wait { + select { + case <-s.ctx.Done(): + return + case <-s.listOrEpochMetaCh: + } + } + + s.listOrEpochMu.Lock() + loading := s.listOrEpochLoadsWaiting + s.listOrEpochLoadsLoading.mergeFrom(loading) + s.listOrEpochLoadsWaiting = listOrEpochLoads{} + s.listOrEpochMetaCh = nil + s.listOrEpochMu.Unlock() + + brokerLoads := s.mapLoadsToBrokers(loading) + + results := make(chan loadedOffsets, 2*len(brokerLoads)) // each broker can receive up to two requests + + var issued, received int + for broker, brokerLoad := range brokerLoads { + s.c.cl.cfg.logger.Log(LogLevelDebug, "offsets to load broker", "broker", broker.meta.NodeID, "load", brokerLoad) + if len(brokerLoad.List) > 0 { + issued++ + go s.c.cl.listOffsetsForBrokerLoad(s.ctx, broker, brokerLoad.List, s.tps, results) + } + if len(brokerLoad.Epoch) > 0 { + issued++ + go s.c.cl.loadEpochsForBrokerLoad(s.ctx, broker, brokerLoad.Epoch, s.tps, results) + } + } + + var reloads listOrEpochLoads + defer func() { + if !reloads.isEmpty() { + s.incWorker() + go func() { + // Before we dec our worker, we must add the + // reloads back into the session's waiting loads. + // Doing so allows a concurrent stopSession to + // track the waiting loads, whereas if we did not + // add things back to the session, we could abandon + // loading these offsets and have a stuck cursor. + defer s.decWorker() + defer reloads.loadWithSession(s, "reload offsets from load failure") + after := time.NewTimer(time.Second) + defer after.Stop() + select { + case <-after.C: + case <-s.ctx.Done(): + return + } + }() + } + }() + + // We must drain all results before returning so that the + // sub-goroutines complete within this worker's lifetime. If the + // session is stopped, the context cancellation propagates to each + // sub-goroutine's broker.waitResp, so they will finish quickly. + // Without draining, stopSession can return (having seen workers=0) + // and purgeTopics can modify tps while a sub-goroutine still + // references it. + for received != issued { + loaded := <-results + received++ + reloads.mergeFrom(s.handleListOrEpochResults(loaded)) + } +} + +// Called within a consumer session, this function handles results from list +// offsets or epoch loads and returns any loads that should be retried. +// +// To us, all errors are reloadable. We either have request level retryable +// errors (unknown partition, etc) or non-retryable errors (auth), or we have +// request issuing errors (no dial, connection cut repeatedly). +// +// For retryable request errors, we may as well back off a little bit to allow +// Kafka to harmonize if the topic exists / etc. +// +// For non-retryable request errors, we may as well retry to both (a) allow the +// user more signals about a problem that they can maybe fix within Kafka (i.e. +// the auth), and (b) force the user to notice errors. +// +// For request issuing errors, we may as well continue to retry because there +// is not much else we can do. RequestWith already retries, but returns when +// the retry limit is hit. We will backoff 1s and then allow RequestWith to +// continue requesting and backing off. +func (s *consumerSession) handleListOrEpochResults(loaded loadedOffsets) (reloads listOrEpochLoads) { + // This function can be running twice concurrently, so we need to guard + // listOrEpochLoadsLoading and usingCursors. For simplicity, we just + // guard this entire function. + + debug := s.c.cl.cfg.logger.Level() >= LogLevelDebug + + var using map[string]map[int32]EpochOffset + type epochOffsetWhy struct { + EpochOffset + error + } + var reloading map[string]map[int32]epochOffsetWhy + if debug { + using = make(map[string]map[int32]EpochOffset) + reloading = make(map[string]map[int32]epochOffsetWhy) + defer func() { + t := "list" + if loaded.loadType == loadTypeEpoch { + t = "epoch" + } + s.c.cl.cfg.logger.Log(LogLevelDebug, fmt.Sprintf("handled %s results", t), "broker", logID(loaded.broker), "using", using, "reloading", reloading) + }() + } + + s.listOrEpochMu.Lock() + defer s.listOrEpochMu.Unlock() + + for _, load := range loaded.loaded { + s.listOrEpochLoadsLoading.removeLoad(load.topic, load.partition) // remove the tracking of this load from our session + + use := func() { + if debug { + tusing := using[load.topic] + if tusing == nil { + tusing = make(map[int32]EpochOffset) + using[load.topic] = tusing + } + tusing[load.partition] = EpochOffset{load.leaderEpoch, load.offset} + } + + // Preserve lastConsumedTime (for AfterMilli fallback on + // subsequent OffsetOutOfRange) and hwm (for lag metrics) + // across validation loads. The cursor is currently + // unusable (we're about to allowUsable it), so reading + // the prior cursorOffset here is safe: no concurrent + // fetch observes useState=true yet. + prior := load.cursor.cursorOffset + load.cursor.setOffset(cursorOffset{ + offset: load.offset, + lastConsumedEpoch: load.leaderEpoch, + lastConsumedTime: prior.lastConsumedTime, + hwm: prior.hwm, + }) + load.cursor.allowUsable() + s.c.usingCursors.use(load.cursor) + } + + var edl *ErrDataLoss + switch { + case errors.As(load.err, &edl): + s.c.addFakeReadyForDraining(load.topic, load.partition, load.err, "notification of data loss") // signal we lost data, but set the cursor to what we can + use() + + case load.err == nil: + use() + + default: // from ErrorCode in a response, or broker request err, or request is canceled as our session is ending + reloads.addLoad(load.topic, load.partition, loaded.loadType, load.request) + if !kerr.IsRetriable(load.err) && !isRetryableBrokerErr(load.err) && !isDialNonTimeoutErr(load.err) && !isContextErr(load.err) { // non-retryable response error; signal such in a response + s.c.addFakeReadyForDraining(load.topic, load.partition, load.err, fmt.Sprintf("notification of non-retryable error from %s request", loaded.loadType)) + } + + if debug { + treloading := reloading[load.topic] + if treloading == nil { + treloading = make(map[int32]epochOffsetWhy) + reloading[load.topic] = treloading + } + treloading[load.partition] = epochOffsetWhy{EpochOffset{load.leaderEpoch, load.offset}, load.err} + } + } + } + + return reloads +} + +// Splits the loads into per-broker loads, mapping each partition to the broker +// that leads that partition. +func (s *consumerSession) mapLoadsToBrokers(loads listOrEpochLoads) map[*broker]listOrEpochLoads { + brokerLoads := make(map[*broker]listOrEpochLoads) + + s.c.cl.brokersMu.RLock() // hold mu so we can check if partition leaders exist + defer s.c.cl.brokersMu.RUnlock() + + brokers := s.c.cl.brokers + seed := s.c.cl.loadSeeds()[0] + + topics := s.tps.load() + for _, loads := range []struct { + m offsetLoadMap + loadType listOrEpochLoadType + }{ + {loads.List, loadTypeList}, + {loads.Epoch, loadTypeEpoch}, + } { + for topic, partitions := range loads.m { + topicPartitions := topics.loadTopic(topic) // this must exist, it not existing would be a bug + for partition, offset := range partitions { + // We default to the first seed broker if we have no loaded + // the broker leader for this partition (we should have). + // Worst case, we get an error for the partition and retry. + broker := seed + if partition >= 0 && partition < int32(len(topicPartitions.partitions)) { + topicPartition := topicPartitions.partitions[partition] + brokerID := topicPartition.leader + if offset.replica != -1 { + // If we are fetching from a follower, we can list + // offsets against the follower itself. The replica + // being non-negative signals that. + // + // Note this is not actually true (i.e. KIP-392 lies), + // but we keep this logic in case we can revert + // to using non-leaders someday. + brokerID = offset.replica + } + if tryBroker := findBroker(brokers, brokerID); tryBroker != nil { + broker = tryBroker + } + offset.currentEpoch = topicPartition.leaderEpoch // ensure we set our latest epoch for the partition + } + + brokerLoad := brokerLoads[broker] + brokerLoad.addLoad(topic, partition, loads.loadType, offset) + brokerLoads[broker] = brokerLoad + } + } + } + + return brokerLoads +} + +// The result of ListOffsets or OffsetForLeaderEpoch for an individual +// partition. +type loadedOffset struct { + topic string + partition int32 + + // The following three are potentially unset if the error is non-nil + // and not ErrDataLoss; these are what we loaded. + cursor *cursor + offset int64 + leaderEpoch int32 + + // Any error encountered for loading this partition, or for epoch + // loading, potentially ErrDataLoss. If this error is not retryable, we + // avoid reloading the offset and instead inject a fake partition for + // PollFetches containing this error. + err error + + // The original request. + request offsetLoad +} + +// The results of ListOffsets or OffsetForLeaderEpoch for an individual broker. +type loadedOffsets struct { + broker int32 + loaded []loadedOffset + loadType listOrEpochLoadType +} + +func (l *loadedOffsets) add(a loadedOffset) { l.loaded = append(l.loaded, a) } +func (l *loadedOffsets) addAll(as []loadedOffset) loadedOffsets { + l.loaded = append(l.loaded, as...) + return *l +} + +func (cl *Client) listOffsetsForBrokerLoad(ctx context.Context, broker *broker, load offsetLoadMap, tps *topicsPartitions, results chan<- loadedOffsets) { + loaded := loadedOffsets{broker: broker.meta.NodeID, loadType: loadTypeList} + + req1, req2 := load.buildListReq(cl.cfg.isolationLevel) + var ( + wg sync.WaitGroup + kresp2 kmsg.Response + err2 error + ) + if req2 != nil { + wg.Add(1) + go func() { + defer wg.Done() + kresp2, err2 = broker.waitResp(ctx, req2) + }() + } + kresp, err := broker.waitResp(ctx, req1) + wg.Wait() + if err != nil || err2 != nil { + if err == nil { + err = err2 + } + results <- loaded.addAll(load.errToLoaded(err)) + return + } + + topics := tps.load() + resp := kresp.(*kmsg.ListOffsetsResponse) + + // If we issued a second req to check that an exact offset is in + // bounds, then regrettably for safety, we have to ensure that the + // shapes of both responses match, and the topic & partition at each + // index matches. Anything that does not match is skipped (and would be + // a bug from Kafka), and we at the end return UnknownTopicOrPartition. + var resp2 *kmsg.ListOffsetsResponse + if req2 != nil { + resp2 = kresp2.(*kmsg.ListOffsetsResponse) + for _, r := range []*kmsg.ListOffsetsResponse{ + resp, + resp2, + } { + ts := r.Topics + sort.Slice(ts, func(i, j int) bool { + return ts[i].Topic < ts[j].Topic + }) + for i := range ts { + ps := ts[i].Partitions + sort.Slice(ps, func(i, j int) bool { + return ps[i].Partition < ps[j].Partition + }) + } + } + + lt := resp.Topics + rt := resp2.Topics + lkeept := lt[:0] + rkeept := rt[:0] + // Over each response, we only keep the topic if the topics match. + for len(lt) > 0 && len(rt) > 0 { + if lt[0].Topic < rt[0].Topic { + lt = lt[1:] + continue + } + if rt[0].Topic < lt[0].Topic { + rt = rt[1:] + continue + } + // As well, for topics that match, we only keep + // partitions that match. In this case, we also want + // both partitions to be error free, otherwise we keep + // an error on both. If one has old style offsets, + // both must. + lp := lt[0].Partitions + rp := rt[0].Partitions + lkeepp := lp[:0] + rkeepp := rp[:0] + for len(lp) > 0 && len(rp) > 0 { + if lp[0].Partition < rp[0].Partition { + lp = lp[1:] + continue + } + if rp[0].Partition < lp[0].Partition { + rp = rp[1:] + continue + } + if len(lp[0].OldStyleOffsets) > 0 && len(rp[0].OldStyleOffsets) == 0 || + len(lp[0].OldStyleOffsets) == 0 && len(rp[0].OldStyleOffsets) > 0 { + lp = lp[1:] + rp = rp[1:] + continue + } + if lp[0].ErrorCode != 0 { + rp[0].ErrorCode = lp[0].ErrorCode + } else if rp[0].ErrorCode != 0 { + lp[0].ErrorCode = rp[0].ErrorCode + } + lkeepp = append(lkeepp, lp[0]) + rkeepp = append(rkeepp, rp[0]) + lp = lp[1:] + rp = rp[1:] + } + // Now we update the partitions in the topic we are + // keeping, and keep our topic. + lt[0].Partitions = lkeepp + rt[0].Partitions = rkeepp + lkeept = append(lkeept, lt[0]) + rkeept = append(rkeept, rt[0]) + lt = lt[1:] + rt = rt[1:] + } + // Finally, update each response with the topics we kept. The + // shapes and indices are the same. + resp.Topics = lkeept + resp2.Topics = rkeept + } + + poffset := func(p *kmsg.ListOffsetsResponseTopicPartition) int64 { + offset := p.Offset + if len(p.OldStyleOffsets) > 0 { + offset = p.OldStyleOffsets[0] // list offsets v0 + } + return offset + } + + for i, rTopic := range resp.Topics { + topic := rTopic.Topic + loadParts, ok := load[topic] + if !ok { + continue // should not happen: kafka replied with something we did not ask for + } + + topicPartitions := topics.loadTopic(topic) // must be non-nil at this point + for j, rPartition := range rTopic.Partitions { + partition := rPartition.Partition + loadPart, ok := loadParts[partition] + if !ok { + continue // should not happen: kafka replied with something we did not ask for + } + + if err := kerr.ErrorForCode(rPartition.ErrorCode); err != nil { + loaded.add(loadedOffset{ + topic: topic, + partition: partition, + err: err, + request: loadPart, + }) + continue // partition err: handled in results + } + + if partition < 0 || partition >= int32(len(topicPartitions.partitions)) { + continue // should not happen: we have not seen this partition from a metadata response + } + topicPartition := topicPartitions.partitions[partition] + + delete(loadParts, partition) + if len(loadParts) == 0 { + delete(load, topic) + } + + offset := poffset(&rPartition) + end := func() int64 { return poffset(&resp2.Topics[i].Partitions[j]) } + + // We ensured the resp2 shape is as we want and has no + // error, so resp2 lookups are safe. + if loadPart.afterMilli { + // If after a milli, if the milli is after the + // end of a partition, the offset is -1. We use + // our end offset request: anything after the + // end offset *now* is after our milli. + if offset == -1 { + offset = end() + } + } else if loadPart.at >= 0 { + // If an exact offset, we listed start and end. + // We validate the offset is within bounds. + end := end() + want := loadPart.at + loadPart.relative + if want >= offset { + offset = want + } + if want >= end { + offset = end + } + } else if loadPart.at == -2 && loadPart.relative > 0 { + // Relative to the start: both start & end were + // issued, and we bound to the end. + offset += loadPart.relative + if end := end(); offset >= end { + offset = end + } + } else if loadPart.at == -1 && loadPart.relative < 0 { + // Relative to the end: both start & end were + // issued, offset is currently the start, so we + // set to the end and then bound to the start. + start := offset + offset = end() + offset += loadPart.relative + if offset <= start { + offset = start + } + } + if offset < 0 { + offset = 0 // sanity + } + + loaded.add(loadedOffset{ + topic: topic, + partition: partition, + cursor: topicPartition.cursor, + offset: offset, + leaderEpoch: rPartition.LeaderEpoch, + request: loadPart, + }) + } + } + + results <- loaded.addAll(load.errToLoaded(kerr.UnknownTopicOrPartition)) +} + +func (*Client) loadEpochsForBrokerLoad(ctx context.Context, broker *broker, load offsetLoadMap, tps *topicsPartitions, results chan<- loadedOffsets) { + loaded := loadedOffsets{broker: broker.meta.NodeID, loadType: loadTypeEpoch} + + kresp, err := broker.waitResp(ctx, load.buildEpochReq()) + if err != nil { + results <- loaded.addAll(load.errToLoaded(err)) + return + } + + // If the version is < 2, we are speaking to an old broker. We should + // not have an old version, but we could have spoken to a new broker + // first then an old broker in the middle of a broker roll. For now, we + // will just loop retrying until the broker is upgraded. + + topics := tps.load() + resp := kresp.(*kmsg.OffsetForLeaderEpochResponse) + for _, rTopic := range resp.Topics { + topic := rTopic.Topic + loadParts, ok := load[topic] + if !ok { + continue // should not happen: kafka replied with something we did not ask for + } + + topicPartitions := topics.loadTopic(topic) // must be non-nil at this point + for _, rPartition := range rTopic.Partitions { + partition := rPartition.Partition + loadPart, ok := loadParts[partition] + if !ok { + continue // should not happen: kafka replied with something we did not ask for + } + + if err := kerr.ErrorForCode(rPartition.ErrorCode); err != nil { + loaded.add(loadedOffset{ + topic: topic, + partition: partition, + err: err, + request: loadPart, + }) + continue // partition err: handled in results + } + + if partition < 0 || partition >= int32(len(topicPartitions.partitions)) { + continue // should not happen: we have not seen this partition from a metadata response + } + topicPartition := topicPartitions.partitions[partition] + + delete(loadParts, partition) + if len(loadParts) == 0 { + delete(load, topic) + } + + // Epoch loading never uses noReset nor afterMilli; + // this at is the offset we wanted to consume and are + // validating. + offset := loadPart.at + var err error + if rPartition.EndOffset < offset { + err = &ErrDataLoss{topic, partition, offset, loadPart.epoch, rPartition.EndOffset, rPartition.LeaderEpoch} + offset = rPartition.EndOffset + } + + loaded.add(loadedOffset{ + topic: topic, + partition: partition, + cursor: topicPartition.cursor, + offset: offset, + leaderEpoch: rPartition.LeaderEpoch, + err: err, + request: loadPart, + }) + } + } + + results <- loaded.addAll(load.errToLoaded(kerr.UnknownTopicOrPartition)) +} + +// In general this returns one request, but if the user is using exact offsets +// rather than start/end, then we issue both the start and end requests to +// ensure the user's requested offset is within bounds. +func (o offsetLoadMap) buildListReq(isolationLevel int8) (r1, r2 *kmsg.ListOffsetsRequest) { + r1 = kmsg.NewPtrListOffsetsRequest() + r1.ReplicaID = -1 + r1.IsolationLevel = isolationLevel + r1.Topics = make([]kmsg.ListOffsetsRequestTopic, 0, len(o)) + var createEnd bool + for topic, partitions := range o { + parts := make([]kmsg.ListOffsetsRequestTopicPartition, 0, len(partitions)) + for partition, offset := range partitions { + // If this is a milli request, we issue two lists: if + // our milli is after the end of a partition, we get no + // offset back and we want to know the start offset + // (since it will be after our milli). + // + // If we are using an exact offset request, we issue + // the start and end so that we can bound the exact + // offset to being within that range. + // + // If we are using a relative offset, we potentially + // issue the end request because relative may shift us + // too far in the other direction. + timestamp := offset.at + if offset.afterMilli { + createEnd = true + } else if timestamp >= 0 || timestamp == -2 && offset.relative > 0 || timestamp == -1 && offset.relative < 0 { + timestamp = -2 + createEnd = true + } + p := kmsg.NewListOffsetsRequestTopicPartition() + p.Partition = partition + p.CurrentLeaderEpoch = offset.currentEpoch // KIP-320 + p.Timestamp = timestamp + p.MaxNumOffsets = 1 + + parts = append(parts, p) + } + t := kmsg.NewListOffsetsRequestTopic() + t.Topic = topic + t.Partitions = parts + r1.Topics = append(r1.Topics, t) + } + + if createEnd { + r2 = kmsg.NewPtrListOffsetsRequest() + *r2 = *r1 + r2.Topics = slices.Clone(r1.Topics) + for i := range r1.Topics { + l := &r2.Topics[i] + r := &r1.Topics[i] + *l = *r + l.Partitions = slices.Clone(r.Partitions) + for i := range l.Partitions { + l.Partitions[i].Timestamp = -1 + } + } + } + + return r1, r2 +} + +func (o offsetLoadMap) buildEpochReq() *kmsg.OffsetForLeaderEpochRequest { + req := kmsg.NewPtrOffsetForLeaderEpochRequest() + req.ReplicaID = -1 + req.Topics = make([]kmsg.OffsetForLeaderEpochRequestTopic, 0, len(o)) + for topic, partitions := range o { + parts := make([]kmsg.OffsetForLeaderEpochRequestTopicPartition, 0, len(partitions)) + for partition, offset := range partitions { + p := kmsg.NewOffsetForLeaderEpochRequestTopicPartition() + p.Partition = partition + p.CurrentLeaderEpoch = offset.currentEpoch + p.LeaderEpoch = offset.epoch + parts = append(parts, p) + } + t := kmsg.NewOffsetForLeaderEpochRequestTopic() + t.Topic = topic + t.Partitions = parts + req.Topics = append(req.Topics, t) + } + return req +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_direct.go b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_direct.go new file mode 100644 index 00000000000..ca2c6d9b792 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_direct.go @@ -0,0 +1,141 @@ +package kgo + +import "maps" + +type directConsumer struct { + cfg *cfg + tps *topicsPartitions // data for topics that the user assigned + using mtmps // topics we are currently using + m mtmps // mirrors cfg.topics and cfg.partitions, but can change with Purge or Add + ps map[string]map[int32]Offset // mirrors cfg.partitions, changed in Purge or Add, for direct partition consuming + reSeen map[string]bool // topics we evaluated against regex, and whether we want them or not +} + +func (c *consumer) initDirect() { + d := &directConsumer{ + cfg: &c.cl.cfg, + tps: newTopicsPartitions(), + reSeen: make(map[string]bool), + using: make(mtmps), + m: make(mtmps), + ps: make(map[string]map[int32]Offset), + } + c.d = d + + if d.cfg.regex { + return + } + + var topics []string + for topic, partitions := range d.cfg.partitions { + topics = append(topics, topic) + for partition := range partitions { + d.m.add(topic, partition) + } + p := make(map[int32]Offset, len(partitions)) + maps.Copy(p, partitions) + d.ps[topic] = p + } + for topic := range d.cfg.topics { + topics = append(topics, topic) + d.m.addt(topic) + } + d.tps.storeTopics(topics) // prime topics to load if non-regex (this is of no benefit if regex) +} + +// applySetOffsets for a direct consumer blindly translates EpochOffsets into +// Offsets. Unlike the group consumer, there is no uncommitted map to check. +func (*directConsumer) applySetOffsets(setOffsets map[string]map[int32]EpochOffset) (assigns map[string]map[int32]Offset) { + assigns = make(map[string]map[int32]Offset) + for topic, partitions := range setOffsets { + set := make(map[int32]Offset) + for partition, eo := range partitions { + set[partition] = Offset{ + at: eo.Offset, + epoch: eo.Epoch, + } + } + assigns[topic] = set + } + return assigns +} + +// findNewAssignments returns new partitions to consume at given offsets +// based off the current topics. +func (d *directConsumer) findNewAssignments() map[string]map[int32]Offset { + topics := d.tps.load() + + toUse := make(map[string]map[int32]Offset, 10) + for topic, topicPartitions := range topics { + var useTopic bool + if d.cfg.regex { + useTopic = d.reSeen[topic] + } else { + useTopic = d.m.onlyt(topic) + } + + // If the above detected that we want to keep this topic, we + // set all partitions as usable. + // + // For internal partitions, we only allow consuming them if + // the topic is explicitly specified. + if !useTopic { + continue + } + partitions := topicPartitions.load() + if d.cfg.regex && partitions.isInternal || len(partitions.partitions) == 0 { + continue + } + toUseTopic := make(map[int32]Offset, len(partitions.partitions)) + for partition := range partitions.partitions { + toUseTopic[int32(partition)] = d.cfg.startOffset + } + toUse[topic] = toUseTopic + } + + // If any topic has specific partitions pinned (from ConsumePartitions + // or AddConsumePartitions), add them. + for topic := range d.m { + for partition, offset := range d.ps[topic] { + toUseTopic, exists := toUse[topic] + if !exists { + toUseTopic = make(map[int32]Offset, 10) + toUse[topic] = toUseTopic + } + toUseTopic[partition] = offset + } + } + + // With everything we want to consume, remove what we are already. + for topic, partitions := range d.using { + toUseTopic, exists := toUse[topic] + if !exists { + continue // metadata update did not return this topic (regex or failing load) + } + for partition := range partitions { + delete(toUseTopic, partition) + } + if len(toUseTopic) == 0 { + delete(toUse, topic) + } + } + + if len(toUse) == 0 { + return nil + } + + // Finally, toUse contains new partitions that we must consume. + // Add them to our using map and assign them. + for topic, partitions := range toUse { + topicUsing, exists := d.using[topic] + if !exists { + topicUsing = make(map[int32]struct{}) + d.using[topic] = topicUsing + } + for partition := range partitions { + topicUsing[partition] = struct{}{} + } + } + + return toUse +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group.go b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group.go new file mode 100644 index 00000000000..7ccc5d195e6 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group.go @@ -0,0 +1,3437 @@ +package kgo + +import ( + "bytes" + "context" + "errors" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +type groupConsumer struct { + c *consumer // used to change consumer state; generally c.mu is grabbed on access + cl *Client // used for running requests / adding to topics map + cfg *cfg + + ctx context.Context + cancel func() + manageDone chan struct{} // closed once when the manage goroutine quits + + cooperative atomic.Bool // true if the group balancer chosen during Join is cooperative + + // The data for topics that the user assigned. Metadata updates the + // atomic.Value in each pointer atomically. + // + // We initialize tps with zero-value *topicPartitions in initGroup + // if we are directly consuming topics. If we are regex consuming, + // the metadata loop itself requests all topics, filters the topics + // against our regex, and then adds matching topics to tps. + // + // This, effectively, is the set of all candidate topics we could + // theoretically consume. Purging topics removes from this. + tps *topicsPartitions + + reSeen map[string]bool // topics we evaluated against regex, and whether we want them or not + + // Full lock grabbed in CommitOffsetsSync, read lock grabbed in + // CommitOffsets, this lock ensures that only one sync commit can + // happen at once, and if it is happening, no other commit can be + // happening. + syncCommitMu xsync.RWMutex + + rejoinCh chan string // cap 1; sent to if subscription changes (regex) + + // For EOS, before we commit, we force a heartbeat. If the client and + // group member are both configured properly, then the transactional + // timeout will be less than the session timeout. By forcing a + // heartbeat before the commit, if the heartbeat was successful, then + // we ensure that we will complete the transaction within the group + // session, meaning we will not commit after the group has rebalanced. + heartbeatForceCh chan func(error) + + // The following two are only updated in the manager / join&sync loop + // The nowAssigned map is read when commits fail: if the commit fails + // with ILLEGAL_GENERATION and it contains only partitions that are in + // nowAssigned, we re-issue. + lastAssigned map[string][]int32 + nowAssigned amtps + + // Fetching ensures we continue fetching offsets across cooperative + // rebalance if an offset fetch returns early due to an immediate + // rebalance. See the large comment on adjustCooperativeFetchOffsets + // for more details. + // + // This is modified only in that function, or in the manage loop on a + // hard error once the heartbeat/fetch has returned. + fetching map[string]map[int32]struct{} + + // onFetchedMu ensures we do not call onFetched nor adjustOffsets + // concurrent with onRevoked. + // + // The group session itself ensures that OnPartitions functions are + // serial, but offset fetching is concurrent with heartbeating and can + // finish before or after heartbeating has already detected a revoke. + // To make user lives easier, we guarantee that offset fetch callbacks + // cannot be concurrent with onRevoked with this mu. If fetch callbacks + // are present, we hook this mu into onRevoked, and we grab it in the + // locations fetch callbacks are called. We only have to worry about + // onRevoked because fetching offsets occurs after onAssigned, and + // onLost happens after fetching offsets is done. + onFetchedMu xsync.Mutex + + // leader is whether we are the leader right now. This is set to false + // + // - set to false at the beginning of a join group session + // - set to true if join group response indicates we are leader + // - read on metadata updates in findNewAssignments + leader atomic.Bool + + // Set to true when ending a transaction committing transaction + // offsets, and then set to false immediately after before calling + // EndTransaction. + offsetsAddedToTxn bool + + // If we are leader, then other members may express interest to consume + // topics that we are not interested in consuming. We track the entire + // group's topics in external, and our fetchMetadata loop uses this. + // We store this as a pointer for address comparisons. + // Not relevant if using KIP-848. + external atomic.Value // *groupExternal + + // See the big comment on `commit`. If we allow committing between + // join&sync, we occasionally see RebalanceInProgress or + // IllegalGeneration errors while cooperative consuming. + // Not relevant if using KIP-848. + noCommitDuringJoinAndSync xsync.RWMutex + + ////////////// + // mu block // + ////////////// + mu xsync.Mutex + + // using is updated when finding new assignments, we always add to this + // if we want to consume a topic (or see there are more potential + // partitions). The difference between 'using' and 'tps' is that + // 'using' is used FOR joining. We add topics to this when we learn + // about them and want to consume them, and the topics here + // are used in the JoinGroup metadata. There may be a small delta + // between 'tps' before topics are in 'using', and 'using' tracks + // the last known partition count for if we are leader. + // + // This is read when joining a group or leaving a group. + using map[string]int // topics *we* are currently using => # partitions known in that topic + + // uncommitted is read and updated all over: + // - updated before PollFetches returns + // - updated when directly setting offsets (to rewind, for transactions) + // - emptied when leaving a group + // - updated when revoking + // - updated after fetching offsets once we receive our group assignment + // - updated after we commit + // - read when getting uncommitted or committed + uncommitted uncommitted + + // memberID and generation are written to in the join and sync loop, + // and mostly read within that loop. This can be read during commits, + // which can happy any time. It is **recommended** to be done within + // the context of a group session, but (a) users may have some unique + // use cases, and (b) the onRevoke hook may take longer than a user + // expects, which would rotate a session. + memberGen groupMemberGen + + // commitDone is set under mu before firing off an async commit + // request. If another commit happens, it waits for the prior to be + // done, and then starts its own. + commitDone chan struct{} + + // blockAuto is set and cleared in CommitOffsets{,Sync} to block + // autocommitting if autocommitting is active. This ensures that an + // autocommit does not cancel the user's manual commit. + blockAuto bool + + // We set this once to manage the group lifecycle once. + // If we detect we should run in 848 mode, we set is848 true. + managing bool + is848 bool + g848 *g848 + + dying bool // set when closing, read in findNewAssignments + left chan struct{} + leaveErr error // set before left is closed +} + +type groupMemberGen struct { + v atomic.Value // *groupMemberGenT +} + +type groupMemberGenT struct { + memberID string + generation int32 +} + +func (g *groupMemberGen) memberID() string { + memberID, _ := g.load() + return memberID +} + +func (g *groupMemberGen) generation() int32 { + _, generation := g.load() + return generation +} + +func (g *groupMemberGen) load() (memberID string, generation int32) { + v := g.v.Load() + if v == nil { + return "", -1 + } + t := v.(*groupMemberGenT) + return t.memberID, t.generation +} + +func (g *groupMemberGen) store(memberID string, generation int32) { + g.v.Store(&groupMemberGenT{memberID, generation}) +} + +func (g *groupMemberGen) storeMember(memberID string) { + g.store(memberID, g.generation()) +} + +func (g *groupMemberGen) storeGeneration(generation int32) { + g.store(g.memberID(), generation) +} + +// LeaveGroup is equivalent to calling [Client.LeaveGroupContext] with +// the client's context; it discards any returned error. See +// LeaveGroupContext for the full behavior. +func (cl *Client) LeaveGroup() { + cl.LeaveGroupContext(cl.ctx) +} + +// LeaveGroupContext leaves a classic consumer group or a share group. +// Close automatically leaves the group, so this is only necessary to +// call if you plan to leave the group but continue to use the client. +// +// The context can be used to avoid waiting for the client to leave the +// group. Not waiting may result in your client being stuck in the group +// and the partitions this client was consuming being stuck until the +// session timeout. This function returns any leave-group error or +// context cancel error. If the context is nil, this immediately +// triggers the leave and does not wait, returning nil. In either the +// ctx-expired or nil-ctx cases, the shutdown work continues in the +// background to keep client state consistent. +// +// For classic consumer groups: if a rebalance is in progress, this +// function waits for the rebalance to complete before the group can +// be left. This is necessary to allow you to safely issue one final +// offset commit in OnPartitionsRevoked. If you have overridden the +// default revoke, you must manually commit offsets before leaving the +// group. If you have configured the group with an InstanceID, this +// does not leave the group. +// +// For share groups: this drains any pending acks, releases records that +// were acquired but never finalized, closes each per-broker share +// session, and sends the final ShareGroupHeartbeat with MemberEpoch=-1 +// to leave the group. LeaveGroupContext is safe to call concurrently +// with [Client.PollRecords], [Client.MarkAcks], and [Record.Ack]: +// after the leave begins, PollRecords will either return any records +// that were already buffered (the caller may still ack them) or +// return an ErrClientClosed fetch; concurrent MarkAcks and Record.Ack +// either land before the leave's release pass (and succeed) or land +// after (and are reported via the configured [ShareAckCallback] as +// failed with an internal "consumer left" error). +// +// LeaveGroupContext is a no-op for direct (non-group) consumers. +func (cl *Client) LeaveGroupContext(ctx context.Context) error { + c := &cl.consumer + if c.g == nil && c.s == nil { + return nil + } + var immediate bool + if ctx == nil { + var cancel func() + ctx, cancel = context.WithCancel(context.Background()) + cancel() + immediate = true + } + + if c.s != nil { + go c.s.leave(ctx) + select { + case <-ctx.Done(): + if immediate { + return nil + } + return ctx.Err() + case <-c.s.left: + return c.s.leaveErr + } + } + + go func() { + c.waitAndAddRebalanceSilent() + c.mu.Lock() // lock for assign + c.assignPartitions(nil, assignInvalidateAll, nil, "invalidating all assignments in LeaveGroup") + c.g.leave(ctx) + c.mu.Unlock() + c.unaddRebalance() + }() + + select { + case <-ctx.Done(): + if immediate { + return nil + } + return ctx.Err() + case <-c.g.left: + return c.g.leaveErr + } +} + +// GroupMetadata returns the current group member ID and generation, or an +// empty string and -1 if not in the group. +func (cl *Client) GroupMetadata() (string, int32) { + g := cl.consumer.g + if g == nil { + return "", -1 + } + return g.memberGen.load() +} + +func (c *consumer) initGroup() { + ctx, cancel := context.WithCancel(c.cl.ctx) + g := &groupConsumer{ + c: c, + cl: c.cl, + cfg: &c.cl.cfg, + + ctx: ctx, + cancel: cancel, + + reSeen: make(map[string]bool), + + manageDone: make(chan struct{}), + tps: newTopicsPartitions(), + rejoinCh: make(chan string, 1), + heartbeatForceCh: make(chan func(error)), + using: make(map[string]int), + + left: make(chan struct{}), + } + c.g = g + if g.cfg.commitCallback == nil { + g.cfg.commitCallback = g.defaultCommitCallback + } + + if g.cfg.txnID == nil { + // We only override revoked / lost if they were not explicitly + // set by options. + if g.cfg.onRevoked == nil { + g.cfg.onRevoked = g.defaultRevoke + } + // For onLost, we do not want to commit in onLost, so we + // explicitly set onLost to an empty function to avoid the + // fallback to onRevoked. + if g.cfg.onLost == nil { + g.cfg.onLost = func(context.Context, *Client, map[string][]int32) {} + } + } else { + g.cfg.autocommitDisable = true + } + + // Capture whether the user actually registered onAssigned before the + // wrapper below replaces it with a non-nil entry-logger. assign() uses + // this to decide whether the BlockRebalanceOnPoll gate applies. + g.cfg.userHasOnAssign = g.cfg.onAssigned != nil + + // INVARIANT: after this loop, on{Assigned,Revoked,Lost} are all non-nil. + // The wrapper unconditionally replaces each callback so we can log entry + // even when the user did not provide one. Downstream callers rely on this + // and skip nil-checks. + for _, logOn := range []struct { + name string + set *func(context.Context, *Client, map[string][]int32) + }{ + {"OnPartitionsAssigned", &g.cfg.onAssigned}, + {"OnPartitionsRevoked", &g.cfg.onRevoked}, + {"OnPartitionsLost", &g.cfg.onLost}, + } { + user := *logOn.set + name := logOn.name + *logOn.set = func(ctx context.Context, cl *Client, m map[string][]int32) { + var ctxExpired bool + select { + case <-ctx.Done(): + ctxExpired = true + default: + } + if ctxExpired { + cl.cfg.logger.Log(LogLevelDebug, "entering "+name, "with", m, "context_expired", ctxExpired) + } else { + cl.cfg.logger.Log(LogLevelDebug, "entering "+name, "with", m) + } + if user != nil { + dup := make(map[string][]int32) + for k, vs := range m { + dup[k] = slices.Clone(vs) + } + user(ctx, cl, dup) + } + } + } + + if g.cfg.onFetched != nil || g.cfg.adjustOffsetsBeforeAssign != nil { + revoked := g.cfg.onRevoked + g.cfg.onRevoked = func(ctx context.Context, cl *Client, m map[string][]int32) { + g.onFetchedMu.Lock() + defer g.onFetchedMu.Unlock() + revoked(ctx, cl, m) + } + } + + // For non-regex topics, we explicitly ensure they exist for loading + // metadata. This is of no impact if we are *also* consuming via regex, + // but that is no problem. + if len(g.cfg.topics) > 0 && !g.cfg.regex { + topics := make([]string, 0, len(g.cfg.topics)) + for topic := range g.cfg.topics { + topics = append(topics, topic) + } + g.tps.storeTopics(topics) + } +} + +func (g *groupConsumer) manageFailWait(consecutiveErrors int, err error) (ctxCanceled bool) { + // If the user has BlockPollOnRebalance enabled, we have to + // block around the onLost and assigning. + g.c.waitAndAddRebalance() + + if errors.Is(err, context.Canceled) { + // The cooperative consumer does not revoke everything + // while rebalancing, meaning if our context is + // canceled, we may have uncommitted data. Rather than + // diving into onLost, we should go into onRevoked, + // because for the most part, a context cancelation + // means we are leaving the group. Going into onRevoked + // gives us an opportunity to commit outstanding + // offsets. For the eager consumer, since we always + // revoke before exiting the heartbeat loop, we do not + // really care so much about *needing* to call + // onRevoked, but since we are handling this case for + // the cooperative consumer we may as well just also + // include the eager consumer. + g.cfg.onRevoked(g.cl.ctx, g.cl, g.nowAssigned.read()) + } else { + // Any other error is perceived as a fatal error, + // and we go into onLost as appropriate. + g.cfg.onLost(g.cl.ctx, g.cl, g.nowAssigned.read()) + g.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookGroupManageError); ok { + h.OnGroupManageError(err) + } + }) + g.c.addFakeReadyForDraining("", 0, &ErrGroupSession{err}, "notification of group management loop error") + } + + // If we are eager, we should have invalidated everything + // before getting here, but we do so doubly just in case. + // + // If we are cooperative, the join and sync could have failed + // during the cooperative rebalance where we were still + // consuming. We need to invalidate everything. Waiting to + // resume from poll is necessary, but the user will likely be + // unable to commit. + { + g.c.mu.Lock() + g.c.assignPartitions(nil, assignInvalidateAll, nil, "clearing assignment at end of group management session") + g.mu.Lock() // before allowing poll to touch uncommitted, lock the group + g.c.mu.Unlock() // now part of poll can continue + g.uncommitted = nil + g.mu.Unlock() + + g.nowAssigned.store(nil) + g.lastAssigned = nil + g.fetching = nil + + g.leader.Store(false) + g.resetExternal() + } + + // Unblock bolling now that we have called onLost and + // re-assigned. + g.c.unaddRebalance() + + if errors.Is(err, context.Canceled) { // context was canceled, quit now + return true + } + + // Waiting for the backoff is a good time to update our + // metadata; maybe the error is from stale metadata. + backoff := g.cfg.retryBackoff(consecutiveErrors) + g.cfg.logger.Log(LogLevelError, "group manage loop errored", + "group", g.cfg.group, + "err", err, + "consecutive_errors", consecutiveErrors, + "backoff", backoff, + ) + deadline := time.Now().Add(backoff) + g.cl.waitmeta(g.ctx, backoff, "waitmeta during group manage backoff") + after := time.NewTimer(time.Until(deadline)) + select { + case <-g.ctx.Done(): + after.Stop() + return true + case <-after.C: + } + return false +} + +// abandonAssignment fires onLost, invalidates cursors, and clears local +// assignment state in preparation for re-joining. Used by the 848 manage +// loop when the server-side member state was lost (Fenced/UnknownMember/ +// StaleMember/GroupMaxSize/UnsupportedAssignor): the partitions are no +// longer ours, so cursors must stop fetching before the next initialJoin +// to prevent dual-processing records that have been reassigned to another +// member. Mirrors manageFailWait's cleanup without the consecutive-error +// backoff. Must be called with memberGen still holding the OLD member id +// so onLost callbacks see the membership context that was active when the +// loss occurred. +func (g *groupConsumer) abandonAssignment(why string) { + g.c.waitAndAddRebalance() + + g.cfg.onLost(g.cl.ctx, g.cl, g.nowAssigned.read()) + + g.c.mu.Lock() + g.c.assignPartitions(nil, assignInvalidateAll, nil, why) + g.mu.Lock() // before allowing poll to touch uncommitted, lock the group + g.c.mu.Unlock() // now part of poll can continue + g.uncommitted = nil + g.mu.Unlock() + + g.nowAssigned.store(nil) + g.lastAssigned = nil + g.fetching = nil + + g.c.unaddRebalance() +} + +// Manages the group consumer's join / sync / heartbeat / fetch offset flow. +// +// Once a group is assigned, we fire a metadata request for all topics the +// assignment specified interest in. Only after we finally have some topic +// metadata do we join the group, and once joined, this management runs in a +// dedicated goroutine until the group is left. +func (g *groupConsumer) manage() { + defer close(g.manageDone) + g.cfg.logger.Log(LogLevelInfo, "beginning to manage the group lifecycle", "group", g.cfg.group) + if !g.cfg.autocommitDisable && g.cfg.autocommitInterval > 0 { + g.cfg.logger.Log(LogLevelInfo, "beginning autocommit loop", "group", g.cfg.group) + go g.loopCommit() + } + + var consecutiveErrors int + joinWhy := "beginning to manage the group lifecycle" + for { + if joinWhy == "" { + joinWhy = "rejoining from normal rebalance" + } + err := g.joinAndSync(joinWhy) + if err == nil { + if joinWhy, err = g.setupAssignedAndHeartbeat(g.cfg.heartbeatInterval, g.heartbeatFn()); err != nil { + if errors.Is(err, kerr.RebalanceInProgress) { + err = nil + } + } + } + if err == nil { + consecutiveErrors = 0 + continue + } + joinWhy = "rejoining after we previously errored and backed off" + + consecutiveErrors++ + ctxCanceled := g.manageFailWait(consecutiveErrors, err) + if ctxCanceled { + return + } + } +} + +func (g *groupConsumer) leave(ctx context.Context) { + // If g.using is nonzero before this check, then a manage goroutine has + // started. If not, it will never start because we set dying. + g.mu.Lock() + wasDead := g.dying + g.dying = true + wasManaging := g.managing + is848 := g.is848 + g.cancel() + g.mu.Unlock() + + go func() { + if wasManaging { + // We want to wait for the manage goroutine to be done + // so that we call the user's on{Assign,RevokeLost}. + <-g.manageDone + } + if wasDead { + // If we already called leave(), then we just wait for + // the prior leave to finish and we avoid re-issuing a + // LeaveGroup request. + return + } + + defer close(g.left) + + // If we JUST started a group but do not yet have a + // member ID, there's nothing we can do. + memberID := g.memberGen.memberID() + if memberID == "" { + g.cfg.logger.Log(LogLevelInfo, "tried to leave group but we have no member ID yet, returning early", "group", g.cfg.group) + return + } + + if is848 { + g.leave848(ctx) + return + } + + if g.cfg.instanceID != nil { + return + } + + g.cfg.logger.Log(LogLevelInfo, "leaving group", + "group", g.cfg.group, + "member_id", memberID, + ) + // If we error when leaving, there is not much + // we can do. We may as well just return. + req := kmsg.NewPtrLeaveGroupRequest() + req.Group = g.cfg.group + req.MemberID = memberID + member := kmsg.NewLeaveGroupRequestMember() + member.MemberID = memberID + member.Reason = kmsg.StringPtr("client leaving group per normal operation") + req.Members = append(req.Members, member) + + resp, err := req.RequestWith(ctx, g.cl) + if err != nil { + g.leaveErr = err + return + } + g.leaveErr = kerr.ErrorForCode(resp.ErrorCode) + }() +} + +// returns the difference of g.nowAssigned and g.lastAssigned. +func (g *groupConsumer) diffAssigned() (added, lost map[string][]int32) { + nowAssigned := g.nowAssigned.clone() + if !g.cooperative.Load() { + return nowAssigned, nil + } + + added = make(map[string][]int32, len(nowAssigned)) + lost = make(map[string][]int32, len(nowAssigned)) + + // First, we diff lasts: any topic in last but not now is lost, + // otherwise, (1) new partitions are added, (2) common partitions are + // ignored, and (3) partitions no longer in now are lost. + lasts := make(map[int32]struct{}, 100) + for topic, lastPartitions := range g.lastAssigned { + nowPartitions, exists := nowAssigned[topic] + if !exists { + lost[topic] = lastPartitions + continue + } + + for _, lastPartition := range lastPartitions { + lasts[lastPartition] = struct{}{} + } + + // Anything now that does not exist in last is new, + // otherwise it is in common and we ignore it. + for _, nowPartition := range nowPartitions { + if _, exists := lasts[nowPartition]; !exists { + added[topic] = append(added[topic], nowPartition) + } else { + delete(lasts, nowPartition) + } + } + + // Anything remanining in last does not exist now + // and is thus lost. + for last := range lasts { + lost[topic] = append(lost[topic], last) + delete(lasts, last) // reuse lasts + } + } + + // Finally, any new topics in now assigned are strictly added. + for topic, nowPartitions := range nowAssigned { + if _, exists := g.lastAssigned[topic]; !exists { + added[topic] = nowPartitions + } + } + + return added, lost +} + +type revokeStage int8 + +const ( + revokeLastSession = iota + revokeThisSession +) + +// revoke calls onRevoked for partitions that this group member is losing and +// updates the uncommitted map after the revoke. +// +// For eager consumers, this simply revokes g.assigned. This will only be +// called at the end of a group session. +// +// For cooperative consumers, this either +// +// (1) if revoking lost partitions from a prior session (i.e., after sync), +// this revokes the passed in lost +// (2) if revoking at the end of a session, this revokes topics that the +// consumer is no longer interested in consuming +// +// Lastly, for cooperative consumers, this must selectively delete what was +// lost from the uncommitted map. +func (g *groupConsumer) revoke(stage revokeStage, lost map[string][]int32, leaving bool) { + g.c.waitAndAddRebalance() + defer g.c.unaddRebalance() + + if !g.cooperative.Load() || leaving { // stage == revokeThisSession if not cooperative + // If we are an eager consumer, we stop fetching all of our + // current partitions as we will be revoking them. + g.c.mu.Lock() + if leaving { + g.c.assignPartitions(nil, assignInvalidateAll, nil, "revoking all assignments because we are leaving the group") + } else { + g.c.assignPartitions(nil, assignInvalidateAll, nil, "revoking all assignments because we are not cooperative") + } + g.c.mu.Unlock() + + if !g.cooperative.Load() { + g.cfg.logger.Log(LogLevelInfo, "eager consumer revoking prior assigned partitions", "group", g.cfg.group, "revoking", g.nowAssigned.read()) + } else { + g.cfg.logger.Log(LogLevelInfo, "cooperative consumer revoking prior assigned partitions because leaving group", "group", g.cfg.group, "revoking", g.nowAssigned.read()) + } + g.cfg.onRevoked(g.cl.ctx, g.cl, g.nowAssigned.read()) + g.nowAssigned.store(nil) + g.lastAssigned = nil + + // After nilling uncommitted here, nothing should recreate + // uncommitted until a future fetch after the group is + // rejoined. This _can_ be broken with a manual SetOffsets or + // with CommitOffsets{,Sync} but we explicitly document not + // to do that outside the context of a live group session. + g.mu.Lock() + g.uncommitted = nil + g.mu.Unlock() + return + } + + switch stage { + case revokeLastSession: + // we use lost in this case; this is the case where we are + // rejoining after losing some partitions (cooperative or KIP-848) + + case revokeThisSession: + // lost is nil for cooperative assigning. Instead, we determine + // lost by finding subscriptions we are no longer interested + // in. This would be from a user's PurgeConsumeTopics call. + // + // We just paused metadata, but purging triggers a rebalance + // which causes a new metadata request -- in short, this could + // be concurrent with a metadata findNewAssignments, so we + // lock. + g.nowAssigned.write(func(nowAssigned map[string][]int32) { + g.mu.Lock() + for topic, partitions := range nowAssigned { + if _, exists := g.using[topic]; !exists { + if lost == nil { + lost = make(map[string][]int32) + } + lost[topic] = partitions + delete(nowAssigned, topic) + } + } + g.mu.Unlock() + }) + } + + if len(lost) > 0 { + // We must now stop fetching anything we lost and invalidate + // any buffered fetches before falling into onRevoked. + // + // We want to invalidate buffered fetches since they may + // contain partitions that we lost, and we do not want a future + // poll to return those fetches. + lostOffsets := make(map[string]map[int32]Offset, len(lost)) + + for lostTopic, lostPartitions := range lost { + lostPartitionOffsets := make(map[int32]Offset, len(lostPartitions)) + for _, lostPartition := range lostPartitions { + lostPartitionOffsets[lostPartition] = Offset{} + } + lostOffsets[lostTopic] = lostPartitionOffsets + } + + // We must invalidate before revoking and before updating + // uncommitted, because we want any commits in onRevoke to be + // for the final polled offsets. We do not want to allow the + // logical race of allowing fetches for revoked partitions + // after a revoke but before an invalidation. + g.c.mu.Lock() + g.c.assignPartitions(lostOffsets, assignInvalidateMatching, g.tps, "revoking assignments from cooperative consuming") + g.c.mu.Unlock() + } + + if len(lost) > 0 || stage == revokeThisSession { + if len(lost) == 0 { + g.cfg.logger.Log(LogLevelInfo, "consumer calling onRevoke at the end of a session; consumer did not change any client-side subscription", "group", g.cfg.group) + } else { + g.cfg.logger.Log(LogLevelInfo, "calling onRevoke at the end of a session", "group", g.cfg.group, "lost", lost, "stage", stage) + } + g.cfg.onRevoked(g.cl.ctx, g.cl, lost) + } + + if len(lost) == 0 { // if we lost nothing, do nothing + return + } + + if stage != revokeThisSession { // cooperative consumers rejoin after revoking what they lost + defer g.rejoin("after revoking what we lost from a rebalance") + } + + // The block below deletes everything lost from our uncommitted map. + // All commits should be **completed** by the time this runs. An async + // commit can undo what we do below. The default revoke runs a sync + // commit. + g.mu.Lock() + defer g.mu.Unlock() + if g.uncommitted == nil { + return + } + for lostTopic, lostPartitions := range lost { + uncommittedPartitions := g.uncommitted[lostTopic] + if uncommittedPartitions == nil { + continue + } + for _, lostPartition := range lostPartitions { + delete(uncommittedPartitions, lostPartition) + } + if len(uncommittedPartitions) == 0 { + delete(g.uncommitted, lostTopic) + } + } + if len(g.uncommitted) == 0 { + g.uncommitted = nil + } +} + +// assignRevokeSession aids in sequencing prerevoke/assign/revoke. +type assignRevokeSession struct { + prerevokeDone chan struct{} + assignDone chan struct{} + revokeDone chan struct{} +} + +func newAssignRevokeSession() *assignRevokeSession { + return &assignRevokeSession{ + prerevokeDone: make(chan struct{}), + assignDone: make(chan struct{}), + revokeDone: make(chan struct{}), + } +} + +// For cooperative consumers, the first thing a cooperative consumer does is to +// diff its last assignment and its new assignment and revoke anything lost. +// We call this a "prerevoke". +func (s *assignRevokeSession) prerevoke(g *groupConsumer, lost map[string][]int32) <-chan struct{} { + // For 848, set prerevoking before the goroutine starts so the + // very first concurrent heartbeat sends keepalive. + g.mu.Lock() + g848 := g.g848 + g.mu.Unlock() + if g848 != nil { + g848.prerevoking.Store(true) + } + go func() { + defer close(s.prerevokeDone) + if g.cooperative.Load() && len(lost) > 0 { + g.revoke(revokeLastSession, lost, false) + } + // Now that prerevoke is complete, clear prerevoking so + // subsequent heartbeats resume sending full requests. + if g848 != nil { + g848.prerevoking.Store(false) + } + }() + return s.prerevokeDone +} + +func (s *assignRevokeSession) assign(g *groupConsumer, newAssigned map[string][]int32) <-chan struct{} { + go func() { + defer close(s.assignDone) + <-s.prerevokeDone + // We always call onAssigned, even if nothing new is assigned. + // This allows consumers to know that assignment is done and do + // setup logic. + // + // The BlockRebalanceOnPoll gate exists to prevent a poll loop + // from committing offsets across a rebalance for partitions it + // no longer owns. Assign only adds partitions, so the user's + // in-flight commit cannot reference anything they don't still + // own. We therefore gate only when the user registered an + // OnPartitionsAssigned callback and needs it serialized with + // poll; the entry-log wrapper alone is fine to run concurrent + // with poll. If you ever add internal work here that races + // with poll (e.g., touching nowAssigned or commit state), + // remove this conditional. + if g.cfg.userHasOnAssign { + g.c.waitAndAddRebalance() + defer g.c.unaddRebalance() + } + g.cfg.onAssigned(g.cl.ctx, g.cl, newAssigned) + }() + return s.assignDone +} + +// At the end of a group session, before we leave the heartbeat loop, we call +// revoke. For non-cooperative consumers, this revokes everything in the +// current session, and before revoking, we invalidate all partitions. For the +// cooperative consumer, this does nothing but does notify the client that a +// revoke has begun / the group session is ending. +// +// This may not run before returning from the heartbeat loop: if we encounter a +// fatal error, we return before revoking so that we can instead call onLost in +// the manage loop. +func (s *assignRevokeSession) revoke(g *groupConsumer, leaving bool) <-chan struct{} { + go func() { + defer close(s.revokeDone) + <-s.assignDone + g.revoke(revokeThisSession, nil, leaving) + }() + return s.revokeDone +} + +// This chunk of code "pre" revokes lost partitions for the cooperative +// consumer and then begins heartbeating while fetching offsets. This returns +// when heartbeating errors (or if fetch offsets errors). +// +// Before returning, this function ensures that +// - onAssigned is complete +// - which ensures that pre revoking is complete +// - fetching is complete +// - heartbeating is complete +func (g *groupConsumer) setupAssignedAndHeartbeat(initialHb time.Duration, hbfn func() (time.Duration, error)) (string, error) { + type hbquit struct { + rejoinWhy string + err error + } + hbErrCh := make(chan hbquit, 1) + fetchErrCh := make(chan error, 1) + + s := newAssignRevokeSession() + added, lost := g.diffAssigned() + g.lastAssigned = g.nowAssigned.clone() // now that we are done with our last assignment, update it per the new assignment + + g.cfg.logger.Log(LogLevelInfo, "new group session begun", "group", g.cfg.group, "added", mtps(added), "lost", mtps(lost)) + s.prerevoke(g, lost) // for cooperative consumers + + // Since we have joined the group, we immediately begin heartbeating. + // This will continue until the heartbeat errors, the group is killed, + // or the fetch offsets below errors. + ctx, cancel := context.WithCancel(g.ctx) + go func() { + defer cancel() // potentially kill offset fetching + g.cfg.logger.Log(LogLevelInfo, "beginning heartbeat loop", "group", g.cfg.group) + rejoinWhy, err := g.heartbeat(initialHb, fetchErrCh, s, hbfn) + hbErrCh <- hbquit{rejoinWhy, err} + }() + + // We immediately begin fetching offsets. We want to wait until the + // fetch function returns, since it assumes within it that another + // assign cannot happen (it assigns partitions itself). Returning + // before the fetch completes would be not good. + // + // The difference between fetchDone and fetchErrCh is that fetchErrCh + // can kill heartbeating, or signal it to continue, while fetchDone + // is specifically used for this function's return. + fetchDone := make(chan struct{}) + defer func() { <-fetchDone }() + + // Before we fetch offsets, we wait for the user's onAssign callback to + // be done. This ensures a few things: + // + // * that we wait for prerevoking to be done, which updates the + // uncommitted field. Waiting for that ensures that a rejoin and poll + // does not have weird concurrent interaction. + // + // * that our onLost will not be concurrent with onAssign + // + // * that the user can start up any per-partition processors necessary + // before we begin consuming that partition. + // + // We especially need to wait here because heartbeating may not + // necessarily run onRevoke before returning (because of a fatal + // error). + s.assign(g, added) + + // If cooperative consuming, we may have to resume fetches. See the + // comment on adjustCooperativeFetchOffsets. + // + // We do this AFTER the user's callback. If we add more partitions + // to `added` that are from a previously canceled fetch, we do NOT + // want to pass those fetch-resumed partitions to the user callback + // again. See #705. + if g.cooperative.Load() { + added = g.adjustCooperativeFetchOffsets(added, lost) + } + + <-s.assignDone + + if len(added) > 0 { + go func() { + defer close(fetchDone) + defer close(fetchErrCh) + fetchErrCh <- g.fetchOffsets(ctx, added) + }() + } else { + close(fetchDone) + close(fetchErrCh) + } + + // Finally, we simply return whatever the heartbeat error is. This will + // be the fetch offset error if that function is what killed this. + + done := <-hbErrCh + return done.rejoinWhy, done.err +} + +func (g *groupConsumer) heartbeatFn() func() (time.Duration, error) { + return func() (time.Duration, error) { + req := kmsg.NewPtrHeartbeatRequest() + req.Group = g.cfg.group + memberID, generation := g.memberGen.load() + req.Generation = generation + req.MemberID = memberID + req.InstanceID = g.cfg.instanceID + var resp *kmsg.HeartbeatResponse + resp, err := req.RequestWith(g.ctx, g.cl) + if err == nil { + err = kerr.ErrorForCode(resp.ErrorCode) + } + return g.cfg.heartbeatInterval, err + } +} + +// heartbeat issues heartbeat requests to Kafka for the duration of a group +// session. +// +// This function begins before fetching offsets to allow the consumer's +// onAssigned to be called before fetching. If the eventual offset fetch +// errors, we continue heartbeating until onRevoked finishes. +// If the error is not RebalanceInProgress, we return immediately. +// +// If the offset fetch is successful, then we basically sit in this function +// until a heartbeat errors or we, being the leader, decide to re-join. +func (g *groupConsumer) heartbeat(initialHb time.Duration, fetchErrCh <-chan error, s *assignRevokeSession, hbfn func() (time.Duration, error)) (string, error) { + g.mu.Lock() + is848 := g.is848 + g.mu.Unlock() + + timer := time.NewTimer(initialHb) + defer timer.Stop() + + // We issue one heartbeat quickly if we are cooperative because + // cooperative consumers rejoin the group immediately, and we want to + // detect that in 500ms rather than 3s. We only want this is non-848 + // mode. + var cooperativeFastCheck <-chan time.Time + if g.cooperative.Load() && !is848 { + cooperativeFastCheck = time.After(500 * time.Millisecond) + } + + var revoked <-chan struct{} + var heartbeat, didRevoke, stopHeartbeating bool + var rejoinWhy string + var lastErr error + var hbBrokerRetries int + heartbeatForceCh := g.heartbeatForceCh + + ctxCh := g.ctx.Done() + + for { + var err error + var force func(error) + heartbeat = false + select { + case <-cooperativeFastCheck: + heartbeat = true + case <-timer.C: + heartbeat = true + case force = <-heartbeatForceCh: + heartbeat = true + case rejoinWhy = <-g.rejoinCh: + // If a metadata update changes our subscription, + // we just pretend we are rebalancing. + g.cfg.logger.Log(LogLevelInfo, "forced rejoin quitting heartbeat loop", "why", rejoinWhy, "is848", is848) + err = kerr.RebalanceInProgress + case err = <-fetchErrCh: + fetchErrCh = nil + case <-revoked: + revoked = nil + didRevoke = true + case <-ctxCh: + // Even if the group is left, we need to wait for our + // revoke to finish before returning, otherwise the + // manage goroutine will race with us setting + // nowAssigned. + ctxCh = nil + err = context.Canceled + } + + if heartbeat && !stopHeartbeating { + g.cfg.logger.Log(LogLevelDebug, "heartbeating", "group", g.cfg.group) + var reset time.Duration + reset, err = hbfn() + timer.Reset(reset) + g.cfg.logger.Log(LogLevelDebug, "heartbeat complete", "group", g.cfg.group, "err", err) + if force != nil { + force(err) + } + } + + // The first error either triggers a clean revoke or it returns + // immediately. If we triggered the revoke, we wait for it to + // complete regardless of any future error. + if didRevoke { + return rejoinWhy, lastErr + } + + if err == nil { + hbBrokerRetries = 0 + continue + } + + // For KIP-848, retryable broker errors (connection closed, + // EOF) and coordinator errors (NOT_COORDINATOR, etc.) are + // transient and do not invalidate the member's state on + // the broker. The classic protocol retries these + // transparently via the client's retryable request + // wrapper, but 848 heartbeats bypass that and manage + // retries here. We use exponential backoff matching the + // retryable wrapper and retry in-place, avoiding the + // session teardown/rebuild that would occur if the error + // propagated to the manage848 loop. + // + // We reset the counter on each success so that intermittent + // failures do not accumulate across the session. Without + // resetting, a few scattered failures per heartbeat cycle + // compound until the counter hits the cap, triggering an + // unnecessary session restart even though most heartbeats + // succeed and the broker-side session is healthy. + // + // If cfg.retries consecutive failures occur without any + // success, the error propagates to manage848 which + // rebuilds the session. + if is848 && (isRetryableBrokerErr(err) || isAnyDialErr(err) || g.cl.maybeDeleteStaleCoordinator(g.cfg.group, coordinatorTypeGroup, err)) { + if int64(hbBrokerRetries) < g.cfg.retries { + hbBrokerRetries++ + backoff := g.cfg.retryBackoff(hbBrokerRetries) + g.cfg.logger.Log(LogLevelInfo, "heartbeat hit retryable error, retrying", + "group", g.cfg.group, + "err", err, + "backoff", backoff, + "retries", hbBrokerRetries, + ) + timer.Reset(backoff) + continue + } + g.cfg.logger.Log(LogLevelInfo, "heartbeat hit retryable error, max retries reached", + "group", g.cfg.group, + "err", err, + "retries", hbBrokerRetries, + ) + } + + // When the 848 closure detects an assignment change, it + // returns errReassigned848. We suppress further heartbeats + // so we cannot see stale state and miss a server-side + // revocation, but we still run the revoke path below. + isReassign := errors.Is(err, errReassigned848) + if isReassign { + stopHeartbeating = true + heartbeatForceCh = nil + err = kerr.RebalanceInProgress + } + + if lastErr == nil { + if is848 && errors.Is(err, kerr.RebalanceInProgress) { + g.cfg.logger.Log(LogLevelInfo, "heartbeat saw a change in group status; partitions were added or lost", "group", g.cfg.group) + } else { + g.cfg.logger.Log(LogLevelInfo, "heartbeat errored", "group", g.cfg.group, "err", err) + } + } else { + g.cfg.logger.Log(LogLevelInfo, "heartbeat errored again while waiting for user revoke to finish", "group", g.cfg.group, "err", err) + } + + // Since we errored, we must revoke. + if !didRevoke && revoked == nil { + // If our error is not from rebalancing, then we + // encountered IllegalGeneration or UnknownMemberID or + // our context closed all of which are unexpected and + // unrecoverable. + // + // We return early rather than revoking and updating + // metadata; the groupConsumer's manage function will + // call onLost with all partitions. + // + // setupAssignedAndHeartbeat still waits for onAssigned + // to be done so that we avoid calling onLost + // concurrently. + if !errors.Is(err, kerr.RebalanceInProgress) && revoked == nil { + return "", err + } + + // Now we call the user provided revoke callback, even + // if cooperative: if cooperative, this only revokes + // partitions we no longer want to consume. + // + // If the err is context.Canceled, the group is being + // left and we revoke everything. + revoked = s.revoke(g, errors.Is(err, context.Canceled)) + } + + // We always save the latest error; generally this should be + // REBALANCE_IN_PROGRESS, but if the revoke takes too long, + // Kafka may boot us and we will get a different error. + lastErr = err + } +} + +// ForceRebalance quits a group member's heartbeat loop so that the member +// rejoins with a JoinGroupRequest. +// +// This function is only useful if you either (a) know that the group member is +// a leader, and want to force a rebalance for any particular reason, or (b) +// are using a custom group balancer, and have changed the metadata that will +// be returned from its JoinGroupMetadata method. This function has no other +// use; see KIP-568 for more details around this function's motivation. +// +// If neither of the cases above are true (this member is not a leader, and the +// join group metadata has not changed), then Kafka will not actually trigger a +// rebalance and will instead reply to the member with its current assignment. +func (cl *Client) ForceRebalance() { + if g := cl.consumer.g; g != nil { + g.rejoin("from ForceRebalance") + } +} + +// rejoin is called after a cooperative member revokes what it lost at the +// beginning of a session, or if we are leader and detect new partitions to +// consume. +func (g *groupConsumer) rejoin(why string) { + select { + case g.rejoinCh <- why: + default: + } +} + +// Joins and then syncs, issuing the two slow requests in goroutines to allow +// for group cancelation to return early. +func (g *groupConsumer) joinAndSync(joinWhy string) error { + g.noCommitDuringJoinAndSync.Lock() + g.cfg.logger.Log(LogLevelDebug, "blocking commits from join&sync") + defer g.noCommitDuringJoinAndSync.Unlock() + defer g.cfg.logger.Log(LogLevelDebug, "unblocking commits from join&sync") + + g.cfg.logger.Log(LogLevelInfo, "joining group", "group", g.cfg.group) + g.leader.Store(false) + g.getAndResetExternalRejoin() + defer func() { + // If we are not leader, we clear any tracking of external + // topics from when we were previously leader, since tracking + // these is just a waste. + if !g.leader.Load() { + g.resetExternal() + } + }() + +start: + select { + case <-g.rejoinCh: // drain to avoid unnecessary rejoins + default: + } + + joinReq := kmsg.NewPtrJoinGroupRequest() + joinReq.Group = g.cfg.group + joinReq.SessionTimeoutMillis = int32(g.cfg.sessionTimeout.Milliseconds()) + joinReq.RebalanceTimeoutMillis = int32(g.cfg.rebalanceTimeout.Milliseconds()) + joinReq.ProtocolType = g.cfg.protocol + joinReq.MemberID = g.memberGen.memberID() + joinReq.InstanceID = g.cfg.instanceID + joinReq.Protocols = g.joinGroupProtocols() + if joinWhy != "" { + joinReq.Reason = kmsg.StringPtr(joinWhy) + } + var ( + joinResp *kmsg.JoinGroupResponse + err error + joined = make(chan struct{}) + ) + + // NOTE: For this function, we have to use the client context, not the + // group context. We want to allow people to issue one final commit in + // OnPartitionsRevoked before leaving a group, so we need to block + // commits during join&sync. If we used the group context, we would be + // cancled immediately when leaving while a join or sync is inflight, + // and then our final commit will receive either REBALANCE_IN_PROGRESS + // or ILLEGAL_GENERATION. + + go func() { + defer close(joined) + joinResp, err = joinReq.RequestWith(g.cl.ctx, g.cl) + }() + + select { + case <-joined: + case <-g.cl.ctx.Done(): + return g.cl.ctx.Err() // client closed + } + if err != nil { + return err + } + + restart, protocol, plan, err := g.handleJoinResp(joinResp) + if restart { + goto start + } + if err != nil { + g.cfg.logger.Log(LogLevelWarn, "join group failed", "group", g.cfg.group, "err", err) + return err + } + + syncReq := kmsg.NewPtrSyncGroupRequest() + syncReq.Group = g.cfg.group + memberID, generation := g.memberGen.load() + syncReq.Generation = generation + syncReq.MemberID = memberID + syncReq.InstanceID = g.cfg.instanceID + syncReq.ProtocolType = &g.cfg.protocol + syncReq.Protocol = &protocol + if !joinResp.SkipAssignment { + syncReq.GroupAssignment = plan // nil unless we are the leader + } + var ( + syncResp *kmsg.SyncGroupResponse + synced = make(chan struct{}) + ) + + g.cfg.logger.Log(LogLevelInfo, "syncing", "group", g.cfg.group, "protocol_type", g.cfg.protocol, "protocol", protocol) + go func() { + defer close(synced) + syncResp, err = syncReq.RequestWith(g.cl.ctx, g.cl) + }() + + select { + case <-synced: + case <-g.cl.ctx.Done(): + return g.cl.ctx.Err() + } + if err != nil { + return err + } + + if err = g.handleSyncResp(protocol, syncResp); err != nil { + if errors.Is(err, kerr.RebalanceInProgress) { + g.cfg.logger.Log(LogLevelInfo, "sync failed with RebalanceInProgress, rejoining", "group", g.cfg.group) + goto start + } + g.cfg.logger.Log(LogLevelWarn, "sync group failed", "group", g.cfg.group, "err", err) + return err + } + + // KIP-814 fixes one limitation with KIP-345, but has another + // fundamental limitation. When an instance ID leader restarts, its + // first join always gets its old assignment *even if* the member's + // topic interests have changed. The broker tells us to skip doing + // assignment ourselves, but we ignore that for our well known + // balancers. Instead, we balance (but avoid sending it while syncing, + // as we are supposed to), and if our sync assignment differs from our + // own calculated assignment, We know we have a stale broker assignment + // and must trigger a rebalance. + if plan != nil && joinResp.SkipAssignment { + for _, assign := range plan { + if assign.MemberID == memberID { + if !bytes.Equal(assign.MemberAssignment, syncResp.MemberAssignment) { + g.rejoin("instance group leader restarted and was reassigned old plan, our topic interests changed and we must rejoin to force a rebalance") + } + break + } + } + } + + return nil +} + +func (g *groupConsumer) handleJoinResp(resp *kmsg.JoinGroupResponse) (restart bool, protocol string, plan []kmsg.SyncGroupRequestGroupAssignment, err error) { + if err = kerr.ErrorForCode(resp.ErrorCode); err != nil { + switch err { + case kerr.MemberIDRequired: + g.memberGen.storeMember(resp.MemberID) // KIP-394 + g.cfg.logger.Log(LogLevelInfo, "join returned MemberIDRequired, rejoining with response's MemberID", "group", g.cfg.group, "member_id", resp.MemberID) + return true, "", nil, nil + case kerr.UnknownMemberID: + g.memberGen.storeMember("") + g.cfg.logger.Log(LogLevelInfo, "join returned UnknownMemberID, rejoining without a member id", "group", g.cfg.group) + return true, "", nil, nil + } + return restart, protocol, plan, err // Request retries as necessary, so this must be a failure + } + g.memberGen.store(resp.MemberID, resp.Generation) + + if resp.Protocol != nil { + protocol = *resp.Protocol + } + + for _, balancer := range g.cfg.balancers { + if protocol == balancer.ProtocolName() { + cooperative := balancer.IsCooperative() + if !cooperative && g.cooperative.Load() { + g.cfg.logger.Log(LogLevelWarn, "downgrading from cooperative group to eager group, this is not supported per KIP-429!") + } + g.cooperative.Store(cooperative) + break + } + } + + // KIP-345 has a fundamental limitation that KIP-814 also does not + // solve. + // + // When using instance IDs, if a leader restarts, its first join + // receives its old assignment no matter what. KIP-345 resulted in + // leaderless consumer groups, KIP-814 fixes this by notifying the + // restarted leader that it is still leader but that it should not + // balance. + // + // If the join response is <= v8, we hackily work around the leaderless + // situation by checking if the LeaderID is prefixed with our + // InstanceID. This is how Kafka and Redpanda are both implemented. At + // worst, if we mis-predict the leader, then we may accidentally try to + // cause a rebalance later and it will do nothing. That's fine. At + // least we can cause rebalances now, rather than having a leaderless, + // not-ever-rebalancing client. + // + // KIP-814 does not solve our problem fully: if we restart and rejoin, + // we always get our old assignment even if we changed what topics we + // were interested in. Because we have our old assignment, we think + // that the plan is fine *even with* our new interests, and we wait for + // some external rebalance trigger. We work around this limitation + // above (see "KIP-814") only for well known balancers; we cannot work + // around this limitation for not well known balancers because they may + // do so weird things we cannot control nor reason about. + leader := resp.LeaderID == resp.MemberID + leaderNoPlan := !leader && resp.Version <= 8 && g.cfg.instanceID != nil && strings.HasPrefix(resp.LeaderID, *g.cfg.instanceID+"-") + if leader { + g.leader.Store(true) + g.cfg.logger.Log(LogLevelInfo, "joined, balancing group", + "group", g.cfg.group, + "member_id", resp.MemberID, + "instance_id", strptr{g.cfg.instanceID}, + "generation", resp.Generation, + "balance_protocol", protocol, + "leader", true, + ) + plan, err = g.balanceGroup(protocol, resp.Members, resp.SkipAssignment) + } else if leaderNoPlan { + g.leader.Store(true) + g.cfg.logger.Log(LogLevelInfo, "joined as leader but unable to balance group due to KIP-345 limitations", + "group", g.cfg.group, + "member_id", resp.MemberID, + "instance_id", strptr{g.cfg.instanceID}, + "generation", resp.Generation, + "balance_protocol", protocol, + "leader", true, + ) + } else { + g.cfg.logger.Log(LogLevelInfo, "joined", + "group", g.cfg.group, + "member_id", resp.MemberID, + "instance_id", strptr{g.cfg.instanceID}, + "generation", resp.Generation, + "leader", false, + ) + } + return restart, protocol, plan, err +} + +type strptr struct { + s *string +} + +func (s strptr) String() string { + if s.s == nil { + return "" + } + return *s.s +} + +// If other group members consume topics we are not interested in, we track the +// entire group's topics in this groupExternal type. On metadata update, we see +// if any partitions for any of these topics have changed, and if so, we as +// leader rejoin the group. +// +// Our external topics are cleared whenever we join and are not leader. We keep +// our previous external topics if we are leader: on the first balance as +// leader, we request metadata for all topics, then on followup balances, we +// already have that metadata and do not need to reload it when balancing. +// +// Whenever metadata updates, we detect if a rejoin is needed and always reset +// the rejoin status. +type groupExternal struct { + tps atomic.Value // map[string]int32 + rejoin atomic.Bool +} + +func (g *groupConsumer) loadExternal() *groupExternal { + e := g.external.Load() + if e != nil { + return e.(*groupExternal) + } + return nil +} + +// We reset our external topics whenever join&sync loop errors, or when we join +// and are not leader. +func (g *groupConsumer) resetExternal() { + g.external.Store((*groupExternal)(nil)) +} + +// If this is our first join as leader, or if a new member joined with new +// topics we were not tracking, we re-initialize external with the all-topics +// metadata refresh. +func (g *groupConsumer) initExternal(current map[string]int32) { + var e groupExternal + e.tps.Store(dupmsi32(current)) + g.external.Store(&e) +} + +// Reset whenever we join, & potentially used to rejoin when finding new +// assignments (i.e., end of metadata). +func (g *groupConsumer) getAndResetExternalRejoin() bool { + e := g.loadExternal() + if e == nil { + return false + } + defer e.rejoin.Store(false) + return e.rejoin.Load() +} + +// Runs fn over a load, not copy, of our map. +func (g *groupExternal) fn(fn func(map[string]int32)) { + if g == nil { + return + } + v := g.tps.Load() + if v == nil { + return + } + tps := v.(map[string]int32) + fn(tps) +} + +// Runs fn over a clone of our external map and updates the map. +func (g *groupExternal) cloned(fn func(map[string]int32)) { + g.fn(func(tps map[string]int32) { + dup := dupmsi32(tps) + fn(dup) + g.tps.Store(dup) + }) +} + +func (g *groupExternal) eachTopic(fn func(string)) { + g.fn(func(tps map[string]int32) { + for t := range tps { + fn(t) + } + }) +} + +func (g *groupExternal) updateLatest(meta map[string]*metadataTopic) { + g.cloned(func(tps map[string]int32) { + var rejoin bool + for t, ps := range tps { + latest, exists := meta[t] + if !exists || latest.loadErr != nil { + continue + } + if psLatest := int32(len(latest.partitions)); psLatest != ps { + rejoin = true + tps[t] = psLatest + } + } + if rejoin { + g.rejoin.Store(true) + } + }) +} + +func (g *groupConsumer) handleSyncResp(protocol string, resp *kmsg.SyncGroupResponse) error { + if err := kerr.ErrorForCode(resp.ErrorCode); err != nil { + return err + } + + b, err := g.findBalancer("sync assignment", protocol) + if err != nil { + return err + } + + assigned, err := b.ParseSyncAssignment(resp.MemberAssignment) + if err != nil { + g.cfg.logger.Log(LogLevelError, "sync assignment parse failed", "group", g.cfg.group, "err", err) + return err + } + for _, v := range assigned { + slices.Sort(v) + } + + g.cfg.logger.Log(LogLevelInfo, "synced", "group", g.cfg.group, "assigned", mtps(assigned)) + + // Past this point, we will fall into the setupAssigned prerevoke code, + // meaning for cooperative, we will revoke what we need to. + g.nowAssigned.store(assigned) + return nil +} + +func (g *groupConsumer) joinGroupProtocols() []kmsg.JoinGroupRequestProtocol { + g.mu.Lock() + + topics := make([]string, 0, len(g.using)) + for topic := range g.using { + topics = append(topics, topic) + } + lastDup := make(map[string][]int32, len(g.lastAssigned)) + for t, ps := range g.lastAssigned { + lastDup[t] = slices.Clone(ps) // deep copy to allow modifications + } + + g.mu.Unlock() + + sort.Strings(topics) // we guarantee to JoinGroupMetadata that the input strings are sorted + for _, partitions := range lastDup { + slices.Sort(partitions) // same for partitions + } + + gen := g.memberGen.generation() + var protos []kmsg.JoinGroupRequestProtocol + for _, balancer := range g.cfg.balancers { + proto := kmsg.NewJoinGroupRequestProtocol() + proto.Name = balancer.ProtocolName() + proto.Metadata = balancer.JoinGroupMetadata(topics, lastDup, gen) + + // KIP-881: inject our rack into the consumer metadata so the + // leader can do rack-aware assignment. We only set Rack if + // the balancer did not already set it (a user's custom + // balancer might use the field for something else). + if g.cfg.rack != "" { + var meta kmsg.ConsumerMemberMetadata + if err := meta.ReadFrom(proto.Metadata); err == nil && meta.Rack == nil { + meta.Rack = &g.cfg.rack + if meta.Version < 3 { + meta.Version = 3 + } + proto.Metadata = meta.AppendTo(nil) + } + } + + protos = append(protos, proto) + } + return protos +} + +// If we are cooperatively consuming, we have a potential problem: if fetch +// offsets is canceled due to an immediate rebalance, when we resume, we will +// not re-fetch offsets for partitions we were previously assigned and are +// still assigned. We will only fetch offsets for new assignments. +// +// To work around that issue, we track everything we are fetching in g.fetching +// and only clear g.fetching if fetchOffsets returns with no error. +// +// Now, if fetching returns early due to an error, when we rejoin and re-fetch, +// we will resume fetching what we were previously: +// +// - first we remove what was lost +// - then we add anything new +// - then we translate our total set into the "added" list to be fetched on return +// +// Any time a group is completely lost, the manage loop clears fetching. When +// cooperative consuming, a hard error is basically losing the entire state and +// rejoining from scratch. +func (g *groupConsumer) adjustCooperativeFetchOffsets(added, lost map[string][]int32) map[string][]int32 { + if g.fetching != nil { + // We were fetching previously: remove anything lost. + for topic, partitions := range lost { + ft := g.fetching[topic] + if ft == nil { + continue // we were not fetching this topic + } + for _, partition := range partitions { + delete(ft, partition) + } + if len(ft) == 0 { + delete(g.fetching, topic) + } + } + } else { + // We were not fetching previously: start a new map for what we + // are adding. + g.fetching = make(map[string]map[int32]struct{}) + } + + // Merge everything we are newly fetching to our fetching map. + for topic, partitions := range added { + ft := g.fetching[topic] + if ft == nil { + ft = make(map[int32]struct{}, len(partitions)) + g.fetching[topic] = ft + } + for _, partition := range partitions { + ft[partition] = struct{}{} + } + } + + // Now translate our full set (previously fetching ++ newly fetching -- + // lost) into a new "added" map to be fetched. + added = make(map[string][]int32, len(g.fetching)) + for topic, partitions := range g.fetching { + ps := make([]int32, 0, len(partitions)) + for partition := range partitions { + ps = append(ps, partition) + } + added[topic] = ps + } + return added +} + +// fetchOffsets is issued once we join a group to see what the prior commits +// were for the partitions we were assigned. +func (g *groupConsumer) fetchOffsets(ctx context.Context, added map[string][]int32) (rerr error) { // we must use "rerr"! see introducing commit + // If we fetch successfully, we can clear the cross-group-cycle + // fetching tracking. + defer func() { + if rerr == nil { + g.fetching = nil + } + }() + + // Our client maps the v0 to v7 format to v8+ when sharding this + // request, if we are only requesting one group. We iterate the v8+ + // Groups format in the response rather than the v0-v7 resp.Topics + // because the sharder's onResp resolves TopicID -> Topic in the + // Groups format, and resp.Topics is a copy that may lose TopicID. + var staleRetries int + var unknownTopicIDRetries int +start: + member, gen := g.memberGen.load() + req := kmsg.NewPtrOffsetFetchRequest() + req.RequireStable = true + reqg := kmsg.NewOffsetFetchRequestGroup() + reqg.Group = g.cfg.group + if member != "" { + reqg.MemberID = &member + reqg.MemberEpoch = gen + } + groupTopics := g.tps.load() + pinV9 := false + for topic, partitions := range added { + reqTopic := kmsg.NewOffsetFetchRequestGroupTopic() + reqTopic.Topic = topic + reqTopic.TopicID = groupTopics.loadTopic(topic).id + if reqTopic.TopicID == ([16]byte{}) { + pinV9 = true + } + reqTopic.Partitions = partitions + reqg.Topics = append(reqg.Topics, reqTopic) + } + req.Groups = append(req.Groups, reqg) + + // OffsetFetch v10 switched Topic to TopicID. If we have no TopicID + // for some topic (e.g. broker caps Metadata below v10, like Azure + // Event Hubs), v10+ would put a zero TopicID on the wire. Pin to + // v9 so the broker continues to match by name. See #1312. + reqCtx := ctx + if pinV9 { + reqCtx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 9}) + } + + var resp *kmsg.OffsetFetchResponse + var err error + + g.cfg.logger.Log(LogLevelDebug, "fetching offsets", + "group", g.cfg.group, + "require_stable", req.RequireStable, + "num_topics", len(reqg.Topics), + ) + fetchDone := make(chan struct{}) + go func() { + defer close(fetchDone) + resp, err = req.RequestWith(reqCtx, g.cl) + }() + select { + case <-fetchDone: + g.cfg.logger.Log(LogLevelDebug, "fetch offsets returned", "group", g.cfg.group, "err", err) + case <-ctx.Done(): + g.cfg.logger.Log(LogLevelInfo, "fetch offsets failed due to context cancelation", "group", g.cfg.group) + return ctx.Err() + } + if err != nil { + g.cfg.logger.Log(LogLevelError, "fetch offsets failed with non-retryable error", "group", g.cfg.group, "err", err) + return err + } + + // Check the group-level error code. For 848 consumers, the + // server validates MemberEpoch on OffsetFetch and returns + // STALE_MEMBER_EPOCH if the epoch changed between the + // heartbeat that assigned partitions and this OffsetFetch. + // Under rebalance churn STALE can fire several times in a row + // while the server is still advancing epochs; force a + // heartbeat and retry up to 10 times (each attempt pauses for + // the HB roundtrip, so this is paced, not a spin). Beyond 10, + // surface the error so manage848 can reset the member and + // re-initialJoin rather than looping here forever. + if err = kerr.ErrorForCode(resp.ErrorCode); err != nil { + if errors.Is(err, kerr.StaleMemberEpoch) { + staleRetries++ + if staleRetries > 10 { + g.cfg.logger.Log(LogLevelError, "fetch offsets: stale member epoch after 10 retries, giving up", "group", g.cfg.group) + return err + } + g.cfg.logger.Log(LogLevelInfo, "fetch offsets returned stale member epoch, forcing heartbeat and retrying", + "group", g.cfg.group, + "attempt", staleRetries, + ) + done := make(chan error, 1) + select { + case g.heartbeatForceCh <- func(err error) { done <- err }: + case <-ctx.Done(): + return ctx.Err() + } + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + goto start + } + g.cfg.logger.Log(LogLevelError, "fetch offsets failed with group-level error", "group", g.cfg.group, "err", err) + return err + } + + // Even if a leader epoch is returned, if brokers do not support + // OffsetForLeaderEpoch for some reason (odd set of supported reqs), we + // cannot use the returned leader epoch. + kip320 := g.cl.supportsOffsetForLeaderEpoch() + + id2t := g.cl.id2tMap() + offsets := make(map[string]map[int32]Offset) + for _, rTopic := range resp.Groups[0].Topics { + topic := rTopic.Topic + if topic == "" { + topic = id2t[rTopic.TopicID] + if topic == "" { + for _, reqTopic := range req.Groups[0].Topics { + if reqTopic.TopicID == rTopic.TopicID { + topic = reqTopic.Topic + break + } + } + if topic == "" { + g.cfg.logger.Log(LogLevelError, "fetch offsets has an empty topic even after a TopicID lookup, this is unexpected, skipping response partition", "topic_id", rTopic.TopicID) + continue + } + g.cfg.logger.Log(LogLevelError, "fetch offsets response had an empty topic name for a TopicID that was in the request, using the request's topic name", "topic", topic, "topic_id", rTopic.TopicID) + } + } + topicOffsets := make(map[int32]Offset) + offsets[topic] = topicOffsets + for _, rPartition := range rTopic.Partitions { + if err = kerr.ErrorForCode(rPartition.ErrorCode); err != nil { + // Some partition errors are retryable: + // + // - UnstableOffsetCommit (KIP-447): a pending + // transaction should be committing soon. + // + // - UnknownTopicID: the broker has not yet + // propagated the topic ID for a newly created + // topic. We now send TopicIDs in OffsetFetch + // v10+. We cap retries because the topic may + // have been legitimately deleted. + retryable := errors.Is(err, kerr.UnstableOffsetCommit) || + errors.Is(err, kerr.UnknownTopicID) && unknownTopicIDRetries < 3 + if errors.Is(err, kerr.UnknownTopicID) { + unknownTopicIDRetries++ + } + if retryable { + g.cfg.logger.Log(LogLevelInfo, "fetch offsets failed with retryable partition error, waiting 1s and retrying", + "group", g.cfg.group, + "topic", topic, + "partition", rPartition.Partition, + "err", err, + ) + select { + case <-ctx.Done(): + // Cancellation here means the + // heartbeat exited (rebalance, new + // assignment, client close); the + // session is tearing down. Returning + // ctx.Err() prevents falling through + // to the non-retryable injection path + // below, which would surface a + // transient retryable error as a fake + // fetch error to the user. + return ctx.Err() + case <-time.After(time.Second): + goto start + } + } + // A single non-retryable partition error (e.g. + // TopicAuthorizationFailed) used to abort the + // entire fetchOffsets and tear the session down, + // which then immediately rejoined and hit the same + // error: a spin loop driven by one bad partition. + // Instead we surface the error to the user via a + // fake fetch and drop the partition from this + // assignment; the rest of the session proceeds. + g.cfg.logger.Log(LogLevelError, "fetch offsets failed for partition; injecting error and continuing with remaining partitions", + "group", g.cfg.group, + "topic", topic, + "partition", rPartition.Partition, + "err", err, + ) + g.c.addFakeReadyForDraining(topic, rPartition.Partition, err, "fetch offsets returned a non-retryable partition error") + continue + } + offset := Offset{ + at: rPartition.Offset, + epoch: -1, + } + if resp.Version >= 5 && kip320 { // KIP-320 + offset.epoch = rPartition.LeaderEpoch + } + if rPartition.Offset == -1 { + offset = g.cfg.startOffset + } + topicOffsets[rPartition.Partition] = offset + } + } + + // Validate the response against what we requested: drop any + // topic or partition the broker returned that we did not ask for. + // A buggy broker returning extra partitions can cause a data race + // if those partitions are already being consumed (see #1271). + // groupTopics was already loaded above to populate reqTopic.TopicID; + // reuse that snapshot so we validate against the same view that + // built the request. + for fetchedTopic, topicOffsets := range offsets { + if !groupTopics.hasTopic(fetchedTopic) { + delete(offsets, fetchedTopic) + g.cfg.logger.Log(LogLevelWarn, "member was assigned topic that we did not ask for in ConsumeTopics! skipping assigning this topic!", "group", g.cfg.group, "topic", fetchedTopic) + continue + } + addedParts, ok := added[fetchedTopic] + if !ok { + delete(offsets, fetchedTopic) + g.cfg.logger.Log(LogLevelWarn, "broker returned topic in OffsetFetch response that we did not request, skipping", "group", g.cfg.group, "topic", fetchedTopic) + continue + } + requested := make(map[int32]struct{}, len(addedParts)) + for _, p := range addedParts { + requested[p] = struct{}{} + } + for partition := range topicOffsets { + if _, ok := requested[partition]; !ok { + delete(topicOffsets, partition) + g.cfg.logger.Log(LogLevelWarn, "broker returned partition in OffsetFetch response that we did not request, skipping", "group", g.cfg.group, "topic", fetchedTopic, "partition", partition) + } + } + } + + if g.cfg.onFetched != nil { + g.onFetchedMu.Lock() + err = g.cfg.onFetched(ctx, g.cl, resp) + g.onFetchedMu.Unlock() + if err != nil { + return err + } + } + if g.cfg.adjustOffsetsBeforeAssign != nil { + g.onFetchedMu.Lock() + offsets, err = g.cfg.adjustOffsetsBeforeAssign(ctx, offsets) + g.onFetchedMu.Unlock() + if err != nil { + return err + } + } + + // Lock for assign and then updating uncommitted. + g.c.mu.Lock() + defer g.c.mu.Unlock() + g.mu.Lock() + defer g.mu.Unlock() + + // Eager: we already invalidated everything; nothing to re-invalidate. + // Cooperative: assign without invalidating what we are consuming. + g.c.assignPartitions(offsets, assignWithoutInvalidating, g.tps, fmt.Sprintf("newly fetched offsets for group %s", g.cfg.group)) + + // We need to update the uncommitted map so that SetOffsets(Committed) + // does not rewind before the committed offsets we just fetched. + if g.uncommitted == nil { + g.uncommitted = make(uncommitted, 10) + } + for topic, partitions := range offsets { + topicUncommitted := g.uncommitted[topic] + if topicUncommitted == nil { + topicUncommitted = make(map[int32]uncommit, 20) + g.uncommitted[topic] = topicUncommitted + } + for partition, offset := range partitions { + if offset.at < 0 { + continue // not yet committed + } + committed := EpochOffset{ + Epoch: offset.epoch, + Offset: offset.at, + } + topicUncommitted[partition] = uncommit{ + dirty: committed, + head: committed, + committed: committed, + } + } + } + return nil +} + +// findNewAssignments updates topics the group wants to use and other metadata. +// We only grab the group mu at the end if we need to. +// +// This joins the group if +// - the group has never been joined +// - new topics are found for consuming (changing this consumer's join metadata) +// +// Additionally, if the member is the leader, this rejoins the group if the +// leader notices new partitions in an existing topic. +// +// This does not rejoin if the leader notices a partition is lost, which is +// finicky. +func (g *groupConsumer) findNewAssignments() { + topics := g.tps.load() + + type change struct { + isNew bool + delta int + } + + var numNewTopics int + toChange := make(map[string]change, len(topics)) + for topic, topicPartitions := range topics { + parts := topicPartitions.load() + numPartitions := len(parts.partitions) + // If we are already using this topic, add that it changed if + // there are more partitions than we were using prior. + if used, exists := g.using[topic]; exists { + if added := numPartitions - used; added > 0 { + toChange[topic] = change{delta: added} + } + continue + } + + // We are iterating over g.tps, which is initialized in the + // group.init from the config's topics, but can also be added + // to in AddConsumeTopics. By default, we use the topic. If + // this is regex based, the config's topics are regular + // expressions that we need to evaluate against (and we do not + // support adding new regex). + useTopic := true + if g.cfg.regex { + useTopic = g.reSeen[topic] + } + + // We only track using the topic if there are partitions for + // it; if there are none, then the topic was set by _us_ as "we + // want to load the metadata", but the topic was not returned + // in the metadata (or it was returned with an error). + if useTopic && numPartitions > 0 { + if g.cfg.regex && parts.isInternal { + continue + } + toChange[topic] = change{isNew: true, delta: numPartitions} + numNewTopics++ + } + } + + externalRejoin := g.leader.Load() && g.getAndResetExternalRejoin() + + if len(toChange) == 0 && !externalRejoin { + return + } + + g.mu.Lock() + defer g.mu.Unlock() + + if g.dying { + return + } + + for topic, change := range toChange { + g.using[topic] += change.delta + } + + if !g.managing { + g.managing = true + if g.should848() { + g.is848 = true + go g.manage848() + return + } + go g.manage() + return + } + + if numNewTopics > 0 { + g.rejoin("rejoining because there are more topics to consume, our interests have changed") + } else if g.leader.Load() { + if len(toChange) > 0 { + g.rejoin("rejoining because we are the leader and noticed some topics have new partitions") + } else if externalRejoin { + g.rejoin("leader detected that partitions on topics another member is consuming have changed, rejoining to trigger rebalance") + } + } +} + +// uncommit tracks the latest offset polled (+1) and the latest commit. +// The reason head is just past the latest offset is because we want +// to commit TO an offset, not BEFORE an offset. +type uncommit struct { + dirty EpochOffset // if autocommitting, what will move to head on next Poll + head EpochOffset // ready to commit + committed EpochOffset // what is committed +} + +// EpochOffset combines a record offset with the leader epoch the broker +// was at when the record was written. +type EpochOffset struct { + // Epoch is the leader epoch of the record being committed. Truncation + // detection relies on the epoch of the CURRENT record. For truncation + // detection, the client asks "what is the end of this epoch?", + // which returns one after the end offset (see the next field, and + // check the docs on kmsg.OffsetForLeaderEpochRequest). + Epoch int32 + + // Offset is the offset of a record. If committing, this should be one + // AFTER a record's offset. Clients start consuming at the offset that + // is committed. + Offset int64 +} + +// Less returns whether the this EpochOffset is less than another. This is less +// than the other if this one's epoch is less, or the epoch's are equal and +// this one's offset is less. +func (e EpochOffset) Less(o EpochOffset) bool { + ee, oe := max(e.Epoch, -1), max(o.Epoch, -1) + return ee < oe || ee == oe && e.Offset < o.Offset +} + +type uncommitted map[string]map[int32]uncommit + +// updateUncommitted sets the latest uncommitted offset. +func (g *groupConsumer) updateUncommitted(fetches Fetches) { + var b bytes.Buffer + debug := g.cfg.logger.Level() >= LogLevelDebug + + // We set the head offset if autocommitting is disabled (because we + // only use head / committed in that case), or if we are greedily + // autocommitting (so that the latest head is available to autocommit). + setHead := g.cfg.autocommitDisable || g.cfg.autocommitGreedy + + g.mu.Lock() + defer g.mu.Unlock() + + for _, fetch := range fetches { + for _, topic := range fetch.Topics { + if debug { + fmt.Fprintf(&b, "%s[", topic.Topic) + } + var topicOffsets map[int32]uncommit + for _, partition := range topic.Partitions { + if len(partition.Records) == 0 { + continue + } + final := partition.Records[len(partition.Records)-1] + + if topicOffsets == nil { + if g.uncommitted == nil { + g.uncommitted = make(uncommitted, 10) + } + topicOffsets = g.uncommitted[topic.Topic] + if topicOffsets == nil { + topicOffsets = make(map[int32]uncommit, 20) + g.uncommitted[topic.Topic] = topicOffsets + } + } + + // Our new head points just past the final consumed offset, + // that is, if we rejoin, this is the offset to begin at. + set := EpochOffset{ + final.LeaderEpoch, // -1 if old message / unknown + final.Offset + 1, + } + prior, ok := topicOffsets[partition.Partition] + if !ok { + uninit := EpochOffset{-1, 0} + uncommit := uncommit{uninit, uninit, uninit} + prior, topicOffsets[partition.Partition] = uncommit, uncommit + } + + if debug { + if setHead { + fmt.Fprintf(&b, "%d{%d=>%d r%d e%d}, ", partition.Partition, prior.head.Offset, set.Offset, len(partition.Records), set.Epoch) + } else { + fmt.Fprintf(&b, "%d{%d=>%d=>%d r%d e%d}, ", partition.Partition, prior.head.Offset, prior.dirty.Offset, set.Offset, len(partition.Records), set.Epoch) + } + } + + prior.dirty = set + if setHead { + prior.head = set + } + topicOffsets[partition.Partition] = prior + } + + if debug { + if bytes.HasSuffix(b.Bytes(), []byte(", ")) { + b.Truncate(b.Len() - 2) + } + b.WriteString("], ") + } + } + } + + if debug { + update := b.String() + update = strings.TrimSuffix(update, ", ") // trim trailing comma and space after final topic + g.cfg.logger.Log(LogLevelDebug, "updated uncommitted", "group", g.cfg.group, "to", update) + } +} + +// Called at the start of PollXyz only if autocommitting is enabled and we are +// not committing greedily, this ensures that when we enter poll, everything +// previously consumed is a candidate for autocommitting. +func (g *groupConsumer) undirtyUncommitted() { + if g == nil { + return + } + // Disabling autocommit means we do not use the dirty offset: we always + // update head, and then manual commits use that. + if g.cfg.autocommitDisable { + return + } + // Greedy autocommitting does not use dirty offsets, because we always + // just set head to the latest. + if g.cfg.autocommitGreedy { + return + } + // If we are autocommitting marked records only, then we do not + // automatically un-dirty our offsets. + if g.cfg.autocommitMarks { + return + } + + g.mu.Lock() + defer g.mu.Unlock() + + for _, partitions := range g.uncommitted { + for partition, uncommit := range partitions { + if uncommit.dirty != uncommit.head { + uncommit.head = uncommit.dirty + partitions[partition] = uncommit + } + } + } +} + +// updateCommitted updates the group's uncommitted map. This function triply +// verifies that the resp matches the req as it should and that the req does +// not somehow contain more than what is in our uncommitted map. +func (g *groupConsumer) updateCommitted( + req *kmsg.OffsetCommitRequest, + resp *kmsg.OffsetCommitResponse, +) { + g.mu.Lock() + defer g.mu.Unlock() + + if req.Generation != g.memberGen.generation() { + return + } + if g.uncommitted == nil { + g.cfg.logger.Log(LogLevelWarn, "received an OffsetCommitResponse after our group session has ended, unable to handle this (were we kicked from the group?)") + return + } + if len(req.Topics) != len(resp.Topics) { // bad kafka + g.cfg.logger.Log(LogLevelError, fmt.Sprintf("broker replied to our OffsetCommitRequest incorrectly! Num topics in request: %d, in reply: %d, we cannot handle this!", len(req.Topics), len(resp.Topics)), "group", g.cfg.group) + return + } + + // v10+: response uses TopicID instead of Topic. Resolve TopicIDs to + // names using the request (which has both) so sorting/matching works. + if resp.Version >= 10 { + id2t := make(map[[16]byte]string, len(req.Topics)) + for _, t := range req.Topics { + id2t[t.TopicID] = t.Topic + } + for i := range resp.Topics { + if resp.Topics[i].Topic == "" { + resp.Topics[i].Topic = id2t[resp.Topics[i].TopicID] + } + } + } + + sort.Slice(req.Topics, func(i, j int) bool { + return req.Topics[i].Topic < req.Topics[j].Topic + }) + sort.Slice(resp.Topics, func(i, j int) bool { + return resp.Topics[i].Topic < resp.Topics[j].Topic + }) + + var b bytes.Buffer + debug := g.cfg.logger.Level() >= LogLevelDebug + + for i := range resp.Topics { + reqTopic := &req.Topics[i] + respTopic := &resp.Topics[i] + topic, exists := g.uncommitted[respTopic.Topic] + if !exists { + continue // just in case; concurrent rebalance lost the topic while commit was in flight + } + if reqTopic.Topic != respTopic.Topic || // bad kafka + len(reqTopic.Partitions) != len(respTopic.Partitions) { // same + g.cfg.logger.Log(LogLevelError, fmt.Sprintf("broker replied to our OffsetCommitRequest incorrectly! Topic at request index %d: %s, reply at index: %s; num partitions on request topic: %d, in reply: %d, we cannot handle this!", i, reqTopic.Topic, respTopic.Topic, len(reqTopic.Partitions), len(respTopic.Partitions)), "group", g.cfg.group) + continue + } + + sort.Slice(reqTopic.Partitions, func(i, j int) bool { + return reqTopic.Partitions[i].Partition < reqTopic.Partitions[j].Partition + }) + sort.Slice(respTopic.Partitions, func(i, j int) bool { + return respTopic.Partitions[i].Partition < respTopic.Partitions[j].Partition + }) + + if debug { + fmt.Fprintf(&b, "%s[", respTopic.Topic) + } + for i := range respTopic.Partitions { + reqPart := &reqTopic.Partitions[i] + respPart := &respTopic.Partitions[i] + uncommit, exists := topic[respPart.Partition] + if !exists { // just in case + continue + } + if reqPart.Partition != respPart.Partition { // bad kafka + g.cfg.logger.Log(LogLevelError, fmt.Sprintf("broker replied to our OffsetCommitRequest incorrectly! Topic %s partition %d != resp partition %d", reqTopic.Topic, reqPart.Partition, respPart.Partition), "group", g.cfg.group) + continue + } + if respPart.ErrorCode != 0 { + g.cfg.logger.Log(LogLevelWarn, "unable to commit offset for topic partition", + "group", g.cfg.group, + "topic", reqTopic.Topic, + "partition", reqPart.Partition, + "commit_from", uncommit.committed.Offset, + "commit_to", reqPart.Offset, + "commit_epoch", reqPart.LeaderEpoch, + "error_code", respPart.ErrorCode, + ) + continue + } + + if debug { + fmt.Fprintf(&b, "%d{%d=>%d}, ", reqPart.Partition, uncommit.committed.Offset, reqPart.Offset) + } + + set := EpochOffset{ + reqPart.LeaderEpoch, + reqPart.Offset, + } + uncommit.committed = set + + // head is set in four places: + // (1) if manually committing or greedily autocommitting, + // then head is bumped on poll + // (2) if autocommitting normally, then head is bumped + // to the prior poll on poll + // (3) if using marks, head is bumped on mark + // (4) here, and we can be here on autocommit or on + // manual commit (usually manual in an onRevoke) + // + // head is usually at or past the commit: usually, head + // is used to build the commit itself. However, in case 4 + // when the user manually commits in onRevoke, the user + // is likely committing with UncommittedOffsets, i.e., + // the dirty offsets that are past the current head. + // We want to ensure we forward the head so that using + // it later does not rewind the manual commit. + // + // This does not affect the first case, because dirty == head, + // and manually committing dirty changes nothing. + // + // This does not affect the second case, because effectively, + // this is just bumping head early (dirty == head, no change). + // + // This *could* affect the third case, because an + // autocommit could begin, followed by a mark rewind, + // followed by autocommit completion. We document that + // using marks to rewind is not recommended. + // + // The user could also muck the offsets with SetOffsets. + // We document that concurrent committing is not encouraged, + // we do not attempt to guard past that. + // + // w.r.t. leader epoch's, we document that modifying + // leader epoch's is not recommended. + if uncommit.head.Less(set) { + uncommit.head = set + } + + topic[respPart.Partition] = uncommit + } + + if debug { + if bytes.HasSuffix(b.Bytes(), []byte(", ")) { + b.Truncate(b.Len() - 2) + } + b.WriteString("], ") + } + } + + if debug { + update := b.String() + update = strings.TrimSuffix(update, ", ") // trim trailing comma and space after final topic + g.cfg.logger.Log(LogLevelDebug, "updated committed", "group", g.cfg.group, "to", update) + } +} + +func (g *groupConsumer) defaultCommitCallback(_ *Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + if err != nil { + if !errors.Is(err, context.Canceled) { + g.cfg.logger.Log(LogLevelError, "default commit failed", "group", g.cfg.group, "err", err) + } else { + g.cfg.logger.Log(LogLevelDebug, "default commit canceled", "group", g.cfg.group) + } + return + } + for _, topic := range resp.Topics { + for _, partition := range topic.Partitions { + if err := kerr.ErrorForCode(partition.ErrorCode); err != nil { + g.cfg.logger.Log(LogLevelError, "in default commit: unable to commit offsets for topic partition", + "group", g.cfg.group, + "topic", topic.Topic, + "partition", partition.Partition, + "error", err) + } + } + } +} + +func (g *groupConsumer) loopCommit() { + ticker := time.NewTicker(g.cfg.autocommitInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-g.ctx.Done(): + return + } + + // We use the group context for the default autocommit; revokes + // use the client context so that we can be sure we commit even + // after the group context is canceled (which is the first + // thing that happens so as to quit the manage loop before + // leaving a group). + // + // We always commit only the head. If we are autocommitting + // dirty, then updateUncommitted updates the head to dirty + // offsets. + g.noCommitDuringJoinAndSync.RLock() + g.mu.Lock() + if !g.blockAuto { + uncommitted := g.getUncommittedLocked(true, false) + if len(uncommitted) == 0 { + g.cfg.logger.Log(LogLevelDebug, "skipping autocommit due to no offsets to commit", "group", g.cfg.group) + g.noCommitDuringJoinAndSync.RUnlock() + } else { + g.cfg.logger.Log(LogLevelDebug, "autocommitting", "group", g.cfg.group) + g.commit(g.ctx, uncommitted, func(cl *Client, req *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + g.noCommitDuringJoinAndSync.RUnlock() + g.cfg.commitCallback(cl, req, resp, err) + }) + } + } else { + g.noCommitDuringJoinAndSync.RUnlock() + } + g.mu.Unlock() + } +} + +// applySetOffsets applies the given offsets to g.uncommitted for partitions +// that are currently being consumed, updating their dirty, head, and committed +// fields. Partitions not in g.uncommitted are skipped (per SetOffsets docs: +// "any extra partitions are skipped"). If any partition's dirty field actually +// changed, the partition is returned in assigns so the caller can reassign +// cursors. If all partitions already had the same dirty offset, this returns +// nil and no cursor reassignment is needed - this is a key optimization for +// transactions resetting state on abort. +func (g *groupConsumer) applySetOffsets(setOffsets map[string]map[int32]EpochOffset) (assigns map[string]map[int32]Offset) { + g.mu.Lock() + defer g.mu.Unlock() + + groupTopics := g.tps.load() + + if g.uncommitted == nil { + return nil + } + for topic, partitions := range setOffsets { + if !groupTopics.hasTopic(topic) { + continue // trying to set a topic that was not assigned... + } + topicUncommitted := g.uncommitted[topic] + if topicUncommitted == nil { + continue // topic was not being consumed + } + var topicAssigns map[int32]Offset + for partition, epochOffset := range partitions { + current, exists := topicUncommitted[partition] + if !exists { + continue // partition was not being consumed + } + topicUncommitted[partition] = uncommit{ + dirty: epochOffset, + head: epochOffset, + committed: epochOffset, + } + if current.dirty == epochOffset { + continue + } else if topicAssigns == nil { + topicAssigns = make(map[int32]Offset, len(partitions)) + } + topicAssigns[partition] = Offset{ + at: epochOffset.Offset, + epoch: epochOffset.Epoch, + } + } + if len(topicAssigns) > 0 { + if assigns == nil { + assigns = make(map[string]map[int32]Offset, 10) + } + assigns[topic] = topicAssigns + } + } + + return assigns +} + +// UncommittedOffsets returns the latest uncommitted offsets. Uncommitted +// offsets are always updated on calls to PollFetches. +// +// If there are no uncommitted offsets, this returns nil. +func (cl *Client) UncommittedOffsets() map[string]map[int32]EpochOffset { + if g := cl.consumer.g; g != nil { + return g.getUncommitted(true) + } + return nil +} + +// MarkedOffsets returns the latest marked offsets. When autocommitting, a +// marked offset is an offset that can be committed, in comparison to a dirty +// offset that cannot yet be committed. MarkedOffsets returns nil if you are +// not using AutoCommitMarks. +func (cl *Client) MarkedOffsets() map[string]map[int32]EpochOffset { + g := cl.consumer.g + if g == nil || !cl.cfg.autocommitMarks { + return nil + } + return g.getUncommitted(false) +} + +// CommittedOffsets returns the latest committed offsets. Committed offsets are +// updated from commits or from joining a group and fetching offsets. +// +// If there are no committed offsets, this returns nil. +func (cl *Client) CommittedOffsets() map[string]map[int32]EpochOffset { + g := cl.consumer.g + if g == nil { + return nil + } + g.mu.Lock() + defer g.mu.Unlock() + + return g.getUncommittedLocked(false, false) +} + +func (g *groupConsumer) getUncommitted(dirty bool) map[string]map[int32]EpochOffset { + g.mu.Lock() + defer g.mu.Unlock() + return g.getUncommittedLocked(true, dirty) +} + +func (g *groupConsumer) getUncommittedLocked(head, dirty bool) map[string]map[int32]EpochOffset { + if g.uncommitted == nil { + return nil + } + + var uncommitted map[string]map[int32]EpochOffset + for topic, partitions := range g.uncommitted { + var topicUncommitted map[int32]EpochOffset + for partition, uncommit := range partitions { + if head && (dirty && uncommit.dirty == uncommit.committed || !dirty && uncommit.head == uncommit.committed) { + continue + } + if topicUncommitted == nil { + if uncommitted == nil { + uncommitted = make(map[string]map[int32]EpochOffset, len(g.uncommitted)) + } + topicUncommitted = uncommitted[topic] + if topicUncommitted == nil { + topicUncommitted = make(map[int32]EpochOffset, len(partitions)) + uncommitted[topic] = topicUncommitted + } + } + if head { + if dirty { + topicUncommitted[partition] = uncommit.dirty + } else { + topicUncommitted[partition] = uncommit.head + } + } else { + topicUncommitted[partition] = uncommit.committed + } + } + } + return uncommitted +} + +var commitContextFn = func() *string { s := "commit_ctx"; return &s }() + +// PreCommitFnContext attaches fn to the context through WithValue. Using the +// context while committing allows fn to be called just before the commit is +// issued. This can be used to modify the actual commit, such as by associating +// metadata with partitions. If fn returns an error, the commit is not +// attempted. +func PreCommitFnContext(ctx context.Context, fn func(*kmsg.OffsetCommitRequest) error) context.Context { + return context.WithValue(ctx, commitContextFn, fn) +} + +var txnCommitContextFn = func() *string { s := "txn_commit_ctx"; return &s }() + +// PreTxnCommitFnContext attaches fn to the context through WithValue. Using +// the context while committing a transaction allows fn to be called just +// before the commit is issued. This can be used to modify the actual commit, +// such as by associating metadata with partitions (for transactions, the +// default internal metadata is the client's current member ID). If fn returns +// an error, the commit is not attempted. This context can be used in either +// GroupTransactSession.End or in Client.EndTransaction. +func PreTxnCommitFnContext(ctx context.Context, fn func(*kmsg.TxnOffsetCommitRequest) error) context.Context { + return context.WithValue(ctx, txnCommitContextFn, fn) +} + +// CommitRecords issues a synchronous offset commit for the offsets contained +// within rs. Retryable errors are retried up to the configured retry limit, +// and any unretryable error is returned. +// +// This function is useful as a simple way to commit offsets if you have +// disabled autocommitting. As an alternative if you always want to commit +// everything, see CommitUncommittedOffsets. +// +// Simple usage of this function may lead to duplicate records if a consumer +// group rebalance occurs before or while this function is being executed. You +// can avoid this scenario by calling CommitRecords in a custom +// OnPartitionsRevoked, but for most workloads, a small bit of potential +// duplicate processing is fine. See the documentation on DisableAutoCommit +// for more details. You can also avoid this problem by using +// BlockRebalanceOnPoll, but that option comes with its own tradeoffs (refer to +// its documentation). +// +// It is recommended to always commit records in order (per partition). If you +// call this function twice with record for partition 0 at offset 999 +// initially, and then with record for partition 0 at offset 4, you will rewind +// your commit. +// +// A use case for this function may be to partially process a batch of records, +// commit, and then continue to process the rest of the records. It is not +// recommended to call this for every record processed in a high throughput +// scenario, because you do not want to unnecessarily increase load on Kafka. +// +// If you do not want to wait for this function to complete before continuing +// processing records, you can call this function in a goroutine. +func (cl *Client) CommitRecords(ctx context.Context, rs ...*Record) error { + // First build the offset commit map. We favor the latest epoch, then + // offset, if any records map to the same topic / partition. + offsets := make(map[string]map[int32]EpochOffset) + for _, r := range rs { + toffsets := offsets[r.Topic] + if toffsets == nil { + toffsets = make(map[int32]EpochOffset) + offsets[r.Topic] = toffsets + } + + set := EpochOffset{ + r.LeaderEpoch, + r.Offset + 1, // need to advice to next offset to move forward + } + + if at, exists := toffsets[r.Partition]; exists { + if set.Less(at) { + continue + } + } + + toffsets[r.Partition] = set + } + + var rerr error // return error + + // Our client retries an OffsetCommitRequest as necessary if the first + // response partition has a retryable group error (group coordinator + // loading, etc), so any partition error is fatal. + cl.CommitOffsetsSync(ctx, offsets, func(_ *Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + if err != nil { + rerr = err + return + } + + for _, topic := range resp.Topics { + for _, partition := range topic.Partitions { + if err := kerr.ErrorForCode(partition.ErrorCode); err != nil { + rerr = err + return + } + } + } + }) + + return rerr +} + +// MarkCommitRecords marks records to be available for autocommitting. This +// function is only useful if you use the AutoCommitMarks config option, see +// the documentation on that option for more details. This function does not +// allow rewinds. +func (cl *Client) MarkCommitRecords(rs ...*Record) { + g := cl.consumer.g + if g == nil || !cl.cfg.autocommitMarks { + return + } + + sort.Slice(rs, func(i, j int) bool { + return rs[i].Topic < rs[j].Topic || + rs[i].Topic == rs[j].Topic && rs[i].Partition < rs[j].Partition + }) + + // protect g.uncommitted map + g.mu.Lock() + defer g.mu.Unlock() + + if g.uncommitted == nil { + g.uncommitted = make(uncommitted) + } + var curTopic string + var curPartitions map[int32]uncommit + for _, r := range rs { + if curPartitions == nil || r.Topic != curTopic { + curPartitions = g.uncommitted[r.Topic] + if curPartitions == nil { + curPartitions = make(map[int32]uncommit) + g.uncommitted[r.Topic] = curPartitions + } + curTopic = r.Topic + } + + current, ok := curPartitions[r.Partition] + if newHead := (EpochOffset{ + r.LeaderEpoch, + r.Offset + 1, + }); !ok || current.head.Less(newHead) { + curPartitions[r.Partition] = uncommit{ + dirty: current.dirty, + committed: current.committed, + head: newHead, + } + } + } +} + +// MarkCommitOffsets marks offsets to be available for autocommitting. This +// function is only useful if you use the AutoCommitMarks config option, see +// the documentation on that option for more details. This function does not +// allow rewinds. +func (cl *Client) MarkCommitOffsets(unmarked map[string]map[int32]EpochOffset) { + g := cl.consumer.g + if g == nil || !cl.cfg.autocommitMarks { + return + } + + // protect g.uncommitted map + g.mu.Lock() + defer g.mu.Unlock() + + if g.uncommitted == nil { + g.uncommitted = make(uncommitted) + } + + for topic, partitions := range unmarked { + curPartitions := g.uncommitted[topic] + if curPartitions == nil { + curPartitions = make(map[int32]uncommit) + g.uncommitted[topic] = curPartitions + } + + for partition, newHead := range partitions { + current, ok := curPartitions[partition] + if !ok || current.head.Less(newHead) { + curPartitions[partition] = uncommit{ + dirty: current.dirty, + committed: current.committed, + head: newHead, + } + } + } + } +} + +// CommitUncommittedOffsets issues a synchronous offset commit for any +// partition that has been consumed from that has uncommitted offsets. +// Retryable errors are retried up to the configured retry limit, and any +// unretryable error is returned. +// +// The recommended pattern for using this function is to have a poll / process +// / commit loop. First PollFetches, then process every record, then call +// CommitUncommittedOffsets. +// +// As an alternative if you want to commit specific records, see CommitRecords. +func (cl *Client) CommitUncommittedOffsets(ctx context.Context) error { + // This function is just the tail end of CommitRecords just above. + return cl.commitOffsets(ctx, cl.UncommittedOffsets()) +} + +// CommitMarkedOffsets issues a synchronous offset commit for any partition +// that has been consumed from that has marked offsets. Retryable errors are +// retried up to the configured retry limit, and any unretryable error is +// returned. +// +// This function is only useful if you have marked offsets with +// MarkCommitRecords when using AutoCommitMarks, otherwise this is a no-op. +// +// The recommended pattern for using this function is to have a poll / process +// / commit loop. First PollFetches, then process every record, +// call MarkCommitRecords for the records you wish the commit and then call +// CommitMarkedOffsets. +// +// As an alternative if you want to commit specific records, see CommitRecords. +func (cl *Client) CommitMarkedOffsets(ctx context.Context) error { + // This function is just the tail end of CommitRecords just above. + marked := cl.MarkedOffsets() + if len(marked) == 0 { + return nil + } + return cl.commitOffsets(ctx, marked) +} + +func (cl *Client) commitOffsets(ctx context.Context, offsets map[string]map[int32]EpochOffset) error { + var rerr error + cl.CommitOffsetsSync(ctx, offsets, func(_ *Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + if err != nil { + rerr = err + return + } + + for _, topic := range resp.Topics { + for _, partition := range topic.Partitions { + if err := kerr.ErrorForCode(partition.ErrorCode); err != nil { + rerr = err + return + } + } + } + }) + return rerr +} + +// CommitOffsetsSync cancels any active CommitOffsets, begins a commit that +// cannot be canceled, and waits for that commit to complete. This function +// will not return until the commit is done and the onDone callback is +// complete. +// +// The purpose of this function is for use in OnPartitionsRevoked or committing +// before leaving a group, because you do not want to have a commit issued in +// OnPartitionsRevoked canceled. +// +// This is an advanced function, and for simpler, more easily understandable +// committing, see CommitRecords and CommitUncommittedOffsets. +// +// For more information about committing and committing asynchronously, see +// CommitOffsets. +func (cl *Client) CommitOffsetsSync( + ctx context.Context, + uncommitted map[string]map[int32]EpochOffset, + onDone func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error), +) { + if onDone == nil { + onDone = func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error) {} + } + + g := cl.consumer.g + if g == nil { + onDone(cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), errNotGroup) + return + } + if len(uncommitted) == 0 { + onDone(cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), nil) + return + } + g.commitOffsetsSync(ctx, uncommitted, onDone) +} + +// waitJoinSyncMu is a rather insane way to try to grab a lock, but also return +// early if we have to wait and the context is canceled. +func (g *groupConsumer) waitJoinSyncMu(ctx context.Context) error { + if g.noCommitDuringJoinAndSync.TryRLock() { + g.cfg.logger.Log(LogLevelDebug, "grabbed join/sync mu on first try") + return nil + } + + var ( + blockJoinSyncCh = make(chan struct{}) + mu xsync.Mutex + returned bool + maybeRUnlock = func() { + mu.Lock() + defer mu.Unlock() + if returned { + g.noCommitDuringJoinAndSync.RUnlock() + } + returned = true + } + ) + + go func() { + g.noCommitDuringJoinAndSync.RLock() + close(blockJoinSyncCh) + maybeRUnlock() + }() + + select { + case <-blockJoinSyncCh: + g.cfg.logger.Log(LogLevelDebug, "grabbed join/sync mu after waiting") + return nil + case <-ctx.Done(): + g.cfg.logger.Log(LogLevelDebug, "not grabbing mu because context canceled") + maybeRUnlock() + return ctx.Err() + } +} + +func (g *groupConsumer) commitOffsetsSync( + ctx context.Context, + uncommitted map[string]map[int32]EpochOffset, + onDone func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error), +) { + g.cfg.logger.Log(LogLevelDebug, "in CommitOffsetsSync", "group", g.cfg.group, "with", uncommitted) + defer g.cfg.logger.Log(LogLevelDebug, "left CommitOffsetsSync", "group", g.cfg.group) + + done := make(chan struct{}) + defer func() { <-done }() + + if onDone == nil { + onDone = func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error) {} + } + + if err := g.waitJoinSyncMu(ctx); err != nil { + onDone(g.cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), err) + close(done) + return + } + + g.syncCommitMu.Lock() // block all other concurrent commits until our OnDone is done. + unblockCommits := func(cl *Client, req *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + g.noCommitDuringJoinAndSync.RUnlock() + defer close(done) + defer g.syncCommitMu.Unlock() + onDone(cl, req, resp, err) + } + + g.mu.Lock() + defer g.mu.Unlock() + + g.blockAuto = true + unblockAuto := func(cl *Client, req *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + unblockCommits(cl, req, resp, err) + g.mu.Lock() + defer g.mu.Unlock() + g.blockAuto = false + } + + g.commit(ctx, uncommitted, unblockAuto) +} + +// CommitOffsets commits the given offsets for a group, calling onDone with the +// commit request and either the response or an error if the response was not +// issued. If uncommitted is empty or the client is not consuming as a group, +// onDone is called with (nil, nil, nil) and this function returns immediately. +// It is OK if onDone is nil, but you will not know if your commit succeeded. +// +// This is an advanced function and is difficult to use correctly. For simpler, +// more easily understandable committing, see CommitRecords and +// CommitUncommittedOffsets. +// +// This function itself does not wait for the commit to finish. By default, +// this function is an asynchronous commit. You can use onDone to make it sync. +// If autocommitting is enabled, this function blocks autocommitting until this +// function is complete and the onDone has returned. +// +// It is invalid to use this function to commit offsets for a transaction. +// +// Note that this function ensures absolute ordering of commit requests by +// canceling prior requests and ensuring they are done before executing a new +// one. This means, for absolute control, you can use this function to +// periodically commit async and then issue a final sync commit before quitting +// (this is the behavior of autocommiting and using the default revoke). This +// differs from the Java async commit, which does not retry requests to avoid +// trampling on future commits. +// +// It is highly recommended to check the response's partition's error codes if +// the response is non-nil. While unlikely, individual partitions can error. +// This is most likely to happen if a commit occurs too late in a rebalance +// event. +// +// Do not use this async CommitOffsets in OnPartitionsRevoked, instead use +// CommitOffsetsSync. If you commit async, the rebalance will proceed before +// this function executes, and you will commit offsets for partitions that have +// moved to a different consumer. +func (cl *Client) CommitOffsets( + ctx context.Context, + uncommitted map[string]map[int32]EpochOffset, + onDone func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error), +) { + cl.cfg.logger.Log(LogLevelDebug, "in CommitOffsets", "with", uncommitted) + defer cl.cfg.logger.Log(LogLevelDebug, "left CommitOffsets") + if onDone == nil { + onDone = func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error) {} + } + + g := cl.consumer.g + if g == nil { + onDone(cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), errNotGroup) + return + } + if len(uncommitted) == 0 { + onDone(cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), nil) + return + } + + if err := g.waitJoinSyncMu(ctx); err != nil { + onDone(g.cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), err) + return + } + + g.syncCommitMu.RLock() // block sync commit, but allow other concurrent Commit to cancel us + unblockJoinSync := func(cl *Client, req *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + g.noCommitDuringJoinAndSync.RUnlock() + defer g.syncCommitMu.RUnlock() + onDone(cl, req, resp, err) + } + + g.mu.Lock() + defer g.mu.Unlock() + + g.blockAuto = true + unblockAuto := func(cl *Client, req *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) { + unblockJoinSync(cl, req, resp, err) + g.mu.Lock() + defer g.mu.Unlock() + g.blockAuto = false + } + + g.commit(ctx, uncommitted, unblockAuto) +} + +// defaultRevoke commits the last fetched offsets and waits for the commit to +// finish. This is the default onRevoked function which, when combined with the +// default autocommit, ensures we never miss committing everything. +// +// Note that the heartbeat loop invalidates all buffered, unpolled fetches +// before revoking, meaning this truly will commit all polled fetches. +func (g *groupConsumer) defaultRevoke(context.Context, *Client, map[string][]int32) { + if !g.cfg.autocommitDisable { + // We use the client's context rather than the group context, + // because this could come from the group being left. The group + // context will already be canceled. + g.commitOffsetsSync(g.cl.ctx, g.getUncommitted(false), g.cfg.commitCallback) + } +} + +// The actual logic to commit. This is called under two locks: +// - g.noCommitDuringJoinAndSync.RLock() +// - g.mu.Lock() +// +// By blocking the JoinGroup from being issued, or blocking the commit on join +// & sync finishing, we avoid RebalanceInProgress and IllegalGeneration. The +// former error happens if a commit arrives to the broker between the two, the +// latter error happens when a commit arrives to the broker with the old +// generation (it was in flight before sync finished). +// +// Practically, what this means is that a user's commits will be blocked if +// they try to commit between join and sync. +// +// For eager consuming, the user should not have any partitions to commit +// anyway. For cooperative consuming, a rebalance can happen after at any +// moment. We block only revokation aspects of rebalances with +// BlockRebalanceOnPoll; we want to allow the cooperative part of rebalancing +// to occur. +func (g *groupConsumer) commit( + ctx context.Context, + uncommitted map[string]map[int32]EpochOffset, + onDone func(*Client, *kmsg.OffsetCommitRequest, *kmsg.OffsetCommitResponse, error), +) { + // The user could theoretically give us topics that have no partitions + // to commit. We strip those: Kafka does not reply to them, and we + // expect all partitions in our request to be replied to in + // updateCommitted. If any topic is empty, we deeply clone and then + // strip everything empty. See #186. + var clone bool + for _, ps := range uncommitted { + if len(ps) == 0 { + clone = true + break + } + } + if clone { + dup := make(map[string]map[int32]EpochOffset, len(uncommitted)) + for t, ps := range uncommitted { + if len(ps) == 0 { + continue + } + dupPs := make(map[int32]EpochOffset, len(ps)) + dup[t] = dupPs + maps.Copy(dupPs, ps) + } + uncommitted = dup + } + + if len(uncommitted) == 0 { // only empty if called thru autocommit / default revoke + // We have to do this concurrently because the expectation is + // that commit itself does not block. + go onDone(g.cl, kmsg.NewPtrOffsetCommitRequest(), kmsg.NewPtrOffsetCommitResponse(), nil) + return + } + + priorDone := g.commitDone + + commitCtx, commitCancel := context.WithCancel(ctx) // enable ours to be canceled and waited for + commitDone := make(chan struct{}) + + g.commitDone = commitDone + + req := kmsg.NewPtrOffsetCommitRequest() + req.Group = g.cfg.group + memberID, generation := g.memberGen.load() + req.Generation = generation + req.MemberID = memberID + req.InstanceID = g.cfg.instanceID + + if ctx.Done() != nil { + go func() { + select { + case <-ctx.Done(): + commitCancel() + case <-commitCtx.Done(): + } + }() + } + + go func() { + defer close(commitDone) // allow future commits to continue when we are done + defer commitCancel() + if priorDone != nil { // wait for any prior request to finish + // We must NOT cancel the prior commit. Canceling an + // in-flight request kills the TCP connection. Our + // subsequent request would then use a new connection, + // and the broker can process the two requests out of + // order (different connections have no ordering + // guarantee). If the prior commit had a lower offset + // (e.g. autocommit HEAD vs. sync commit DIRTY), the + // broker could process it AFTER ours and overwrite + // our higher offset, causing duplicate consumption. + // + // By waiting for the prior commit to complete + // naturally, we keep the connection alive and + // guarantee our request is queued behind the prior on + // the same connection, preserving ordering. + select { + case <-priorDone: + default: + g.cfg.logger.Log(LogLevelDebug, "waiting for prior commit to finish before issuing another", "group", g.cfg.group) + <-priorDone + } + } + g.cfg.logger.Log(LogLevelDebug, "issuing commit", "group", g.cfg.group, "uncommitted", uncommitted) + + groupTopics := g.tps.load() + pinV9 := false + for topic, partitions := range uncommitted { + reqTopic := kmsg.NewOffsetCommitRequestTopic() + reqTopic.Topic = topic + if td := groupTopics.loadTopic(topic); td != nil { + reqTopic.TopicID = td.id + } + if reqTopic.TopicID == ([16]byte{}) { + pinV9 = true + } + for partition, eo := range partitions { + reqPartition := kmsg.NewOffsetCommitRequestTopicPartition() + reqPartition.Partition = partition + reqPartition.Offset = eo.Offset + reqPartition.LeaderEpoch = eo.Epoch // KIP-320 + reqPartition.Metadata = &req.MemberID + reqTopic.Partitions = append(reqTopic.Partitions, reqPartition) + } + req.Topics = append(req.Topics, reqTopic) + } + + // OffsetCommit v10 switched Topic to TopicID. If we have no + // TopicID for some topic (e.g. broker caps Metadata below v10, + // like Azure Event Hubs), v10+ would put a zero TopicID on the + // wire. Pin to v9 so the broker continues to match by name. + // See #1312. Held in a separate variable from commitCtx so + // the cancel/retry-sleep paths keep using the unwrapped ctx. + // + // pinV9 is computed once here and not recomputed inside the + // STALE_MEMBER_EPOCH retry loop below because that loop only + // DROPS partitions from req.Topics; it never re-adds topics + // whose TopicID state could change pinV9. If a future change + // allows re-adding topics on retry, recompute pinV9 (and rebuild + // reqCtx) inside the loop so a topic-id-less topic never lands + // on a v10+ wire by accident. + reqCtx := commitCtx + if pinV9 { + reqCtx = context.WithValue(commitCtx, ctxPinReq, &pinReq{pinMax: true, max: 9}) + } + + if fn, ok := ctx.Value(commitContextFn).(func(*kmsg.OffsetCommitRequest) error); ok { + if err := fn(req); err != nil { + onDone(g.cl, req, nil, err) + return + } + } + + var resp *kmsg.OffsetCommitResponse + var err error + // KIP-848 STALE_MEMBER_EPOCH is expected under rebalance + // churn: the heartbeat loop needs to observe the new epoch + // (via the paired HB response on the same connection) before + // we can rebuild the commit. Retry up to 10 times (each + // attempt sleeps 500ms between STALE responses) so that + // auto-commit before a rebalance does not surface a benign + // STALE to the user. + // + // On gen change, filter req.Topics down to partitions we + // still own before retrying so we do not race the new owner's + // commits for revoked partitions. The filtered-out partitions + // get synthesized REBALANCE_IN_PROGRESS responses after the + // loop so the caller still sees a per-partition result for + // everything they asked us to commit. We restore the original + // req.Topics before invoking onDone so the callback sees the + // caller's full request. + origReqTopics := req.Topics + // dropped tracks (topic name, topic id) -> partitions that + // were filtered out of req.Topics during retries. + type droppedTopic struct { + name string + id [16]byte + partitions []int32 + } + var dropped []droppedTopic + + staleRetries := 0 + for { + start := time.Now() + resp, err = req.RequestWith(reqCtx, g.cl) + if err != nil { + req.Topics = origReqTopics + onDone(g.cl, req, nil, err) + return + } + g.cl.metrics.observeTime(&g.cl.metrics.cCommitLatency, time.Since(start).Milliseconds()) + + // KIP-848 returns STALE_MEMBER_EPOCH when the + // commit's epoch doesn't match the server's current + // epoch. This commonly happens when a heartbeat and + // OffsetCommit are pipelined on the same connection + // and the server bumps the epoch before seeing the + // commit. The heartbeat response carrying the new + // epoch arrives before the commit response (same + // connection, ordered), but goroutine scheduling + // may let us process the commit response first. We + // sleep briefly to give the heartbeat loop time to + // update memberGen, then reload and retry. + // + // If the heartbeat loop has stopped (e.g. leave + // path with a canceled context), memberGen is not + // updated and we break without retrying. This is a + // known limitation without KIP-1251 (assignment + // epochs for consumer groups). + stale := false + for _, t := range resp.Topics { + for _, p := range t.Partitions { + if p.ErrorCode == kerr.StaleMemberEpoch.Code { + stale = true + } + } + } + if stale { + staleRetries++ + if staleRetries > 10 { + g.cfg.logger.Log(LogLevelError, "offset commit: stale member epoch after 10 retries, giving up", "group", g.cfg.group) + break + } + t := time.NewTimer(500 * time.Millisecond) + select { + case <-t.C: + case <-commitCtx.Done(): + t.Stop() + } + if commitCtx.Err() != nil { + break + } + _, newGen := g.memberGen.load() + if newGen != req.Generation { + owned := g.nowAssigned.read() + keptTopics := make([]kmsg.OffsetCommitRequestTopic, 0, len(req.Topics)) + droppedParts := 0 + for _, rt := range req.Topics { + ownedParts := owned[rt.Topic] + kept := make([]kmsg.OffsetCommitRequestTopicPartition, 0, len(rt.Partitions)) + var drop []int32 + for _, rp := range rt.Partitions { + if slices.Contains(ownedParts, rp.Partition) { + kept = append(kept, rp) + } else { + drop = append(drop, rp.Partition) + } + } + if len(drop) > 0 { + dropped = append(dropped, droppedTopic{rt.Topic, rt.TopicID, drop}) + droppedParts += len(drop) + } + if len(kept) > 0 { + rt.Partitions = kept + keptTopics = append(keptTopics, rt) + } + } + req.Topics = keptTopics + g.cfg.logger.Log(LogLevelInfo, "offset commit got stale member epoch, retrying with updated epoch", + "group", g.cfg.group, + "old_epoch", req.Generation, + "new_epoch", newGen, + "lost_partitions_filtered", droppedParts, + ) + req.Generation = newGen + if len(req.Topics) == 0 { + // Nothing left on the wire; synthesize an empty + // response that the dropped-partition fill below + // will populate. + resp = kmsg.NewPtrOffsetCommitResponse() + resp.Version = req.Version + break + } + continue + } + } + break + } + + // Inject synthetic responses for partitions filtered out + // during retries so the caller sees a per-partition result + // for everything they asked us to commit. + for _, d := range dropped { + var rt *kmsg.OffsetCommitResponseTopic + for i := range resp.Topics { + if resp.Topics[i].Topic == d.name && resp.Topics[i].TopicID == d.id { + rt = &resp.Topics[i] + break + } + } + if rt == nil { + resp.Topics = append(resp.Topics, kmsg.OffsetCommitResponseTopic{ + Topic: d.name, + TopicID: d.id, + }) + rt = &resp.Topics[len(resp.Topics)-1] + } + for _, p := range d.partitions { + rt.Partitions = append(rt.Partitions, kmsg.OffsetCommitResponseTopicPartition{ + Partition: p, + ErrorCode: kerr.RebalanceInProgress.Code, + }) + } + } + + // Restore so updateCommitted and onDone see the caller's + // original request, not the wire-filtered one. + req.Topics = origReqTopics + + g.updateCommitted(req, resp) + onDone(g.cl, req, resp, nil) + }() +} + +type reNews struct { + added map[string][]string + skipped []string +} + +func (r *reNews) add(re, match string) { + if r.added == nil { + r.added = make(map[string][]string) + } + r.added[re] = append(r.added[re], match) +} + +func (r *reNews) skip(topic string) { + r.skipped = append(r.skipped, topic) +} + +func (r *reNews) log(cfg *cfg) { + if len(r.added) == 0 && len(r.skipped) == 0 { + return + } + var addeds []string + for re, matches := range r.added { + sort.Strings(matches) + addeds = append(addeds, fmt.Sprintf("%s[%s]", re, strings.Join(matches, " "))) + } + added := strings.Join(addeds, " ") + sort.Strings(r.skipped) + cfg.logger.Log(LogLevelInfo, "consumer regular expressions evaluated on new topics", "added", added, "evaluated_and_skipped", r.skipped) +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group_848.go b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group_848.go new file mode 100644 index 00000000000..756d45122e7 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_group_848.go @@ -0,0 +1,661 @@ +package kgo + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "reflect" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func (g *groupConsumer) should848() bool { + if wantBeta := g.cl.ctx.Value("opt_in_kafka_next_gen_balancer_beta"); wantBeta == nil { // !!! TODO REMOVE ONCE BROKER IMPROVES + return false + } + if g.cl.cfg.disableNextGenBalancer { + return false + } + // We pin to v1, introduced in Kafka 4, which fully stabilizes KIP-848. + if !g.cl.supportsKIP848v1() { + return false + } + switch g.cfg.balancers[0].(type) { + case *stickyBalancer: + case *rangeBalancer: + default: + return false + } + return true +} + +func (g *groupConsumer) manage848() { + var serverAssignor string + switch g.cfg.balancers[0].(type) { + case *stickyBalancer: + serverAssignor = "uniform" + case *rangeBalancer: + serverAssignor = "range" + } + + // fallbackToClassic is set when manage848 hands off to the classic + // manage() goroutine, which takes ownership of closing manageDone. + var fallbackToClassic bool + defer func() { + if !fallbackToClassic { + close(g.manageDone) + } + }() + var known848Support bool + optInKnown := func() { + if known848Support { + return + } + known848Support = true + g.cfg.logger.Log(LogLevelInfo, "beginning to manage the next-gen group lifecycle", "group", g.cfg.group) + g.cooperative.Store(true) // next gen is always cooperative + if !g.cfg.autocommitDisable && g.cfg.autocommitInterval > 0 { + g.cfg.logger.Log(LogLevelInfo, "beginning autocommit loop", "group", g.cfg.group) + go g.loopCommit() + } + } + + g.mu.Lock() + g848 := &g848{g: g, serverAssignor: serverAssignor} + g.g848 = g848 + g.mu.Unlock() + + // v1+ requires the client to generate their own memberID. + // On v0, the server provides the ID and we join twice. + // We pin to v1+. + g.memberGen.store(newStringUUID(), 0) // 0 joins the group + + // consecutiveErrors tracks failures in the outer loop: failed + // initialJoin attempts and manageFailWait invocations. Used for + // backoff scaling. Reset when the inner heartbeat loop is entered + // (meaning initialJoin succeeded). + var consecutiveErrors int + var unreleasedInstanceRetries int // capped at 3; see retryable-error branch below +outer: + for { + initialHb, err := g848.initialJoin() + + // Even if Kafka replies that the API is available, if we use it + // and the broker is not configured to support it, we receive + // UnsupportedVersion. On the first loop + if !known848Support { + if err != nil { + var ke *kerr.Error + if errors.As(err, &ke) { + if ke.Code == kerr.UnsupportedVersion.Code { + // It's okay to update is848 here. This is used while leaving + // and while heartbeating. We have not yet entered heartbeating, + // and if the user is concurrently leaving, the lack of a memberID + // means both 848 and old group mgmt leaves return early. + g.mu.Lock() + g.is848 = false + g.mu.Unlock() + g.cfg.logger.Log(LogLevelInfo, "falling back to standard consumer group management due to lack of broker support", "group", g.cfg.group) + fallbackToClassic = true + go g.manage() + return + } + optInKnown() // A kerr that is NOT UnsupportVersion means this is supported. + } + // For non-kerr errors, we fall into normal logic below and retry. + } else { + optInKnown() + } + } + + // Retryable errors from initialJoin should not be surfaced + // to the user: coordinator errors and broker-level errors + // (connection closed, EOF) are transient, so we backoff and + // retry without going through manageFailWait. + // + // UnreleasedInstanceID is retryable-with-a-cap: the broker + // briefly keeps old static-instance state after + // FencedMemberEpoch, so an immediate rejoin can race for + // 1-2 cycles. Beyond 3 attempts it indicates a real + // cross-process InstanceID conflict and we fall through to + // manageFailWait so the user sees it (matches Java's + // handleFatalFailure semantics, with a small race budget). + retryable := err != nil && (g.cl.maybeDeleteStaleCoordinator(g.cfg.group, coordinatorTypeGroup, err) || + isRetryableBrokerErr(err) || isAnyDialErr(err) || + errors.Is(err, kerr.UnreleasedInstanceID) || errors.Is(err, kerr.StaleMemberEpoch)) + + giveUp := false + if retryable && errors.Is(err, kerr.UnreleasedInstanceID) { + unreleasedInstanceRetries++ + if unreleasedInstanceRetries > 3 { + g.cfg.logger.Log(LogLevelError, + "UnreleasedInstanceID after 3 retries - static instance is held by another member; check for duplicate InstanceID config or an unclean peer shutdown", + "group", g.cfg.group, + "instance", g.cfg.instanceID, + ) + giveUp = true + } + } + + if retryable && !giveUp { + // StaleMemberEpoch on initialJoin means the server + // remembers our memberID at a later epoch than the 0 + // we sent. Reset to a fresh member id so the retry + // joins as a new member. + if errors.Is(err, kerr.StaleMemberEpoch) { + g.memberGen.store(newStringUUID(), 0) + } + consecutiveErrors++ + g.cfg.logger.Log(LogLevelInfo, "consumer group initial heartbeat hit retryable error, backing off and retrying", + "group", g.cfg.group, + "err", err, + "consecutive_errors", consecutiveErrors, + "unreleased_instance_retries", unreleasedInstanceRetries, + ) + backoff := g.cfg.retryBackoff(consecutiveErrors) + g.cl.waitmeta(g.ctx, backoff, "waitmeta during 848 retryable error backoff") + after := time.NewTimer(backoff) + select { + case <-g.ctx.Done(): + after.Stop() + return + case <-after.C: + } + continue + } + + // consecutiveTransientRestarts tracks how many times the + // inner heartbeat session has silently restarted due to + // transient errors (EOF, connection refused, etc.) without + // any successful heartbeat in between. Every cfg.retries + // consecutive restarts, we inject a fake fetch error so the + // user knows the broker is unreachable. Reset on success. + var consecutiveTransientRestarts int + for err == nil { + consecutiveErrors = 0 + unreleasedInstanceRetries = 0 + var nowAssigned map[string][]int32 + + // In heartbeating, if we lose or gain partitions, we need to + // exit the old heartbeat and re-enter setupAssignedAndHeartbeat. + // Otherwise, we heartbeat exactly the same as the old. + // + // This results in a few more heartbeats than necessary + // when things are changing, but keeps all the old + // logic that handles all edge conditions. + // + // setupAssignedAndHeartbeat starts prerevoke (for + // lost partitions) and heartbeating concurrently. + // While prerevoking, the heartbeat sends keepalive + // (Topics=nil) so the server does not see partitions + // released before the user commits their offsets in + // any OnPartitionsRevoked callback. + // The prerevoke goroutine clears prerevoking when + // revocation and offset commits are complete, and + // subsequent heartbeats resume sending full requests. + _, err = g.setupAssignedAndHeartbeat(initialHb, func() (time.Duration, error) { + req := g848.mkreq() + prerevoking := g848.prerevoking.Load() + // When the client has no partitions (Topics is empty), + // always send a full request rather than a keepalive. + // This ensures the server sees our actual empty + // assignment state. Without this, a lost response + // containing our assignment can leave the server + // thinking we acknowledged partitions we never + // received: the server marks the assignment as + // delivered, but we never got it. Keepalive + // (Topics=nil) means "no change", which doesn't + // correct the stale server state. Sending Topics=[] + // tells the server "I have nothing", forcing it to + // re-deliver. + topicsMatch := len(req.Topics) > 0 && reflect.DeepEqual(g848.lastSubscribedTopics, req.SubscribedTopicNames) && reflect.DeepEqual(g848.lastTopics, req.Topics) + if prerevoking || topicsMatch { + req.InstanceID = nil + req.RackID = nil + req.RebalanceTimeoutMillis = -1 + req.ServerAssignor = nil + req.SubscribedTopicRegex = nil + req.SubscribedTopicNames = nil + req.Topics = nil + } + resp, err := req.RequestWith(g.ctx, g.cl) + sleep := g.cfg.heartbeatInterval + if err == nil { + err = errCodeMessage(resp.ErrorCode, resp.ErrorMessage) + sleep = time.Duration(resp.HeartbeatIntervalMillis) * time.Millisecond + } + if err != nil { + // Reset last-sent state so the next attempt + // is a full request. If the server processed + // our request but we lost the response, the + // full retry corrects the server's view of + // our current partitions. + g848.lastTopics = nil + g848.lastSubscribedTopics = nil + return sleep, err + } + hbAssigned := g848.handleResp(req, resp) + if hbAssigned != nil { + err = errReassigned848 + nowAssigned = hbAssigned + } + return sleep, err + }) + + switch { + case errors.Is(err, kerr.RebalanceInProgress): + err = nil + + // Retryable broker errors (connection closed, EOF), + // dial errors (connection refused - the broker may be + // restarting), and coordinator errors + // (NOT_COORDINATOR, etc.) are retried in the + // heartbeat loop up to cfg.retries. If the cap is + // hit, the error propagates here. We silently restart + // the session and keep retrying. Every cfg.retries + // consecutive restarts, we inject a fake fetch error + // so the user knows the broker is unreachable. + case isRetryableBrokerErr(err), + isAnyDialErr(err), + g.cl.maybeDeleteStaleCoordinator(g.cfg.group, coordinatorTypeGroup, err): + consecutiveTransientRestarts++ + if int64(consecutiveTransientRestarts) >= g.cfg.retries && int64(consecutiveTransientRestarts)%g.cfg.retries == 0 { + g.c.addFakeReadyForDraining("", 0, &ErrGroupSession{ + Err: fmt.Errorf("consumer group %s heartbeat has been failing for %d consecutive attempts, still retrying: %w", g.cfg.group, consecutiveTransientRestarts, err), + }, "consumer group heartbeat persistently failing") + } + err = nil + + case errors.Is(err, kerr.UnknownMemberID), + errors.Is(err, kerr.StaleMemberEpoch): + // UnknownMemberID: server forgot us. + // StaleMemberEpoch: our epoch drifted (e.g. a + // heartbeat response was lost). Either way, the + // fix is identical: abandon the assignment and + // re-initialJoin with a fresh member id so the + // server hands us back a current epoch. + member, gen := g.memberGen.load() + g.cfg.logger.Log(LogLevelInfo, "consumer group heartbeat error, abandoning assignment and rejoining with new member id", + "group", g.cfg.group, + "member_id", member, + "generation", gen, + "err", err, + ) + g.abandonAssignment(fmt.Sprintf("abandoning assignment after %v", err)) + g.memberGen.store(newStringUUID(), 0) + continue outer + + case errors.Is(err, kerr.FencedMemberEpoch), + errors.Is(err, kerr.GroupMaxSizeReached), + errors.Is(err, kerr.UnsupportedAssignor): + lvl := LogLevelInfo + if errors.Is(err, kerr.GroupMaxSizeReached) { + lvl = LogLevelWarn + } else if errors.Is(err, kerr.UnsupportedAssignor) { + lvl = LogLevelError + } + member, gen := g.memberGen.load() + g.cfg.logger.Log(lvl, "consumer group heartbeat error, abandoning assignment and rejoining", + "group", g.cfg.group, + "member_id", member, + "generation", gen, + "err", err, + ) + g.abandonAssignment(fmt.Sprintf("abandoning assignment after %v", err)) + continue outer + } + + if err == nil { + consecutiveTransientRestarts = 0 + } + if nowAssigned != nil { + member, gen := g.memberGen.load() + g.cfg.logger.Log(LogLevelInfo, "consumer group heartbeat detected an updated assignment; exited heartbeat loop to assign & reentering", + "group", g.cfg.group, + "member_id", member, + "generation", gen, + "now_assigned", nowAssigned, + ) + // handleResp already stored nowAssigned (before memberGen) + // so concurrent readers see a consistent (gen, assignment) + // pair; no additional store needed here. + } + } + + // The errors we have to handle are: + // * UnknownMemberID: abandon partitions, rejoin + // * FencedMemberEpoch: abandon partitions, rejoin + // * UnreleasedInstanceID: fatal error, do not rejoin + // * General error: fatal error, do not rejoin + // + // In the latter two cases, we fall into rejoining anyway + // because it is both non-problematic (we will keep failing + // with the same error) and because it will cause the user + // to repeatedly get error logs. + // + // Note that manageFailWait calls nowAssigned.store(nil), + // meaning our initialJoin should *always* have an empty + // Topics in the request. + consecutiveErrors++ + ctxCanceled := g.manageFailWait(consecutiveErrors, err) + if ctxCanceled { + return + } + } +} + +func (g *groupConsumer) leave848(ctx context.Context) { + memberID := g.memberGen.memberID() + g.cfg.logger.Log(LogLevelInfo, "leaving next-gen group", + "group", g.cfg.group, + "member_id", memberID, + "instance_id", g.cfg.instanceID, + ) + // If we error when leaving, there is not much + // we can do. We may as well just return. + req := kmsg.NewPtrConsumerGroupHeartbeatRequest() + req.Group = g.cfg.group + req.MemberID = memberID + req.MemberEpoch = -1 + if g.cfg.instanceID != nil { + req.MemberEpoch = -2 + } + + resp, err := req.RequestWith(ctx, g.cl) + if err != nil { + g.leaveErr = err + return + } + g.leaveErr = errCodeMessage(resp.ErrorCode, resp.ErrorMessage) +} + +type g848 struct { + g *groupConsumer + + serverAssignor string + + lastSubscribedTopics []string + lastTopics []kmsg.ConsumerGroupHeartbeatRequestTopic + + // unresolvedAssigned holds topic IDs from a heartbeat + // response that could not be mapped to a name via id2t. + // These are included in subsequent heartbeat Topics so the + // server sees them acknowledged. When metadata resolves the + // ID, the topic is moved into newAssigned. + unresolvedAssigned map[topicID][]int32 + + // prerevoking is true while prerevoke is running: the + // assignment has changed and lost partitions are being + // revoked and their offsets committed. While true, the + // heartbeat closure sends keepalive (Topics=nil) so the + // server does not see partitions as released before offsets + // are committed. Cleared by the prerevoke goroutine. + prerevoking atomic.Bool +} + +// v1+ requires the end user to generate their own MemberID, with the +// recommendation being v4 uuid base64 encoded so it can be put in URLs. We +// roughly do that (no version nor variant bits). crypto/rand does not fail +// and, in future Go versions, will panic on internal errors. +func newStringUUID() string { + var uuid [16]byte + io.ReadFull(rand.Reader, uuid[:]) // even more random than adding version & variant bits is having full randomness + return base64.URLEncoding.EncodeToString(uuid[:]) +} + +func (g *g848) initialJoin() (time.Duration, error) { + // handleResp publishes nowAssigned BEFORE memberGen so that any + // concurrent commit retry observing the new gen also sees the new + // assignment (see comment in handleResp). Verify the invariant up + // front: a clean initialJoin must enter with no assignment so that + // the publish order remains the source of truth. + if g.g.nowAssigned.read() != nil { + panic("nowAssigned is not nil in our initial join, invalid invariant!") + } + g.g.memberGen.storeGeneration(0) + g.lastSubscribedTopics = nil + g.lastTopics = nil + g.prerevoking.Store(false) + req := g.mkreq() + resp, err := req.RequestWith(g.g.ctx, g.g.cl) + if err == nil { + err = errCodeMessage(resp.ErrorCode, resp.ErrorMessage) + } + if err != nil { + return 0, err + } + nowAssigned := g.handleResp(req, resp) + member, gen := g.g.memberGen.load() + g.g.cfg.logger.Log(LogLevelInfo, "consumer group initial heartbeat received assignment", + "group", g.g.cfg.group, + "member_id", member, + "generation", gen, + "now_assigned", nowAssigned, + ) + + return time.Duration(resp.HeartbeatIntervalMillis) * time.Millisecond, nil +} + +func (g *g848) handleResp(req *kmsg.ConsumerGroupHeartbeatRequest, resp *kmsg.ConsumerGroupHeartbeatResponse) map[string][]int32 { + id2t := g.g.cl.id2tMap() + newAssigned := make(map[string][]int32) + + // Only update the last-sent fields when Topics was actually + // included in the request. When the request was a keepalive + // (Topics=nil), we preserve the previous values so the next + // comparison still matches and produces another keepalive. + // Without this guard, storing nil after a keepalive causes + // DeepEqual(nil, []) to fail on the next heartbeat, re-sending + // Topics=[] - creating an alternating full/keepalive pattern. + // The server clears pendingRevocations when Topics does not + // contain a pending partition, so the alternating stale full + // heartbeats can cause premature revocation clearing and dual + // assignment. + if req.Topics != nil { + g.lastSubscribedTopics = req.SubscribedTopicNames + g.lastTopics = req.Topics + } + + if resp.Assignment != nil { + // Fresh assignment from server - replace unresolved state. + g.unresolvedAssigned = nil + for _, t := range resp.Assignment.Topics { + name := id2t[t.TopicID] + if name == "" { + if g.unresolvedAssigned == nil { + g.unresolvedAssigned = make(map[topicID][]int32) + } + g.unresolvedAssigned[topicID(t.TopicID)] = slices.Clone(t.Partitions) + continue + } + slices.Sort(t.Partitions) + newAssigned[name] = t.Partitions + } + } + + // Try to resolve previously-unresolved topic IDs now that + // metadata may have refreshed since the last heartbeat. + // We don't proactively hook into the metadata update to + // resolve immediately - waiting for the next heartbeat is + // simpler and only costs one heartbeat interval (~5s). + for id, ps := range g.unresolvedAssigned { + if name := id2t[[16]byte(id)]; name != "" { + slices.Sort(ps) + newAssigned[name] = ps + delete(g.unresolvedAssigned, id) + } + } + if len(g.unresolvedAssigned) > 0 { + g.g.cl.triggerUpdateMetadataNow("consumer group heartbeat has unresolved topic IDs in assignment") + } + + // storeMember publishes the new memberGen. We defer it so it runs AFTER + // any nowAssigned.store below. Atomic stores in Go are sequentially + // consistent, so concurrent readers (e.g. the commit() STALE retry + // filter at consumer_group.go) cannot observe new memberGen with + // stale nowAssigned: seeing the new gen happens-after seeing the new + // assignment. This closes the race that would otherwise leak revoked + // partitions into a retried commit. + storeMember := func() { + if resp.MemberID != nil { + g.g.memberGen.store(*resp.MemberID, resp.MemberEpoch) + g.g.cl.cfg.logger.Log(LogLevelDebug, "storing member and epoch", "group", g.g.cfg.group, "member", *resp.MemberID, "epoch", resp.MemberEpoch) + } else { + g.g.memberGen.storeGeneration(resp.MemberEpoch) + g.g.cl.cfg.logger.Log(LogLevelDebug, "storing epoch", "group", g.g.cfg.group, "epoch", resp.MemberEpoch) + } + } + defer storeMember() + + // Only return nil (no change) when the response had no + // assignment at all (keepalive) or when all topics in the + // assignment are still unresolved. When the server explicitly + // sends an empty assignment (resp.Assignment != nil with no + // topics), fall through to the comparison so the client + // detects it as "revoke everything". + // + // The unresolved check handles the case where the server + // assigned topics whose IDs the client can't map to names + // yet (e.g. newly created topic, metadata not refreshed). + // Those went into unresolvedAssigned rather than + // newAssigned. Without this guard, we'd fall through with + // newAssigned={} and tell the client to revoke everything, + // when really the server did assign partitions - we just + // need to wait for metadata resolution. + if len(newAssigned) == 0 && (resp.Assignment == nil || len(g.unresolvedAssigned) > 0) { + return nil + } + + // Merge with current assignment: newAssigned only has topics + // from the response or freshly resolved; fill in any existing + // topics the server didn't mention (resp.Assignment == nil). + current := g.g.nowAssigned.read() + if resp.Assignment == nil { + for t, ps := range current { + if _, ok := newAssigned[t]; !ok { + newAssigned[t] = ps + } + } + } + + if !mapi32sDeepEq(current, newAssigned) { + // Store BEFORE the deferred storeMember runs, so an observer that + // sees new memberGen via memberGen.load() is guaranteed to also + // see the matching nowAssigned via nowAssigned.read(). + g.g.nowAssigned.store(newAssigned) + return newAssigned + } + return nil +} + +func (g *g848) mkreq() *kmsg.ConsumerGroupHeartbeatRequest { + req := kmsg.NewPtrConsumerGroupHeartbeatRequest() + req.Group = g.g.cfg.group + req.MemberID, req.MemberEpoch = g.g.memberGen.load() + + // Most fields in the request can be null if the field is equal to the + // last time we sent the request. The first time we write, we include + // all information. We always return all information here; the caller + // of mkreq may strip fields as needed. + // + // Our initial set of subscribed topics is specified in our config. + // For non-regex consuming, topics are directly created into g.tps. + // As well, g.tps is added to or purged from in AddConsumeTopics or + // PurgeConsumeTopics. We can always use g.tps for direct topics the + // user wants to consume. + // + // For regex topics, they cannot add or remove after client creation. + // We just use the initial config field. + + req.InstanceID = g.g.cfg.instanceID + if g.g.cfg.rack != "" { + req.RackID = &g.g.cfg.rack + } + req.RebalanceTimeoutMillis = int32(g.g.cfg.rebalanceTimeout.Milliseconds()) + req.ServerAssignor = &g.serverAssignor + + tps := g.g.tps.load() + if g.g.cl.cfg.regex && len(g.g.cl.cfg.excludeTopics) > 0 { + // KIP-848's SubscribedTopicRegex is include-only with no exclude + // counterpart. When excludes are configured, fall back to sending + // the already-resolved topic names from g.tps (which + // filterMetadataAllTopics populates with excludes applied). + // New topics are picked up on the next metadata refresh. + subscribedTopics := make([]string, 0, len(tps)) + for t := range tps { + subscribedTopics = append(subscribedTopics, t) + } + slices.Sort(subscribedTopics) + req.SubscribedTopicNames = subscribedTopics + } else if g.g.cl.cfg.regex { + topics := g.g.cl.cfg.topics + patterns := make([]string, 0, len(topics)) + for topic := range topics { + patterns = append(patterns, "(?:"+topic+")") + } + slices.Sort(patterns) + pattern := strings.Join(patterns, "|") + req.SubscribedTopicRegex = &pattern + } else { + // SubscribedTopics must always exist when epoch == 0. + // We specifically 'make' the slice to ensure it is non-nil. + subscribedTopics := make([]string, 0, len(tps)) + for t := range tps { + subscribedTopics = append(subscribedTopics, t) + } + slices.Sort(subscribedTopics) + req.SubscribedTopicNames = subscribedTopics + } + // Build Topics from our current assignment. The heartbeat closure + // uses prerevoking to strip this to nil during prerevoke, + // preventing the server from seeing released partitions before + // offsets are committed. + // + // A topic may be in nowAssigned but absent from tps if the user + // just called PurgeFetchTopics: purge removes from tps, but + // nowAssigned reflects the server's view and is only updated when + // the server acknowledges the revoke in a future heartbeat + // response. We skip such topics here; SubscribedTopicNames (built + // from tps above) signals the unsubscribe and the server revokes + // on the next response. + nowAssigned := g.g.nowAssigned.read() + req.Topics = []kmsg.ConsumerGroupHeartbeatRequestTopic{} // ALWAYS initialize: len 0 is significantly different than nil (nil means same as last time) + for t, ps := range nowAssigned { + tp, ok := tps[t] + if !ok { + continue + } + rt := kmsg.NewConsumerGroupHeartbeatRequestTopic() + rt.Partitions = slices.Clone(ps) + rt.TopicID = tp.load().id + req.Topics = append(req.Topics, rt) + } + // Include unresolved topic IDs so the server sees them + // acknowledged. This also makes topicsMatch false (since + // lastTopics won't contain these), forcing a full request + // whose isFullRequest triggers a re-send of the assignment. + for id, ps := range g.unresolvedAssigned { + rt := kmsg.NewConsumerGroupHeartbeatRequestTopic() + rt.TopicID = [16]byte(id) + rt.Partitions = slices.Clone(ps) + req.Topics = append(req.Topics, rt) + } + + // Canonicalize ordering by TopicID so the heartbeat closure's + // DeepEqual against lastTopics is stable across map-iteration + // orderings, avoiding unnecessary full heartbeats. + slices.SortFunc(req.Topics, func(a, b kmsg.ConsumerGroupHeartbeatRequestTopic) int { + return bytes.Compare(a.TopicID[:], b.TopicID[:]) + }) + + return req +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_share.go b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_share.go new file mode 100644 index 00000000000..f97bf9797a6 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/consumer_share.go @@ -0,0 +1,2994 @@ +package kgo + +import ( + "cmp" + "context" + "errors" + "fmt" + "maps" + "slices" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +type ( + shareConsumer struct { + c *consumer + cl *Client + cfg *cfg + + fm fetchManager + + // Our subscribed topic set and their partition metadata, + // updated atomically after metadata responses. The keys are + // the topic names we send as SubscribedTopicNames in the + // heartbeat; the values hold per-partition state including + // the shareCursor and its current source. What we ACTUALLY + // consume is whatever the broker assigns us out of this set, + // tracked in nowAssigned. + tps *topicsPartitions + + reSeen map[string]bool // topics we evaluated against regex, and whether we want them + + memberGen groupMemberGen + nowAssigned amtps + + // topic IDs from heartbeat responses that we cannot yet map to + // names. Owned by the manage goroutine. + unresolvedAssigns map[topicID][]int32 + + lastSentSubscribedTopics []string + lastSentRack bool + + // ackMu/ackC/pendingAcks: used by FlushAcks to wait for + // all in-flight acks to drain. pendingAcks is an + // atomic counter (free reads/writes on the Record.Ack + // hot path); ackMu+ackC are only taken when waking + // FlushAcks waiters (broadcast happens under mu only + // when pendingAcks reaches 0, to keep the cond + // broadcast and the wait-loop's Load consistent). + ackMu xsync.Mutex + ackC *sync.Cond + pendingAcks atomic.Int64 + + // callbackRing serializes shareAckCallback invocations + // via the same ring + spawn-on-empty pattern that + // producer.go uses for batchPromises. Each entry carries + // a pendingAcks count that is subtracted AFTER the user + // callback returns, so FlushAcks blocks until callbacks + // have completed. + callbackRing ring[shareCallbackEntry] + + // Per-record ack state for every record returned by the + // last poll. On the next poll, any state still pending + // (or AckRenew) is auto-accepted. Holds state pointers, + // not *Record, so the auto-accept pass is safe against + // Record.Recycle + pool reuse aliasing the *Record memory. + // Access serialized via sc.c.mu. + lastPolled []*shareAckState + + ////////////// + // mu block // + ////////////// + + mu xsync.Mutex + cond *sync.Cond + dying bool // single-shot leave guard + incWorker gate + workers int // active goroutines (manage + source loops) + left chan struct{} + leaveErr error + + manageRunning atomic.Bool // CAS prevents double-spawn + } + + shareCursor struct { + topic string + topicID [16]byte + partition int32 + + // source is atomic to support cursors moving between + // sources concurrent with user acking. + source atomic.Pointer[source] + + cursorsIdx int + + // assigned is true when the cursor's partition is currently + // part of our share-group assignment (via assignPartitions), + // false otherwise (default state or via revoke). + // + // Unlike classic source's cursor.useState, assigned is NOT + // flipped during request build. The fetch loop is single- + // threaded per source and the sem mechanism prevents + // concurrent fetches, so the request-time flip-and-restore + // dance from classic is unnecessary here and orchestrating + // a similar flow actually makes the code worse. + assigned atomic.Bool + + ackMu xsync.Mutex + pendingAcks []*shareAckState // user acks (r.Ack, finalizePreviousPoll, batchAckRecords) + pendingGaps []shareAckRange // internal acks (gap acks, release-undeliverable) + closed bool // set to reject user-side acks arriving after shutdown + } + + // AckStatus defines how the broker should handle an acquired share group + // record: accept, release for redelivery, reject, or renew the lock. + // The zero value is not valid; MarkAcks and Record.Ack silently + // ignore it. + AckStatus int8 + + // ShareAckResult is a per-partition result from a share group acknowledge; + // Err is nil on success. + ShareAckResult struct { + Topic string + Partition int32 + Err error + } + + // ShareAckResults are the per-partition outcomes from one share-group + // acknowledge, as reported by the broker. Use [ShareAckResults.Ok] for + // an all-or-nothing check, or iterate the slice directly for + // per-partition handling. A nil [ShareAckResult.Err] means the broker + // successfully processed that partition's acks; a non-nil Err means the + // broker rejected them, the client could not deliver them, or the + // broker omitted the partition from its response. + ShareAckResults []ShareAckResult + + // shareMove records a partition that should be moved to a new leader, + // collected from CurrentLeader hints in ShareFetch responses. + shareMove struct { + topicID [16]byte + partition int32 + leaderID int32 + } + + // shareAckRange is a contiguous offset range with a fixed ack + // type. Used for: + // - wire format: merged ranges sent in ShareFetch/ShareAcknowledge + // - internal acks: gap acks and release-undeliverable (cursor's + // pendingGaps queue); these have no status pointer because + // the user never saw the records + // + // source and sessionEpoch are used by the staleness filter to + // drop acks the broker would reject (session reset or cursor + // migration). + shareAckRange struct { + firstOffset int64 + lastOffset int64 + source *source + sessionEpoch int32 + ackType int8 // uniform type for the entire range + } + + // shareAckState is per-record ack state (24 bytes), used as + // the ack handle directly: pendingAcks, lastPolled, and the + // cursorsAcks accumulator all hold *shareAckState. offset and + // slab are populated only for ACQUIRED records (set at slab + // construction); non-acquired records leave them zero and must + // never reach the ack path. + // + // A single record can appear multiple times in pendingAcks + // (e.g., Ack(AckRenew) then Ack(AckAccept) both append the + // same pointer; the terminal CAS overwrites status, but the + // original entry remains). buildAckRanges reads status.Load() + // and dedupes by offset so the broker sees one wire range per + // offset; duplicates would be rejected with INVALID_RECORD_STATE. + // The sc.pendingAcks counter still increments once per appended + // entry; subtraction at callback time uses the entry count. + shareAckState struct { + status atomic.Int32 // CAS target for ack transitions + deliveryCount int32 // broker's delivery count for this record (>= 1) + offset int64 // record's Kafka offset + slab *shareAckSlab // back-ref: gives ackSource, sessionEpoch, cursor + } + + // shareAckSlab holds the per-record shareAckState array for one + // ShareFetch batch. One slab per batch is attached to every + // record's Context under shareAckKey. Records within a batch + // are contiguous in memory (one []Record allocation), so + // pointer arithmetic from records0 gives the slab index for + // any *Record in the batch (see shareAckFromCtx). + // + // ackSource, sessionEpoch are the source identity and session + // epoch at decode time; the staleness filter compares these + // against the source actually sending the ack, dropping acks + // the broker would reject on cursor migration or session reset. + // cursor routes acks to the right partition. acqLockDeadlineNanos + // is the broker's acquisition-lock deadline, exposed via + // Record.AcquisitionDeadline. + shareAckSlab struct { + states []shareAckState + records0 *Record // first record in the batch's backing array + ackSource *source + cursor *shareCursor + acqLockDeadlineNanos int64 + sessionEpoch int32 + } + + // shareCallbackEntry is pushed onto the callbackRing. The drainer + // invokes the user callback, then subtracts nAcks from + // sc.pendingAcks. This ensures FlushAcks blocks until all + // callbacks have completed. + shareCallbackEntry struct { + results ShareAckResults + nAcks int64 + } + + // shareFetchResult is returned by handleShareReqResp for + // shareFetch to act on at the request level. + shareFetchResult struct { + fetch Fetch + moves []shareMove + moveBrokers []BrokerMetadata // NodeEndpoints from response; used by applyMoves to add unknown brokers + ackResults ShareAckResults + ackRequeued int64 + discardErr error // non-nil: response thrown away, error all pending acks + allErrsStripped bool // every partition had errors, nothing buffered + } + + // cursorAckDrain is a cursor + the entries/gaps drained from it. + cursorAckDrain struct { + cursor *shareCursor + entries []*shareAckState // user acks + gaps []shareAckRange // internal acks (gap/release) + } + + // cursorsAcks groups user ack entries by cursor so a flush can + // take each cursor's mutex once. + cursorsAcks map[*shareCursor][]*shareAckState +) + +// isShareAckRetryable returns true only for errors where retrying the ack +// on the same broker could succeed. Narrower than kerr.IsRetriable: +// leader-change errors (NotLeaderForPartition, FencedLeaderEpoch) are +// excluded because the acquisition lock was released at leader change; +// unknown-topic errors (UnknownTopicOrPartition, UnknownTopicID) are +// excluded because the records will be re-delivered via broker-side +// acquisition-lock timeout. +func isShareAckRetryable(err error) bool { + if !kerr.IsRetriable(err) { + return false + } + switch { + case errors.Is(err, kerr.NotLeaderForPartition), + errors.Is(err, kerr.FencedLeaderEpoch), + errors.Is(err, kerr.UnknownTopicOrPartition), + errors.Is(err, kerr.UnknownTopicID): + return false + } + return true +} + +// add accumulates state under its cursor (state.slab.cursor). +func (m *cursorsAcks) add(state *shareAckState) { + cursor := state.slab.cursor + if *m == nil { + *m = make(cursorsAcks) + } + (*m)[cursor] = append((*m)[cursor], state) +} + +const ( + // AckAccept marks a record as successfully processed. The broker + // advances the share group's cursor past this record. + AckAccept AckStatus = 1 + + // AckRelease releases a record back to the broker for redelivery + // to another consumer. The delivery count is incremented. + AckRelease AckStatus = 2 + + // AckReject marks a record as permanently unprocessable. The broker + // archives the record and does not redeliver it. + AckReject AckStatus = 3 + + // AckRenew extends the acquisition lock on a record without completing + // it (KIP-1222, requires Kafka 4.2+). Renews do not persist across + // polls: any renewed record that is not explicitly acked with a + // terminal status before the next [Client.PollRecords] is + // auto-accepted. On client close, any records still in AckRenew + // state are automatically converted to AckRelease so the broker + // can redeliver them immediately. + AckRenew AckStatus = 4 +) + +func (s AckStatus) String() string { + switch s { + case AckAccept: + return "accept" + case AckRelease: + return "release" + case AckReject: + return "reject" + case AckRenew: + return "renew" + default: + return fmt.Sprintf("AckStatus(%d)", s) + } +} + +// Error returns the first non-nil error in the results, or nil. +func (rs ShareAckResults) Error() error { + for _, r := range rs { + if r.Err != nil { + return r.Err + } + } + return nil +} + +// Ok returns true if all results succeeded. +func (rs ShareAckResults) Ok() bool { + return rs.Error() == nil +} + +///////////////////////// +// INIT / LEAVE / POLL // +///////////////////////// + +func (c *consumer) initShare() { + ctx, cancel := context.WithCancel(c.cl.ctx) + sc := &shareConsumer{ + c: c, + cl: c.cl, + cfg: &c.cl.cfg, + + fm: newFetchManager(ctx, cancel, &c.pollActive, c.pollWake, c.cl.cfg.maxConcurrentFetches), + + reSeen: make(map[string]bool), + tps: newTopicsPartitions(), + + left: make(chan struct{}), + } + sc.ackC = sync.NewCond(&sc.ackMu) + sc.cond = sync.NewCond(&sc.mu) + c.s = sc + + if len(sc.cfg.topics) > 0 && !sc.cfg.regex { + topics := make([]string, 0, len(sc.cfg.topics)) + for topic := range sc.cfg.topics { + topics = append(topics, topic) + } + sc.tps.storeTopics(topics) + } +} + +// poll is the share consumer's implementation of PollRecords, with the added +// behavior of auto-accepting records from the prior poll that were not +// explicitly ack'd or that were only renewed (AckRenew does not persist +// across polls). +func (sc *shareConsumer) poll(ctx context.Context, maxPollRecords int) Fetches { + // Guard race conditions in case people use the API incorrectly + // (concurrent leave with poll). + // + // Our lock pattern is slightly different than Client.PollRecords + // because of, well, other races (finalizePreviousPoll, etc). + sc.c.mu.Lock() + sc.finalizePreviousPoll() + + var fetches Fetches + fill := func() { + paused := sc.c.loadPaused() + + sc.c.sourcesReadyMu.Lock() + if maxPollRecords < 0 { + for _, s := range sc.c.sourcesReadyForDraining { + fetches = append(fetches, s.share.takeBuffered(paused)) + } + sc.c.sourcesReadyForDraining = nil + } else { + remaining := maxPollRecords + for len(sc.c.sourcesReadyForDraining) > 0 && remaining > 0 { + s := sc.c.sourcesReadyForDraining[0] + f, taken, drained := s.share.takeNBuffered(paused, remaining) + if drained { + sc.c.sourcesReadyForDraining = sc.c.sourcesReadyForDraining[1:] + } + remaining -= taken + fetches = append(fetches, f) + } + } + fetches = append(fetches, sc.c.fakeReadyForDraining...) + sc.c.fakeReadyForDraining = nil + sc.c.sourcesReadyMu.Unlock() + + if len(fetches) > 0 { + sc.trackLastPolled(fetches) + } + } + + fill() + sc.c.mu.Unlock() + if len(fetches) > 0 || ctx == nil { + return fetches + } + + done := make(chan struct{}) + quit := false + go func() { + sc.c.sourcesReadyMu.Lock() + defer sc.c.sourcesReadyMu.Unlock() + defer close(done) + for !quit && len(sc.c.sourcesReadyForDraining) == 0 && len(sc.c.fakeReadyForDraining) == 0 { + sc.c.sourcesReadyCond.Wait() + } + }() + + exit := func() { + sc.c.sourcesReadyMu.Lock() + quit = true + sc.c.sourcesReadyMu.Unlock() + sc.c.sourcesReadyCond.Broadcast() + } + + select { + case <-sc.fm.ctx.Done(): + exit() + return NewErrFetch(ErrClientClosed) + case <-ctx.Done(): + exit() + return NewErrFetch(ctx.Err()) + case <-done: + sc.c.mu.Lock() + fill() + sc.c.mu.Unlock() + } + + return fetches +} + +// finalizePreviousPoll auto-accepts every record from the previous +// poll whose status is still pending (or AckRenew -- renews do not +// persist across polls). +func (sc *shareConsumer) finalizePreviousPoll() { + if len(sc.lastPolled) == 0 { + return + } + batchAckStates(sc, sc.lastPolled, AckAccept, false) + clear(sc.lastPolled) + sc.lastPolled = sc.lastPolled[:0] +} + +// trackLastPolled snapshots state pointers (not *Record) so the +// next-poll auto-accept is safe against Record.Recycle + pool reuse +// aliasing the *Record memory to a later batch. +func (sc *shareConsumer) trackLastPolled(fetches Fetches) { + sc.lastPolled = slices.Grow(sc.lastPolled[:0], fetches.NumRecords())[:0] + fetches.EachRecord(func(r *Record) { + if st := shareAckFromCtx(r); st != nil { + sc.lastPolled = append(sc.lastPolled, st) + } + }) +} + +// MarkAcks sets the acknowledgement status for share group records. +// If no records are provided, all records from the last poll that have +// not already been marked are marked with the given status. +// If specific records are provided, the status is applied to each +// using the same per-record CAS rules as [Record.Ack]: terminal +// statuses override an existing [AckRenew] but not another terminal, +// and a renew is rejected if the record is already in any non-zero +// state. +func (cl *Client) MarkAcks(status AckStatus, rs ...*Record) { + if status < AckAccept || status > AckRenew { + return // unknown AckStatus value; silently ignore + } + sc := cl.consumer.s + if sc == nil { + return + } + if len(rs) > 0 { + batchAckRecords(sc, rs, status, false) + return + } + // Fill in the gaps: only touch records that are still PENDING. + // This preserves any explicit ack the caller already set. + // + // c.mu serializes lastPolled access against PollRecords and + // sc.leave's reject pass; without it, a -race run with a user + // who wrongly calls MarkAcks concurrently with PollRecords + // would flag a data race on the slice header. + sc.c.mu.Lock() + defer sc.c.mu.Unlock() + batchAckStates(sc, sc.lastPolled, status, true) +} + +// FlushAcks synchronously flushes all pending share group acks. It +// blocks until the pending-ack counter reaches zero (every queued ack +// has been sent to its broker and the corresponding [ShareAckCallback] +// has been invoked), or the ctx is canceled. +// +// Do not call FlushAcks from inside a [ShareAckCallback]. +func (cl *Client) FlushAcks(ctx context.Context) error { + sc := cl.consumer.s + if sc == nil { + return nil + } + + // Signal all sources to flush immediately. + cl.allSources(func(s *source) { + s.signalShareAckFlush() + }) + + // Wait for all pending acks to drain, or context cancellation. + quit := false + done := make(chan struct{}) + go func() { + sc.ackMu.Lock() + defer sc.ackMu.Unlock() + defer close(done) + for !quit && sc.pendingAcks.Load() > 0 { + sc.ackC.Wait() + } + }() + + select { + case <-done: + return nil + case <-ctx.Done(): + sc.ackMu.Lock() + quit = true + sc.ackMu.Unlock() + sc.ackC.Broadcast() + return ctx.Err() + } +} + +// leave performs the graceful share-group shutdown: +// +// 1. Set dying (single-shot guard + blocks new goroutines). +// 2. Cancel fm.ctx so manage + source loops exit. +// 3. Wait for all workers (manage + source loops) to reach 0. +// 4. Release lastPolled and buffered records, drain fetchManager slots. +// 5. Per-source FINAL_EPOCH ShareFetch with remaining acks. +// 6. Leave heartbeat (MemberEpoch=-1). +// +// Called from a fresh goroutine by LeaveGroupContext. The caller +// selects on sc.left or ctx.Done. +func (sc *shareConsumer) leave(ctx context.Context) { + sc.mu.Lock() + if sc.dying { + sc.mu.Unlock() + return + } + sc.dying = true + sc.mu.Unlock() + defer close(sc.left) + + // Cancel fm.ctx: manage and source loops all select on + // fm.ctx.Done and exit promptly. + sc.fm.cancel() + + // Wait for every worker (manage + source loops) to exit. + // After this we have exclusive access to all share state. + sc.mu.Lock() + for sc.workers > 0 { + sc.cond.Wait() + } + sc.mu.Unlock() + + // A user PollRecords could still be in flight: even after + // fm.cancel, the non-deterministic select may wake on the + // sourcesReady channel instead of fm.ctx.Done. c.mu blocks + // poll entirely. Under it we clear the source queue (so any + // racing poll sees nothing -- the records are still on + // s.share.buffered; closeShareSession finds and releases them) + // and release unacked lastPolled records (AckRelease so another + // consumer gets them, not auto-accept). + sc.c.mu.Lock() + sc.c.sourcesReadyMu.Lock() + sc.c.sourcesReadyForDraining = nil + sc.c.sourcesReadyMu.Unlock() + if len(sc.lastPolled) > 0 { + batchAckStates(sc, sc.lastPolled, AckRelease, true) + sc.lastPolled = nil + } + sc.c.mu.Unlock() + + // Per-source: release buffered records, drain fetchManager + // slots, then close the share session (FINAL_EPOCH ShareAcknowledge + // with all remaining acks piggybacked). + var wg sync.WaitGroup + sc.cl.allSources(func(s *source) { + wg.Add(1) + go func() { + defer wg.Done() + s.closeShareSession(ctx) + }() + }) + wg.Wait() + + // Leave heartbeat. + memberID, _ := sc.memberGen.load() + if memberID == "" { + return + } + sc.cfg.logger.Log(LogLevelInfo, "leaving share group", + "group", sc.cfg.shareGroup, + "member_id", memberID, + ) + req := kmsg.NewPtrShareGroupHeartbeatRequest() + req.GroupID = sc.cfg.shareGroup + req.MemberID = memberID + req.MemberEpoch = -1 + resp, err := req.RequestWith(ctx, sc.cl) + if err != nil { + sc.leaveErr = err + return + } + sc.leaveErr = errCodeMessage(resp.ErrorCode, resp.ErrorMessage) +} + +// closeShareSession releases any buffered records on this source, +// drains fetchManager slots, then sends a FINAL_EPOCH ShareAcknowledge +// (-1) with all remaining cursor.pendingAcks piggybacked. Called +// from sc.leave after all workers have exited. +func (s *source) closeShareSession(ctx context.Context) { + sc := s.share.sc + // Release buffered-but-never-polled records. Their acks land + // on cursor.pendingAcks and get piggybacked on the FINAL_EPOCH + // request below. + b := s.share.buffered + s.share.buffered = shareBufferedFetch{} + if b.doneFetch != nil { // source had a buffered fetch + b.doneFetch <- false + for ti := range b.fetch.Topics { + t := &b.fetch.Topics[ti] + for pi := range t.Partitions { + batchAckRecords(sc, t.Partitions[pi].Records, AckRelease, true) + } + } + } + + // Drain all cursor acks. drainAcks(true) sets the closed flag + // atomically with the drain so post-leave Record.Ack gets a + // callback instead of being silently lost. + // + // Renew entries are converted to release: we are shutting down, + // so extending the lock is pointless. Explicitly releasing + // ensures the broker can redeliver immediately rather than + // waiting for the acquisition lock to expire. + s.share.mu.Lock() + epoch := s.share.sessionEpoch + drains := s.drainAllShareAcks(true) + s.share.mu.Unlock() + var nAcks int64 + for _, d := range drains { + for _, st := range d.entries { + st.status.CompareAndSwap(int32(AckRenew), int32(AckRelease)) + } + nAcks += int64(len(d.entries)) + } + + // Filter stale-epoch batches for consistency with shareAck. + // The broker would reject them per-partition + // anyway (their original session is gone), and we notify the + // user via the shareAckCallback for consistency. + nLive, nStaleAcks, staleResults := filterStaleEntries(s, epoch, drains) + sc.enqueueCallback(staleResults, nStaleAcks) + if nStaleAcks > 0 { + sc.cfg.logger.Log(LogLevelInfo, "share session close: dropped stale-epoch acks", + "broker", logID(s.nodeID), + "stale_ack_records", nStaleAcks, + ) + } + + // If we never established a session and there are no live + // acks, there is nothing to tell the broker. Skip. + if epoch == 0 && nLive == 0 { + return + } + + memberID, _ := sc.memberGen.load() + req := kmsg.NewPtrShareAcknowledgeRequest() + req.GroupID = &sc.cfg.shareGroup + req.MemberID = &memberID + req.ShareSessionEpoch = -1 // KIP-932: close the session + + // drainIdx tracks the (topicID, partition) keys that actually + // appeared on the wire so the broker-omitted scan in the + // response handler doesn't flag partitions we never sent (e.g. + // a drain whose entries all have status=0 produces an empty + // range list from buildAckRanges and is skipped below). + var drainIdx map[tidp]int + if nLive > 0 { + drainIdx = make(map[tidp]int, len(drains)) + topicIdx := make(map[[16]byte]int) + for i, d := range drains { + if len(d.entries) == 0 && len(d.gaps) == 0 { + continue + } + ranges, _ := buildAckRanges(d.entries, d.gaps) + if len(ranges) == 0 { + continue + } + drainIdx[tidp{d.cursor.topicID, d.cursor.partition}] = i + tidx, ok := topicIdx[d.cursor.topicID] + if !ok { + tidx = len(req.Topics) + topicIdx[d.cursor.topicID] = tidx + req.Topics = append(req.Topics, kmsg.ShareAcknowledgeRequestTopic{ + TopicID: d.cursor.topicID, + }) + } + rt := &req.Topics[tidx] + rp := kmsg.ShareAcknowledgeRequestTopicPartition{ + Partition: d.cursor.partition, + } + for _, r := range ranges { + rp.AcknowledgementBatches = append(rp.AcknowledgementBatches, + kmsg.ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch{ + FirstOffset: r.firstOffset, + LastOffset: r.lastOffset, + AcknowledgeTypes: ackTypes(r.ackType), + }) + } + rt.Partitions = append(rt.Partitions, rp) + } + } + + sc.cfg.logger.Log(LogLevelDebug, "closing share session on broker", + "broker", logID(s.nodeID), + "group", sc.cfg.shareGroup, + "piggybacked_ack_records", nLive, + ) + + // nil ctx on brokerOrErr is intentional: this matches the + // classic killSessionOnClose idiom (source.go's + // killSessionOnClose). We want to find the broker without + // getting stuck waiting on a broker-lookup ctx; the actual + // request below uses the caller-supplied ctx for its timeout. + br, err := s.cl.brokerOrErr(nil, s.nodeID, errUnknownBroker) + if err != nil { + sc.cfg.logger.Log(LogLevelDebug, "share session close: unable to find broker", + "broker", logID(s.nodeID), + "err", err, + ) + sc.enqueueAckErrors(drains, err, nAcks-nStaleAcks) // piggybacked acks lost, decrement pendingAcks + return + } + done := make(chan struct{}) + var respErr error + var resp *kmsg.ShareAcknowledgeResponse + br.do(ctx, req, func(k kmsg.Response, e error) { + respErr = e + if k != nil { + resp, _ = k.(*kmsg.ShareAcknowledgeResponse) + } + close(done) + }) + select { + case <-done: + case <-ctx.Done(): + sc.enqueueAckErrors(drains, ctx.Err(), nAcks-nStaleAcks) // ctx expired before response; acks lost, decrement pendingAcks + s.resetShareSession() + return + } + + // Per-partition callback dispatch. + if respErr != nil { + sc.cfg.logger.Log(LogLevelDebug, "share session close request failed", + "broker", logID(s.nodeID), + "err", respErr, + ) + // Network failure: report all live drains as failed. + // enqueueAckErrors skips empty entries/gaps internally, + // so stale-only drains (already reported via staleResults) + // are not double-fired. + sc.enqueueAckErrors(drains, respErr, nAcks-nStaleAcks) + } else if resp != nil { + if topErr := kerr.ErrorForCode(resp.ErrorCode); topErr != nil { + sc.cfg.logger.Log(LogLevelInfo, "share session close: top-level error from broker", + "broker", logID(s.nodeID), + "err", topErr, + ) + sc.enqueueAckErrors(drains, topErr, nAcks-nStaleAcks) + s.resetShareSession() + return + } + seen := make(map[tidp]struct{}, len(drainIdx)) + var results ShareAckResults + var respErrCount int + for _, rt := range resp.Topics { + for _, rp := range rt.Partitions { + key := tidp{rt.TopicID, rp.Partition} + if _, dup := seen[key]; dup { + sc.cfg.logger.Log(LogLevelWarn, "broker returned duplicate partition in share session close response, ignoring duplicate", + "broker", logID(s.nodeID), + "partition", rp.Partition, + ) + continue + } + seen[key] = struct{}{} + i, ok := drainIdx[key] + if !ok { + continue // partition we did not ack + } + drained := drains[i] + var err error + if rp.ErrorCode != 0 { + err = kerr.ErrorForCode(rp.ErrorCode) + respErrCount++ + } + results = append(results, ShareAckResult{ + Topic: drained.cursor.topic, + Partition: rp.Partition, + Err: err, + }) + } + } + if respErrCount > 0 { + sc.cfg.logger.Log(LogLevelInfo, "share session close: per-partition errors from broker", + "broker", logID(s.nodeID), + "partition_errors", respErrCount, + ) + } + for tp, i := range drainIdx { + if _, processed := seen[tp]; !processed { + drained := drains[i] + sc.cfg.logger.Log(LogLevelWarn, "broker omitted partition from share session close response", + "broker", logID(s.nodeID), + "topic", drained.cursor.topic, + "partition", tp.p, + ) + results = append(results, ShareAckResult{ + Topic: drained.cursor.topic, + Partition: tp.p, + Err: errBrokerOmittedAckPartition, + }) + } + } + sc.enqueueCallback(results, nAcks-nStaleAcks) + } + + // Reset our local view of the session so a hypothetical reuse + // (this client cannot reuse, but be defensive) starts fresh. + s.resetShareSession() +} + +// purgeTopics removes topics from the share consumer's subscription +// and revokes any cursors for them. Called from consumer.purgeTopics +// inside blockingMetadataFn, serializing with metadata updates and +// applyMoves. +// +// The share consumer does not own its assignment: the server does. +// So the purge is two-phase: +// +// 1. Local (here): revoke cursors, drain-and-close them (so +// post-purge Record.Ack returns errShareConsumerLeft), remove +// from tps and reSeen. +// 2. Server (async): the next heartbeat sends the updated +// SubscribedTopicNames; the coordinator revokes; the next +// heartbeat response drives assignPartitions to clean up +// nowAssigned. +func (sc *shareConsumer) purgeTopics(topics []string) { + sc.cfg.logger.Log(LogLevelDebug, "purging share group topics", + "group", sc.cfg.shareGroup, + "topics", topics, + ) + tps := sc.tps.load() + for _, topic := range topics { + tp, ok := tps[topic] + if !ok { + continue + } + td := tp.load() + for i := range td.partitions { + cursor := td.partitions[i].shareCursor + cursor.assigned.Store(false) + cursor.source.Load().removeShareCursor(cursor) + entries, _ := cursor.drainAcks(true) + if n := int64(len(entries)); n > 0 { + sc.enqueueCallback(ShareAckResults{{cursor.topic, cursor.partition, errShareConsumerLeft}}, n) + } + } + } + for _, topic := range topics { + delete(sc.reSeen, topic) + } + sc.tps.purgeTopics(topics) +} + +//////////// +// MANAGE // +//////////// + +// maybeStartManage is the share equivalent of findNewPartitions; +// we don't need to do find logic here (heartbeat manages that), +// so this just starts the manage goro if it is not yet started +// and if topic data has been loaded. +func (sc *shareConsumer) maybeStartManage() { + if !sc.manageRunning.CompareAndSwap(false, true) { + return + } + for _, topicPartitions := range sc.tps.load() { + if len(topicPartitions.load().partitions) > 0 { + go sc.manage() + return + } + } + sc.manageRunning.Store(false) +} + +func (sc *shareConsumer) manage() { + defer sc.manageRunning.Store(false) + if !sc.incWorker() { + return // dying, bail + } + defer sc.decWorker() + + sc.cfg.logger.Log(LogLevelInfo, "beginning to manage the share group lifecycle", + "group", sc.cfg.shareGroup, + ) + + sc.memberGen.store(newStringUUID(), 0) // 0 joins the group + + var consecutiveErrors int + for { + sleep, err := sc.heartbeat() + if err == nil { + consecutiveErrors = 0 + + timer := time.NewTimer(sleep) + select { + case <-sc.fm.ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + continue + } + + switch { + case errors.Is(err, context.Canceled): + return + + case errors.Is(err, kerr.UnknownMemberID), + errors.Is(err, kerr.FencedMemberEpoch): + // Keep the same UUID (matches the Java client) and reset + // to epoch 0 so the next heartbeat re-joins. + member, gen := sc.memberGen.load() + sc.memberGen.storeGeneration(0) + sc.cfg.logger.Log(LogLevelInfo, "share group heartbeat lost membership, resetting epoch", + "group", sc.cfg.shareGroup, + "member_id", member, + "generation", gen, + "err", err, + ) + sc.lastSentSubscribedTopics = nil + sc.lastSentRack = false + sc.assignPartitions(nil) + sc.unresolvedAssigns = nil + sc.cl.allSources(func(s *source) { + s.resetShareSession() + }) + consecutiveErrors = 0 + continue + + case isRetryableBrokerErr(err), + isAnyDialErr(err), + sc.cl.maybeDeleteStaleCoordinator(sc.cfg.shareGroup, coordinatorTypeShare, err): + // Retryable -- fall through to shared backoff below. + + default: + // Fatal or unknown -- inject error into poll path and + // fire hooks so the user knows something is wrong. + // We do NOT assignPartitions(nil) here: we don't know + // if the broker kept or dropped our assignment. If it + // dropped us, the next heartbeat returns + // UnknownMemberID/FencedMemberEpoch which already + // clears. Clearing eagerly when the broker still has + // our assignment would cause unnecessary churn + // (revoke + re-assign on the next heartbeat). + sc.c.addFakeReadyForDraining("", 0, &ErrGroupSession{err}, "notification of share group management error") + sc.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookGroupManageError); ok { + h.OnGroupManageError(err) + } + }) + } + + // Reset sent-field tracking on any error so we re-send + // SubscribedTopicNames and RackID on the next heartbeat. + // If the coordinator changed, the new coordinator may not + // have our state yet. + sc.lastSentSubscribedTopics = nil + sc.lastSentRack = false + + consecutiveErrors++ + backoff := sc.cfg.retryBackoff(consecutiveErrors) + sc.cfg.logger.Log(LogLevelInfo, "share group manage loop hit a retryable error, retrying", + "group", sc.cfg.shareGroup, + "err", err, + "consecutive_errors", consecutiveErrors, + "backoff", backoff, + ) + deadline := time.Now().Add(backoff) + sc.cl.waitmeta(sc.fm.ctx, backoff, "waitmeta during share group manage backoff") + after := time.NewTimer(time.Until(deadline)) + select { + case <-sc.fm.ctx.Done(): + after.Stop() + return + case <-after.C: + } + } +} + +// Issues a heartbeat request, handles the response, and returns +// how long to wait until the next heartbeat. +func (sc *shareConsumer) heartbeat() (time.Duration, error) { + req := kmsg.NewPtrShareGroupHeartbeatRequest() + req.GroupID = sc.cfg.shareGroup + req.MemberID, req.MemberEpoch = sc.memberGen.load() + + // Only send RackID once; reset on fence/unknown forces re-send. + if sc.cfg.rack != "" && !sc.lastSentRack { + req.RackID = &sc.cfg.rack + } + + tps := sc.tps.load() + subscribedTopics := slices.Sorted(maps.Keys(tps)) + if sc.lastSentSubscribedTopics == nil || !slices.Equal(subscribedTopics, sc.lastSentSubscribedTopics) { + req.SubscribedTopicNames = subscribedTopics + } + + sc.cfg.logger.Log(LogLevelDebug, "share group heartbeating", + "group", sc.cfg.shareGroup, + "member_id", req.MemberID, + "member_epoch", req.MemberEpoch, + ) + resp, err := req.RequestWith(sc.fm.ctx, sc.cl) + sleep := sc.cfg.heartbeatInterval + if err == nil { + err = errCodeMessage(resp.ErrorCode, resp.ErrorMessage) + sleep = time.Duration(resp.HeartbeatIntervalMillis) * time.Millisecond + if sleep < time.Second { + sleep = time.Second // sanity + } + } + sc.cfg.logger.Log(LogLevelDebug, "share group heartbeat complete", + "group", sc.cfg.shareGroup, + "new_member_epoch", func() int32 { + if resp != nil { + return resp.MemberEpoch + } + return 0 + }(), + "err", err, + ) + if err != nil { + return sleep, err + } + + sc.lastSentSubscribedTopics = subscribedTopics + sc.lastSentRack = req.RackID != nil + + newAssigned := sc.handleHeartbeatResp(resp) + if newAssigned != nil { + sc.assignPartitions(newAssigned) + } + return sleep, nil +} + +// Returns a new assignment IF the server gives us a new assignment +// OR if we were previously assigned topic IDs we could not map to topics, +// and now we can. +func (sc *shareConsumer) handleHeartbeatResp(resp *kmsg.ShareGroupHeartbeatResponse) map[string][]int32 { + // The member ID is client-generated and immutable; only + // update the epoch from the response. + sc.memberGen.storeGeneration(resp.MemberEpoch) + + if resp.Assignment == nil { + if len(sc.unresolvedAssigns) == 0 { + return nil + } + // No assignment: try to resolve prior un-resolvable + // topics. If we can, that updates our current assignment + // and we return the new update. + resolved := sc.resolveUnresolvedTopicIDs() + if len(resolved) == 0 { + return nil + } + current := sc.nowAssigned.read() + merged := make(map[string][]int32, len(current)+len(resolved)) + maps.Copy(merged, current) + maps.Copy(merged, resolved) + return merged + } + + // Fresh assignment: discard prior unresolved state since it does not + // matter; parse assignment, and perhaps again add to unresolved. + id2t := sc.cl.id2tMap() + newAssigned := make(map[string][]int32) + sc.unresolvedAssigns = nil + for _, t := range resp.Assignment.TopicPartitions { + name := id2t[t.TopicID] + if name == "" { + if sc.unresolvedAssigns == nil { + sc.unresolvedAssigns = make(map[topicID][]int32) + } + sc.unresolvedAssigns[topicID(t.TopicID)] = t.Partitions + continue + } + newAssigned[name] = t.Partitions + } + + if len(sc.unresolvedAssigns) > 0 { + sc.cl.triggerUpdateMetadataNow("share group heartbeat has unresolved topic IDs in assignment") + } + + return newAssigned +} + +// resolveUnresolvedTopicIDs maps unknown topic IDs to topic names. +// Anything that cannot be mapped stays unresolved. +// Returns newly resolved topics in a new map. +func (sc *shareConsumer) resolveUnresolvedTopicIDs() map[string][]int32 { + if len(sc.unresolvedAssigns) == 0 { + return nil + } + id2t := sc.cl.id2tMap() + var dst map[string][]int32 + for id, ps := range sc.unresolvedAssigns { + name := id2t[[16]byte(id)] + if name == "" { + continue + } + if dst == nil { + dst = make(map[string][]int32) + } + dst[name] = append(dst[name], ps...) // shouldn't exist, but append anyway + delete(sc.unresolvedAssigns, id) + } + return dst +} + +func (sc *shareConsumer) assignPartitions(assignments map[string][]int32) { + sc.cfg.logger.Log(LogLevelInfo, "assigning share partitions", + "group", sc.cfg.shareGroup, + "assignments", assignments, + ) + + old := sc.nowAssigned.read() // manage loop is the sole writer; can be concurrently read by source goros + tps := sc.tps.load() + + sourcesToWake := make(map[*source]struct{}) + defer func() { + for src := range sourcesToWake { + src.maybeShareConsume() + } + }() + + // Revoke what was lost. + for t, oldPs := range old { + newPs := assignments[t] + tp, ok := tps[t] + if !ok { + continue // if it's not in tps (or similar checks below), we weren't using it + } + td := tp.load() + for _, p := range oldPs { + if slices.Contains(newPs, p) { // linear, *usually* fast... + continue + } + if int(p) >= len(td.partitions) { + continue + } + cursor := td.partitions[p].shareCursor + cursor.assigned.Store(false) + sourcesToWake[cursor.source.Load()] = struct{}{} + } + } + + // Add what is new. + var needsMetaUpdate bool + for t, newPs := range assignments { + oldPs := old[t] + tp, ok := tps[t] + if !ok { + needsMetaUpdate = true + continue // if we don't know the tps data, we can't assign it; force a meta refresh + } + td := tp.load() + for _, p := range newPs { + if slices.Contains(oldPs, p) { + continue // already was assigned, no-op + } + if int(p) >= len(td.partitions) { + needsMetaUpdate = true + continue + } + cursor := td.partitions[p].shareCursor + cursor.assigned.Store(true) + sourcesToWake[cursor.source.Load()] = struct{}{} + } + } + if needsMetaUpdate { + sc.cl.triggerUpdateMetadataNow("share assignment has unknown topic or partition, or leader has no source") + } + + sc.nowAssigned.store(assignments) +} + +// applyMoves moves share cursors to new leaders based on CurrentLeader +// hints from a ShareFetch response. When the response's NodeEndpoints +// list includes brokers we do not yet know about, register them first +// (via the kip951move pattern) so we can move the cursor without +// waiting for a metadata refresh. The move runs asynchronously inside +// blockingMetadataFn so it serializes with metadata updates and purge, +// matching the kip951move pattern for producers. +func (sc *shareConsumer) applyMoves(moves []shareMove, endpoints []BrokerMetadata) { + if len(moves) == 0 { + return + } + go sc.cl.blockingMetadataFn(func() { + // Seed any brokers from the response's NodeEndpoints that we + // do not yet know about. Same merge-without-remove invariant + // as kip951move.ensureBrokers. + if len(endpoints) > 0 { + k := kip951move{brokers: endpoints} + k.ensureBrokers(sc.cl) + } + // Ensure per-broker sink+source exist for every move target, + // and snapshot the sink+source pointer per leader to avoid + // repeated locks. + leaderSrcs := make(map[int32]*source, len(moves)) + sc.cl.sinksAndSourcesMu.Lock() + for _, m := range moves { + if _, exists := leaderSrcs[m.leaderID]; exists { + continue + } + sns, ok := sc.cl.sinksAndSources[m.leaderID] + if !ok { + sns = sinkAndSource{ + sink: sc.cl.newSink(m.leaderID), + source: sc.cl.newSource(m.leaderID), + } + sc.cl.sinksAndSources[m.leaderID] = sns + } + leaderSrcs[m.leaderID] = sns.source + } + sc.cl.sinksAndSourcesMu.Unlock() + + // Safety: Any leader whose endpoint we didn't see needs + // metadata to populate cl.brokers. + seenEndpoint := make(map[int32]struct{}, len(endpoints)) + for _, ep := range endpoints { + seenEndpoint[ep.NodeID] = struct{}{} + } + needsMeta := false + for _, m := range moves { + if _, ok := seenEndpoint[m.leaderID]; !ok { + needsMeta = true + break + } + } + if needsMeta { + sc.cl.triggerUpdateMetadataNow("share cursor CurrentLeader hint points to broker not in NodeEndpoints") + } + + id2t := sc.cl.id2tMap() + tps := sc.tps.load() + + var moved int + for _, m := range moves { + topicName := id2t[m.topicID] + if topicName == "" { + continue + } + tp, ok := tps[topicName] + if !ok { + continue + } + td := tp.load() + if int(m.partition) >= len(td.partitions) { + continue + } + cursor := td.partitions[m.partition].shareCursor + oldSource := cursor.source.Load() + if oldSource.nodeID == m.leaderID { + continue + } + newSource := leaderSrcs[m.leaderID] + oldSource.removeShareCursor(cursor) + cursor.source.Store(newSource) + newSource.addShareCursor(cursor) + moved++ + } + if moved > 0 { + sc.cfg.logger.Log(LogLevelInfo, "migrated share cursors via CurrentLeader hints", + "total_hints", len(moves), + "applied", moved, + ) + } + }) +} + +//////////// +// SOURCE // +//////////// + +func (s *source) maybeShareConsume() { + if s.fetchState.maybeBegin() { + go s.loopShareFetch() + } +} + +// incWorker is called at the very top of loopShareFetch. If the +// share consumer is dying, it returns false and the loop must +// hardFinish. +// +// This mirrors the consumerSession worker-count pattern in +// source.go's loopFetch but uses sc.mu (shared with sc.cond) instead +// of a per-session mutex, since shareConsumer has no per-session +// lifecycle. +func (sc *shareConsumer) incWorker() bool { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.dying { + return false + } + sc.workers++ + return true +} + +func (sc *shareConsumer) decWorker() { + sc.mu.Lock() + sc.workers-- + if sc.workers == 0 { + sc.cond.Broadcast() + } + sc.mu.Unlock() +} + +func (s *source) loopShareFetch() { + sc := s.share.sc + + if !sc.incWorker() { + s.fetchState.hardFinish() + return + } + defer sc.decWorker() + + var ( + canFetch = make(chan chan bool, 1) + ackTimer *time.Timer + ackTimerC <-chan time.Time // nil until acks are pending + + resetAckTimer = func() { + if ackTimer == nil { + ackTimer = time.NewTimer(time.Second) + } else { + ackTimer.Reset(time.Second) + } + ackTimerC = ackTimer.C + } + stopAckTimer = func() { + if ackTimer != nil { + ackTimer.Stop() + } + ackTimerC = nil + } + flushAcks = func() { + stopAckTimer() + s.shareAck(nil) + } + ) + + // If this function is signaled by acks and we have nothing to consume, + // we will EVENTUALLY exit once we loop through seeing there is nothing + // to fetch and the workState allows us to exit. + // + // On fm.ctx cancellation we do NOT flush acks: shareAck would fail on + // the cancelled ctx and enqueueAckErrors would drop them. Instead, we + // leave them on the cursor for closeShareSession to drain and + // piggyback on the FINAL_EPOCH request (which uses the caller ctx). + for { + unbuffered: + for { + select { + case <-sc.fm.ctx.Done(): + return + case <-s.share.ackCh: + resetAckTimer() + case <-s.share.ackFlushCh: + flushAcks() + case <-ackTimerC: + flushAcks() + // No more acks: We try finishing by doing a + // quick check of cursors, the check that is + // done when actually building a ShareFetch. + // + // If we DO loop back to the start (maybe two + // acks bumped workState to continueWorking), + // we will go through the full request flow and + // exit after creating an empty request, same + // as the normal consumer. + if again := s.fetchState.maybeFinish(false); !again { + return + } + case <-s.sem: + break unbuffered + } + } + + desire: + for { + select { + case <-sc.fm.ctx.Done(): + return + case <-s.share.ackCh: + resetAckTimer() + case <-s.share.ackFlushCh: + flushAcks() + case <-ackTimerC: + flushAcks() + case sc.fm.desireFetch() <- canFetch: + break desire + } + } + + allowed: + for { + select { + case <-sc.fm.ctx.Done(): + sc.fm.cancelFetchCh <- canFetch + return + case <-s.share.ackCh: + resetAckTimer() + case <-s.share.ackFlushCh: + flushAcks() + case <-ackTimerC: + flushAcks() + case doneFetch := <-canFetch: + if sc.fm.ctx.Err() != nil { + doneFetch <- false + return + } + fetched := s.shareFetch(doneFetch) + // If we fetched, any pending acks from this source's + // cursors were piggybacked on the request. Stop the + // ack timer; if more acks arrive between here and + // the next loop iteration, a signal is waiting in + // share.ackCh that will restart the timer. + if fetched { + stopAckTimer() + } + if !s.fetchState.maybeFinish(fetched || ackTimerC != nil) { + return + } + break allowed + } + } + } +} + +func (s *sourceShare) takeBuffered(paused pausedTopics) Fetch { + sc := s.sc + b := s.buffered + s.buffered = shareBufferedFetch{} + b.doneFetch <- true + close(s.s.sem) + + f := b.fetch + s.s.hook(&f, false, true) // unbuffered, polled + + // Strip paused partitions from the returned fetch and release + // their records back to the broker for redelivery. + keep := f.Topics[:0] + for _, t := range f.Topics { + pps, topicPaused := paused.t(t.Topic) + if topicPaused && pps.all { + for i := range t.Partitions { + batchAckRecords(sc, t.Partitions[i].Records, AckRelease, false) + } + continue + } + if topicPaused { + keepp := t.Partitions[:0] + for _, p := range t.Partitions { + if _, ok := pps.m[p.Partition]; ok { + batchAckRecords(sc, p.Records, AckRelease, false) + continue + } + keepp = append(keepp, p) + } + t.Partitions = keepp + } + if len(t.Partitions) > 0 { + keep = append(keep, t) + } + } + f.Topics = keep + + return f +} + +func (s *sourceShare) takeNBuffered(paused pausedTopics, n int) (Fetch, int, bool) { + sc := s.sc + var ( + r Fetch + rstrip Fetch + taken int + ) + + b := &s.buffered + bf := &b.fetch + for len(bf.Topics) > 0 && n > 0 { + t := &bf.Topics[0] + + if paused.has(t.Topic, -1) { + rstrip.Topics = append(rstrip.Topics, *t) + bf.Topics = bf.Topics[1:] + for i := range t.Partitions { + batchAckRecords(sc, t.Partitions[i].Records, AckRelease, false) + } + continue + } + + var rt *FetchTopic + ensureTopicAdded := func() { + if rt != nil { + return + } + r.Topics = append(r.Topics, *t) + rt = &r.Topics[len(r.Topics)-1] + rt.Partitions = nil + } + var rtstrip *FetchTopic + ensureTopicStripped := func() { + if rtstrip != nil { + return + } + rstrip.Topics = append(rstrip.Topics, *t) + rtstrip = &rstrip.Topics[len(rstrip.Topics)-1] + rtstrip.Partitions = nil + } + + for len(t.Partitions) > 0 && n > 0 { + p := &t.Partitions[0] + + if paused.has(t.Topic, p.Partition) { + ensureTopicStripped() + rtstrip.Partitions = append(rtstrip.Partitions, *p) + batchAckRecords(sc, p.Records, AckRelease, false) + t.Partitions = t.Partitions[1:] + continue + } + + ensureTopicAdded() + rt.Partitions = append(rt.Partitions, *p) + rp := &rt.Partitions[len(rt.Partitions)-1] + + take := min(n, len(p.Records)) + rp.Records = p.Records[:take:take] + p.Records = p.Records[take:] + n -= take + taken += take + + if len(p.Records) == 0 { + t.Partitions = t.Partitions[1:] + } + } + + if len(t.Partitions) == 0 { + bf.Topics = bf.Topics[1:] + } + } + + if len(rstrip.Topics) > 0 { + s.s.hook(&rstrip, false, true) + } + s.s.hook(&r, false, true) // unbuffered, polled + + drained := len(bf.Topics) == 0 + if drained { + s.takeBuffered(nil) + } + return r, taken, drained +} + +///////// +// ACK // +///////// + +// shareAck sends a ShareAcknowledge request. If predrained is non-nil, +// those acks are used directly; otherwise acks are drained from cursors. +// On success, bumps the share session epoch. On failure, either +// re-queues to cursors (retryable) or drops (session destroyed / +// transport error) and notifies the callback. +func (s *source) shareAck(predrained []cursorAckDrain) { + sc := s.share.sc + + var drains []cursorAckDrain + s.share.mu.Lock() // guard concurrent cursor movement + epoch := s.share.sessionEpoch + if predrained != nil { + drains = predrained + } else { + drains = s.drainAllShareAcks(false) + } + s.share.mu.Unlock() + + if len(drains) == 0 { + return + } + + nAcks, nStaleAcks, staleResults := filterStaleEntries(s, epoch, drains) + + sc.enqueueCallback(staleResults, nStaleAcks) + if nAcks == 0 { + return // everything was stale + } + + if epoch == 0 { + sc.cfg.logger.Log(LogLevelInfo, "dropping stale acks, share session epoch is 0", "broker", s.nodeID) + sc.enqueueAckErrors(drains, kerr.InvalidShareSessionEpoch, nAcks) + return + } + + memberID, _ := sc.memberGen.load() + req := kmsg.NewPtrShareAcknowledgeRequest() + req.GroupID = &sc.cfg.shareGroup + req.MemberID = &memberID + req.ShareSessionEpoch = epoch + + topicIdx := make(map[[16]byte]int) + drainIdx := make(map[tidp]int) // maps topic+partition to drains index + for i, d := range drains { + if len(d.entries) == 0 && len(d.gaps) == 0 { + continue + } + ranges, hasRenew := buildAckRanges(d.entries, d.gaps) + if len(ranges) == 0 { + continue + } + drainIdx[tidp{d.cursor.topicID, d.cursor.partition}] = i + if hasRenew { + req.IsRenewAck = true + } + tidx, ok := topicIdx[d.cursor.topicID] + if !ok { + tidx = len(req.Topics) + topicIdx[d.cursor.topicID] = tidx + req.Topics = append(req.Topics, kmsg.ShareAcknowledgeRequestTopic{TopicID: d.cursor.topicID}) + } + rp := kmsg.ShareAcknowledgeRequestTopicPartition{Partition: d.cursor.partition} + for _, r := range ranges { + rp.AcknowledgementBatches = append(rp.AcknowledgementBatches, + kmsg.ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch{ + FirstOffset: r.firstOffset, + LastOffset: r.lastOffset, + AcknowledgeTypes: ackTypes(r.ackType), + }) + } + req.Topics[tidx].Partitions = append(req.Topics[tidx].Partitions, rp) + } + sc.cfg.logger.Log(LogLevelDebug, "sending share acknowledge", + "broker", logID(s.nodeID), + "group", sc.cfg.shareGroup, + "session_epoch", req.ShareSessionEpoch, + "n_topics", len(req.Topics), + "n_ack_records", nAcks, + "is_renew_ack", req.IsRenewAck, + ) + // A retry is likely useless if the request actually left the client, + // the broker rejects duplicate acks with INVALID_RECORD_STATE. + // But, if the request did not leave the client, we benefit, and + // failing early if the req didn't is worse. + kresp, err := s.cl.retryableBrokerFn(func() (*broker, error) { + return s.cl.brokerOrErr(sc.cl.ctx, s.nodeID, errUnknownBroker) + }).Request(sc.cl.ctx, req) + if err != nil { + sc.cfg.logger.Log(LogLevelDebug, "share ack request failed", + "broker", s.nodeID, + "err", err, + ) + s.resetShareSession() + sc.enqueueAckErrors(drains, err, nAcks) + return + } + + resp := kresp.(*kmsg.ShareAcknowledgeResponse) + if topErr := kerr.ErrorForCode(resp.ErrorCode); topErr != nil { + s.resetShareSession() + sc.cfg.logger.Log(LogLevelInfo, "share ack top-level error, acks not delivered", + "broker", s.nodeID, + "err", topErr, + ) + sc.enqueueAckErrors(drains, topErr, nAcks) + return + } + s.bumpShareSessionEpochIfCurrent(epoch) // concurrent session reset / disconnect could have reset us + + var ( + seen = make(map[tidp]struct{}, len(drains)) + results ShareAckResults + moves []shareMove + requeued int64 + ) + for _, rt := range resp.Topics { + for _, rp := range rt.Partitions { + tpKey := tidp{rt.TopicID, rp.Partition} + if _, dup := seen[tpKey]; dup { + sc.cfg.logger.Log(LogLevelWarn, "broker returned duplicate partition in share ack response, ignoring", + "broker", logID(s.nodeID), + "partition", rp.Partition, + ) + continue + } + seen[tpKey] = struct{}{} + i, ok := drainIdx[tpKey] + if !ok { + continue // broker returned a partition we didn't ack + } + drained := drains[i] + + var partErr error + if rp.ErrorCode != 0 { + partErr = kerr.ErrorForCode(rp.ErrorCode) + if rp.CurrentLeader.LeaderID >= 0 && rp.CurrentLeader.LeaderEpoch >= 0 { + moves = append(moves, shareMove{ + topicID: rt.TopicID, + partition: rp.Partition, + leaderID: rp.CurrentLeader.LeaderID, + }) + } + if isShareAckRetryable(partErr) { + drained.requeue(sc) + requeued += int64(len(drained.entries)) + continue + } + } + + // On success (or non-retryable error): reset renew + // statuses so the user can renew the same record again. + // Only walk entries when the request actually carried + // renew acks -- otherwise the CAS is guaranteed to + // no-op on every entry. + if req.IsRenewAck { + for _, e := range drained.entries { + e.status.CompareAndSwap(int32(AckRenew), 0) + } + } + + results = append(results, ShareAckResult{ + Topic: drained.cursor.topic, + Partition: rp.Partition, + Err: partErr, + }) + } + } + + for tp, i := range drainIdx { + if _, processed := seen[tp]; !processed { + drained := drains[i] + sc.cfg.logger.Log(LogLevelWarn, "broker omitted partition from share ack response", + "broker", logID(s.nodeID), + "topic", drained.cursor.topic, + "partition", tp.p, + ) + results = append(results, ShareAckResult{ + Topic: drained.cursor.topic, + Partition: tp.p, + Err: errBrokerOmittedAckPartition, + }) + } + } + if len(moves) > 0 { + var moveBrokers []BrokerMetadata + if len(resp.NodeEndpoints) > 0 { + moveBrokers = make([]BrokerMetadata, 0, len(resp.NodeEndpoints)) + for _, ep := range resp.NodeEndpoints { + moveBrokers = append(moveBrokers, BrokerMetadata{ + NodeID: ep.NodeID, + Host: ep.Host, + Port: ep.Port, + Rack: ep.Rack, + }) + } + } + sc.applyMoves(moves, moveBrokers) + } + sc.enqueueCallback(results, nAcks-requeued) +} + +// releaseUndeliverable releases records that were "acquired" but for which the +// broker gave us no record data (i.e. protocol violation). +func (s *source) releaseUndeliverable(cursor *shareCursor, acquired []kmsg.ShareFetchResponseTopicPartitionAcquiredRecord, epoch int32) { + if len(acquired) == 0 { + return + } + releases := make([]shareAckRange, 0, len(acquired)) + for _, ar := range acquired { + if ar.LastOffset < ar.FirstOffset { + continue // inverted range, skip + } + releases = append(releases, shareAckRange{ + firstOffset: ar.FirstOffset, + lastOffset: ar.LastOffset, + source: s, + sessionEpoch: epoch, + ackType: int8(AckRelease), + }) + } + cursor.enqueueGaps(releases) +} + +func (sc *shareConsumer) enqueueAllAcks(byCursor cursorsAcks) { + if len(byCursor) == 0 { + return + } + var ( + sourcesToWake = make(map[*source]struct{}) + closedRes ShareAckResults + ) + for cursor, entries := range byCursor { + cursor.ackMu.Lock() + if cursor.closed { + cursor.ackMu.Unlock() + closedRes = append(closedRes, ShareAckResult{cursor.topic, cursor.partition, errShareConsumerLeft}) + continue + } + cursor.pendingAcks = append(cursor.pendingAcks, entries...) + sc.pendingAcks.Add(int64(len(entries))) + cursor.ackMu.Unlock() + sourcesToWake[cursor.source.Load()] = struct{}{} + } + for src := range sourcesToWake { + src.signalShareAcks() + } + if len(closedRes) > 0 { + sc.enqueueCallback(closedRes, 0) + } +} + +func (sc *shareConsumer) subtractPendingAcks(n int64) { + if n <= 0 { + return + } + if sc.pendingAcks.Add(-n) == 0 { + sc.ackMu.Lock() + sc.ackMu.Unlock() //nolint:staticcheck,gocritic // intentional Lock/Unlock: serialization point before ackC.Broadcast + sc.ackC.Broadcast() + } +} + +// enqueueAckErrors builds per-partition error results from drains +// and enqueues them. Empty-batch drains are skipped: they were +// fully stale-filtered and already reported via the stale callback. +func (sc *shareConsumer) enqueueAckErrors(drains []cursorAckDrain, err error, nAcks int64) { + if len(drains) == 0 { + return + } + var results ShareAckResults + for _, d := range drains { + if len(d.entries) == 0 && len(d.gaps) == 0 { + continue + } + results = append(results, ShareAckResult{d.cursor.topic, d.cursor.partition, err}) + } + sc.enqueueCallback(results, nAcks) +} + +// enqueueCallback pushes a callback + pending-count entry onto the ring. +func (sc *shareConsumer) enqueueCallback(results ShareAckResults, nAcks int64) { + entry := shareCallbackEntry{results: results, nAcks: nAcks} + if first, _ := sc.callbackRing.push(entry); first { + go sc.drainCallbacks(entry) + } +} + +func (sc *shareConsumer) drainCallbacks(entry shareCallbackEntry) { + cb := sc.cfg.shareAckCallback + for { + if cb != nil && len(entry.results) > 0 { + cb(sc.cl, entry.results) + } + sc.subtractPendingAcks(entry.nAcks) + var more bool + entry, more, _ = sc.callbackRing.dropPeek() + if !more { + return + } + } +} + +// requeue puts a drain's entries and gaps back onto its cursor in +// one mutex acquisition. If the cursor is closed (e.g. after +// PurgeTopics), entries fire the left-group callback; gaps are +// silently dropped. +func (d *cursorAckDrain) requeue(sc *shareConsumer) { + c := d.cursor + c.ackMu.Lock() + if c.closed { + c.ackMu.Unlock() + if len(d.entries) > 0 { + sc.enqueueCallback(ShareAckResults{{c.topic, c.partition, errShareConsumerLeft}}, int64(len(d.entries))) + } + return + } + c.pendingAcks = append(c.pendingAcks, d.entries...) + c.pendingGaps = append(c.pendingGaps, d.gaps...) + c.ackMu.Unlock() +} + +// enqueueGaps appends gap/release ranges to the cursor's pendingGaps. +// Does NOT increment sc.pendingAcks - internal acks are invisible to +// FlushAcks. Returns false if the cursor is closed. +func (c *shareCursor) enqueueGaps(gaps []shareAckRange) bool { + c.ackMu.Lock() + if c.closed { + c.ackMu.Unlock() + return false + } + c.pendingGaps = append(c.pendingGaps, gaps...) + c.ackMu.Unlock() + return true +} + +// drainAcks removes and returns all pending user acks and gaps +// under c.ackMu. If close is true, marks the cursor closed so +// later appendAck / enqueueGaps return a left-group error. +func (c *shareCursor) drainAcks(close bool) ([]*shareAckState, []shareAckRange) { + c.ackMu.Lock() + entries := c.pendingAcks + gaps := c.pendingGaps + c.pendingAcks = nil + c.pendingGaps = nil + if close { + c.closed = true + } + c.ackMu.Unlock() + return entries, gaps +} + +// drainAllShareAcks drains every cursor under s.share, returning +// only non-empty drains. Caller must hold s.share.mu. close is +// forwarded to drainAcks. +func (s *source) drainAllShareAcks(close bool) []cursorAckDrain { + var drains []cursorAckDrain + for _, c := range s.share.cursors { + entries, gaps := c.drainAcks(close) + if len(entries) > 0 || len(gaps) > 0 { + drains = append(drains, cursorAckDrain{cursor: c, entries: entries, gaps: gaps}) + } + } + return drains +} + +var shareAckKey = strp("share-ack") + +// shareAckFromCtx resolves the per-record ack state from the share-fetch +// slab attached to r.Context. Returns nil for records that did not come +// from a share fetch, or whose pointer is out of the slab's range. +// state.slab gives the enclosing slab when callers need it. +func shareAckFromCtx(r *Record) *shareAckState { + if r.Context == nil { + return nil + } + v := r.Context.Value(shareAckKey) + if v == nil { + return nil + } + slab := v.(*shareAckSlab) + idx := int((uintptr(unsafe.Pointer(r)) - uintptr(unsafe.Pointer(slab.records0))) / recSize) //nolint:gosec // pointer arithmetic to index the slab; idx is bounds-checked below + if idx < 0 || idx >= len(slab.states) { + return nil + } + return &slab.states[idx] +} + +// appendAck appends this state to its cursor's pendingAcks and +// increments sc.pendingAcks. +func (st *shareAckState) appendAck() { + c := st.slab.cursor + sc := st.slab.ackSource.share.sc + c.ackMu.Lock() + if c.closed { + c.ackMu.Unlock() + sc.enqueueCallback(ShareAckResults{{c.topic, c.partition, errShareConsumerLeft}}, 0) + return + } + wake := len(c.pendingAcks) == 0 // only empty->non-empty needs a wake signal + c.pendingAcks = append(c.pendingAcks, st) + sc.pendingAcks.Add(1) // must happen inside mu, otherwise drainAcks(true) could race before we inc and FlushAcks could end early + c.ackMu.Unlock() + + if wake { + c.source.Load().signalShareAcks() + } +} + +// batchAckRecords applies status to acks resolvable from the given +// records. Callers must only pass records whose backing memory is +// still valid; for the lastPolled / pendingAcks paths, use +// batchAckStates which operates on pre-resolved state pointers. +// +// strictZero=true narrows the CAS to "status was 0", used by +// closeShareSession / leave / MarkAcks(no-args) so a bulk action +// does not override an explicit user ack. +func batchAckRecords(sc *shareConsumer, rs []*Record, status AckStatus, strictZero bool) { + var byCursor cursorsAcks + for _, r := range rs { + st := shareAckFromCtx(r) + if st == nil { + continue + } + if !st.tryAck(status, strictZero) { + continue + } + byCursor.add(st) + } + sc.enqueueAllAcks(byCursor) +} + +// batchAckStates is batchAckRecords for pre-resolved state pointers. +// Safe under Record.Recycle + pool-reuse aliasing: never reads any +// *Record. +func batchAckStates(sc *shareConsumer, sts []*shareAckState, status AckStatus, strictZero bool) { + var byCursor cursorsAcks + for _, st := range sts { + if !st.tryAck(status, strictZero) { + continue + } + byCursor.add(st) + } + sc.enqueueAllAcks(byCursor) +} + +// tryAck CASes this state's status to the given AckStatus. +// - strictZero or AckRenew: succeeds only on status==0. +// - terminal otherwise: succeeds on 0 or AckRenew; fails if +// already terminal. +func (st *shareAckState) tryAck(status AckStatus, strictZero bool) bool { + if strictZero || status == AckRenew { + return st.status.CompareAndSwap(0, int32(status)) + } + for { + cur := st.status.Load() + if cur != 0 && cur != int32(AckRenew) { + return false + } + if st.status.CompareAndSwap(cur, int32(status)) { + return true + } + } +} + +// filterStaleEntries mutates drains in-place to drop ack entries +// (and gap ranges) that the broker cannot honor, reporting them via +// the user callback as pre-filtered drops. Two categories, both +// derived from the per-entry slab's ackSource and sessionEpoch: +// +// 1. slab.ackSource == s && slab.sessionEpoch > epoch: same source, +// entry stamped with an epoch higher than the current session +// epoch: a session reset happened and the broker lost record state. +// +// 2. slab.ackSource != s: the cursor migrated from the original +// source to s after the acks were queued. Acquisition state is +// not transferred to the new broker. +// +// Edge cases: enough epoch bumps happen, or the leader transfer from +// A to B then back; it's fine, we'll just have a wasted round trip +// and the broker rejects the acks. +// +// Returns: +// - nAcks: count of deliverable (non-stale, non-migrated) records +// - nStaleAcks: count of pre-filtered records (stale + migrated) +// - staleResults: per-cursor pre-filter results for the user callback +func filterStaleEntries(s *source, epoch int32, drains []cursorAckDrain) (nUserAcks, nStaleUserAcks int64, staleResults ShareAckResults) { + for i := range drains { + d := &drains[i] + + // Filter user ack entries. + filteredEntries := d.entries[:0] + var dropErr error + for _, e := range d.entries { + switch { + case e.slab.ackSource == s && e.slab.sessionEpoch > epoch: + nStaleUserAcks++ + if dropErr == nil { + dropErr = kerr.InvalidShareSessionEpoch + } + case e.slab.ackSource != s: + nStaleUserAcks++ + if dropErr == nil { + dropErr = kerr.InvalidRecordState + } + default: + nUserAcks++ + filteredEntries = append(filteredEntries, e) + } + } + d.entries = filteredEntries + + // Filter gap/release ranges (same source/epoch check). + filteredGaps := d.gaps[:0] + for _, g := range d.gaps { + switch { + case g.source == s && g.sessionEpoch > epoch: + // stale gap; drop silently (not counted in pendingAcks) + case g.source != s: + // migrated gap; drop silently + default: + filteredGaps = append(filteredGaps, g) + } + } + d.gaps = filteredGaps + + if dropErr != nil { + staleResults = append(staleResults, ShareAckResult{d.cursor.topic, d.cursor.partition, dropErr}) + } + } + return +} + +var ackTypeSlices = [...][]int8{ + 0: {0}, // gap (compaction skip) + int8(AckAccept): {int8(AckAccept)}, + int8(AckRelease): {int8(AckRelease)}, + int8(AckReject): {int8(AckReject)}, + int8(AckRenew): {int8(AckRenew)}, +} // eliminates many tiny allocs + +func ackTypes(t int8) []int8 { + if t >= 0 && int(t) < len(ackTypeSlices) { + return ackTypeSlices[t] + } + return []int8{t} +} + +// buildAckRanges converts user ack entries + gap ranges into merged +// wire-format ranges. User entries are read via their live status +// pointer; entries with status 0 (reset or unhandled) are skipped. +// hasRenew is true if any entry has AckRenew status. +// +// Entries and gaps are sorted by offset before coalescing so that +// contiguous same-type ranges merge regardless of insertion order. +// The two are built separately (gaps are acked immediately so they +// rarely coalesce with user entries). +func buildAckRanges(entries []*shareAckState, gaps []shareAckRange) (ranges []shareAckRange, hasRenew bool) { + slices.SortFunc(entries, func(a, b *shareAckState) int { + return cmp.Compare(a.offset, b.offset) + }) + slices.SortFunc(gaps, func(a, b shareAckRange) int { + return cmp.Compare(a.firstOffset, b.firstOffset) + }) + // Dedupe: a single record can have multiple entries for the same offset + // (e.g. Ack(AckRenew) then Ack(AckAccept) both append; the terminal + // CAS overwrites the renew but the renew entry remains in the slice). + // Both entries read the same final status, so emit only one. Without + // this, the request carries two adjacent [X,X,T] batches and the + // broker rejects with INVALID_RECORD_STATE. + var lastOffset int64 = -1 + for _, e := range entries { + t := int8(e.status.Load()) + if t == 0 { + continue // status was reset or not yet decided + } + if e.offset == lastOffset { + continue // duplicate from a renew-then-terminal sequence + } + lastOffset = e.offset + if t == int8(AckRenew) { + hasRenew = true + } + ranges = coalesceAppendRange(ranges, shareAckRange{ + firstOffset: e.offset, + lastOffset: e.offset, + source: e.slab.ackSource, + sessionEpoch: e.slab.sessionEpoch, + ackType: t, + }) + } + for _, g := range gaps { + ranges = coalesceAppendRange(ranges, g) + } + return +} + +// coalesceAppendRange appends r to the slice, merging with the last +// element if they are contiguous, same type, same source, and same epoch. +func coalesceAppendRange(out []shareAckRange, r shareAckRange) []shareAckRange { + if n := len(out); n > 0 { + last := &out[n-1] + if last.ackType == r.ackType && last.source == r.source && + last.sessionEpoch == r.sessionEpoch && last.lastOffset+1 == r.firstOffset { + last.lastOffset = r.lastOffset + return out + } + } + return append(out, r) +} + +/////////// +// FETCH // -- methods on source rather than shareSource b/c most need source fields +/////////// + +// shareFetch orchestrates a share fetch: send the request, handle +// errors and backoff, apply leader moves, dispatch ack callbacks, +// and buffer the result. Per-partition handling lives in +// handleShareReqResp. +func (s *source) shareFetch(doneFetch chan<- bool) (fetched bool) { + sc := s.share.sc + req, usable, piggybackAcks, sentPiggyback, nAcks, staleResults, nStaleAcks, hasRenew := s.createShareReq(false) + + // Renew acks (type 4) cannot be piggybacked on a ShareFetch + // (the broker requires IsRenewAck + zero fetch params). If any + // are present, send ALL drained acks via standalone + // ShareAcknowledge first, then rebuild the request without + // re-draining acks (they were already sent, new ones COULD + // have happened but we need forward progress...). + if hasRenew { + sc.enqueueCallback(staleResults, nStaleAcks) + s.shareAck(piggybackAcks) + req, usable, piggybackAcks, sentPiggyback, nAcks, staleResults, nStaleAcks, _ = s.createShareReq(true) // clears piggyBack, nacks, stale + } + + var ( + buffered bool + alreadySentToDoneFetch bool + ) + defer func() { + if !buffered && !alreadySentToDoneFetch { + doneFetch <- false + } + }() + + // Handle stale-filtered acks from createShareReq. + sc.enqueueCallback(staleResults, nStaleAcks) + + // Drop piggybacked acks on a fresh session (epoch 0): the + // broker rejects acks on an initial fetch. + if req != nil && req.ShareSessionEpoch == 0 && len(piggybackAcks) > 0 { + sc.cfg.logger.Log(LogLevelInfo, "dropping stale piggybacked acks, share session epoch is 0", + "broker", s.nodeID, + ) + sc.enqueueAckErrors(piggybackAcks, kerr.InvalidShareSessionEpoch, nAcks) + piggybackAcks = nil + nAcks = 0 + } + + if req == nil { // nothing to fetch or forget; fallback to a shareAck + s.shareAck(nil) + return false + } + + sc.cfg.logger.Log(LogLevelDebug, "sending share fetch", + "broker", logID(s.nodeID), + "group", sc.cfg.shareGroup, + "session_epoch", req.ShareSessionEpoch, + "n_topics", len(req.Topics), + "n_forgotten_topics", len(req.ForgottenTopicsData), + "max_wait_ms", req.MaxWaitMillis, + "max_records", req.MaxRecords, + "batch_size", req.BatchSize, + "n_piggyback_acks", nAcks, + ) + + // Bound the broker round-trip to MaxWait + 5s. The broker caps + // its own wait at MaxWaitMillis, so a healthy response arrives + // within MaxWait + RTT; 5s of grace tolerates slow links while + // still giving LeaveGroup a predictable shutdown ceiling instead + // of cancelling mid-fetch (which strands piggybacked acks that + // the broker already processed, causing double-ack or stale- + // epoch errors on the FINAL_EPOCH resend path). + timeout := time.Duration(req.MaxWaitMillis)*time.Millisecond + 5*time.Second + var ( + kresp kmsg.Response + err error + requested = make(chan struct{}) + ctx, cancel = context.WithTimeout(sc.cl.ctx, timeout) + ) + defer cancel() + + br, brerr := s.cl.brokerOrErr(ctx, s.nodeID, errUnknownBroker) + if brerr != nil { + close(requested) + err = brerr + } else { + br.do(ctx, req, func(k kmsg.Response, e error) { + kresp, err = k, e + close(requested) + }) + } + + // Wait for the response. ctx carries a MaxWait+5s deadline rather + // than fm.ctx cancellation: cancelling mid-fetch is lossy (the + // broker may have already consumed piggybacked acks and bumped + // the session epoch), so we let in-flight fetches finish within + // their natural window. LeaveGroup's worker barrier tolerates + // this because the deadline caps total shutdown delay. Only the + // hard-stuck case (broker unresponsive past the deadline) falls + // into the requeue path below, where closeShareSession re-sends + // the acks on the userCtx-bounded FINAL_EPOCH ShareAcknowledge. + select { + case <-requested: + if isContextErr(err) && ctx.Err() != nil { + // Deadline hit or cl.ctx cancelled. Requeue piggybacked + // acks so closeShareSession or a later fetch can retry. + for _, pa := range piggybackAcks { + pa.requeue(sc) + } + return false + } + fetched = true + case <-ctx.Done(): + // cl.ctx cancelled or deadline. Requeue for closeShareSession. + for _, pa := range piggybackAcks { + pa.requeue(sc) + } + return false + } + + var didBackoff bool + backoff := func() { + // Release fetch slot before sleeping so other sources + // can fetch during our backoff. + doneFetch <- false + alreadySentToDoneFetch = true + didBackoff = true + s.consecutiveFailures++ + after := time.NewTimer(sc.cfg.retryBackoff(s.consecutiveFailures)) + defer after.Stop() + select { + case <-after.C: + case <-ctx.Done(): + } + } + defer func() { + if !didBackoff { + s.consecutiveFailures = 0 + } + }() + + if err != nil { + s.resetShareSession() + backoff() + sc.enqueueAckErrors(piggybackAcks, err, nAcks) + return fetched + } + + resp := kresp.(*kmsg.ShareFetchResponse) + + res := s.handleShareReqResp(req, resp, usable, piggybackAcks, sentPiggyback) + + if res.discardErr != nil { + sc.enqueueAckErrors(piggybackAcks, res.discardErr, nAcks) + return fetched + } + + if len(res.moves) > 0 { + sc.applyMoves(res.moves, res.moveBrokers) + } + + sc.enqueueCallback(res.ackResults, nAcks-res.ackRequeued) + + if res.fetch.hasErrorsOrRecords() { + buffered = true + s.share.buffered = shareBufferedFetch{ + fetch: res.fetch, + doneFetch: doneFetch, + } + s.sem = make(chan struct{}) + s.hook(&res.fetch, true, false) + sc.c.addSourceReadyForDraining(s) + } else if res.allErrsStripped { + backoff() + } + return fetched +} + +// handleShareReqResp decodes partitions, bumps the session epoch, +// and handles gap acks, undeliverable releases, and ack requeues. +// Returns a shareFetchResult for shareFetch to act on. +func (s *source) handleShareReqResp(req *kmsg.ShareFetchRequest, resp *kmsg.ShareFetchResponse, usable []*shareCursor, piggybackAcks []cursorAckDrain, piggybackIdx map[tidp]int) shareFetchResult { + sc := s.share.sc + + if topErr := kerr.ErrorForCode(resp.ErrorCode); topErr != nil { + s.resetShareSession() + sc.cfg.logger.Log(LogLevelInfo, "share fetch top-level error", + "broker", s.nodeID, + "err", topErr, + ) + return shareFetchResult{discardErr: topErr} + } + + // Mid-flight reset guard: if manage reset the session while + // our request was in flight, the records are unackable (their + // slab epoch won't match the new session) but piggybacked ack + // results are still valid -- the broker processed them. We + // extract ack results and discard records. + epoch := req.ShareSessionEpoch + s.share.mu.Lock() + sessionStale := s.share.sessionEpoch != epoch + if !sessionStale { + s.share.sessionEpoch++ + // Only add cursors that were in the WANT set (usable) to + // sessionParts. req.Topics may also contain piggyback-only + // partitions for cursors that got revoked after we drained + // their acks: adding those to sessionParts would force us + // to forget them on the next request, generating an extra + // round trip of ForgottenTopicsData. + for _, c := range usable { + s.share.sessionParts[tidp{c.topicID, c.partition}] = struct{}{} + } + for _, ft := range req.ForgottenTopicsData { + for _, p := range ft.Partitions { + delete(s.share.sessionParts, tidp{ft.TopicID, p}) + } + } + } + newEpoch := s.share.sessionEpoch + s.share.mu.Unlock() + if sessionStale { + sc.cfg.logger.Log(LogLevelInfo, "share fetch session was reset mid-flight, extracting ack results only", + "broker", s.nodeID, + "sent_epoch", epoch, + "current_epoch", newEpoch, + ) + } + + acqLockMillis := resp.AcquisitionLockTimeoutMillis + if acqLockMillis < 1000 { + if acqLockMillis <= 0 { + s.cl.cfg.logger.Log(LogLevelWarn, "broker share fetch response has non-positive AcquisitionLockTimeoutMillis; clamping to 1s", + "broker", logID(s.nodeID), + "value", acqLockMillis, + ) + } + acqLockMillis = 1000 + } + acqLockDeadlineNanos := time.Now().Add(time.Duration(acqLockMillis) * time.Millisecond).UnixNano() + + // cursorMap includes both usable and piggyback-only cursors so + // ack-only partitions in the response are not treated as unknown. + cursorMap := make(map[tidp]*shareCursor, len(usable)+len(piggybackAcks)) + for _, c := range usable { + cursorMap[tidp{c.topicID, c.partition}] = c + } + for _, d := range piggybackAcks { + cursorMap[tidp{d.cursor.topicID, d.cursor.partition}] = d.cursor + } + + var ( + fetch Fetch + ackResults ShareAckResults + moves []shareMove + ackRequeued int64 + partitionsWithErrs int + seen = make(map[tidp]struct{}, len(usable)+len(piggybackAcks)) + ) + + for i := range resp.Topics { + rt := &resp.Topics[i] + + var topicName string + var partitions []FetchPartition + + for j := range rt.Partitions { + rp := &rt.Partitions[j] + tpKey := tidp{rt.TopicID, rp.Partition} + if _, dup := seen[tpKey]; dup { + s.cl.cfg.logger.Log(LogLevelWarn, "broker returned duplicate partition in share fetch response, ignoring duplicate", + "broker", logID(s.nodeID), + "topic_id", rt.TopicID, + "partition", rp.Partition, + ) + continue + } + seen[tpKey] = struct{}{} + cursor := cursorMap[tpKey] + if cursor == nil { + s.cl.cfg.logger.Log(LogLevelWarn, "broker returned partition from share fetch that we did not ask for", + "broker", logID(s.nodeID), + "topic_id", rt.TopicID, + "partition", rp.Partition, + ) + continue + } + topicName = cursor.topic + + // Handle acks first, + pi, hadAcks := piggybackIdx[tpKey] + if rp.AcknowledgeErrorCode != 0 { + ackErr := kerr.ErrorForCode(rp.AcknowledgeErrorCode) + if !hadAcks { + s.cl.cfg.logger.Log(LogLevelWarn, "broker returned ack error for partition we did not ack", + "broker", logID(s.nodeID), + "topic", topicName, + "partition", rp.Partition, + "err", ackErr, + ) + } else if isShareAckRetryable(ackErr) { + pa := piggybackAcks[pi] + pa.requeue(sc) + ackRequeued += int64(len(pa.entries)) + } else { + ackResults = append(ackResults, ShareAckResult{topicName, rp.Partition, ackErr}) + } + } else if hadAcks { + ackResults = append(ackResults, ShareAckResult{topicName, rp.Partition, nil}) + } + + if sessionStale { + continue // records are unackable, extract ack results only + } + + // then records. + if rp.ErrorCode != 0 { + partitionsWithErrs++ + if rp.CurrentLeader.LeaderID >= 0 && rp.CurrentLeader.LeaderEpoch >= 0 { + moves = append(moves, shareMove{ + topicID: rt.TopicID, + partition: rp.Partition, + leaderID: rp.CurrentLeader.LeaderID, + }) + continue + } + partErr := kerr.ErrorForCode(rp.ErrorCode) + partitions = append(partitions, FetchPartition{ + Partition: rp.Partition, + Err: partErr, + }) + continue + } + + if len(rp.Records) == 0 && len(rp.AcquiredRecords) == 0 { + continue + } + + if len(rp.Records) == 0 && len(rp.AcquiredRecords) > 0 { // buggy broker guard + s.cl.cfg.logger.Log(LogLevelWarn, "broker share fetch response has acquired records but no record data; releasing for redelivery", + "broker", logID(s.nodeID), + "topic", topicName, + "partition", rp.Partition, + "acquired_ranges", len(rp.AcquiredRecords), + ) + s.releaseUndeliverable(cursor, rp.AcquiredRecords, newEpoch) + continue + } + + // We could be delivered acks that are no longer + // assigned (concurrent heartbeat update); that's + // fine, we deliver anyway and user will ack. + // The records are acquired for us on the broker, + // not auto-released. + + fp, gapAcks := s.processSharePartition(topicName, cursor, newEpoch, rp, acqLockDeadlineNanos) + if len(gapAcks) > 0 { + cursor.enqueueGaps(gapAcks) + } + + if len(fp.Records) == 0 && fp.Err == nil { + continue + } + partitions = append(partitions, fp) + } + if len(partitions) > 0 { + fetch.Topics = append(fetch.Topics, FetchTopic{ + Topic: topicName, + Partitions: partitions, + }) + } + } + + for tp, pi := range piggybackIdx { + if _, processed := seen[tp]; !processed { + topic := piggybackAcks[pi].cursor.topic + s.cl.cfg.logger.Log(LogLevelWarn, "broker omitted partition from share fetch response that we sent acks for", + "broker", logID(s.nodeID), + "topic", topic, + "partition", tp.p, + ) + ackResults = append(ackResults, ShareAckResult{topic, tp.p, errBrokerOmittedAckPartition}) + } + } + + var moveBrokers []BrokerMetadata + if len(moves) > 0 && len(resp.NodeEndpoints) > 0 { + moveBrokers = make([]BrokerMetadata, 0, len(resp.NodeEndpoints)) + for _, ep := range resp.NodeEndpoints { + moveBrokers = append(moveBrokers, BrokerMetadata{ + NodeID: ep.NodeID, + Host: ep.Host, + Port: ep.Port, + Rack: ep.Rack, + }) + } + } + + return shareFetchResult{ + fetch: fetch, + moves: moves, + moveBrokers: moveBrokers, + ackResults: ackResults, + ackRequeued: ackRequeued, + allErrsStripped: partitionsWithErrs > 0 && !fetch.hasErrorsOrRecords(), + } +} + +// processSharePartition decodes a single partition from a ShareFetch +// response, filters to acquired records, and generates gap acks. +// +// Gap acks cover offsets in AcquiredRecords ranges that have no +// corresponding record data (compaction holes or decode errors). +// The broker tracks acks in blocks - regardless of whether there are +// actually underlying records. We ack the gaps immediately to free +// up the acquired count on the broker. +func (s *source) processSharePartition(topicName string, cursor *shareCursor, sessionEpoch int32, rp *kmsg.ShareFetchResponseTopicPartition, acqLockDeadlineNanos int64) (FetchPartition, []shareAckRange) { + sc := s.share.sc + // Build a synthetic FetchResponseTopicPartition because ShareFetch + // uses the same wire format for records. + fakePart := kmsg.NewFetchResponseTopicPartition() + fakePart.Partition = rp.Partition + fakePart.RecordBatches = rp.Records + fakePart.HighWatermark = -1 + fakePart.LastStableOffset = -1 + fakePart.LogStartOffset = -1 + + fp, _ := ProcessFetchPartition(ProcessFetchPartitionOpts{ + KeepControlRecords: sc.cfg.keepControl, + DisableCRCValidation: sc.cfg.disableFetchCRCValidation, + Topic: topicName, + Partition: rp.Partition, + Pools: sc.cfg.pools, + shareAckSlab: func(numRecords int, firstRecord *Record) *shareAckSlab { + return &shareAckSlab{ + states: make([]shareAckState, numRecords), + records0: firstRecord, + ackSource: s, + cursor: cursor, + acqLockDeadlineNanos: acqLockDeadlineNanos, + sessionEpoch: sessionEpoch, + } + }, + }, &fakePart, sc.cfg.decompressor, nil) + + // Defense: validate AcquiredRecords. The two-pointer scan + // below assumes the ranges are sorted by FirstOffset, non- + // overlapping, and have FirstOffset <= LastOffset. Buggy + // brokers may violate any of these. + // + // Salvage strategy: + // - Sort by FirstOffset (handles unsorted). + // - Drop ranges where FirstOffset > LastOffset (degenerate; + // unrecoverable per-range, but the rest of the partition + // is still usable). + // - Merge overlapping ranges into their union (overlapping + // acquisition windows shouldn't exist, but if they do, + // treat as one wider acquisition). + if len(rp.AcquiredRecords) > 0 { + acquired := rp.AcquiredRecords + + // Drop degenerate ranges in-place and detect + // unsorted/overlapping in the same pass. + n := 0 + needsFix := false + for _, ar := range acquired { + if ar.FirstOffset < 0 || ar.FirstOffset > ar.LastOffset { + continue + } + if n > 0 && ar.FirstOffset <= acquired[n-1].LastOffset { + needsFix = true + } + acquired[n] = ar + n++ + } + if n < len(acquired) { + s.cl.cfg.logger.Log(LogLevelWarn, "broker share fetch response had degenerate AcquiredRecords ranges; skipping", + "broker", logID(s.nodeID), + "topic", topicName, + "partition", rp.Partition, + "dropped", len(acquired)-n, + ) + acquired = acquired[:n] + rp.AcquiredRecords = acquired + } + if needsFix { + s.cl.cfg.logger.Log(LogLevelWarn, "broker share fetch response had unsorted or overlapping AcquiredRecords; salvaging via sort+merge", + "broker", logID(s.nodeID), + "topic", topicName, + "partition", rp.Partition, + ) + fixed := slices.Clone(acquired) + slices.SortFunc(fixed, func(a, b kmsg.ShareFetchResponseTopicPartitionAcquiredRecord) int { + return cmp.Compare(a.FirstOffset, b.FirstOffset) + }) + out := fixed[:0] + for _, ar := range fixed { + if len(out) > 0 && ar.FirstOffset <= out[len(out)-1].LastOffset+1 { + if ar.LastOffset > out[len(out)-1].LastOffset { + out[len(out)-1].LastOffset = ar.LastOffset + } + continue + } + out = append(out, ar) + } + rp.AcquiredRecords = out + } + } + + // Filter to acquired & gaps using a two-pointer scan. + gapType := int8(0) // 0 = gap + if fp.Err != nil { // error codes are handled before entering; an error here is a decode error + // fp.Err here is a whole-batch decode failure: the batch + // header, CRC, or compressed payload could not be parsed. kgo + // does not have per-record deserializers, so there is no + // per-record decode error to surface -- any future + // per-record error (e.g. key/value decompression of an + // individual record inside a decoded batch) is not reported + // via fp.Err. Current enumeration of causes: + // - batch CRC mismatch + // - whole-batch decompression failure + // - malformed batch header / length + // Because the entire batch failed to decode, we can't + // distinguish which acquired offsets belonged to the bad + // batch vs a successfully-decoded one. We RELEASE the + // unfilled offsets so the broker re-delivers them after + // acquisition-lock expiry; Reject would permanently archive + // records that could be fine on a redelivery from another + // consumer (e.g. transient corruption on the wire). + // + // If the underlying error indicates irrecoverable corruption + // (not a network/transport issue), reject would be more + // correct -- but kgo does not currently distinguish those + // classes, so RELEASE is the safe default. + gapType = int8(AckRelease) + sc.cl.cfg.logger.Log(LogLevelWarn, "share fetch decode error on batch; releasing affected offsets for broker redelivery", + "topic", topicName, + "partition", rp.Partition, + "err", fp.Err, + "acquired_ranges", len(rp.AcquiredRecords), + "records_decoded", len(fp.Records), + ) + } + var ( + gapAcks []shareAckRange + ri, n int + ) + + for _, ar := range rp.AcquiredRecords { + nextExpected := ar.FirstOffset + for ri < len(fp.Records) && fp.Records[ri].Offset < ar.FirstOffset { + ri++ + } + for ri < len(fp.Records) && fp.Records[ri].Offset <= ar.LastOffset { + r := fp.Records[ri] + if r.Offset > nextExpected { + gapAcks = append(gapAcks, shareAckRange{ + firstOffset: nextExpected, + lastOffset: r.Offset - 1, + source: s, + sessionEpoch: sessionEpoch, + ackType: gapType, + }) + } + slab := r.Context.Value(shareAckKey).(*shareAckSlab) + idx := int((uintptr(unsafe.Pointer(r)) - uintptr(unsafe.Pointer(slab.records0))) / recSize) //nolint:gosec // pointer arithmetic to index the slab (same as pools.go) + deliveryCount := int32(ar.DeliveryCount) + if deliveryCount < 1 { + deliveryCount = 1 // handle buggy broker; we guarantee 1 min in DeliveryCount docs + } + slab.states[idx] = shareAckState{ + deliveryCount: deliveryCount, + offset: r.Offset, + slab: slab, + } + fp.Records[n] = r + n++ + nextExpected = r.Offset + 1 + ri++ + } + if nextExpected <= ar.LastOffset { + gapAcks = append(gapAcks, shareAckRange{ + firstOffset: nextExpected, + lastOffset: ar.LastOffset, + source: s, + sessionEpoch: sessionEpoch, + ackType: gapType, + }) + } + } + + clear(fp.Records[n:]) + fp.Records = fp.Records[:n] + return fp, gapAcks +} + +// createShareReq builds a ShareFetch request under share.mu. +// Returns nil req if there is nothing to fetch or forget. +// Longer than source.go's createReq because share also drains +// piggybacked acks, filters stale batches, and computes the +// forget set (session diff) inline under the same lock. +func (s *source) createShareReq(skipAckDrain bool) ( + req *kmsg.ShareFetchRequest, + usable []*shareCursor, + piggybackAcks []cursorAckDrain, + sentPiggyback map[tidp]int, + nAcks int64, + staleResults ShareAckResults, + nStaleAcks int64, + hasRenew bool, +) { + sc := s.share.sc + paused := s.cl.consumer.loadPaused() + + s.share.mu.Lock() + defer s.share.mu.Unlock() + + // Build the WANT set: cursors with assigned=true and not paused. + // usable (the returned slice) preserves round-robin fairness for + // req.Topics ordering. wantSet is the set form for the diff + // against sessionParts below. + wantSet := make(map[tidp]struct{}, len(s.share.cursors)) + usable = make([]*shareCursor, 0, len(s.share.cursors)) + nShareCursors := len(s.share.cursors) + ci := s.share.cursorsStart + for range s.share.cursors { + c := s.share.cursors[ci] + ci = (ci + 1) % nShareCursors + if !c.assigned.Load() || paused.has(c.topic, c.partition) { + continue + } + wantSet[tidp{c.topicID, c.partition}] = struct{}{} + usable = append(usable, c) + } + if nShareCursors > 0 { + s.share.cursorsStart = (s.share.cursorsStart + 1) % nShareCursors + } + + // Compute toForget = sessionParts - wantSet. Anything currently + // in the broker session that we no longer want gets a + // ForgottenTopicsData entry. Only meaningful at epoch > 0 + // (sessionParts is empty at epoch 0). + var toForget []tidp + epoch := s.share.sessionEpoch + if epoch > 0 { + for sp := range s.share.sessionParts { + if _, want := wantSet[sp]; !want { + toForget = append(toForget, sp) + } + } + } + + // Nothing to fetch AND nothing to forget: hand off to shareAck. + if len(usable) == 0 && len(toForget) == 0 { + return + } + + // Drain acks from ALL cursors on this source (not just usable + // ones) to piggyback on the ShareFetch request. + if !skipAckDrain { + piggybackAcks = s.drainAllShareAcks(false) + nAcks, nStaleAcks, staleResults = filterStaleEntries(s, epoch, piggybackAcks) + // Compute hasRenew once here so callers don't have to re-walk + // every entry's status atomic in a separate hasRenewAck pass. + scanRenew: + for i := range piggybackAcks { + for _, e := range piggybackAcks[i].entries { + if int8(e.status.Load()) == int8(AckRenew) { + hasRenew = true + break scanRenew + } + } + } + } + + memberID, _ := sc.memberGen.load() + req = kmsg.NewPtrShareFetchRequest() + req.GroupID = &sc.cfg.shareGroup + req.MemberID = &memberID + req.ShareSessionEpoch = epoch + req.MaxWaitMillis = sc.cfg.maxWait + if nAcks > 0 && req.MaxWaitMillis > 500 { + req.MaxWaitMillis = 500 // the broker holds the entire response (ack results included) until MaxWait expires or records appear; cap so FlushAcks isn't blocked + } + req.MinBytes = sc.cfg.minBytes + req.MaxBytes = sc.cfg.maxBytes.load() + maxRecs := sc.cfg.shareMaxRecords + if maxRecs <= 0 { + maxRecs = 500 // KIP-1206 default + } + req.MaxRecords = maxRecs + req.BatchSize = maxRecs + if sc.cfg.shareMaxRecordsStrict { + req.ShareAcquireMode = 1 // KIP-1206 record-limit mode + } + + topicIdx := make(map[[16]byte]int, len(usable)) + partIdx := make(map[tidp]int, len(usable)) + + // Add usable cursors, skipping those already in the broker + // session (incremental fetch). + for _, c := range usable { + if epoch > 0 { + if _, inSession := s.share.sessionParts[tidp{c.topicID, c.partition}]; inSession { + continue + } + } + tidx, ok := topicIdx[c.topicID] + if !ok { + tidx = len(req.Topics) + topicIdx[c.topicID] = tidx + req.Topics = append(req.Topics, kmsg.ShareFetchRequestTopic{ + TopicID: c.topicID, + }) + } + partIdx[tidp{c.topicID, c.partition}] = len(req.Topics[tidx].Partitions) + req.Topics[tidx].Partitions = append(req.Topics[tidx].Partitions, kmsg.ShareFetchRequestTopicPartition{ + Partition: c.partition, + PartitionMaxBytes: sc.cfg.maxPartBytes.load(), + }) + } + + // Piggyback acks (epoch 0 is a fresh session; the caller + // handles the drop + callback). If any renew entries are + // present, the pre-flight in shareFetch sends all acks via + // standalone ShareAcknowledge and rebuilds without acks. + if epoch > 0 && len(piggybackAcks) > 0 { + sentPiggyback = make(map[tidp]int, len(piggybackAcks)) + for i := range piggybackAcks { + d := &piggybackAcks[i] + ranges, _ := buildAckRanges(d.entries, d.gaps) // renew unnecessary per comment just above + if len(ranges) == 0 { + continue + } + tid := d.cursor.topicID + partition := d.cursor.partition + sentPiggyback[tidp{tid, partition}] = i + tidx, ok := topicIdx[tid] + if !ok { + tidx = len(req.Topics) + topicIdx[tid] = tidx + req.Topics = append(req.Topics, kmsg.ShareFetchRequestTopic{TopicID: tid}) + } + rt := &req.Topics[tidx] + key := tidp{tid, partition} + pi, ok := partIdx[key] + if !ok { + pi = len(rt.Partitions) + partIdx[key] = pi + rt.Partitions = append(rt.Partitions, kmsg.ShareFetchRequestTopicPartition{ + Partition: partition, + }) + } + rp := &rt.Partitions[pi] + for _, r := range ranges { + rp.AcknowledgementBatches = append(rp.AcknowledgementBatches, + kmsg.ShareFetchRequestTopicPartitionAcknowledgementBatch{ + FirstOffset: r.firstOffset, + LastOffset: r.lastOffset, + AcknowledgeTypes: ackTypes(r.ackType), + }) + } + } + } + + // Forget partitions no longer wanted (sessionParts - wantSet). + for _, key := range toForget { + var found bool + for i := range req.ForgottenTopicsData { + if req.ForgottenTopicsData[i].TopicID == key.id { + req.ForgottenTopicsData[i].Partitions = append(req.ForgottenTopicsData[i].Partitions, key.p) + found = true + break + } + } + if !found { + req.ForgottenTopicsData = append(req.ForgottenTopicsData, + kmsg.ShareFetchRequestForgottenTopicsData{ + TopicID: key.id, + Partitions: []int32{key.p}, + }) + } + } + + return +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/errors.go b/vendor/github.com/twmb/franz-go/pkg/kgo/errors.go new file mode 100644 index 00000000000..131c47485b6 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/errors.go @@ -0,0 +1,455 @@ +package kgo + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "syscall" + + "github.com/twmb/franz-go/pkg/kerr" +) + +// IsRetryableBrokerErr returns whether the client considers an error from a +// broker retrayble. This returns true specifically if the client thinks it can +// retry whatever it was just trying to do with a broker. It returns false in +// all other cases. +// +// This can used external to the library to help filter errors if use kgo +// hooks: errors may be sent to hooks before the client retries whatever it was +// just attempting. +func IsRetryableBrokerErr(err error) bool { + return isRetryableBrokerErr(err) +} + +func isRetryableBrokerErr(err error) bool { + // The error could be nil if we are evaluating multiple errors at once, + // and only one is non-nil. The intent of this function is to evaluate + // whether an **error** is retryable, not a non-error. We return that + // nil is not retryable -- the calling code evaluating multiple errors + // at once would not call into this function if all errors were nil. + if err == nil { + return false + } + // https://github.com/golang/go/issues/45729 + // + // Temporary is relatively useless. We will still check for the + // temporary interface, and in all cases, even with timeouts, we want + // to retry. + // + // More generally, we will retry for any error that unwraps into an + // os.SyscallError. Looking at Go's net package, the error we care + // about is net.OpError. Looking into that further, any error that + // reaches into the operating system return a syscall error, which is + // then put in net.OpError's Err field as an os.SyscallError. There are + // a few non-os.SyscallError errors, these are where Go itself detects + // a hard failure. We do not retry those. + // + // We blanket retry os.SyscallError because a lot of the times, what + // appears as a hard failure can actually be retried. For example, a + // failed dial can be retried, maybe the resolver temporarily had a + // problem. + // + // We favor testing os.SyscallError first, because net.OpError _always_ + // implements Temporary, so if we test that first, it'll return false + // in many cases when we want to return true from os.SyscallError. + if se := (*os.SyscallError)(nil); errors.As(err, &se) { + // Non-timeout dial errors are deliberately *not* retryable here. + // The carve-out forces every caller that wants dial-error retry + // behavior to opt in explicitly, because the right recovery + // strategy varies by call site: + // + // - Single bad seed bootstrap: should fail fast so the user + // finds out about the typo'd address. + // - cl.broker() unpinned admin: should rotate to a different + // broker via shouldRetryNext, which calls cl.broker()'s + // built-in rotation. + // - Cached controller/coordinator: should clear the cache and + // re-resolve, then retry (handled by failDial). + // - Broker pinned by ID (Broker.RetriableRequest): should retry + // the same broker bounded by retryTimeout (also failDial). + // - Sink (produce) and source (fetch): should refresh metadata + // and remap to the new partition leader (handled at the + // sink/source call sites). + // + // Returning true here would lump all of these into a generic + // retry loop and silently mask the call-site recovery behavior. + return !isDialNonTimeoutErr(err) + } + // EOF can be returned if a broker kills a connection unexpectedly, and + // we can retry that. Same for ErrClosed. + if errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) { + // If the FIRST read is EOF, that is usually not a good sign, + // often it's from bad SASL. We err on the side of pessimism + // and do not retry. + if ee := (*ErrFirstReadEOF)(nil); errors.As(err, &ee) && !ee.retry { + return false + } + return true + } + // We could have a retryable producer ID failure, which then bubbled up + // as errProducerIDLoadFail so as to be retried later. + if pe := (*errProducerIDLoadFail)(nil); errors.As(err, &pe) { + return true + } + // We could have chosen a broker, and then a concurrent metadata update + // could have removed it. + if errors.Is(err, errChosenBrokerDead) { + return true + } + // A broker kept giving us short sasl lifetimes, so we killed the + // connection ourselves. We can retry on a new connection. + if errors.Is(err, errSaslReauthLoop) { + return true + } + // We really should not get correlation mismatch, but if we do, we can + // retry. + if errors.Is(err, errCorrelationIDMismatch) { + return true + } + // We sometimes load the controller before issuing requests, and the + // cluster may not yet be ready and will return -1 for the controller. + // We can backoff and retry and hope the cluster has stabilized. + if ce := (*errUnknownController)(nil); errors.As(err, &ce) { + return true + } + // Same thought for a non-existing coordinator. + if ce := (*errUnknownCoordinator)(nil); errors.As(err, &ce) { + return true + } + var tempErr interface{ Temporary() bool } + if errors.As(err, &tempErr) { + return tempErr.Temporary() + } + return false +} + +func isDialNonTimeoutErr(err error) bool { + var ne *net.OpError + return errors.As(err, &ne) && ne.Op == "dial" && !ne.Timeout() +} + +func isAnyDialErr(err error) bool { + var ne *net.OpError + return errors.As(err, &ne) && ne.Op == "dial" +} + +// isPermanentDialErr reports whether a dial error is a hard configuration +// problem that no amount of retrying will fix. We treat the following as +// permanent: +// +// - DNS NXDOMAIN: the host genuinely does not exist. +// - EACCES / EPERM: local socket permission denied. +// +// Everything else dial-shaped (ECONNREFUSED, EHOSTUNREACH, ENETUNREACH, +// dial timeouts, ...) is considered transient. ECONNREFUSED in particular +// is the canonical signal that a broker is mid-restart (the listener is +// closed before the JVM exits, and not yet bound after it starts) and is +// the most common dial error worth retrying. +func isPermanentDialErr(err error) bool { + if !isAnyDialErr(err) { + return false + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && dnsErr.IsNotFound { + return true + } + if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) { + return true + } + return false +} + +func isContextErr(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +func isSkippableBrokerErr(err error) bool { + // Some broker errors are not retryable for the given broker itself, + // but we *could* skip the broker and try again on the next broker. For + // example, if the user input an invalid address and a valid address + // for seeds, when we fail dialing the first seed, we cannot retry that + // broker, but we can skip to the next. + // + // We take anything that returns an OpError that *is not* a context + // error deep inside. + if errors.Is(err, errUnknownBroker) { + return true + } + var ne *net.OpError + if errors.As(err, &ne) && !isContextErr(err) { + return true + } + return false +} + +var ( + ////////////// + // INTERNAL // -- when used multiple times or checked in different areas of the client + ////////////// + + // Returned when issuing a request to a broker that the client does not + // know about (maybe missing from metadata responses now). + errUnknownBroker = errors.New("unknown broker") + + // A temporary error returned when a broker connection has died, + // either from a metadata update or from the connection closing + // while a request was in-flight. + errChosenBrokerDead = errors.New("the broker connection has died and the request will be retried on a new connection") + + // If a broker repeatedly gives us tiny sasl lifetimes, we fail a + // request after a few tries to forcefully kill the connection and + // restart a new connection ourselves. + errSaslReauthLoop = errors.New("the broker is repeatedly giving us sasl lifetimes that are too short to write a request") + + // A temporary error returned when Kafka replies with a different + // correlation ID than we were expecting for the request the client + // issued. + // + // If this error happens, the client closes the broker connection. + errCorrelationIDMismatch = errors.New("correlation ID mismatch") + + // Returned when using a kmsg.Request with a key larger than kmsg.MaxKey. + errUnknownRequestKey = errors.New("request key is unknown") + + // Returned if a connection has loaded broker ApiVersions and knows + // that the broker cannot handle the request to-be-issued request. + errBrokerTooOld = errors.New("broker is too old; the broker has already indicated it will not know how to handle the request") + + // Returned when trying to call group functions when the client is not + // assigned a group. + errNotGroup = errors.New("invalid group function call when not assigned a group") + + // Returned when trying to begin a transaction with a client that does + // not have a transactional ID. + errNotTransactional = errors.New("invalid attempt to begin a transaction with a non-transactional client") + + // Returned when trying to produce a record outside of a transaction. + errNotInTransaction = errors.New("cannot produce record transactionally if not in a transaction") + + errNoTopic = errors.New("cannot produce record with no topic and no default topic") + + // Returned for all buffered produce records when a user purges topics. + errPurged = errors.New("topic purged while buffered") + + errMissingMetadataPartition = errors.New("metadata update is missing a partition that we were previously using") + + errNoCommittedOffset = errors.New("partition has no prior committed offset") + + // Returned by the 848 heartbeat closure when it detects an assignment + // change. The heartbeat loop treats this like RebalanceInProgress but + // suppresses further heartbeat requests so that a second heartbeat + // cannot see stale assignment state and miss a revocation. + errReassigned848 = errors.New("848 reassignment detected") + + ////////////// + // EXTERNAL // + ////////////// + + // ErrRecordTimeout is passed to produce promises when records are + // unable to be produced within the RecordDeliveryTimeout. + ErrRecordTimeout = errors.New("records have timed out before they were able to be produced") + + // ErrRecordRetries is passed to produce promises when records are + // unable to be produced after RecordRetries attempts. + ErrRecordRetries = errors.New("record failed after being retried too many times") + + // ErrMaxBuffered is returned when the maximum amount of records are + // buffered and either manual flushing is enabled or you are using + // TryProduce. + ErrMaxBuffered = errors.New("the maximum amount of records are buffered, cannot buffer more") + + // ErrAborting is returned for all buffered records while + // AbortBufferedRecords is being called. + ErrAborting = errors.New("client is aborting buffered records") + + // ErrClientClosed is returned in various places when the client's + // Close function has been called. + // + // For producing, records are failed with this error. + // + // For consuming, a fake partition is injected into a poll response + // that has this error. + // + // For any request, the request is failed with this error. + ErrClientClosed = errors.New("client closed") +) + +// ErrFirstReadEOF is returned for responses that immediately error with +// io.EOF. This is the client's guess as to why a read from a broker is +// failing with io.EOF. Two cases are currently handled, +// +// - When the client is using TLS but brokers are not, brokers close +// connections immediately because the incoming request looks wrong. +// - When SASL is required but missing, brokers close connections immediately. +// +// There may be other reasons that an immediate io.EOF is encountered (perhaps +// the connection truly was severed before a response was received), but this +// error can help you quickly check common problems. +type ErrFirstReadEOF struct { + kind uint8 + err error + retry bool +} + +type errProducerIDLoadFail struct { + err error +} + +func (e *errProducerIDLoadFail) Error() string { + if e.err == nil { + return "unable to initialize a producer ID due to request failures" + } + return fmt.Sprintf("unable to initialize a producer ID due to request failures: %v", e.err) +} + +func (e *errProducerIDLoadFail) Unwrap() error { return e.err } + +const ( + firstReadDial uint8 = iota + firstReadTLS + firstReadSASL +) + +func (e *ErrFirstReadEOF) Error() string { + switch e.kind { + case firstReadDial: + return "broker closed the connection immediately after a dial, which often happens if the client is using TLS when the broker is not expecting it: is TLS misconfigured on the client or the broker?" + case firstReadTLS: + return "broker closed the connection immediately during api versions negotiation, which often happens when the broker requires TLS but the client is using plaintext: is TLS missing?" + default: // firstReadSASL + return "broker closed the connection immediately after a request was issued, which often happens when SASL is required but not provided: is SASL missing?" + } +} + +// Unwrap returns io.EOF (or, if a custom dialer returned a wrapped io.EOF, +// this returns the custom dialer's wrapped error). +func (e *ErrFirstReadEOF) Unwrap() error { return e.err } + +// ErrDataLoss is returned for Kafka >=2.1 when data loss is detected and the +// client is able to reset to the last valid offset. +type ErrDataLoss struct { + // Topic is the topic data loss was detected on. + Topic string + // Partition is the partition data loss was detected on. + Partition int32 + // ConsumedTo is what the client had consumed to for this partition before + // data loss was detected. + ConsumedTo int64 + // ConsumedToEpoch is the epoch for the offset the client was currently + // consuming. + ConsumedToEpoch int32 + // ResetTo is what the client reset the partition to; everything from + // ResetTo to ConsumedTo was lost. + ResetTo int64 + // ResetToEpoch is the epoch the client was reset to. + ResetToEpoch int32 +} + +func (e *ErrDataLoss) Error() string { + return fmt.Sprintf("topic %s partition %d lost records;"+ + " the client consumed to offset %d epoch %d but was reset to offset %d epoch %d", + e.Topic, e.Partition, e.ConsumedTo, e.ConsumedToEpoch, e.ResetTo, e.ResetToEpoch) +} + +type errUnknownController struct { + id int32 +} + +func (e *errUnknownController) Error() string { + if e.id == -1 { + return "broker replied that the controller broker is not available" + } + return fmt.Sprintf("broker replied that the controller broker is %d,"+ + " but did not reply with that broker in the broker list", e.id) +} + +type errUnknownCoordinator struct { + coordinator int32 + key coordinatorKey +} + +func (e *errUnknownCoordinator) Error() string { + switch e.key.typ { + case coordinatorTypeGroup: + return fmt.Sprintf("broker replied that group %s has broker coordinator %d,"+ + " but did not reply with that broker in the broker list", + e.key.name, e.coordinator) + case coordinatorTypeTxn: + return fmt.Sprintf("broker replied that txn id %s has broker coordinator %d,"+ + " but did not reply with that broker in the broker list", + e.key.name, e.coordinator) + default: + return fmt.Sprintf("broker replied to an unknown coordinator key %s (type %d) that it has a broker coordinator %d,"+ + " but did not reply with that broker in the broker list", e.key.name, e.key.typ, e.coordinator) + } +} + +// ErrGroupSession is injected into a poll if an error occurred such that your +// consumer group member was kicked from the group or was never able to join +// the group. +type ErrGroupSession struct { + Err error +} + +func (e *ErrGroupSession) Error() string { + return fmt.Sprintf("unable to join group session: %v", e.Err) +} + +func (e *ErrGroupSession) Unwrap() error { return e.Err } + +type errDecompress struct { + err error +} + +func (e *errDecompress) Error() string { + return fmt.Sprintf("unable to decompress batch: %v", e.err) +} + +func (e *errDecompress) Unwrap() error { return e.err } + +func isDecompressErr(err error) bool { + var ed *errDecompress + return errors.As(err, &ed) +} + +func errCodeMessage(code int16, errMessage *string) error { + if err := kerr.ErrorForCode(code); err != nil { + if errMessage != nil { + return fmt.Errorf("%w: %s", err, *errMessage) + } + return err + } + return nil +} + +type errApiVersionsReset struct { + err error +} + +// errShareConsumerLeft is reported via shareAckCallback for any +// Record.Ack or MarkAcks call made after LeaveGroup has begun +// closing the share consumer. Unexported on purpose: there is no +// importable sentinel for users to errors.Is against, because the +// situation is non-actionable -- the broker session is gone. Users +// should treat this as fatal-for-this-record and not retry. +var errShareConsumerLeft = errors.New("share consumer has left the group; ack will not be delivered") + +// errBrokerOmittedAckPartition is reported via shareAckCallback when +// the broker's ShareFetch or ShareAcknowledge response did not echo a +// partition we sent acks for. The broker did not return a Kafka error +// code for the partition; it simply omitted it from the response, +// which is a protocol violation. Unexported on purpose: like +// errShareConsumerLeft, there is no importable sentinel for users to +// errors.Is against. The condition is non-actionable per-record; +// the right user response is to log and treat the ack as failed. +// +// We previously surfaced this as kerr.UnknownServerError, which was +// misleading because no error code was returned by the broker. The +// dedicated sentinel makes the actual situation explicit. +var errBrokerOmittedAckPartition = errors.New("broker omitted partition from share fetch response that we sent acks for") + +func (e *errApiVersionsReset) Error() string { return e.err.Error() } +func (e *errApiVersionsReset) Unwrap() error { return e.err } diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/group_balancer.go b/vendor/github.com/twmb/franz-go/pkg/kgo/group_balancer.go new file mode 100644 index 00000000000..a754ee67972 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/group_balancer.go @@ -0,0 +1,1068 @@ +package kgo + +import ( + "bytes" + "fmt" + "slices" + "sort" + "strings" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/sticky" + "github.com/twmb/franz-go/pkg/kmsg" +) + +// GroupBalancer balances topics and partitions among group members. +// +// A GroupBalancer is roughly equivalent to Kafka's PartitionAssignor. +type GroupBalancer interface { + // ProtocolName returns the name of the protocol, e.g. roundrobin, + // range, sticky. + ProtocolName() string + + // JoinGroupMetadata returns the metadata to use in JoinGroup, given + // the topic interests and the current assignment and group generation. + // + // It is safe to modify the input topics and currentAssignment. The + // input topics are guaranteed to be sorted, as are the partitions for + // each topic in currentAssignment. It is recommended for your output + // to be ordered by topic and partitions. Since Kafka uses the output + // from this function to determine whether a rebalance is needed, a + // deterministic output will avoid accidental rebalances. + JoinGroupMetadata( + topicInterests []string, + currentAssignment map[string][]int32, + generation int32, + ) []byte + + // ParseSyncAssignment returns assigned topics and partitions from an + // encoded SyncGroupResponse's MemberAssignment. + ParseSyncAssignment(assignment []byte) (map[string][]int32, error) + + // MemberBalancer returns a GroupMemberBalancer for the given group + // members, as well as the topics that all the members are interested + // in. If the client does not have some topics in the returned topics, + // the client issues a metadata request to load the number of + // partitions in those topics before calling the GroupMemberBalancer's + // Balance function. + // + // The input group members are guaranteed to be sorted first by + // instance ID, if non-nil, and then by member ID. + // + // It is up to the user to decide how to decode each member's + // ProtocolMetadata field. The default client group protocol of + // "consumer" by default uses join group metadata's of type + // kmsg.ConsumerMemberMetadata. If this is the case for you, it may be + // useful to use the ConsumerBalancer type to help parse the metadata + // and balance. + // + // If the member metadata cannot be deserialized correctly, this should + // return a relevant error. + MemberBalancer(members []kmsg.JoinGroupResponseMember) (b GroupMemberBalancer, topics map[string]struct{}, err error) + + // IsCooperative returns if this is a cooperative balance strategy. + IsCooperative() bool +} + +// GroupMemberBalancer balances topics amongst group members. If your balancing +// can fail, you can implement GroupMemberBalancerOrError. +type GroupMemberBalancer interface { + // Balance balances topics and partitions among group members, where + // the int32 in the topics map corresponds to the number of partitions + // known to be in each topic. + Balance(topics map[string]int32) IntoSyncAssignment +} + +// GroupMemberBalancerOrError is an optional extension interface for +// GroupMemberBalancer. This can be implemented if your balance function can +// fail. +// +// For interface purposes, it is required to implement GroupMemberBalancer, but +// Balance will never be called. +type GroupMemberBalancerOrError interface { + GroupMemberBalancer + BalanceOrError(topics map[string]int32) (IntoSyncAssignment, error) +} + +// IntoSyncAssignment takes a balance plan and returns a list of assignments to +// use in a kmsg.SyncGroupRequest. +// +// It is recommended to ensure the output is deterministic and ordered by +// member / topic / partitions. +type IntoSyncAssignment interface { + IntoSyncAssignment() []kmsg.SyncGroupRequestGroupAssignment +} + +// ConsumerBalancer is a helper type for writing balance plans that use the +// "consumer" protocol, such that each member uses a kmsg.ConsumerMemberMetadata +// in its join group request. +type ConsumerBalancer struct { + b ConsumerBalancerBalance + members []kmsg.JoinGroupResponseMember + metadatas []kmsg.ConsumerMemberMetadata + topics map[string]struct{} + + // partitionRacks maps topic => partition index => leader rack. + // nil when rack-aware assignment is not active. + partitionRacks map[string][]string + + err error +} + +// Balance satisfies the GroupMemberBalancer interface, but is never called +// because GroupMemberBalancerOrError exists. +func (*ConsumerBalancer) Balance(map[string]int32) IntoSyncAssignment { + panic("unreachable") +} + +// BalanceOrError satisfies the GroupMemberBalancerOrError interface. +func (b *ConsumerBalancer) BalanceOrError(topics map[string]int32) (IntoSyncAssignment, error) { + return b.b.Balance(b, topics), b.err +} + +// Members returns the list of input members for this group balancer. +func (b *ConsumerBalancer) Members() []kmsg.JoinGroupResponseMember { + return b.members +} + +// EachMember calls fn for each member and its corresponding metadata in the +// consumer group being balanced. +func (b *ConsumerBalancer) EachMember(fn func(member *kmsg.JoinGroupResponseMember, meta *kmsg.ConsumerMemberMetadata)) { + for i := range b.members { + fn(&b.members[i], &b.metadatas[i]) + } +} + +// MemberAt returns the nth member and its corresponding metadata. +func (b *ConsumerBalancer) MemberAt(n int) (*kmsg.JoinGroupResponseMember, *kmsg.ConsumerMemberMetadata) { + return &b.members[n], &b.metadatas[n] +} + +// SetError allows you to set any error that occurred while balancing. This +// allows you to fail balancing and return nil from Balance. +func (b *ConsumerBalancer) SetError(err error) { + b.err = err +} + +// PartitionRacks returns the partition rack mapping for rack-aware assignment +// (KIP-881). The map is topic => partition index => leader rack. Returns nil +// if no rack info is available. +func (b *ConsumerBalancer) PartitionRacks() map[string][]string { + return b.partitionRacks +} + +// MemberTopics returns the unique set of topics that all members are +// interested in. +// +// This can safely be called if the balancer is nil; if so, this will return +// nil. +func (b *ConsumerBalancer) MemberTopics() map[string]struct{} { + if b == nil { + return nil + } + return b.topics +} + +// NewPlan returns a type that can be used to build a balance plan. The return +// satisfies the IntoSyncAssignment interface. +func (b *ConsumerBalancer) NewPlan() *BalancePlan { + plan := make(map[string]map[string][]int32, len(b.members)) + for i := range b.members { + plan[b.members[i].MemberID] = make(map[string][]int32) + } + return &BalancePlan{plan} +} + +// ConsumerBalancerBalance is what the ConsumerBalancer invokes to balance a +// group. +// +// This is a complicated interface, but in short, this interface has one +// function that implements the actual balancing logic: using the input +// balancer, balance the input topics and partitions. If your balancing can +// fail, you can use ConsumerBalancer.SetError(...) to return an error from +// balancing, and then you can simply return nil from Balance. +type ConsumerBalancerBalance interface { + Balance(*ConsumerBalancer, map[string]int32) IntoSyncAssignment +} + +// ParseConsumerSyncAssignment parses `assignment` as kmsg.ConsumerMemberAssignment +// and returns the mapped topic => partitions assignment. +func ParseConsumerSyncAssignment(assignment []byte) (map[string][]int32, error) { + var kassignment kmsg.ConsumerMemberAssignment + if err := kassignment.ReadFrom(assignment); err != nil { + return nil, fmt.Errorf("sync assignment parse failed: %v", err) + } + + m := make(map[string][]int32, len(kassignment.Topics)) + for _, topic := range kassignment.Topics { + m[topic.Topic] = topic.Partitions + } + return m, nil +} + +// NewConsumerBalancer parses the each member's metadata as a +// kmsg.ConsumerMemberMetadata and returns a ConsumerBalancer to use in balancing. +// +// If any metadata parsing fails, this returns an error. +func NewConsumerBalancer(balance ConsumerBalancerBalance, members []kmsg.JoinGroupResponseMember) (*ConsumerBalancer, error) { + b := &ConsumerBalancer{ + b: balance, + members: members, + metadatas: make([]kmsg.ConsumerMemberMetadata, len(members)), + topics: make(map[string]struct{}), + } + + for i, member := range members { + meta := &b.metadatas[i] + meta.Default() + memberMeta := member.ProtocolMetadata + if err := meta.ReadFrom(memberMeta); err != nil { + // Some buggy clients claimed support for v1 but then + // did not add OwnedPartitions, resulting in a short + // metadata. If we fail at reading and the version is + // v1, we retry again as v0. We do not support other + // versions because hopefully other clients stop + // claiming higher and higher version support and not + // actually supporting them. Sarama has a similarish + // workaround. See #493. + if bytes.HasPrefix(memberMeta, []byte{0, 1}) { + memberMeta[0] = 0 + memberMeta[1] = 0 + if err = meta.ReadFrom(memberMeta); err != nil { + return nil, fmt.Errorf("unable to read member metadata: %v", err) + } + } + } + for _, topic := range meta.Topics { + b.topics[topic] = struct{}{} + } + sort.Strings(meta.Topics) + } + + return b, nil +} + +// BalancePlan is a helper type to build the result of balancing topics +// and partitions among group members. +type BalancePlan struct { + plan map[string]map[string][]int32 // member => topic => partitions +} + +// AsMemberIDMap returns the plan as a map of member IDs to their topic & +// partition assignments. +// +// Internally, a BalancePlan is currently represented as this map. Any +// modification to the map modifies the plan. The internal representation of a +// plan may change in the future to include more metadata. If this happens, the +// map returned from this function may not represent all aspects of a plan. +// The client will attempt to mirror modifications to the map directly back +// into the underlying plan as best as possible. +func (p *BalancePlan) AsMemberIDMap() map[string]map[string][]int32 { + return p.plan +} + +func (p *BalancePlan) String() string { + var sb strings.Builder + + var membersWritten int + for member, topics := range p.plan { + membersWritten++ + sb.WriteString(member) + sb.WriteString("{") + + var topicsWritten int + for topic, partitions := range topics { + fmt.Fprintf(&sb, "%s%v", topic, partitions) + topicsWritten++ + if topicsWritten < len(topics) { + sb.WriteString(", ") + } + } + + sb.WriteString("}") + if membersWritten < len(p.plan) { + sb.WriteString(", ") + } + } + + return sb.String() +} + +// AddPartition assigns a partition for the topic to a given member. +func (p *BalancePlan) AddPartition(member *kmsg.JoinGroupResponseMember, topic string, partition int32) { + memberPlan := p.plan[member.MemberID] + memberPlan[topic] = append(memberPlan[topic], partition) +} + +// AddPartitions assigns many partitions for a topic to a given member. +func (p *BalancePlan) AddPartitions(member *kmsg.JoinGroupResponseMember, topic string, partitions []int32) { + memberPlan := p.plan[member.MemberID] + memberPlan[topic] = append(memberPlan[topic], partitions...) +} + +// IntoSyncAssignment satisfies the IntoSyncAssignment interface. +func (p *BalancePlan) IntoSyncAssignment() []kmsg.SyncGroupRequestGroupAssignment { + kassignments := make([]kmsg.SyncGroupRequestGroupAssignment, 0, len(p.plan)) + for member, assignment := range p.plan { + var kassignment kmsg.ConsumerMemberAssignment + for topic, partitions := range assignment { + slices.Sort(partitions) + assnTopic := kmsg.NewConsumerMemberAssignmentTopic() + assnTopic.Topic = topic + assnTopic.Partitions = partitions + kassignment.Topics = append(kassignment.Topics, assnTopic) + } + sort.Slice(kassignment.Topics, func(i, j int) bool { return kassignment.Topics[i].Topic < kassignment.Topics[j].Topic }) + syncAssn := kmsg.NewSyncGroupRequestGroupAssignment() + syncAssn.MemberID = member + syncAssn.MemberAssignment = kassignment.AppendTo(nil) + kassignments = append(kassignments, syncAssn) + } + sort.Slice(kassignments, func(i, j int) bool { return kassignments[i].MemberID < kassignments[j].MemberID }) + return kassignments +} + +func joinMemberLess(l, r *kmsg.JoinGroupResponseMember) bool { + if l.InstanceID != nil { + if r.InstanceID == nil { + return true + } + return *l.InstanceID < *r.InstanceID + } + if r.InstanceID != nil { + return false + } + return l.MemberID < r.MemberID +} + +func sortJoinMembers(members []kmsg.JoinGroupResponseMember) { + sort.Slice(members, func(i, j int) bool { return joinMemberLess(&members[i], &members[j]) }) +} + +func sortJoinMemberPtrs(members []*kmsg.JoinGroupResponseMember) { + sort.Slice(members, func(i, j int) bool { return joinMemberLess(members[i], members[j]) }) +} + +func (g *groupConsumer) findBalancer(from, proto string) (GroupBalancer, error) { + for _, b := range g.cfg.balancers { + if b.ProtocolName() == proto { + return b, nil + } + } + var ours []string + for _, b := range g.cfg.balancers { + ours = append(ours, b.ProtocolName()) + } + g.cl.cfg.logger.Log(LogLevelError, fmt.Sprintf("%s could not find broker-chosen balancer", from), "kafka_choice", proto, "our_set", strings.Join(ours, ", ")) + return nil, fmt.Errorf("unable to balance: none of our balancers have a name equal to the balancer chosen for balancing (%s)", proto) +} + +// balanceGroup returns a balancePlan from a join group response. +// +// If the group has topics this leader does not want to consume, this also +// returns all topics and partitions; the leader will then periodically do its +// own metadata update to see if partition counts have changed for these random +// topics. +func (g *groupConsumer) balanceGroup(proto string, members []kmsg.JoinGroupResponseMember, skipBalance bool) ([]kmsg.SyncGroupRequestGroupAssignment, error) { + g.cl.cfg.logger.Log(LogLevelInfo, "balancing group as leader") + + b, err := g.findBalancer("balance group", proto) + if err != nil { + return nil, err + } + + sortJoinMembers(members) + + memberBalancer, topics, err := b.MemberBalancer(members) + if err != nil { + return nil, fmt.Errorf("unable to create group member balancer: %v", err) + } + + myTopics := g.tps.load() + var needMeta bool + topicPartitionCount := make(map[string]int32, len(topics)) + for topic := range topics { + data, exists := myTopics[topic] + if !exists { + needMeta = true + continue + } + topicPartitionCount[topic] = int32(len(data.load().partitions)) + } + + // If our consumer metadata does not contain all topics, the group is + // expressing interests in topics we are not consuming. Perhaps we have + // those topics saved in our external topics map. + if needMeta { + g.loadExternal().fn(func(m map[string]int32) { + needMeta = false + for topic := range topics { + partitions, exists := m[topic] + if !exists { + needMeta = true + continue + } + topicPartitionCount[topic] = partitions + } + }) + } + + if needMeta { + g.cl.cfg.logger.Log(LogLevelInfo, "group members indicated interest in topics the leader is not assigned, fetching metadata for all group topics") + var metaTopics []string + for topic := range topics { + metaTopics = append(metaTopics, topic) + } + + _, resp, err := g.cl.fetchMetadataByName(g.ctx, false, metaTopics, nil) + if err != nil { + return nil, fmt.Errorf("unable to fetch metadata for group topics: %v", err) + } + for i := range resp.Topics { + t := &resp.Topics[i] + if t.Topic == nil { + g.cl.cfg.logger.Log(LogLevelWarn, "metadata resp in balance for topic has nil topic, skipping...", "err", kerr.ErrorForCode(t.ErrorCode)) + continue + } + if t.ErrorCode != 0 { + g.cl.cfg.logger.Log(LogLevelWarn, "metadata resp in balance for topic has error, skipping...", "topic", t.Topic, "err", kerr.ErrorForCode(t.ErrorCode)) + continue + } + topicPartitionCount[*t.Topic] = int32(len(t.Partitions)) + } + + g.initExternal(topicPartitionCount) + } + + // KIP-881: build partition rack info for rack-aware assignment. + // We use cached broker racks and partition leaders from local + // metadata. This requires no extra fetches. + if cb, ok := memberBalancer.(*ConsumerBalancer); ok { + cb.partitionRacks = g.buildPartitionRacks(cb, topicPartitionCount) + } + + // If the returned balancer is a ConsumerBalancer (which it likely + // always will be), then we can print some useful debugging information + // about what member interests are. + if b, ok := memberBalancer.(*ConsumerBalancer); ok { + interests := new(bytes.Buffer) + b.EachMember(func(member *kmsg.JoinGroupResponseMember, meta *kmsg.ConsumerMemberMetadata) { + interests.Reset() + fmt.Fprintf(interests, "interested topics: %v, previously owned: ", meta.Topics) + for _, owned := range meta.OwnedPartitions { + slices.Sort(owned.Partitions) + fmt.Fprintf(interests, "%s%v, ", owned.Topic, owned.Partitions) + } + strInterests := interests.String() + strInterests = strings.TrimSuffix(strInterests, ", ") + + if member.InstanceID == nil { + g.cl.cfg.logger.Log(LogLevelInfo, "balance group member", "id", member.MemberID, "interests", strInterests) + } else { + g.cl.cfg.logger.Log(LogLevelInfo, "balance group member", "id", member.MemberID, "instance_id", *member.InstanceID, "interests", strInterests) + } + }) + } else { + g.cl.cfg.logger.Log(LogLevelInfo, "unable to log information about group member interests: the user has defined a custom balancer (not a *ConsumerBalancer)") + } + + // KIP-814: we are leader and we know what the entire group is + // consuming. Crucially, we parsed topics that we are potentially not + // interested in and are now tracking them for metadata updates. We + // have logged the current interests, we do not need to actually + // balance. + if skipBalance { + switch proto := b.ProtocolName(); proto { + case RangeBalancer().ProtocolName(), + RoundRobinBalancer().ProtocolName(), + StickyBalancer().ProtocolName(), + CooperativeStickyBalancer().ProtocolName(): + default: + return nil, nil + } + } + + // If the returned IntoSyncAssignment is a BalancePlan, which it likely + // is if the balancer is a ConsumerBalancer, then we can again print + // more useful debugging information. + var into IntoSyncAssignment + if memberBalancerOrErr, ok := memberBalancer.(GroupMemberBalancerOrError); ok { + if into, err = memberBalancerOrErr.BalanceOrError(topicPartitionCount); err != nil { + g.cl.cfg.logger.Log(LogLevelError, "balance failed", "err", err) + return nil, err + } + } else { + into = memberBalancer.Balance(topicPartitionCount) + } + + if p, ok := into.(*BalancePlan); ok { + g.cl.cfg.logger.Log(LogLevelInfo, "balanced", "plan", p.String()) + } else { + g.cl.cfg.logger.Log(LogLevelInfo, "unable to log balance plan: the user has returned a custom IntoSyncAssignment (not a *BalancePlan)") + } + + return into.IntoSyncAssignment(), nil +} + +// buildPartitionRacks builds a topic => partition => rack map for rack-aware +// assignment (KIP-881). It uses cached broker racks and partition leader info +// from local metadata. Returns nil if no rack info is available. +func (g *groupConsumer) buildPartitionRacks(b *ConsumerBalancer, topicPartitionCount map[string]int32) map[string][]string { + // Check if any member has a rack. + var hasRack bool + for i := range b.metadatas { + if b.metadatas[i].Rack != nil { + hasRack = true + break + } + } + if !hasRack { + return nil + } + + // Get broker ID => rack from cached broker metadata. + brokerRacks := g.cl.brokerRacks() + if len(brokerRacks) == 0 { + return nil + } + + // Build partition racks from local topic metadata. Each + // partition's rack is determined by its leader broker's rack. + partitionRacks := make(map[string][]string, len(topicPartitionCount)) + myTopics := g.tps.load() + for topic, numPartitions := range topicPartitionCount { + racks := make([]string, numPartitions) + if data, ok := myTopics[topic]; ok { + tpd := data.load() + for i, p := range tpd.partitions { + if p == nil || int32(i) >= numPartitions { + continue + } + if rack, ok := brokerRacks[p.leader]; ok { + racks[i] = rack + } + } + } + partitionRacks[topic] = racks + } + + // Only return rack info if at least one partition has a rack. + for _, racks := range partitionRacks { + for _, r := range racks { + if r != "" { + return partitionRacks + } + } + } + return nil +} + +// helper func; range and roundrobin use v0 +func simpleMemberMetadata(interests []string, generation int32) []byte { + meta := kmsg.NewConsumerMemberMetadata() + meta.Version = 3 // BUMP ME WHEN NEW FIELDS ARE ADDED, AND BUMP BELOW + meta.Topics = interests // input interests are already sorted + // meta.OwnedPartitions is nil, since simple protocols are not cooperative + meta.Generation = generation + return meta.AppendTo(nil) +} + +/////////////////// +// Balance Plans // +/////////////////// + +// RoundRobinBalancer returns a group balancer that evenly maps topics and +// partitions to group members. +// +// Suppose there are two members M0 and M1, two topics t0 and t1, and each +// topic has three partitions p0, p1, and p2. The partition balancing will be +// +// M0: [t0p0, t0p2, t1p1] +// M1: [t0p1, t1p0, t1p2] +// +// If all members subscribe to all topics equally, the roundrobin balancer +// will give a perfect balance. However, if topic subscriptions are quite +// unequal, the roundrobin balancer may lead to a bad balance. See KIP-49 +// for one example (note that the fair strategy mentioned in KIP-49 does +// not exist). +// +// This is equivalent to the Java roundrobin balancer. +func RoundRobinBalancer() GroupBalancer { + return new(roundRobinBalancer) +} + +type roundRobinBalancer struct{} + +func (*roundRobinBalancer) ProtocolName() string { return "roundrobin" } +func (*roundRobinBalancer) IsCooperative() bool { return false } +func (*roundRobinBalancer) JoinGroupMetadata(interests []string, _ map[string][]int32, generation int32) []byte { + return simpleMemberMetadata(interests, generation) +} + +func (*roundRobinBalancer) ParseSyncAssignment(assignment []byte) (map[string][]int32, error) { + return ParseConsumerSyncAssignment(assignment) +} + +func (r *roundRobinBalancer) MemberBalancer(members []kmsg.JoinGroupResponseMember) (GroupMemberBalancer, map[string]struct{}, error) { + b, err := NewConsumerBalancer(r, members) + return b, b.MemberTopics(), err +} + +func (*roundRobinBalancer) Balance(b *ConsumerBalancer, topics map[string]int32) IntoSyncAssignment { + type topicPartition struct { + topic string + partition int32 + } + var nparts int + for _, partitions := range topics { + nparts += int(partitions) + } + // Order all partitions available to balance, filtering out those that + // no members are subscribed to. + allParts := make([]topicPartition, 0, nparts) + for topic := range b.MemberTopics() { + for partition := int32(0); partition < topics[topic]; partition++ { + allParts = append(allParts, topicPartition{ + topic, + partition, + }) + } + } + sort.Slice(allParts, func(i, j int) bool { + l, r := allParts[i], allParts[j] + return l.topic < r.topic || l.topic == r.topic && l.partition < r.partition + }) + + plan := b.NewPlan() + // While parts are unassigned, assign them. + var memberIdx int + for len(allParts) > 0 { + next := allParts[0] + allParts = allParts[1:] + + // The Java roundrobin strategy walks members circularly until + // a member can take this partition, and then starts the next + // partition where the circular iterator left off. + assigned: + for { + member, meta := b.MemberAt(memberIdx) + memberIdx = (memberIdx + 1) % len(b.Members()) + for _, topic := range meta.Topics { + if topic == next.topic { + plan.AddPartition(member, next.topic, next.partition) + break assigned + } + } + } + } + + return plan +} + +// RangeBalancer returns a group balancer that, per topic, maps partitions to +// group members. Since this works on a topic level, uneven partitions per +// topic to the number of members can lead to slight partition consumption +// disparities. +// +// Suppose there are two members M0 and M1, two topics t0 and t1, and each +// topic has three partitions p0, p1, and p2. The partition balancing will be +// +// M0: [t0p0, t0p1, t1p0, t1p1] +// M1: [t0p2, t1p2] +// +// This is equivalent to the Java range balancer. +func RangeBalancer() GroupBalancer { + return new(rangeBalancer) +} + +type rangeBalancer struct{} + +func (*rangeBalancer) ProtocolName() string { return "range" } +func (*rangeBalancer) IsCooperative() bool { return false } +func (*rangeBalancer) JoinGroupMetadata(interests []string, _ map[string][]int32, generation int32) []byte { + return simpleMemberMetadata(interests, generation) +} + +func (*rangeBalancer) ParseSyncAssignment(assignment []byte) (map[string][]int32, error) { + return ParseConsumerSyncAssignment(assignment) +} + +func (r *rangeBalancer) MemberBalancer(members []kmsg.JoinGroupResponseMember) (GroupMemberBalancer, map[string]struct{}, error) { + b, err := NewConsumerBalancer(r, members) + return b, b.MemberTopics(), err +} + +func (*rangeBalancer) Balance(b *ConsumerBalancer, topics map[string]int32) IntoSyncAssignment { + // Build member rack lookup for rack-aware assignment. + memberRack := make(map[string]string, len(b.members)) + for i := range b.members { + if b.metadatas[i].Rack != nil { + memberRack[b.members[i].MemberID] = *b.metadatas[i].Rack + } + } + + topics2PotentialConsumers := make(map[string][]*kmsg.JoinGroupResponseMember) + b.EachMember(func(member *kmsg.JoinGroupResponseMember, meta *kmsg.ConsumerMemberMetadata) { + for _, topic := range meta.Topics { + topics2PotentialConsumers[topic] = append(topics2PotentialConsumers[topic], member) + } + }) + + plan := b.NewPlan() + for topic, potentialConsumers := range topics2PotentialConsumers { + sortJoinMemberPtrs(potentialConsumers) + numPartitions := int(topics[topic]) + nConsumers := len(potentialConsumers) + div, rem := numPartitions/nConsumers, numPartitions%nConsumers + + assigned := make([]bool, numPartitions) + assignCount := make([]int, nConsumers) + + // Phase 1: if rack info is available, assign rack-matching + // partitions first. This is a no-op when partitionRacks is nil. + if topicRacks := b.partitionRacks[topic]; topicRacks != nil { + for ci, consumer := range potentialConsumers { + rack := memberRack[consumer.MemberID] + if rack == "" { + continue + } + maxForConsumer := div + if ci < rem { + maxForConsumer++ + } + for p := 0; p < numPartitions && assignCount[ci] < maxForConsumer; p++ { + if assigned[p] { + continue + } + if p < len(topicRacks) && topicRacks[p] == rack { + plan.AddPartition(consumer, topic, int32(p)) + assigned[p] = true + assignCount[ci]++ + } + } + } + } + + // Phase 2: assign remaining partitions using range. + pIdx := 0 + for ci := 0; ci < nConsumers; ci++ { + maxForConsumer := div + if ci < rem { + maxForConsumer++ + } + quota := maxForConsumer - assignCount[ci] + for pIdx < numPartitions && quota > 0 { + if !assigned[pIdx] { + plan.AddPartition(potentialConsumers[ci], topic, int32(pIdx)) + quota-- + } + pIdx++ + } + } + } + + return plan +} + +// StickyBalancer returns a group balancer that ensures minimal partition +// movement on group changes while also ensuring optimal balancing. +// +// Suppose there are three members M0, M1, and M2, and two topics t0 and t1 +// each with three partitions p0, p1, and p2. If the initial balance plan looks +// like +// +// M0: [t0p0, t0p1, t0p2] +// M1: [t1p0, t1p1, t1p2] +// M2: [t2p0, t2p2, t2p2] +// +// If M2 disappears, both roundrobin and range would have mostly destructive +// reassignments. +// +// Range would result in +// +// M0: [t0p0, t0p1, t1p0, t1p1, t2p0, t2p1] +// M1: [t0p2, t1p2, t2p2] +// +// which is imbalanced and has 3 partitions move from members that did not need +// to move (t0p2, t1p0, t1p1). +// +// RoundRobin would result in +// +// M0: [t0p0, t0p2, t1p1, t2p0, t2p2] +// M1: [t0p1, t1p0, t1p2, t2p1] +// +// which is balanced, but has 2 partitions move when they do not need to +// (t0p1, t1p1). +// +// Sticky balancing results in +// +// M0: [t0p0, t0p1, t0p2, t2p0, t2p2] +// M1: [t1p0, t1p1, t1p2, t2p1] +// +// which is balanced and does not cause any unnecessary partition movement. +// The actual t2 partitions may not be in that exact combination, but they +// will be balanced. +// +// An advantage of the sticky consumer is that it allows API users to +// potentially avoid some cleanup until after the consumer knows which +// partitions it is losing when it gets its new assignment. Users can +// then only cleanup state for partitions that changed, which will be +// minimal (see KIP-54; this client also includes the KIP-351 bugfix). +// +// Note that this API implements the sticky partitioning quite differently from +// the Java implementation. The Java implementation is difficult to reason +// about and has many edge cases that result in non-optimal balancing (albeit, +// you likely have to be trying to hit those edge cases). This API uses a +// different algorithm to ensure optimal balancing while being an order of +// magnitude faster. +// +// Since the new strategy is a strict improvement over the Java strategy, it is +// entirely compatible. Any Go client sharing a group with a Java client will +// not have its decisions undone on leadership change from a Go consumer to a +// Java one. Java balancers do not apply the strategy it comes up with if it +// deems the balance score equal to or worse than the original score (the score +// being effectively equal to the standard deviation of the mean number of +// assigned partitions). This Go sticky balancer is optimal and extra sticky. +// Thus, the Java balancer will never back out of a strategy from this +// balancer. +func StickyBalancer() GroupBalancer { + return &stickyBalancer{cooperative: false} +} + +type stickyBalancer struct { + cooperative bool +} + +func (s *stickyBalancer) ProtocolName() string { + if s.cooperative { + return "cooperative-sticky" + } + return "sticky" +} +func (s *stickyBalancer) IsCooperative() bool { return s.cooperative } +func (s *stickyBalancer) JoinGroupMetadata(interests []string, currentAssignment map[string][]int32, generation int32) []byte { + meta := kmsg.NewConsumerMemberMetadata() + meta.Version = 3 // BUMP ME WHEN NEW FIELDS ARE ADDED, AND BUMP ABOVE + meta.Topics = interests + meta.Generation = generation + stickyMeta := kmsg.NewStickyMemberMetadata() + stickyMeta.Generation = generation + for topic, partitions := range currentAssignment { + if s.cooperative { + metaPart := kmsg.NewConsumerMemberMetadataOwnedPartition() + metaPart.Topic = topic + metaPart.Partitions = partitions + meta.OwnedPartitions = append(meta.OwnedPartitions, metaPart) + } + stickyAssn := kmsg.NewStickyMemberMetadataCurrentAssignment() + stickyAssn.Topic = topic + stickyAssn.Partitions = partitions + stickyMeta.CurrentAssignment = append(stickyMeta.CurrentAssignment, stickyAssn) + } + + // KAFKA-12898: ensure our topics are sorted + metaOwned := meta.OwnedPartitions + stickyCurrent := stickyMeta.CurrentAssignment + sort.Slice(metaOwned, func(i, j int) bool { return metaOwned[i].Topic < metaOwned[j].Topic }) + sort.Slice(stickyCurrent, func(i, j int) bool { return stickyCurrent[i].Topic < stickyCurrent[j].Topic }) + + meta.UserData = stickyMeta.AppendTo(nil) + return meta.AppendTo(nil) +} + +func (*stickyBalancer) ParseSyncAssignment(assignment []byte) (map[string][]int32, error) { + return ParseConsumerSyncAssignment(assignment) +} + +func (s *stickyBalancer) MemberBalancer(members []kmsg.JoinGroupResponseMember) (GroupMemberBalancer, map[string]struct{}, error) { + b, err := NewConsumerBalancer(s, members) + return b, b.MemberTopics(), err +} + +func (s *stickyBalancer) Balance(b *ConsumerBalancer, topics map[string]int32) IntoSyncAssignment { + // Since our input into balancing is already sorted by instance ID, + // the sticky strategy does not need to worry about instance IDs at all. + // See my (slightly rambling) comment on KAFKA-8432. + stickyMembers := make([]sticky.GroupMember, 0, len(b.Members())) + b.EachMember(func(member *kmsg.JoinGroupResponseMember, meta *kmsg.ConsumerMemberMetadata) { + var rack string + if meta.Rack != nil { + rack = *meta.Rack + } + stickyMembers = append(stickyMembers, sticky.GroupMember{ + ID: member.MemberID, + Topics: meta.Topics, + UserData: meta.UserData, + Owned: meta.OwnedPartitions, + Generation: meta.Generation, + Cooperative: s.cooperative, + Rack: rack, + }) + }) + + p := &BalancePlan{sticky.BalanceWithRacks(stickyMembers, topics, b.partitionRacks)} + if s.cooperative { + p.AdjustCooperative(b) + } + return p +} + +// CooperativeStickyBalancer performs the sticky balancing strategy, but +// additionally opts the consumer group into "cooperative" rebalancing. +// +// Cooperative rebalancing differs from "eager" (the original) rebalancing in +// that group members do not stop processing partitions during the rebalance. +// Instead, once they receive their new assignment, each member determines +// which partitions it needs to revoke. If any, they send a new join request +// (before syncing), and the process starts over. This should ultimately end up +// in only two join rounds, with the major benefit being that processing never +// needs to stop. +// +// NOTE once a group is collectively using cooperative balancing, it is unsafe +// to have a member join the group that does not support cooperative balancing. +// If the only-eager member is elected leader, it will not know of the new +// multiple join strategy and things will go awry. Thus, once a group is +// entirely on cooperative rebalancing, it cannot go back. +// +// Migrating an eager group to cooperative balancing requires two rolling +// bounce deploys. The first deploy should add the cooperative-sticky strategy +// as an option (that is, each member goes from using one balance strategy to +// two). During this deploy, Kafka will tell leaders to continue using the old +// eager strategy, since the old eager strategy is the only one in common among +// all members. The second rolling deploy removes the old eager strategy. At +// this point, Kafka will tell the leader to use cooperative-sticky balancing. +// During this roll, all members in the group that still have both strategies +// continue to be eager and give up all of their partitions every rebalance. +// However, once a member only has cooperative-sticky, it can begin using this +// new strategy and things will work correctly. See KIP-429 for more details. +func CooperativeStickyBalancer() GroupBalancer { + return &stickyBalancer{cooperative: true} +} + +// AdjustCooperative performs the final adjustment to a plan for cooperative +// balancing. +// +// Over the plan, we remove all partitions that migrated from one member (where +// it was assigned) to a new member (where it is now planned). +// +// This allows members that had partitions removed to revoke and rejoin, which +// will then do another rebalance, and in that new rebalance, the planned +// partitions are now on the free list to be assigned. +func (p *BalancePlan) AdjustCooperative(b *ConsumerBalancer) { + allAdded := make(map[string]map[int32]string, 100) // topic => partition => member + allRevoked := make(map[string]map[int32]struct{}, 100) + + addT := func(t string) map[int32]string { + addT := allAdded[t] + if addT == nil { + addT = make(map[int32]string, 20) + allAdded[t] = addT + } + return addT + } + revokeT := func(t string) map[int32]struct{} { + revokeT := allRevoked[t] + if revokeT == nil { + revokeT = make(map[int32]struct{}, 20) + allRevoked[t] = revokeT + } + return revokeT + } + + tmap := make(map[string]struct{}) // reusable topic existence map + pmap := make(map[int32]struct{}) // reusable partitions existence map + + plan := p.plan + + // First, on all members, we find what was added and what was removed + // to and from that member. + b.EachMember(func(member *kmsg.JoinGroupResponseMember, meta *kmsg.ConsumerMemberMetadata) { + planned := plan[member.MemberID] + + // added := planned - current + // revoked := current - planned + + for ptopic := range planned { // set existence for all planned topics + tmap[ptopic] = struct{}{} + } + for _, otopic := range meta.OwnedPartitions { // over all prior owned topics, + topic := otopic.Topic + delete(tmap, topic) + ppartitions, exists := planned[topic] + if !exists { // any topic that is no longer planned was entirely revoked, + allRevokedT := revokeT(topic) + for _, opartition := range otopic.Partitions { + allRevokedT[opartition] = struct{}{} + } + continue + } + // calculate what was added by creating a planned existence map, + // then removing what was owned, and anything that remains is new, + for _, ppartition := range ppartitions { + pmap[ppartition] = struct{}{} + } + for _, opartition := range otopic.Partitions { + delete(pmap, opartition) + } + if len(pmap) > 0 { + allAddedT := addT(topic) + for ppartition := range pmap { + delete(pmap, ppartition) + allAddedT[ppartition] = member.MemberID + } + } + // then calculate removal by creating owned existence map, + // then removing what was planned, anything remaining was revoked. + for _, opartition := range otopic.Partitions { + pmap[opartition] = struct{}{} + } + for _, ppartition := range ppartitions { + delete(pmap, ppartition) + } + if len(pmap) > 0 { + allRevokedT := revokeT(topic) + for opartition := range pmap { + delete(pmap, opartition) + allRevokedT[opartition] = struct{}{} + } + } + } + for ptopic := range tmap { // finally, anything remaining in tmap is a new planned topic. + delete(tmap, ptopic) + allAddedT := addT(ptopic) + for _, ppartition := range planned[ptopic] { + allAddedT[ppartition] = member.MemberID + } + } + }) + + // Over all revoked, if the revoked partition was added to a different + // member, we remove that partition from the new member. + for topic, rpartitions := range allRevoked { + atopic, exists := allAdded[topic] + if !exists { + continue + } + for rpartition := range rpartitions { + amember, exists := atopic[rpartition] + if !exists { + continue + } + + ptopics := plan[amember] + ppartitions := ptopics[topic] + for i, ppartition := range ppartitions { + if ppartition == rpartition { + ppartitions[i] = ppartitions[len(ppartitions)-1] + ppartitions = ppartitions[:len(ppartitions)-1] + break + } + } + if len(ppartitions) > 0 { + ptopics[topic] = ppartitions + } else { + delete(ptopics, topic) + } + } + } +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/hooks.go b/vendor/github.com/twmb/franz-go/pkg/kgo/hooks.go new file mode 100644 index 00000000000..3ba13db0eab --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/hooks.go @@ -0,0 +1,439 @@ +package kgo + +import ( + "context" + "net" + "time" +) + +//////////////////////////////////////////////////////////////// +// NOTE: // +// NOTE: Make sure new hooks are checked in implementsAnyHook // +// NOTE: // +//////////////////////////////////////////////////////////////// + +// Hook is a hook to be called when something happens in kgo. +// +// The base Hook interface is meaningless, but wherever a hook can occur in kgo, +// the client checks if your hook implements an appropriate interface. If so, +// your hook is called. +// +// This allows you to only hook in to behavior you care about, and it allows +// the client to add more hooks in the future. +// +// All hook interfaces in this package have Hook in the name. Hooks must be +// safe for concurrent use. It is expected that hooks are fast; if a hook needs +// to take time, then copy what you need and ensure the hook is async. +type Hook any + +type hooks []Hook + +func (hs hooks) each(fn func(Hook)) { + for _, h := range hs { + fn(h) + } +} + +// HookNewClient is called in NewClient after a client is initialized. This +// hook can be used to perform final setup work in your hooks. +type HookNewClient interface { + // OnNewClient is passed the newly initialized client, before any + // client goroutines are started. + OnNewClient(*Client) +} + +// HookClientClosed is called in Close or CloseAfterRebalance after a client +// has been closed. This hook can be used to perform final cleanup work. +type HookClientClosed interface { + // OnClientClosed is passed the client that has been closed, after + // all client-internal close cleanup has happened. + OnClientClosed(*Client) +} + +////////////////// +// BROKER HOOKS // +////////////////// + +// HookBrokerConnect is called after a connection to a broker is opened. +type HookBrokerConnect interface { + // OnBrokerConnect is passed the broker metadata, how long it took to + // dial and initialize the connection (issue ApiVersions and run through + // any SASL flow), and either the dial's resulting net.Conn or any error. + OnBrokerConnect(meta BrokerMetadata, initDur time.Duration, conn net.Conn, err error) +} + +// HookBrokerDisconnect is called when a connection to a broker is closed. +type HookBrokerDisconnect interface { + // OnBrokerDisconnect is passed the broker metadata and the connection + // that is closing. + OnBrokerDisconnect(meta BrokerMetadata, conn net.Conn) +} + +// HookBrokerWrite is called after a write to a broker. +// +// Kerberos SASL does not cause write hooks, since it directly writes to the +// connection. +type HookBrokerWrite interface { + // OnBrokerWrite is passed the broker metadata, the key for the request + // that was written, the number of bytes that were written (may not be + // the whole request if there was an error), how long the request + // waited before being written (including throttling waiting), how long + // it took to write the request, and any error. + // + // The bytes written does not count any tls overhead. + OnBrokerWrite(meta BrokerMetadata, key int16, bytesWritten int, writeWait, timeToWrite time.Duration, err error) +} + +// HookBrokerRead is called after a read from a broker. +// +// Kerberos SASL does not cause read hooks, since it directly reads from the +// connection. +type HookBrokerRead interface { + // OnBrokerRead is passed the broker metadata, the key for the response + // that was read, the number of bytes read (may not be the whole read + // if there was an error), how long the client waited before reading + // the response, how long it took to read the response, and any error. + // + // The bytes read does not count any tls overhead. + OnBrokerRead(meta BrokerMetadata, key int16, bytesRead int, readWait, timeToRead time.Duration, err error) +} + +// BrokerE2E tracks complete information for a write of a request followed by a +// read of that requests's response. +// +// Note that if this is for a produce request with no acks, there will be no +// read wait / time to read. +type BrokerE2E struct { + // BytesWritten is the number of bytes written for this request. + // + // This may not be the whole request if there was an error while writing. + BytesWritten int + + // BytesRead is the number of bytes read for this requests's response. + // + // This may not be the whole response if there was an error while + // reading, and this will be zero if there was a write error. + BytesRead int + + // WriteWait is the time spent waiting from when this request was + // generated internally in the client to just before the request is + // written to the connection. This number is not included in the + // DurationE2E method. + WriteWait time.Duration + // TimeToWrite is how long a request took to be written on the wire. + // This specifically tracks only how long conn.Write takes. + TimeToWrite time.Duration + // ReadWait tracks the span of time immediately following conn.Write + // until conn.Read begins. + ReadWait time.Duration + // TimeToRead tracks how long conn.Read takes for this request to be + // entirely read. This includes the time it takes to allocate a buffer + // for the response after the initial four size bytes are read. + TimeToRead time.Duration + + // WriteErr is any error encountered during writing. If a write error is + // encountered, no read will be attempted. + WriteErr error + // ReadErr is any error encountered during reading. + ReadErr error +} + +// DurationE2E returns the e2e time from the start of when a request is written +// to the end of when the response for that request was fully read. If a write +// or read error occurs, this hook is called with all information possible at +// the time (e.g., if a write error occurs, all write info is specified). +// +// Kerberos SASL does not cause this hook, since it directly reads from the +// connection. +func (e *BrokerE2E) DurationE2E() time.Duration { + return e.TimeToWrite + e.ReadWait + e.TimeToRead +} + +// Err returns the first of either the write err or the read err. If this +// return is non-nil, the request/response had an error. +func (e *BrokerE2E) Err() error { + if e.WriteErr != nil { + return e.WriteErr + } + return e.ReadErr +} + +// HookBrokerE2E is called after a write to a broker that errors, or after a +// read to a broker. +// +// This differs from HookBrokerRead and HookBrokerWrite by tracking all E2E +// info for a write and a read, which allows for easier e2e metrics. This hook +// can replace both the read and write hook. +type HookBrokerE2E interface { + // OnBrokerE2E is passed the broker metadata, the key for the + // request/response that was written/read, and the e2e info for the + // request and response. + OnBrokerE2E(meta BrokerMetadata, key int16, e2e BrokerE2E) +} + +// HookBrokerThrottle is called after a response to a request is read +// from a broker, and the response identifies throttling in effect. +type HookBrokerThrottle interface { + // OnBrokerThrottle is passed the broker metadata, the imposed + // throttling interval, and whether the throttle was applied before + // Kafka responded to them request or after. + // + // For Kafka < 2.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0, the throttle is applied after issuing a response. + // + // If throttledAfterResponse is false, then Kafka already applied the + // throttle. If it is true, the client internally will not send another + // request until the throttle deadline has passed. + OnBrokerThrottle(meta BrokerMetadata, throttleInterval time.Duration, throttledAfterResponse bool) +} + +////////// +// MISC // +////////// + +// HookGroupManageError is called after every error that causes the client, +// operating as a group member, to break out of the group managing loop and +// backoff temporarily. +// +// Specifically, any error that would result in OnPartitionsLost being called +// will result in this hook being called. +type HookGroupManageError interface { + // OnGroupManageError is passed the error that killed a group session. + // This can be used to detect potentially fatal errors and act on them + // at runtime to recover (such as group auth errors, or group max size + // reached). + OnGroupManageError(error) +} + +/////////////////////////////// +// PRODUCE & CONSUME BATCHES // +/////////////////////////////// + +// ProduceBatchMetrics tracks information about successful produces to +// partitions. +type ProduceBatchMetrics struct { + // NumRecords is the number of records that were produced in this + // batch. + NumRecords int + + // UncompressedBytes is the number of bytes the records serialized as + // before compression. + // + // For record batches (Kafka v0.11.0+), this is the size of the records + // in a batch, and does not include record batch overhead. + // + // For message sets, this size includes message set overhead. + UncompressedBytes int + + // CompressedBytes is the number of bytes actually written for this + // batch, after compression. If compression is not used, this will be + // equal to UncompresedBytes. + // + // For record batches, this is the size of the compressed records, and + // does not include record batch overhead. + // + // For message sets, this is the size of the compressed message set. + CompressedBytes int + + // CompressionType signifies which algorithm the batch was compressed + // with. + // + // 0 is no compression, 1 is gzip, 2 is snappy, 3 is lz4, and 4 is + // zstd. + CompressionType uint8 +} + +// HookProduceBatchWritten is called whenever a batch is known to be +// successfully produced. +type HookProduceBatchWritten interface { + // OnProduceBatchWritten is called per successful batch written to a + // topic partition + OnProduceBatchWritten(meta BrokerMetadata, topic string, partition int32, metrics ProduceBatchMetrics) +} + +// FetchBatchMetrics tracks information about fetches of batches. +type FetchBatchMetrics struct { + // NumRecords is the number of records that were fetched in this batch. + // + // Note that this number includes transaction markers, which are not + // actually returned to the user. + // + // If the batch has an encoding error, this will be 0. + NumRecords int + + // UncompressedBytes is the number of bytes the records deserialized + // into after decompresion. + // + // For record batches (Kafka v0.11.0+), this is the size of the records + // in a batch, and does not include record batch overhead. + // + // For message sets, this size includes message set overhead. + // + // Note that this number may be higher than the corresponding number + // when producing, because as an "optimization", Kafka can return + // partial batches when fetching. + UncompressedBytes int + + // CompressedBytes is the number of bytes actually read for this batch, + // before decompression. If the batch was not compressed, this will be + // equal to UncompressedBytes. + // + // For record batches, this is the size of the compressed records, and + // does not include record batch overhead. + // + // For message sets, this is the size of the compressed message set. + CompressedBytes int + + // CompressionType signifies which algorithm the batch was compressed + // with. + // + // 0 is no compression, 1 is gzip, 2 is snappy, 3 is lz4, and 4 is + // zstd. + CompressionType uint8 +} + +// HookFetchBatchRead is called whenever a batch is read within the client. +// +// Note that this hook is called when processing, but a batch may be internally +// discarded after processing in some uncommon specific circumstances. +// +// If the client reads v0 or v1 message sets, and they are not compressed, then +// this hook will be called per record. +type HookFetchBatchRead interface { + // OnFetchBatchRead is called per batch read from a topic partition. + OnFetchBatchRead(meta BrokerMetadata, topic string, partition int32, metrics FetchBatchMetrics) +} + +/////////////////////////////// +// PRODUCE & CONSUME RECORDS // +/////////////////////////////// + +// HookProduceRecordBuffered is called when a record is buffered internally in +// the client from a call to Produce. +// +// This hook can be used to write metrics that gather the number of records or +// bytes buffered, or the hook can be used to write interceptors that modify a +// record's key / value / headers before being produced. If you just want a +// metric for the number of records buffered, use the client's +// BufferedProduceRecords method, as it is faster. +// +// Note that this hook may slow down high-volume producing a bit. +type HookProduceRecordBuffered interface { + // OnProduceRecordBuffered is passed a record that is buffered. + // + // This hook is called immediately after Produce is called, after the + // function potentially sets the default topic. + OnProduceRecordBuffered(*Record) +} + +// HookProduceRecordPartitioned is called when a record is partitioned and +// internally ready to be flushed. +// +// This hook can be used to create metrics of buffered records per partition, +// and then you can correlate that to partition leaders and determine which +// brokers are having problems. +// +// Note that this hook will slow down high-volume producing and it is +// recommended to only use this temporarily or if you are ok with the +// performance hit. +type HookProduceRecordPartitioned interface { + // OnProduceRecordPartitioned is passed a record that has been + // partitioned and the current broker leader for the partition + // (note that the leader may change if the partition is moved). + // + // This hook is called once a record is queued to be flushed. The + // record's Partition and Timestamp fields are safe to read. + OnProduceRecordPartitioned(*Record, int32) +} + +// HookProduceRecordUnbuffered is called just before a record's promise is +// finished; this is effectively a mirror of a record promise. +// +// As an example, if using HookProduceRecordBuffered for a gauge of how many +// record bytes are buffered, this hook can be used to decrement the gauge. +// +// Note that this hook will slow down high-volume producing a bit. As well, +// records that were buffered but are paused (and stripped internally before +// being returned to the user) will still be passed to this hook. +type HookProduceRecordUnbuffered interface { + // OnProduceRecordUnbuffered is passed a record that is just about to + // have its produce promise called, as well as the error that the + // promise will be called with. + OnProduceRecordUnbuffered(*Record, error) +} + +// HookFetchRecordBuffered is called when a record is internally buffered after +// fetching, ready to be polled. +// +// This hook can be used to write gauge metrics regarding the number of records +// or bytes buffered, or to write interceptors that modify a record before +// being returned from polling. If you just want a metric for the number of +// records buffered, use the client's BufferedFetchRecords method, as it is +// faster. +// +// Note that this hook will slow down high-volume consuming a bit. +type HookFetchRecordBuffered interface { + // OnFetchRecordBuffered is passed a record that is now buffered, ready + // to be polled. + OnFetchRecordBuffered(*Record) +} + +// HookFetchRecordUnbuffered is called when a fetched record is unbuffered. +// +// A record can be internally discarded after being in some scenarios without +// being polled, such as when the internal assignment changes. +// +// As an example, if using HookFetchRecordBuffered for a gauge of how many +// record bytes are buffered ready to be polled, this hook can be used to +// decrement the gauge. +// +// Note that this hook may slow down high-volume consuming a bit. +type HookFetchRecordUnbuffered interface { + // OnFetchRecordUnbuffered is passed a record that is being + // "unbuffered" within the client, and whether the record is being + // returned from polling. + OnFetchRecordUnbuffered(r *Record, polled bool) +} + +// HookPollStart is called at the beginning of every PollFetches or +// PollRecords call, before any records are drained from internal buffers and +// before any HookFetchRecordUnbuffered hooks fire for the same poll. +// +// This hook is useful for instrumenting the poll boundary: for example, a +// tracing integration that opens a span per record via HookFetchRecordUnbuffered +// can implement this hook to finish the previous poll's spans before new ones +// are created. +type HookPollStart interface { + // OnPollStart is called at the start of every PollFetches or + // PollRecords call with the context passed by the caller. + OnPollStart(ctx context.Context) +} + +///////////// +// HELPERS // +///////////// + +// implementsAnyHook will check the incoming Hook for any Hook implementation +func implementsAnyHook(h Hook) bool { + switch h.(type) { + case HookNewClient, + HookClientClosed, + HookBrokerConnect, + HookBrokerDisconnect, + HookBrokerWrite, + HookBrokerRead, + HookBrokerE2E, + HookBrokerThrottle, + HookGroupManageError, + HookProduceBatchWritten, + HookFetchBatchRead, + HookProduceRecordBuffered, + HookProduceRecordPartitioned, + HookProduceRecordUnbuffered, + HookFetchRecordBuffered, + HookFetchRecordUnbuffered, + HookPollStart: + return true + } + return false +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/graph.go b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/graph.go new file mode 100644 index 00000000000..d6bbb587ed2 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/graph.go @@ -0,0 +1,226 @@ +package sticky + +import "container/heap" + +// Graph maps members to partitions they want to steal. +// +// The representation was chosen so as to avoid updating all members on any +// partition move; move updates are one map update. +type graph struct { + b *balancer + + // node => edges out + // "from a node (member), which topicNum could we steal?" + out [][]uint32 + + // edge => who owns this edge; built in balancer's assignUnassigned + cxns []partitionConsumer + + // scores are all node scores from a search node. The distance field + // is reset on findSteal to infinityScore.. + scores pathScores + + // heapBuf and pathBuf are backing buffers that are reused every + // findSteal; note that pathBuf must be done being used before + // the next find steal, but it always is. + heapBuf pathHeap + pathBuf []stealSegment +} + +func (b *balancer) newGraph( + partitionConsumers []partitionConsumer, + topicPotentials [][]uint16, +) graph { + g := graph{ + b: b, + out: make([][]uint32, len(b.members)), + cxns: partitionConsumers, + scores: make([]pathScore, len(b.members)), + heapBuf: make([]*pathScore, len(b.members)), + } + outBufs := make([]uint32, len(b.members)*len(topicPotentials)) + for memberNum := range b.plan { + out := outBufs[:0:len(topicPotentials)] + outBufs = outBufs[len(topicPotentials):] + // In the worst case, if every node is linked to each other, + // each node will have nparts edges. We preallocate the worst + // case. It is common for the graph to be highly connected. + g.out[memberNum] = out + } + for topicNum, potentials := range topicPotentials { + for _, potential := range potentials { + g.out[potential] = append(g.out[potential], uint32(topicNum)) + } + } + return g +} + +func (g *graph) changeOwnership(edge int32, newDst uint16) { + g.cxns[edge].memberNum = newDst +} + +// findSteal uses Dijkstra search to find a path from the best node it can reach. +func (g *graph) findSteal(from uint16) ([]stealSegment, bool) { + // First, we must reset our scores from any prior run. This is O(M), + // but is fast and faster than making a map and extending it a lot. + for i := range g.scores { + g.scores[i].distance = infinityScore + g.scores[i].done = false + } + + first, _ := g.getScore(from) + + first.distance = 0 + first.done = true + + g.heapBuf = append(g.heapBuf[:0], first) + rem := &g.heapBuf + for rem.Len() > 0 { + current := heap.Pop(rem).(*pathScore) + if current.level > first.level+1 { + path := g.pathBuf[:0] + for current.parent != nil { + path = append(path, stealSegment{ + current.node, + current.parent.node, + current.srcEdge, + }) + current = current.parent + } + g.pathBuf = path + return path, true + } + + current.done = true + + for _, topicNum := range g.out[current.node] { + info := g.b.topicInfos[topicNum] + firstPartNum, lastPartNum := info.partNum, info.partNum+info.partitions + for edge := firstPartNum; edge < lastPartNum; edge++ { + neighborNode := g.cxns[edge].memberNum + neighbor, isNew := g.getScore(neighborNode) + if neighbor.done { + continue + } + + distance := current.distance + 1 + + // The neighbor is the current node that owns this edge. + // If our node originally owned this partition, then it + // would be preferable to steal edge back. + srcIsOriginal := g.cxns[edge].originalNum == current.node + + // If this is a new neighbor (our first time seeing the neighbor + // in our search), this is also the shortest path to reach them, + // where shortest defers preference to original sources THEN distance. + if isNew { + neighbor.parent = current + neighbor.srcIsOriginal = srcIsOriginal + neighbor.srcEdge = edge + neighbor.distance = distance + neighbor.heapIdx = len(*rem) + heap.Push(rem, neighbor) + } else if !neighbor.srcIsOriginal && srcIsOriginal { + // If the search path has seen this neighbor before, but + // we now are evaluating a partition that would increase + // stickiness if stolen, then fixup the neighbor's parent + // and srcEdge. + neighbor.parent = current + neighbor.srcIsOriginal = true + neighbor.srcEdge = edge + neighbor.distance = distance + heap.Fix(rem, neighbor.heapIdx) + } + } + } + } + + return nil, false +} + +type stealSegment struct { + src uint16 // member num + dst uint16 // member num + part int32 // partNum +} + +// As we traverse a graph, we assign each node a path score, which tracks a few +// numbers for what it would take to reach this node from our first node. +type pathScore struct { + // Done is set to true when we pop a node off of the graph. Once we + // pop a node, it means we have found a best path to that node and + // we do not want to revisit it for processing if any other future + // nodes reach back to this one. + done bool + + // srcIsOriginal is true if, were our parent to steal srcEdge, would + // that put srcEdge back on the original member. That is, if we are B + // and our parent is A, does our srcEdge originally belong do A? + // + // This field exists to work around a very slim edge case where a + // partition is stolen by B and then needs to be stolen back by A + // later. + srcIsOriginal bool + + node uint16 // our member num + distance int32 // how many steals it would take to get here + srcEdge int32 // the partition used to reach us + level int32 // partitions owned on this segment + parent *pathScore + heapIdx int +} + +type pathScores []pathScore + +const infinityScore = 1<<31 - 1 + +func (g *graph) getScore(node uint16) (*pathScore, bool) { + r := &g.scores[node] + exists := r.distance != infinityScore + if !exists { + *r = pathScore{ + node: node, + level: int32(len(g.b.plan[node])), + distance: infinityScore, + } + } + return r, !exists +} + +type pathHeap []*pathScore + +func (p *pathHeap) Len() int { return len(*p) } +func (p *pathHeap) Swap(i, j int) { + h := *p + l, r := h[i], h[j] + l.heapIdx, r.heapIdx = r.heapIdx, l.heapIdx + h[i], h[j] = r, l +} + +// For our path, we always want to prioritize stealing a partition we +// originally owned. This may result in a longer steal path, but it will +// increase stickiness. +// +// Next, our real goal, which is to find a node we can steal from. Because of +// this, we always want to sort by the highest level. The pathHeap stores +// reachable paths, so by sorting by the highest level, we terminate quicker: +// we always check the most likely candidates to quit our search. +// +// Finally, we simply prefer searching through shorter paths and, barring that, +// just sort by node. +func (p *pathHeap) Less(i, j int) bool { + l, r := (*p)[i], (*p)[j] + return l.srcIsOriginal && !r.srcIsOriginal || !l.srcIsOriginal && !r.srcIsOriginal && + (l.level > r.level || l.level == r.level && + (l.distance < r.distance || l.distance == r.distance && + l.node < r.node)) +} + +func (p *pathHeap) Push(x any) { *p = append(*p, x.(*pathScore)) } +func (p *pathHeap) Pop() any { + h := *p + l := len(h) + r := h[l-1] + *p = h[:l-1] + return r +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/rbtree.go b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/rbtree.go new file mode 100644 index 00000000000..8d563b7873f --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/rbtree.go @@ -0,0 +1,392 @@ +package sticky + +// This file contains a vendoring of github.com/twmb/go-rbtree, with interface +// types replaced with *partitionLevel. We do this to simplify (and slightly) +// speed up the rbtree, get rid of a bunch of code we do not need, and to drop +// a dep. + +type color bool + +const red, black color = true, false + +// Tree is a red-black tree. +type treePlan struct { + root *treePlanNode + size int +} + +type treePlanNode struct { + left *treePlanNode + right *treePlanNode + parent *treePlanNode + color color + item *partitionLevel +} + +// liftRightSideOf is rotateLeft. +// +// Graphically speaking, this takes the node on the right and lifts it above +// ourselves. IMO trying to visualize a "rotation" is confusing. +func (t *treePlan) liftRightSideOf(n *treePlanNode) { + r := n.right + t.relinkParenting(n, r) + + // lift the right + n.right = r.left + n.parent = r + + // fix the lifted right's left + if r.left != nil { + r.left.parent = n + } + r.left = n +} + +// liftLeftSideOf is rotateRight, renamed to aid my visualization. +func (t *treePlan) liftLeftSideOf(n *treePlanNode) { + l := n.left + t.relinkParenting(n, l) + + n.left = l.right + n.parent = l + + if l.right != nil { + l.right.parent = n + } + l.right = n +} + +// relinkParenting is called to fix a former child c of node n's parent +// relationship to the parent of n. +// +// After this, the n node can be considered to have no parent. +func (t *treePlan) relinkParenting(n, c *treePlanNode) { + p := n.parent + if c != nil { + c.parent = p + } + if p == nil { + t.root = c + return + } + if n == p.left { + p.left = c + } else { + p.right = c + } +} + +func (n *treePlanNode) sibling() *treePlanNode { + if n.parent == nil { + return nil + } + if n == n.parent.left { + return n.parent.right + } + return n.parent.left +} + +func (n *treePlanNode) uncle() *treePlanNode { + p := n.parent + if p.parent == nil { + return nil + } + return p.sibling() +} + +func (n *treePlanNode) grandparent() *treePlanNode { + return n.parent.parent +} + +func (n *treePlanNode) isBlack() bool { + return n == nil || n.color == black +} + +func (t *treePlan) insert(i *partitionLevel) *treePlanNode { + r := &treePlanNode{item: i} + t.reinsert(r) + return r +} + +func (t *treePlan) reinsert(n *treePlanNode) { + *n = treePlanNode{ + color: red, + item: n.item, + } + t.size++ + if t.root == nil { + n.color = black + t.root = n + return + } + + on := t.root + var set **treePlanNode + for { + if n.item.less(on.item) { + if on.left == nil { + set = &on.left + break + } + on = on.left + } else { + if on.right == nil { + set = &on.right + break + } + on = on.right + } + } + + n.parent = on + *set = n + +repair: + // Case 1: we have jumped back to the root. Paint it black. + if n.parent == nil { + n.color = black + return + } + + // Case 2: if our parent is black, us being red does not add a new black + // to the chain and cannot increase the maximum number of blacks from + // root, so we are done. + if n.parent.color == black { + return + } + + // Case 3: if we have an uncle and it is red, then we flip our + // parent's, uncle's, and grandparent's color. + // + // This stops the red-red from parent to us, but may introduce + // a red-red from grandparent to its parent, so we set ourselves + // to the grandparent and go back to the repair beginning. + if uncle := n.uncle(); uncle != nil && uncle.color == red { + n.parent.color = black + uncle.color = black + n = n.grandparent() + n.color = red + goto repair + } + + // Case 4 step 1: our parent is red but uncle is black. Step 2 relies + // on the node being on the "outside". If we are on the inside, our + // parent lifts ourselves above itself, thus making the parent the + // outside, and then we become that parent. + p := n.parent + g := p.parent + if n == p.right && p == g.left { + t.liftRightSideOf(p) + n = n.left + } else if n == p.left && p == g.right { + t.liftLeftSideOf(p) + n = n.right + } + + // Care 4 step 2: we are on the outside, and we and our parent are red. + // If we are on the left, our grandparent lifts its left and then swaps + // its and our parent's colors. + // + // This fixes the red-red situation while preserving the number of + // blacks from root to leaf property. + p = n.parent + g = p.parent + + if n == p.left { + t.liftLeftSideOf(g) + } else { + t.liftRightSideOf(g) + } + p.color = black + g.color = red +} + +func (t *treePlan) delete(n *treePlanNode) { + t.size-- + + // We only want to delete nodes with at most one child. If this has + // two, we find the max node on the left, set this node's item to that + // node's item, and then delete that max node. + if n.left != nil && n.right != nil { + remove := n.left.max() + n.item, remove.item = remove.item, n.item + n = remove + } + + // Determine which child to elevate into our position now that we know + // we have at most one child. + c := n.right + if n.right == nil { + c = n.left + } + + t.doDelete(n, c) + t.relinkParenting(n, c) +} + +// Since we do not represent leave nodes with objects, we relink the parent +// after deleting. See the Wikipedia note. Most of our deletion operations +// on n (the dubbed "shadow" node) rather than c. +func (t *treePlan) doDelete(n, c *treePlanNode) { + // If the node was red, we deleted a red node; the number of black + // nodes along any path is the same and we can quit. + if n.color != black { + return + } + + // If the node was black, then, if we have a child and it is red, + // we switch the child to black to preserve the path number. + if c != nil && c.color == red { + c.color = black + return + } + + // We either do not have a child (nil is black), or we do and it + // is black. We must preserve the number of blacks. + +case1: + // Case 1: if the child is the new root, then the tree must have only + // had up to two elements and now has one or zero. We are done. + if n.parent == nil { + return + } + + // Note that if we are here, we must have a sibling. + // + // The first time through, from the deleted node, the deleted node was + // black and the child was black. This being two blacks meant that the + // original node's parent required two blacks on the other side. + // + // The second time through, through case 3, the sibling was repainted + // red... so it must still exist. + + // Case 2: if the child's sibling is red, we recolor the parent and + // sibling and lift the sibling, ensuring we have a black sibling. + s := n.sibling() + if s.color == red { + n.parent.color = red + s.color = black + if n == n.parent.left { + t.liftRightSideOf(n.parent) + } else { + t.liftLeftSideOf(n.parent) + } + s = n.sibling() + } + + // Right here, we know the sibling is black. If both sibling children + // are black or nil leaves (black), we enter cases 3 and 4. + if s.left.isBlack() && s.right.isBlack() { + // Case 3: if the parent, sibling, sibling's children are + // black, we can paint the sibling red to fix the imbalance. + // However, the same black imbalance can exist on the other + // side of the parent, so we go back to case 1 on the parent. + s.color = red + if n.parent.color == black { + n = n.parent + goto case1 + } + + // Case 4: if the sibling and sibling's children are black, but + // the parent is red, We can swap parent and sibling colors to + // fix our imbalance. We have no worry of further imbalances up + // the tree since we deleted a black node, replaced it with a + // red node, and then painted that red node black. + n.parent.color = black + return + } + + // Now we know the sibling is black and one of its children is red. + + // Case 5: in preparation for 6, if we are on the left, we want our + // sibling, if it has a right child, for that child's color to be red. + // We swap the sibling and sibling's left's color (since we know the + // sibling has a red child and that the right is black) and we lift the + // left child. + // + // This keeps the same number of black nodes and under the sibling. + if n == n.parent.left && s.right.isBlack() { + s.color = red + s.left.color = black + t.liftLeftSideOf(s) + } else if n == n.parent.right && s.left.isBlack() { + s.color = red + s.right.color = black + t.liftRightSideOf(s) + } + s = n.sibling() // can change from the above case + + // At this point, we know we have a black sibling and, if we are on + // the left, it has a red child on its right. + + // Case 6: we lift the sibling above the parent, swap the sibling's and + // parent's color, and change the sibling's right's color from red to + // black. + // + // This brings in a black above our node to replace the one we deleted, + // while preserves the number of blacks on the other side of the path. + s.color = n.parent.color + n.parent.color = black + if n == n.parent.left { + s.right.color = black + t.liftRightSideOf(n.parent) + } else { + s.left.color = black + t.liftLeftSideOf(n.parent) + } +} + +func (t *treePlan) findWith(cmp func(*partitionLevel) int) *treePlanNode { + on := t.root + for on != nil { + way := cmp(on.item) + switch { + case way < 0: + on = on.left + case way == 0: + return on + case way > 0: + on = on.right + } + } + return nil +} + +func (t *treePlan) findWithOrInsertWith( + find func(*partitionLevel) int, + insert func() *partitionLevel, +) *treePlanNode { + found := t.findWith(find) + if found == nil { + return t.insert(insert()) + } + return found +} + +func (t *treePlan) min() *treePlanNode { + if t.root == nil { + return nil + } + return t.root.min() +} + +func (n *treePlanNode) min() *treePlanNode { + for n.left != nil { + n = n.left + } + return n +} + +func (t *treePlan) max() *treePlanNode { + if t.root == nil { + return nil + } + return t.root.max() +} + +func (n *treePlanNode) max() *treePlanNode { + for n.right != nil { + n = n.right + } + return n +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/sticky.go b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/sticky.go new file mode 100644 index 00000000000..06ac816937a --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/sticky/sticky.go @@ -0,0 +1,932 @@ +// Package sticky provides sticky partitioning strategy for Kafka, with a +// complete overhaul to be faster, more understandable, and optimal. +// +// For some points on how Java's strategy is flawed, see +// https://github.com/IBM/sarama/pull/1416/files/b29086bdaae0da7ce71eae3f854d50685fd6b631#r315005878 +package sticky + +import ( + "math" + "slices" + + "github.com/twmb/franz-go/pkg/kbin" + "github.com/twmb/franz-go/pkg/kmsg" +) + +// Sticky partitioning has two versions, the latter from KIP-341 preventing a +// bug. The second version introduced generations with the default generation +// from the first generation's consumers defaulting to -1. + +// We can support up to 65533 members; two slots are reserved. +// We can support up to 2,147,483,647 partitions. +// I expect a server to fall over before reaching either of these numbers. + +// GroupMember is a Kafka group member. +type GroupMember struct { + ID string + Topics []string + UserData []byte + Owned []kmsg.ConsumerMemberMetadataOwnedPartition + Generation int32 + Cooperative bool + Rack string // KIP-881: empty if not set +} + +// Plan is the plan this package came up with (member => topic => partitions). +type Plan map[string]map[string][]int32 + +type balancer struct { + // members are the members in play for this balance. + // This is built in newBalancer mapping member IDs to the GroupMember. + members []GroupMember + + memberNums map[string]uint16 // member id => index into members + + topicNums map[string]uint32 // topic name => index into topicInfos + topicInfos []topicInfo + partOwners []uint32 // partition => owning topicNum + + // Stales tracks partNums that are doubly subscribed in this join + // where one of the subscribers is on an old generation. + // + // The newer generation goes into plan directly, the older gets + // stuffed here. + stales map[int32]uint16 // partNum => stale memberNum + + plan membersPartitions // what we are building and balancing + + // planByNumPartitions orders plan members into partition count levels. + // + // The nodes in the tree reference values in plan, meaning updates in + // this field are visible in plan. + planByNumPartitions treePlan + + // if the subscriptions are complex (all members do _not_ consume the + // same partitions), then we build a graph and use that for assigning. + isComplex bool + + // stealGraph is a graphical representation of members and partitions + // they want to steal. + stealGraph graph + + // KIP-881: rack-aware assignment. partRacks is indexed by flat + // partNum (same as partOwners), memberRacks by memberNum. Rack + // indices are 1-based so that zero-initialized slices naturally + // mean "no rack" (noRack == 0). When no rack info is available, + // both slices are nil. The nRacks field is the count of distinct + // racks; rackHeaps in assignRackAware is indexed by rack-1. + memberRacks []uint16 + partRacks []uint16 + nRacks int +} + +type topicInfo struct { + partNum int32 // base part num + partitions int32 // number of partitions in the topic + topic string +} + +func newBalancer(members []GroupMember, topics map[string]int32, partitionRacks map[string][]string) *balancer { + var ( + nparts int + topicNums = make(map[string]uint32, len(topics)) + topicInfos = make([]topicInfo, len(topics)) + ) + for topic, partitions := range topics { + topicNum := uint32(len(topicNums)) + topicNums[topic] = topicNum + topicInfos[topicNum] = topicInfo{ + partNum: int32(nparts), + partitions: partitions, + topic: topic, + } + nparts += int(partitions) + } + partOwners := make([]uint32, 0, nparts) + for topicNum, info := range topicInfos { + for i := int32(0); i < info.partitions; i++ { + partOwners = append(partOwners, uint32(topicNum)) + } + } + memberNums := make(map[string]uint16, len(members)) + for num, member := range members { + memberNums[member.ID] = uint16(num) + } + + b := &balancer{ + members: members, + memberNums: memberNums, + topicNums: topicNums, + topicInfos: topicInfos, + + partOwners: partOwners, + stales: make(map[int32]uint16), + plan: make(membersPartitions, len(members)), + } + + evenDivvy := nparts/len(members) + 1 + planBuf := make(memberPartitions, evenDivvy*len(members)) + for num := range members { + b.plan[num] = planBuf[:0:evenDivvy] + planBuf = planBuf[evenDivvy:] + } + + b.initRacks(partitionRacks) + return b +} + +// initRacks sets up rack-aware fields if any member and any partition +// have rack info. Both sides must have racks for rack-aware assignment +// to be useful. +func (b *balancer) initRacks(partitionRacks map[string][]string) { + if len(partitionRacks) == 0 { + return + } + + // Map rack strings to small 1-based indices (0 = noRack). + rackIndex := make(map[string]uint16) + rackOf := func(s string) uint16 { + if s == "" { + return noRack + } + idx, ok := rackIndex[s] + if !ok { + idx = uint16(len(rackIndex) + 1) + rackIndex[s] = idx + } + return idx + } + + memberRacks := make([]uint16, len(b.members)) + var anyMemberRack bool + for i := range b.members { + memberRacks[i] = rackOf(b.members[i].Rack) + if memberRacks[i] != noRack { + anyMemberRack = true + } + } + if !anyMemberRack { + return + } + + // Build flat partRacks indexed by partNum. Zero-init is noRack. + partRacks := make([]uint16, cap(b.partOwners)) + var anyPartRack bool + for topic, racks := range partitionRacks { + topicNum, ok := b.topicNums[topic] + if !ok { + continue + } + info := b.topicInfos[topicNum] + for i, rack := range racks { + if int32(i) >= info.partitions { + break + } + if rack != "" { + idx := rackOf(rack) + partRacks[info.partNum+int32(i)] = idx + anyPartRack = true + } + } + } + if !anyPartRack { + return + } + + b.partRacks = partRacks + b.memberRacks = memberRacks + b.nRacks = len(rackIndex) +} + +func (b *balancer) into() Plan { + plan := make(Plan, len(b.plan)) + ntopics := 5 * len(b.topicNums) / 4 + + for memberNum, partNums := range b.plan { + member := b.members[memberNum].ID + if len(partNums) == 0 { + plan[member] = make(map[string][]int32, 0) + continue + } + topics := make(map[string][]int32, ntopics) + plan[member] = topics + + // partOwners is created by topic, and partNums refers to + // indices in partOwners. If we sort by partNum, we have sorted + // topics and partitions. + slices.Sort(partNums) + + // We can reuse partNums for our topic partitions. + topicParts := partNums[:0] + + lastTopicNum := b.partOwners[partNums[0]] + lastTopicInfo := b.topicInfos[lastTopicNum] + for _, partNum := range partNums { + topicNum := b.partOwners[partNum] + + if topicNum != lastTopicNum { + topics[lastTopicInfo.topic] = topicParts[:len(topicParts):len(topicParts)] + topicParts = topicParts[len(topicParts):] + + lastTopicNum = topicNum + lastTopicInfo = b.topicInfos[topicNum] + } + + partition := partNum - lastTopicInfo.partNum + topicParts = append(topicParts, partition) + } + topics[lastTopicInfo.topic] = topicParts[:len(topicParts):len(topicParts)] + } + return plan +} + +func (b *balancer) partNumByTopic(topic string, partition int32) (int32, bool) { + topicNum, exists := b.topicNums[topic] + if !exists { + return 0, false + } + topicInfo := b.topicInfos[topicNum] + if partition >= topicInfo.partitions { + return 0, false + } + return topicInfo.partNum + partition, true +} + +// memberPartitions contains partitions for a member. +type memberPartitions []int32 + +func (m *memberPartitions) remove(needle int32) { + s := *m + var d int + for i, check := range s { + if check == needle { + d = i + break + } + } + s[d] = s[len(s)-1] + *m = s[:len(s)-1] +} + +func (m *memberPartitions) takeEnd() int32 { + s := *m + r := s[len(s)-1] + *m = s[:len(s)-1] + return r +} + +func (m *memberPartitions) add(partNum int32) { + *m = append(*m, partNum) +} + +// membersPartitions maps members to their partitions. +type membersPartitions []memberPartitions + +type partitionLevel struct { + level int + members []uint16 +} + +// partitionLevel's members field used to be a map, but removing it gains a +// slight perf boost at the cost of removing members being O(M). +// Even with the worse complexity, scanning a short list can be faster +// than managing a map, and we expect groups to not be _too_ large. +func (l *partitionLevel) removeMember(memberNum uint16) { + for i, v := range l.members { + if v == memberNum { + l.members[i] = l.members[len(l.members)-1] + l.members = l.members[:len(l.members)-1] + return + } + } +} + +func (b *balancer) findLevel(level int) *partitionLevel { + return b.planByNumPartitions.findWithOrInsertWith( + func(n *partitionLevel) int { return level - n.level }, + func() *partitionLevel { return newPartitionLevel(level) }, + ).item +} + +func (b *balancer) fixMemberLevel( + src *treePlanNode, + memberNum uint16, + partNums memberPartitions, +) { + b.removeLevelingMember(src, memberNum) + newLevel := len(partNums) + partLevel := b.findLevel(newLevel) + partLevel.members = append(partLevel.members, memberNum) +} + +func (b *balancer) removeLevelingMember( + src *treePlanNode, + memberNum uint16, +) { + src.item.removeMember(memberNum) + if len(src.item.members) == 0 { + b.planByNumPartitions.delete(src) + } +} + +func (l *partitionLevel) less(r *partitionLevel) bool { + return l.level < r.level +} + +func newPartitionLevel(level int) *partitionLevel { + return &partitionLevel{level: level} +} + +func (b *balancer) initPlanByNumPartitions() { + for memberNum, partNums := range b.plan { + partLevel := b.findLevel(len(partNums)) + partLevel.members = append(partLevel.members, uint16(memberNum)) + } +} + +// Balance performs sticky partitioning for the given group members and topics, +// returning the determined plan. +func Balance(members []GroupMember, topics map[string]int32) Plan { + return BalanceWithRacks(members, topics, nil) +} + +// BalanceWithRacks performs sticky partitioning with rack-aware assignment +// (KIP-881). partitionRacks maps topic => partition index => rack of the +// partition leader. When non-nil and members also have racks, unassigned +// partitions are preferentially placed on rack-matching members before +// falling back to normal assignment. +func BalanceWithRacks(members []GroupMember, topics map[string]int32, partitionRacks map[string][]string) Plan { + if len(members) == 0 { + return make(Plan) + } + b := newBalancer(members, topics, partitionRacks) + if cap(b.partOwners) == 0 { + return b.into() + } + b.parseMemberMetadata() + b.assignUnassignedAndInitGraph() + b.initPlanByNumPartitions() + b.balance() + return b.into() +} + +// parseMemberMetadata parses all member userdata to initialize the prior plan. +func (b *balancer) parseMemberMetadata() { + // all partitions => members that are consuming those partitions + // Each partition should only have one consumer, but a flaky member + // could rejoin with an old generation (stale user data) and say it + // is consuming something a different member is. See KIP-341. + partitionConsumersByGeneration := make([]memberGeneration, cap(b.partOwners)) + + const highBit uint32 = 1 << 31 + var memberPlan []topicPartition + var gen uint32 + + for _, member := range b.members { + // KAFKA-13715 / KIP-792: cooperative-sticky now includes a + // generation directly with the currently-owned partitions, and + // we can avoid deserializing UserData. This guards against + // some zombie issues (see KIP). + // + // The eager (sticky) balancer revokes all partitions before + // rejoining, so we cannot use Owned. + if member.Cooperative && member.Generation >= 0 { + memberPlan = memberPlan[:0] + for _, t := range member.Owned { + for _, p := range t.Partitions { + memberPlan = append(memberPlan, topicPartition{t.Topic, p}) + } + } + gen = uint32(member.Generation) + } else { + memberPlan, gen = deserializeUserData(member.UserData, memberPlan[:0]) + } + gen |= highBit + memberNum := b.memberNums[member.ID] + for _, topicPartition := range memberPlan { + partNum, exists := b.partNumByTopic(topicPartition.topic, topicPartition.partition) + if !exists { + continue + } + + // We keep the highest generation, and at most two generations. + // If something is doubly consumed, we skip it. + pcs := &partitionConsumersByGeneration[partNum] + switch { + case gen > pcs.genNew: // one consumer already, but new member has higher generation + pcs.memberOld, pcs.genOld = pcs.memberNew, pcs.genNew + pcs.memberNew, pcs.genNew = memberNum, gen + + case gen > pcs.genOld: // one consumer already, we could be second, or if there is a second, we have a high generation + pcs.memberOld, pcs.genOld = memberNum, gen + } + } + } + + for partNum, pcs := range partitionConsumersByGeneration { + if pcs.genNew&highBit != 0 { + b.plan[pcs.memberNew].add(int32(partNum)) + if pcs.genOld&highBit != 0 { + b.stales[int32(partNum)] = pcs.memberOld + } + } + } +} + +type memberGeneration struct { + memberNew uint16 + memberOld uint16 + genNew uint32 + genOld uint32 +} + +type topicPartition struct { + topic string + partition int32 +} + +// deserializeUserData returns the topic partitions a member was consuming and +// the join generation it was consuming from. +// +// If anything fails or we do not understand the userdata parsing generation, +// we return empty defaults. The member will just be assumed to have no +// history. +func deserializeUserData(userdata []byte, base []topicPartition) (memberPlan []topicPartition, generation uint32) { + memberPlan = base[:0] + b := kbin.Reader{Src: userdata} + for numAssignments := b.ArrayLen(); numAssignments > 0; numAssignments-- { + topic := b.UnsafeString() + for numPartitions := b.ArrayLen(); numPartitions > 0; numPartitions-- { + memberPlan = append(memberPlan, topicPartition{ + topic, + b.Int32(), + }) + } + } + if len(b.Src) > 0 { + // A generation of -1 is just as good of a generation as 0, so we use 0 + // and then use the high bit to signify this generation has been set. + if generationI32 := b.Int32(); generationI32 > 0 { + generation = uint32(generationI32) + } + } + if b.Complete() != nil { + memberPlan = memberPlan[:0] + } + return memberPlan, generation +} + +func (b *balancer) sortMemberByLiteralPartNum(memberNum int) { + partNums := b.plan[memberNum] + slices.SortFunc(partNums, func(lpNum, rpNum int32) int { + ltNum, rtNum := b.partOwners[lpNum], b.partOwners[rpNum] + li, ri := b.topicInfos[ltNum], b.topicInfos[rtNum] + lt, rt := li.topic, ri.topic + lp, rp := lpNum-li.partNum, rpNum-ri.partNum + if lp < rp { + return -1 + } else if lp > rp { + return 1 + } else if lt < rt { + return -1 + } + return 1 + }) +} + +// assignUnassignedAndInitGraph is a long function that assigns unassigned +// partitions to the least loaded members and initializes our steal graph. +// +// Doing so requires a bunch of metadata, and in the process we want to remove +// partitions from the plan that no longer exist in the client. +func (b *balancer) assignUnassignedAndInitGraph() { + // First, over all members in this assignment, map each partition to + // the members that can consume it. We will use this for assigning. + // + // To do this mapping efficiently, we first map each topic to the + // memberNums that can consume those topics, and then use the results + // below in the partition mapping. Doing this two step process allows + // for a 10x speed boost rather than ranging over all partitions many + // times. + topicPotentialsBuf := make([]uint16, len(b.topicNums)*len(b.members)) + topicPotentials := make([][]uint16, len(b.topicNums)) + for memberNum, member := range b.members { + for _, topic := range member.Topics { + topicNum, exists := b.topicNums[topic] + if !exists { + continue + } + memberNums := topicPotentials[topicNum] + if cap(memberNums) == 0 { + memberNums = topicPotentialsBuf[:0:len(b.members)] + topicPotentialsBuf = topicPotentialsBuf[len(b.members):] + } + topicPotentials[topicNum] = append(memberNums, uint16(memberNum)) + } + } + + for _, topicMembers := range topicPotentials { + // If the number of members interested in this topic is not the + // same as the number of members in this group, then **other** + // members are interested in other topics and not this one, and + // we must go to complex balancing. + // + // We could accidentally fall into isComplex if any member is + // not interested in anything, but realistically we do not + // expect members to join with no interests. + if len(topicMembers) != len(b.members) { + b.isComplex = true + } + } + + // Next, over the prior plan, un-map deleted topics or topics that + // members no longer want. This is where we determine what is now + // unassigned. + partitionConsumers := make([]partitionConsumer, cap(b.partOwners)) // partNum => consuming member + for i := range partitionConsumers { + partitionConsumers[i] = partitionConsumer{unassignedPart, unassignedPart} + } + for memberNum := range b.plan { + partNums := &b.plan[memberNum] + for _, partNum := range *partNums { + topicNum := b.partOwners[partNum] + if len(topicPotentials[topicNum]) == 0 { // all prior subscriptions stopped wanting this partition + partNums.remove(partNum) + continue + } + memberTopics := b.members[memberNum].Topics + var memberStillWantsTopic bool + if slices.Contains(memberTopics, b.topicInfos[topicNum].topic) { + memberStillWantsTopic = true + } + if !memberStillWantsTopic { + partNums.remove(partNum) + continue + } + partitionConsumers[partNum] = partitionConsumer{uint16(memberNum), uint16(memberNum)} + } + } + + b.tryRestickyStales(topicPotentials, partitionConsumers) + + // For each member, we now sort their current partitions by partition, + // then topic. Sorting the lowest numbers first means that once we + // steal from the end (when adding a member), we steal equally across + // all topics. This benefits the standard case the most, where all + // members consume equally. + for memberNum := range b.plan { + b.sortMemberByLiteralPartNum(memberNum) + } + + // KIP-881: rack-aware pre-assignment for the simple path. For + // unassigned partitions with rack info, preferentially assign to + // rack-matched members via per-rack heaps. Anything left unassigned + // falls through to the normal heap below. The complex path handles + // rack preference inline via tie-breaking (see below). + if b.partRacks != nil && !b.isComplex { + b.assignRackAware(partitionConsumers, topicPotentials) + } + + if !b.isComplex && len(topicPotentials) > 0 { + potentials := topicPotentials[0] + (&membersByPartitions{potentials, b.plan}).init() + for partNum, owner := range partitionConsumers { + if owner.memberNum != unassignedPart { + continue + } + assigned := potentials[0] + b.plan[assigned].add(int32(partNum)) + (&membersByPartitions{potentials, b.plan}).fix0() + partitionConsumers[partNum].memberNum = assigned + } + } else { + for partNum, owner := range partitionConsumers { + if owner.memberNum != unassignedPart { + continue + } + potentials := topicPotentials[b.partOwners[partNum]] + if len(potentials) == 0 { + continue + } + // KIP-881: when partition racks are available, break + // ties among equally-loaded members by preferring a + // rack-matched member. This preserves optimal balance + // while improving rack locality without needing a + // separate pre-assignment pass. + var pRack uint16 // noRack (0) when no rack info + if b.partRacks != nil { + pRack = b.partRacks[partNum] + } + leastConsumingPotential := potentials[0] + leastConsuming := len(b.plan[leastConsumingPotential]) + bestRackMatch := pRack != noRack && b.memberRacks[leastConsumingPotential] == pRack + for _, potential := range potentials[1:] { + potentialConsuming := len(b.plan[potential]) + rackMatch := pRack != noRack && b.memberRacks[potential] == pRack + if potentialConsuming < leastConsuming || + potentialConsuming == leastConsuming && rackMatch && !bestRackMatch { + leastConsumingPotential = potential + leastConsuming = potentialConsuming + bestRackMatch = rackMatch + } + } + b.plan[leastConsumingPotential].add(int32(partNum)) + partitionConsumers[partNum].memberNum = leastConsumingPotential + } + } + + // Lastly, with everything assigned, we build our steal graph for + // balancing if needed. + if b.isComplex { + b.stealGraph = b.newGraph( + partitionConsumers, + topicPotentials, + ) + } +} + +// unassignedPart is a fake member number that we use to track if a partition +// is deleted or unassigned. +const unassignedPart = math.MaxUint16 - 1 + +// noRack is the sentinel for "no rack info" in memberRacks / partRacks. +// Rack indices are 1-based so that zero-initialized slices default to noRack. +const noRack = 0 + +// tryRestickyStales is a pre-assigning step where, for all stale members, +// we give partitions back to them if the partition is currently on an +// over loaded member or unassigned. +// +// This effectively re-stickies members before we balance further. +func (b *balancer) tryRestickyStales( + topicPotentials [][]uint16, + partitionConsumers []partitionConsumer, +) { + for staleNum, lastOwnerNum := range b.stales { + potentials := topicPotentials[b.partOwners[staleNum]] // there must be a potential consumer if we are here + var canTake bool + for _, potentialNum := range potentials { + if potentialNum == lastOwnerNum { + canTake = true + } + } + if !canTake { + return + } + + // The part cannot be unassigned here; a stale member + // would just have it. The part also cannot be deleted; + // if it is, there are no potential consumers and the + // logic above continues before getting here. The part + // must be on a different owner (cannot be lastOwner), + // otherwise it would not be a lastOwner in the stales + // map; it would just be the current owner. + currentOwner := partitionConsumers[staleNum].memberNum + lastOwnerPartitions := &b.plan[lastOwnerNum] + currentOwnerPartitions := &b.plan[currentOwner] + if len(*lastOwnerPartitions)+1 < len(*currentOwnerPartitions) { + currentOwnerPartitions.remove(staleNum) + lastOwnerPartitions.add(staleNum) + } + } +} + +// assignRackAware pre-assigns unassigned partitions to members whose rack +// matches the partition's leader rack, without exceeding the balanced +// quota. Uses per-rack min-heaps for O(log M) per assignment. Only called +// for the simple (uniform subscription) path; the complex path handles +// rack preference inline via tie-breaking in the main assignment loop. +func (b *balancer) assignRackAware( + partitionConsumers []partitionConsumer, + topicPotentials [][]uint16, +) { + if len(topicPotentials) == 0 { + return + } + maxQuota := (cap(b.partOwners) + len(b.members) - 1) / len(b.members) + + // Build per-rack heaps indexed by rack index. We track which + // indices are populated so setup is O(members) not O(nRacks). + type rackHeap struct { + mbp membersByPartitions + } + rackHeaps := make([]rackHeap, b.nRacks) + rackCount := make([]int, b.nRacks) + var populated []uint16 + // Count members per rack and track which rack indices are populated. + for _, m := range topicPotentials[0] { + rack := b.memberRacks[m] + if rack != noRack { + ri := rack - 1 + if rackCount[ri] == 0 { + populated = append(populated, ri) + } + rackCount[ri]++ + } + } + // Preallocate each populated rack's member slice. + for _, i := range populated { + rackHeaps[i].mbp.members = make([]uint16, 0, rackCount[i]) + } + // Place each member into its rack's heap. + for _, m := range topicPotentials[0] { + rack := b.memberRacks[m] + if rack != noRack { + rackHeaps[rack-1].mbp.members = append(rackHeaps[rack-1].mbp.members, m) + } + } + // Initialize each rack's min-heap by partition count. + for _, i := range populated { + rackHeaps[i].mbp.plan = b.plan + rackHeaps[i].mbp.init() + } + // Assign unassigned partitions to rack-matched members under quota. + for partNum, owner := range partitionConsumers { + if owner.memberNum != unassignedPart { + continue + } + pRack := b.partRacks[partNum] + if pRack == noRack { + continue + } + rh := &rackHeaps[pRack-1] + if len(rh.mbp.members) == 0 { + continue + } + candidate := rh.mbp.members[0] + if len(b.plan[candidate]) >= maxQuota { + continue + } + b.plan[candidate].add(int32(partNum)) + rh.mbp.fix0() + partitionConsumers[partNum] = partitionConsumer{candidate, candidate} + } +} + +type partitionConsumer struct { + memberNum uint16 + originalNum uint16 +} + +// While assigning, we keep members per topic heap sorted by the number of +// partitions they are currently consuming. This allows us to have quick +// assignment vs. always scanning to see the min loaded member. +// +// Our process is to init the heap and then always fix the 0th index after +// making it larger, so we only ever need to sift down. +type membersByPartitions struct { + members []uint16 + plan membersPartitions +} + +func (m *membersByPartitions) init() { + n := len(m.members) + for i := n/2 - 1; i >= 0; i-- { + m.down(i, n) + } +} + +func (m *membersByPartitions) fix0() { + m.down(0, len(m.members)) +} + +func (m *membersByPartitions) down(i0, n int) { + node := i0 + for { + left := 2*node + 1 + if left >= n || left < 0 { // left < 0 after int overflow + break + } + swap := left // left child + swapLen := len(m.plan[m.members[left]]) + if right := left + 1; right < n { + if rightLen := len(m.plan[m.members[right]]); rightLen < swapLen { + swapLen = rightLen + swap = right + } + } + nodeLen := len(m.plan[m.members[node]]) + if nodeLen <= swapLen { + break + } + m.members[node], m.members[swap] = m.members[swap], m.members[node] + node = swap + } +} + +// balance loops trying to move partitions until the plan is as balanced +// as it can be. +func (b *balancer) balance() { + if b.isComplex { + b.balanceComplex() + return + } + + // If all partitions are consumed equally, we have a very easy + // algorithm to balance: while the min and max levels are separated + // by over two, take from the top and give to the bottom. + min := b.planByNumPartitions.min().item + max := b.planByNumPartitions.max().item + for { + if max.level <= min.level+1 { + return + } + + minMems := min.members + maxMems := max.members + for len(minMems) > 0 && len(maxMems) > 0 { + dst := minMems[0] + src := maxMems[0] + + minMems = minMems[1:] + maxMems = maxMems[1:] + + srcPartitions := &b.plan[src] + dstPartitions := &b.plan[dst] + + dstPartitions.add(srcPartitions.takeEnd()) + } + + nextUp := b.findLevel(min.level + 1) + nextDown := b.findLevel(max.level - 1) + + endOfUps := len(min.members) - len(minMems) + endOfDowns := len(max.members) - len(maxMems) + + nextUp.members = append(nextUp.members, min.members[:endOfUps]...) + nextDown.members = append(nextDown.members, max.members[:endOfDowns]...) + + min.members = min.members[endOfUps:] + max.members = max.members[endOfDowns:] + + if len(min.members) == 0 { + b.planByNumPartitions.delete(b.planByNumPartitions.min()) + min = b.planByNumPartitions.min().item + } + if len(max.members) == 0 { + b.planByNumPartitions.delete(b.planByNumPartitions.max()) + max = b.planByNumPartitions.max().item + } + } +} + +func (b *balancer) balanceComplex() { + for min := b.planByNumPartitions.min(); b.planByNumPartitions.size > 1; min = b.planByNumPartitions.min() { + level := min.item + // If this max level is within one of this level, then nothing + // can steal down so we return early. + max := b.planByNumPartitions.max().item + if max.level <= level.level+1 { + return + } + // We continually loop over this level until every member is + // static (deleted) or bumped up a level. + for len(level.members) > 0 { + memberNum := level.members[0] + if stealPath, found := b.stealGraph.findSteal(memberNum); found { + for _, segment := range stealPath { + b.reassignPartition(segment.src, segment.dst, segment.part) + } + if len(max.members) == 0 { + break + } + continue + } + + // If we could not find a steal path, this + // member is not static (will never grow). + level.removeMember(memberNum) + if len(level.members) == 0 { + b.planByNumPartitions.delete(b.planByNumPartitions.min()) + } + } + } +} + +func (b *balancer) reassignPartition(src, dst uint16, partNum int32) { + srcPartitions := &b.plan[src] + dstPartitions := &b.plan[dst] + + oldSrcLevel := len(*srcPartitions) + oldDstLevel := len(*dstPartitions) + + srcPartitions.remove(partNum) + dstPartitions.add(partNum) + + b.fixMemberLevel( + b.planByNumPartitions.findWith(func(n *partitionLevel) int { + return oldSrcLevel - n.level + }), + src, + *srcPartitions, + ) + b.fixMemberLevel( + b.planByNumPartitions.findWith(func(n *partitionLevel) int { + return oldDstLevel - n.level + }), + dst, + *dstPartitions, + ) + + b.stealGraph.changeOwnership(partNum, dst) +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/sync_mutex.go b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/sync_mutex.go new file mode 100644 index 00000000000..1bf42c4ec5e --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/sync_mutex.go @@ -0,0 +1,13 @@ +//go:build !synctests + +// Package xsync provides Mutex and RWMutex types that are aliases for +// sync.Mutex / sync.RWMutex in normal builds and channel-backed +// reimplementations under the "synctests" build tag. The channel-backed +// versions let testing/synctest treat mutex waits as durably blocked so the +// bubble's virtual clock can advance. +package xsync + +import "sync" + +type Mutex = sync.Mutex +type RWMutex = sync.RWMutex diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/synctest_mutex.go b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/synctest_mutex.go new file mode 100644 index 00000000000..6813a875f4e --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/internal/xsync/synctest_mutex.go @@ -0,0 +1,169 @@ +//go:build synctests + +package xsync + +import "sync" + +// Mutex is a channel-based mutex for use with synctest. +// A goroutine blocked on a sync.Mutex is not "durably +// blocked" from synctest's perspective, which prevents time +// from advancing. Channel-based blocking is durably blocked. +type Mutex struct { + once sync.Once + ch chan struct{} +} + +func (m *Mutex) init() { + m.once.Do(func() { + m.ch = make(chan struct{}, 1) + m.ch <- struct{}{} + }) +} + +func (m *Mutex) Lock() { + m.init() + <-m.ch +} + +func (m *Mutex) TryLock() bool { + m.init() + select { + case <-m.ch: + return true + default: + return false + } +} + +func (m *Mutex) Unlock() { + m.init() + select { + case m.ch <- struct{}{}: + default: + panic("sync: unlock of unlocked mutex") + } +} + +// RWMutex is a channel-based readers-writer mutex with +// writer priority. +// +// The design uses a "gate token" pattern: a buffered-1 +// channel holds a single token. Readers take the token and +// immediately return it (passing through the gate). Writers +// take the token and hold it, which blocks all new readers +// until the writer unlocks. This provides writer priority +// naturally — the moment a writer calls Lock, new RLock +// calls block on the gate. +// +// A separate writerSignal channel (buffered 1) is used by +// the last active reader to wake a waiting writer. A stale +// signal can accumulate if readers drain while no writer is +// waiting; Lock drains any stale signal before checking the +// reader count. +type RWMutex struct { + once sync.Once + mu Mutex + gate chan struct{} + readerCount int + writerSignal chan struct{} +} + +func (rw *RWMutex) init() { + rw.once.Do(func() { + rw.gate = make(chan struct{}, 1) + rw.gate <- struct{}{} + rw.writerSignal = make(chan struct{}, 1) + rw.mu.init() + }) +} + +func (rw *RWMutex) RLock() { + rw.init() + <-rw.gate + rw.mu.Lock() + rw.readerCount++ + rw.mu.Unlock() + rw.gate <- struct{}{} +} + +func (rw *RWMutex) TryRLock() bool { + rw.init() + select { + case <-rw.gate: + default: + return false + } + rw.mu.Lock() + rw.readerCount++ + rw.mu.Unlock() + rw.gate <- struct{}{} + return true +} + +func (rw *RWMutex) RUnlock() { + rw.mu.Lock() + rw.readerCount-- + if rw.readerCount < 0 { + rw.mu.Unlock() + panic("sync: RUnlock of unlocked RWMutex") + } + if rw.readerCount == 0 { + select { + case rw.writerSignal <- struct{}{}: + default: + } + } + rw.mu.Unlock() +} + +func (rw *RWMutex) Lock() { + rw.init() + <-rw.gate + select { + case <-rw.writerSignal: + default: + } + rw.mu.Lock() + if rw.readerCount > 0 { + rw.mu.Unlock() + <-rw.writerSignal + } else { + rw.mu.Unlock() + } +} + +func (rw *RWMutex) TryLock() bool { + rw.init() + select { + case <-rw.gate: + default: + return false + } + select { + case <-rw.writerSignal: + default: + } + rw.mu.Lock() + if rw.readerCount > 0 { + rw.mu.Unlock() + rw.gate <- struct{}{} + return false + } + rw.mu.Unlock() + return true +} + +func (rw *RWMutex) Unlock() { + select { + case rw.gate <- struct{}{}: + default: + panic("sync: Unlock of unlocked RWMutex") + } +} + +func (rw *RWMutex) RLocker() sync.Locker { return (*rlocker)(rw) } + +type rlocker RWMutex + +func (r *rlocker) Lock() { (*RWMutex)(r).RLock() } +func (r *rlocker) Unlock() { (*RWMutex)(r).RUnlock() } diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/logger.go b/vendor/github.com/twmb/franz-go/pkg/kgo/logger.go new file mode 100644 index 00000000000..b9f1cd2465f --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/logger.go @@ -0,0 +1,140 @@ +package kgo + +import ( + "bytes" + "fmt" + "io" + "strings" +) + +// LogLevel designates which level the logger should log at. +type LogLevel int8 + +const ( + // LogLevelNone disables logging. + LogLevelNone LogLevel = iota + // LogLevelError logs all errors. Generally, these should not happen. + LogLevelError + // LogLevelWarn logs all warnings, such as request failures. + LogLevelWarn + // LogLevelInfo logs informational messages, such as requests. This is + // usually the default log level. + LogLevelInfo + // LogLevelDebug logs verbose information, and is usually not used in + // production. + LogLevelDebug +) + +func (l LogLevel) String() string { + switch l { + case LogLevelError: + return "ERROR" + case LogLevelWarn: + return "WARN" + case LogLevelInfo: + return "INFO" + case LogLevelDebug: + return "DEBUG" + default: + return "NONE" + } +} + +// Logger is used to log informational messages. +type Logger interface { + // Level returns the log level to log at. + // + // Implementations can change their log level on the fly, but this + // function must be safe to call concurrently. + Level() LogLevel + + // Log logs a message with key, value pair arguments for the given log + // level. Keys are always strings, while values can be any type. + // + // This must be safe to call concurrently. + Log(level LogLevel, msg string, keyvals ...any) +} + +// BasicLogger returns a logger that will print to dst in the following format: +// +// prefix [LEVEL] message; key: val, key: val +// +// prefixFn is optional; if non-nil, it is called for a per-message prefix. +// +// Writes to dst are not checked for errors. +func BasicLogger(dst io.Writer, level LogLevel, prefixFn func() string) Logger { + return &basicLogger{dst, level, prefixFn} +} + +type basicLogger struct { + dst io.Writer + level LogLevel + pfxFn func() string +} + +func (b *basicLogger) Level() LogLevel { return b.level } +func (b *basicLogger) Log(level LogLevel, msg string, keyvals ...any) { + buf := byteBuffers.Get().(*bytes.Buffer) + defer byteBuffers.Put(buf) + + buf.Reset() + if b.pfxFn != nil { + buf.WriteString(b.pfxFn()) + } + buf.WriteByte('[') + buf.WriteString(level.String()) + buf.WriteString("] ") + buf.WriteString(msg) + + if len(keyvals) > 0 { + buf.WriteString("; ") + format := strings.Repeat("%v: %v, ", len(keyvals)/2) + if len(format) > 1 { + format = format[:len(format)-2] // trim trailing comma and space + } + fmt.Fprintf(buf, format, keyvals...) + } + + buf.WriteByte('\n') + b.dst.Write(buf.Bytes()) +} + +// nopLogger, the default logger, drops everything. +type nopLogger struct{} + +func (*nopLogger) Level() LogLevel { return LogLevelNone } +func (*nopLogger) Log(LogLevel, string, ...any) { +} + +// wrappedLogger wraps the config logger for convenience at logging callsites. +type wrappedLogger struct { + inner Logger +} + +func (w *wrappedLogger) Level() LogLevel { + if w.inner == nil { + return LogLevelNone + } + return w.inner.Level() +} + +func (w *wrappedLogger) Log(level LogLevel, msg string, keyvals ...any) { + if w.Level() < level { + return + } + w.inner.Log(level, msg, keyvals...) +} + +// LoggerFn returns an anonymous function that can be used in other packages +// that support their own anonymous logger functions. +// +// Notably, this was added so that you can easily use a kgo.Logger in the +// sister 'sr' and 'kfake' packages. Both clients can be initialized with a +// 'LogFn' option. This function makes it easy to use the same kgo.Logger +// across the other packages. +// +// func LoggerFn(l Logger) func(int8, string, ...any) { +// return func(lvl int8, msg string, keyvals ...any) { +// l.Log(LogLevel(lvl), msg, keyvals...) +// } +// } diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/metadata.go b/vendor/github.com/twmb/franz-go/pkg/kgo/metadata.go new file mode 100644 index 00000000000..c5615c1ff72 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/metadata.go @@ -0,0 +1,1153 @@ +package kgo + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" +) + +type metawait struct { + mu xsync.Mutex + c *sync.Cond + lastUpdate time.Time +} + +func (m *metawait) init() { m.c = sync.NewCond(&m.mu) } +func (m *metawait) signal() { + m.mu.Lock() + m.lastUpdate = time.Now() + m.mu.Unlock() + m.c.Broadcast() +} + +// ForceMetadataRefresh triggers the client to update the metadata that is +// currently used for producing & consuming. +// +// Internally, the client already properly triggers metadata updates whenever a +// partition is discovered to be out of date (leader moved, epoch is old, etc). +// However, when partitions are added to a topic through a CreatePartitions +// request, it may take up to MetadataMaxAge for the new partitions to be +// discovered. In this case, you may want to forcefully refresh metadata +// manually to discover these new partitions sooner. +func (cl *Client) ForceMetadataRefresh() { + cl.triggerUpdateMetadataNow("from user ForceMetadataRefresh") +} + +// PartitionLeader returns the given topic partition's leader, leader epoch and +// load error. This returns -1, -1, nil if the partition has not been loaded. +func (cl *Client) PartitionLeader(topic string, partition int32) (leader, leaderEpoch int32, err error) { + if partition < 0 { + return -1, -1, errors.New("invalid negative partition") + } + + var t *topicPartitions + + m := cl.producer.topics.load() + if len(m) > 0 { + t = m[topic] + } + if t == nil { + if cl.consumer.g != nil { + if m = cl.consumer.g.tps.load(); len(m) > 0 { + t = m[topic] + } + } else if cl.consumer.d != nil { + if m = cl.consumer.d.tps.load(); len(m) > 0 { + t = m[topic] + } + } + if t == nil { + return -1, -1, nil + } + } + + tv := t.load() + if len(tv.partitions) <= int(partition) { + return -1, -1, tv.loadErr + } + p := tv.partitions[partition] + return p.leader, p.leaderEpoch, p.loadErr +} + +var noid2t = make(map[[16]byte]string) + +func (cl *Client) id2tMap() map[[16]byte]string { + v := cl.id2t.Load() + if v == nil { + return noid2t + } + m := v.(map[[16]byte]string) + if m == nil { + return noid2t + } + return m +} + +// waitmeta returns immediately if metadata was updated within the last second, +// otherwise this waits for up to wait for a metadata update to complete. +func (cl *Client) waitmeta(ctx context.Context, wait time.Duration, why string) { + cl.dowaitmeta(ctx, wait, false, why) +} + +func (cl *Client) dowaitmeta(ctx context.Context, wait time.Duration, force bool, why string) { + now := time.Now() + + if !force { + cl.metawait.mu.Lock() + if now.Sub(cl.metawait.lastUpdate) < cl.cfg.metadataMinAge { + cl.metawait.mu.Unlock() + return + } + cl.metawait.mu.Unlock() + } + + cl.triggerUpdateMetadataNow(why) + + quit := false + done := make(chan struct{}) + timeout := time.NewTimer(wait) + defer timeout.Stop() + + go func() { + defer close(done) + cl.metawait.mu.Lock() + defer cl.metawait.mu.Unlock() + + for !quit { + if now.Sub(cl.metawait.lastUpdate) < cl.cfg.metadataMinAge { + return + } + cl.metawait.c.Wait() + } + }() + + select { + case <-done: + return + case <-timeout.C: + case <-ctx.Done(): + case <-cl.ctx.Done(): + } + + cl.metawait.mu.Lock() + quit = true + cl.metawait.mu.Unlock() + cl.metawait.c.Broadcast() +} + +func (cl *Client) triggerUpdateMetadata(must bool, why string) bool { + if !must { + cl.metawait.mu.Lock() + defer cl.metawait.mu.Unlock() + if time.Since(cl.metawait.lastUpdate) < cl.cfg.metadataMinAge { + return false + } + } + + select { + case cl.updateMetadataCh <- why: + default: + } + return true +} + +func (cl *Client) triggerUpdateMetadataNow(why string) { + select { + case cl.updateMetadataNowCh <- why: + default: + } +} + +func (cl *Client) blockingMetadataFn(fn func()) { + var wg sync.WaitGroup + wg.Add(1) + waitfn := func() { + defer wg.Done() + fn() + } + select { + case cl.blockingMetadataFnCh <- waitfn: + wg.Wait() + case <-cl.ctx.Done(): + } +} + +// updateMetadataLoop updates metadata whenever the update ticker ticks, +// or whenever deliberately triggered. +func (cl *Client) updateMetadataLoop() { + defer close(cl.metadone) + var consecutiveErrors int + var lastAt time.Time + + ticker := time.NewTicker(cl.cfg.metadataMaxAge) + defer ticker.Stop() +loop: + for { + var now bool + select { + case <-cl.ctx.Done(): + return + case <-ticker.C: + // We do not log on the standard update case. + case why := <-cl.updateMetadataCh: + cl.cfg.logger.Log(LogLevelInfo, "metadata update triggered", "why", why) + case why := <-cl.updateMetadataNowCh: + cl.cfg.logger.Log(LogLevelInfo, "immediate metadata update triggered", "why", why) + now = true + case fn := <-cl.blockingMetadataFnCh: + fn() + continue loop + } + + var nowTries int + start: + nowTries++ + if !now { + if wait := cl.cfg.metadataMinAge - time.Since(lastAt); wait > 0 { + timer := time.NewTimer(wait) + prewait: + select { + case <-cl.ctx.Done(): + timer.Stop() + return + case why := <-cl.updateMetadataNowCh: + timer.Stop() + cl.cfg.logger.Log(LogLevelInfo, "immediate metadata update triggered, bypassing normal wait", "why", why) + case <-timer.C: + case fn := <-cl.blockingMetadataFnCh: + fn() + goto prewait + } + } + } + + // Even with an "update now", we sleep just a bit to allow some + // potential pile on now triggers. + time.Sleep(time.Until(lastAt.Add(10 * time.Millisecond))) + + // Drain any refires that occurred during our waiting. + out: + for { + select { + case <-cl.updateMetadataCh: + case <-cl.updateMetadataNowCh: + case fn := <-cl.blockingMetadataFnCh: + fn() + default: + break out + } + } + + retryWhy, err := cl.updateMetadata() + lastAt = time.Now() + if retryWhy != nil || err != nil { + // If err is non-nil, the metadata request failed + // itself and already retried 3x; we do not loop more. + // + // If err is nil, the a topic or partition had a load + // error and is perhaps still being created. We retry a + // few more times to give Kafka a chance to figure + // things out. By default this will put us at 2s of + // looping+waiting (250ms per wait, 8x), and if things + // still fail we will fall into the slower update below + // which waits (default) 5s between tries. + if now && err == nil && nowTries < 8 { + wait := min(cl.cfg.metadataMinAge, 250*time.Millisecond) + cl.cfg.logger.Log(LogLevelDebug, "immediate metadata update had inner errors, re-updating", + "errors", retryWhy.reason(""), + "update_after", wait, + ) + timer := time.NewTimer(wait) + quickbackoff: + select { + case <-cl.ctx.Done(): + timer.Stop() + return + case <-timer.C: + case fn := <-cl.blockingMetadataFnCh: + fn() + goto quickbackoff + } + goto start + } + if err != nil { + cl.triggerUpdateMetadata(true, fmt.Sprintf("re-updating metadata due to err: %s", err)) + } else { + cl.triggerUpdateMetadata(true, retryWhy.reason("re-updating due to inner errors")) + } + } + if err == nil { + cl.metawait.signal() + cl.consumer.doOnMetadataUpdate() + consecutiveErrors = 0 + continue + } + + consecutiveErrors++ + // We sleep a bit in case the max metadata age is very small; + // typically this sleep is inconsequential. + after := time.NewTimer(cl.cfg.retryBackoff(consecutiveErrors)) + backoff: + select { + case <-cl.ctx.Done(): + after.Stop() + return + case <-after.C: + case fn := <-cl.blockingMetadataFnCh: + fn() + goto backoff + } + } +} + +var errMissingTopic = errors.New("topic_missing") + +// Updates all producer and consumer partition data, returning whether a new +// update needs scheduling or if an error occurred. +// +// The producer and consumer use different topic maps and underlying +// topicPartitionsData pointers, but we update those underlying pointers +// equally. +func (cl *Client) updateMetadata() (retryWhy multiUpdateWhy, err error) { + var ( + tpsProducerLoad = cl.producer.topics.load() + tpsConsumer *topicsPartitions + groupExternal *groupExternal + all = cl.cfg.regex + reqTopics []string + ) + c := &cl.consumer + switch { + case c.g != nil: + tpsConsumer = c.g.tps + groupExternal = c.g.loadExternal() + case c.s != nil: + tpsConsumer = c.s.tps + case c.d != nil: + tpsConsumer = c.d.tps + } + + if !all { + reqTopicsSet := make(map[string]struct{}) + for _, m := range []map[string]*topicPartitions{ + tpsProducerLoad, + tpsConsumer.load(), + } { + for topic := range m { + reqTopicsSet[topic] = struct{}{} + } + } + groupExternal.eachTopic(func(t string) { + reqTopicsSet[t] = struct{}{} + }) + reqTopics = make([]string, 0, len(reqTopicsSet)) + for topic := range reqTopicsSet { + reqTopics = append(reqTopics, topic) + } + } + + // If the user has auto topic creation while producing AND is consuming + // via regex, we need to send a separate metadata request with unknown + // produce topics before our standard metadata request. The standard + // metadata request is send with a nil Topics field (requesting all), + // which prevents us from ever creating the topics. + var unknownCreateResp map[string]*metadataTopic + if all && cl.cfg.allowAutoTopicCreation { + cl.producer.unknownTopicsMu.Lock() + if len(cl.producer.unknownTopics) > 0 { + unknownTopics := make([]string, 0, len(cl.producer.unknownTopics)) + for unknown := range cl.producer.unknownTopics { + unknownTopics = append(unknownTopics, unknown) + } + var err error + unknownCreateResp, err = cl.fetchTopicMetadata(false, unknownTopics) + if err != nil { + // We bump all produce topics even though we + // only explicitly requested unknown ones; this + // is a general request failure and we want to + // note it (also this is simpler...). + cl.bumpMetadataFailForTopics( + tpsProducerLoad, + err, + ) + } + } + cl.producer.unknownTopicsMu.Unlock() + } + + latest, err := cl.fetchTopicMetadata(all, reqTopics) + if err != nil { + cl.bumpMetadataFailForTopics( // bump load failures for all topics + tpsProducerLoad, + err, + ) + return nil, err + } + groupExternal.updateLatest(latest) + + // If regex consuming AND we issued a metadata request to forcefully + // create topics, we merge any topics missing into the all-request from + // the create-request. It is possible we want to keep failed creation + // errors. + for t, mt := range unknownCreateResp { + if _, ok := latest[t]; !ok { + latest[t] = mt + } + } + + // Merge topic ID mappings into the existing id2t map. We clone + // rather than rebuild so that topics missing from this particular + // metadata response (transient broker omission, non-all request) + // retain their ID mapping from prior responses. + // + // If a topic was deleted and recreated, the broker returns a new + // ID for the same name. We do NOT add the new ID if the old ID + // is still present - the old mapping is preserved until the user + // explicitly purges via PurgeTopicsFromClient. This avoids having + // two IDs for the same topic name. + { + old := cl.id2tMap() + merged := make(map[[16]byte]string, len(old)+len(latest)) + maps.Copy(merged, old) + + // Build the set of topic names that already have an ID. + knownNames := make(map[string]struct{}, len(merged)) + for _, name := range merged { + knownNames[name] = struct{}{} + } + + for _, mt := range latest { + if mt.id == ([16]byte{}) { + continue + } + if _, exists := knownNames[mt.topic]; exists { + // This name already has an ID in the map. + // Only update if it's the same ID (normal + // case), skip if it's a different ID + // (recreated topic). + if _, sameID := merged[mt.id]; sameID { + merged[mt.id] = mt.topic + } + continue + } + merged[mt.id] = mt.topic + knownNames[mt.topic] = struct{}{} + } + cl.id2t.Store(merged) + } + + // If we are consuming with regex and fetched all topics, the metadata + // may have returned topics the consumer is not yet tracking. We ensure + // that we will store the topics at the end of our metadata update. + tpsConsumerLoad := tpsConsumer.load() + if all { + allTopics := make([]string, 0, len(latest)) + for topic, mt := range latest { + // loadErr should only be non-nil when requesting all + // topics if this is with auto-topic-creation && the + // creation failed. That is, we should not consume the + // topic since we just tried creating it and creating + // it failed. + if mt.loadErr == nil { + allTopics = append(allTopics, topic) + } + } + + // We filter out topics will not match any of our regex's. + // This ensures that the `tps` field does not contain topics + // we will never use (the client works with misc. topics in + // there, but it's better to avoid it -- and allows us to use + // `tps` in GetConsumeTopics). + allTopics = c.filterMetadataAllTopics(allTopics) + + tpsConsumerLoad = tpsConsumer.ensureTopics(allTopics) + defer tpsConsumer.storeData(tpsConsumerLoad) + + // For regex consuming, if a topic is not returned in the + // response and for at least missingTopicDelete from when we + // first discovered it, we assume the topic has been deleted + // and purge it. We allow for missingTopicDelete because (in + // testing locally) Kafka can originally broadcast a newly + // created topic exists and then fail to broadcast that info + // again for a while. + var purgeTopics []string + for topic, tps := range tpsConsumerLoad { + if _, ok := latest[topic]; !ok { + if td := tps.load(); td.when != 0 && time.Since(time.Unix(td.when, 0)) > cl.cfg.missingTopicDelete { + purgeTopics = append(purgeTopics, td.topic) + } else { + retryWhy.add(topic, -1, errMissingTopic) + } + } + } + if len(purgeTopics) > 0 { + // We have to `go` because Purge issues a blocking + // metadata fn; this will wait for our current + // execution to finish then purge. + cl.cfg.logger.Log(LogLevelInfo, "regex consumer purging topics that were previously consumed because they are missing in a metadata response, we are assuming they are deleted", "topics", purgeTopics) + go cl.PurgeTopicsFromClient(purgeTopics...) + } + } + + css := &consumerSessionStopper{cl: cl} + defer css.maybeRestart() + + consumerKind := partitionKindConsume + if cl.consumer.s != nil { + consumerKind = partitionKindShare + } + var missingProduceTopics []*topicPartitions + for _, m := range []struct { + priors map[string]*topicPartitions + kind partitionKind + }{ + {tpsProducerLoad, partitionKindProduce}, + {tpsConsumerLoad, consumerKind}, + } { + for topic, priorParts := range m.priors { + newParts, exists := latest[topic] + if !exists { + if m.kind == partitionKindProduce { + missingProduceTopics = append(missingProduceTopics, priorParts) + } + continue + } + cl.mergeTopicPartitions( + topic, + priorParts, + newParts, + m.kind, + css, + &retryWhy, + ) + } + } + + // For all produce topics that were missing, we want to bump their + // retries that a failure happened. However, if we are regex consuming, + // then it is possible in a rare scenario for the broker to not return + // a topic that actually does exist and that we previously received a + // metadata response for. This is handled above for consuming, we now + // handle it the same way for consuming. + if len(missingProduceTopics) > 0 { + var bumpFail []string + for _, tps := range missingProduceTopics { + if all { + if td := tps.load(); td.when != 0 && time.Since(time.Unix(td.when, 0)) > cl.cfg.missingTopicDelete { + bumpFail = append(bumpFail, td.topic) + } else { + retryWhy.add(td.topic, -1, errMissingTopic) + } + } else { + bumpFail = append(bumpFail, tps.load().topic) + } + } + if len(bumpFail) > 0 { + cl.bumpMetadataFailForTopics( + tpsProducerLoad, + fmt.Errorf("metadata request did not return topics: %v", bumpFail), + bumpFail..., + ) + } + } + + return retryWhy, nil +} + +// We use a special structure to represent metadata before we *actually* convert +// it to topicPartitionsData. This helps avoid any pointer reuse problems +// because we want to keep the client's producer and consumer maps completely +// independent. If we just returned map[string]*topicPartitionsData, we could +// end up in some really weird pointer reuse scenario that ultimately results +// in a bug. +// +// See #190 for more details, as well as the commit message introducing this. +type metadataTopic struct { + loadErr error + isInternal bool + topic string + id [16]byte + partitions []metadataPartition +} + +func (mt *metadataTopic) newPartitions(cl *Client, kind partitionKind) *topicPartitionsData { + n := len(mt.partitions) + ps := &topicPartitionsData{ + loadErr: mt.loadErr, + isInternal: mt.isInternal, + partitions: make([]*topicPartition, 0, n), + writablePartitions: make([]*topicPartition, 0, n), + topic: mt.topic, + id: mt.id, + when: time.Now().Unix(), + } + for i := range mt.partitions { + p := mt.partitions[i].newPartition(cl, kind) + ps.partitions = append(ps.partitions, p) + if p.loadErr == nil { + ps.writablePartitions = append(ps.writablePartitions, p) + } + } + return ps +} + +type metadataPartition struct { + topic string + topicID [16]byte + partition int32 + loadErr int16 + leader int32 + leaderEpoch int32 + sns sinkAndSource +} + +func (mp metadataPartition) newPartition(cl *Client, kind partitionKind) *topicPartition { + td := topicPartitionData{ + leader: mp.leader, + leaderEpoch: mp.leaderEpoch, + } + p := &topicPartition{ + loadErr: kerr.ErrorForCode(mp.loadErr), + topicPartitionData: td, + } + switch kind { + case partitionKindProduce: + r := &recBuf{ + cl: cl, + topic: mp.topic, + topicID: mp.topicID, + partition: mp.partition, + maxRecordBatchBytes: cl.maxRecordBatchBytesForTopic(mp.topic), + recBufsIdx: -1, + failing: mp.loadErr != 0, + sink: mp.sns.sink, + topicPartitionData: td, + lastAckedOffset: -1, + } + r.lingerFn = r.unlingerAndManuallyDrain + p.records = r + case partitionKindShare: + p.shareCursor = &shareCursor{ + topic: mp.topic, + topicID: mp.topicID, + partition: mp.partition, + cursorsIdx: -1, // sentinel: not yet added to a source + } + p.shareCursor.source.Store(mp.sns.source) + default: + p.cursor = &cursor{ + topic: mp.topic, + topicID: mp.topicID, + partition: mp.partition, + keepControl: cl.cfg.keepControl, + cursorsIdx: -1, + source: mp.sns.source, + topicPartitionData: td, + cursorOffset: cursorOffset{ + offset: -1, // required to not consume until needed + lastConsumedEpoch: -1, // required sentinel + }, + } + } + return p +} + +// fetchTopicMetadata fetches metadata for all reqTopics and returns new +// topicPartitionsData for each topic. +func (cl *Client) fetchTopicMetadata(all bool, reqTopics []string) (map[string]*metadataTopic, error) { + _, meta, err := cl.fetchMetadataByName(cl.ctx, all, reqTopics, nil) + if err != nil { + return nil, err + } + + topics := make(map[string]*metadataTopic, len(meta.Topics)) + + // Even if metadata returns a leader epoch, we do not use it unless we + // can validate it per OffsetForLeaderEpoch. Some brokers may have an + // odd set of support. + useLeaderEpoch := cl.supportsOffsetForLeaderEpoch() + + for i := range meta.Topics { + topicMeta := &meta.Topics[i] + if topicMeta.Topic == nil { + cl.cfg.logger.Log(LogLevelWarn, "metadata response contained nil topic name even though we did not request with topic IDs, skipping") + continue + } + topic := *topicMeta.Topic + + mt := &metadataTopic{ + loadErr: kerr.ErrorForCode(topicMeta.ErrorCode), + isInternal: topicMeta.IsInternal, + topic: topic, + id: topicMeta.TopicID, + partitions: make([]metadataPartition, 0, len(topicMeta.Partitions)), + } + + topics[topic] = mt + + if mt.loadErr != nil { + continue + } + + // This 249 limit is in Kafka itself, we copy it here to rely on it while producing. + if len(topic) > 249 { + mt.loadErr = fmt.Errorf("invalid long topic name of (len %d) greater than max allowed 249", len(topic)) + continue + } + + // Kafka partitions are strictly increasing from 0. We enforce + // that here; if any partition is missing, we consider this + // topic a load failure. + sort.Slice(topicMeta.Partitions, func(i, j int) bool { + return topicMeta.Partitions[i].Partition < topicMeta.Partitions[j].Partition + }) + for i := range topicMeta.Partitions { + if got := topicMeta.Partitions[i].Partition; got != int32(i) { + mt.loadErr = fmt.Errorf("kafka did not reply with a comprehensive set of partitions for a topic; we expected partition %d but saw %d", i, got) + break + } + } + + if mt.loadErr != nil { + continue + } + + for i := range topicMeta.Partitions { + partMeta := &topicMeta.Partitions[i] + leaderEpoch := partMeta.LeaderEpoch + if meta.Version < 7 || !useLeaderEpoch { + leaderEpoch = -1 + } + mp := metadataPartition{ + topic: topic, + topicID: topicMeta.TopicID, + partition: partMeta.Partition, + loadErr: partMeta.ErrorCode, + leader: partMeta.Leader, + leaderEpoch: leaderEpoch, + } + if mp.loadErr != 0 { + mp.leader = unknownSeedID(0) // ensure every records & cursor can use a sink or source + } + cl.sinksAndSourcesMu.Lock() + sns, exists := cl.sinksAndSources[mp.leader] + if !exists { + sns = sinkAndSource{ + sink: cl.newSink(mp.leader), + source: cl.newSource(mp.leader), + } + cl.sinksAndSources[mp.leader] = sns + } + for _, replica := range partMeta.Replicas { + if replica < 0 { + continue + } + if _, exists = cl.sinksAndSources[replica]; !exists { + cl.sinksAndSources[replica] = sinkAndSource{ + sink: cl.newSink(replica), + source: cl.newSource(replica), + } + } + } + cl.sinksAndSourcesMu.Unlock() + mp.sns = sns + mt.partitions = append(mt.partitions, mp) + } + } + + return topics, nil +} + +// mergeTopicPartitions merges a new topicPartition into an old and returns +// whether the metadata update that caused this merge needs to be retried. +// +// Retries are necessary if the topic or any partition has a retryable error. +func (cl *Client) mergeTopicPartitions( + topic string, + l *topicPartitions, + mt *metadataTopic, + kind partitionKind, + css *consumerSessionStopper, + retryWhy *multiUpdateWhy, +) { + isProduce := kind == partitionKindProduce + isShare := kind == partitionKindShare + lv := *l.load() // copy so our field writes do not collide with reads + + r := mt.newPartitions(cl, kind) + + // Producers must store the update through a special function that + // manages unknown topic waiting, whereas consumers can just simply + // store the update. + if isProduce { + hadPartitions := len(lv.partitions) != 0 + defer func() { cl.storePartitionsUpdate(topic, l, &lv, hadPartitions) }() + } else { + defer l.v.Store(&lv) + } + + lv.loadErr = r.loadErr + lv.isInternal = r.isInternal + lv.topic = r.topic + lv.id = r.id + if lv.when == 0 { + lv.when = r.when + } + + // If the load had an error for the entire topic, we set the load error + // but keep our stale partition information. For anything being + // produced, we bump the respective error or fail everything. There is + // nothing to be done in a consumer. + if r.loadErr != nil { + if isProduce { + for _, topicPartition := range lv.partitions { + topicPartition.records.bumpRepeatedLoadErr(lv.loadErr) + } + } else if !kerr.IsRetriable(r.loadErr) || cl.cfg.keepRetryableFetchErrors { + cl.consumer.addFakeReadyForDraining(topic, -1, r.loadErr, "metadata refresh has a load error on this entire topic") + } + retryWhy.add(topic, -1, r.loadErr) + return + } + + // Before the atomic update, we keep the latest partitions / writable + // partitions. All updates happen in r's slices, and we keep the + // results and store them in lv. + defer func() { + lv.partitions = r.partitions + lv.writablePartitions = r.writablePartitions + }() + + // We should have no deleted partitions, but there are two cases where + // we could. + // + // 1) an admin added partitions, we saw, then we re-fetched metadata + // from an out of date broker that did not have the new partitions + // + // 2) a topic was deleted and recreated with fewer partitions + // + // Both of these scenarios should be rare to non-existent, and we do + // nothing if we encounter them. + + // Migrating topicPartitions is a little tricky because we have to + // worry about underlying pointers that may currently be loaded. + for part, oldTP := range lv.partitions { + exists := part < len(r.partitions) + if !exists { + // This is the "deleted" case; see the comment above. + // + // We need to keep the partition around. For producing, + // the partition could be loaded and a record could be + // added to it after we bump the load error. For + // consuming, the partition is part of a group or part + // of what was loaded for direct consuming. + // + // We only clear a partition if it is purged from the + // client (which can happen automatically for consumers + // if the user opted into ConsumeRecreatedTopics). + dup := *oldTP + newTP := &dup + newTP.loadErr = errMissingMetadataPartition + + r.partitions = append(r.partitions, newTP) + + cl.cfg.logger.Log(LogLevelDebug, "metadata update is missing partition in topic, we are keeping the partition around for safety -- use PurgeTopicsFromClient if you wish to remove the topic", + "topic", topic, + "partition", part, + ) + if isProduce { + oldTP.records.bumpRepeatedLoadErr(errMissingMetadataPartition) + } + retryWhy.add(topic, int32(part), errMissingMetadataPartition) + continue + } + newTP := r.partitions[part] + + // Like above for the entire topic, an individual partition + // can have a load error. Unlike for the topic, individual + // partition errors are always retryable. + // + // If the load errored, we keep all old information minus the + // load error itself (the new load will have no information). + if newTP.loadErr != nil { + err := newTP.loadErr + *newTP = *oldTP + newTP.loadErr = err + if isProduce { + newTP.records.bumpRepeatedLoadErr(newTP.loadErr) + } else if !kerr.IsRetriable(newTP.loadErr) || cl.cfg.keepRetryableFetchErrors { + cl.consumer.addFakeReadyForDraining(topic, int32(part), newTP.loadErr, "metadata refresh has a load error on this partition") + } + retryWhy.add(topic, int32(part), newTP.loadErr) + continue + } + + // If the new partition has an older leader epoch, then we + // fetched from an out of date broker. We just keep the old + // information. + if newTP.leaderEpoch < oldTP.leaderEpoch { + // If we repeatedly rewind, then perhaps the cluster + // entered some bad state and lost forward progress. + // We will log & allow the rewind to allow the client + // to continue; other requests may encounter fenced + // epoch errors (and respectively recover). + // + // Five is a pretty low amount of retries, but since + // we iterate through known brokers, this basically + // means we keep stale metadata if five brokers all + // agree things rewound. + const maxEpochRewinds = 5 + if oldTP.epochRewinds < maxEpochRewinds { + cl.cfg.logger.Log(LogLevelDebug, "metadata leader epoch went backwards, ignoring update", + "topic", topic, + "partition", part, + "old_leader_epoch", oldTP.leaderEpoch, + "new_leader_epoch", newTP.leaderEpoch, + "current_num_rewinds", oldTP.epochRewinds+1, + ) + *newTP = *oldTP + newTP.epochRewinds++ + retryWhy.add(topic, int32(part), errEpochRewind) + continue + } + + cl.cfg.logger.Log(LogLevelInfo, "metadata leader epoch went backwards repeatedly, we are now keeping the metadata to allow forward progress", + "topic", topic, + "partition", part, + "old_leader_epoch", oldTP.leaderEpoch, + "new_leader_epoch", newTP.leaderEpoch, + ) + } + + if !isProduce { + var noID [16]byte + var newID, oldID [16]byte + if isShare { + newID = newTP.shareCursor.topicID + oldID = oldTP.shareCursor.topicID + } else { + newID = newTP.cursor.topicID + oldID = oldTP.cursor.topicID + } + if newID == noID && oldID != noID { + cl.cfg.logger.Log(LogLevelWarn, "metadata update is missing the topic ID when we previously had one, ignoring update", + "topic", topic, + "partition", part, + ) + *newTP = *oldTP + retryWhy.add(topic, int32(part), errMissingTopicID) + continue + } + } + + // If the tp data is the same, we simply copy over the records + // and cursor pointers. + // + // If the tp data equals the old, then the sink / source is the + // same, because the sink/source is from the tp leader. + if newTP.topicPartitionData == oldTP.topicPartitionData { + cl.cfg.logger.Log(LogLevelDebug, "metadata refresh has identical topic partition data", + "topic", topic, + "partition", part, + "leader", newTP.leader, + "leader_epoch", newTP.leaderEpoch, + ) + switch kind { + case partitionKindProduce: + newTP.records = oldTP.records + newTP.records.clearFailing() // always clear failing state for producing after meta update + case partitionKindShare: + newTP.shareCursor = oldTP.shareCursor + default: + newTP.cursor = oldTP.cursor // unlike records, there is no failing state for a cursor + } + } else { + cl.cfg.logger.Log(LogLevelDebug, "metadata refresh topic partition data changed", + "topic", topic, + "partition", part, + "new_leader", newTP.leader, + "new_leader_epoch", newTP.leaderEpoch, + "old_leader", oldTP.leader, + "old_leader_epoch", oldTP.leaderEpoch, + ) + switch kind { + case partitionKindProduce: + oldTP.migrateProductionTo(newTP) // migration clears failing state + case partitionKindShare: + oldTP.migrateShareCursorTo(cl, newTP) + default: + oldTP.migrateCursorTo(newTP, css) + } + } + } + + // For any partitions **not currently in use**, we need to add them to + // the sink or source. If they are in use, they could be getting + // managed or moved by the sink or source itself, so we should not + // check the index field (which may be concurrently modified). + if len(lv.partitions) > len(r.partitions) { + return + } + newPartitions := r.partitions[len(lv.partitions):] + + // Anything left with a negative recBufsIdx / cursorsIdx is a new topic + // partition and must be added to the sink / source. + for _, newTP := range newPartitions { + if newTP.loadErr != nil { + if isProduce { + newTP.records.bumpRepeatedLoadErr(newTP.loadErr) + } else if !kerr.IsRetriable(newTP.loadErr) || cl.cfg.keepRetryableFetchErrors { + cl.consumer.addFakeReadyForDraining(topic, newTP.partition(), newTP.loadErr, "metadata refresh has a load error on a new partition") + } + retryWhy.add(topic, newTP.partition(), newTP.loadErr) + } + switch kind { + case partitionKindProduce: + if newTP.records.recBufsIdx == -1 { + newTP.records.sink.addRecBuf(newTP.records) + cl.cfg.logger.Log(LogLevelDebug, "metadata refresh new produce partition", + "topic", topic, + "partition", newTP.partition(), + "leader", newTP.leader, + "leader_epoch", newTP.leaderEpoch, + ) + } + case partitionKindShare: + if newTP.shareCursor.cursorsIdx == -1 { + newTP.shareCursor.source.Load().addShareCursor(newTP.shareCursor) + cl.cfg.logger.Log(LogLevelDebug, "metadata refresh new share consume partition", + "topic", topic, + "partition", newTP.partition(), + "leader", newTP.leader, + "leader_epoch", newTP.leaderEpoch, + ) + } + default: + if newTP.cursor.cursorsIdx == -1 { + newTP.cursor.source.addCursor(newTP.cursor) + cl.cfg.logger.Log(LogLevelDebug, "metadata refresh new consume partition", + "topic", topic, + "partition", newTP.partition(), + "leader", newTP.leader, + "leader_epoch", newTP.leaderEpoch, + ) + } + } + } +} + +var ( + errEpochRewind = errors.New("epoch rewind") + errMissingTopicID = errors.New("missing topic ID") +) + +type multiUpdateWhy map[kerrOrString]map[string]map[int32]struct{} + +type kerrOrString struct { + k *kerr.Error + s string +} + +func (m *multiUpdateWhy) isOnly(err error) bool { + if m == nil { + return false + } + for e := range *m { + if !errors.Is(err, e.k) { + return false + } + } + return true +} + +func (m *multiUpdateWhy) add(t string, p int32, err error) { + if err == nil { + return + } + + if *m == nil { + *m = make(map[kerrOrString]map[string]map[int32]struct{}) + } + var ks kerrOrString + if ke := (*kerr.Error)(nil); errors.As(err, &ke) { + ks = kerrOrString{k: ke} + } else { + ks = kerrOrString{s: err.Error()} + } + + ts := (*m)[ks] + if ts == nil { + ts = make(map[string]map[int32]struct{}) + (*m)[ks] = ts + } + + ps := ts[t] + if ps == nil { + ps = make(map[int32]struct{}) + ts[t] = ps + } + // -1 signals that the entire topic had an error. + if p != -1 { + ps[p] = struct{}{} + } +} + +// err{topic[1 2 3] topic2[4 5 6]} err2{...} +func (m multiUpdateWhy) reason(reason string) string { + if len(m) == 0 { + return "" + } + + ksSorted := make([]kerrOrString, 0, len(m)) + for err := range m { + ksSorted = append(ksSorted, err) + } + sort.Slice(ksSorted, func(i, j int) bool { // order by non-nil kerr's code, otherwise the string + l, r := ksSorted[i], ksSorted[j] + return l.k != nil && (r.k == nil || l.k.Code < r.k.Code) || r.k == nil && l.s < r.s + }) + + var errorStrings []string + for _, ks := range ksSorted { + ts := m[ks] + tsSorted := make([]string, 0, len(ts)) + for t := range ts { + tsSorted = append(tsSorted, t) + } + sort.Strings(tsSorted) + + var topicStrings []string + for _, t := range tsSorted { + ps := ts[t] + if len(ps) == 0 { + topicStrings = append(topicStrings, t) + } else { + psSorted := make([]int32, 0, len(ps)) + for p := range ps { + psSorted = append(psSorted, p) + } + slices.Sort(psSorted) + topicStrings = append(topicStrings, fmt.Sprintf("%s%v", t, psSorted)) + } + } + + if ks.k != nil { + errorStrings = append(errorStrings, fmt.Sprintf("%s{%s}", ks.k.Message, strings.Join(topicStrings, " "))) + } else { + errorStrings = append(errorStrings, fmt.Sprintf("%s{%s}", ks.s, strings.Join(topicStrings, " "))) + } + } + if reason == "" { + return strings.Join(errorStrings, " ") + } + return reason + ": " + strings.Join(errorStrings, " ") +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/metrics_714.go b/vendor/github.com/twmb/franz-go/pkg/kgo/metrics_714.go new file mode 100644 index 00000000000..927f0f9fdbe --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/metrics_714.go @@ -0,0 +1,986 @@ +package kgo + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "math/rand" + "strings" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func (cl *Client) pushMetrics() { + defer cl.metrics.ctxCancel() + + if cl.cfg.disableClientMetrics { + return + } + + m := &cl.metrics + + select { + case <-cl.ctx.Done(): + return + case <-m.firstObserve: + } + + if !cl.supportsClientMetrics() { + m.mu.Lock() + m.unsupported.Store(true) + m.pReqLatency = nil + m.cReqLatency = nil + m.mu.Unlock() + return + } + + var clientInstanceID [16]byte + var terminating bool + for !terminating { + greq := kmsg.NewGetTelemetrySubscriptionsRequest() + greq.ClientInstanceID = clientInstanceID + + gresp, err := greq.RequestWith(m.ctx, cl) + if err == nil { + err = kerr.ErrorForCode(gresp.ErrorCode) + } + if err != nil { + cl.cfg.logger.Log(LogLevelDebug, "unable to get telemetry subscriptions, retrying in 30s", "err", err) + after := time.NewTimer(30 * time.Second) + select { + case <-cl.ctx.Done(): + after.Stop() + return + case <-after.C: + } + continue + } + + clientInstanceID = gresp.ClientInstanceID + + // If there are no requested metrics, we wait the push interval + // and re-get. + if len(gresp.RequestedMetrics) == 0 { + wait := time.Duration(gresp.PushIntervalMillis) * time.Millisecond + cl.cfg.logger.Log(LogLevelDebug, "no metrics requested, sleeping and asking again later", "sleep", wait) + after := time.NewTimer(wait) + select { + case <-cl.ctx.Done(): + terminating = true + case <-after.C: + } + continue + } + cl.cfg.logger.Log(LogLevelInfo, "received client metrics subscription, beginning periodic send loop", + "client_instance_id", fmt.Sprintf("%x", gresp.ClientInstanceID), + "subscription_id", gresp.SubscriptionID, + "accepted_compression_types", gresp.AcceptedCompressionTypes, + "telemetry_max_bytes", gresp.TelemetryMaxBytes, + "push_interval", time.Duration(gresp.PushIntervalMillis)*time.Millisecond, + "requested_metrics", gresp.RequestedMetrics, + ) + + var codecs []CompressionCodec + for _, accepted := range gresp.AcceptedCompressionTypes { + switch CompressionCodecType(accepted) { + case CodecNone: + case CodecGzip: + codecs = append(codecs, GzipCompression()) + case CodecSnappy: + codecs = append(codecs, SnappyCompression()) + case CodecLz4: + codecs = append(codecs, Lz4Compression()) + case CodecZstd: + codecs = append(codecs, ZstdCompression()) + } + } + compressor, _ := DefaultCompressor(codecs...) + allowedNames := buildNameFilter(gresp.RequestedMetrics) + + // We want to pin pushing metrics to a single broker until that + // broker goes away. To do this, we first issue RequestSharded, + // figure out the broker that responded, and then use that + // broker until we receive errUnknownBroker. + var br *Broker + + push: + for i := 0; !terminating; i++ { + wait := max(time.Duration(gresp.PushIntervalMillis)*time.Millisecond, time.Second) + if i == 0 { // for the first request, jitter 0.5 <= wait <= 1.5 + cl.rng(func(r *rand.Rand) { + wait = time.Duration(float64(wait) * (0.5 + r.Float64())) + }) + } + + // Wait until our push interval; if the client is quitting, + // we immediately send a push with Terminating=true. + after := time.NewTimer(wait) + var terminating bool + select { + case <-cl.ctx.Done(): + terminating = true + case <-after.C: + } + + // Create our request. + preq := kmsg.NewPushTelemetryRequest() + preq.ClientInstanceID = clientInstanceID + preq.SubscriptionID = gresp.SubscriptionID + preq.Terminating = terminating + serialized, compression, nmetrics := m.appendTo(nil, gresp.DeltaTemporality, gresp.TelemetryMaxBytes, allowedNames, compressor) + if len(serialized) > int(gresp.TelemetryMaxBytes) { + cl.cfg.logger.Log(LogLevelWarn, "serialized metrics are larger than the broker provided max limit, sending anyway and hoping for the best", + "max_bytes", gresp.TelemetryMaxBytes, + "serialized_bytes", len(serialized), + ) + } + preq.CompressionType = compression + preq.Metrics = serialized + + cl.cfg.logger.Log(LogLevelDebug, "sending client metrics to broker", "num_metrics", nmetrics) + + // Send our request, pinning to the broker we get a response + // from or using the pinned broker if possible. + var resp kmsg.Response + var err error + doreq: + if br == nil { + shard := cl.RequestSharded(m.ctx, &preq)[0] + resp, err = shard.Resp, shard.Err + if err == nil { + br = cl.Broker(int(shard.Meta.NodeID)) + } + } else { + resp, err = br.RetriableRequest(m.ctx, &preq) + if errors.Is(err, errUnknownBroker) { + br = nil + goto doreq + } + } + if err != nil { + cl.cfg.logger.Log(LogLevelDebug, "unable to send client metrics, resetting subscription", "err", err) + break + } + + // Not much to do on the response besides a bunch of + // error handling. + presp := resp.(*kmsg.PushTelemetryResponse) + switch err := kerr.ErrorForCode(presp.ErrorCode); err { + case nil: + case kerr.InvalidRequest: + cl.cfg.logger.Log(LogLevelError, "client metrics was sent at the wrong time (after sending terminating request already?), exiting metrics loop", "err", err) + return + case kerr.InvalidRecord: + cl.cfg.logger.Log(LogLevelError, "broker could not understand our client metrics serialization, this is perhaps a franz-go bug", "err", err) + return + case kerr.TelemetryTooLarge: + cl.cfg.logger.Log(LogLevelWarn, "client metrics payload was too large (check your broker max telemetry bytes)", "err", err) + // Do nothing; continue aggregating + case kerr.UnknownSubscriptionID: + cl.cfg.logger.Log(LogLevelInfo, "client metrics has an outdated subscription, re-getting our subscription information", "err", err) + break push + case kerr.UnsupportedCompressionType: + cl.cfg.logger.Log(LogLevelInfo, "client metrics compression is not supported by the broker even though we only used previously supported compressors, re-getting our subscription information", "err", err) + default: + if !kerr.IsRetriable(err) { + cl.cfg.logger.Log(LogLevelWarn, "client metrics received an unknown error we do not know how to handle, exiting metrics loop", "err", err) + return + } + cl.cfg.logger.Log(LogLevelWarn, "client metrics received an unknown error that is retriable, continuing to next push cycle", "err", err) + } + } + } +} + +func buildNameFilter(requested []string) func(string) bool { + if len(requested) == 0 { + return func(string) bool { return false } + } + if len(requested) == 1 && requested[0] == "" { + return func(string) bool { return true } + } + return func(name string) bool { + for _, pfx := range requested { + if strings.HasPrefix(name, pfx) { + return true + } + } + return false + } +} + +const ( + // MetricTypeSum is a sum type metric. Every time the metric is + // collected, the number you are returning is cumulative from the time + // the client / your application was initialized. + MetricTypeSum = 1 + iota + + // MetricTypeGauge is a gauge type metric. It is a recording of a value + // at a point in time. + MetricTypeGauge +) + +type ( + // MetricType is the type of metric you are providing: Type is the type + // of metric this is: either a gauge or a sum type. + MetricType uint8 + + // Metric is a user-defined client side metric so that you can send + // user-defined client metrics to the broker and give your cluster + // operator insight into your client. + // + // This type exists to support KIP-1076, which is an extension to + // KIP-714. Read either of those for more detail. + Metric struct { + // Name is a user provided metric name. + // + // KIP-714 prescribes how to name metrics: lowercase with dots, + // no dashes, and interoperable with the OpenTelemetry + // ecosystem. It is recommended follow the KIP-714 guidance and + // to namespace your metrics ("my.specific.metric.1; + // my.specific.metric.2). This client does not attempt to + // further santize your user provided name. + Name string + + // Type is the type of metric this is: either a gauge or a sum + // type. + // + // Note that for sum types, the client internally caches all + // sum types by name for one extra collection period so that + // the client can calculate "delta" metrics if the broker + // requests them. You must provide the sum type metric on every + // collection; skipping a cycle means the client will be unable + // to calculate the delta once you provide it again in the + // future. + Type MetricType + + // ValueInt is the value to record. Only one of ValueInt or + // ValueFloat should be non-zero; if both are non-zero or if + // both are zero, the metric is ignored. + // + // Sum metrics only support ValueInt, and the number should + // never go down. If the value goes down or ValueFloat is + // used, the metric is skipped. + ValueInt int64 + + // ValueFloat is the value to record. Only one of ValueInt or + // ValueFloat should be non-zero; if both are non-zero or if + // both are zero, the metric is ignored. + // + // Sum metrics only support ValueInt, and the number should + // never go down. If the value goes down or ValueFloat is + // used, the metric is skipped. + ValueFloat float64 + + // Attrs are optional attributes to add to this metric, such as + // a node ID. The supported `any` types are strings, booleans, + // numbers, and byte slices. All other attributes are silently + // skipped. Attributes should be lowercase with underscores + // (see KIP-714). + Attrs map[string]any + } + + // metricRate is a count per second and a total. + metricRate struct { + count atomic.Int64 // Sum of events this period; rate == float64(count/time) at rollup + tot atomic.Int64 // Total events over all time. + lastTot int64 // Updated when encoding; the last value for tot in case broker requests DELTA. + } + + // metricTime reports average latency, max latency, and total latency. + // The unit is in milliseconds. + metricTime struct { + sum atomic.Int64 // With separate aggDur field, avg = sum/aggDur at rollup. + max atomic.Int64 // Max latency seen during this window. + } + + // We skip: + // * producer.record.queue.time.{avg,max} : medium signal; requires more wiring in the producer + // * consumer.poll.idle.ratio.avg : less relevant in this client + // * consumer.coordinator.rebalance.latency : underspecified: should this just track leader rebalance time? or from when we notice rebalance in progress to the next ok heartbeat? or..? + // * consumer.fetch.manager.fetch.latency : seems to duplicate consumer.node.request.latency ?? + // * consumer.coordinator.assigned.partitions : honestly just tedious to work in given incremental changes from cooperative rebalancing; open to being convinced to add this + metrics struct { + cl *Client + + closedFirstObserve atomic.Bool + + // mu is grabbed when accessing the map fields. + mu xsync.Mutex + unsupported atomic.Bool // set to true if the broker does not support client metrics; guards nil-ing the maps + + pConnCreation metricRate + pReqLatency map[int32]*metricTime + pThrottle metricTime + + cConnCreation metricRate + cReqLatency map[int32]*metricTime + cCommitLatency metricTime + + initNano int64 + lastPushNano int64 + + userSumLast map[string]int64 + + firstObserve chan struct{} + + ctx context.Context + ctxCancel func() + } +) + +func (m *metrics) init(cl *Client) { + m.cl = cl + m.initNano = time.Now().UnixNano() + m.firstObserve = make(chan struct{}) + m.ctx, m.ctxCancel = context.WithCancel(context.Background()) // for graceful shutdown +} + +func safeDiv[T ~int64 | ~float64](num, denom T) T { + if denom == 0 { + return 0 + } + return num / denom +} + +// Internally we use the metrics.observeRate function so that +// triggerFirstObserve is always called. +func (t *metricRate) observe() { + t.count.Add(1) + t.tot.Add(1) +} + +func (t *metricRate) rollNums() (rate float64, tot, lastTot int64) { + count := t.count.Swap(0) + lastTot = t.lastTot + tot = t.tot.Load() + t.lastTot = tot + + rate = safeDiv(float64(count), float64(tot-lastTot)) + return rate, tot, lastTot +} + +// Internally we use the metrics.observeTime function so that +// triggerFirstObserve is always called. +func (t *metricTime) observe(millis int64) { + t.sum.Add(millis) + for { + max := t.max.Load() + if millis < max { + return + } + if t.max.CompareAndSwap(max, millis) { + return + } + } +} + +func (t *metricTime) rollNums(aggDur time.Duration) (avg float64, max int64) { + sum := t.sum.Swap(0) + max = t.max.Swap(0) + avg = safeDiv(float64(sum), float64(aggDur.Milliseconds())) + return avg, max +} + +func (m *metrics) triggerFirstObserve() { + if !m.closedFirstObserve.Swap(true) { + close(m.firstObserve) + } +} + +func (m *metrics) observeRate(field *metricRate) { + m.triggerFirstObserve() + field.observe() +} + +func (m *metrics) observeTime(field *metricTime, millis int64) { + m.triggerFirstObserve() + field.observe(millis) +} + +func (m *metrics) observeNodeTime(node int32, field *map[int32]*metricTime, millis int64) { + m.triggerFirstObserve() + if m.unsupported.Load() { + return + } + m.mu.Lock() + defer m.mu.Unlock() + if m.unsupported.Load() { // one more check inside the lock to avoid a race where the maps are set nil after storing true + return + } + if *field == nil { + *field = make(map[int32]*metricTime) + } + t := (*field)[node] + if t == nil { + t = new(metricTime) + (*field)[node] = t + } + t.observe(millis) +} + +func (m *metrics) appendTo(b []byte, useDeltaSums bool, maxBytes int32, allowedNames func(string) bool, compressor Compressor) ([]byte, int8, int) { + //////////// + // VARIABLE INITIALIZATION + //////////// + var ( + metricsData otelMetricsData + resourceMetric = &metricsData.resourceMetric + resource = &resourceMetric.resource + scopeMetric = &resourceMetric.scopeMetric + scope = &scopeMetric.scope + metrics = &scopeMetric.metrics + labels = make(map[string]any) + ) + + if m.cl.cfg.rack != "" { + labels["client_rack"] = m.cl.cfg.rack + } + if m.cl.cfg.group != "" { + labels["group_id"] = m.cl.cfg.group + } + if m.cl.cfg.instanceID != nil { + labels["group_instance_id"] = *m.cl.cfg.instanceID + } + if memberID, _ := m.cl.GroupMetadata(); memberID != "" { + labels["group_member_id"] = memberID + } + if m.cl.cfg.txnID != nil { + labels["transactional_id"] = *m.cl.cfg.txnID + } + if len(labels) > 0 { + resource.attributes = labels + } + scope.name = "kgo" + scope.version = softwareVersion() + + now := time.Now() + nowNano := now.UnixNano() + lastPush := m.lastPushNano + aggDur := now.Sub(time.Unix(0, lastPush)) + if lastPush == 0 { + aggDur = now.Sub(time.Unix(0, m.initNano)) + } + m.lastPushNano = nowNano + + //////////// + // FUNCTIONS THAT APPEND TO `metrics`. + //////////// + + appendGauge := func(name string, vi64 int64, vf64 float64, attrs map[string]any) { + if vi64 == 0 && vf64 == 0 || vi64 != 0 && vf64 != 0 { + return + } + if !allowedNames(name) { + return + } + *metrics = append(*metrics, otelMetric{ + name: name, + gauge: otelGauge{ + otelNumDataPoint{ + attributes: attrs, + vInt: vi64, + vDouble: vf64, + startNano: lastPush, + timeNano: nowNano, + }, + }, + }) + } + appendSum := func(name string, tot, lastTot int64, attrs map[string]any) { + if tot-lastTot == 0 { + return + } + if !allowedNames(name) { + return + } + if useDeltaSums { + *metrics = append(*metrics, otelMetric{ + name: name, + sum: otelSum{ + dataPoint: otelNumDataPoint{ + attributes: attrs, + vInt: tot - lastTot, + startNano: lastPush, + timeNano: nowNano, + }, + aggregationTemporality: otelTempDelta, + }, + }) + } else { + *metrics = append(*metrics, otelMetric{ + name: name, + sum: otelSum{ + dataPoint: otelNumDataPoint{ + attributes: attrs, + vInt: tot, + startNano: m.initNano, + timeNano: nowNano, + }, + aggregationTemporality: otelTempCumulative, + }, + }) + } + } + + //////////// + // COLLECTING ALL METRICS TO SEND + //////////// + + m.mu.Lock() + for _, s := range []struct { + name string + v any + }{ + {"org.apache.kafka.producer.connection.creation", &m.pConnCreation}, + {"org.apache.kafka.producer.node.request.latency", &m.pReqLatency}, + {"org.apache.kafka.producer.produce.throttle.time", &m.pThrottle}, + {"org.apache.kafka.consumer.connection.creation", &m.cConnCreation}, + {"org.apache.kafka.consumer.node.request.latency", &m.cReqLatency}, + {"org.apache.kafka.consumer.coordinator.commit.latency", &m.cCommitLatency}, + } { + switch t := s.v.(type) { + case *metricRate: + rate, tot, lastTot := t.rollNums() + appendGauge(s.name+".rate", 0, rate, nil) + appendSum(s.name+".total", tot, lastTot, nil) + + case *metricTime: + avg, max := t.rollNums(aggDur) + appendGauge(s.name+".avg", 0, avg, nil) + appendGauge(s.name+".max", max, 0, nil) + + case *map[int32]*metricTime: + for broker, m := range *t { + avg, max := m.rollNums(aggDur) + attrs := map[string]any{"node_id": broker} + appendGauge(s.name+".avg", 0, avg, attrs) + appendGauge(s.name+".max", max, 0, attrs) + // By-node metrics do not write the forever-total-latency. + // We only have per-push increments, so we can safely delete + // the node every round (i.e., clean up if a broker goes away). + if avg == 0 && max == 0 { + delete(*t, broker) + } + } + + default: + m.mu.Unlock() + panic("unsupported type") + } + } + m.mu.Unlock() + + userAt := len(*metrics) + if m.cl.cfg.userMetrics != nil { + var userSkipped []string + last := m.userSumLast + if last == nil { + last = make(map[string]int64) + } + m.userSumLast = make(map[string]int64) + for um := range m.cl.cfg.userMetrics() { + if um.ValueInt != 0 && um.ValueFloat != 0 || um.ValueInt == 0 && um.ValueFloat == 0 { + userSkipped = append(userSkipped, um.Name) + continue + } + + switch um.Type { + case MetricTypeSum: + lastTot := last[um.Name] + if um.ValueFloat != 0 || um.ValueInt < 0 || lastTot > um.ValueInt { + userSkipped = append(userSkipped, um.Name) + continue + } + appendSum(um.Name, um.ValueInt, lastTot, um.Attrs) + m.userSumLast[um.Name] = um.ValueInt + + case MetricTypeGauge: + appendGauge(um.Name, um.ValueInt, um.ValueFloat, um.Attrs) + } + } + if len(userSkipped) > 0 { + m.cl.cfg.logger.Log(LogLevelWarn, "skipped serialization of some user provided metrics", "skipped", userSkipped) + } + } + + //////////// + // SERIALIZING - finally append metricsData to b + //////////// + + serialized, codec := metricsData.appendTo(b[:0]), CodecNone + + if compressor != nil { + serialized, codec = compressor.Compress(new(bytes.Buffer), serialized) + } + + if len(serialized) > int(maxBytes) && userAt < len(*metrics) { + m.cl.cfg.logger.Log(LogLevelWarn, "adding user metrics results in a too-large payload; dropping user metrics and serializing only standard client metrics", + "max_bytes", maxBytes, + "serialized_bytes_with_user", len(serialized), + ) + *metrics = (*metrics)[:userAt] + serialized, codec = metricsData.appendTo(b[:0]), CodecNone + if compressor != nil { + serialized, codec = compressor.Compress(new(bytes.Buffer), serialized) + } + } + return serialized, int8(codec), len(*metrics) +} + +// The types below are encoded in protobuf format; canonical +// definitions can be found at: +// https://github.com/open-telemetry/opentelemetry-proto/blob/f24da8deeb50118271c9435972791ef05ec003b1/opentelemetry/proto/metrics/v1/metrics.proto +type ( + otelMetricsData struct { + // resourceMetrics is repeated, but we always use one element. + resourceMetric otelResourceMetric // = 1 + } + + otelResourceMetric struct { + // We do not encode resource if resource.attributes is empty; + // eliding resource just means it is not known. + resource otelResource // = 1 + + // scopeMetric is repeated, but we always use one element. + scopeMetric otelScopeMetric // = 2 + } + + // From a separate file: + // https://github.com/open-telemetry/opentelemetry-proto/blob/f24da8deeb50118271c9435972791ef05ec003b1/opentelemetry/proto/resource/v1/resource.proto + // We only use attributes, and only if any exist. + otelResource struct { + attributes map[string]any // = 1 + } + + otelScopeMetric struct { + scope otelInstrumentationScope // = 1 + metrics []otelMetric // = 2 + } + + // From a separate file: + // https://github.com/open-telemetry/opentelemetry-proto/blob/f24da8deeb50118271c9435972791ef05ec003b1/opentelemetry/proto/common/v1/common.proto + // We use "kgo" and our released version (via the `softwareVersion()` func). + otelInstrumentationScope struct { + name string // = 1 + version string // = 2 + } + + otelMetric struct { + name string // = 1 + + // The metric data can be oneOf the following two: + gauge otelGauge // = 5 + sum otelSum // = 7 + // We use the non-empty struct; one must be non-empty. + } + + otelGauge struct { + // The .proto defines this as `repeated`, but we always use + // only one data point. + dataPoint otelNumDataPoint // = 1 + } + + otelSum struct { + // Same as otelGauge, the .proto defines this as `repeated` + // but we only use one data point. + dataPoint otelNumDataPoint // = 1 + + // aggregationTemporality is an enum; + // 0 is unspecified + // 1 is delta + // 2 is cumulative + aggregationTemporality uint8 // = 2 + + // We always set isMonotonic to true. + isMonotonic bool // = 3 + } + + otelNumDataPoint struct { + // Encoded as key/value attributes; we currently only use int32 + // broker IDs. This will need to change if other types are + // needed. + attributes map[string]any // = 7 + + startNano int64 // = 2 + timeNano int64 // = 3 + + // Only one of vDouble or vInt is non-zero, only + // one is used (this is oneof). + vDouble float64 // = 4 + vInt int64 // = 6 + } +) + +const ( + otelTempDelta = 1 + otelTempCumulative = 2 +) + +//////////// +// SERIALIZATION +//////////// + +const ( + protoTypeVarint = 0 + protoType64bit = 1 + protoTypeLength = 2 +) + +// appendProtoTag adds a Protocol Buffer tag (field number + proto type) +func appendProtoTag(b []byte, fieldNumber, protoType int) []byte { + return binary.AppendUvarint(b, uint64((fieldNumber<<3)|protoType)) +} + +// appendProtoString appends a string as length-prefixed bytes +func appendProtoString(b []byte, s string) []byte { + b = binary.AppendUvarint(b, uint64(len(s))) + return append(b, s...) +} + +func (d *otelMetricsData) appendTo(b []byte) []byte { + // Field 1: resourceMetric (message) + b = appendProtoTag(b, 1, protoTypeLength) + resourceBytes := d.resourceMetric.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(resourceBytes))) + b = append(b, resourceBytes...) + + return b +} + +func (m *otelResourceMetric) appendTo(b []byte) []byte { + // Field 1: resource (message) - only if attributes are present + if len(m.resource.attributes) > 0 { + b = appendProtoTag(b, 1, protoTypeLength) + resourceBytes := m.resource.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(resourceBytes))) + b = append(b, resourceBytes...) + } + + // Field 2: scopeMetric (message) + b = appendProtoTag(b, 2, protoTypeLength) + scopeBytes := m.scopeMetric.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(scopeBytes))) + b = append(b, scopeBytes...) + + return b +} + +func (r *otelResource) appendTo(b []byte) []byte { + // Field 1: attributes (repeated KeyValue) + return appendOtelAttributesTo(b, 1, r.attributes) +} + +func (s *otelScopeMetric) appendTo(b []byte) []byte { + // Field 1: scope (message) + b = appendProtoTag(b, 1, protoTypeLength) + scopeBytes := s.scope.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(scopeBytes))) + b = append(b, scopeBytes...) + + // Field 2: metrics (repeated message) + for _, m := range s.metrics { + b = appendProtoTag(b, 2, protoTypeLength) + metricBytes := m.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(metricBytes))) + b = append(b, metricBytes...) + } + + return b +} + +func (s *otelInstrumentationScope) appendTo(b []byte) []byte { + // Field 1: name (string) + if s.name != "" { + b = appendProtoTag(b, 1, protoTypeLength) + b = appendProtoString(b, s.name) + } + // Field 2: version (string) + if s.version != "" { + b = appendProtoTag(b, 2, protoTypeLength) + b = appendProtoString(b, s.version) + } + return b +} + +func (m *otelMetric) appendTo(b []byte) []byte { + // Field 1: name (string) + if m.name != "" { + b = appendProtoTag(b, 1, protoTypeLength) + b = appendProtoString(b, m.name) + } + + // Field 5: gauge (message) - if used + if m.gauge.dataPoint.timeNano != 0 { + b = appendProtoTag(b, 5, protoTypeLength) + gaugeBytes := m.gauge.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(gaugeBytes))) + b = append(b, gaugeBytes...) + } + + // Field 7: sum (message) - if used + if m.sum.dataPoint.timeNano != 0 { + b = appendProtoTag(b, 7, protoTypeLength) + sumBytes := m.sum.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(sumBytes))) + b = append(b, sumBytes...) + } + return b +} + +func (g *otelGauge) appendTo(b []byte) []byte { + // Field 1: dataPoints (repeated message) + b = appendProtoTag(b, 1, protoTypeLength) + dataPointBytes := g.dataPoint.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(dataPointBytes))) + b = append(b, dataPointBytes...) + + return b +} + +func (s *otelSum) appendTo(b []byte) []byte { + // Field 1: dataPoints (repeated message) + b = appendProtoTag(b, 1, protoTypeLength) + dataPointBytes := s.dataPoint.appendTo(nil) + b = binary.AppendUvarint(b, uint64(len(dataPointBytes))) + b = append(b, dataPointBytes...) + + // Field 2: aggregationTemporality (enum) + if s.aggregationTemporality != 0 { + b = appendProtoTag(b, 2, protoTypeVarint) + b = binary.AppendUvarint(b, uint64(s.aggregationTemporality)) + } + + // Field 3: isMonotonic (bool) + if s.isMonotonic { + b = appendProtoTag(b, 3, protoTypeVarint) + b = binary.AppendUvarint(b, 1) // true + } + + return b +} + +func (d *otelNumDataPoint) appendTo(b []byte) []byte { + // Field 2: startTimeUnixNano (fixed64) + if d.startNano != 0 { + b = appendProtoTag(b, 2, protoType64bit) + b = binary.LittleEndian.AppendUint64(b, uint64(d.startNano)) + } + + // Field 3: timeUnixNano (fixed64) + b = appendProtoTag(b, 3, protoType64bit) + b = binary.LittleEndian.AppendUint64(b, uint64(d.timeNano)) + + // Field 4: asDouble (double) + if d.vDouble != 0 { + b = appendProtoTag(b, 4, protoType64bit) + b = binary.LittleEndian.AppendUint64(b, math.Float64bits(d.vDouble)) + } + + // Field 6: asInt (sfixed64) + if d.vInt != 0 { + b = appendProtoTag(b, 6, protoType64bit) + b = binary.LittleEndian.AppendUint64(b, uint64(d.vInt)) + } + + // Field 7: attributes + b = appendOtelAttributesTo(b, 7, d.attributes) + + return b +} + +func appendOtelAttributesTo(b []byte, fieldNumber int, attrs map[string]any) []byte { +outer: + for key, value := range attrs { + b = appendProtoTag(b, fieldNumber, protoTypeLength) + + kvBytes := []byte{} + // Field 1: key (string) + kvBytes = appendProtoTag(kvBytes, 1, protoTypeLength) + kvBytes = appendProtoString(kvBytes, key) + + // Field 2: value (AnyValue) + kvBytes = appendProtoTag(kvBytes, 2, protoTypeLength) + + var anyValueBytes []byte + switch t := value.(type) { + case *string, string: + var v string + switch t := t.(type) { + case string: + v = t + case *string: + if t != nil { + v = *t + } + } + anyValueBytes = appendProtoTag(anyValueBytes, 1, protoTypeLength) // string_value = 1 + anyValueBytes = appendProtoString(anyValueBytes, v) + case bool: + anyValueBytes = appendProtoTag(anyValueBytes, 2, protoTypeVarint) // bool_value = 2 + if t { + anyValueBytes = binary.AppendUvarint(anyValueBytes, 1) + } else { + anyValueBytes = binary.AppendUvarint(anyValueBytes, 0) + } + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr: + anyValueBytes = appendProtoTag(anyValueBytes, 3, protoTypeVarint) // int_value = 3 + var v uint64 + switch t := t.(type) { + case int: + v = uint64(t) + case int8: + v = uint64(t) + case int16: + v = uint64(t) + case int32: + v = uint64(t) + case int64: + v = uint64(t) + case uint: + v = uint64(t) + case uint8: + v = uint64(t) + case uint16: + v = uint64(t) + case uint32: + v = uint64(t) + case uint64: + v = t + case uintptr: + v = uint64(t) + } + anyValueBytes = binary.AppendUvarint(anyValueBytes, v) + case float32, float64: + var v float64 + switch t := t.(type) { + case float32: + v = float64(t) + case float64: + v = t + } + anyValueBytes = appendProtoTag(anyValueBytes, 4, protoType64bit) // double_value = 4 + anyValueBytes = binary.LittleEndian.AppendUint64(anyValueBytes, math.Float64bits(v)) + case []byte: + anyValueBytes = appendProtoTag(anyValueBytes, 7, protoTypeLength) // bytes_value = 7 + anyValueBytes = binary.AppendUvarint(anyValueBytes, uint64(len(t))) + anyValueBytes = append(anyValueBytes, t...) + default: + continue outer + } + + kvBytes = binary.AppendUvarint(kvBytes, uint64(len(anyValueBytes))) + kvBytes = append(kvBytes, anyValueBytes...) + + b = binary.AppendUvarint(b, uint64(len(kvBytes))) + b = append(b, kvBytes...) + } + return b +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/partitioner.go b/vendor/github.com/twmb/franz-go/pkg/kgo/partitioner.go new file mode 100644 index 00000000000..46e7d11d124 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/partitioner.go @@ -0,0 +1,614 @@ +package kgo + +import ( + "math" + "math/rand" + "time" + + "github.com/twmb/franz-go/pkg/kbin" +) + +// Partitioner creates topic partitioners to determine which partition messages +// should be sent to. +// +// Note that a record struct is unmodified (minus a potential default topic) +// from producing through partitioning, so you can set fields in the record +// struct before producing to aid in partitioning with a custom partitioner. +type Partitioner interface { + // forTopic returns a partitioner for an individual topic. It is + // guaranteed that only one record will use the an individual topic's + // topicPartitioner at a time, meaning partitioning within a topic does + // not require locks. + ForTopic(string) TopicPartitioner +} + +// TopicPartitioner partitions records in an individual topic. +type TopicPartitioner interface { + // RequiresConsistency returns true if a record must hash to the same + // partition even if a partition is down. + // If true, a record may hash to a partition that cannot be written to + // and will error until the partition comes back. + RequiresConsistency(*Record) bool + // Partition determines, among a set of n partitions, which index should + // be chosen to use for the partition for r. + Partition(r *Record, n int) int +} + +// TopicPartitionerOnNewBatch is an optional extension interface to +// TopicPartitioner that calls OnNewBatch before any new batch is created. If +// buffering a record would cause a new batch, OnNewBatch is called. +// +// This interface allows for partitioner implementations that effectively pin +// to a partition until a new batch is created, after which the partitioner can +// choose which next partition to use. +type TopicPartitionerOnNewBatch interface { + // OnNewBatch is called when producing a record if that record would + // trigger a new batch on its current partition. + OnNewBatch() +} + +// TopicBackupPartitioner is an optional extension interface to +// TopicPartitioner that can partition by the number of records buffered. +// +// If a partitioner implements this interface, the Partition function will +// never be called. +type TopicBackupPartitioner interface { + TopicPartitioner + + // PartitionByBackup is similar to Partition, but has an additional + // backupIter. This iterator will return the number of buffered records + // per partition index. The iterator's Next function can only be called + // up to n times, calling it any more will panic. + PartitionByBackup(r *Record, n int, backupIter TopicBackupIter) int +} + +// TopicBackupIter is an iterates through partition indices. +type TopicBackupIter interface { + // Next returns the next partition index and the total buffered records + // for the partition. If Rem returns 0, calling this function again + // will panic. + Next() (int, int64) + // Rem returns the number of elements left to iterate through. + Rem() int +} + +//////////// +// SIMPLE // - BasicConsistent, Manual, RoundRobin +//////////// + +// BasicConsistentPartitioner wraps a single function to provide a Partitioner +// and TopicPartitioner (that function is essentially a combination of +// Partitioner.ForTopic and TopicPartitioner.Partition). +// +// As a minimal example, if you do not care about the topic and you set the +// partition before producing: +// +// kgo.BasicConsistentPartitioner(func(topic) func(*Record, int) int { +// return func(r *Record, n int) int { +// return int(r.Partition) +// } +// }) +func BasicConsistentPartitioner(partition func(string) func(r *Record, n int) int) Partitioner { + return &basicPartitioner{partition} +} + +type ( + basicPartitioner struct { + fn func(string) func(*Record, int) int + } + + basicTopicPartitioner struct { + fn func(*Record, int) int + } +) + +func (b *basicPartitioner) ForTopic(t string) TopicPartitioner { + return &basicTopicPartitioner{b.fn(t)} +} + +func (*basicTopicPartitioner) RequiresConsistency(*Record) bool { return true } +func (b *basicTopicPartitioner) Partition(r *Record, n int) int { return b.fn(r, n) } + +// ManualPartitioner is a partitioner that simply returns the Partition field +// that is already set on any record. +// +// Any record with an invalid partition will be immediately failed. This +// partitioner is simply the partitioner that is demonstrated in the +// BasicConsistentPartitioner documentation. +func ManualPartitioner() Partitioner { + return BasicConsistentPartitioner(func(string) func(*Record, int) int { + return func(r *Record, _ int) int { + return int(r.Partition) + } + }) +} + +// RoundRobinPartitioner is a partitioner that round-robin's through all +// available partitions. This algorithm has lower throughput and causes higher +// CPU load on brokers, but can be useful if you want to ensure an even +// distribution of records to partitions. +func RoundRobinPartitioner() Partitioner { + return new(roundRobinPartitioner) +} + +type ( + roundRobinPartitioner struct{} + + roundRobinTopicPartitioner struct { + on int + } +) + +func (*roundRobinPartitioner) ForTopic(string) TopicPartitioner { + return new(roundRobinTopicPartitioner) +} + +func (*roundRobinTopicPartitioner) RequiresConsistency(*Record) bool { return false } +func (r *roundRobinTopicPartitioner) Partition(_ *Record, n int) int { + if r.on >= n { + r.on = 0 + } + ret := r.on + r.on++ + return ret +} + +////////////////// +// LEAST BACKUP // +////////////////// + +// LeastBackupPartitioner prioritizes partitioning by three factors, in order: +// +// 1. pin to the current pick until there is a new batch +// 2. on new batch, choose the least backed up partition (the partition with +// the fewest amount of buffered records) +// 3. if multiple partitions are equally least-backed-up, choose one at random +// +// This algorithm prioritizes least-backed-up throughput, which may result in +// unequal partitioning. It is likely that this algorithm will talk most to the +// broker that it has the best connection to. +// +// This algorithm is resilient to brokers going down: if a few brokers die, it +// is possible your throughput will be so high that the maximum buffered +// records will be reached in the now-offline partitions before metadata +// responds that the broker is offline. With the standard partitioning +// algorithms, the only recovery is if the partition is remapped or if the +// broker comes back online. With the least backup partitioner, downed +// partitions will see slight backup, but then the other partitions that are +// still accepting writes will get all of the writes and your client will not +// be blocked. +// +// Under ideal scenarios (no broker / connection issues), StickyPartitioner is +// equivalent to LeastBackupPartitioner. This partitioner is only recommended +// if you are a producer consistently dealing with flaky connections or +// problematic brokers and do not mind uneven load on your brokers. +func LeastBackupPartitioner() Partitioner { + return new(leastBackupPartitioner) +} + +type ( + leastBackupInput struct{ mapping []*topicPartition } + + leastBackupPartitioner struct{} + + leastBackupTopicPartitioner struct { + onPart int + rng *rand.Rand + } +) + +func (i *leastBackupInput) Next() (int, int64) { + last := len(i.mapping) - 1 + buffered := i.mapping[last].records.buffered.Load() + i.mapping = i.mapping[:last] + return last, buffered +} + +func (i *leastBackupInput) Rem() int { + return len(i.mapping) +} + +func (*leastBackupPartitioner) ForTopic(string) TopicPartitioner { + return &leastBackupTopicPartitioner{ + onPart: -1, + rng: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +func (p *leastBackupTopicPartitioner) OnNewBatch() { p.onPart = -1 } +func (*leastBackupTopicPartitioner) RequiresConsistency(*Record) bool { return false } +func (*leastBackupTopicPartitioner) Partition(*Record, int) int { panic("unreachable") } + +func (p *leastBackupTopicPartitioner) PartitionByBackup(_ *Record, n int, backup TopicBackupIter) int { + if p.onPart == -1 || p.onPart >= n { + leastBackup := int64(math.MaxInt64) + npicked := 0 + for ; n > 0; n-- { + pick, backup := backup.Next() + if backup < leastBackup { + leastBackup = backup + p.onPart = pick + npicked = 1 + } else { + npicked++ // reservoir sampling with k = 1 + if p.rng.Intn(npicked) == 0 { + p.onPart = pick + } + } + } + } + return p.onPart +} + +/////////////////// +// UNIFORM BYTES // +/////////////////// + +// UniformBytesPartitioner is a redux of the StickyPartitioner, proposed in +// KIP-794 and release with the Java client in Kafka 3.3. This partitioner +// returns the same partition until 'bytes' is hit. At that point, a +// re-partitioning happens. If adaptive is false, this chooses a new random +// partition, otherwise this chooses a broker based on the inverse of the +// backlog currently buffered for that broker. If keys is true, this uses +// standard hashing based on record key for records with non-nil keys. hasher +// is optional; if nil, the default hasher murmur2 (Kafka's default). +// +// The point of this hasher is to create larger batches while producing the +// same amount to all partitions over the long run. Adaptive opts in to a +// slight imbalance so that this can produce more to brokers that are less +// loaded. +// +// This implementation differs slightly from Kafka's because this does not +// account for the compressed size of a batch, nor batch overhead. For +// overhead, in practice, the overhead is relatively constant so it would +// affect all batches equally. For compression, this client does not compress +// until after a batch is created and frozen, so it is not possible to track +// compression. This client also uses the number of records for backup +// calculation rather than number of bytes, but the heuristic should be +// similar. Lastly, this client does not have a timeout for partition +// availability. Realistically, these will be the most backed up partitions so +// they should be chosen the least. +// +// NOTE: This implementation may create sub-optimal batches if lingering is +// enabled. This client's default is to disable lingering. The patch used to +// address this in Kafka is KAFKA-14156 (which itself is not perfect in the +// context of disabling lingering). For more details, read KAFKA-14156. +func UniformBytesPartitioner(bytes int, adaptive, keys bool, hasher PartitionerHasher) Partitioner { + if hasher == nil { + hasher = KafkaHasher(murmur2) + } + return &uniformBytesPartitioner{ + bytes, + adaptive, + keys, + hasher, + } +} + +type ( + uniformBytesPartitioner struct { + bytes int + adaptive bool + keys bool + hasher PartitionerHasher + } + + uniformBytesTopicPartitioner struct { + u uniformBytesPartitioner + bytes int + onPart int + rng *rand.Rand + + calc []struct { + f float64 + n int + } + } +) + +func (u *uniformBytesPartitioner) ForTopic(string) TopicPartitioner { + return &uniformBytesTopicPartitioner{ + u: *u, + onPart: -1, + rng: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +func (p *uniformBytesTopicPartitioner) RequiresConsistency(r *Record) bool { + return p.u.keys && r.Key != nil +} +func (*uniformBytesTopicPartitioner) Partition(*Record, int) int { panic("unreachable") } + +func (p *uniformBytesTopicPartitioner) PartitionByBackup(r *Record, n int, backup TopicBackupIter) int { + if p.u.keys && r.Key != nil { + return p.u.hasher(r.Key, n) + } + + l := 1 + // attributes, int8 unused + 1 + // ts delta, 1 minimum (likely 2 or 3) + 1 + // offset delta, likely 1 + kbin.VarintLen(int32(len(r.Key))) + + len(r.Key) + + kbin.VarintLen(int32(len(r.Value))) + + len(r.Value) + + kbin.VarintLen(int32(len(r.Headers))) // varint array len headers + + for _, h := range r.Headers { + l += kbin.VarintLen(int32(len(h.Key))) + + len(h.Key) + + kbin.VarintLen(int32(len(h.Value))) + + len(h.Value) + } + + p.bytes += l + if p.bytes >= p.u.bytes { + p.bytes = l + p.onPart = -1 + } + + if p.onPart >= 0 && p.onPart < n { + return p.onPart + } + + if !p.u.adaptive { + p.onPart = p.rng.Intn(n) + } else { + p.calc = p.calc[:0] + + // For adaptive, the logic is that we pick by broker according + // to the inverse of the queue size. Presumably this means + // bytes, but we use records for simplicity. + // + // We calculate 1/recs for all brokers and choose the first one + // in this ordering that takes us negative. + // + // e.g., 1/1 + 1/3; pick is 0.2; 0.2*1.3333 = 0.26666; minus 1 + // is negative, meaning our pick is the first. If rng was 0.9, + // scaled is 1.2, meaning our pick is the second (-1, still + // positive, second pick takes us negative). + // + // To guard floating rounding problems, if we pick nothing, + // then this means we pick our last. + var t float64 + for ; n > 0; n-- { + n, backup := backup.Next() + backup++ // ensure non-zero + f := 1 / float64(backup) + t += f + p.calc = append(p.calc, struct { + f float64 + n int + }{f, n}) + } + r := p.rng.Float64() + pick := r * t + for _, c := range p.calc { + pick -= c.f + if pick <= 0 { + p.onPart = c.n + break + } + } + if p.onPart == -1 { + p.onPart = p.calc[len(p.calc)-1].n + } + } + return p.onPart +} + +///////////////////// +// STICKY & COMPAT // - Sticky, Kafka (custom hash), Sarama (custom hash) +///////////////////// + +// StickyPartitioner is the same as StickyKeyPartitioner, but with no logic to +// consistently hash keys. That is, this only partitions according to the +// sticky partition strategy. +func StickyPartitioner() Partitioner { + return new(stickyPartitioner) +} + +type ( + stickyPartitioner struct{} + + stickyTopicPartitioner struct { + lastPart int + onPart int + rng *rand.Rand + } +) + +func (*stickyPartitioner) ForTopic(string) TopicPartitioner { + p := newStickyTopicPartitioner() + return &p +} + +func newStickyTopicPartitioner() stickyTopicPartitioner { + return stickyTopicPartitioner{ + lastPart: -1, + onPart: -1, + rng: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +func (p *stickyTopicPartitioner) OnNewBatch() { p.lastPart, p.onPart = p.onPart, -1 } +func (*stickyTopicPartitioner) RequiresConsistency(*Record) bool { return false } +func (p *stickyTopicPartitioner) Partition(_ *Record, n int) int { + if p.onPart == -1 || p.onPart >= n { + p.onPart = p.rng.Intn(n) + if p.onPart == p.lastPart { + p.onPart = (p.onPart + 1) % n + } + } + return p.onPart +} + +// StickyKeyPartitioner mirrors the default Java partitioner from Kafka's 2.4 +// release (see KIP-480 and KAFKA-8601) until their 3.3 release. This was +// replaced in 3.3 with the uniform sticky partitioner (KIP-794), which is +// reimplemented in this client as the UniformBytesPartitioner. +// +// This is the same "hash the key consistently, if no key, choose random +// partition" strategy that the Java partitioner has always used, but rather +// than always choosing a random partition, the partitioner pins a partition to +// produce to until that partition rolls over to a new batch. Only when rolling +// to new batches does this partitioner switch partitions. +// +// The benefit with this pinning is less CPU utilization on Kafka brokers. +// Over time, the random distribution is the same, but the brokers are handling +// on average larger batches. +// +// hasher is optional; if nil, this will return a partitioner that partitions +// exactly how Kafka does. Specifically, the partitioner will use murmur2 to +// hash keys, will mask out the 32nd bit, and then will mod by the number of +// potential partitions. +func StickyKeyPartitioner(hasher PartitionerHasher) Partitioner { + if hasher == nil { + hasher = KafkaHasher(murmur2) + } + return &keyPartitioner{hasher} +} + +// PartitionerHasher returns a partition to use given the input data and number +// of partitions. +type PartitionerHasher func([]byte, int) int + +// KafkaHasher returns a PartitionerHasher using hashFn that mirrors how Kafka +// partitions after hashing data. In Kafka, after hashing into a uint32, the +// hash is converted to an int32 and the high bit is stripped. Kafka by default +// uses murmur2 hashing, and the StickyKeyPartiitoner uses this by default. +// Using this KafkaHasher function is only necessary if you want to change the +// underlying hashing algorithm. +func KafkaHasher(hashFn func([]byte) uint32) PartitionerHasher { + return func(key []byte, n int) int { + // https://github.com/apache/kafka/blob/d91a94e/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java#L59 + // https://github.com/apache/kafka/blob/d91a94e/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L865-L867 + // Masking before or after the int conversion makes no difference. + return int(hashFn(key)&0x7fffffff) % n + } +} + +// SaramaHasher is a historical misnamed partitioner. This library's original +// implementation of the SaramaHasher was incorrect, if you want an exact +// match for the Sarama partitioner, use the [SaramaCompatHasher]. +// +// This partitioner remains because as it turns out, other ecosystems provide +// a similar partitioner and this partitioner is useful for compatibility. +// +// In particular, using this function with a crc32.ChecksumIEEE hasher makes +// this partitioner match librdkafka's consistent partitioner, or the +// zendesk/ruby-kafka partitioner. +func SaramaHasher(hashFn func([]byte) uint32) PartitionerHasher { + return func(key []byte, n int) int { + p := int(hashFn(key)) % n + if p < 0 { + p = -p + } + return p + } +} + +// SaramaCompatHasher returns a PartitionerHasher using hashFn that mirrors how +// Sarama partitions after hashing data. +// +// Sarama has two differences from Kafka when partitioning: +// +// 1) In Kafka, when converting the uint32 hash to an int32, Kafka masks the +// high bit. In Sarama, if the high bit is 1 (i.e., the number as an int32 is +// negative), Sarama negates the number. +// +// 2) Kafka by default uses the murmur2 hashing algorithm. Sarama by default +// uses fnv-1a. +// +// Sarama added a NewReferenceHashPartitioner function that attempted to align +// with Kafka, but the reference partitioner only fixed the first difference, +// not the second. Further customization options were added later that made it +// possible to exactly match Kafka when hashing. +// +// In short, to *exactly* match the Sarama defaults, use the following: +// +// kgo.StickyKeyPartitioner(kgo.SaramaCompatHasher(fnv32a)) +// +// Where fnv32a is a function returning a new 32 bit fnv-1a hasher. +// +// func fnv32a(b []byte) uint32 { +// h := fnv.New32a() +// h.Reset() +// h.Write(b) +// return h.Sum32() +// } +func SaramaCompatHasher(hashFn func([]byte) uint32) PartitionerHasher { + return func(key []byte, n int) int { + p := int32(hashFn(key)) % int32(n) + if p < 0 { + p = -p + } + return int(p) + } +} + +type ( + keyPartitioner struct { + hasher PartitionerHasher + } + + stickyKeyTopicPartitioner struct { + hasher PartitionerHasher + stickyTopicPartitioner + } +) + +func (k *keyPartitioner) ForTopic(string) TopicPartitioner { + return &stickyKeyTopicPartitioner{k.hasher, newStickyTopicPartitioner()} +} + +func (*stickyKeyTopicPartitioner) RequiresConsistency(r *Record) bool { return r.Key != nil } +func (p *stickyKeyTopicPartitioner) Partition(r *Record, n int) int { + if r.Key != nil { + return p.hasher(r.Key, n) + } + return p.stickyTopicPartitioner.Partition(r, n) +} + +///////////// +// MURMUR2 // +///////////// + +// Straight from the C++ code and from the Java code duplicating it. +// https://github.com/apache/kafka/blob/d91a94e/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L383-L421 +// https://github.com/aappleby/smhasher/blob/61a0530f/src/MurmurHash2.cpp#L37-L86 +// +// The Java code uses ints but with unsigned shifts; we do not need to. +func murmur2(b []byte) uint32 { + const ( + seed uint32 = 0x9747b28c + m uint32 = 0x5bd1e995 + r = 24 + ) + h := seed ^ uint32(len(b)) + for len(b) >= 4 { + k := uint32(b[3])<<24 + uint32(b[2])<<16 + uint32(b[1])<<8 + uint32(b[0]) + b = b[4:] + k *= m + k ^= k >> r + k *= m + + h *= m + h ^= k + } + switch len(b) { + case 3: + h ^= uint32(b[2]) << 16 + fallthrough + case 2: + h ^= uint32(b[1]) << 8 + fallthrough + case 1: + h ^= uint32(b[0]) + h *= m + } + + h ^= h >> 13 + h *= m + h ^= h >> 15 + return h +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/pools.go b/vendor/github.com/twmb/franz-go/pkg/kgo/pools.go new file mode 100644 index 00000000000..f4449c3db45 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/pools.go @@ -0,0 +1,201 @@ +package kgo + +import ( + "context" + "fmt" + "sync/atomic" + "unsafe" + + "github.com/twmb/franz-go/pkg/kmsg" +) + +//////////////////////////////////////////////////////////////// +// NOTE: // +// NOTE: Make sure new hooks are checked in implementsAnyPool // +// NOTE: // +//////////////////////////////////////////////////////////////// + +// Pool is a memory pool to be used where relevant. +// +// The base Pool interface is meaningless, but wherever a type can be pooled in +// kgo, the client checks if your pool implements an appropriate pool interface. +// If so, the pool is received from (Get), and when the data is done being used, +// the pool is put back into (Put). +// +// All pool interfaces in this package have Pool in the name. Pools must be safe +// for concurrent use. +type Pool any + +type pools []Pool + +func (ps pools) each(fn func(Pool) bool) { + for _, p := range ps { + found := fn(p) + if found { + return + } + } +} + +// PoolDecompressBytes is a pool that returns a slice that decompressed data +// will be decoded into. +type PoolDecompressBytes interface { + // GetDecompressBytes returns a slice to decompress into. This + // interface is given the compressed data and the codec that will be + // used for decompressing. + // + // For many decompression algorithms, it is not possible to accurately + // know the size that data will be once decompressed. You can guess a + // multiplier, or you can use rolling statistics via FetchBatchMetrics. + // If the slice is not large enough, it is grown and the grown slice is + // put back into the pool (not the original slice, since decompression + // libraries often internally discard the original slice). + // + // NOTE: If you provide your own Decompressor, this function will not + // be called. However, PutDecompressBytes will still be called with the + // slice that is returned from Decompress if Decompress was called. It + // is expected that you use your own GetDecompressBytes in your own + // Decompress if you provide this pool, however, if you do not, you'll + // just have extra data slices put back into your pool that you never + // created. + GetDecompressBytes(compressed []byte, codec CompressionCodecType) []byte + // PutDecompressBytes puts a slice of that was used for decompression + // back into the pool. The slice is zeroed before it is put back. + PutDecompressBytes([]byte) +} + +// PoolKRecords is a pool that returns a slice that raw kmsg.Record's are +// decoded into. +type PoolKRecords interface { + // GetKRecords returns a slice with capacity n. + GetKRecords(n int) []kmsg.Record + // PutKRecords puts a slice back into the pool. + PutKRecords([]kmsg.Record) +} + +// PoolRecords is a pool that returns a slice of Record's. +type PoolRecords interface { + // GetRecords returns a slice with capacity n. + GetRecords(n int) []Record + // PutRecords puts a slice back into the pool. + PutRecords([]Record) +} + +func strp(s string) *string { return &s } + +var ctxRecRecycle = strp("rec-recycle") + +const recSize = unsafe.Sizeof(Record{}) + +func recordPoolsCtx(pools []Pool, decompressBytes []byte, recs []Record) (*recordPools, context.Context) { + if len(recs) == 0 { // ...just in case + return nil, nil + } + p := &recordPools{ + pools: pools, + decompressBytes: decompressBytes, + recs: recs, + } + return p, context.WithValue(context.Background(), ctxRecRecycle, p) +} + +// Recycle "recycles" this record if it was taken from a pool, and frees its +// attachment to any underlying pooled slices. If the pooled slice no longer +// has any records attached, the slices are put back into their pools. +// +// This method is only relevant if you are using the [WithPools] option. +// +// For share group records (KIP-932): Recycle releases the record's +// backing memory but does NOT by itself ack. Any record not explicitly +// acked is auto-accepted on the next [Client.PollRecords] via the +// standard share-consumer next-poll auto-accept pass; this is true +// regardless of whether the record was Recycled. If you need a +// non-Accept outcome (Release/Reject/Renew), call [Record.Ack] with +// the appropriate status BEFORE Recycle. +// +// NOTE: It is invalid to continue using the record after calling recycle; +// doing so may result in corruption and data races. If you use +// PoolDecompressBytes, you cannot continue to use a shallow copy of any +// fields, you must clone them! +func (r *Record) Recycle() { + if r.Context == nil { + return + } + v := r.Context.Value(ctxRecRecycle) + if v == nil { + return + } + *r = Record{} // prevent the Record from hanging onto anything; *kmsg.Record is already cleared during processing + ps := v.(*recordPools) + + r0 := &ps.recs[0] + + idx := int((uintptr(unsafe.Pointer(r)) - uintptr(unsafe.Pointer(r0))) / recSize) //nolint:gosec // unsafe rule (3) roughly (we don't need to convert back) + if idx < 0 || idx >= len(ps.recs) { + panic(fmt.Sprintf("recycling a record and the index %v is invalid (max pool len %v)", idx, len(ps.recs))) + } + if &ps.recs[idx] != r { + panic("sanity check pointer comparison index for recycling record is not the same!") + } + + rem := ps.n.Add(-1) + if rem > 0 { + return + } + + // Reset the length of slices to max to ensure we zero things and so + // that users can avoid this resetting. + ps.decompressBytes = ps.decompressBytes[:cap(ps.decompressBytes)] + ps.recs = ps.recs[:cap(ps.recs)] + + for i := range ps.decompressBytes { + ps.decompressBytes[i] = 0 // this loop is optimized in the compiler to a memclr: https://go.dev/wiki/CompilerOptimizations#optimized-memclr + } + + // We use len checks below because we should NOT put empty slices back + // into the pool (especially since the decompressBytes may never have + // been created!). + pools(ps.pools).each(func(p Pool) bool { + if pdecom, ok := p.(PoolDecompressBytes); ok { + if len(ps.decompressBytes) > 0 { + pdecom.PutDecompressBytes(ps.decompressBytes) + ps.decompressBytes = nil + } + } + if precs, ok := p.(PoolRecords); ok { + if len(ps.recs) > 0 { + precs.PutRecords(ps.recs) + ps.recs = nil + } + } + return false // always loop through all pools; we put each slice into a max of one pool + }) +} + +type recordPools struct { + pools []Pool + n atomic.Int64 + + decompressBytes []byte + recs []Record +} + +// implementsAnyPool will check the incoming Pool for any Pool implementation +func implementsAnyPool(p Pool) bool { + switch p.(type) { + case /*PoolRequestBuffer,*/ + PoolDecompressBytes, + PoolKRecords, + PoolRecords: + return true + } + return false +} + +func ensureLen[S ~[]E, E any](s S, n int) S { + s = s[:cap(s)] + if len(s) >= n { + return s[:n] + } + return append(s, make([]E, n-len(s))...) +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/produce-bugs-prompt.md b/vendor/github.com/twmb/franz-go/pkg/kgo/produce-bugs-prompt.md new file mode 100644 index 00000000000..8fcd18ab86f --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/produce-bugs-prompt.md @@ -0,0 +1,64 @@ +# franz-go produce: bugs & races + +You are analyzing the franz-go Kafka client library (pkg/kgo). Find correctness +bugs and race conditions in the PRODUCE codepath. Do not flag style, naming, +missing tests, or refactor opportunities. + +Files in scope (read in this order): +- sink.go per-broker produce loop; owns recBufs; recBufs own recBatch +- producer.go producer abstraction, sink selection, promise completion +- partitioner.go record -> partition assignment +- txn.go GroupTransactSession; transactional epoch lifecycle +- metadata.go mergeTopicPartitions (new-partitions loop), writablePartitions, + partitionsForTopicProduce, doPartition +- record_and_fetch.go Record / Promise types +- client.go cross-cutting: broker selection, retry, close + +Invariants and conventions (assume these hold; don't re-derive): +- A recBuf has exactly one active writer at a time: the sink that owns it. +- Sequence numbers must be monotonic per (PID, epoch, partition). +- Within a partition, in-order delivery is REQUIRED; reordering between + in-flight batches must be prevented. +- bufferedRecords semaphore bounds total in-flight records globally. +- PID/epoch lifecycle: InitProducerID -> [AddPartitionsToTxn ->] Produce + -> [EndTxn]. KIP-890 (EndTxn v5+) bumps the epoch in EndTxn; retries + must accept stale_epoch+1 == current. +- Lock ordering: c.mu -> g.mu. +- Context-key idiom is pointer-to-string (ctxPinReq, ctxRecRecycle, etc.); + not a bug. + +Known intentional behavior - DO NOT flag any of these: +- Produce capped at v12 when any partition lacks a TopicID (Event Hubs guard). +- OffsetFetch / OffsetCommit pinned to v9 when any topic lacks a TopicID + (same reason); OffsetFetch topic-id resolution skipped below v10. +- recBatch field ordering chosen for atomic-64 alignment on 32-bit platforms. +- Per-partition sequence reset after a fatal idempotent error (e.g. + UNKNOWN_PRODUCER_ID) - intentional recovery path. +- The "first finished promise wins" pattern for aborted batches. + +Find only: +1. Data races: unsynchronized reads/writes, especially around sink/recBuf + migration during metadata refresh. +2. Lost or duplicate writes: a record buffered but never produced; a record + produced twice across retry/leader-change/txn-abort paths. +3. Out-of-order delivery: any sequence where two batches for the same + partition can be acked in a different order than produced. +4. Idempotent/transactional lifecycle bugs: bad PID/epoch transitions, + AddPartitions / EndTxn ordering, retries after stale epoch, fenced + producer recovery, AddPartitions never sent for a partition we produce to. +5. Promise leaks: a record promise that never fires (success OR error) + under some path - context cancel, client close, txn abort, etc. +6. Goroutine leaks past client.Close(). +7. Off-by-one / boundary errors in batch sizing, sequences, partition counts. +8. TopicID resolution races with metadata refresh during in-flight produce. + +Output per finding: +- Severity: critical (data loss / dup / corruption) | high (hang / leak) + | medium (rare race, recoverable) | low +- File:line +- What: one sentence +- How: numbered walkthrough of goroutines/events that triggers it +- Fix: one-paragraph sketch (not full code) + +If a category yields nothing, say "none found" - don't pad. If you cannot +trace a finding to a concrete sequence of events, omit it. diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/produce-efficiency-prompt.md b/vendor/github.com/twmb/franz-go/pkg/kgo/produce-efficiency-prompt.md new file mode 100644 index 00000000000..57f4b8d3ed8 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/produce-efficiency-prompt.md @@ -0,0 +1,58 @@ +# franz-go produce: efficiency + +Find efficiency improvements in the PRODUCE codepath of franz-go (pkg/kgo). + +Files in scope: +- sink.go per-broker produce loop; owns recBufs; recBufs own recBatch +- producer.go producer abstraction, sink selection, promise completion +- partitioner.go record -> partition assignment +- txn.go GroupTransactSession; transactional epoch lifecycle +- metadata.go mergeTopicPartitions (new-partitions loop), writablePartitions, + partitionsForTopicProduce, doPartition +- record_and_fetch.go Record / Promise types +- client.go cross-cutting + +Hot paths in priority order: +1. PER-RECORD: Client.Produce, partitioner, append to recBuf. +2. PER-BATCH: assemble Produce request, encode/compress records, send. +3. PER-RESPONSE: parse Produce response, fire promises, advance recBatch. +4. PER-FLUSH: drain partitions, wait for in-flight to settle. + +Already tuned - do not suggest these: +- recBatch reuse / sticky pooling. +- Sticky partitioner. +- bufferedRecords semaphore for bounded memory. +- Compression options (LZ4/Snappy/Gzip/Zstd) at batch granularity. +- TopicID-keyed produce (KIP-516) when broker supports it. + +Find only: +1. Allocations on the PER-RECORD path: slice growth without preallocation, + string<->[]byte conversion, interface boxing of concrete types, closure + captures, time.Now overhead, log line construction not gated by level. +2. Allocations on the PER-BATCH path: byte slices that could come from a + pool, repeated header building, varint encoding scratch space, repeated + compressor instantiation. +3. Lock hold time in sink.go / recBuf: work under lock that could be done + outside; broad locks that could be narrower; sleep/IO under lock. +4. Wasted CPU: redundant validation, recomputed values, unnecessary + copying between buffers, recompression patterns. +5. Goroutine churn: per-record or per-batch goroutine spawn that could + be amortized to per-broker. +6. Map/slice access patterns on hot path: lookups replaceable with indexed + access; linear scans over partitions when an index exists. +7. Cache-line false sharing: fields written by different goroutines + sitting in the same cache line. +8. Channel overhead on hot path: small unbuffered channels, repeated + send/recv that could batch. + +Output per finding: +- Cost class: per-record | per-batch | per-response | per-flush | startup +- File:line +- What: one sentence +- Cost: e.g. "allocates a 64-byte slice per record; 1M rec/s -> 60MB/s garbage" +- Fix: sketch + +Skip: micro-optimizations on cold paths (config parsing, client init, +error paths). Skip "consider sync.Pool" unless you've identified the +specific allocation hot spot it would address. Skip anything where you +can't name the cost class. diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/producer.go b/vendor/github.com/twmb/franz-go/pkg/kgo/producer.go new file mode 100644 index 00000000000..c4a13618f92 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/producer.go @@ -0,0 +1,1336 @@ +package kgo + +import ( + "context" + "errors" + "fmt" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +type producer struct { + // mu and c are used for flush and drain notifications; mu is used for + // a few other tight locks. + mu xsync.Mutex + c *sync.Cond + + bufferedRecords int64 + bufferedBytes int64 + + cl *Client + + topicsMu xsync.Mutex // locked to prevent concurrent updates; reads are always atomic + topics *topicsPartitions + + // Hooks exist behind a pointer because likely they are not used. + // We only take up one byte vs. 6. + hooks *struct { + buffered []HookProduceRecordBuffered + partitioned []HookProduceRecordPartitioned + unbuffered []HookProduceRecordUnbuffered + } + + // unknownTopics buffers all records for topics that are not loaded. + // The map is to a pointer to a slice for reasons documented in + // waitUnknownTopic. + unknownTopicsMu xsync.Mutex + unknownTopics map[string]*unknownTopicProduces + + id atomic.Value + producingTxn atomic.Bool + + // We must have a producer field for flushing; we cannot just have a + // field on recBufs that is toggled on flush. If we did, then a new + // recBuf could be created and records sent to while we are flushing. + flushing atomic.Int32 // >0 if flushing, can Flush many times concurrently + blocked atomic.Int32 // >0 if over max recs or bytes + blockedBytes int64 + + aborting atomic.Int32 // >0 if aborting, can abort many times concurrently + + idMu xsync.Mutex + idVersion int16 + + batchPromises ring[batchPromise] // we never call die() on it + + txnMu xsync.Mutex + inTxn bool + tx890p2 atomic.Bool +} + +// BufferedProduceRecords returns the number of records currently buffered for +// producing within the client. +// +// This can be used as a gauge to determine how far behind the client is for +// flushing records produced by your client (which can help determine network / +// cluster health). +func (cl *Client) BufferedProduceRecords() int64 { + cl.producer.mu.Lock() + defer cl.producer.mu.Unlock() + return cl.producer.bufferedRecords + int64(cl.producer.blocked.Load()) +} + +// BufferedProduceBytes returns the number of bytes currently buffered for +// producing within the client. This is the sum of all keys, values, and header +// keys/values. See the related [BufferedProduceRecords] for more information. +func (cl *Client) BufferedProduceBytes() int64 { + cl.producer.mu.Lock() + defer cl.producer.mu.Unlock() + return cl.producer.bufferedBytes + cl.producer.blockedBytes +} + +// EnsureProduceConnectionIsOpen attempts to open a produce connection to all +// specified brokers, or all brokers if `brokers` is empty or contains -1. +// +// This can be used in an attempt to reduce the latency when producing if your +// application produces infrequently: you can force open a produce connection a +// bit earlier than you intend to produce, rather than at the moment you +// produce. In rare circumstances, it is possible that a connection that was +// ensured to be open may close before you produce. +// +// This returns an errors.Join'd error that merges a message for all brokers +// that failed to be opened as well as why. +func (cl *Client) EnsureProduceConnectionIsOpen(ctx context.Context, brokers ...int32) error { + var ( + keep = brokers[:0] + all bool + wg sync.WaitGroup + mu xsync.Mutex + errs []error + ) + for _, b := range brokers { + switch { + case b < -1: + case b == -1: + all = true + case b > -1: + keep = append(keep, b) + } + } + var toOpen []*broker + if all || len(brokers) == 0 { + cl.brokersMu.RLock() + toOpen = cl.brokers + if len(toOpen) == 0 { + cl.brokersMu.RUnlock() + if err := cl.fetchBrokerMetadata(ctx); err != nil { + return err + } + cl.brokersMu.RLock() + toOpen = cl.brokers + } + cl.brokersMu.RUnlock() + } else { + for _, b := range brokers { + wg.Add(1) + go func() { + defer wg.Done() + + br, err := cl.brokerOrErr(ctx, b, errUnknownBroker) + + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, fmt.Errorf("%d: %w", b, err)) + return + } + toOpen = append(toOpen, br) + }() + } + wg.Wait() + } + + for _, br := range toOpen { + wg.Add(1) + br.do(ctx, &forceOpenReq{new(kmsg.ProduceRequest)}, func(_ kmsg.Response, err error) { + defer wg.Done() + if err != nil { + mu.Lock() + errs = append(errs, fmt.Errorf("%d: %w", br.meta.NodeID, err)) + mu.Unlock() + } + }) + } + wg.Wait() + + return errors.Join(errs...) +} + +type unknownTopicProduces struct { + buffered []promisedRec + wait chan error // retryable errors + fatal chan error // must-signal quit errors; capacity 1 +} + +func (p *producer) init(cl *Client) { + p.cl = cl + p.topics = newTopicsPartitions() + p.unknownTopics = make(map[string]*unknownTopicProduces) + p.idVersion = -1 + p.id.Store(&producerID{ + id: -1, + epoch: -1, + err: errReloadProducerID, + }) + p.c = sync.NewCond(&p.mu) + p.batchPromises.initMaxLen(max(int(cl.cfg.maxBufferedRecords), 8192)) + + inithooks := func() { + if p.hooks == nil { + p.hooks = &struct { + buffered []HookProduceRecordBuffered + partitioned []HookProduceRecordPartitioned + unbuffered []HookProduceRecordUnbuffered + }{} + } + } + + cl.cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookProduceRecordBuffered); ok { + inithooks() + p.hooks.buffered = append(p.hooks.buffered, h) + } + if h, ok := h.(HookProduceRecordPartitioned); ok { + inithooks() + p.hooks.partitioned = append(p.hooks.partitioned, h) + } + if h, ok := h.(HookProduceRecordUnbuffered); ok { + inithooks() + p.hooks.unbuffered = append(p.hooks.unbuffered, h) + } + }) +} + +func (p *producer) purgeTopics(topics []string) { + p.topicsMu.Lock() + defer p.topicsMu.Unlock() + + p.unknownTopicsMu.Lock() + for _, topic := range topics { + if unknown, exists := p.unknownTopics[topic]; exists { + delete(p.unknownTopics, topic) + close(unknown.wait) + p.promiseBatch(batchPromise{ + recs: unknown.buffered, + err: errPurged, + }) + } + } + p.unknownTopicsMu.Unlock() + + toStore := p.topics.clone() + defer p.topics.storeData(toStore) + + for _, topic := range topics { + d := toStore.loadTopic(topic) + if d == nil { + continue + } + delete(toStore, topic) + for _, p := range d.partitions { + r := p.records + + // First we set purged, so that anything in the process + // of being buffered will immediately fail when it goes + // to buffer. + r.mu.Lock() + r.purged = true + r.mu.Unlock() + + // Now we remove from the sink. When we do, the recBuf + // is effectively abandoned. Any active produces may + // finish before we fail the records; if they finish + // after they will no longer belong in the batch, but + // they may have been produced. This is the duplicate + // risk a user runs when purging. + // + // We do not need to lock for `r.sink` access because + // this is run in a blocking metadata fn, meaning the + // sink cannot change. We do not WANT to lock because + // r.mu => r.sink.recBufsMu would cause lock inversion. + r.sink.removeRecBuf(r) + + // Once abandoned, we now need to fail anything that + // was buffered. + r.mu.Lock() + r.failAllRecords(errPurged) + r.mu.Unlock() + } + } +} + +func (p *producer) isAborting() bool { return p.aborting.Load() > 0 } + +func noPromise(*Record, error) {} + +// ProduceResult is the result of producing a record in a synchronous manner. +type ProduceResult struct { + // Record is the produced record. It is always non-nil. + // + // If this record was produced successfully, its attrs / offset / id / + // epoch / etc. fields are filled in on return if possible (i.e. when + // producing with acks required). + Record *Record + + // Err is a potential produce error. If this is non-nil, the record was + // not produced successfully. + Err error +} + +// ProduceResults is a collection of produce results. +type ProduceResults []ProduceResult + +// FirstErr returns the first erroring result, if any. +func (rs ProduceResults) FirstErr() error { + for _, r := range rs { + if r.Err != nil { + return r.Err + } + } + return nil +} + +// First the first record and error in the produce results. +// +// This function is useful if you only passed one record to ProduceSync. +func (rs ProduceResults) First() (*Record, error) { + return rs[0].Record, rs[0].Err +} + +// ProduceSync is a synchronous produce. See the [Produce] documentation for an +// in depth description of how producing works. +// +// This function produces all records and waits for them all to be produced +// before returning. If the client has a non-zero linger configured, after all +// records are enqueued, this function stops lingering and triggers an immediate +// drain on all partitions that records were produced to. This avoids +// unnecessarily waiting for linger timers when the caller is synchronously +// waiting for results. Partitions that are lingering due to concurrent +// [Produce] calls are not affected. +func (cl *Client) ProduceSync(ctx context.Context, rs ...*Record) ProduceResults { + var ( + wg sync.WaitGroup + results = make(ProduceResults, 0, len(rs)) + promise = func(r *Record, err error) { + results = append(results, ProduceResult{r, err}) + wg.Done() + } + ) + + wg.Add(len(rs)) + + // After each Produce call for a known topic, the record's Partition + // field is already set (see bufferRecord), allowing us to collect + // which recBufs to unlinger without a second pass over the records. + // We use a [16] base array to avoid heap allocation in the common + // case, and linear dedup since the number of unique partitions is + // typically small. + // + // We load partition data BEFORE calling Produce to avoid a data + // race on r.Partition. If partitions exist before Produce, + // partitionsForTopicProduce will also see them (partition counts + // are monotonically increasing) and will partition the record + // synchronously in bufferRecord, making r.Partition safe to read + // after Produce returns. If pd is nil, we never read r.Partition, + // avoiding a race with the metadata goroutine which partitions + // unknownTopics records asynchronously. + var ( + buf [16]*recBuf + unlinger = buf[:0] + topics topicsPartitionsData + + lastTopic string + lastPD *topicPartitionsData + ) + if cl.cfg.linger > 0 { + topics = cl.producer.topics.load() + } + + for _, r := range rs { + var pd *topicPartitionsData + if topics != nil { + if r.Topic == "" || cl.cfg.defaultProduceTopicAlways { + r.Topic = cl.cfg.defaultProduceTopic + } + if r.Topic == lastTopic { + pd = lastPD + } else if parts, ok := topics[r.Topic]; ok { + if v := parts.load(); len(v.partitions) > 0 { + pd = v + } + lastTopic = r.Topic + lastPD = pd + } + } + + cl.Produce(ctx, r, promise) + + if pd == nil { + continue + } + if r.Partition < 0 || int(r.Partition) >= len(pd.partitions) { + continue + } + rb := pd.partitions[r.Partition].records + var seen bool + for _, have := range unlinger { + if have == rb { + seen = true + break + } + } + if !seen { + unlinger = append(unlinger, rb) + } + } + + for _, rb := range unlinger { + rb.unlingerAndManuallyDrain() + } + + wg.Wait() + + return results +} + +// FirstErrPromise is a helper type to capture only the first failing error +// when producing a batch of records with this type's Promise function. +// +// This is useful for when you only care about any record failing, and can use +// that as a signal (i.e., to abort a batch). The AbortingFirstErrPromise +// function can be used to abort all records as soon as the first error is +// encountered. If you do not need to abort, you can use this type with no +// constructor. +// +// This is similar to using ProduceResult's FirstErr function. +type FirstErrPromise struct { + wg sync.WaitGroup + once atomic.Bool + err error + cl *Client +} + +// AbortingFirstErrPromise returns a FirstErrPromise that will call the +// client's AbortBufferedRecords function if an error is encountered. +// +// This can be used to quickly exit when any error is encountered, rather than +// waiting while flushing only to discover things errored. +func AbortingFirstErrPromise(cl *Client) *FirstErrPromise { + return &FirstErrPromise{ + cl: cl, + } +} + +// Promise is a promise for producing that will store the first error +// encountered. +func (f *FirstErrPromise) promise(_ *Record, err error) { + defer f.wg.Done() + if err != nil && !f.once.Swap(true) { + f.err = err + if f.cl != nil { + f.wg.Add(1) + go func() { + defer f.wg.Done() + f.cl.AbortBufferedRecords(context.Background()) + }() + } + } +} + +// Promise returns a promise for producing that will store the first error +// encountered. +// +// The returned promise must eventually be called, because a FirstErrPromise +// does not return from 'Err' until all promises are completed. +func (f *FirstErrPromise) Promise() func(*Record, error) { + f.wg.Add(1) + return f.promise +} + +// Err waits for all promises to complete and then returns any stored error. +func (f *FirstErrPromise) Err() error { + f.wg.Wait() + return f.err +} + +// TryProduce is similar to Produce, but rather than blocking if the client +// currently has MaxBufferedRecords or MaxBufferedBytes buffered, this fails +// immediately with ErrMaxBuffered. See the Produce documentation for more +// details. +func (cl *Client) TryProduce( + ctx context.Context, + r *Record, + promise func(*Record, error), +) { + cl.produce(ctx, r, promise, false) +} + +// Produce sends a Kafka record to the topic in the record's Topic field, +// calling an optional `promise` with the record and a potential error when +// Kafka replies. For a synchronous produce, see ProduceSync. Records are +// produced in order per partition if the record is produced successfully. +// Successfully produced records will have their attributes, offset, and +// partition set before the promise is called. All promises are called serially +// (and should be relatively fast). If a record's timestamp is unset, this +// sets the timestamp to time.Now. +// +// If the topic field is empty, the client will use the DefaultProduceTopic; if +// that is also empty, the record is failed immediately. If the record is too +// large to fit in a batch on its own in a produce request, the record will be +// failed with immediately kerr.MessageTooLarge. +// +// If the client is configured to automatically flush the client currently has +// the configured maximum amount of records buffered, Produce will block. The +// context can be used to cancel waiting while records flush to make space. In +// contrast, if manual flushing is configured, the record will be failed +// immediately with ErrMaxBuffered (this same behavior can be had with +// TryProduce). +// +// Once a record is buffered into a batch, it can be canceled in three ways: +// canceling the context, the record timing out, or hitting the maximum +// retries. If any of these conditions are hit and it is currently safe to fail +// records, all buffered records for the relevant partition are failed. Only +// the first record's context in a batch is considered when determining whether +// the batch should be canceled. A record is not safe to fail if the client +// is idempotently producing and a request has been sent; in this case, the +// client cannot know if the broker actually processed the request (if so, then +// removing the records from the client will create errors the next time you +// produce). +// +// If the client is transactional and a transaction has not been begun, the +// promise is immediately called with an error corresponding to not being in a +// transaction. +func (cl *Client) Produce( + ctx context.Context, + r *Record, + promise func(*Record, error), +) { + cl.produce(ctx, r, promise, true) +} + +func (cl *Client) produce( + ctx context.Context, + r *Record, + promise func(*Record, error), + block bool, +) { + if ctx == nil { + ctx = context.Background() + } + if r.Context == nil { + r.Context = ctx + } + if promise == nil { + promise = noPromise + } + if r.Topic == "" || cl.cfg.defaultProduceTopicAlways { + r.Topic = cl.cfg.defaultProduceTopic + } + + p := &cl.producer + if p.hooks != nil && len(p.hooks.buffered) > 0 { + for _, h := range p.hooks.buffered { + h.OnProduceRecordBuffered(r) + } + } + + // We can now fail the rec after the buffered hook. + if r.Topic == "" { + p.promiseRecordBeforeBuf(promisedRec{ctx, promise, r}, errNoTopic) + return + } + if cl.cfg.txnID != nil && !p.producingTxn.Load() { + p.promiseRecordBeforeBuf(promisedRec{ctx, promise, r}, errNotInTransaction) + return + } + + userSize := r.userSize() + if cl.cfg.maxBufferedBytes > 0 && userSize > cl.cfg.maxBufferedBytes { + p.promiseRecordBeforeBuf( + promisedRec{ctx, promise, r}, + fmt.Errorf("%w (uncompressed_bytes=%d)", kerr.MessageTooLarge, userSize), + ) + return + } + + // We have to grab the produce lock to check if this record will exceed + // configured limits. We try to keep the logic tight since this is + // effectively a global lock around producing. + p.mu.Lock() + overMaxRecs := p.bufferedRecords >= cl.cfg.maxBufferedRecords + overMaxBytes := cl.cfg.maxBufferedBytes > 0 && p.bufferedBytes+userSize > cl.cfg.maxBufferedBytes + if overMaxRecs || overMaxBytes { + if !block || cl.cfg.manualFlushing { + p.mu.Unlock() + p.promiseRecordBeforeBuf(promisedRec{ctx, promise, r}, ErrMaxBuffered) + return + } + + // Before we potentially unlinger, add that we are blocked to + // ensure we do NOT start a linger anymore. We THEN wakeup + // anything that is actively lingering. Note that blocked is + // also used when finishing promises to see if we need to be + // notified. + p.blocked.Add(1) + p.blockedBytes += userSize + p.mu.Unlock() + + cl.cfg.logger.Log(LogLevelDebug, "blocking Produce because we are either over max buffered records or max buffered bytes", + "over_max_records", overMaxRecs, + "over_max_bytes", overMaxBytes, + ) + + cl.unlingerDueToMaxRecsBuffered() + + // We keep the lock when we exit. If we are flushing, we want + // this blocked record to be produced before we return from + // flushing. This blocked record will be accounted for in the + // bufferedRecords addition below, after being removed from + // blocked in the goroutine. + wait := make(chan struct{}) + var quit bool + go func() { + defer close(wait) + p.mu.Lock() + for !quit && (p.bufferedRecords >= cl.cfg.maxBufferedRecords || + (cl.cfg.maxBufferedBytes > 0 && p.bufferedBytes+userSize > cl.cfg.maxBufferedBytes)) { + p.c.Wait() + } + p.blocked.Add(-1) + p.blockedBytes -= userSize + }() + + drainBuffered := func(err error) { + // The expected case here is that a context was + // canceled while we waiting for space, so we are + // exiting and need to kill the goro above. + // + // However, it is possible that the goro above has + // already exited AND the context was canceled, and + // `select` chose the context-canceled case. + // + // So, to avoid a deadlock, we need to wakeup the + // goro above in another goroutine. + go func() { + p.mu.Lock() + quit = true + p.mu.Unlock() + p.c.Broadcast() + }() + <-wait // we wait for the goroutine to exit, then unlock again (since the goroutine leaves the mutex locked) + p.mu.Unlock() + p.promiseRecordBeforeBuf(promisedRec{ctx, promise, r}, err) + } + + select { + case <-wait: + cl.cfg.logger.Log(LogLevelDebug, "Produce block awoken, we now have space to produce, continuing to partition and produce") + case <-cl.ctx.Done(): + drainBuffered(ErrClientClosed) + cl.cfg.logger.Log(LogLevelDebug, "client ctx canceled while blocked in Produce, returning") + return + case <-ctx.Done(): + drainBuffered(ctx.Err()) + cl.cfg.logger.Log(LogLevelDebug, "produce ctx canceled while blocked in Produce, returning") + return + } + } + p.bufferedRecords++ + p.bufferedBytes += userSize + p.mu.Unlock() + + cl.loadPartsAndPartition(promisedRec{ctx, promise, r}) +} + +type batchPromise struct { + baseOffset int64 + pid int64 + epoch int16 + attrs RecordAttrs + beforeBuf bool + recs []promisedRec + err error +} + +func (p *producer) promiseBatch(b batchPromise) { + if first, _ := p.batchPromises.push(b); first { + go p.finishPromises(b) + } +} + +func (p *producer) promiseRecord(pr promisedRec, err error) { + p.promiseBatch(batchPromise{recs: []promisedRec{pr}, err: err}) +} + +func (p *producer) promiseRecordBeforeBuf(pr promisedRec, err error) { + p.promiseBatch(batchPromise{recs: []promisedRec{pr}, beforeBuf: true, err: err}) +} + +func (p *producer) finishPromises(b batchPromise) { + cl := p.cl + var more bool + var broadcast bool + defer func() { + if broadcast { + p.c.Broadcast() + } + }() +start: + for i, pr := range b.recs { + pr.LeaderEpoch = -1 + if b.baseOffset == -1 { + // if the base offset is invalid/unknown (-1), all record offsets should + // be treated as unknown + pr.Offset = -1 + } else { + pr.Offset = b.baseOffset + int64(i) + } + pr.ProducerID = b.pid + pr.ProducerEpoch = b.epoch + pr.Attrs = b.attrs + recBroadcast := cl.finishRecordPromise(pr, b.err, b.beforeBuf) + broadcast = broadcast || recBroadcast + } + clear(b.recs) // drop references so the pooled slice does not retain them + if cap(b.recs) > 4 { + cl.prsPool.put(b.recs) + } + + b, more, _ = p.batchPromises.dropPeek() + if more { + goto start + } +} + +func (cl *Client) finishRecordPromise(pr promisedRec, err error, beforeBuffering bool) (broadcast bool) { + p := &cl.producer + + if p.hooks != nil && len(p.hooks.unbuffered) > 0 { + for _, h := range p.hooks.unbuffered { + h.OnProduceRecordUnbuffered(pr.Record, err) + } + } + + // Capture user size before potential modification by the promise. + // + // We call the promise before finishing the flush notification, + // allowing users of Flush to know all buf recs are done by the + // time we notify flush below. + userSize := pr.userSize() + pr.promise(pr.Record, err) + + // If this record was never buffered, it's size was never accounted + // for on any p field: return early. + if beforeBuffering { + return broadcast + } + + // Keep the lock as tight as possible: the broadcast can come after. + p.mu.Lock() + p.bufferedBytes -= userSize + p.bufferedRecords-- + broadcast = p.blocked.Load() > 0 || p.bufferedRecords == 0 && p.flushing.Load() > 0 + p.mu.Unlock() + + return broadcast +} + +// loadPartsAndPartition loads the partitions for a topic and produce to them. +// If the topic does not currently exist, the record is buffered in +// unknownTopics for a metadata update to deal with. +func (cl *Client) loadPartsAndPartition(pr promisedRec) { + parts, partsData := cl.partitionsForTopicProduce(pr) + if parts == nil { // saved in unknownTopics + return + } + cl.doPartition(parts, partsData, pr) +} + +func (cl *Client) doPartition(parts *topicPartitions, partsData *topicPartitionsData, pr promisedRec) { + if partsData.loadErr != nil && !kerr.IsRetriable(partsData.loadErr) { + cl.producer.promiseRecord(pr, partsData.loadErr) + return + } + + parts.partsMu.Lock() + defer parts.partsMu.Unlock() + if parts.partitioner == nil { + parts.partitioner = cl.cfg.partitioner.ForTopic(pr.Topic) + } + + mapping := partsData.writablePartitions + if parts.partitioner.RequiresConsistency(pr.Record) { + mapping = partsData.partitions + } + if len(mapping) == 0 { + cl.producer.promiseRecord(pr, errors.New("unable to partition record due to no usable partitions")) + return + } + + var pick int + tlp, _ := parts.partitioner.(TopicBackupPartitioner) + if tlp != nil { + if parts.lb == nil { + parts.lb = new(leastBackupInput) + } + parts.lb.mapping = mapping + pick = tlp.PartitionByBackup(pr.Record, len(mapping), parts.lb) + } else { + pick = parts.partitioner.Partition(pr.Record, len(mapping)) + } + if pick < 0 || pick >= len(mapping) { + cl.producer.promiseRecord(pr, fmt.Errorf("invalid record partitioning choice of %d from %d available", pick, len(mapping))) + return + } + + partition := mapping[pick] + + onNewBatch, _ := parts.partitioner.(TopicPartitionerOnNewBatch) + abortOnNewBatch := onNewBatch != nil + processed := partition.records.bufferRecord(pr, abortOnNewBatch) // KIP-480 + if !processed { + onNewBatch.OnNewBatch() + + if tlp != nil { + parts.lb.mapping = mapping + pick = tlp.PartitionByBackup(pr.Record, len(mapping), parts.lb) + } else { + pick = parts.partitioner.Partition(pr.Record, len(mapping)) + } + + if pick < 0 || pick >= len(mapping) { + cl.producer.promiseRecord(pr, fmt.Errorf("invalid record partitioning choice of %d from %d available", pick, len(mapping))) + return + } + partition = mapping[pick] + partition.records.bufferRecord(pr, false) // KIP-480 + } +} + +// ProducerID returns, loading if necessary, the current producer ID and epoch. +// This returns an error if the producer ID could not be loaded, if the +// producer ID has fatally errored, or if the context is canceled. +func (cl *Client) ProducerID(ctx context.Context) (int64, int16, error) { + var ( + id int64 + epoch int16 + err error + + done = make(chan struct{}) + ) + + go func() { + defer close(done) + id, epoch, err = cl.producerID(ctx2fn(ctx)) + }() + + select { + case <-ctx.Done(): + return 0, 0, ctx.Err() + case <-done: + return id, epoch, err + } +} + +type producerID struct { + id int64 + epoch int16 + err error +} + +var errReloadProducerID = errors.New("producer id needs reloading") + +// initProducerID initializes the client's producer ID for idempotent +// producing only (no transactions, which are more special). After the first +// load, this clears all buffered unknown topics. +func (cl *Client) producerID(ctxFn func() context.Context) (int64, int16, error) { + p := &cl.producer + + id := p.id.Load().(*producerID) + if errors.Is(id.err, errReloadProducerID) { + p.idMu.Lock() + defer p.idMu.Unlock() + + if id = p.id.Load().(*producerID); errors.Is(id.err, errReloadProducerID) { + if cl.cfg.disableIdempotency { + cl.cfg.logger.Log(LogLevelInfo, "skipping producer id initialization because the client was configured to disable idempotent writes") + id = &producerID{ + id: -1, + epoch: -1, + err: nil, + } + p.id.Store(id) + } else if cl.cfg.txnID == nil && id.id >= 0 && id.epoch < math.MaxInt16-1 { + // For the idempotent producer, as specified in KIP-360, + // if we had an ID, we can bump the epoch locally. + // If we are at the max epoch, we will ask for a new ID. + cl.cfg.logger.Log(LogLevelInfo, "locally bumping idempotent producer epoch to recover from a prior produce error", + "id", id.id, "old_epoch", id.epoch, "new_epoch", id.epoch+1, + ) + cl.resetAllProducerSequences() + id = &producerID{ + id: id.id, + epoch: id.epoch + 1, + err: nil, + } + p.id.Store(id) + } else { + newID, keep := cl.doInitProducerID(ctxFn, id.id, id.epoch) + if keep { + id = newID + // Whenever we have a new producer ID, we need + // our sequence numbers to be 0. On the first + // record produced, this will be true, but if + // we were signaled to reset the producer ID, + // then we definitely still need to reset here. + cl.resetAllProducerSequences() + p.id.Store(id) + } else { + // If we are not keeping the producer ID, + // we will return our old ID but with a + // static error that we can check or bubble + // up where needed. + id = &producerID{ + id: id.id, + epoch: id.epoch, + err: &errProducerIDLoadFail{newID.err}, + } + } + } + } + } + + return id.id, id.epoch, id.err +} + +// As seen in KAFKA-12152, if we bump an epoch, we have to reset sequence nums +// for every partition. Otherwise, we will use a new id/epoch for a partition +// and trigger OOOSN errors. +// +// Pre 2.5, this function is only be called if it is acceptable to continue +// on data loss (idempotent producer with no StopOnDataLoss option). +// +// 2.5+, it is safe to call this if the producer ID can be reset (KIP-360), +// in EndTransaction. +func (cl *Client) resetAllProducerSequences() { + for _, tp := range cl.producer.topics.load() { + for _, p := range tp.load().partitions { + p.records.mu.Lock() + p.records.needSeqReset = true + p.records.mu.Unlock() + } + } +} + +func (cl *Client) failProducerID(id int64, epoch int16, err error) { + p := &cl.producer + + // We do not lock the idMu when failing a producer ID, for two reasons. + // + // 1) With how we store below, we do not need to. We only fail if the + // ID we are failing has not changed and if the ID we are failing has + // not failed already. Failing outside the lock is the same as failing + // within the lock. + // + // 2) Locking would cause a deadlock, because producerID locks + // idMu=>recBuf.Mu, whereas we failing while locked within a recBuf in + // sink.go. + new := &producerID{ + id: id, + epoch: epoch, + err: err, + } + for { + current := p.id.Load().(*producerID) + if current.id != id || current.epoch != epoch { + cl.cfg.logger.Log(LogLevelInfo, "ignoring a fail producer id request due to current id being different", + "current_id", current.id, + "current_epoch", current.epoch, + "current_err", current.err, + "fail_id", id, + "fail_epoch", epoch, + "fail_err", err, + ) + return + } + if current.err != nil { + cl.cfg.logger.Log(LogLevelInfo, "ignoring a fail producer id because our producer id has already been failed", + "current_id", current.id, + "current_epoch", current.epoch, + "current_err", current.err, + "fail_err", err, + ) + return + } + if p.id.CompareAndSwap(current, new) { + cl.cfg.logger.Log(LogLevelInfo, "producer ID failed due to a produce error, next produce will reinitialize the producer epoch", + "id", id, "epoch", epoch, "err", err, + ) + return + } + } +} + +// doInitProducerID inits the idempotent ID and potentially the transactional +// producer epoch, returning whether to keep the result. +func (cl *Client) doInitProducerID(ctxFn func() context.Context, lastID int64, lastEpoch int16) (*producerID, bool) { + cl.cfg.logger.Log(LogLevelInfo, "initializing producer id") + req := kmsg.NewPtrInitProducerIDRequest() + req.TransactionalID = cl.cfg.txnID + req.ProducerID = lastID + req.ProducerEpoch = lastEpoch + if cl.cfg.txnID != nil { + req.TransactionTimeoutMillis = int32(cl.cfg.txnTimeout.Milliseconds()) + } + + ctx := ctxFn() + var resp *kmsg.InitProducerIDResponse + err := cl.doWithConcurrentTransactions(ctx, "InitProducerID", func() error { + var err error + resp, err = req.RequestWith(ctx, cl) + if err != nil { + return err + } + return nil // resp.ErrorCode handled below + }) + if err != nil { + if errors.Is(err, errUnknownRequestKey) || errors.Is(err, errBrokerTooOld) { + cl.cfg.logger.Log(LogLevelInfo, "unable to initialize a producer id because the broker is too old or the client is pinned to an old version, continuing without a producer id") + return &producerID{-1, -1, nil}, true + } + if errors.Is(err, errChosenBrokerDead) { + select { + case <-cl.ctx.Done(): + cl.cfg.logger.Log(LogLevelInfo, "producer id initialization failure due to dying client", "err", err) + return &producerID{lastID, lastEpoch, ErrClientClosed}, true + default: + } + } + cl.cfg.logger.Log(LogLevelInfo, "producer id initialization failure, discarding initialization attempt", "err", err) + return &producerID{lastID, lastEpoch, err}, false + } + + if err = kerr.ErrorForCode(resp.ErrorCode); err != nil { + // We could receive concurrent transactions; this is ignorable + // and we just want to re-init. + if kerr.IsRetriable(err) || errors.Is(err, kerr.ConcurrentTransactions) { + cl.cfg.logger.Log(LogLevelInfo, "producer id initialization resulted in retryable error, discarding initialization attempt", "err", err) + return &producerID{lastID, lastEpoch, err}, false + } + cl.cfg.logger.Log(LogLevelInfo, "producer id initialization errored", "err", err) + return &producerID{lastID, lastEpoch, err}, true + } + + cl.cfg.logger.Log(LogLevelInfo, "producer id initialization success", "id", resp.ProducerID, "epoch", resp.ProducerEpoch) + + // We track if this was v3. We do not need to gate this behind a mutex, + // because the only other use is EndTransaction's read, which is + // documented to only be called sequentially after producing. + if cl.producer.idVersion == -1 { + cl.producer.idVersion = req.Version + } + + return &producerID{resp.ProducerID, resp.ProducerEpoch, nil}, true +} + +// partitionsForTopicProduce returns the topic partitions for a record. +// If the topic is not loaded yet, this buffers the record and returns +// nil, nil. +func (cl *Client) partitionsForTopicProduce(pr promisedRec) (*topicPartitions, *topicPartitionsData) { + p := &cl.producer + topic := pr.Topic + + topics := p.topics.load() + parts, exists := topics[topic] + if exists { + if v := parts.load(); len(v.partitions) > 0 { + return parts, v + } + } + + if !exists { // topic did not exist: check again under mu and potentially create it + p.topicsMu.Lock() + defer p.topicsMu.Unlock() + + if parts, exists = p.topics.load()[topic]; !exists { // update parts for below + // Before we store the new topic, we lock unknown + // topics to prevent a concurrent metadata update + // seeing our new topic before we are waiting from the + // addUnknownTopicRecord fn. Otherwise, we would wait + // and never be re-notified. + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + p.topics.storeTopics([]string{topic}) + cl.addUnknownTopicRecord(pr) + cl.triggerUpdateMetadataNow("forced load because we are producing to a topic for the first time") + return nil, nil + } + } + + // Here, the topic existed, but maybe has not loaded partitions yet. We + // have to lock unknown topics first to ensure ordering just in case a + // load has not happened. + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + if v := parts.load(); len(v.partitions) > 0 { + return parts, v + } + cl.addUnknownTopicRecord(pr) + cl.triggerUpdateMetadata(false, "reload trigger due to produce topic still not known") + + return nil, nil // our record is buffered waiting for metadata update; nothing to return +} + +// addUnknownTopicRecord adds a record to a topic whose partitions are +// currently unknown. This is always called with the unknownTopicsMu held. +func (cl *Client) addUnknownTopicRecord(pr promisedRec) { + unknown := cl.producer.unknownTopics[pr.Topic] + if unknown == nil { + unknown = &unknownTopicProduces{ + buffered: make([]promisedRec, 0, 100), + wait: make(chan error, 5), + fatal: make(chan error, 1), + } + cl.producer.unknownTopics[pr.Topic] = unknown + } + unknown.buffered = append(unknown.buffered, pr) + if len(unknown.buffered) == 1 { + go cl.waitUnknownTopic(pr.ctx, pr.Context, pr.Topic, unknown) + } +} + +// waitUnknownTopic waits for a notification +func (cl *Client) waitUnknownTopic( + pctx context.Context, // context passed to Produce + rctx context.Context, // context on the record itself + topic string, + unknown *unknownTopicProduces, +) { + cl.cfg.logger.Log(LogLevelInfo, "producing to a new topic for the first time, fetching metadata to learn its partitions", "topic", topic) + + var ( + tries int + unknownTries int64 + err error + lastRetryErr error + after <-chan time.Time + ) + + if timeout := cl.cfg.recordTimeout; timeout > 0 { + timer := time.NewTimer(cl.cfg.recordTimeout) + defer timer.Stop() + after = timer.C + } + + // Ordering: aborting is set first, then unknown topics are manually + // canceled in a lock. New unknown topics after that lock will see + // aborting here and immediately cancel themselves. + if cl.producer.isAborting() { + err = ErrAborting + } + + for err == nil { + select { + case <-pctx.Done(): + err = pctx.Err() + case <-rctx.Done(): + err = rctx.Err() + case <-cl.ctx.Done(): + err = ErrClientClosed + case <-after: + if lastRetryErr != nil { + err = fmt.Errorf("%w, last err: %w", ErrRecordTimeout, lastRetryErr) + } else { + err = ErrRecordTimeout + } + case err = <-unknown.fatal: + case retryableErr, ok := <-unknown.wait: + if !ok { + cl.cfg.logger.Log(LogLevelInfo, "done waiting for metadata for new topic", "topic", topic) + return // metadata was successful! + } + cl.cfg.logger.Log(LogLevelInfo, "new topic metadata wait failed, retrying wait", "topic", topic, "err", retryableErr) + lastRetryErr = retryableErr + tries++ + if int64(tries) > cl.cfg.recordRetries { + err = fmt.Errorf("no partitions available after attempting to refresh metadata %d times, last err: %w", tries, retryableErr) + } + if cl.cfg.maxUnknownFailures >= 0 && errors.Is(retryableErr, kerr.UnknownTopicOrPartition) { + unknownTries++ + if unknownTries > cl.cfg.maxUnknownFailures { + err = retryableErr + } + } + } + } + + // If we errored above, we come down here to potentially clear the + // topic wait and fail all buffered records. However, under some + // extreme conditions, a quickly following metadata update could delete + // our unknown topic, and then a produce could recreate a new unknown + // topic. We only delete and finish promises if the pointer in the + // unknown topic map is still the same. + p := &cl.producer + + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + nowUnknown := p.unknownTopics[topic] + if nowUnknown != unknown { + return + } + cl.cfg.logger.Log(LogLevelInfo, "new topic metadata wait failed, done retrying, failing all records", "topic", topic, "err", err) + + delete(p.unknownTopics, topic) + p.promiseBatch(batchPromise{ + recs: unknown.buffered, + err: err, + }) +} + +func (cl *Client) unlingerDueToMaxRecsBuffered() { + if cl.cfg.linger <= 0 { + return + } + for _, parts := range cl.producer.topics.load() { + for _, part := range parts.load().partitions { + part.records.unlingerAndManuallyDrain() + } + } + cl.cfg.logger.Log(LogLevelDebug, "unlingered all partitions due to hitting max buffered") +} + +// Flush hangs waiting for all buffered records to be flushed, stopping all +// lingers if necessary. +// +// If the context finishes (Done), this returns the context's error. +// +// This function is safe to call multiple times concurrently, and safe to call +// concurrent with Flush. +func (cl *Client) Flush(ctx context.Context) error { + p := &cl.producer + + // Signal to finishRecord that we want to be notified once buffered hits 0. + // Also forbid any new producing to start a linger. + p.flushing.Add(1) + defer p.flushing.Add(-1) + + cl.cfg.logger.Log(LogLevelInfo, "flushing") + defer cl.cfg.logger.Log(LogLevelDebug, "flushed") + + // At this point, if lingering is configured, nothing will _start_ a + // linger because the producer's flushing atomic int32 is nonzero. We + // must wake anything that could be lingering up, after which all sinks + // will loop draining. + if cl.cfg.linger > 0 || cl.cfg.manualFlushing { + for _, parts := range p.topics.load() { + for _, part := range parts.load().partitions { + part.records.unlingerAndManuallyDrain() + } + } + } + + quit := false + done := make(chan struct{}) + go func() { + p.mu.Lock() + defer p.mu.Unlock() + defer close(done) + + for !quit && p.bufferedRecords+int64(p.blocked.Load()) > 0 { + p.c.Wait() + } + }() + + select { + case <-done: + return nil + case <-ctx.Done(): + p.mu.Lock() + quit = true + p.mu.Unlock() + p.c.Broadcast() + return ctx.Err() + } +} + +// Bumps the tries for all buffered records in the client. +// +// This is called whenever there is a problematic error that would affect the +// state of all buffered records as a whole: +// +// - if we cannot init a producer ID due to RequestWith errors, producing is useless +// - if we cannot add partitions to a txn due to RequestWith errors, producing is useless +// +// Note that these are specifically due to RequestWith errors, not due to +// receiving a response that has a retryable error code. That is, if our +// request keeps dying. +func (cl *Client) bumpRepeatedLoadErr(err error) { + p := &cl.producer + + for _, partitions := range p.topics.load() { + for _, partition := range partitions.load().partitions { + partition.records.bumpRepeatedLoadErr(err) + } + } + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + for _, unknown := range p.unknownTopics { + select { + case unknown.wait <- err: + default: + } + } +} + +// Clears all buffered records in the client with the given error. +// +// - closing client +// - aborting transaction +// - fatal AddPartitionsToTxn +// +// Because the error fails everything, we also empty our unknown topics and +// delete any topics that were still unknown from the producer's topics. +func (cl *Client) failBufferedRecords(err error) { + p := &cl.producer + + for _, partitions := range p.topics.load() { + for _, partition := range partitions.load().partitions { + recBuf := partition.records + recBuf.mu.Lock() + recBuf.failAllRecords(err) + recBuf.mu.Unlock() + } + } + + p.topicsMu.Lock() + defer p.topicsMu.Unlock() + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + toStore := p.topics.clone() + defer p.topics.storeData(toStore) + + var toFail [][]promisedRec + for topic, unknown := range p.unknownTopics { + delete(toStore, topic) + delete(p.unknownTopics, topic) + close(unknown.wait) + toFail = append(toFail, unknown.buffered) + } + + for _, fail := range toFail { + p.promiseBatch(batchPromise{ + recs: fail, + err: err, + }) + } +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/record_and_fetch.go b/vendor/github.com/twmb/franz-go/pkg/kgo/record_and_fetch.go new file mode 100644 index 00000000000..2c0275b16d1 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/record_and_fetch.go @@ -0,0 +1,734 @@ +package kgo + +import ( + "context" + "errors" + "iter" + "time" + "unsafe" +) + +// RecordHeader contains extra information that can be sent with Records. +type RecordHeader struct { + Key string + Value []byte +} + +// RecordAttrs contains additional meta information about a record, such as its +// compression or timestamp type. +type RecordAttrs struct { + // 6 bits are used right now for record batches, and we use the high + // bit to signify no timestamp due to v0 message set. + // + // bits 1 thru 3: + // 000 no compression + // 001 gzip + // 010 snappy + // 011 lz4 + // 100 zstd + // bit 4: timestamp type + // bit 5: is transactional + // bit 6: is control + // bit 8: no timestamp type + attrs uint8 +} + +// TimestampType specifies how Timestamp was determined. +// +// The default, 0, means that the timestamp was determined in a client +// when the record was produced. +// +// An alternative is 1, which is when the Timestamp is set in Kafka. +// +// Records pre 0.10.0 did not have timestamps and have value -1. +func (a RecordAttrs) TimestampType() int8 { + if a.attrs&0b1000_0000 != 0 { + return -1 + } + return int8(a.attrs&0b0000_1000) >> 3 +} + +// CompressionType signifies with which algorithm this record was compressed. +// +// 0 is no compression, 1 is gzip, 2 is snappy, 3 is lz4, and 4 is zstd. +// The returned uint8 can be converted directly to a [CompressionCodecType]. +func (a RecordAttrs) CompressionType() uint8 { + return a.attrs & 0b0000_0111 +} + +// IsTransactional returns whether a record is a part of a transaction. +func (a RecordAttrs) IsTransactional() bool { + return a.attrs&0b0001_0000 != 0 +} + +// IsControl returns whether a record is a "control" record (ABORT or COMMIT). +// These are generally not visible unless explicitly opted into. +func (a RecordAttrs) IsControl() bool { + return a.attrs&0b0010_0000 != 0 +} + +// Record is a record to write to Kafka. +type Record struct { + // Key is an optional field that can be used for partition assignment. + // + // This is generally used with a hash partitioner to cause all records + // with the same key to go to the same partition. + Key []byte + // Value is blob of data to write to Kafka. + Value []byte + + // Headers are optional key/value pairs that are passed along with + // records. + // + // These are purely for producers and consumers; Kafka does not look at + // this field and only writes it to disk. + Headers []RecordHeader + + // NOTE: if logAppendTime, timestamp is MaxTimestamp, not first + delta + // zendesk/ruby-kafka#706 + + // Timestamp is the timestamp that will be used for this record. + // + // Record batches are always written with "CreateTime", meaning that + // timestamps are generated by clients rather than brokers. + // + // When producing, if this field is not yet set, it is set to time.Now. + Timestamp time.Time + + // Topic is the topic that a record is written to. + // + // This must be set for producing. + Topic string + + // Partition is the partition that a record is written to. + // + // For producing, this is left unset. This will be set by the client + // before the record is unbuffered. If you use the ManualPartitioner, + // the value of this field is always the partition chosen when + // producing (i.e., you partition manually ahead of time). + Partition int32 + + // Attrs specifies what attributes were on this record. + // + // For producing, this is left unset. This will be set by the client + // before the record is unbuffered. + Attrs RecordAttrs + + // ProducerEpoch is the producer epoch of this message if it was + // produced with a producer ID. An epoch and ID of 0 means it was not. + // + // For producing, this is left unset. This will be set by the client + // before the record is unbuffered. + ProducerEpoch int16 + + // ProducerID is the producer ID of this message if it was produced + // with a producer ID. An epoch and ID of 0 means it was not. + // + // For producing, this is left unset. This will be set by the client + // before the record is unbuffered. + ProducerID int64 + + // LeaderEpoch is the leader epoch of the broker at the time this + // record was written, or -1 if on message sets. When producing, + // this is always set to -1 (producers do not use this field and the + // broker does not reply with the epoch). + // + // For committing records, it is not recommended to modify the + // LeaderEpoch. Clients use the LeaderEpoch for data loss detection. + LeaderEpoch int32 + + // Offset is the offset that a record is written as. + // + // For producing, this is left unset. This will be set by the client + // before the record is unbuffered. If you are producing with no acks, + // this will just be the offset used in the produce request and does + // not mirror the offset actually stored within Kafka. + Offset int64 + + // Context is an optional field that is used for enriching records. + // + // If this field is nil when producing, it is set to the Produce ctx + // arg. This field can be used to propagate record enrichment across + // producer hooks. It can also be set in a consumer hook to propagate + // enrichment to consumer clients. + Context context.Context +} + +func (r *Record) userSize() int64 { + s := len(r.Key) + len(r.Value) + for _, h := range r.Headers { + s += len(h.Key) + len(h.Value) + } + return int64(s) +} + +// When buffering records, we calculate the length and tsDelta ahead of time +// (also because number width affects encoding length). We repurpose the Offset +// field to save space. +func (r *Record) setLengthAndTimestampDelta(length int32, tsDelta int64) { + r.LeaderEpoch = length + r.Offset = tsDelta +} + +func (r *Record) lengthAndTimestampDelta() (length int32, tsDelta int64) { + return r.LeaderEpoch, r.Offset +} + +// AppendFormat appends a record to b given the layout or returns an error if +// the layout is invalid. This is a one-off shortcut for using +// NewRecordFormatter. See that function's documentation for the layout +// specification. +func (r *Record) AppendFormat(b []byte, layout string) ([]byte, error) { + f, err := NewRecordFormatter(layout) + if err != nil { + return b, err + } + return f.AppendRecord(b, r), nil +} + +// DeliveryCount returns the share group delivery count for this record: 1 on +// first delivery, incremented each time the broker re-delivers the record +// after an AckRelease or acquisition lock timeout. Returns 0 for records that +// are not from a share group fetch. +func (r *Record) DeliveryCount() int32 { + st := shareAckFromCtx(r) + if st == nil { + return 0 + } + return st.deliveryCount +} + +// AcquisitionDeadline returns the share group acquisition lock deadline for +// this record: the broker will release the record for re-delivery if the +// consumer has not acknowledged it (Accept, Release, Reject, or Renew) by this +// time. Returns the zero time for records that are not from a share group +// fetch. +// +// Note this is computed from the acquisition timeout millis in the Kafka +// ShareFetch response when the client decodes the response. The milliseconds +// are computed on the broker, and the timeout begins before the fetch is sent +// to the client. As well, actually sending an ack back to the broker will have +// network overhead. It is best to assume your actual practical deadline may be +// a few seconds before the deadline returned from this function. +func (r *Record) AcquisitionDeadline() time.Time { + if r.Context == nil { + return time.Time{} + } + slab, _ := r.Context.Value(shareAckKey).(*shareAckSlab) + if slab == nil { + return time.Time{} + } + return time.Unix(0, slab.acqLockDeadlineNanos) +} + +// Ack sets the acknowledgement status for a share group record. This is a +// no-op for records that are not from a share group fetch. Acks flush +// periodically in the background; to force a flush, call +// [Client.FlushAcks]. +// +// Terminal statuses (AckAccept, AckRelease, AckReject) commit the record for +// the broker. A record that is never explicitly acked is auto-accepted when +// you poll. +// +// AckRenew extends the broker's acquisition lock (requires Kafka 4.2+) so the +// caller can process a single record for longer than the broker's acquisition +// lock timeout (30s default) without the record being released to another +// consumer. Use AckRenew when processing one record takes longer than the +// lock window and you do not intend to poll for more records until you are +// done. AckRenew does NOT defer the terminal ack across polls: if you call +// the next [Client.PollRecords] without first providing a terminal ack, the +// renewed record is auto-accepted by the next poll's finalize pass, exactly +// as if you had never renewed it. AckRenew may be called repeatedly within a +// processing window, but only after each prior renew has been acknowledged +// by the broker; a renew issued while a prior renew is still in flight is a +// no-op (the broker would coalesce them either way). After the broker +// confirms a renew, the next call extends the lock again. +// +// Renewals are best-effort. If the partition's leader changes or the +// broker connection drops between the renewal and the terminal ack, the +// held record is lost and the broker will redeliver it (possibly to +// another consumer). Share group consumers are at-least-once regardless: +// any TCP hiccup can cause re-delivery, so user processing must be +// idempotent. +func (r *Record) Ack(status AckStatus) { + if status < AckAccept || status > AckRenew { + return + } + st := shareAckFromCtx(r) + if st == nil || !st.tryAck(status, false) { + return + } + // Every successful CAS appends an entry and increments + // pendingAcks. Multiple calls on the same record (e.g. + // renew then accept) produce multiple entries that coalesce + // at request-build time. + st.appendAck() +} + +// StringRecord returns a Record with the Value field set to the input value +// string. For producing, this function is useful in tandem with the +// client-level DefaultProduceTopic option. +// +// This function uses the 'unsafe' package to avoid copying value into a slice. +// +// NOTE: It is NOT SAFE to modify the record's value. This function should only +// be used if you only ever read record fields. This function can safely be used +// for producing; the client never modifies a record's key nor value fields. +func StringRecord(value string) *Record { + return &Record{Value: unsafe.Slice(unsafe.StringData(value), len(value))} //nolint:gosec // G103 safe string-to-[]byte without copy; caller must not modify the slice +} + +// KeyStringRecord returns a Record with the Key and Value fields set to the +// input key and value strings. For producing, this function is useful in +// tandem with the client-level DefaultProduceTopic option. +// +// This function uses the 'unsafe' package to avoid copying value into a slice. +// +// NOTE: It is NOT SAFE to modify the record's value. This function should only +// be used if you only ever read record fields. This function can safely be used +// for producing; the client never modifies a record's key nor value fields. +func KeyStringRecord(key, value string) *Record { + return &Record{ + Key: unsafe.Slice(unsafe.StringData(key), len(key)), //nolint:gosec // G103 safe string-to-[]byte without copy; caller must not modify the slice + Value: unsafe.Slice(unsafe.StringData(value), len(value)), //nolint:gosec // G103 safe string-to-[]byte without copy; caller must not modify the slice + } +} + +// SliceRecord returns a Record with the Value field set to the input value +// slice. For producing, this function is useful in tandem with the +// client-level DefaultProduceTopic option. +func SliceRecord(value []byte) *Record { + return &Record{Value: value} +} + +// KeySliceRecord returns a Record with the Key and Value fields set to the +// input key and value slices. For producing, this function is useful in +// tandem with the client-level DefaultProduceTopic option. +func KeySliceRecord(key, value []byte) *Record { + return &Record{Key: key, Value: value} +} + +// FetchPartition is a response for a partition in a fetched topic from a +// broker. +type FetchPartition struct { + // Partition is the partition this is for. + Partition int32 + // Err is an error for this partition in the fetch. + // + // Note that if this is a fatal error, such as data loss or non + // retryable errors, this partition will never be fetched again. + Err error + // HighWatermark is the current high watermark for this partition, that + // is, the current offset that is on all in sync replicas. + HighWatermark int64 + // LastStableOffset is the offset at which all prior offsets have been + // "decided". Non transactional records are always decided immediately, + // but transactional records are only decided once they are committed + // or aborted. + // + // The LastStableOffset will always be at or under the HighWatermark. + LastStableOffset int64 + // LogStartOffset is the low watermark of this partition, otherwise + // known as the earliest offset in the partition. + LogStartOffset int64 + // Records contains feched records for this partition. + Records []*Record +} + +// EachRecord calls fn for each record in the partition. +func (p *FetchPartition) EachRecord(fn func(*Record)) { + for _, r := range p.Records { + fn(r) + } +} + +// FetchTopic is a response for a fetched topic from a broker. +type FetchTopic struct { + // Topic is the topic this is for. + Topic string + // TopicID is the ID of the topic, if your cluster supports returning + // topic IDs in fetch responses (Kafka 3.1+). + TopicID [16]byte + // Partitions contains individual partitions in the topic that were + // fetched. + Partitions []FetchPartition +} + +// EachPartition calls fn for each partition in Fetches. +func (t *FetchTopic) EachPartition(fn func(FetchPartition)) { + for i := range t.Partitions { + fn(t.Partitions[i]) + } +} + +// EachRecord calls fn for each record in the topic, in any partition order. +func (t *FetchTopic) EachRecord(fn func(*Record)) { + for i := range t.Partitions { + for _, r := range t.Partitions[i].Records { + fn(r) + } + } +} + +// Records returns all records in all partitions in this topic. +// +// This is a convenience function that does a single slice allocation. If you +// can process records individually, it is far more efficient to use the Each +// functions. +func (t *FetchTopic) Records() []*Record { + var n int + t.EachPartition(func(p FetchPartition) { + n += len(p.Records) + }) + rs := make([]*Record, 0, n) + t.EachPartition(func(p FetchPartition) { + rs = append(rs, p.Records...) + }) + return rs +} + +// Fetch is an individual response from a broker. +type Fetch struct { + // Topics are all topics being responded to from a fetch to a broker. + Topics []FetchTopic +} + +// Fetches is a group of fetches from brokers. +type Fetches []Fetch + +// FetchError is an error in a fetch along with the topic and partition that +// the error was on. +type FetchError struct { + Topic string + Partition int32 + Err error +} + +// Errors returns all errors in a fetch with the topic and partition that +// errored. +// +// There are a few classes of errors possible, in order from most-retryable +// (or ignorable) to least retryable: +// +// 1. a normal kerr.Error; these are usually the non-retryable kerr.Errors, +// but theoretically a non-retryable error can be fixed at runtime (auth +// error? fix auth). It is worth restarting the client for these errors if +// you do not intend to fix this problem at runtime. These can also be +// returned when metadata loading of the topic or partition has a +// non-retryable error. +// +// 2. an injected *ErrDataLoss; these are informational, the client +// automatically resets consuming to where it should and resumes. This +// error is worth logging and investigating, but not worth restarting the +// client for. +// +// 3. an injected context error; this can be present if the context you were +// using for polling timed out or was canceled. +// +// 4. an injected ErrGroupSession; this is an informational error that is +// injected once a group session is lost in a way that is not the standard +// rebalance. This error can signify that your consumer member is not able +// to connect to the group (ACL problems, unreachable broker), or you +// blocked rebalancing for too long, or your callbacks took too long. +// +// 5. an injected ErrClientClosed; this is a fatal informational error that +// is returned from every Poll call if the client has been closed. +// A corresponding helper function IsClientClosed can be used to detect +// this error. +// +// 6. If using NewOffset().AtCommitted(), an untyped error is injected if a +// partition the client wants to consume has no commit. +// +// 7. an untyped batch parse failure; these are usually unrecoverable by +// restarts, and it may be best to just let the client continue. +// Restarting is an option, but you may need to manually repair your +// partition. This usually implies data corruption on the broker. +// +// 8. An untyped non-retryable error that the client does not know how to +// handle when it was trying to validate some aspect of fetching (something +// failed very unexpectedly when listing offsets to learn where to fetch, or +// when validating the epoch in offsets). The client still internally will +// retry what it was doing, but odds are not great. +// +// This list may grow over time. Generally, untyped, non-context errors are not +// retryable. Typed errors are usually retryable given time or given wider +// system fixes (perhaps live-updating auth or certs or ACLs, or restarting a +// down broker). +func (fs Fetches) Errors() []FetchError { + var errs []FetchError + fs.EachError(func(t string, p int32, err error) { + errs = append(errs, FetchError{t, p, err}) + }) + return errs +} + +// When we fetch, it is possible for Kafka to reply with topics / partitions +// that have no records and no errors. This will definitely happen outside of +// fetch sessions, but may also happen at other times (for some reason). +// When that happens we want to ignore the fetch. +func (f Fetch) hasErrorsOrRecords() bool { + for i := range f.Topics { + t := &f.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + if p.Err != nil || len(p.Records) > 0 { + return true + } + } + } + return false +} + +// IsClientClosed returns whether the fetches include an error indicating that +// the client is closed. +// +// This function is useful to break out of a poll loop; you likely want to call +// this function before calling Errors. If you may cancel the context to poll, +// you may want to use Err0 and manually check errors.Is(ErrClientClosed) or +// errors.Is(context.Canceled). +func (fs Fetches) IsClientClosed() bool { + // An injected ErrClientClosed is a single fetch with one topic and + // one partition. We can use this to make IsClientClosed do less work. + return len(fs) == 1 && len(fs[0].Topics) == 1 && len(fs[0].Topics[0].Partitions) == 1 && errors.Is(fs[0].Topics[0].Partitions[0].Err, ErrClientClosed) +} + +// Err0 returns the error at the 0th index fetch, topic, and partition. This +// can be used to quickly check if polling returned early because the client +// was closed or the context was canceled and is faster than performing a +// linear scan over all partitions with Err. When the client is closed or the +// context is canceled, fetches will contain only one partition whose Err field +// indicates the close / cancel. Note that this returns whatever the first +// error is, nil or non-nil, and does not check for a specific error value. +func (fs Fetches) Err0() error { + if len(fs) > 0 && len(fs[0].Topics) > 0 && len(fs[0].Topics[0].Partitions) > 0 { + return fs[0].Topics[0].Partitions[0].Err + } + return nil +} + +// Err returns the first error in all fetches, if any. This can be used to +// quickly check if the client is closed or your poll context was canceled, or +// to check if there's some other error that requires deeper investigation with +// EachError. This function performs a linear scan over all fetched partitions. +// It is recommended to always check all errors. If you would like to more +// quickly check ahead of time if a poll was canceled because of closing the +// client or canceling the context, you can use Err0. +func (fs Fetches) Err() error { + for _, f := range fs { + for i := range f.Topics { + ft := &f.Topics[i] + for j := range ft.Partitions { + fp := &ft.Partitions[j] + if fp.Err != nil { + return fp.Err + } + } + } + } + return nil +} + +// EachError calls fn for every partition that had a fetch error with the +// topic, partition, and error. +// +// This function has the same semantics as the Errors function; refer to the +// documentation on that function for what types of errors are possible. +func (fs Fetches) EachError(fn func(string, int32, error)) { + for _, f := range fs { + for i := range f.Topics { + ft := &f.Topics[i] + for j := range ft.Partitions { + fp := &ft.Partitions[j] + if fp.Err != nil { + fn(ft.Topic, fp.Partition, fp.Err) + } + } + } + } +} + +// RecordIter returns an iterator over all records in a fetch. +// +// Note that errors should be inspected as well. +// +// Alternatively, use [RecordsAll] for a native Go iterator over records in the fetch. +func (fs Fetches) RecordIter() *FetchesRecordIter { + iter := &FetchesRecordIter{fetches: fs} + iter.prepareNext() + return iter +} + +// FetchesRecordIter iterates over records in a fetch. +type FetchesRecordIter struct { + fetches []Fetch + ti int // index to current topic in fetches[0] + pi int // index to current partition in current topic + ri int // index to current record in current partition +} + +// Done returns whether there are any more records to iterate over. +func (i *FetchesRecordIter) Done() bool { + return len(i.fetches) == 0 +} + +// Next returns the next record from a fetch. +func (i *FetchesRecordIter) Next() *Record { + next := i.fetches[0].Topics[i.ti].Partitions[i.pi].Records[i.ri] + i.ri++ + i.prepareNext() + return next +} + +func (i *FetchesRecordIter) prepareNext() { +beforeFetch0: + if len(i.fetches) == 0 { + return + } + + fetch0 := &i.fetches[0] +beforeTopic: + if i.ti >= len(fetch0.Topics) { + i.fetches = i.fetches[1:] + i.ti = 0 + goto beforeFetch0 + } + + topic := &fetch0.Topics[i.ti] +beforePartition: + if i.pi >= len(topic.Partitions) { + i.ti++ + i.pi = 0 + goto beforeTopic + } + + partition := &topic.Partitions[i.pi] + if i.ri >= len(partition.Records) { + i.pi++ + i.ri = 0 + goto beforePartition + } +} + +// RecordsAll returns a Go native iterator that yields the records in a fetch. +// +// Similarly to [RecordIter], the errors should be inspected separately. +func (fs Fetches) RecordsAll() iter.Seq[*Record] { + return func(yield func(*Record) bool) { + for iter := fs.RecordIter(); !iter.Done(); { + if !yield(iter.Next()) { + return + } + } + } +} + +// EachPartition calls fn for each partition in Fetches. +// +// Partitions are not visited in any specific order, and a topic may be visited +// multiple times if it is spread across fetches. +func (fs Fetches) EachPartition(fn func(FetchTopicPartition)) { + for _, fetch := range fs { + for _, topic := range fetch.Topics { + for i := range topic.Partitions { + fn(FetchTopicPartition{ + Topic: topic.Topic, + FetchPartition: topic.Partitions[i], + }) + } + } + } +} + +// EachTopic calls fn for each topic in Fetches. +// +// This is a convenience function that groups all partitions for the same topic +// from many fetches into one FetchTopic. A map is internally allocated to +// group partitions per topic before calling fn. +func (fs Fetches) EachTopic(fn func(FetchTopic)) { + switch len(fs) { + case 0: + return + case 1: + for _, topic := range fs[0].Topics { + fn(topic) + } + return + } + + topics := make(map[string][]FetchPartition) + for _, fetch := range fs { + for _, topic := range fetch.Topics { + topics[topic.Topic] = append(topics[topic.Topic], topic.Partitions...) + } + } + + for topic, partitions := range topics { + fn(FetchTopic{ + topic, + [16]byte{}, + partitions, + }) + } +} + +// EachRecord calls fn for each record in Fetches. +// +// This is very similar to using a record iter, and is solely a convenience +// function depending on which style you prefer. +func (fs Fetches) EachRecord(fn func(*Record)) { + for iter := fs.RecordIter(); !iter.Done(); { + fn(iter.Next()) + } +} + +// Records returns all records in all fetches. +// +// This is a convenience function that does a single slice allocation. If you +// can process records individually, it is far more efficient to use the Each +// functions or the RecordIter. +func (fs Fetches) Records() []*Record { + rs := make([]*Record, 0, fs.NumRecords()) + fs.EachPartition(func(p FetchTopicPartition) { + rs = append(rs, p.Records...) + }) + return rs +} + +// NumRecords returns the total number of records across all fetched partitions. +func (fs Fetches) NumRecords() (n int) { + fs.EachPartition(func(p FetchTopicPartition) { + n += len(p.Records) + }) + return n +} + +// Empty checks whether the fetch result empty. This method is faster than NumRecords() == 0. +func (fs Fetches) Empty() bool { + for i := range fs { + for j := range fs[i].Topics { + for k := range fs[i].Topics[j].Partitions { + if len(fs[i].Topics[j].Partitions[k].Records) > 0 { + return false + } + } + } + } + + return true +} + +// FetchTopicPartition is similar to FetchTopic, but for an individual +// partition. +type FetchTopicPartition struct { + // Topic is the topic this is for. + Topic string + // FetchPartition is an individual partition within this topic. + FetchPartition +} + +// EachRecord calls fn for each record in the topic's partition. +func (r *FetchTopicPartition) EachRecord(fn func(*Record)) { + for _, r := range r.Records { + fn(r) + } +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/record_formatter.go b/vendor/github.com/twmb/franz-go/pkg/kgo/record_formatter.go new file mode 100644 index 00000000000..59f9d75d82d --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/record_formatter.go @@ -0,0 +1,2274 @@ +package kgo + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/twmb/franz-go/pkg/kbin" +) + +//////////// +// WRITER // +//////////// + +// RecordFormatter formats records. +type RecordFormatter struct { + calls atomic.Int64 + fns []func([]byte, *FetchPartition, *Record) []byte +} + +// AppendRecord appends a record to b given the parsed format and returns the +// updated slice. +func (f *RecordFormatter) AppendRecord(b []byte, r *Record) []byte { + for _, fn := range f.fns { + b = fn(b, nil, r) + } + return b +} + +// AppendPartitionRecord appends a record and partition to b given the parsed +// format and returns the updated slice. +func (f *RecordFormatter) AppendPartitionRecord(b []byte, p *FetchPartition, r *Record) []byte { + for _, fn := range f.fns { + b = fn(b, p, r) + } + return b +} + +// NewRecordFormatter returns a formatter for the given layout, or an error if +// the layout is invalid. +// +// The formatter is very powerful, as such there is a lot to describe. This +// documentation attempts to be as succinct as possible. +// +// Similar to the fmt package, record formatting is based off of slash escapes +// and percent "verbs" (copying fmt package lingo). Slashes are used for common +// escapes, +// +// \t \n \r \\ \xNN +// +// printing tabs, newlines, carriage returns, slashes, and hex encoded +// characters. +// +// Percent encoding opts in to printing aspects of either a record or a fetch +// partition: +// +// %t topic +// %T topic length +// %k key +// %K key length +// %v value +// %V value length +// %h begin the header specification +// %H number of headers +// %p partition +// %o offset +// %e leader epoch +// %d timestamp (date, formatting described below) +// %a record attributes (formatting required, described below) +// %x producer id +// %y producer epoch +// %D share group delivery count (0 if not from a share group) +// %A share group acquisition deadline (timestamp, 0 if not from a share group) +// +// For AppendPartitionRecord, the formatter also undersands the following three +// formatting options: +// +// %[ partition log start offset +// %| partition last stable offset +// %] partition high watermark +// +// The formatter internally tracks the number of times AppendRecord or +// AppendPartitionRecord have been called. The special option %i prints the +// iteration / call count: +// +// %i format iteration number (starts at 1) +// +// Lastly, there are three escapes to print raw characters that are usually +// used for formatting options: +// +// %% percent sign +// %{ left brace (required if a brace is after another format option) +// %} right brace +// +// # Header specification +// +// Specifying headers is essentially a primitive nested format option, +// accepting the key and value escapes above: +// +// %K header key length +// %k header key +// %V header value length +// %v header value +// +// For example, "%H %h{%k %v }" will print the number of headers, and then each +// header key and value with a space after each. +// +// # Verb modifiers +// +// Most of the previous verb specifications can be modified by adding braces +// with a given modifier, e.g., "%V{ascii}". All modifiers are described below. +// +// # Numbers +// +// All number verbs accept braces that control how the number is printed: +// +// %v{ascii} the default, print the number as ascii +// %v{number} alias for ascii +// +// %v{hex64} print 16 hex characters for the number +// %v{hex32} print 8 hex characters for the number +// %v{hex16} print 4 hex characters for the number +// %v{hex8} print 2 hex characters for the number +// %v{hex4} print 1 hex characters for the number +// %v{hex} print as many hex characters as necessary for the number +// +// %v{big64} print the number in big endian uint64 format +// %v{big32} print the number in big endian uint32 format +// %v{big16} print the number in big endian uint16 format +// %v{big8} alias for byte +// +// %v{little64} print the number in little endian uint64 format +// %v{little32} print the number in little endian uint32 format +// %v{little16} print the number in little endian uint16 format +// %v{little8} alias for byte +// +// %v{byte} print the number as a single byte +// %v{bool} print "true" if the number is non-zero, otherwise "false" +// +// All numbers are truncated as necessary per each given format. +// +// # Timestamps +// +// Timestamps can be specified in three formats: plain number formatting, +// native Go timestamp formatting, or strftime formatting. Number formatting is +// follows the rules above using the millisecond timestamp value. Go and +// strftime have further internal format options: +// +// %d{go##2006-01-02T15:04:05Z07:00##} +// %d{strftime[%F]} +// +// An arbitrary amount of pounds, braces, and brackets are understood before +// beginning the actual timestamp formatting. For Go formatting, the format is +// simply passed to the time package's AppendFormat function. For strftime, all +// "man strftime" options are supported. Time is always in UTC. +// +// # Attributes +// +// Records attributes require formatting, where each formatting option selects +// which attribute to print and how to print it. +// +// %a{compression} +// %a{compression;number} +// %a{compression;big64} +// %a{compression;hex8} +// +// By default, prints the compression as text ("none", "gzip", ...). +// Compression can be printed as a number with ";number", where number is any +// number formatting option described above. +// +// %a{timestamp-type} +// %a{timestamp-type;big64} +// +// Prints -1 for pre-0.10 records, 0 for client generated timestamps, and 1 for +// broker generated. Number formatting can be controlled with ";number". +// +// %a{transactional-bit} +// %a{transactional-bit;bool} +// +// Prints 1 if the record is a part of a transaction or 0 if it is not. Number +// formatting can be controlled with ";number". +// +// %a{control-bit} +// %a{control-bit;bool} +// +// Prints 1 if the record is a commit marker or 0 if it is not. Number +// formatting can be controlled with ";number". +// +// # Text +// +// Topics, keys, and values have "base64", "base64raw", "hex", and "unpack" +// formatting options: +// +// %t{hex} +// %k{unpack{iIqQc.$}} +// %v{base64} +// %v{base64raw} +// +// Unpack formatting is inside of enclosing pounds, braces, or brackets, the +// same way that timestamp formatting is understood. The syntax roughly follows +// Python's struct packing/unpacking rules: +// +// x pad character (does not parse input) +// < parse what follows as little endian +// > parse what follows as big endian +// +// b signed byte +// B unsigned byte +// h int16 ("half word") +// H uint16 ("half word") +// i int32 +// I uint32 +// q int64 ("quad word") +// Q uint64 ("quad word") +// +// c any character +// . alias for c +// s consume the rest of the input as a string +// $ match the end of the line (append error string if anything remains) +// +// Unlike python, a '<' or '>' can appear anywhere in the format string and +// affects everything that follows. It is possible to switch endianness +// multiple times. If the parser needs more data than available, or if the more +// input remains after '$', an error message will be appended. +func NewRecordFormatter(layout string) (*RecordFormatter, error) { + var f RecordFormatter + + var literal []byte // non-formatted raw text to output + var i int + for len(layout) > 0 { + i++ + c, size := utf8.DecodeRuneInString(layout) + rawc := layout[:size] + layout = layout[size:] + switch c { + default: + literal = append(literal, rawc...) + continue + + case '\\': + c, n, err := parseLayoutSlash(layout) + if err != nil { + return nil, err + } + layout = layout[n:] + literal = append(literal, c) + continue + + case '%': + } + + if len(layout) == 0 { + return nil, errors.New("invalid escape sequence at end of layout string") + } + + cNext, size := utf8.DecodeRuneInString(layout) + if cNext == '%' || cNext == '{' || cNext == '}' { + literal = append(literal, byte(cNext)) + layout = layout[size:] + continue + } + + var ( + isOpenBrace = len(layout) > 2 && layout[1] == '{' + handledBrace bool + escaped = layout[0] + ) + layout = layout[1:] + + // We are entering a format string. If we have any built + // literal before, this is now raw text that we will append. + if len(literal) > 0 { + l := literal + literal = nil + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, _ *Record) []byte { return append(b, l...) }) + } + + if isOpenBrace { // opening a brace: layout continues after + layout = layout[1:] + } + + switch escaped { + default: + return nil, fmt.Errorf("unknown escape sequence %%%s", string(escaped)) + + case 'T', 'K', 'V', 'H', 'p', 'o', 'e', 'i', 'x', 'y', 'D', '[', '|', ']': + // Numbers default to ascii, but we support a bunch of + // formatting options. We parse the format here, and + // then below is switching on which field to print. + var numfn func([]byte, int64) []byte + if handledBrace = isOpenBrace; handledBrace { + numfn2, n, err := parseNumWriteLayout(layout) + if err != nil { + return nil, err + } + layout = layout[n:] + numfn = numfn2 + } else { + numfn = writeNumASCII + } + switch escaped { + case 'T': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(len(r.Topic))) }) + }) + case 'K': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(len(r.Key))) }) + }) + case 'V': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(len(r.Value))) }) + }) + case 'H': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(len(r.Headers))) }) + }) + case 'p': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(r.Partition)) }) + }) + case 'o': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, r.Offset) }) + }) + case 'e': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(r.LeaderEpoch)) }) + }) + case 'i': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, _ *Record) []byte { + return numfn(b, f.calls.Add(1)) + }) + case 'x': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, r.ProducerID) }) + }) + case 'y': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(r.ProducerEpoch)) }) + }) + case 'D': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, int64(r.DeliveryCount())) }) + }) + case '[': + f.fns = append(f.fns, func(b []byte, p *FetchPartition, _ *Record) []byte { + return writeP(b, p, func(b []byte, p *FetchPartition) []byte { return numfn(b, p.LogStartOffset) }) + }) + case '|': + f.fns = append(f.fns, func(b []byte, p *FetchPartition, _ *Record) []byte { + return writeP(b, p, func(b []byte, p *FetchPartition) []byte { return numfn(b, p.LastStableOffset) }) + }) + case ']': + f.fns = append(f.fns, func(b []byte, p *FetchPartition, _ *Record) []byte { + return writeP(b, p, func(b []byte, p *FetchPartition) []byte { return numfn(b, p.HighWatermark) }) + }) + } + + case 't', 'k', 'v': + var appendFn func([]byte, []byte) []byte + if handledBrace = isOpenBrace; handledBrace { + switch { + case strings.HasPrefix(layout, "}"): + layout = layout[len("}"):] + appendFn = appendPlain + case strings.HasPrefix(layout, "base64}"): + appendFn = appendBase64 + layout = layout[len("base64}"):] + case strings.HasPrefix(layout, "base64raw}"): + appendFn = appendBase64raw + layout = layout[len("base64raw}"):] + case strings.HasPrefix(layout, "hex}"): + appendFn = appendHex + layout = layout[len("hex}"):] + case strings.HasPrefix(layout, "unpack"): + unpack, rem, err := nomOpenClose(layout[len("unpack"):]) + if err != nil { + return nil, fmt.Errorf("unpack parse err: %v", err) + } + if len(rem) == 0 || rem[0] != '}' { + return nil, fmt.Errorf("unpack missing closing } in %q", layout) + } + layout = rem[1:] + appendFn, err = parseUnpack(unpack) + if err != nil { + return nil, fmt.Errorf("unpack formatting parse err: %v", err) + } + + default: + return nil, fmt.Errorf("unknown %%%s{ escape", string(escaped)) + } + } else { + appendFn = appendPlain + } + switch escaped { + case 't': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return appendFn(b, []byte(r.Topic)) }) + }) + case 'k': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return appendFn(b, r.Key) }) + }) + case 'v': + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return appendFn(b, r.Value) }) + }) + } + + case 'a': + if !isOpenBrace { + return nil, errors.New("missing open brace sequence on %a signifying how attributes should be written") + } + handledBrace = true + + num := func(skipText string, rfn func(*Record) int64) error { + layout = layout[len(skipText):] + numfn, n, err := parseNumWriteLayout(layout) + if err != nil { + return err + } + layout = layout[n:] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, rfn(r)) }) + }) + return nil + } + bi64 := func(b bool) int64 { + if b { + return 1 + } + return 0 + } + + switch { + case strings.HasPrefix(layout, "compression}"): + layout = layout[len("compression}"):] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { + switch CompressionCodecType(r.Attrs.CompressionType()) { + case CodecNone: + return append(b, "none"...) + case CodecGzip: + return append(b, "gzip"...) + case CodecSnappy: + return append(b, "snappy"...) + case CodecLz4: + return append(b, "lz4"...) + case CodecZstd: + return append(b, "zstd"...) + default: + return append(b, "unknown"...) + } + }) + }) + case strings.HasPrefix(layout, "compression;"): + if err := num("compression;", func(r *Record) int64 { return int64(r.Attrs.CompressionType()) }); err != nil { + return nil, err + } + + case strings.HasPrefix(layout, "timestamp-type}"): + layout = layout[len("timestamp-type}"):] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { + return strconv.AppendInt(b, int64(r.Attrs.TimestampType()), 10) + }) + }) + case strings.HasPrefix(layout, "timestamp-type;"): + if err := num("timestamp-type;", func(r *Record) int64 { return int64(r.Attrs.TimestampType()) }); err != nil { + return nil, err + } + + case strings.HasPrefix(layout, "transactional-bit}"): + layout = layout[len("transactional-bit}"):] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { + if r.Attrs.IsTransactional() { + return append(b, '1') + } + return append(b, '0') + }) + }) + case strings.HasPrefix(layout, "transactional-bit;"): + if err := num("transactional-bit;", func(r *Record) int64 { return bi64(r.Attrs.IsTransactional()) }); err != nil { + return nil, err + } + + case strings.HasPrefix(layout, "control-bit}"): + layout = layout[len("control-bit}"):] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { + if r.Attrs.IsControl() { + return append(b, '1') + } + return append(b, '0') + }) + }) + case strings.HasPrefix(layout, "control-bit;"): + if err := num("control-bit;", func(r *Record) int64 { return bi64(r.Attrs.IsControl()) }); err != nil { + return nil, err + } + + default: + return nil, errors.New("unknown %a formatting") + } + + case 'h': + if !isOpenBrace { + return nil, errors.New("missing open brace sequence on %h signifying how headers are written") + } + handledBrace = true + // Headers can have their own internal braces, so we + // must look for a matching end brace. + braces := 1 + at := 0 + for braces != 0 && len(layout[at:]) > 0 { + switch layout[at] { + case '{': + if at > 0 && layout[at-1] != '%' { + braces++ + } + case '}': + if at > 0 && layout[at-1] != '%' { + braces-- + } + } + at++ + } + if braces > 0 { + return nil, fmt.Errorf("invalid header specification: missing closing brace in %q", layout) + } + + spec := layout[:at-1] + layout = layout[at:] + inf, err := NewRecordFormatter(spec) + if err != nil { + return nil, fmt.Errorf("invalid header specification %q: %v", spec, err) + } + + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + reuse := new(Record) + for _, header := range r.Headers { + reuse.Key = []byte(header.Key) + reuse.Value = header.Value + b = inf.AppendRecord(b, reuse) + } + return b + }) + + case 'd', 'A': + // For datetime parsing, we support plain millis in any + // number format, strftime, or go formatting. We + // default to plain ascii millis. + // + // %d uses the record timestamp; %A uses the share + // group acquisition deadline. For non-share records, + // AcquisitionDeadline returns the zero time; we map + // that to the Unix epoch so all sub-formats print a + // sensible value (millis=0, year=1970) rather than + // time.Time{}'s native year-1 representation. + isDeadline := escaped == 'A' + getTime := func(r *Record) time.Time { + if isDeadline { + t := r.AcquisitionDeadline() + if t.IsZero() { + return time.Unix(0, 0) + } + return t + } + return r.Timestamp + } + + handledBrace = isOpenBrace + if !handledBrace { + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return strconv.AppendInt(b, getTime(r).UnixNano()/1e6, 10) }) + }) + continue + } + + escChar := string(escaped) + switch { + case strings.HasPrefix(layout, "strftime"): + tfmt, rem, err := nomOpenClose(layout[len("strftime"):]) + if err != nil { + return nil, fmt.Errorf("strftime parse err: %v", err) + } + if len(rem) == 0 || rem[0] != '}' { + return nil, fmt.Errorf("%%%s{strftime missing closing } in %q", escChar, layout) + } + layout = rem[1:] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return strftimeAppendFormat(b, tfmt, getTime(r).UTC()) }) + }) + + case strings.HasPrefix(layout, "go"): + tfmt, rem, err := nomOpenClose(layout[len("go"):]) + if err != nil { + return nil, fmt.Errorf("go parse err: %v", err) + } + if len(rem) == 0 || rem[0] != '}' { + return nil, fmt.Errorf("%%%s{go missing closing } in %q", escChar, layout) + } + layout = rem[1:] + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return getTime(r).UTC().AppendFormat(b, tfmt) }) + }) + + default: + numfn, n, err := parseNumWriteLayout(layout) + if err != nil { + return nil, fmt.Errorf("unknown %%%s{ time specification in %q", escChar, layout) + } + layout = layout[n:] + + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, r *Record) []byte { + return writeR(b, r, func(b []byte, r *Record) []byte { return numfn(b, getTime(r).UnixNano()/1e6) }) + }) + } + } + + // If we opened a brace, we require a closing brace. + if isOpenBrace && !handledBrace { + return nil, fmt.Errorf("unhandled open brace %q", layout) + } + } + + // Ensure we print any trailing text. + if len(literal) > 0 { + f.fns = append(f.fns, func(b []byte, _ *FetchPartition, _ *Record) []byte { return append(b, literal...) }) + } + + return &f, nil +} + +func appendPlain(dst, src []byte) []byte { + return append(dst, src...) +} + +func appendBase64(dst, src []byte) []byte { + fin := append(dst, make([]byte, base64.StdEncoding.EncodedLen(len(src)))...) + base64.StdEncoding.Encode(fin[len(dst):], src) + return fin +} + +func appendBase64raw(dst, src []byte) []byte { + fin := append(dst, make([]byte, base64.RawStdEncoding.EncodedLen(len(src)))...) + base64.RawStdEncoding.Encode(fin[len(dst):], src) + return fin +} + +func appendHex(dst, src []byte) []byte { + fin := append(dst, make([]byte, hex.EncodedLen(len(src)))...) + hex.Encode(fin[len(dst):], src) + return fin +} + +// nomOpenClose extracts a middle section from a string beginning with repeated +// delimiters and returns it as with remaining (past end delimiters) string. +func nomOpenClose(src string) (middle, remaining string, err error) { + if len(src) == 0 { + return "", "", errors.New("empty layout") + } + delim := src[0] + openers := 1 + for openers < len(src) && src[openers] == delim { + openers++ + } + switch delim { + case '{': + delim = '}' + case '[': + delim = ']' + case '(': + delim = ')' + } + src = src[openers:] + end := strings.Repeat(string(delim), openers) + idx := strings.Index(src, end) + if idx < 0 { + return "", "", fmt.Errorf("missing end delim %q", end) + } + middle = src[:idx] + return middle, src[idx+len(end):], nil +} + +func parseUnpack(layout string) (func([]byte, []byte) []byte, error) { + // take dst, src; return dst + // %!q(eof) + // take 8 bytes, decode it, print decoded + var fns []func([]byte, []byte) ([]byte, int) + little := true + var sawEnd bool + for i := range layout { + if sawEnd { + return nil, errors.New("already saw end-of-input parsing character") + } + + var need int + var signed bool + cs := layout[i : i+1] + switch cs[0] { + case 'x': + continue + + case '<': + little = true + continue + case '>': + little = false + continue + + case 'b': + need = 1 + signed = true + case 'B': + need = 1 + case 'h': + need = 2 + signed = true + case 'H': + need = 2 + case 'i': + need = 4 + signed = true + case 'I': + need = 4 + case 'q': + need = 8 + signed = true + case 'Q': + need = 8 + + case 'c', '.': + fns = append(fns, func(dst, src []byte) ([]byte, int) { + if len(src) < 1 { + return append(dst, "%!c(no bytes available)"...), 0 + } + return append(dst, src[0]), 1 + }) + continue + + case 's': + sawEnd = true + fns = append(fns, func(dst, src []byte) ([]byte, int) { + return append(dst, src...), len(src) + }) + continue + + case '$': + fns = append(fns, func(dst, src []byte) ([]byte, int) { + if len(src) != 0 { + dst = append(dst, "%!$(not end-of-input)"...) + } + return dst, len(src) + }) + sawEnd = true + continue + + default: + return nil, fmt.Errorf("invalid unpack parsing character %s", cs) + } + + islittle := little + fns = append(fns, func(dst, src []byte) ([]byte, int) { + if len(src) < need { + return append(dst, fmt.Sprintf("%%!%%s(have %d bytes, need %d)", len(src), need)...), len(src) + } + + var ul, ub uint64 + var il, ib int64 + switch need { + case 1: + ul = uint64(src[0]) + ub = ul + il = int64(byte(ul)) + ib = int64(byte(ub)) + case 2: + ul = uint64(binary.LittleEndian.Uint16(src)) + ub = uint64(binary.BigEndian.Uint16(src)) + il = int64(int16(ul)) + ib = int64(int16(ub)) + case 4: + ul = uint64(binary.LittleEndian.Uint32(src)) + ub = uint64(binary.BigEndian.Uint32(src)) + il = int64(int32(ul)) + ib = int64(int32(ub)) + case 8: + ul = binary.LittleEndian.Uint64(src) + ub = binary.BigEndian.Uint64(src) + il = int64(ul) + ib = int64(ub) + } + u := ub + i := ib + if islittle { + u = ul + i = il + } + + if signed { + return strconv.AppendInt(dst, i, 10), need + } + return strconv.AppendUint(dst, u, 10), need + }) + } + + return func(dst, src []byte) []byte { + for _, fn := range fns { + var n int + dst, n = fn(dst, src) + src = src[n:] + } + return dst + }, nil +} + +func parseNumWriteLayout(layout string) (func([]byte, int64) []byte, int, error) { + braceEnd := strings.IndexByte(layout, '}') + if braceEnd == -1 { + return nil, 0, errors.New("missing brace end } to close number format specification") + } + end := braceEnd + 1 + switch layout = layout[:braceEnd]; layout { + case "ascii", "number": + return writeNumASCII, end, nil + case "hex64": + return writeNumHex64, end, nil + case "hex32": + return writeNumHex32, end, nil + case "hex16": + return writeNumHex16, end, nil + case "hex8": + return writeNumHex8, end, nil + case "hex4": + return writeNumHex4, end, nil + case "hex": + return writeNumHex, end, nil + case "big64": + return writeNumBig64, end, nil + case "big32": + return writeNumBig32, end, nil + case "big16": + return writeNumBig16, end, nil + case "byte", "big8", "little8": + return writeNumByte, end, nil + case "little64": + return writeNumLittle64, end, nil + case "little32": + return writeNumLittle32, end, nil + case "little16": + return writeNumLittle16, end, nil + case "bool": + return writeNumBool, end, nil + default: + return nil, 0, fmt.Errorf("invalid output number layout %q", layout) + } +} + +func writeR(b []byte, r *Record, fn func([]byte, *Record) []byte) []byte { + if r == nil { + return append(b, ""...) + } + return fn(b, r) +} + +func writeP(b []byte, p *FetchPartition, fn func([]byte, *FetchPartition) []byte) []byte { + if p == nil { + return append(b, ""...) + } + return fn(b, p) +} +func writeNumASCII(b []byte, n int64) []byte { return strconv.AppendInt(b, n, 10) } + +const hexc = "0123456789abcdef" + +func writeNumHex64(b []byte, n int64) []byte { + u := uint64(n) + return append(b, + hexc[(u>>60)&0xf], + hexc[(u>>56)&0xf], + hexc[(u>>52)&0xf], + hexc[(u>>48)&0xf], + hexc[(u>>44)&0xf], + hexc[(u>>40)&0xf], + hexc[(u>>36)&0xf], + hexc[(u>>32)&0xf], + hexc[(u>>28)&0xf], + hexc[(u>>24)&0xf], + hexc[(u>>20)&0xf], + hexc[(u>>16)&0xf], + hexc[(u>>12)&0xf], + hexc[(u>>8)&0xf], + hexc[(u>>4)&0xf], + hexc[u&0xf], + ) +} + +func writeNumHex32(b []byte, n int64) []byte { + u := uint64(n) + return append(b, + hexc[(u>>28)&0xf], + hexc[(u>>24)&0xf], + hexc[(u>>20)&0xf], + hexc[(u>>16)&0xf], + hexc[(u>>12)&0xf], + hexc[(u>>8)&0xf], + hexc[(u>>4)&0xf], + hexc[u&0xf], + ) +} + +func writeNumHex16(b []byte, n int64) []byte { + u := uint64(n) + return append(b, + hexc[(u>>12)&0xf], + hexc[(u>>8)&0xf], + hexc[(u>>4)&0xf], + hexc[u&0xf], + ) +} + +func writeNumHex8(b []byte, n int64) []byte { + u := uint64(n) + return append(b, + hexc[(u>>4)&0xf], + hexc[u&0xf], + ) +} + +func writeNumHex4(b []byte, n int64) []byte { + u := uint64(n) + return append(b, + hexc[u&0xf], + ) +} + +func writeNumHex(b []byte, n int64) []byte { + return strconv.AppendUint(b, uint64(n), 16) +} + +func writeNumBig64(b []byte, n int64) []byte { + u := uint64(n) + return append(b, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +func writeNumLittle64(b []byte, n int64) []byte { + u := uint64(n) + return append(b, byte(u), byte(u>>8), byte(u>>16), byte(u>>24), byte(u>>32), byte(u>>40), byte(u>>48), byte(u>>56)) +} + +func writeNumBig32(b []byte, n int64) []byte { + u := uint64(n) + return append(b, byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +func writeNumLittle32(b []byte, n int64) []byte { + u := uint64(n) + return append(b, byte(u), byte(u>>8), byte(u>>16), byte(u>>24)) +} +func writeNumBig16(b []byte, n int64) []byte { u := uint64(n); return append(b, byte(u>>8), byte(u)) } +func writeNumLittle16(b []byte, n int64) []byte { + u := uint64(n) + return append(b, byte(u), byte(u>>8)) +} +func writeNumByte(b []byte, n int64) []byte { u := uint64(n); return append(b, byte(u)) } + +func writeNumBool(b []byte, n int64) []byte { + if n == 0 { + return append(b, "false"...) + } + return append(b, "true"...) +} + +//////////// +// READER // +//////////// + +// RecordReader reads records from an io.Reader. +type RecordReader struct { + r *bufio.Reader + + buf []byte + fns []readParse + + done bool +} + +// NewRecordReader returns a record reader for the given layout, or an error if +// the layout is invalid. +// +// Similar to the RecordFormatter, the RecordReader parsing is quite powerful. +// There is a bit less to describe in comparison to RecordFormatter, but still, +// this documentation attempts to be as succinct as possible. +// +// Similar to the fmt package, record parsing is based off of slash escapes and +// percent "verbs" (copying fmt package lingo). Slashes are used for common +// escapes, +// +// \t \n \r \\ \xNN +// +// reading tabs, newlines, carriage returns, slashes, and hex encoded +// characters. +// +// Percent encoding reads into specific values of a Record: +// +// %t topic +// %T topic length +// %k key +// %K key length +// %v value +// %V value length +// %h begin the header specification +// %H number of headers +// %p partition +// %o offset +// %e leader epoch +// %d timestamp +// %x producer id +// %y producer epoch +// +// If using length / number verbs (i.e., "sized" verbs), they must occur before +// what they are sizing. +// +// There are three escapes to parse raw characters, rather than opting into +// some formatting option: +// +// %% percent sign +// %{ left brace +// %} right brace +// +// Unlike record formatting, timestamps can only be read as numbers because Go +// or strftime formatting can both be variable length and do not play too well +// with delimiters. Timestamps numbers are read as milliseconds. +// +// # Numbers +// +// All size numbers can be parsed in the following ways: +// +// %v{ascii} parse numeric digits until a non-numeric +// %v{number} alias for ascii +// +// %v{hex64} read 16 hex characters for the number +// %v{hex32} read 8 hex characters for the number +// %v{hex16} read 4 hex characters for the number +// %v{hex8} read 2 hex characters for the number +// %v{hex4} read 1 hex characters for the number +// +// %v{big64} read the number as big endian uint64 format +// %v{big32} read the number as big endian uint32 format +// %v{big16} read the number as big endian uint16 format +// %v{big8} alias for byte +// +// %v{little64} read the number as little endian uint64 format +// %v{little32} read the number as little endian uint32 format +// %v{little16} read the number as little endian uint16 format +// %v{little8} read the number as a byte +// +// %v{byte} read the number as a byte +// %v{bool} read "true" as 1, "false" as 0 +// %v{3} read 3 characters (any number) +// +// # Header specification +// +// Similar to number formatting, headers are parsed using a nested primitive +// format option, accepting the key and value escapes previously mentioned. +// +// # Text +// +// Topics, keys, and values can be decoded using "base64", "hex", and "json" +// formatting options. Any size specification is the size of the encoded value +// actually being read (i.e., size as seen, not size when decoded). JSON values +// are compacted after being read. +// +// %T%t{hex} - 4abcd reads four hex characters "abcd" +// %V%v{base64} - 2z9 reads two base64 characters "z9" +// %v{json} %k - {"foo" : "bar"} foo reads a JSON object and then "foo" +// +// As well, these text options can be parsed with regular expressions: +// +// %k{re[\d*]}%v{re[\s+]} +func NewRecordReader(reader io.Reader, layout string) (*RecordReader, error) { + r := &RecordReader{r: bufio.NewReader(reader)} + if err := r.parseReadLayout(layout); err != nil { + return nil, err + } + return r, nil +} + +// ReadRecord reads the next record in the reader and returns it, or returns a +// parsing error. +// +// This will return io.EOF only if the underlying reader returns io.EOF at the +// start of a new record. If an io.EOF is returned mid record, this returns +// io.ErrUnexpectedEOF. It is expected for this function to be called until it +// returns io.EOF. +func (r *RecordReader) ReadRecord() (*Record, error) { + rec := new(Record) + return rec, r.ReadRecordInto(rec) +} + +// ReadRecordInto reads the next record into the given record and returns any +// parsing error +// +// This will return io.EOF only if the underlying reader returns io.EOF at the +// start of a new record. If an io.EOF is returned mid record, this returns +// io.ErrUnexpectedEOF. It is expected for this function to be called until it +// returns io.EOF. +func (r *RecordReader) ReadRecordInto(rec *Record) error { + if r.done { + return io.EOF + } + return r.next(rec) +} + +// SetReader replaces the underlying reader with the given reader. +func (r *RecordReader) SetReader(reader io.Reader) { + r.r = bufio.NewReader(reader) + r.done = false +} + +const ( + parsesTopic parseRecordBits = 1 << iota + parsesTopicSize + parsesKey + parsesKeySize + parsesValue + parsesValueSize + parsesHeaders + parsesHeadersNum +) + +// The record reading format must be either entirely sized or entirely unsized. +// This type helps us track what's what. +type parseRecordBits uint8 + +func (p *parseRecordBits) set(r parseRecordBits) { *p |= r } +func (p parseRecordBits) has(r parseRecordBits) bool { return p&r != 0 } + +func (r *RecordReader) parseReadLayout(layout string) error { + if len(layout) == 0 { + return errors.New("RecordReader: invalid empty format") + } + + var ( + // If we are reading by size, we parse the layout size into one + // of these variables. When reading, we use the captured + // variable's value. + topicSize = new(uint64) + keySize = new(uint64) + valueSize = new(uint64) + headersNum = new(uint64) + + bits parseRecordBits + + literal []byte // raw literal we are currently working on + addLiteral = func() { + if len(r.fns) > 0 && r.fns[len(r.fns)-1].read.empty() { + r.fns[len(r.fns)-1].read.delim = literal + } else if len(literal) > 0 { + r.fns = append(r.fns, readParse{ + read: readKind{exact: literal}, + }) + } + literal = nil + } + ) + + for len(layout) > 0 { + c, size := utf8.DecodeRuneInString(layout) + rawc := layout[:size] + layout = layout[size:] + switch c { + default: + literal = append(literal, rawc...) + continue + + case '\\': + c, n, err := parseLayoutSlash(layout) + if err != nil { + return err + } + layout = layout[n:] + literal = append(literal, c) + continue + + case '%': + } + + if len(layout) == 0 { + literal = append(literal, rawc...) + continue + } + + cNext, size := utf8.DecodeRuneInString(layout) + if cNext == '%' || cNext == '{' || cNext == '}' { + literal = append(literal, byte(cNext)) + layout = layout[size:] + continue + } + + var ( + isOpenBrace = len(layout) > 2 && layout[1] == '{' + handledBrace bool + escaped = layout[0] + ) + layout = layout[1:] + addLiteral() + + if isOpenBrace { // opening a brace: layout continues after + layout = layout[1:] + } + + switch escaped { + default: + return fmt.Errorf("unknown percent escape sequence %q", layout[:1]) + + case 'T', 'K', 'V', 'H': + var dst *uint64 + var bit parseRecordBits + switch escaped { + case 'T': + dst, bit = topicSize, parsesTopicSize + case 'K': + dst, bit = keySize, parsesKeySize + case 'V': + dst, bit = valueSize, parsesValueSize + case 'H': + dst, bit = headersNum, parsesHeadersNum + } + if bits.has(bit) { + return fmt.Errorf("%%%s is doubly specified", string(escaped)) + } + if bits.has(bit >> 1) { + return fmt.Errorf("size specification %%%s cannot come after value specification %%%s", string(escaped), strings.ToLower(string(escaped))) + } + bits.set(bit) + fn, n, err := r.parseReadSize("ascii", dst, false) + if handledBrace = isOpenBrace; handledBrace { + fn, n, err = r.parseReadSize(layout, dst, true) + } + if err != nil { + return fmt.Errorf("unable to parse %%%s: %s", string(escaped), err) + } + layout = layout[n:] + r.fns = append(r.fns, fn) + + case 'p', 'o', 'e', 'd', 'x', 'y': + dst := new(uint64) + fn, n, err := r.parseReadSize("ascii", dst, false) + if handledBrace = isOpenBrace; handledBrace { + fn, n, err = r.parseReadSize(layout, dst, true) + } + if err != nil { + return fmt.Errorf("unable to parse %%%s: %s", string(escaped), err) + } + layout = layout[n:] + numParse := fn.parse + switch escaped { + case 'p': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.Partition = int32(*dst) + return nil + } + case 'o': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.Offset = int64(*dst) + return nil + } + case 'e': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.LeaderEpoch = int32(*dst) + return nil + } + case 'd': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.Timestamp = time.Unix(0, int64(*dst)*1e6) + return nil + } + case 'x': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.ProducerID = int64(*dst) + return nil + } + case 'y': + fn.parse = func(b []byte, rec *Record) error { + if err := numParse(b, nil); err != nil { + return err + } + rec.ProducerEpoch = int16(*dst) + return nil + } + } + r.fns = append(r.fns, fn) + + case 't', 'k', 'v': + var decodeFn func([]byte) ([]byte, error) + var re *regexp.Regexp + var isJson bool + if handledBrace = isOpenBrace; handledBrace { + switch { + case strings.HasPrefix(layout, "}"): + layout = layout[len("}"):] + case strings.HasPrefix(layout, "base64}"): + decodeFn = decodeBase64 + layout = layout[len("base64}"):] + case strings.HasPrefix(layout, "hex}"): + decodeFn = decodeHex + layout = layout[len("hex}"):] + case strings.HasPrefix(layout, "json}"): + isJson = true + decodeFn = func(b []byte) ([]byte, error) { + var buf bytes.Buffer + err := json.Compact(&buf, b) + return buf.Bytes(), err + } + layout = layout[len("json}"):] + case strings.HasPrefix(layout, "re"): + restr, rem, err := nomOpenClose(layout[len("re"):]) + if err != nil { + return fmt.Errorf("re parse err: %v", err) + } + if len(rem) == 0 || rem[0] != '}' { + return fmt.Errorf("re missing closing } in %q", layout) + } + layout = rem[1:] + if !strings.HasPrefix(restr, "^") { + restr = "^" + restr + } + re, err = regexp.Compile(restr) + if err != nil { + return fmt.Errorf("re parse err: %v", err) + } + + default: + return fmt.Errorf("unknown %%%s{ escape", string(escaped)) + } + } + + var bit, bitSize parseRecordBits + var inner func([]byte, *Record) + var size *uint64 + switch escaped { + case 't': + bit, bitSize, size = parsesTopic, parsesTopicSize, topicSize + inner = func(b []byte, r *Record) { r.Topic = string(b) } + case 'k': + bit, bitSize, size = parsesKey, parsesKeySize, keySize + inner = func(b []byte, r *Record) { r.Key = dupslice(b) } + case 'v': + bit, bitSize, size = parsesValue, parsesValueSize, valueSize + inner = func(b []byte, r *Record) { r.Value = dupslice(b) } + } + + fn := readParse{parse: func(b []byte, r *Record) error { + if decodeFn != nil { + dec, err := decodeFn(b) + if err != nil { + return err + } + b = dec + } + inner(b, r) + return nil + }} + bit.set(bit) + if bits.has(bitSize) { + if re != nil { + return errors.New("cannot specify exact size and regular expression") + } + if isJson { + return errors.New("cannot specify exact size and json") + } + fn.read = readKind{sizefn: func() int { return int(*size) }} + } else if re != nil { + fn.read = readKind{re: re} + } else if isJson { + fn.read = readKind{condition: new(jsonReader).read} + } + r.fns = append(r.fns, fn) + + case 'h': + bits.set(parsesHeaders) + if !bits.has(parsesHeadersNum) { + return errors.New("missing header count specification %H before header specification %h") + } + if !isOpenBrace { + return errors.New("missing open brace sequence on %h signifying how headers are encoded") + } + handledBrace = true + // Similar to above, headers can have their own + // internal braces, so we look for a matching end. + braces := 1 + at := 0 + for braces != 0 && len(layout[at:]) > 0 { + switch layout[at] { + case '{': + if at > 0 && layout[at-1] != '%' { + braces++ + } + case '}': + if at > 0 && layout[at-1] != '%' { + braces-- + } + } + at++ + } + if braces > 0 { + return fmt.Errorf("invalid header specification: missing closing brace in %q", layout) + } + + // We parse the header specification recursively, but + // we require that it is sized and contains only keys + // and values. Checking the delimiter checks sizing. + var inr RecordReader + if err := inr.parseReadLayout(layout[:at-1]); err != nil { + return fmt.Errorf("invalid header specification: %v", err) + } + layout = layout[at:] + + // To parse headers, we save the inner reader's parsing + // function stash the current record's key/value before + // parsing, and then capture the key/value as a header. + r.fns = append(r.fns, readParse{read: readKind{handoff: func(r *RecordReader, rec *Record) error { + k, v := rec.Key, rec.Value + defer func() { rec.Key, rec.Value = k, v }() + inr.r = r.r + for i := uint64(0); i < *headersNum; i++ { + rec.Key, rec.Value = nil, nil + if err := inr.next(rec); err != nil { + return err + } + rec.Headers = append(rec.Headers, RecordHeader{Key: string(rec.Key), Value: rec.Value}) + } + return nil + }}}) + } + + if isOpenBrace && !handledBrace { + return fmt.Errorf("unhandled open brace %q", layout) + } + } + + addLiteral() + + // We must sort noreads to the front, we use this guarantee when + // reading to handle EOF properly. + var noreads, reads []readParse + for _, fn := range r.fns { + if fn.read.noread { + noreads = append(noreads, fn) + } else { + reads = append(reads, fn) + } + } + r.fns = make([]readParse, 0, len(noreads)+len(reads)) + r.fns = append(r.fns, noreads...) + r.fns = append(r.fns, reads...) + + return nil +} + +// Returns a function that parses a number from the internal reader into dst. +// +// If needBrace is true, the user is specifying how to read the number, +// otherwise we default to ascii. Reading ascii requires us to peek at bytes +// until we get to a non-number byte. +func (*RecordReader) parseReadSize(layout string, dst *uint64, needBrace bool) (readParse, int, error) { + var end int + if needBrace { + braceEnd := strings.IndexByte(layout, '}') + if braceEnd == -1 { + return readParse{}, 0, errors.New("missing brace end } to close number size specification") + } + layout = layout[:braceEnd] + end = braceEnd + 1 + } + + switch layout { + default: + num, err := strconv.Atoi(layout) + if err != nil { + return readParse{}, 0, fmt.Errorf("unrecognized number reading layout %q: %v", layout, err) + } + if num <= 0 { + return readParse{}, 0, fmt.Errorf("invalid zero or negative number %q when parsing read size", layout) + } + return readParse{ + readKind{noread: true}, + func([]byte, *Record) error { *dst = uint64(num); return nil }, + }, end, nil + + case "ascii", "number": + return readParse{ + readKind{condition: func(b byte) int8 { + if b < '0' || b > '9' { + return -1 + } + return 2 // ignore EOF if we hit it after this + }}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 10, 64) + return err + }, + }, end, nil + + case "big64": + return readParse{ + readKind{size: 8}, + func(b []byte, _ *Record) error { *dst = binary.BigEndian.Uint64(b); return nil }, + }, end, nil + case "big32": + return readParse{ + readKind{size: 4}, + func(b []byte, _ *Record) error { *dst = uint64(binary.BigEndian.Uint32(b)); return nil }, + }, end, nil + case "big16": + return readParse{ + readKind{size: 2}, + func(b []byte, _ *Record) error { *dst = uint64(binary.BigEndian.Uint16(b)); return nil }, + }, end, nil + + case "little64": + return readParse{ + readKind{size: 8}, + func(b []byte, _ *Record) error { *dst = binary.LittleEndian.Uint64(b); return nil }, + }, end, nil + case "little32": + return readParse{ + readKind{size: 4}, + func(b []byte, _ *Record) error { *dst = uint64(binary.LittleEndian.Uint32(b)); return nil }, + }, end, nil + case "little16": + return readParse{ + readKind{size: 2}, + func(b []byte, _ *Record) error { *dst = uint64(binary.LittleEndian.Uint16(b)); return nil }, + }, end, nil + + case "byte", "big8", "little8": + return readParse{ + readKind{size: 1}, + func(b []byte, _ *Record) error { *dst = uint64(b[0]); return nil }, + }, end, nil + + case "hex64": + return readParse{ + readKind{size: 16}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 16, 64) + return err + }, + }, end, nil + case "hex32": + return readParse{ + readKind{size: 8}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 16, 64) + return err + }, + }, end, nil + case "hex16": + return readParse{ + readKind{size: 4}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 16, 64) + return err + }, + }, end, nil + case "hex8": + return readParse{ + readKind{size: 2}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 16, 64) + return err + }, + }, end, nil + case "hex4": + return readParse{ + readKind{size: 1}, + func(b []byte, _ *Record) (err error) { + *dst, err = strconv.ParseUint(kbin.UnsafeString(b), 16, 64) + return err + }, + }, end, nil + + case "bool": + const ( + stateUnknown uint8 = iota + stateTrue + stateFalse + ) + var state uint8 + var last byte + return readParse{ + readKind{condition: func(b byte) (done int8) { + defer func() { + if done <= 0 { + state = stateUnknown + last = 0 + } + }() + + switch state { + default: // stateUnknown + switch b { + case 't': + state = stateTrue + last = b + return 1 + case 'f': + state = stateFalse + last = b + return 1 + } + return -1 + + case stateTrue: + if last == 't' && b == 'r' || last == 'r' && b == 'u' { + last = b + return 1 + } else if last == 'u' && b == 'e' { + return 0 + } + return -1 + + case stateFalse: + if last == 'f' && b == 'a' || last == 'a' && b == 'l' || last == 'l' && b == 's' { + last = b + return 1 + } else if last == 's' && b == 'e' { + return 0 + } + return -1 + } + }}, + func(b []byte, _ *Record) error { + switch string(b) { + case "true": + *dst = 1 + case "false": + *dst = 0 + default: + return fmt.Errorf("invalid bool %s", b) + } + return nil + }, + }, end, nil + } +} + +func decodeBase64(b []byte) ([]byte, error) { + n, err := base64.StdEncoding.Decode(b[:base64.StdEncoding.DecodedLen(len(b))], b) + return b[:n], err +} + +func decodeHex(b []byte) ([]byte, error) { + n, err := hex.Decode(b[:hex.DecodedLen(len(b))], b) + return b[:n], err +} + +type readKind struct { + noread bool + exact []byte + condition func(byte) int8 // -2: error, -1: stop, do not consume input; 0: stop, consume input; 1: keep going, consume input, 2: keep going, consume input, can EOF + size int + sizefn func() int + handoff func(*RecordReader, *Record) error + delim []byte + re *regexp.Regexp +} + +func (r *readKind) empty() bool { + return !r.noread && + r.exact == nil && + r.condition == nil && + r.size == 0 && + r.sizefn == nil && + r.handoff == nil && + r.delim == nil && + r.re == nil +} + +type readParse struct { + read readKind + parse func([]byte, *Record) error +} + +func dupslice(b []byte) []byte { + if len(b) == 0 { + return nil + } + dup := make([]byte, len(b)) + copy(dup, b) + return dup +} + +func (r *RecordReader) next(rec *Record) error { + for i, fn := range r.fns { + r.buf = r.buf[:0] + + var err error + switch { + case fn.read.noread: + // do nothing + case fn.read.exact != nil: + err = r.readExact(fn.read.exact) + case fn.read.condition != nil: + err = r.readCondition(fn.read.condition) + case fn.read.size > 0: + err = r.readSize(fn.read.size) + case fn.read.sizefn != nil: + err = r.readSize(fn.read.sizefn()) + case fn.read.handoff != nil: + err = fn.read.handoff(r, rec) + case fn.read.re != nil: + err = r.readRe(fn.read.re) + default: + err = r.readDelim(fn.read.delim) // we *always* fall back to delim parsing + } + + switch err { + default: + return err + case nil: + case io.EOF, io.ErrUnexpectedEOF: + r.done = true + // We guarantee that all noread parses are at + // the front, so if we io.EOF on the first + // non-noread, then we bubble it up. + if len(r.buf) == 0 && (i == 0 || r.fns[i-1].read.noread) { + return io.EOF + } + if i != len(r.fns)-1 || err == io.ErrUnexpectedEOF { + return io.ErrUnexpectedEOF + } + } + + if fn.parse == nil { + continue + } + + if err := fn.parse(r.buf, rec); err != nil { + return err + } + } + return nil +} + +func (r *RecordReader) readCondition(fn func(byte) int8) error { + var ignoreEOF bool + for { + peek, err := r.r.Peek(1) + if err != nil { + if err == io.EOF && ignoreEOF { + err = nil + } + return err + } + ignoreEOF = false + c := peek[0] + switch fn(c) { + case -2: + return fmt.Errorf("invalid input %q", c) + case -1: + return nil + case 0: + r.r.Discard(1) + r.buf = append(r.buf, c) + return nil + case 1: + case 2: + ignoreEOF = true + } + r.r.Discard(1) + r.buf = append(r.buf, c) + } +} + +type reReader struct { + r *RecordReader + peek []byte + err error +} + +func (re *reReader) ReadRune() (r rune, size int, err error) { + re.peek, re.err = re.r.r.Peek(len(re.peek) + 1) + if re.err != nil { + return 0, 0, re.err + } + return rune(re.peek[len(re.peek)-1]), 1, nil +} + +func (r *RecordReader) readRe(re *regexp.Regexp) error { + reader := reReader{r: r} + loc := re.FindReaderIndex(&reader) + if loc == nil { + if reader.err == io.EOF && len(reader.peek) > 0 { + return fmt.Errorf("regexp text mismatch, saw %q", reader.peek) + } + return reader.err + } + n := loc[1] // we ensure the regexp begins with ^, so we only need the end + r.buf = append(r.buf, reader.peek[:n]...) + r.r.Discard(n) + if n == len(reader.peek) { + return reader.err + } + return nil +} + +func (r *RecordReader) readSize(n int) error { + r.buf = append(r.buf, make([]byte, n)...) + n, err := io.ReadFull(r.r, r.buf) + r.buf = r.buf[:n] + return err +} + +func (r *RecordReader) readExact(d []byte) error { + if err := r.readSize(len(d)); err != nil { + return err + } + if !bytes.Equal(d, r.buf) { + return fmt.Errorf("exact text mismatch, read %q when expecting %q", r.buf, d) + } + return nil +} + +func (r *RecordReader) readDelim(d []byte) error { + // Empty delimiters opt in to reading the rest of the text. + if len(d) == 0 { + b, err := io.ReadAll(r.r) + r.buf = b + // ReadAll stops at io.EOF, but we need to bubble that up. + if err == nil { + return io.EOF + } + return err + } + + // We use the simple inefficient search algorithm, which can be O(nm), + // but we aren't expecting huge search spaces. Long term we could + // convert to a two-way search. + for { + peek, err := r.r.Peek(len(d)) + if err != nil { + // If we peek an io.EOF, we were looking for our delim + // and hit the end. This is unexpected. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + if !bytes.Equal(peek, d) { + // We did not find our delim. Skip the first char + // then continue again. + r.buf = append(r.buf, peek[0]) + r.r.Discard(1) + continue + } + // We found our delim. We discard it and return. + r.r.Discard(len(d)) + return nil + } +} + +type jsonReader struct { + state int8 + n int8 // misc. + nexts []int8 +} + +func (*jsonReader) isHex(c byte) bool { + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', + 'A', 'B', 'C', 'D', 'E', 'F': + return true + default: + return false + } +} + +func (*jsonReader) isNum(c byte) bool { + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +func (*jsonReader) isNat(c byte) bool { + switch c { + case '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +func (*jsonReader) isE(c byte) bool { + return c == 'e' || c == 'E' +} + +const ( + jrstAny int8 = iota + jrstObj + jrstObjSep + jrstObjFin + jrstArr + jrstArrFin + jrstStrBegin + jrstStr + jrstStrEsc + jrstStrEscU + jrstTrue + jrstFalse + jrstNull + jrstNeg + jrstOne + jrstDotOrE + jrstDot + jrstE +) + +func (r *jsonReader) read(c byte) (rr int8) { +start: + switch r.state { + case jrstAny: + switch c { + case ' ', '\t', '\n', '\r': + return 1 // skip whitespace, need more + case '{': + r.state = jrstObj + return 1 // object open, need more + case '[': + r.state = jrstArr + return 1 // array open, need more + case '"': + r.state = jrstStr + return 1 // string open, need more + case 't': + r.state = jrstTrue + r.n = 0 + return 1 // beginning of true, need more + case 'f': + r.state = jrstFalse + r.n = 0 + return 1 // beginning of false, need more + case 'n': + r.state = jrstNull + r.n = 0 + return 1 // beginning of null, need more + case '-': + r.state = jrstNeg + return 1 // beginning of negative number, need more + case '0': + r.state = jrstDotOrE + return 1 // beginning of 0e or 0., need more + case '1', '2', '3', '4', '5', '6', '7', '8', '9': + r.state = jrstOne + return 1 // beginning of number, need more + default: + return -2 // invalid json + } + + case jrstObj: + switch c { + case ' ', '\t', '\n', '\r': + return 1 // skip whitespace in json object, need more + case '"': + r.pushState(jrstStr, jrstObjSep) + return 1 // beginning of object key, need to finish, transition to obj sep + case '}': + return r.popState() // end of object, this is valid json end, pop state + default: + return -2 // invalid json: expected object key + } + case jrstObjSep: + switch c { + case ' ', '\t', '\n', '\r': + return 1 // skip whitespace in json object, need more + case ':': + r.pushState(jrstAny, jrstObjFin) + return 1 // beginning of object value, need to finish, transition to obj fin + default: + return -2 // invalid json: expected object separator + } + case jrstObjFin: + switch c { + case ' ', '\r', '\t', '\n': + return 1 // skip whitespace in json object, need more + case ',': + r.pushState(jrstStrBegin, jrstObjSep) + return 1 // beginning of new object key, need to finish, transition to obj sep + case '}': + return r.popState() // end of object, this is valid json end, pop state + default: + return -2 // invalid json + } + + case jrstArr: + switch c { + case ' ', '\r', '\t', '\n': + return 1 // skip whitespace in json array, need more + case ']': + return r.popState() // end of array, this is valid json end, pop state + default: + r.pushState(jrstAny, jrstArrFin) + goto start // array value began: immediately transition to it + } + case jrstArrFin: + switch c { + case ' ', '\r', '\t', '\n': + return 1 // skip whitespace in json array, need more + case ',': + r.state = jrstArr + return 1 // beginning of new array value, need more + case ']': + return r.popState() // end of array, this is valid json end, pop state + default: + return -2 // invalid json + } + + case jrstStrBegin: + switch c { + case ' ', '\r', '\t', '\n': + return 1 // skip whitespace in json object (before beginning of key), need more + case '"': + r.state = jrstStr + return 1 // beginning of object key, need more + default: + return -2 // invalid json + } + + case jrstStr: + switch c { + case 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31: + return -2 // invalid json: control characters not allowed in string + case '"': + return r.popState() // end of string, this is valid json end, pop state + case '\\': + r.state = jrstStrEsc + return 1 // beginning of escape sequence, need more + default: + return 1 // continue string, need more + } + case jrstStrEsc: + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': + r.state = jrstStr + return 1 // end of escape sequence, still need to finish string + case 'u': + r.state = jrstStrEscU + r.n = 0 + return 1 // beginning of unicode escape sequence, need more + default: + return -2 // invalid json: invalid escape sequence + } + case jrstStrEscU: + if !r.isHex(c) { + return -2 // invalid json: invalid unicode escape sequence + } + r.n++ + if r.n == 4 { + r.state = jrstStr + } + return 1 // end of unicode escape sequence, still need to finish string + + case jrstTrue: + switch { + case r.n == 0 && c == 'r': + r.n++ + return 1 + case r.n == 1 && c == 'u': + r.n++ + return 1 + case r.n == 2 && c == 'e': + return r.popState() // end of true, this is valid json end, pop state + } + case jrstFalse: + switch { + case r.n == 0 && c == 'a': + r.n++ + return 1 + case r.n == 1 && c == 'l': + r.n++ + return 1 + case r.n == 2 && c == 's': + r.n++ + return 1 + case r.n == 3 && c == 'e': + return r.popState() // end of false, this is valid json end, pop state + } + case jrstNull: + switch { + case r.n == 0 && c == 'u': + r.n++ + return 1 + case r.n == 1 && c == 'l': + r.n++ + return 1 + case r.n == 2 && c == 'l': + return r.popState() // end of null, this is valid json end, pop state + } + + case jrstNeg: + if c == '0' { + r.state = jrstDotOrE + return r.oneOrTwo() // beginning of -0, need to see if there is more (potentially end) + } else if r.isNat(c) { + r.state = jrstOne + return r.oneOrTwo() // beginning of -1 (or 2,3,..9), need to see if there is more (potentially end) + } + return -2 // invalid, -a or something + case jrstOne: + if r.isNum(c) { + return r.oneOrTwo() // continue the number (potentially end) + } + fallthrough // not a number, check if e or . + case jrstDotOrE: + if r.isE(c) { + r.state = jrstE + return 1 // beginning of exponent, need more + } + if c == '.' { + r.state = jrstDot + r.n = 0 + return 1 // beginning of dot, need more + } + if r.popStateToStart() { + goto start + } + return -1 // done with number, no more state to bubble to: we are done + + case jrstDot: + switch r.n { + case 0: + if !r.isNum(c) { + return -2 // first char after dot must be a number + } + r.n = 1 + return r.oneOrTwo() // saw number, keep and continue (potentially end) + case 1: + if r.isNum(c) { + return r.oneOrTwo() // more number, keep and continue (potentially end) + } + if r.isE(c) { + r.state = jrstE + r.n = 0 + return 1 // beginning of exponent (-0.1e), need more + } + if r.popStateToStart() { + goto start + } + return -1 // done with number, no more state to bubble to: we are done + } + case jrstE: + switch r.n { + case 0: + if c == '+' || c == '-' { + r.n = 1 + return 1 // beginning of exponent sign, need more + } + fallthrough + case 1: + if !r.isNum(c) { + return -2 // first char after exponent must be sign or number + } + r.n = 2 + return r.oneOrTwo() // saw number, keep and continue (potentially end) + case 2: + if r.isNum(c) { + return r.oneOrTwo() // more number, keep and continue (potentially end) + } + if r.popStateToStart() { + goto start + } + return -1 // done with number, no more state to bubble to: we are done + } + } + return -2 // unknown state +} + +func (r *jsonReader) pushState(next, next2 int8) { + r.nexts = append(r.nexts, next2) + r.state = next +} + +func (r *jsonReader) popState() int8 { + if len(r.nexts) == 0 { + r.state = jrstAny + return 0 + } + r.state = r.nexts[len(r.nexts)-1] + r.nexts = r.nexts[:len(r.nexts)-1] + return 1 +} + +func (r *jsonReader) popStateToStart() bool { + if len(r.nexts) == 0 { + r.state = jrstAny + return false + } + r.state = r.nexts[len(r.nexts)-1] + r.nexts = r.nexts[:len(r.nexts)-1] + return true +} + +func (r *jsonReader) oneOrTwo() int8 { + if len(r.nexts) > 0 { + return 1 + } + return 2 +} + +//////////// +// COMMON // +//////////// + +func parseLayoutSlash(layout string) (byte, int, error) { + if len(layout) == 0 { + return 0, 0, errors.New("invalid slash escape at end of delim string") + } + switch layout[0] { + case 't': + return '\t', 1, nil + case 'n': + return '\n', 1, nil + case 'r': + return '\r', 1, nil + case '\\': + return '\\', 1, nil + case 'x': + if len(layout) < 3 { // on x, need two more + return 0, 0, errors.New("invalid non-terminated hex escape sequence at end of delim string") + } + hex := layout[1:3] + n, err := strconv.ParseInt(hex, 16, 8) + if err != nil { + return 0, 0, fmt.Errorf("unable to parse hex escape sequence %q: %v", hex, err) + } + return byte(n), 3, nil + default: + return 0, 0, fmt.Errorf("unknown slash escape sequence %q", layout[:1]) + } +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/ring.go b/vendor/github.com/twmb/franz-go/pkg/kgo/ring.go new file mode 100644 index 00000000000..9aad112547b --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/ring.go @@ -0,0 +1,140 @@ +package kgo + +import ( + "sync" + + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" +) + +// The ring type is a dynamically-sized circular buffer with optional blocking +// when full. The buffer starts at capacity 8 and grows as needed. When the +// buffer empties, it shrinks back to capacity 8 to release memory. +// +// This ring replaces channels in a few places in this client. The *main* +// advantage it provides is to allow loops that terminate. +// +// With channels, we always have to have a goroutine draining the channel. We +// cannot start the goroutine when we add the first element, because the +// goroutine will immediately drain the first and if something produces right +// away, it will start a second concurrent draining goroutine. +// +// We cannot fix that by adding a "working" field, because we would need a lock +// around checking if the goroutine still has elements *and* around setting the +// working field to false. If a push was blocked, it would be holding the lock, +// which would block the worker from grabbing the lock. Any other lock ordering +// has TOCTOU problems as well. +// +// The key insight is that we only pop the front *after* we are done with it. +// If there are still more elements, the worker goroutine can continue working. +// If there are no more elements, it can quit. When pushing, if the pusher +// pushed the first element, it starts the worker. +// +// Pushes fail if the ring is dead, allowing the pusher to fail any promise. +// If a die happens while a worker is running, all future pops will see the +// ring is dead and can fail promises immediately. If a worker is not running, +// then there are no promises that need to be called. + +const minRingCap = 8 + +type ring[T any] struct { + mu xsync.Mutex + + elems []T // circular buffer, min capacity minRingCap + head int // index of first element + l int // number of elements + + maxLen int // if >0, push blocks when l >= maxLen + cond *sync.Cond // used for blocking when at maxLen + dead bool +} + +// initMaxLen sets the maximum number of elements before push blocks. +// This must be called before any concurrent access. +func (r *ring[T]) initMaxLen(max int) { + r.maxLen = max + r.cond = sync.NewCond(&r.mu) +} + +func (r *ring[T]) die() { + r.mu.Lock() + defer r.mu.Unlock() + + r.dead = true + if r.cond != nil { + r.cond.Broadcast() + } +} + +func (r *ring[T]) push(elem T) (first, dead bool) { + r.mu.Lock() + defer r.mu.Unlock() + + // If a max length is set, block until there's space. + for r.maxLen > 0 && r.l >= r.maxLen && !r.dead { + r.cond.Wait() + } + + if r.dead { + return false, true + } + + // Grow: double capacity when full (or initialize to minRingCap). + if r.l == cap(r.elems) { + r.resize(max(cap(r.elems)*2, minRingCap)) + } + + // Write at tail position (head + l, wrapped). + writePos := (r.head + r.l) % cap(r.elems) + r.elems[writePos] = elem + r.l++ + + return r.l == 1, false +} + +// resize changes the buffer capacity, copying elements in linear order. +// Must be called with r.mu held. +func (r *ring[T]) resize(newCap int) { + newElems := make([]T, newCap) + if r.l > 0 { + // Copy elements in order: from head to end, then from start to head. + if r.head+r.l <= len(r.elems) { + copy(newElems, r.elems[r.head:r.head+r.l]) + } else { + n := copy(newElems, r.elems[r.head:]) + copy(newElems[n:], r.elems[:r.l-n]) + } + } + r.elems = newElems + r.head = 0 +} + +func (r *ring[T]) dropPeek() (next T, more, dead bool) { + var zero T + + r.mu.Lock() + defer r.mu.Unlock() + + if r.l == 0 { + return zero, false, r.dead + } + + // Clear current head element. + r.elems[r.head] = zero + r.head = (r.head + 1) % cap(r.elems) + r.l-- + + // Signal any blocked pushers that space is available. + if r.cond != nil { + r.cond.Signal() + } + + // Shrink: reduce to minRingCap when mostly empty to release memory. + if r.l <= minRingCap/2 && cap(r.elems) > minRingCap { + r.resize(minRingCap) + } + + if r.l > 0 { + return r.elems[r.head], true, r.dead + } + return zero, false, r.dead +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/sink.go b/vendor/github.com/twmb/franz-go/pkg/kgo/sink.go new file mode 100644 index 00000000000..29099f6ce15 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/sink.go @@ -0,0 +1,2633 @@ +package kgo + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash/crc32" + "math" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kbin" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +type sink struct { + cl *Client // our owning client, for cfg, metadata triggering, context, etc. + nodeID int32 // the node ID of the broker this sink belongs to + + // inflightSem controls the number of concurrent produce requests. We + // start with a limit of 1, which covers Kafka v0.11.0. On the first + // response, we check what version was set in the request. If it is at + // least 4, which 1.0 introduced, we upgrade the sem size. + inflightSem atomic.Value + produceVersion atomic.Int32 // negative is unset, positive is version + + drainState workLoop + + // seqRespsMu, guarded by seqRespsMu, contains responses that must + // be handled sequentially. These responses are handled asynchronously, + // but sequentially. + seqResps ring[*seqResp] // we never call die() on it + + backoffMu xsync.Mutex // guards the following + needBackoff bool + backoffSeq uint32 // prevents pile on failures + + // consecutiveFailures is incremented every backoff and cleared every + // successful response. For simplicity, if we have a good response + // following an error response before the error response's backoff + // occurs, the backoff is not cleared. + consecutiveFailures atomic.Uint32 + + recBufsMu xsync.Mutex // guards the following + recBufs []*recBuf // contains all partition records for batch building + recBufsStart int // incremented every req to avoid large batch starvation +} + +type seqResp struct { + resp kmsg.Response + err error + done chan struct{} + br *broker + promise func(*broker, kmsg.Response, error) +} + +func (cl *Client) newSink(nodeID int32) *sink { + s := &sink{ + cl: cl, + nodeID: nodeID, + } + s.produceVersion.Store(-1) + maxInflight := 1 + if cl.cfg.disableIdempotency { + maxInflight = cl.cfg.maxProduceInflight + } + s.inflightSem.Store(make(chan struct{}, maxInflight)) + return s +} + +// createReq returns a produceRequest from currently buffered records +// and whether there are more records to create more requests immediately. +func (s *sink) createReq(id int64, epoch int16) (*produceRequest, *kmsg.AddPartitionsToTxnRequest, bool) { + tx890p2 := s.cl.producer.tx890p2.Load() + + // produceMax is the highest Produce version we will let the broker dispatch + // negotiate for this request. Cap at 11 unless we can use v12+ (KIP-890 + // part 2: no AddPartitionsToTxn round-trip), and additionally cap at 12 if + // any recBuf in this request lacks a TopicID, because Produce v13 puts the + // TopicID on the wire (see #1312 - quirky brokers can advertise Produce v13 + // while capping Metadata below v10, leaving us without TopicIDs). + produceMax := int16(11) + if s.cl.cfg.txnID == nil || tx890p2 { + produceMax = 13 + } + + req := &produceRequest{ + produceMax: produceMax, + txnID: s.cl.cfg.txnID, + acks: s.cl.cfg.acks.val, + timeout: int32(s.cl.cfg.produceTimeout.Milliseconds()), + + producerID: id, + producerEpoch: epoch, + + compressor: s.cl.cfg.compressor, + + wireLength: s.cl.baseProduceRequestLength(), // start length with no topics + wireLengthLimit: s.cl.cfg.maxBrokerWriteBytes, + } + txnBuilder := txnReqBuilder{ + txnID: req.txnID, + id: id, + epoch: epoch, + pv12: produceMax >= 12, // v12 means no AddPartitionsToTxn round-trip + } + + var moreToDrain bool + + s.recBufsMu.Lock() + defer s.recBufsMu.Unlock() + + recBufsIdx := s.recBufsStart + for range s.recBufs { + recBuf := s.recBufs[recBufsIdx] + recBufsIdx = (recBufsIdx + 1) % len(s.recBufs) + + recBuf.mu.Lock() + if recBuf.failing || len(recBuf.batches) == recBuf.batchDrainIdx || recBuf.inflightOnSink != nil && recBuf.inflightOnSink != s || recBuf.inflight != 0 && !recBuf.okOnSink { + recBuf.mu.Unlock() + continue + } + + batch := recBuf.batches[recBuf.batchDrainIdx] + if added := req.tryAddBatch(s.produceVersion.Load(), recBuf, batch); !added { + recBuf.mu.Unlock() + moreToDrain = true + continue + } + if req.produceMax > 12 && recBuf.topicID == ([16]byte{}) { + req.produceMax = 12 + } + + if s.cl.cfg.disableIdempotency || s.cl.cfg.allowIdempotentProduceCancellation { + if cctx := batch.records[0].cancelingCtx(); cctx != nil && req.firstCancelingCtx == nil { + req.firstCancelingCtx = cctx //nolint:fatcontext // we are only here if firstCancelingCtx is currently nil + } + } + + recBuf.inflightOnSink = s + recBuf.inflight++ + + recBuf.batchDrainIdx++ + recBuf.lockedStopLinger() + recBuf.seq = incrementSequence(recBuf.seq, int32(len(batch.records))) + moreToDrain = recBuf.checkIfShouldDrainOrStartLinger() || moreToDrain + recBuf.mu.Unlock() + + txnBuilder.add(recBuf) + } + + // We could have lost our only record buffer just before we grabbed the + // lock above, so we have to check there are recBufs. + if len(s.recBufs) > 0 { + s.recBufsStart = (s.recBufsStart + 1) % len(s.recBufs) + } + return req, txnBuilder.req, moreToDrain +} + +func incrementSequence(sequence, increment int32) int32 { + if sequence > math.MaxInt32-increment { + return increment - (math.MaxInt32 - sequence) - 1 + } + + return sequence + increment +} + +type txnReqBuilder struct { + txnID *string + req *kmsg.AddPartitionsToTxnRequest + id int64 + epoch int16 + pv12 bool + addedTopics map[string]int // topic => index into req +} + +func (t *txnReqBuilder) add(rb *recBuf) { + // For produce v12+, we mark the partition as added to the transaction + // if there is no partition error code in a produce response. + // + // For prior versions, we actually issue an AddPartitionsToTxn request. + // The original logic was to mark addedToTxn before issuing the request + // and swap it back to false if the request failed or there was a + // partition error. We *could* swap this to only add to the txn on + // successful request, but there is other logic that needs to run on + // failure and the old code is well tested, so we'll keep it. + // + // We must keep the pv12 check first, otherwise we may accidentally + // mark something as added to the txn while produce requests fail. + if t.txnID == nil || t.pv12 || rb.addedToTxn.Swap(true) { + return + } + if t.req == nil { + req := kmsg.NewPtrAddPartitionsToTxnRequest() + req.TransactionalID = *t.txnID + req.ProducerID = t.id + req.ProducerEpoch = t.epoch + t.req = req + t.addedTopics = make(map[string]int, 10) + } + idx, exists := t.addedTopics[rb.topic] + if !exists { + idx = len(t.req.Topics) + t.addedTopics[rb.topic] = idx + reqTopic := kmsg.NewAddPartitionsToTxnRequestTopic() + reqTopic.Topic = rb.topic + t.req.Topics = append(t.req.Topics, reqTopic) + } + t.req.Topics[idx].Partitions = append(t.req.Topics[idx].Partitions, rb.partition) +} + +func (s *sink) maybeDrain() { + if s.cl.cfg.manualFlushing && s.cl.producer.flushing.Load() == 0 { + return + } + if s.drainState.maybeBegin() { + go s.drain() + } +} + +func (s *sink) maybeBackoff() { + s.backoffMu.Lock() + backoff := s.needBackoff + s.backoffMu.Unlock() + + if !backoff { + return + } + defer s.clearBackoff() + + s.cl.triggerUpdateMetadata(false, "opportunistic load during sink backoff") // as good a time as any + + tries := int(s.consecutiveFailures.Add(1)) + after := time.NewTimer(s.cl.cfg.retryBackoff(tries)) + defer after.Stop() + + select { + case <-after.C: + case <-s.cl.ctx.Done(): + case <-s.anyCtx().Done(): + } +} + +func (s *sink) maybeTriggerBackoff(seq uint32) { + s.backoffMu.Lock() + defer s.backoffMu.Unlock() + if seq == s.backoffSeq { + s.needBackoff = true + } +} + +func (s *sink) clearBackoff() { + s.backoffMu.Lock() + defer s.backoffMu.Unlock() + s.backoffSeq++ + s.needBackoff = false +} + +// drain drains buffered records and issues produce requests. +// +// This function is harmless if there are no records that need draining. +// We rely on that to not worry about accidental triggers of this function. +func (s *sink) drain() { + again := true + for again { + s.maybeBackoff() + + sem := s.inflightSem.Load().(chan struct{}) + select { + case sem <- struct{}{}: + case <-s.cl.ctx.Done(): + s.drainState.hardFinish() + return + } + + again = s.drainState.maybeFinish(s.produce(sem)) + } +} + +// Returns the first context encountered ranging across all records. +// This does not use defers to make it clear at the return that all +// unlocks are called in proper order. Ideally, do not call this func +// due to lock intensity. +func (s *sink) anyCtx() context.Context { + s.recBufsMu.Lock() + for _, recBuf := range s.recBufs { + recBuf.mu.Lock() + if len(recBuf.batches) > 0 { + batch0 := recBuf.batches[0] + batch0.mu.Lock() + // We grab the first context we can. If a batch can't + // be canceled, we skip it since that context is + // irrelevant. It's possible that a future batch also + // can't be canceled after we grab a canceling context; + // that's fine, this is just to cancel a request, we + // will handle retrying those batches when when + // handling the response. + if batch0.canFailFromLoadErrs && (!batch0.unsureIfProduced || s.cl.cfg.allowIdempotentProduceCancellation) && len(batch0.records) > 0 { + r0 := batch0.records[0] + if rctx := r0.cancelingCtx(); rctx != nil { + batch0.mu.Unlock() + recBuf.mu.Unlock() + s.recBufsMu.Unlock() + return rctx + } + } + batch0.mu.Unlock() + } + recBuf.mu.Unlock() + } + s.recBufsMu.Unlock() + return context.Background() +} + +func (s *sink) produce(sem <-chan struct{}) bool { + var produced bool + defer func() { + if !produced { + <-sem + } + }() + + // We could have some spurious produce calls while wrapping up. + if s.cl.BufferedProduceRecords() == 0 { + return false + } + + // producerID can fail from: + // - retry failure + // - auth failure + // - transactional: a produce failure that failed the producer ID + // - AddPartitionsToTxn failure (see just below) + // - some head-of-line context failure + // + // All but the first error is fatal. Recovery may be possible with + // EndTransaction in specific cases, but regardless, all buffered + // records must fail. + // + // NOTE: we init the producer ID before creating a request to ensure we + // are always using the latest id/epoch with the proper sequence + // numbers. (i.e., resetAllSequenceNumbers && producerID logic combo). + // + // For the first-discovered-record-head-of-line context, we want to + // avoid looking it up if possible (which is why producerID takes a + // ctxFn). If we do use one, we want to be sure that the + // context.Canceled error is from *that* context rather than the client + // context or something else. So, we go through some special care to + // track setting the ctx / looking up if it is canceled. + var holCtxMu xsync.Mutex + var holCtx context.Context + ctxFn := func() context.Context { + holCtxMu.Lock() + defer holCtxMu.Unlock() + if holCtx == nil { + holCtx = s.anyCtx() //nolint:fatcontext // not sure why this is flagged + } + return holCtx + } + isHolCtxDone := func() bool { + holCtxMu.Lock() + defer holCtxMu.Unlock() + if holCtx == nil { + return false + } + select { + case <-holCtx.Done(): + return true + default: + } + return false + } + + id, epoch, err := s.cl.producerID(ctxFn) + if err != nil { + var pe *errProducerIDLoadFail + switch { + case errors.As(err, &pe): + if errors.Is(pe.err, context.Canceled) && isHolCtxDone() { + // Some head-of-line record in a partition had a context cancelation. + // We look for any partition with HOL cancelations and fail them all. + s.cl.cfg.logger.Log(LogLevelInfo, "the first record in some partition(s) had a context cancelation; failing all relevant partitions", "broker", logID(s.nodeID)) + s.recBufsMu.Lock() + defer s.recBufsMu.Unlock() + for _, recBuf := range s.recBufs { + recBuf.mu.Lock() + var failAll bool + if len(recBuf.batches) > 0 { + batch0 := recBuf.batches[0] + batch0.mu.Lock() + if batch0.canFailFromLoadErrs && (!batch0.unsureIfProduced || s.cl.cfg.allowIdempotentProduceCancellation) && len(batch0.records) > 0 { + r0 := batch0.records[0] + if rctx := r0.cancelingCtx(); rctx != nil { + select { + case <-rctx.Done(): + failAll = true // we must not call failAllRecords here, because failAllRecords locks batches! + default: + } + } + } + batch0.mu.Unlock() + } + if failAll { + recBuf.failAllRecords(err) + } + recBuf.mu.Unlock() + } + return true + } + s.cl.bumpRepeatedLoadErr(err) + s.cl.cfg.logger.Log(LogLevelWarn, "unable to load producer ID, bumping client's buffered record load errors by 1 and retrying") + return true // whatever caused our produce, we did nothing, so keep going + case errors.Is(err, ErrClientClosed): + s.cl.failBufferedRecords(err) + default: + s.cl.cfg.logger.Log(LogLevelError, "fatal InitProducerID error, failing all buffered records", "broker", logID(s.nodeID), "err", err) + s.cl.failBufferedRecords(err) + } + return false + } + + // NOTE: we create the req AFTER getting our producer ID! + // + // If a prior response caused errReloadProducerID, then calling + // producerID() sets needSeqReset, and creating the request resets + // sequence numbers. We need to have that logic occur before we create + // the request, otherwise we will create a request with the old + // sequence numbers using our new producer ID, which will then again + // fail with OOOSN. + req, txnReq, moreToDrain := s.createReq(id, epoch) + if len(req.batches.bs) == 0 { // everything was failing or lingering, or what is buffered is in flight already + return moreToDrain + } + + // With KIP-890 (EndTxn v5+), the epoch is bumped on every EndTxn. + // There is a possible TOCTOU race between producerID() and + // createReq() above: + // + // - Sink A's drain loop calls producerID(), gets epoch N. + // - Sink A is preempted or otherwise delayed before calling + // createReq(). + // - Meanwhile, the last produce response arrives on another + // sink. bufferedRecords hits 0, Flush returns. + // - EndTxn round-trips, the epoch bumps to N+1. + // - The user produces records for the next transaction. + // - Sink A resumes, calls createReq(epoch=N), and picks up + // those new records stamped with the stale epoch N. + // - The broker rejects with INVALID_PRODUCER_EPOCH, failing + // the records permanently. + // + // We check the epoch after building the request. If it changed, + // we undo the drain state (resetBatchDrainIdx + decInflight) and + // return true so the drain loop retries with the new epoch. + // + // If the epoch has NOT changed, then EndTxn has not completed, + // Flush has not returned, and the user has not produced any + // records for a new transaction. Any records in the request are + // from the current transaction at epoch N, which is correct. + // + // This undo is safe: resetBatchDrainIdx + decInflight rewind + // each recBuf to its pre-drain state, the existing defer releases + // the semaphore (produced is still false), and returning true + // retries produce() with the correct epoch. + // + // We use a raw atomic load rather than producerID() to avoid + // side effects (blocking on idMu, triggering InitProducerID + // reloads). We bail on ANY change to the producer ID - whether + // from an epoch bump, a reload, or an error - so we do not need + // to interpret the stored value beyond id/epoch comparison. The + // next produce() call handles errors through producerID()'s + // normal error paths. + // + // We also bail on cur.err != nil. A parallel sink handling an + // OOOSN response can call failProducerID, which sets err on the + // stored producerID *without* changing id/epoch. The next + // producerID() call on that sink then reloads, and its + // resetAllProducerSequences pass walks every recBuf flipping + // needSeqReset=true BEFORE storing the new (id, epoch). There is + // a narrow window where a concurrent createReq on THIS sink has + // already picked up (id, epoch) from a pre-fail producerID() and + // now observes needSeqReset=true on some recBuf while the stored + // id/epoch still match. Without the err check, that createReq + // would send a request stamped with the old (id, epoch) but a + // freshly-reset seq=0; the broker would reject with OOOSN and we + // would retry. Checking err catches the window: after + // failProducerID, cur.err is non-nil even though id/epoch are + // unchanged, so we bail and the next drain pulls the reloaded + // (id, epoch) with seq=0 consistently. + if cur := s.cl.producer.id.Load().(*producerID); cur.id != id || cur.epoch != epoch || cur.err != nil { + req.batches.sliced().eachOwnerLocked(func(b *recBatch) { + b.owner.resetBatchDrainIdx() + b.decInflight() + }) + return true + } + + if txnReq != nil { + // txnReq can fail from: + // - TransactionAbortable + // - retry failure + // - auth failure + // - producer id mapping / epoch errors + // The latter case can potentially recover with the kip logic + // we have defined in EndTransaction. Regardless, on failure + // here, all buffered records must fail. + batchesStripped, err := s.doTxnReq(req, txnReq) + if err != nil { + switch { + case errors.Is(err, kerr.TransactionAbortable): + // If we get TransactionAbortable, we continue into producing. + // The produce will fail with the same error, and this is the + // only way to notify the user to abort the txn. + case isRetryableBrokerErr(err) || isDialNonTimeoutErr(err): + s.cl.bumpRepeatedLoadErr(err) + s.cl.cfg.logger.Log(LogLevelWarn, "unable to AddPartitionsToTxn due to retryable broker err, bumping client's buffered record load errors by 1 and retrying", "err", err) + s.cl.triggerUpdateMetadata(false, "attempting to refresh broker list due to failed AddPartitionsToTxn requests") + return moreToDrain || len(req.batches.bs) > 0 // nothing stripped if request-issuing error + default: + // Note that err can be InvalidProducerEpoch, which is + // potentially recoverable in EndTransaction. + // + // We do not fail all buffered records here, + // because that can lead to undesirable behavior + // with produce request vs. end txn (KAFKA-12671) + s.cl.failProducerID(id, epoch, err) + s.cl.cfg.logger.Log(LogLevelError, "fatal AddPartitionsToTxn error, failing all buffered records (it is possible the client can recover after EndTransaction)", "broker", logID(s.nodeID), "err", err) + return false + } + } + + // If we stripped everything, ensure we backoff to force a + // metadata load. If not everything was stripped, we issue our + // request and ensure we will retry a producing until + // everything is stripped (and we eventually back off). + if batchesStripped { + moreToDrain = true + if len(req.batches.bs) == 0 { + s.maybeTriggerBackoff(s.backoffSeq) + } + } + } + + if len(req.batches.bs) == 0 { // txn req could have removed some partitions to retry later (unknown topic, etc.) + return moreToDrain + } + + req.backoffSeq = s.backoffSeq // safe to read outside mu since we are in drain loop + + produced = true + + batches := req.batches.sliced() + s.doSequenced(req, func(br *broker, resp kmsg.Response, err error) { + s.handleReqResp(br, req, resp, err) + batches.eachOwnerLocked((*recBatch).decInflight) + <-sem + }) + return moreToDrain +} + +// With handleSeqResps below, this function ensures that all request responses +// are handled in order. We use this guarantee while in handleReqResp below. +func (s *sink) doSequenced( + req *produceRequest, + promise func(*broker, kmsg.Response, error), +) { + wait := &seqResp{ + done: make(chan struct{}), + promise: promise, + } + + // We can NOT use any record context. If we do, we force the request to + // fail while also force the batch to be unfailable (due to no + // response). If and only if the user has disabled idempotency, we + // allow the user to cancel the request via some random record with a + // canceling context. + ctx := req.firstCancelingCtx + if ctx == nil { + ctx = s.cl.ctx + } + br, err := s.cl.brokerOrErr(ctx, s.nodeID, errUnknownBroker) + if err != nil { + wait.err = err + close(wait.done) + } else { + br.do(ctx, req, func(resp kmsg.Response, err error) { + wait.resp = resp + wait.err = err + close(wait.done) + }) + wait.br = br + } + + if first, _ := s.seqResps.push(wait); first { + go s.handleSeqResps(wait) + } +} + +// Ensures that all request responses are processed in order. +func (s *sink) handleSeqResps(wait *seqResp) { + var more bool +start: + <-wait.done + wait.promise(wait.br, wait.resp, wait.err) + + wait, more, _ = s.seqResps.dropPeek() + if more { + goto start + } +} + +// Issues an AddPartitionsToTxnRequest before a produce request for all +// partitions that need to be added to a transaction. +func (s *sink) doTxnReq( + req *produceRequest, + txnReq *kmsg.AddPartitionsToTxnRequest, +) (stripped bool, err error) { + // If we return an unretryable error, then we have to reset everything + // to not be in the transaction and begin draining at the start. + // + // These batches must be the first in their recBuf, because we would + // not be trying to add them to a partition if they were not. + defer func() { + if err != nil { + req.batches.eachOwnerLocked(seqRecBatch.removeFromTxn) + } + }() + // We do NOT let record context cancelations fail this request: doing + // so would put the transactional ID in an unknown state. This is + // similar to the warning we give in the txn.go file, but the + // difference there is the user knows explicitly at the function call + // that canceling the context will opt them into invalid state. + err = s.cl.doWithConcurrentTransactions(s.cl.ctx, fmt.Sprintf("AddPartitionsToTxn-sink%d", s.nodeID), func() error { + stripped, err = s.issueTxnReq(req, txnReq) + return err + }) + return stripped, err +} + +// Removing a batch from the transaction means we will not be issuing it +// inflight, and that it was not added to the txn and that we need to reset the +// drain index. +func (b *recBatch) removeFromTxn() { + b.owner.addedToTxn.Store(false) + b.owner.resetBatchDrainIdx() + b.decInflight() +} + +func (s *sink) issueTxnReq( + req *produceRequest, + txnReq *kmsg.AddPartitionsToTxnRequest, +) (stripped bool, fatalErr error) { + resp, err := txnReq.RequestWith(s.cl.ctx, s.cl) + if err != nil { + return false, err + } + + for _, topic := range resp.Topics { + topicBatches, ok := req.batches.bs[topic.Topic] + if !ok { + s.cl.cfg.logger.Log(LogLevelError, "broker replied with topic in AddPartitionsToTxnResponse that was not in request", "topic", topic.Topic) + continue + } + for _, partition := range topic.Partitions { + if err := kerr.ErrorForCode(partition.ErrorCode); err != nil && err != kerr.TransactionAbortable { // see below for txn abortable + // OperationNotAttempted is set for all partitions that are authorized + // if any partition is unauthorized _or_ does not exist. We simply remove + // unattempted partitions and treat them as retryable. + if !kerr.IsRetriable(err) && !errors.Is(err, kerr.OperationNotAttempted) { + fatalErr = err // auth err, etc + continue + } + + batch, ok := topicBatches[partition.Partition] + if !ok { + s.cl.cfg.logger.Log(LogLevelError, "broker replied with partition in AddPartitionsToTxnResponse that was not in request", "topic", topic.Topic, "partition", partition.Partition) + continue + } + + // We are stripping this retryable-err batch from the request, + // so we must reset that it has been added to the txn. + batch.owner.mu.Lock() + batch.removeFromTxn() + batch.owner.mu.Unlock() + + stripped = true + + delete(topicBatches, partition.Partition) + } + if len(topicBatches) == 0 { + delete(req.batches.bs, topic.Topic) + } + } + } + return stripped, fatalErr +} + +// firstRespCheck is effectively a sink.Once. On the first response, if the +// used request version is at least 4, we upgrade our inflight sem. +// +// Starting on version 4, Kafka allowed five inflight requests while +// maintaining idempotency. Before, only one was allowed. +// +// We go through an atomic because drain can be waiting on the sem (with +// capacity one). We store four here, meaning new drain loops will load the +// higher capacity sem without read/write pointer racing a current loop. +// +// This logic does mean that we will never use the full potential 5 in flight +// outside of a small window during the store, but some pages in the Kafka +// confluence basically show that more than two in flight has marginal benefit +// anyway (although that may be due to their Java API). +// +// https://cwiki.apache.org/confluence/display/KAFKA/An+analysis+of+the+impact+of+max.in.flight.requests.per.connection+and+acks+on+Producer+performance +// https://issues.apache.org/jira/browse/KAFKA-5494 +func (s *sink) firstRespCheck(idempotent bool, version int16) { + if s.produceVersion.Load() < 0 { + s.produceVersion.Store(int32(version)) + if idempotent && version >= 4 { + s.inflightSem.Store(make(chan struct{}, 4)) + } + } +} + +// handleReqClientErr is called when the client errors before receiving a +// produce response. +func (s *sink) handleReqClientErr(req *produceRequest, err error) { + switch { + default: + s.cl.cfg.logger.Log(LogLevelWarn, "random error while producing, requeueing unattempted request", "broker", logID(s.nodeID), "err", err) + fallthrough + + case errors.Is(err, errUnknownBroker), + isDialNonTimeoutErr(err), + isRetryableBrokerErr(err): + updateMeta := !isRetryableBrokerErr(err) + if updateMeta { + s.cl.cfg.logger.Log(LogLevelInfo, "produce request failed, triggering metadata update", "broker", logID(s.nodeID), "err", err) + } else { + s.cl.cfg.logger.Log(LogLevelDebug, "produce request failed with a retryable error, retrying without a metadata update", "broker", logID(s.nodeID), "err", err) + } + s.handleRetryBatches(req.batches, nil, req.backoffSeq, updateMeta, false, "failed produce request triggered metadata update") + + case errors.Is(err, ErrClientClosed): + s.cl.failBufferedRecords(ErrClientClosed) + } +} + +// No acks mean no response. The following block is basically an extremely +// condensed version of the logic in handleReqResp. +func (s *sink) handleReqRespNoack(b *bytes.Buffer, debug bool, req *produceRequest) { + if debug { + fmt.Fprintf(b, "noack ") + } + for topic, partitions := range req.batches.bs { + if debug { + fmt.Fprintf(b, "%s[", topic) + } + for partition, batch := range partitions { + batch.owner.mu.Lock() + if batch.isOwnersFirstBatch() { + if debug { + fmt.Fprintf(b, "%d{0=>%d}, ", partition, len(batch.records)) + } + s.cl.finishBatch(batch.recBatch, req.producerID, req.producerEpoch, 0, nil) + } else if debug { + fmt.Fprintf(b, "%d{skipped}, ", partition) + } + batch.owner.mu.Unlock() + } + if debug { + if bytes.HasSuffix(b.Bytes(), []byte(", ")) { + b.Truncate(b.Len() - 2) + } + b.WriteString("], ") + } + } +} + +func (s *sink) handleReqResp(br *broker, req *produceRequest, resp kmsg.Response, err error) { + // Bump retry counter for every batch in this request. Runs once per + // br.do completion, not per AppendTo call, so the broker-level + // zero-bytes-written retry at broker.go does not double-count. + for _, partitions := range req.batches.bs { + for _, batch := range partitions { + batch.tries.Add(1) + } + } + + if err != nil { + s.handleReqClientErr(req, err) + return + } + s.firstRespCheck(req.idempotent(), req.version) + s.consecutiveFailures.Store(0) + defer req.metrics.hook(&s.cl.cfg, br) // defer to end so that non-written batches are removed + + var b *bytes.Buffer + debug := s.cl.cfg.logger.Level() >= LogLevelDebug + if debug { + b = bytes.NewBuffer(make([]byte, 0, 128)) + defer func() { + update := b.String() + update = strings.TrimSuffix(update, ", ") + s.cl.cfg.logger.Log(LogLevelDebug, "produced", "broker", logID(s.nodeID), "to", update) + }() + } + + if req.acks == 0 { + s.handleReqRespNoack(b, debug, req) + return + } + + var kmove kip951move + var reqRetry seqRecBatches // handled at the end + + kresp := resp.(*kmsg.ProduceResponse) + for i := range kresp.Topics { + rt := &kresp.Topics[i] + topic := rt.Topic + tid := rt.TopicID + if req.version >= 13 { + var ok bool + if topic, ok = req.batches.id2t[rt.TopicID]; !ok { + s.cl.cfg.logger.Log(LogLevelError, "broker erroneously replied with topic id in produce request that we did not produce to", "broker", logID(s.nodeID), "topic", topic, "topic_id", strtid(rt.TopicID)) + delete(req.metrics, topic) + continue + } + } else { + tid = req.batches.t2id[topic] + } + partitions, ok := req.batches.bs[topic] + if !ok { + s.cl.cfg.logger.Log(LogLevelError, "broker erroneously replied with topic in produce request that we did not produce to", "broker", logID(s.nodeID), "topic", topic) + delete(req.metrics, topic) + continue + } + + if debug { + fmt.Fprintf(b, "%s[", topic) + } + + tmetrics := req.metrics[topic] + for j := range rt.Partitions { + rp := &rt.Partitions[j] + partition := rp.Partition + batch, ok := partitions[partition] + if !ok { + s.cl.cfg.logger.Log(LogLevelError, "broker erroneously replied with partition in produce request that we did not produce to", "broker", logID(s.nodeID), "topic", rt.Topic, "partition", partition) + delete(tmetrics, partition) + continue // should not hit this + } + delete(partitions, partition) + + retry, didProduce := s.handleReqRespBatch( + b, + &kmove, + kresp, + topic, + rp, + batch, + req.producerID, + req.producerEpoch, + tmetrics[partition], + ) + if retry { + reqRetry.addSeqBatch(topic, tid, partition, batch) + } + if !didProduce { + delete(tmetrics, partition) + } + } + + if debug { + if bytes.HasSuffix(b.Bytes(), []byte(", ")) { + b.Truncate(b.Len() - 2) + } + b.WriteString("], ") + } + + if len(partitions) == 0 { + delete(req.batches.bs, topic) + } + } + + if len(req.batches.bs) > 0 { + s.cl.cfg.logger.Log(LogLevelError, "broker did not reply to all topics / partitions in the produce request! reenqueuing missing partitions", "broker", logID(s.nodeID)) + s.handleRetryBatches(req.batches, nil, 0, true, false, "broker did not reply to all topics in produce request") + } + if len(reqRetry.bs) > 0 { + s.handleRetryBatches(reqRetry, &kmove, 0, true, true, "produce request had retry batches") + } +} + +func (s *sink) handleReqRespBatch( + b *bytes.Buffer, + kmove *kip951move, + resp *kmsg.ProduceResponse, + topic string, + rp *kmsg.ProduceResponseTopicPartition, + batch seqRecBatch, + producerID int64, + producerEpoch int16, + batchMetrics ProduceBatchMetrics, +) (retry, didProduce bool) { + batch.owner.mu.Lock() + defer batch.owner.mu.Unlock() + + nrec := len(batch.records) + + debug := b != nil + if debug { + fmt.Fprintf(b, "%d{", rp.Partition) + } + + // We only ever operate on the first batch in a record buf. Batches + // work sequentially; if this is not the first batch then an error + // happened and this later batch is no longer a part of a seq chain. + if !batch.isOwnersFirstBatch() { + if debug { + if err := kerr.ErrorForCode(rp.ErrorCode); err == nil { + if nrec > 0 { + fmt.Fprintf(b, "skipped@%d=>%d}, ", rp.BaseOffset, rp.BaseOffset+int64(nrec)) + } else { + fmt.Fprintf(b, "skipped@%d}, ", rp.BaseOffset) + } + } else { + if nrec > 0 { + fmt.Fprintf(b, "skipped@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } else { + fmt.Fprintf(b, "skipped@%d(%s)}, ", rp.BaseOffset, err) + } + } + } + return false, false + } + + // Since we have received a response and we are the first batch, we can + // at this point re-enable failing from load errors. + // + // We do not need the mutex lock on the batch. We already have the + // recBuf mu (guarding most concurrency). The only place batch fields + // are accessed & modified without the recBuf mu is when writing a + // batch, and we only ever use a batch in inflight request at a time + // (regardless of the partition being canceled or moving to a + // different sink). + batch.canFailFromLoadErrs = true + + // If the response was from a timeout, or the record was written but + // not to enough replicas, we actually do not know whether the record + // was persisted or not. We need to poison this batch: if we encounter + // a retryable error the NEXT time we produce, we still are unsure of + // the final state, and we need to block canceling producing. + if rp.ErrorCode == kerr.RequestTimedOut.Code || rp.ErrorCode == kerr.NotEnoughReplicasAfterAppend.Code { + batch.unsureIfProduced = true + } + + // By default, we assume we errored. Non-error updates this back + // to true. + batch.owner.okOnSink = false + + if moving := kmove.maybeAddProducePartition(resp, rp, batch.owner); moving { + if debug { + fmt.Fprintf(b, "move:%d:%d@%d,%d}, ", rp.CurrentLeader.LeaderID, rp.CurrentLeader.LeaderEpoch, rp.BaseOffset, nrec) + } + batch.owner.failing = true + return true, false + } + + err := kerr.ErrorForCode(rp.ErrorCode) + if errors.Is(err, kerr.MessageTooLarge) { + err = fmt.Errorf("%w (uncompressed_bytes=%d, compressed_bytes=%d)", err, batchMetrics.UncompressedBytes, batchMetrics.CompressedBytes) + } + failUnknown := batch.owner.checkUnknownFailLimit(err) + switch { + case err == kerr.ConcurrentTransactions: + // Occasionally this is bubbled back to the producer as of + // KIP-890; we retry this. + fallthrough //nolint:gocritic // easier to read this way + + case kerr.IsRetriable(err) && + !failUnknown && + err != kerr.CorruptMessage && + (batch.tries.Load() <= s.cl.cfg.recordRetries || batch.unsureIfProduced): // we need to bypass the retry limit if we are not sure of the state + if debug { + fmt.Fprintf(b, "retrying@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } + return true, false + + case err == kerr.OutOfOrderSequenceNumber, + err == kerr.UnknownProducerID, + err == kerr.InvalidProducerIDMapping, + err == kerr.InvalidProducerEpoch: + + // OOOSN always means data loss 1.0+ and is ambiguous prior. + // We assume the worst and only continue if requested. + // + // UnknownProducerID was introduced to allow some form of safe + // handling, but KIP-360 demonstrated that resetting sequence + // numbers is fundamentally unsafe, so we treat it like OOOSN. + // + // KAFKA-5793 specifically mentions for OOOSN "when you get it, + // it should always mean data loss". Sometime after KIP-360, + // Kafka changed the client to remove all places + // UnknownProducerID was returned, and then started referring + // to OOOSN as retryable. KIP-890 definitively says OOOSN is + // retryable. However, the Kafka source as of 24-10-10 still + // only retries OOOSN for batches that are NOT the expected + // next batch (i.e., it's next + 1, for when there are multiple + // in flight). With KIP-890, we still just disregard whatever + // supposedly non-retryable / actually-is-retryable error is + // returned if the LogStartOffset is _after_ what we previously + // produced. Specifically, this is step (4) in wiki link + // within KAFKA-5793. + // + // InvalidMapping is similar to UnknownProducerID, but occurs + // when the txnal coordinator timed out our transaction. + // + // 2.5 + // ===== + // 2.5 introduced some behavior to potentially safely reset + // the sequence numbers by bumping an epoch (see KIP-360). + // + // For the idempotent producer, the solution is to fail all + // buffered records and then let the client user reset things + // with the understanding that they cannot guard against + // potential dups / reordering at that point. Realistically, + // that's no better than a config knob that allows the user + // to continue (our stopOnDataLoss flag), so for the idempotent + // producer, if stopOnDataLoss is false, we just continue. + // + // For the transactional producer, we always fail the producerID. + // EndTransaction will trigger recovery if possible. + // + // 2.7 + // ===== + // InvalidProducerEpoch became retryable in 2.7. Prior, it + // was ambiguous (timeout? fenced?). Now, InvalidProducerEpoch + // is only returned on produce, and then we can recover on other + // txn coordinator requests, which have PRODUCER_FENCED vs + // TRANSACTION_TIMED_OUT. + + if batch.owner.lastAckedOffset >= 0 && rp.LogStartOffset > batch.owner.lastAckedOffset { + s.cl.cfg.logger.Log(LogLevelInfo, "partition prefix truncation to after our last produce caused the broker to forget us; no loss occurred, bumping producer epoch and resetting sequence numbers", + "broker", logID(s.nodeID), + "topic", topic, + "partition", rp.Partition, + "producer_id", producerID, + "producer_epoch", producerEpoch, + "err", err, + ) + s.cl.failProducerID(producerID, producerEpoch, errReloadProducerID) + if debug { + fmt.Fprintf(b, "resetting@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } + return true, false + } + + if s.cl.cfg.txnID != nil || s.cl.cfg.stopOnDataLoss { + s.cl.cfg.logger.Log(LogLevelInfo, "batch errored, failing the producer ID", + "broker", logID(s.nodeID), + "topic", topic, + "partition", rp.Partition, + "producer_id", producerID, + "producer_epoch", producerEpoch, + "err", err, + ) + s.cl.failProducerID(producerID, producerEpoch, err) + + s.cl.finishBatch(batch.recBatch, producerID, producerEpoch, rp.BaseOffset, err) + if debug { + fmt.Fprintf(b, "fatal@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } + return false, false + } + if s.cl.cfg.onDataLoss != nil { + s.cl.cfg.onDataLoss(topic, rp.Partition) + } + + // For OOOSN, and UnknownProducerID + // + // The only recovery is to fail the producer ID, which ensures + // that all batches reset sequence numbers and use a new producer + // ID on the next batch. + // + // For InvalidProducerIDMapping && InvalidProducerEpoch, + // + // We should not be here, since this error occurs in the + // context of transactions, which are caught above. + s.cl.cfg.logger.Log(LogLevelInfo, fmt.Sprintf("batch errored with %s, failing the producer ID and resetting all sequence numbers", err.(*kerr.Error).Message), + "broker", logID(s.nodeID), + "topic", topic, + "partition", rp.Partition, + "producer_id", producerID, + "producer_epoch", producerEpoch, + "sequence", batch.seq, + "first_sequence", batch.owner.batch0Seq, + "num_records", nrec, + "base_offset", rp.BaseOffset, + "err", err, + ) + + // After we fail here, any new produce (even new ones + // happening concurrent with this function) will load + // a new epoch-bumped producer ID and all first-batches + // will reset sequence numbers appropriately. + s.cl.failProducerID(producerID, producerEpoch, errReloadProducerID) + if debug { + fmt.Fprintf(b, "resetting@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } + return true, false + + case err == kerr.DuplicateSequenceNumber: // ignorable, but we should not get + s.cl.cfg.logger.Log(LogLevelInfo, "received unexpected duplicate sequence number, ignoring and treating batch as successful", + "broker", logID(s.nodeID), + "topic", topic, + "partition", rp.Partition, + "producer_id", producerID, + "producer_epoch", producerEpoch, + "sequence", batch.seq, + "first_sequence", batch.owner.batch0Seq, + "num_records", nrec, + "base_offset", rp.BaseOffset, + ) + err = nil + fallthrough + default: + if err != nil { + s.cl.cfg.logger.Log(LogLevelInfo, "batch in a produce request failed", + "broker", logID(s.nodeID), + "topic", topic, + "partition", rp.Partition, + "err", err, + "err_is_retryable", kerr.IsRetriable(err), + "max_retries_reached", !failUnknown && batch.tries.Load() > s.cl.cfg.recordRetries, + ) + } else { + // Only credit okOnSink to the sink that earned it. If the + // recBuf migrated mid-flight, this response is from the old + // sink; setting okOnSink=true here would let the new sink + // pipeline a second produce before its own first ack and + // re-introduce the #223 OOOSN-on-retry race the gate exists + // to prevent. lastAckedOffset and addedToTxn track broker- + // side facts and remain unconditional. + if batch.owner.sink == s { + batch.owner.okOnSink = true + } + batch.owner.lastAckedOffset = rp.BaseOffset + int64(len(batch.records)) + if resp.Version >= 12 && s.cl.cfg.txnID != nil { + batch.owner.addedToTxn.Swap(true) + } + } + s.cl.finishBatch(batch.recBatch, producerID, producerEpoch, rp.BaseOffset, err) + didProduce = err == nil + if debug { + if err != nil { + fmt.Fprintf(b, "err@%d,%d(%s)}, ", rp.BaseOffset, nrec, err) + } else { + fmt.Fprintf(b, "%d=>%d}, ", rp.BaseOffset, rp.BaseOffset+int64(nrec)) + } + } + } + return false, didProduce // no retry +} + +// finishBatch removes a batch from its owning record buffer and finishes all +// records in the batch. +// +// This is safe even if the owning recBuf migrated sinks, since we are +// finishing based off the status of an inflight req from the original sink. +func (cl *Client) finishBatch(batch *recBatch, producerID int64, producerEpoch int16, baseOffset int64, err error) { + recBuf := batch.owner + + if err != nil { + // We know that Kafka replied this batch is a failure. We can + // fail this batch and all batches in this partition. + // This will keep sequence numbers correct. + recBuf.failAllRecords(err) + return + } + + // We know the batch made it to Kafka successfully without error. + // We remove this batch and finish all records appropriately. + finished := len(batch.records) + recBuf.batch0Seq = incrementSequence(recBuf.batch0Seq, int32(finished)) + recBuf.buffered.Add(-int64(finished)) + recBuf.batches[0] = nil + recBuf.batches = recBuf.batches[1:] + recBuf.batchDrainIdx-- + + batch.mu.Lock() + records, attrs := batch.records, batch.attrs + batch.records = nil + batch.mu.Unlock() + + cl.producer.promiseBatch(batchPromise{ + baseOffset: baseOffset, + pid: producerID, + epoch: producerEpoch, + // A recBuf.attrs is updated when appending to be written. For + // v0 && v1 produce requests, we set bit 8 in the attrs + // corresponding to our own RecordAttr's bit 8 being no + // timestamp type. Thus, we can directly convert the batch + // attrs to our own RecordAttrs. + attrs: RecordAttrs{uint8(attrs)}, + recs: records, + }) +} + +// handleRetryBatches sets any first-buf-batch to failing and triggers a +// metadata that will eventually clear the failing state and re-drain. +// +// If idempotency is disabled, if a batch is timed out or hit the retry limit, +// we fail it and anything after it. +func (s *sink) handleRetryBatches( + retry seqRecBatches, + kmove *kip951move, + backoffSeq uint32, + updateMeta bool, // if we should maybe update the metadata + canFail bool, // if records can fail if they are at limits + why string, +) { + logger := s.cl.cfg.logger + debug := logger.Level() >= LogLevelDebug + var needsMetaUpdate bool + var shouldBackoff bool + if kmove != nil { + defer kmove.maybeBeginMove(s.cl) + } + var numRetryBatches, numMoveBatches int + retry.eachOwnerLocked(func(batch seqRecBatch) { + numRetryBatches++ + if !batch.isOwnersFirstBatch() { + if debug { + logger.Log(LogLevelDebug, "retry batch is not the first batch in the owner, skipping result", + "topic", batch.owner.topic, + "partition", batch.owner.partition, + ) + } + return + } + + // If the request failed due to a concurrent metadata update + // moving partitions to a different sink (or killing the sink + // this partition was on), we can just reset the drain index + // and trigger draining now the new sink. There is no reason + // to backoff on this sink nor trigger a metadata update. + if batch.owner.sink != s { + if debug { + logger.Log(LogLevelDebug, "transitioned sinks while a request was inflight, retrying immediately on new sink without backoff", + "topic", batch.owner.topic, + "partition", batch.owner.partition, + "old_sink", s.nodeID, + "new_sink", batch.owner.sink.nodeID, + ) + } + batch.owner.resetBatchDrainIdx() + return + } + + if (canFail && !batch.unsureIfProduced) || s.cl.cfg.disableIdempotency || s.cl.cfg.allowIdempotentProduceCancellation { + if err := batch.maybeFailErr(&s.cl.cfg); err != nil { + batch.owner.failAllRecords(err) + return + } + } + + batch.owner.resetBatchDrainIdx() + + // Now that the batch drain index is reset, if this retry is + // caused from a moving batch, return early. We do not need + // to backoff nor do we need to trigger a metadata update. + if kmove.hasRecBuf(batch.owner) { + numMoveBatches++ + return + } + + // If our first batch (seq == 0) fails with unknown topic, we + // retry immediately. Kafka can reply with valid metadata + // immediately after a topic was created, before the leaders + // actually know they are leader. + unknownAndFirstBatch := batch.owner.unknownFailures == 1 && batch.owner.seq == 0 + + if unknownAndFirstBatch { + shouldBackoff = true + return + } + if updateMeta { + batch.owner.failing = true + needsMetaUpdate = true + } + }) + + if debug { + logger.Log(LogLevelDebug, "retry batches processed", + "wanted_metadata_update", updateMeta, + "triggering_metadata_update", needsMetaUpdate, + "should_backoff", shouldBackoff, + ) + } + + // If we do want to metadata update, we only do so if any batch was the + // first batch in its buf / not concurrently failed. + if needsMetaUpdate { + s.cl.triggerUpdateMetadata(true, why) + return + } + + // We could not need a metadata update for two reasons: + // + // * our request died when being issued + // + // * we would update metadata, but what failed was the first batch + // produced and the error was unknown topic / partition. + // + // In either of these cases, we should backoff a little bit to avoid + // spin looping. + // + // If neither of these cases are true, then we entered wanting a + // metadata update, but the batches either were not the first batch, or + // the batches were concurrently failed. + // + // If all partitions are moving, we do not need to backoff nor drain. + if shouldBackoff || (!updateMeta && numRetryBatches != numMoveBatches) { + s.maybeTriggerBackoff(backoffSeq) + s.maybeDrain() + } +} + +// addRecBuf adds a new record buffer to be drained to a sink and clears the +// buffer's failing state. +func (s *sink) addRecBuf(add *recBuf) { + s.recBufsMu.Lock() + add.recBufsIdx = len(s.recBufs) + s.recBufs = append(s.recBufs, add) + s.recBufsMu.Unlock() + + add.clearFailing() +} + +// removeRecBuf removes a record buffer from a sink. +func (s *sink) removeRecBuf(rm *recBuf) { + s.recBufsMu.Lock() + defer s.recBufsMu.Unlock() + + if rm.recBufsIdx != len(s.recBufs)-1 { + s.recBufs[rm.recBufsIdx], s.recBufs[len(s.recBufs)-1] = s.recBufs[len(s.recBufs)-1], nil + s.recBufs[rm.recBufsIdx].recBufsIdx = rm.recBufsIdx + } else { + s.recBufs[rm.recBufsIdx] = nil // do not let this removal hang around + } + + s.recBufs = s.recBufs[:len(s.recBufs)-1] + if s.recBufsStart == len(s.recBufs) { + s.recBufsStart = 0 + } +} + +// recBuf is a buffer of records being produced to a partition and being +// drained by a sink. This is only not drained if the partition has a load +// error and thus does not a have a sink to be drained into. +type recBuf struct { + cl *Client // for cfg, record finishing + + topic string + topicID [16]byte + partition int32 + + // The number of bytes we can buffer in a batch for this particular + // topic/partition. This may be less than the configured + // maxRecordBatchBytes because of produce request overhead. + maxRecordBatchBytes int32 + + // addedToTxn, for transactions only, signifies whether this partition + // has been added to the transaction yet or not. + addedToTxn atomic.Bool + + // For LoadTopicPartitioner partitioning; atomically tracks the number + // of records buffered in total on this recBuf. + buffered atomic.Int64 + + mu xsync.Mutex // guards r/w access to all fields below + + // sink is who is currently draining us. This can be modified + // concurrently during a metadata update. + // + // The first set to a non-nil sink is done without a mutex. + // + // Since only metadata updates can change the sink, metadata updates + // also read this without a mutex. + sink *sink + // recBufsIdx is our index into our current sink's recBufs field. + // This exists to aid in removing the buffer from the sink. + recBufsIdx int + + // A concurrent metadata update can move a recBuf from one sink to + // another while requests are inflight on the original sink. We do not + // want to allow new requests to start on the new sink until they all + // finish on the old, because with some pathological request order + // finishing, we would allow requests to finish out of order: + // handleSeqResps works per sink, not across sinks. + inflightOnSink *sink + // We only want to allow more than 1 inflight on a sink *if* we are + // currently receiving successful responses. Unimportantly, this allows + // us to save resources if the broker is having a problem or just + // recovered from one. Importantly, we work around an edge case in + // Kafka. Kafka will accept the first produce request for a pid/epoch + // with *any* sequence number. Say we sent two requests inflight. The + // first request Kafka replies to with NOT_LEADER_FOR_PARTITION, the + // second, the broker finished setting up and accepts. The broker now + // has the second request but not the first, we will retry both + // requests and receive OOOSN, and the broker has logs out of order. + // By only allowing more than one inflight if we have seen an ok + // response, we largely eliminate risk of this problem. See #223 for + // more details. + okOnSink bool + // Inflight tracks the number of requests inflight using batches from + // this recBuf. Every time this hits zero, if the batchDrainIdx is not + // at the end, we clear inflightOnSink and trigger the *current* sink + // to drain. + inflight uint8 + + lastAckedOffset int64 // last ProduceResponse's BaseOffset + how many records we produced + + topicPartitionData // updated in metadata migrateProductionTo (same spot sink is updated) + + // seq is used for the seq in each record batch. It is incremented when + // produce requests are made and can be reset on errors to batch0Seq. + // + // If idempotency is disabled, we just use "0" for the first sequence + // when encoding our payload. + // + // This is also used to check the first batch produced (disregarding + // seq resets) -- see handleRetryBatches. + seq int32 + // batch0Seq is the seq of the batch at batchDrainIdx 0. If we reset + // the drain index, we reset seq with this number. If we successfully + // finish batch 0, we bump this. + batch0Seq int32 + // If we need to reset sequence numbers, we set needSeqReset, and then + // when we use the **first** batch, we reset sequences to 0. + needSeqReset bool + + // batches is our list of buffered records. Batches are appended as the + // final batch crosses size thresholds or as drain freezes batches from + // further modification. + // + // Most functions in a sink only operate on a batch if the batch is the + // first batch in a buffer. This is necessary to ensure that all + // records are truly finished without error in order. + batches []*recBatch + // batchDrainIdx is where the next batch will drain from. We only + // remove from the head of batches when a batch is finished. + // This is read while buffering and modified in a few places. + batchDrainIdx int + + // If we fail with UNKNOWN_TOPIC_OR_PARTITION, we bump this and fail + // all records once this exceeds the config's unknown topic fail limit. + // If we ever see a different error (or no error), this is reset. + unknownFailures int64 + + // lingering is a timer that avoids starting maybeDrain until expiry, + // allowing for more records to be buffered in a single batch. + // + // Note that if something else starts a drain, if the first batch of + // this buffer fits into the request, it will be used. + // + // This is on recBuf rather than Sink to avoid some complicated + // interactions of triggering the sink to loop or not. Ideally, with + // the sticky partition hashers, we will only have a few partitions + // lingering and that this is on a RecBuf should not matter. + lingering *time.Timer + lingerFn func() // stored once to avoid method value closure alloc per linger cycle + isLingering bool // whether the linger timer is active; lingering may be non-nil but stopped + + // failing is set when we encounter a temporary partition error during + // producing, such as UnknownTopicOrPartition (signifying the partition + // moved to a different broker). + // + // It is always cleared on metadata update. + failing bool + + // Only possibly set in PurgeTopics, this is used to fail anything that + // was in the process of being buffered. + purged bool +} + +// bufferRecord usually buffers a record, but does not if abortOnNewBatch is +// true and if this function would create a new batch. +// +// This returns whether the promised record was processed or not (buffered or +// immediately errored). +func (recBuf *recBuf) bufferRecord(pr promisedRec, abortOnNewBatch bool) bool { + recBuf.mu.Lock() + defer recBuf.mu.Unlock() + + // We truncate to milliseconds to avoid some accumulated rounding error + // problems (see IBM/sarama#1455) + if pr.Timestamp.IsZero() { + pr.Timestamp = time.Now() + } + pr.Timestamp = pr.Timestamp.Truncate(time.Millisecond) + pr.Partition = recBuf.partition // set now, for the hook below + + if recBuf.purged { + recBuf.cl.producer.promiseRecord(pr, errPurged) + return true + } + + var ( + mkNewBatch = true + produceVersion = recBuf.sink.produceVersion.Load() + ) + + if recBuf.batchDrainIdx != len(recBuf.batches) { + batch := recBuf.batches[len(recBuf.batches)-1] + appended, _ := batch.tryBuffer(pr, produceVersion, recBuf.maxRecordBatchBytes, false) + mkNewBatch = !appended + } + + if mkNewBatch { + newBatch := recBuf.newRecordBatch() + appended, aborted := newBatch.tryBuffer(pr, produceVersion, recBuf.maxRecordBatchBytes, abortOnNewBatch) + + switch { + case aborted: // not processed + recBuf.cl.prsPool.put(newBatch.records) + return false + case appended: // we return true below + default: // processed as failure + recBuf.cl.prsPool.put(newBatch.records) + recBuf.cl.producer.promiseRecord(pr, + fmt.Errorf("%w (uncompressed_bytes=%d)", kerr.MessageTooLarge, pr.userSize()), + ) + return true + } + + recBuf.batches = append(recBuf.batches, newBatch) + } + + recBuf.maybeTriggerDrain() + + recBuf.buffered.Add(1) + if recBuf.cl.producer.hooks != nil && len(recBuf.cl.producer.hooks.partitioned) > 0 { + for _, h := range recBuf.cl.producer.hooks.partitioned { + h.OnProduceRecordPartitioned(pr.Record, recBuf.sink.nodeID) + } + } + + return true +} + +// Maybe drains, maybe starts a linger. +// Must be called while locked. +func (recBuf *recBuf) maybeTriggerDrain() { + if recBuf.checkIfShouldDrainOrStartLinger() { + recBuf.lockedStopLinger() + recBuf.sink.maybeDrain() + } +} + +// Checks and returns if we should drain; if not, this potentially starts a +// linger timer. +func (recBuf *recBuf) checkIfShouldDrainOrStartLinger() bool { + nbufBatches := len(recBuf.batches) - recBuf.batchDrainIdx + return recBuf.cl.cfg.linger == 0 && nbufBatches > 0 || // no lingering, and any batch exists? drain + nbufBatches > 1 || // lingering, and more than one batch exists? drain -- one is full + nbufBatches == 1 && !recBuf.lockedMaybeLinger() // only one batch; if we cannot start lingering, we are being flushed or have too much buffered and have to drain +} + +// Begins a linger timer unless the producer is being flushed. +func (recBuf *recBuf) lockedMaybeLinger() bool { + if recBuf.cl.producer.flushing.Load() > 0 || recBuf.cl.producer.blocked.Load() > 0 { + return false + } + if !recBuf.isLingering { + recBuf.isLingering = true + if recBuf.lingering == nil { + recBuf.lingering = time.AfterFunc(recBuf.cl.cfg.linger, recBuf.lingerFn) + } else { + recBuf.lingering.Reset(recBuf.cl.cfg.linger) + } + } + return true +} + +func (recBuf *recBuf) lockedStopLinger() { + if recBuf.isLingering { + recBuf.lingering.Stop() + recBuf.isLingering = false + } +} + +func (recBuf *recBuf) unlingerAndManuallyDrain() { + recBuf.mu.Lock() + defer recBuf.mu.Unlock() + recBuf.lockedStopLinger() + recBuf.sink.maybeDrain() +} + +// bumpRepeatedLoadErr is provided to bump a buffer's number of consecutive +// load errors during metadata updates. +// +// Partition load errors are generally temporary (leader/listener/replica not +// available), and this try bump is not expected to do much. If for some reason +// a partition errors for a long time and we are not idempotent, this function +// drops all buffered records. +func (recBuf *recBuf) bumpRepeatedLoadErr(err error) { + recBuf.mu.Lock() + defer recBuf.mu.Unlock() + if len(recBuf.batches) == 0 { + return + } + batch0 := recBuf.batches[0] + + // We need to lock the batch as well because there could be a buffered + // request about to be written. Writing requests only grabs the batch + // mu, not the recBuf mu. + batch0.tries.Add(1) + batch0.mu.Lock() + var ( + canFail = !recBuf.cl.idempotent() || recBuf.cl.cfg.allowIdempotentProduceCancellation || (batch0.canFailFromLoadErrs && !batch0.unsureIfProduced) // we can only fail if we are not idempotent, cancellation is allowed, or if we have no outstanding requests + batch0Fail = batch0.maybeFailErr(&recBuf.cl.cfg) != nil // timeout, retries, or aborting + netErr = isRetryableBrokerErr(err) || isDialNonTimeoutErr(err) // we can fail if this is *not* a network error + retryableKerr = kerr.IsRetriable(err) // we fail if this is not a retryable kerr, + isUnknownLimit = recBuf.checkUnknownFailLimit(err) // or if it is, but it is UnknownTopicOrPartition and we are at our limit + + willFail = canFail && (batch0Fail || !netErr && (!retryableKerr || retryableKerr && isUnknownLimit)) + ) + batch0.isFailingFromLoadErr = willFail + batch0.mu.Unlock() + + recBuf.cl.cfg.logger.Log(LogLevelWarn, "produce partition load error, bumping error count on first stored batch", + "broker", logID(recBuf.sink.nodeID), + "topic", recBuf.topic, + "partition", recBuf.partition, + "err", err, + "can_fail", canFail, + "batch0_should_fail", batch0Fail, + "is_network_err", netErr, + "is_retryable_kerr", retryableKerr, + "is_unknown_limit", isUnknownLimit, + "will_fail", willFail, + ) + + if willFail { + recBuf.failAllRecords(err) + } +} + +// Called locked, if err is an unknown error, bumps our limit, otherwise resets +// it. This returns if we have reached or exceeded the limit. +func (recBuf *recBuf) checkUnknownFailLimit(err error) bool { + if errors.Is(err, kerr.UnknownTopicOrPartition) { + recBuf.unknownFailures++ + } else { + recBuf.unknownFailures = 0 + } + return recBuf.cl.cfg.maxUnknownFailures >= 0 && recBuf.unknownFailures > recBuf.cl.cfg.maxUnknownFailures +} + +// failAllRecords fails all buffered records in this recBuf. +// This is used anywhere where we have to fail and remove an entire batch, +// if we just removed the one batch, the seq num chain would be broken. +// +// - from fatal InitProducerID or AddPartitionsToTxn +// - from client closing +// - if not idempotent && hit retry / timeout limit +// - if batch fails fatally when producing +func (recBuf *recBuf) failAllRecords(err error) { + recBuf.lockedStopLinger() + for _, batch := range recBuf.batches { + // We need to guard our clearing of records against a + // concurrent produceRequest's write, which can have this batch + // buffered wile we are failing. + // + // We do not need to worry about concurrent recBuf + // modifications to this batch because the recBuf is already + // locked. + batch.mu.Lock() + records := batch.records + batch.records = nil + batch.mu.Unlock() + + recBuf.cl.producer.promiseBatch(batchPromise{ + recs: records, + err: err, + }) + } + recBuf.resetBatchDrainIdx() + recBuf.buffered.Store(0) + recBuf.batches = nil +} + +// clearFailing clears a buffer's failing state if it is failing. +// +// This is called when a buffer is added to a sink (to clear a failing state +// from migrating buffers between sinks) or when a metadata update sees the +// sink is still on the same source. +func (recBuf *recBuf) clearFailing() { + recBuf.mu.Lock() + defer recBuf.mu.Unlock() + + recBuf.failing = false + recBuf.maybeTriggerDrain() +} + +func (recBuf *recBuf) resetBatchDrainIdx() { + recBuf.cl.cfg.logger.Log(LogLevelDebug, "rewinding produce sequence to resend pending batches", + "topic", recBuf.topic, + "partition", recBuf.partition, + "rewind_from", recBuf.seq, + "rewind_to", recBuf.batch0Seq, + "pending_batches", len(recBuf.batches), + ) + recBuf.seq = recBuf.batch0Seq + recBuf.batchDrainIdx = 0 +} + +// promisedRec ties a record with the callback that will be called once +// a batch is finally written and receives a response. +type promisedRec struct { + ctx context.Context + promise func(*Record, error) + *Record +} + +func (pr promisedRec) cancelingCtx() context.Context { + if pr.ctx.Done() != nil { + return pr.ctx + } + if pr.Context.Done() != nil { + return pr.Context + } + return nil +} + +// recBatch is the type used for buffering records before they are written. +type recBatch struct { + owner *recBuf // who owns us + + tries atomic.Int64 // how many times this batch has been sent, or should have been sent but requests leading up to it failed (metadata, add partitions to txn, etc) + + // Once this batch is actually selected to be sent in a produce request, + // we freeze it. No more records can be added. + frozen bool + // We can only fail a batch if we have never issued it, or we have + // issued it and have received a response. If we do not receive a + // response, we cannot know whether we actually wrote bytes that Kafka + // processed or not. So, we set this to false every time we issue a + // request with this batch, and then reset it to true whenever we + // process a response. + canFailFromLoadErrs bool + // If we receive a response, but the error code is REQUEST_TIMED_OUT or + // NOT_ENOUGH_REPLICAS_AFTER_APPEND, we actually do not know the state + // of producing this on the broker. Further, we need to persist this + // state: if we produce a second time and receive a different retryable + // error, we need to ensure we do not allow the record to be canceled + // *then*. Once we do not know the state, we need to block cancelation + // until we definitively produce or definitively fail. + unsureIfProduced bool + // If we are going to fail the batch in bumpRepeatedLoadErr, we need to + // set this bool to true. There could be a concurrent request about to + // be written. See more comments below where this is used. + isFailingFromLoadErr bool + + wireLength int32 // tracks total size this batch would currently encode as, including length prefix + v1wireLength int32 // same as wireLength, but for message set v1 + + attrs int16 // updated during apending; read and converted to RecordAttrs on success + firstTimestamp int64 // since unix epoch, in millis + maxTimestampDelta int64 + + mu xsync.Mutex // guards appendTo's reading of records against failAllRecords emptying it + records []promisedRec // record w/ length, ts calculated +} + +// Returns an error if the batch should fail. +func (b *recBatch) maybeFailErr(cfg *cfg) error { + if len(b.records) > 0 { + r0 := &b.records[0] + select { + case <-r0.ctx.Done(): + return r0.ctx.Err() + case <-r0.Context.Done(): + return r0.Context.Err() + default: + } + } + switch { + case b.isTimedOut(cfg.recordTimeout): + return ErrRecordTimeout + case b.tries.Load() > cfg.recordRetries: + return ErrRecordRetries + case b.owner.cl.producer.isAborting(): + return ErrAborting + } + return nil +} + +func (b *recBatch) v0wireLength() int32 { return b.v1wireLength - 8 } // no timestamp +func (b *recBatch) batchLength() int32 { return b.wireLength - 4 } // no length prefix +func (b *recBatch) flexibleWireLength() int32 { // uvarint length prefix + batchLength := b.batchLength() + return int32(kbin.UvarintLen(uvar32(batchLength))) + batchLength +} + +// appendRecord saves a new record to a batch. +// +// This is called under the owning recBuf's mu, meaning records cannot be +// concurrently modified by failing. This batch cannot actively be used +// in a request, so we do not need to worry about a concurrent read. +func (b *recBatch) appendRecord(pr promisedRec, nums recordNumbers) { + b.wireLength += nums.wireLength() + b.v1wireLength += messageSet1Length(pr.Record) + if len(b.records) == 0 { + b.firstTimestamp = pr.Timestamp.UnixNano() / 1e6 + } else if nums.tsDelta > b.maxTimestampDelta { + b.maxTimestampDelta = nums.tsDelta + } + b.records = append(b.records, pr) +} + +// newRecordBatch returns a new record batch for a topic and partition. +func (recBuf *recBuf) newRecordBatch() *recBatch { + const recordBatchOverhead = 4 + // array len + 8 + // firstOffset + 4 + // batchLength + 4 + // partitionLeaderEpoch + 1 + // magic + 4 + // crc + 2 + // attributes + 4 + // lastOffsetDelta + 8 + // firstTimestamp + 8 + // maxTimestamp + 8 + // producerID + 2 + // producerEpoch + 4 + // seq + 4 // record array length + return &recBatch{ + owner: recBuf, + records: recBuf.cl.prsPool.get()[:0], + wireLength: recordBatchOverhead, + + canFailFromLoadErrs: true, // until we send this batch, we can fail it + } +} + +// prsPool is the one pool we have internally that is hard to expose an +// interface for. That said, ideally batch size is relatively consistent +// over time and using our own internal pool is fine enough. +type prsPool struct{ p *sync.Pool } + +func newPrsPool() prsPool { + return prsPool{ + p: &sync.Pool{New: func() any { r := make([]promisedRec, 10); return &r }}, + } +} + +func (p prsPool) get() []promisedRec { return (*p.p.Get().(*[]promisedRec))[:0] } +func (p prsPool) put(s []promisedRec) { p.p.Put(&s) } + +// isOwnersFirstBatch returns if the batch in a recBatch is the first batch in +// a records. We only ever want to update batch / buffer logic if the batch is +// the first in the buffer. +func (b *recBatch) isOwnersFirstBatch() bool { + return len(b.owner.batches) > 0 && b.owner.batches[0] == b +} + +// Returns whether the first record in a batch is past the limit. +func (b *recBatch) isTimedOut(limit time.Duration) bool { + if limit == 0 { + return false + } + return time.Since(b.records[0].Timestamp) > limit +} + +// Decrements the inflight count for this batch. +// +// If the inflight count hits zero, this potentially re-triggers a drain on the +// *current* sink. A concurrent metadata update could have moved the recBuf to +// a different sink; that sink will not drain this recBuf until all requests on +// the old sink are finished. +// +// This is always called in the produce request path, not anywhere else (i.e. +// not failAllRecords). We want inflight decrementing to be the last thing that +// happens always for every request. It does not matter if the records were +// independently failed: from the request issuing perspective, the batch is +// still inflight. +func (b *recBatch) decInflight() { + recBuf := b.owner + recBuf.inflight-- + if recBuf.inflight != 0 { + return + } + recBuf.inflightOnSink = nil + recBuf.maybeTriggerDrain() +} + +//////////////////// +// produceRequest // +//////////////////// + +// produceRequest is a kmsg.Request that is used when we want to +// flush our buffered records. +// +// It is the same as kmsg.ProduceRequest, but with a custom AppendTo. +type produceRequest struct { + version int16 + produceMax int16 // negotiation cap: 11 if !(non-txn || tx890p2), else 13, lowered to 12 if any added recBuf lacks a TopicID + + backoffSeq uint32 + + txnID *string + acks int16 + timeout int32 + batches seqRecBatches + + firstCancelingCtx context.Context // of all batches added, the first one with a record that has a canceling context; only used with disableIdempotency + + producerID int64 + producerEpoch int16 + + // Initialized in AppendTo, metrics tracks uncompressed & compressed + // sizes (in byteS) of each batch. + // + // We use this in handleReqResp for the OnProduceHook. + metrics produceMetrics + + compressor Compressor + + // wireLength is initially the size of sending a produce request, + // including the request header, with no topics. We start with the + // non-flexible size because it is strictly larger than flexible, but + // we use the proper flexible numbers when calculating. + wireLength int32 + wireLengthLimit int32 +} + +type produceMetrics map[string]map[int32]ProduceBatchMetrics + +func (p produceMetrics) hook(cfg *cfg, br *broker) { + if len(p) == 0 { + return + } + var hooks []HookProduceBatchWritten + cfg.hooks.each(func(h Hook) { + if h, ok := h.(HookProduceBatchWritten); ok { + hooks = append(hooks, h) + } + }) + if len(hooks) == 0 { + return + } + go func() { + for _, h := range hooks { + for topic, partitions := range p { + for partition, metrics := range partitions { + h.OnProduceBatchWritten(br.meta, topic, partition, metrics) + } + } + } + }() +} + +func (p *produceRequest) idempotent() bool { return p.producerID >= 0 } + +func (p *produceRequest) tryAddBatch(produceVersion int32, recBuf *recBuf, batch *recBatch) bool { + batchWireLength, flexible, topicIDs := batch.wireLengthForProduceVersion(produceVersion) + batchWireLength += 4 // int32 partition prefix + + if partitions, exists := p.batches.bs[recBuf.topic]; !exists { + if topicIDs { + batchWireLength += 16 + 1 // topic ID size, compact array len for 1 item (if we are using topic IDs, we are definitely flexible) + } else { + lt := int32(len(recBuf.topic)) + if flexible { + batchWireLength += uvarlen(len(recBuf.topic)) + lt + 1 // compact string len, topic, compact array len for 1 item + } else { + batchWireLength += 2 + lt + 4 // string len, topic, partition array len + } + } + } else if flexible { + // If the topic exists and we are flexible, adding this + // partition may increase the length of our size prefix. + lastPartitionsLen := uvarlen(len(partitions)) + newPartitionsLen := uvarlen(len(partitions) + 1) + batchWireLength += (newPartitionsLen - lastPartitionsLen) + } + // If we are flexible but do not know it yet, adding partitions may + // increase our length prefix. Since we are pessimistically assuming + // non-flexible, we have 200mil partitions to add before we have to + // worry about hitting 5 bytes vs. the non-flexible 4. We do not worry. + + if p.wireLength+batchWireLength > p.wireLengthLimit { + return false + } + + if recBuf.batches[0] == batch { + if !p.idempotent() || recBuf.cl.cfg.allowIdempotentProduceCancellation || (batch.canFailFromLoadErrs && !batch.unsureIfProduced) { + if err := batch.maybeFailErr(&batch.owner.cl.cfg); err != nil { + recBuf.failAllRecords(err) + return false + } + } + if recBuf.needSeqReset { + recBuf.cl.cfg.logger.Log(LogLevelDebug, "resetting produce sequence numbers to 0 for new producer epoch", + "topic", recBuf.topic, + "partition", recBuf.partition, + ) + recBuf.needSeqReset = false + recBuf.seq = 0 + recBuf.batch0Seq = 0 + } + } + + batch.frozen = true + p.wireLength += batchWireLength + p.batches.addBatch( + recBuf.topic, + recBuf.topicID, + recBuf.partition, + recBuf.seq, + batch, + ) + return true +} + +// seqRecBatch: a recBatch with a sequence number. +type seqRecBatch struct { + seq int32 + *recBatch +} + +type seqRecBatches struct { + bs map[string]map[int32]seqRecBatch + t2id map[string][16]byte + id2t map[[16]byte]string +} + +func (rbs *seqRecBatches) addBatch(topic string, topicID [16]byte, part, seq int32, batch *recBatch) { + if rbs.bs == nil { + rbs.bs = make(map[string]map[int32]seqRecBatch) + rbs.t2id = make(map[string][16]byte) + rbs.id2t = make(map[[16]byte]string) + } + topicBatches, exists := rbs.bs[topic] + if !exists { + topicBatches = make(map[int32]seqRecBatch, 1) + rbs.bs[topic] = topicBatches + rbs.t2id[topic] = topicID + rbs.id2t[topicID] = topic + } + topicBatches[part] = seqRecBatch{seq, batch} +} + +func (rbs *seqRecBatches) addSeqBatch(topic string, topicID [16]byte, part int32, batch seqRecBatch) { + if rbs.bs == nil { + rbs.bs = make(map[string]map[int32]seqRecBatch) + rbs.t2id = make(map[string][16]byte) + rbs.id2t = make(map[[16]byte]string) + } + topicBatches, exists := rbs.bs[topic] + if !exists { + topicBatches = make(map[int32]seqRecBatch, 1) + rbs.bs[topic] = topicBatches + rbs.t2id[topic] = topicID + rbs.id2t[topicID] = topic + } + topicBatches[part] = batch +} + +func (rbs seqRecBatches) each(fn func(seqRecBatch)) { + for _, partitions := range rbs.bs { + for _, batch := range partitions { + fn(batch) + } + } +} + +func (rbs seqRecBatches) eachOwnerLocked(fn func(seqRecBatch)) { + rbs.each(func(batch seqRecBatch) { + batch.owner.mu.Lock() + defer batch.owner.mu.Unlock() + fn(batch) + }) +} + +func (rbs seqRecBatches) sliced() recBatches { + var batches []*recBatch + for _, partitions := range rbs.bs { + for _, batch := range partitions { + batches = append(batches, batch.recBatch) + } + } + return batches +} + +type recBatches []*recBatch + +func (bs recBatches) eachOwnerLocked(fn func(*recBatch)) { + for _, b := range bs { + b.owner.mu.Lock() + fn(b) + b.owner.mu.Unlock() + } +} + +////////////// +// COUNTING // - this section is all about counting how bytes lay out on the wire +////////////// + +// Returns the non-flexible base produce request length (the request header and +// the request itself with no topics). +// +// See the large comment on maxRecordBatchBytesForTopic for why we always use +// non-flexible (in short: it is strictly larger). +func (cl *Client) baseProduceRequestLength() int32 { + const messageRequestOverhead int32 = 4 + // int32 length prefix + 2 + // int16 key + 2 + // int16 version + 4 + // int32 correlation ID + 2 // int16 client ID len (always non flexible) + // empty tag section skipped; see below + + const produceRequestBaseOverhead int32 = 2 + // int16 transactional ID len (flexible or not, since we cap at 16382) + 2 + // int16 acks + 4 + // int32 timeout + 4 // int32 topics non-flexible array length + // empty tag section skipped; see below + + baseLength := messageRequestOverhead + produceRequestBaseOverhead + if cl.cfg.id != nil { + baseLength += int32(len(*cl.cfg.id)) + } + if cl.cfg.txnID != nil { + baseLength += int32(len(*cl.cfg.txnID)) + } + return baseLength +} + +// Returns the maximum size a record batch can be for this given topic, such +// that if just a **single partition** is fully stuffed with records and we +// only encode that one partition, we will not overflow our configured limits. +// +// The maximum topic length is 249, which has a 2 byte prefix for flexible or +// non-flexible. +// +// Non-flexible versions will have a 4 byte length topic array prefix, a 4 byte +// length partition array prefix. and a 4 byte records array length prefix. +// +// Flexible versions would have a 1 byte length topic array prefix, a 1 byte +// length partition array prefix, up to 5 bytes for the records array length +// prefix, and three empty tag sections resulting in 3 bytes (produce request +// struct, topic struct, partition struct). As well, for the request header +// itself, we have an additional 1 byte tag section (that we currently keep +// empty). +// +// Thus in the worst case, we have 14 bytes of prefixes for non-flexible vs. +// 11 bytes for flexible. We default to the more limiting size: non-flexible. +func (cl *Client) maxRecordBatchBytesForTopic(topic string) int32 { + topicLen := 16 // topic ID length, if using topic IDs + if len(topic) > topicLen { + topicLen = len(topic) + } + minOnePartitionBatchLength := cl.baseProduceRequestLength() + + 2 + // int16 topic string length prefix length + int32(topicLen) + + 4 + // int32 partitions array length + 4 + // partition int32 encoding length + 4 // int32 record bytes array length + + wireLengthLimit := cl.cfg.maxBrokerWriteBytes + + recordBatchLimit := wireLengthLimit - minOnePartitionBatchLength + if cfgLimit := cl.cfg.maxRecordBatchBytes(topic); cfgLimit < recordBatchLimit { + recordBatchLimit = cfgLimit + } + return recordBatchLimit +} + +func messageSet0Length(r *Record) int32 { + const length = 4 + // array len + 8 + // offset + 4 + // size + 4 + // crc + 1 + // magic + 1 + // attributes + 4 + // key array bytes len + 4 // value array bytes len + return length + int32(len(r.Key)) + int32(len(r.Value)) +} + +func messageSet1Length(r *Record) int32 { + return messageSet0Length(r) + 8 // timestamp +} + +// Returns the numbers for a record if it were added to the record batch. +func (b *recBatch) calculateRecordNumbers(r *Record) recordNumbers { + tsMillis := r.Timestamp.UnixNano() / 1e6 + tsDelta := tsMillis - b.firstTimestamp + + // If this is to be the first record in the batch, then our timestamp + // delta is actually 0. + if len(b.records) == 0 { + tsDelta = 0 + } + + offsetDelta := int32(len(b.records)) // since called before adding record, delta is the current end + + l := 1 + // attributes, int8 unused + kbin.VarlongLen(tsDelta) + + kbin.VarintLen(offsetDelta) + + kbin.VarintLen(int32(len(r.Key))) + + len(r.Key) + + kbin.VarintLen(int32(len(r.Value))) + + len(r.Value) + + kbin.VarintLen(int32(len(r.Headers))) // varint array len headers + + for _, h := range r.Headers { + l += kbin.VarintLen(int32(len(h.Key))) + + len(h.Key) + + kbin.VarintLen(int32(len(h.Value))) + + len(h.Value) + } + + return recordNumbers{ + lengthField: int32(l), + tsDelta: tsDelta, + } +} + +func uvar32(l int32) uint32 { return 1 + uint32(l) } +func uvarlen(l int) int32 { return int32(kbin.UvarintLen(uvar32(int32(l)))) } + +// recordNumbers tracks a few numbers for a record that is buffered. +type recordNumbers struct { + lengthField int32 // the length field prefix of a record encoded on the wire + tsDelta int64 // the ms delta of when the record was added against the first timestamp +} + +// wireLength is the wire length of a record including its length field prefix. +func (n recordNumbers) wireLength() int32 { + return int32(kbin.VarintLen(n.lengthField)) + n.lengthField +} + +func (b *recBatch) wireLengthForProduceVersion(v int32) (batchWireLength int32, flexible, topicIDs bool) { + batchWireLength = b.wireLength + + // If we do not yet know the produce version, we default to the largest + // size. Our request building sizes will always be an overestimate. + if v < 0 { + v1BatchWireLength := b.v1wireLength + if v1BatchWireLength > batchWireLength { + batchWireLength = v1BatchWireLength + } + flexibleBatchWireLength := b.flexibleWireLength() + if flexibleBatchWireLength > batchWireLength { + batchWireLength = flexibleBatchWireLength + } + } else { + switch v { + case 0, 1: + batchWireLength = b.v0wireLength() + case 2: + batchWireLength = b.v1wireLength + case 3, 4, 5, 6, 7, 8: + batchWireLength = b.wireLength + default: + batchWireLength = b.flexibleWireLength() + flexible = true + topicIDs = v >= 13 + } + } + + return batchWireLength, flexible, topicIDs +} + +func (b *recBatch) tryBuffer(pr promisedRec, produceVersion, maxBatchBytes int32, abortOnNewBatch bool) (appended, aborted bool) { + nums := b.calculateRecordNumbers(pr.Record) + + batchWireLength, _, _ := b.wireLengthForProduceVersion(produceVersion) + newBatchLength := batchWireLength + nums.wireLength() + + if b.frozen || newBatchLength > maxBatchBytes { + return false, false + } + if abortOnNewBatch { + return false, true + } + b.appendRecord(pr, nums) + pr.setLengthAndTimestampDelta( + nums.lengthField, + nums.tsDelta, + ) + return true, false +} + +////////////// +// ENCODING // - this section is all about actually writing a produce request +////////////// + +func (*produceRequest) Key() int16 { return 0 } + +func (p *produceRequest) MaxVersion() int16 { return p.produceMax } +func (p *produceRequest) SetVersion(v int16) { p.version = v } +func (p *produceRequest) GetVersion() int16 { return p.version } +func (p *produceRequest) IsFlexible() bool { return p.version >= 9 } +func (p *produceRequest) AppendTo(dst []byte) []byte { + flexible := p.IsFlexible() + + p.metrics = make(map[string]map[int32]ProduceBatchMetrics) + + if p.version >= 3 { + if flexible { + dst = kbin.AppendCompactNullableString(dst, p.txnID) + } else { + dst = kbin.AppendNullableString(dst, p.txnID) + } + } + + dst = kbin.AppendInt16(dst, p.acks) + dst = kbin.AppendInt32(dst, p.timeout) + if flexible { + dst = kbin.AppendCompactArrayLen(dst, len(p.batches.bs)) + } else { + dst = kbin.AppendArrayLen(dst, len(p.batches.bs)) + } + + for topic, partitions := range p.batches.bs { + if p.version >= 13 { + id := p.batches.t2id[topic] + dst = append(dst, id[:]...) + dst = kbin.AppendCompactArrayLen(dst, len(partitions)) + } else if flexible { + dst = kbin.AppendCompactString(dst, topic) + dst = kbin.AppendCompactArrayLen(dst, len(partitions)) + } else { + dst = kbin.AppendString(dst, topic) + dst = kbin.AppendArrayLen(dst, len(partitions)) + } + + tmetrics := make(map[int32]ProduceBatchMetrics) + p.metrics[topic] = tmetrics + + for partition, batch := range partitions { + dst = kbin.AppendInt32(dst, partition) + batch.mu.Lock() + if batch.records == nil || batch.isFailingFromLoadErr { // concurrent failAllRecords OR concurrent bumpRepeatedLoadErr + if flexible { + dst = kbin.AppendCompactNullableBytes(dst, nil) + } else { + dst = kbin.AppendNullableBytes(dst, nil) + } + batch.mu.Unlock() + continue + } + // canFailFromLoadErrs is a monotonic one-way mutation + // (true -> false, never the other way). It is safe inside + // AppendTo even under the broker-level zero-bytes-written + // retry path at broker.go, because the retry re-invokes + // AppendTo on the same request; the second call sets the + // same value with no behavior change. Any mutation in + // AppendTo that affects the serialized bytes must satisfy + // the same monotonicity criterion, or it must move out of + // AppendTo (see tries, moved to handleReqResp). + batch.canFailFromLoadErrs = false // we are going to write this batch: the response status is now unknown + var pmetrics ProduceBatchMetrics + if p.version < 3 { + dst, pmetrics = batch.appendToAsMessageSet(dst, uint8(p.version), p.compressor) + } else { + dst, pmetrics = batch.appendTo(dst, p.version, p.producerID, p.producerEpoch, p.txnID != nil, p.compressor) + } + batch.mu.Unlock() + tmetrics[partition] = pmetrics + if flexible { + dst = append(dst, 0) + } + } + if flexible { + dst = append(dst, 0) + } + } + if flexible { + dst = append(dst, 0) + } + + return dst +} + +func (*produceRequest) ReadFrom([]byte) error { + panic("unreachable -- the client never uses ReadFrom on its internal produceRequest") +} + +func (p *produceRequest) ResponseKind() kmsg.Response { + r := kmsg.NewPtrProduceResponse() + r.Version = p.version + return r +} + +func (b seqRecBatch) appendTo( + in []byte, + version int16, + producerID int64, + producerEpoch int16, + transactional bool, + compressor Compressor, +) (dst []byte, m ProduceBatchMetrics) { // named return so that our defer for flexible versions can modify it + flexible := version >= 9 + dst = in + nullableBytesLen := b.wireLength - 4 // NULLABLE_BYTES leading length, minus itself + nullableBytesLenAt := len(dst) // in case compression adjusting + dst = kbin.AppendInt32(dst, nullableBytesLen) + + // With flexible versions, the array length prefix can be anywhere from + // 1 byte long to 5 bytes long (covering up to 268MB). + // + // We have to add our initial understanding of the array length as a + // uvarint, but if compressing shrinks what that length would encode + // as, we have to shift everything down. + if flexible { + dst = dst[:nullableBytesLenAt] + batchLength := b.batchLength() + dst = kbin.AppendUvarint(dst, uvar32(batchLength)) // compact array non-null prefix + batchAt := len(dst) + defer func() { + batch := dst[batchAt:] + if int32(len(batch)) == batchLength { // we did not compress: simply return + return + } + + // We *only* could have shrunk the batch bytes, so our + // append here will not overwrite anything we need to + // keep. + newDst := kbin.AppendUvarint(dst[:nullableBytesLenAt], uvar32(int32(len(batch)))) + + // If our append did not shorten the length prefix, we + // can just return the prior dst, otherwise we have to + // shift the batch itself down on newDst. + if len(newDst) != batchAt { + dst = append(newDst, batch...) + } + }() + } + + // Below here, we append the actual record batch, which cannot be + // flexible. Everything encodes properly; flexible adjusting is done in + // the defer just above. + + dst = kbin.AppendInt64(dst, 0) // firstOffset, defined as zero for producing + + batchLen := nullableBytesLen - 8 - 4 // length of what follows this field (so, minus what came before and ourself) + batchLenAt := len(dst) // in case compression adjusting + dst = kbin.AppendInt32(dst, batchLen) + + dst = kbin.AppendInt32(dst, -1) // partitionLeaderEpoch, unused in clients + dst = kbin.AppendInt8(dst, 2) // magic, defined as 2 for records v0.11.0+ + + crcStart := len(dst) // fill at end + dst = kbin.AppendInt32(dst, 0) // reserved crc + + attrsAt := len(dst) // in case compression adjusting + b.attrs = 0 + if transactional { + b.attrs |= 0x0010 // bit 5 is the "is transactional" bit + } + dst = kbin.AppendInt16(dst, b.attrs) + dst = kbin.AppendInt32(dst, int32(len(b.records)-1)) // lastOffsetDelta + dst = kbin.AppendInt64(dst, b.firstTimestamp) + dst = kbin.AppendInt64(dst, b.firstTimestamp+b.maxTimestampDelta) + + seq := b.seq + if producerID < 0 { // a negative producer ID means we are not using idempotence + seq = 0 + } + dst = kbin.AppendInt64(dst, producerID) + dst = kbin.AppendInt16(dst, producerEpoch) + dst = kbin.AppendInt32(dst, seq) + + dst = kbin.AppendArrayLen(dst, len(b.records)) + recordsAt := len(dst) + for i, pr := range b.records { + dst = pr.appendTo(dst, int32(i)) + } + + toCompress := dst[recordsAt:] + m.NumRecords = len(b.records) + m.UncompressedBytes = len(toCompress) + m.CompressedBytes = m.UncompressedBytes + + if compressor != nil { + w := byteBuffers.Get().(*bytes.Buffer) + defer byteBuffers.Put(w) + w.Reset() + + compressed, codec := compressor.Compress(w, toCompress, mkCompressFlags(version)...) + if compressed != nil && // nil would be from an error + len(compressed) < len(toCompress) { + // our compressed was shorter: copy over + copy(dst[recordsAt:], compressed) + dst = dst[:recordsAt+len(compressed)] + m.CompressedBytes = len(compressed) + m.CompressionType = uint8(codec) + + // update the few record batch fields we already wrote + savings := int32(len(toCompress) - len(compressed)) + nullableBytesLen -= savings + batchLen -= savings + b.attrs |= int16(codec) + if !flexible { + kbin.AppendInt32(dst[:nullableBytesLenAt], nullableBytesLen) + } + kbin.AppendInt32(dst[:batchLenAt], batchLen) + kbin.AppendInt16(dst[:attrsAt], b.attrs) + } + } + + kbin.AppendInt32(dst[:crcStart], int32(crc32.Checksum(dst[crcStart+4:], crc32c))) + + return dst, m +} + +func (pr promisedRec) appendTo(dst []byte, offsetDelta int32) []byte { + length, tsDelta := pr.lengthAndTimestampDelta() + dst = kbin.AppendVarint(dst, length) + dst = kbin.AppendInt8(dst, 0) // attributes, currently unused + dst = kbin.AppendVarlong(dst, tsDelta) + dst = kbin.AppendVarint(dst, offsetDelta) + dst = kbin.AppendVarintBytes(dst, pr.Key) + dst = kbin.AppendVarintBytes(dst, pr.Value) + dst = kbin.AppendVarint(dst, int32(len(pr.Headers))) + for _, h := range pr.Headers { + dst = kbin.AppendVarintString(dst, h.Key) + dst = kbin.AppendVarintBytes(dst, h.Value) + } + return dst +} + +func (b seqRecBatch) appendToAsMessageSet(dst []byte, version uint8, compressor Compressor) ([]byte, ProduceBatchMetrics) { + var m ProduceBatchMetrics + + nullableBytesLenAt := len(dst) + dst = append(dst, 0, 0, 0, 0) // nullable bytes len + for i, pr := range b.records { + _, tsDelta := pr.lengthAndTimestampDelta() + dst = appendMessageTo( + dst, + version, + 0, + int64(i), + b.firstTimestamp+tsDelta, + pr.Record, + ) + } + + b.attrs = 0 + + // Produce request v0 and v1 uses message set v0, which does not have + // timestamps. We set bit 8 in our attrs which corresponds with our own + // kgo.RecordAttrs's bit. The attrs field is unused in a sink / recBuf + // outside of the appending functions or finishing records; if we use + // more bits in our internal RecordAttrs, the below will need to + // change. + if version == 0 || version == 1 { + b.attrs |= 0b1000_0000 + } + + toCompress := dst[nullableBytesLenAt+4:] // skip nullable bytes leading prefix + m.NumRecords = len(b.records) + m.UncompressedBytes = len(toCompress) + m.CompressedBytes = m.UncompressedBytes + + if compressor != nil { + w := byteBuffers.Get().(*bytes.Buffer) + defer byteBuffers.Put(w) + w.Reset() + + compressed, codec := compressor.Compress(w, toCompress, mkCompressFlags(int16(version))...) + inner := &Record{Value: compressed} + wrappedLength := messageSet0Length(inner) + if version == 2 { + wrappedLength += 8 // timestamp + } + + if compressed != nil && int(wrappedLength) < len(toCompress) { + m.CompressedBytes = int(wrappedLength) + m.CompressionType = uint8(codec) + + b.attrs |= int16(codec) + + dst = appendMessageTo( + dst[:nullableBytesLenAt+4], + version, + int8(codec), + int64(len(b.records)-1), + b.firstTimestamp, + inner, + ) + } + } + + kbin.AppendInt32(dst[:nullableBytesLenAt], int32(len(dst[nullableBytesLenAt+4:]))) + return dst, m +} + +func appendMessageTo( + dst []byte, + version uint8, + attributes int8, + offset int64, + timestamp int64, + r *Record, +) []byte { + magic := version >> 1 + dst = kbin.AppendInt64(dst, offset) + msgSizeStart := len(dst) + dst = append(dst, 0, 0, 0, 0) + crc32Start := len(dst) + dst = append(dst, + 0, 0, 0, 0, + magic, + byte(attributes)) + if magic == 1 { + dst = kbin.AppendInt64(dst, timestamp) + } + dst = kbin.AppendNullableBytes(dst, r.Key) + dst = kbin.AppendNullableBytes(dst, r.Value) + kbin.AppendInt32(dst[:crc32Start], int32(crc32.ChecksumIEEE(dst[crc32Start+4:]))) + kbin.AppendInt32(dst[:msgSizeStart], int32(len(dst[msgSizeStart+4:]))) + return dst +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/source.go b/vendor/github.com/twmb/franz-go/pkg/kgo/source.go new file mode 100644 index 00000000000..4266cbea2d5 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/source.go @@ -0,0 +1,2771 @@ +package kgo + +import ( + "cmp" + "context" + "encoding/binary" + "fmt" + "hash/crc32" + "slices" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/twmb/franz-go/pkg/kbin" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +type readerFrom interface { + ReadFrom([]byte) error +} + +// A source consumes from an individual broker. +// +// As long as there is at least one active cursor, a source aims to have *one* +// buffered fetch at all times. As soon as the fetch is taken, a source issues +// another fetch in the background. +type source struct { + cl *Client // our owning client, for cfg, metadata triggering, context, etc. + nodeID int32 // the node ID of the broker this sink belongs to + + // Tracks how many _failed_ fetch requests we have in a row (unable to + // receive a response). Any response, even responses with an ErrorCode + // set, are successful. This field is used for backoff purposes. + consecutiveFailures int + + fetchState workLoop + sem chan struct{} // closed when fetchable, recreated when a buffered fetch exists + buffered bufferedFetch // contains a fetch the source has buffered for polling + + session fetchSession // supports fetch sessions as per KIP-227 + + cursorsMu xsync.Mutex + cursors []*cursor // contains all partitions being consumed on this source + cursorsStart int // incremented every fetch req to ensure all partitions are fetched + + share sourceShare +} + +type sourceShare struct { + s *source // back-pointer for hook, sem, maybeShareConsume + sc *shareConsumer + + mu xsync.Mutex + cursors []*shareCursor + cursorsStart int + sessionEpoch int32 // 0=new, incremented on success, -1=close + sessionParts map[tidp]struct{} // broker-confirmed session partitions; createShareFetchReq diffs WANT against this to compute add/forget + ackCh chan struct{} // acks pending, batch on timer + ackFlushCh chan struct{} // flush acks immediately + buffered shareBufferedFetch // decoded share fetch waiting for poll +} + +func (cl *Client) newSource(nodeID int32) *source { + s := &source{ + cl: cl, + nodeID: nodeID, + sem: make(chan struct{}), + } + if cl.cfg.disableFetchSessions { + s.session.kill() + } + s.share.s = s + s.share.sc = cl.consumer.s + s.share.ackCh = make(chan struct{}, 1) + s.share.ackFlushCh = make(chan struct{}, 1) + s.share.sessionParts = make(map[tidp]struct{}) + close(s.sem) + return s +} + +func (s *source) addCursor(add *cursor) { + s.cursorsMu.Lock() + add.cursorsIdx = len(s.cursors) + s.cursors = append(s.cursors, add) + s.cursorsMu.Unlock() + + // Adding a new cursor may allow a new partition to be fetched. + // We do not need to cancel any current fetch nor kill the session, + // since adding a cursor is non-destructive to work in progress. + // If the session is currently stopped, this is a no-op. + s.maybeConsume() +} + +// Removes a cursor from the source. +// +// The caller should do this with a stopped session if necessary, which +// should clear any buffered fetch and reset the source's session. +func (s *source) removeCursor(rm *cursor) { + s.cursorsMu.Lock() + defer s.cursorsMu.Unlock() + + if rm.cursorsIdx != len(s.cursors)-1 { + s.cursors[rm.cursorsIdx], s.cursors[len(s.cursors)-1] = s.cursors[len(s.cursors)-1], nil + s.cursors[rm.cursorsIdx].cursorsIdx = rm.cursorsIdx + } else { + s.cursors[rm.cursorsIdx] = nil // do not let the memory hang around + } + + s.cursors = s.cursors[:len(s.cursors)-1] + if s.cursorsStart == len(s.cursors) { + s.cursorsStart = 0 + } +} + +// cursor is where we are consuming from for an individual partition. +type cursor struct { + topic string + topicID [16]byte + partition int32 + + unknownIDFails atomic.Int32 + + keepControl bool // whether to keep control records + + cursorsIdx int // updated under source mutex + + // The source we are currently on. This is modified in two scenarios: + // + // * by metadata when the consumer session is completely stopped + // + // * by a fetch when handling a fetch response that returned preferred + // replicas + // + // This is additionally read within a session when cursor is + // transitioning from used to usable. + source *source + + // If this cursor moves to a preferred replica: unix nano of when. + // This is set in `move`, and read when handling a response. Both are + // independent events in a live session. If we have been consuming from + // the preferred replica for more than RecheckPreferredReplicaInterval, + // we fetch from the original leader again. + moveAt int64 + + // useState is an atomic that has two states: unusable and usable. A + // cursor can be used in a fetch request if it is in the usable state. + // Once used, the cursor is unusable, and will be set back to usable + // one the request lifecycle is complete (a usable fetch response, or + // once listing offsets or loading epochs completes). + // + // A cursor can be set back to unusable when sources are stopped. This + // can be done if a group loses a partition, for example. + // + // The used state is exclusively updated by either building a fetch + // request or when the source is stopped. + useState atomic.Bool + + topicPartitionData // updated in metadata when session is stopped + + // cursorOffset is our epoch/offset that we are consuming. When a fetch + // request is issued, we "freeze" a view of the offset and of the + // leader epoch (see cursorOffsetNext for why the leader epoch). When a + // buffered fetch is taken, we update the cursor. + cursorOffset +} + +// cursorOffset tracks offsets/epochs for a cursor. +type cursorOffset struct { + // What the cursor is at: we request this offset next. + offset int64 + + // The epoch of the last record we consumed. Also used for KIP-320, if + // we are fenced or we have an offset out of range error, we go into + // the OffsetForLeaderEpoch recovery. The last consumed epoch tells the + // broker which offset we want: either (a) the next offset if the last + // consumed epoch is the current epoch, or (b) the offset of the first + // record in the next epoch. This allows for exact offset resetting and + // data loss detection. + // + // See kmsg.OffsetForLeaderEpochResponseTopicPartition for more + // details. + lastConsumedEpoch int32 + + // If we receive OFFSET_OUT_OF_RANGE, and we previously *know* we + // consumed an offset, we reset to the nearest offset after our prior + // known valid consumed offset. + lastConsumedTime time.Time + + // The current high watermark of the partition. Uninitialized (0) means + // we do not know the HWM, or there is no lag. + hwm int64 +} + +// use, for fetch requests, freezes a view of the cursorOffset. +func (c *cursor) use() *cursorOffsetNext { + // A source using a cursor has exclusive access to the use field by + // virtue of that source building a request during a live session, + // or by virtue of the session being stopped. + c.useState.Store(false) + return &cursorOffsetNext{ + cursorOffset: c.cursorOffset, + from: c, + currentLeaderEpoch: c.leaderEpoch, + } +} + +// unset transitions a cursor to an unusable state when the cursor is no longer +// to be consumed. This is called exclusively after sources are stopped. +// This also unsets the cursor offset, which is assumed to be unused now. +func (c *cursor) unset() { + c.useState.Store(false) + c.setOffset(cursorOffset{ + offset: -1, + lastConsumedEpoch: -1, + hwm: 0, + }) +} + +// usable returns whether a cursor can be used for building a fetch request. +func (c *cursor) usable() bool { + return c.useState.Load() +} + +// allowUsable allows a cursor to be fetched, and is called either in assigning +// offsets, or when a buffered fetch is taken or discarded, or when listing / +// epoch loading finishes. +// +// We capture c.source before Swap because Swap makes this cursor immediately +// eligible for fetching. With kfake (in-process), a fetch can complete and +// move() can overwrite c.source before we reach maybeConsume. +func (c *cursor) allowUsable() { + s := c.source + c.useState.Swap(true) + s.maybeConsume() +} + +// setOffset sets the cursors offset which will be used the next time a fetch +// request is built. This function is called under the source mutex while the +// source is stopped, and the caller is responsible for calling maybeConsume +// after. +func (c *cursor) setOffset(o cursorOffset) { + c.cursorOffset = o +} + +// cursorOffsetNext is updated while processing a fetch response. +// +// When a buffered fetch is taken, we update a cursor with the final values in +// the modified cursor offset. +type cursorOffsetNext struct { + cursorOffset + from *cursor + + // The leader epoch at the time we took this cursor offset snapshot. We + // need to copy this rather than accessing it through `from` because a + // fetch request can be canceled while it is being written (and reading + // the epoch). + // + // The leader field itself is only read within the context of a session + // while the session is alive, thus it needs no such guard. + // + // Basically, any field read in AppendTo needs to be copied into + // cursorOffsetNext. + currentLeaderEpoch int32 +} + +type cursorOffsetPreferred struct { + cursorOffsetNext + preferredReplica int32 + ooor bool + recheck bool +} + +// Moves a cursor from one source to another. This is done while handling a +// fetch response or creating a fetch request, which means within the context +// of a live session. +// +// NOTE: We cannot add the cursor to the new source until AFTER we are done +// modifying everything on the cursor. FURTHER, we cannot add the cursor to the +// new source until we are done accessing *any* field on the cursor. As soon +// as we add the cursor to the new source, it is eligible for use in a new +// fetch request -- and, pathologically, the cursor could move again and the +// source can be changed again (and at that point, all bets are off and there +// can be some weird access patterns and modifying of fields that leads to +// races or crashes). Point is: remove, modify, add. Do not modify after add. +// See #1167. +func (p *cursorOffsetPreferred) move() { + c := p.from + + // Before we migrate the cursor, we check if the destination source + // exists. If not, we do not migrate and instead force a metadata. + + c.source.cl.sinksAndSourcesMu.Lock() + sns, exists := c.source.cl.sinksAndSources[p.preferredReplica] + c.source.cl.sinksAndSourcesMu.Unlock() + + if !exists { + c.source.cl.triggerUpdateMetadataNow("cursor moving to a different broker that is not yet known") + return + } + + c.source.removeCursor(c) + c.source = sns.source + c.moveAt = time.Now().UnixNano() + c.useState.Swap(true) + c.source.addCursor(c) +} + +type cursorPreferreds []cursorOffsetPreferred + +func (cs cursorPreferreds) String() string { + type pnext struct { + p int32 + next int32 + ooor bool + recheck bool + } + ts := make(map[string][]pnext) + for _, c := range cs { + t := c.from.topic + p := c.from.partition + ts[t] = append(ts[t], pnext{p, c.preferredReplica, c.ooor, c.recheck}) + } + tsorted := make([]string, 0, len(ts)) + for t, ps := range ts { + tsorted = append(tsorted, t) + slices.SortFunc(ps, func(l, r pnext) int { + return cmp.Or(cmp.Compare(l.p, r.p), cmp.Compare(l.next, r.next)) + }) + } + slices.Sort(tsorted) + + sb := new(strings.Builder) + for i, t := range tsorted { + ps := ts[t] + fmt.Fprintf(sb, "%s{", t) + + for j, p := range ps { + if j < len(ps)-1 { + if p.ooor { + fmt.Fprintf(sb, "p%d=>b%d[ooor], ", p.p, p.next) + } else if p.recheck { + fmt.Fprintf(sb, "p%d=>b%d[recheck], ", p.p, p.next) + } else { + fmt.Fprintf(sb, "p%d=>b%d, ", p.p, p.next) + } + } else { + if p.ooor { + fmt.Fprintf(sb, "p%d=>b%d[ooor]", p.p, p.next) + } else if p.recheck { + fmt.Fprintf(sb, "p%d=>b%d[recheck]", p.p, p.next) + } else { + fmt.Fprintf(sb, "p%d=>b%d", p.p, p.next) + } + } + } + + if i < len(tsorted)-1 { + fmt.Fprint(sb, "}, ") + } else { + fmt.Fprint(sb, "}") + } + } + return sb.String() +} + +func (cs cursorPreferreds) eachPreferred(fn func(cursorOffsetPreferred)) { + for _, c := range cs { + fn(c) + } +} + +type usedOffsets map[string]map[int32]*cursorOffsetNext + +func (os usedOffsets) eachOffset(fn func(*cursorOffsetNext)) { + for _, ps := range os { + for _, o := range ps { + fn(o) + } + } +} + +func (os usedOffsets) finishUsingAllWithSet() { + os.eachOffset(func(o *cursorOffsetNext) { o.from.setOffset(o.cursorOffset); o.from.allowUsable() }) +} + +func (os usedOffsets) finishUsingAll() { + os.eachOffset(func(o *cursorOffsetNext) { o.from.allowUsable() }) +} + +// bufferedFetch is a fetch response waiting to be consumed by the client. +type bufferedFetch struct { + fetch Fetch + + doneFetch chan<- bool // when unbuffered, we send down this + usedOffsets usedOffsets // what the offsets will be next if this fetch is used +} + +func (s *source) hook(f *Fetch, buffered, polled bool) { + // Collect matching hooks once, then fuse dispatch with the metrics walk + // so we visit each record exactly once regardless of hook count. + var ( + bufH []HookFetchRecordBuffered + unbH []HookFetchRecordUnbuffered + ) + s.cl.cfg.hooks.each(func(h Hook) { + if buffered { + if h, ok := h.(HookFetchRecordBuffered); ok { + bufH = append(bufH, h) + } + } else { + if h, ok := h.(HookFetchRecordUnbuffered); ok { + unbH = append(unbH, h) + } + } + }) + + var nrecs int + var nbytes int64 + for i := range f.Topics { + t := &f.Topics[i] + for j := range t.Partitions { + p := &t.Partitions[j] + nrecs += len(p.Records) + for k := range p.Records { + r := p.Records[k] + nbytes += r.userSize() + for _, h := range bufH { + h.OnFetchRecordBuffered(r) + } + for _, h := range unbH { + h.OnFetchRecordUnbuffered(r, polled) + } + } + } + } + if buffered { + s.cl.consumer.bufferedRecords.Add(int64(nrecs)) + s.cl.consumer.bufferedBytes.Add(nbytes) + } else { + s.cl.consumer.bufferedRecords.Add(-int64(nrecs)) + s.cl.consumer.bufferedBytes.Add(-nbytes) + } +} + +// takeBuffered drains a buffered fetch and updates offsets. +func (s *source) takeBuffered(paused pausedTopics) Fetch { + if len(paused) == 0 { + return s.takeBufferedFn(true, usedOffsets.finishUsingAllWithSet) + } + var strip map[string]map[int32]struct{} + f := s.takeBufferedFn(true, func(os usedOffsets) { + for t, ps := range os { + // If the entire topic is paused, we allowUsable all + // and strip the topic entirely. + pps, ok := paused.t(t) + if !ok { + for _, o := range ps { + o.from.setOffset(o.cursorOffset) + o.from.allowUsable() + } + continue + } + if strip == nil { + strip = make(map[string]map[int32]struct{}) + } + if pps.all { + for _, o := range ps { + o.from.allowUsable() + } + strip[t] = nil // initialize key, for existence-but-len-0 check below + continue + } + stript := make(map[int32]struct{}) + for _, o := range ps { + if _, ok := pps.m[o.from.partition]; ok { + o.from.allowUsable() + stript[o.from.partition] = struct{}{} + continue + } + o.from.setOffset(o.cursorOffset) + o.from.allowUsable() + } + // We only add stript to strip if there are any + // stripped partitions. We could have a paused + // partition that is on another broker, while this + // broker has no paused partitions -- if we add stript + // here, our logic below (stripping this entire topic) + // is more confusing (present nil vs. non-present nil). + if len(stript) > 0 { + strip[t] = stript + } + } + }) + if strip != nil { + keep := f.Topics[:0] + for _, t := range f.Topics { + stript, ok := strip[t.Topic] + if ok { + if len(stript) == 0 { + continue // stripping this entire topic + } + keepp := t.Partitions[:0] + for _, p := range t.Partitions { + if _, ok := stript[p.Partition]; ok { + continue + } + keepp = append(keepp, p) + } + t.Partitions = keepp + } + keep = append(keep, t) + } + f.Topics = keep + } + return f +} + +func (s *source) discardBuffered() { + s.takeBufferedFn(false, usedOffsets.finishUsingAll) +} + +// takeNBuffered takes a limited amount of records from a buffered fetch, +// updating offsets in each partition per records taken. +// +// This only allows a new fetch once every buffered record has been taken. +// +// This returns the number of records taken and whether the source has been +// completely drained. +func (s *source) takeNBuffered(paused pausedTopics, n int) (Fetch, int, bool) { + var ( + r Fetch + rstrip Fetch + taken int + ) + + b := &s.buffered + bf := &b.fetch + for len(bf.Topics) > 0 && n > 0 { + t := &bf.Topics[0] + tCursors := b.usedOffsets[t.Topic] + + // If the topic is outright paused, we allowUsable all + // partitions in the topic and skip the topic entirely. + if paused.has(t.Topic, -1) { + rstrip.Topics = append(rstrip.Topics, *t) + bf.Topics = bf.Topics[1:] + for _, pCursor := range tCursors { + pCursor.from.allowUsable() + } + delete(b.usedOffsets, t.Topic) + continue + } + + var rt *FetchTopic + ensureTopicAdded := func() { + if rt != nil { + return + } + r.Topics = append(r.Topics, *t) + rt = &r.Topics[len(r.Topics)-1] + rt.Partitions = nil + } + var rtstrip *FetchTopic + ensureTopicStripped := func() { + if rtstrip != nil { + return + } + rstrip.Topics = append(rstrip.Topics, *t) + rtstrip = &rstrip.Topics[len(rstrip.Topics)-1] + rtstrip.Partitions = nil + } + + for len(t.Partitions) > 0 && n > 0 { + p := &t.Partitions[0] + + if paused.has(t.Topic, p.Partition) { + ensureTopicStripped() + rtstrip.Partitions = append(rtstrip.Partitions, *p) + t.Partitions = t.Partitions[1:] + pCursor := tCursors[p.Partition] + pCursor.from.allowUsable() + delete(tCursors, p.Partition) + if len(tCursors) == 0 { + delete(b.usedOffsets, t.Topic) + } + continue + } + + ensureTopicAdded() + rt.Partitions = append(rt.Partitions, *p) + rp := &rt.Partitions[len(rt.Partitions)-1] + + take := min(n, len(p.Records)) + + rp.Records = p.Records[:take:take] + p.Records = p.Records[take:] + + n -= take + taken += take + + pCursor := tCursors[p.Partition] + + if len(p.Records) == 0 { + t.Partitions = t.Partitions[1:] + + pCursor.from.setOffset(pCursor.cursorOffset) + pCursor.from.allowUsable() + delete(tCursors, p.Partition) + if len(tCursors) == 0 { + delete(b.usedOffsets, t.Topic) + } + continue + } + + lastReturnedRecord := rp.Records[len(rp.Records)-1] + pCursor.from.setOffset(cursorOffset{ + offset: lastReturnedRecord.Offset + 1, + lastConsumedEpoch: lastReturnedRecord.LeaderEpoch, + lastConsumedTime: lastReturnedRecord.Timestamp, + hwm: p.HighWatermark, + }) + } + + if len(t.Partitions) == 0 { + bf.Topics = bf.Topics[1:] + } + } + + if len(rstrip.Topics) > 0 { + s.hook(&rstrip, false, true) + } + s.hook(&r, false, true) // unbuffered, polled + + drained := len(bf.Topics) == 0 + if drained { + s.takeBuffered(nil) + } + return r, taken, drained +} + +func (s *source) takeBufferedFn(polled bool, offsetFn func(usedOffsets)) Fetch { + r := s.buffered + s.buffered = bufferedFetch{} + offsetFn(r.usedOffsets) + r.doneFetch <- true + close(s.sem) + + s.hook(&r.fetch, false, polled) // unbuffered, potentially polled + + return r.fetch +} + +// createReq actually creates a fetch request. +func (s *source) createReq() *fetchRequest { + req := &fetchRequest{ + maxWait: s.cl.cfg.maxWait, + minBytes: s.cl.cfg.minBytes, + maxBytes: s.cl.cfg.maxBytes.load(), + maxPartBytes: s.cl.cfg.maxPartBytes.load(), + rack: s.cl.cfg.rack, + isolationLevel: s.cl.cfg.isolationLevel, + preferLagFn: s.cl.cfg.preferLagFn, + + // We copy a view of the session for the request, which allows + // modify source while the request may be reading its copy. + session: s.session, + } + + paused := s.cl.consumer.loadPaused() + + // While building this request, if any cursor is follow fetching and it + // has been more than the recheck-if-we-should-still-follow interval, + // we skip the cursor and move it back to the leader. + // + // This is safe w.r.t. metadata updates because `createReq` is running + // in the context of a live consumer session. + var rechecks cursorPreferreds + defer func() { + if len(rechecks) > 0 { + if s.cl.cfg.logger.Level() >= LogLevelInfo { + s.cl.cfg.logger.Log(LogLevelInfo, "redirecting follower fetchers back to their leader to re-check if a new follower should be chosen", + "from_broker", s.nodeID, + "moves", rechecks.String(), + ) + } + for _, c := range rechecks { + c.move() + } + } + }() + + s.cursorsMu.Lock() + defer s.cursorsMu.Unlock() + + cursorIdx := s.cursorsStart + for range s.cursors { + c := s.cursors[cursorIdx] + cursorIdx = (cursorIdx + 1) % len(s.cursors) + if !c.usable() { + continue + } + if s.nodeID != c.leader && c.moveAt > 0 && time.Since(time.Unix(0, c.moveAt)) > s.cl.cfg.recheckPreferredReplicaInterval { + rechecks = append(rechecks, cursorOffsetPreferred{ + cursorOffsetNext: *c.use(), + preferredReplica: c.leader, + recheck: true, + }) + continue + } + if paused.has(c.topic, c.partition) { + continue + } + req.addCursor(c) + } + + // We could have lost our only record buffer just before we grabbed the + // source lock above. + if len(s.cursors) > 0 { + s.cursorsStart = (s.cursorsStart + 1) % len(s.cursors) + } + + return req +} + +func (s *source) maybeConsume() { + if s.fetchState.maybeBegin() { + go s.loopFetch() + } +} + +func (s *source) loopFetch() { + consumer := &s.cl.consumer + session := consumer.loadSession() + + if session == noConsumerSession { + s.fetchState.hardFinish() + // It is possible that we were triggered to consume while we + // had no consumer session, and then *after* loopFetch loaded + // noConsumerSession, the session was saved and triggered to + // consume again. If this function is slow the first time + // around, it could still be running and about to hardFinish. + // The second trigger will do nothing, and then we hardFinish + // and block a new session from actually starting consuming. + // + // To guard against this, after we hard finish, we load the + // session again: if it is *not* noConsumerSession, we trigger + // attempting to consume again. Worst case, the trigger is + // useless and it will exit below when it builds an empty + // request. + sessionNow := consumer.loadSession() + if session != sessionNow { + s.maybeConsume() + } + return + } + + session.incWorker() + defer session.decWorker() + + // After our add, check quickly **without** another select case to + // determine if this context was truly canceled. Any other select that + // has another select case could theoretically race with the other case + // also being selected. + select { + case <-session.ctx.Done(): + s.fetchState.hardFinish() + return + default: + } + + // We receive on canFetch when we can fetch, and we send back when we + // are done fetching. + canFetch := make(chan chan bool, 1) + + again := true + for again { + select { + case <-session.ctx.Done(): + s.fetchState.hardFinish() + return + case <-s.sem: + } + + select { + case <-session.ctx.Done(): + s.fetchState.hardFinish() + return + case session.desireFetch() <- canFetch: + } + + select { + case <-session.ctx.Done(): + session.cancelFetchCh <- canFetch + s.fetchState.hardFinish() + return + case doneFetch := <-canFetch: + // Perform another last minute best-effort check on session context + // to avoid calling fetch with a canceled context just because it passed + // the above selects by chance due to pseudo-random select behavior. + if session.ctx.Err() != nil { + doneFetch <- false + s.fetchState.hardFinish() + return + } + again = s.fetchState.maybeFinish(s.fetch(session, doneFetch)) + } + } +} + +func (s *source) killSessionOnClose(ctx context.Context) { + br, err := s.cl.brokerOrErr(nil, s.nodeID, errUnknownBroker) + if err != nil { + return + } + s.session.kill() + req := &fetchRequest{ + maxWait: 1, + minBytes: 1, + maxBytes: 1, + maxPartBytes: 1, + rack: s.cl.cfg.rack, + isolationLevel: s.cl.cfg.isolationLevel, + session: s.session, + } + ch := make(chan struct{}) + br.do(ctx, req, func(kmsg.Response, error) { close(ch) }) + <-ch +} + +// fetch is the main logic center of fetching messages. +// +// This is a long function, made much longer by winded documentation, that +// contains a lot of the side effects of fetching and updating. The function +// consists of two main bulks of logic: +// +// - First, issue a request that can be killed if the source needs to be +// stopped. Processing the response modifies no state on the source. +// +// - Second, we keep the fetch response and update everything relevant +// (session, trigger some list or epoch updates, buffer the fetch). +// +// One small part between the first and second step is to update preferred +// replicas. We always keep the preferred replicas from the fetch response +// *even if* the source needs to be stopped. The knowledge of which preferred +// replica to use would not be out of date even if the consumer session is +// changing. +func (s *source) fetch(consumerSession *consumerSession, doneFetch chan<- bool) (fetched bool) { + req := s.createReq() + + // For all returns, if we do not buffer our fetch, then we want to + // ensure our used offsets are usable again. + var ( + alreadySentToDoneFetch bool + setOffsets bool + buffered bool + ) + defer func() { + if !buffered { + if req.numOffsets > 0 { + if setOffsets { + req.usedOffsets.finishUsingAllWithSet() + } else { + req.usedOffsets.finishUsingAll() + } + } + if !alreadySentToDoneFetch { + doneFetch <- false + } + } + }() + + if req.numOffsets == 0 { // cursors could have been set unusable + return fetched + } + + // If our fetch is killed, we want to cancel waiting for the response. + var ( + kresp kmsg.Response + requested = make(chan struct{}) + ctx, cancel = context.WithCancel(consumerSession.ctx) + ) + defer cancel() + + br, err := s.cl.brokerOrErr(ctx, s.nodeID, errUnknownBroker) + if err != nil { + close(requested) + } else { + br.do(ctx, req, func(k kmsg.Response, e error) { + kresp, err = k, e + close(requested) + }) + } + + select { + case <-requested: + // As `select` is pseudo-random when both cases are ready, + // check for context error in this branch to avoid retrying immediately with a failed context. + if isContextErr(err) && ctx.Err() != nil { + return fetched + } + fetched = true + case <-ctx.Done(): + return fetched + } + + var didBackoff bool + backoff := func(why any) { + // We preemptively allow more fetches (since we are not buffering) + // and reset our session because of the error (who knows if kafka + // processed the request but the client failed to receive it). + doneFetch <- false + alreadySentToDoneFetch = true + s.session.reset() + didBackoff = true + + s.cl.triggerUpdateMetadata(false, fmt.Sprintf("opportunistic load during source backoff: %v", why)) // as good a time as any + s.consecutiveFailures++ + after := time.NewTimer(s.cl.cfg.retryBackoff(s.consecutiveFailures)) + defer after.Stop() + select { + case <-after.C: + case <-ctx.Done(): + } + } + defer func() { + if !didBackoff { + s.consecutiveFailures = 0 + } + }() + + // If we had an error, we backoff. Killing a fetch quits the backoff, + // but that is fine; we may just re-request too early and fall into + // another backoff. + if err != nil { + backoff(err) + return fetched + } + + // The logic below here should be relatively quick. + // + // Note that fetch runs entirely in the context of a consumer session. + // loopFetch does not return until this function does, meaning we + // cannot concurrently issue a second fetch for partitions that are + // being processed below. + + resp := kresp.(*kmsg.FetchResponse) + fetch, reloadOffsets, preferreds, allErrsStripped, updateWhy := s.handleReqResp(br, req, resp) + + deleteReqUsedOffset := func(topic string, partition int32) { + t := req.usedOffsets[topic] + delete(t, partition) + if len(t) == 0 { + delete(req.usedOffsets, topic) + } + } + + // Before updating the source, we move all cursors that have new + // preferred replicas and remove them (and any reload offsets) from + // being tracked in our req offsets. + // + // Polling uses usedOffsets to update cursors; we need to remove moved + // or reloading cursors from being modified when the records that were + // fetched are finally polled. + if len(preferreds) > 0 { + if s.cl.cfg.logger.Level() >= LogLevelInfo { + s.cl.cfg.logger.Log(LogLevelInfo, "fetch partitions returned preferred replicas", + "from_broker", s.nodeID, + "moves", preferreds.String(), + ) + } + preferreds.eachPreferred(func(c cursorOffsetPreferred) { + c.move() + deleteReqUsedOffset(c.from.topic, c.from.partition) + }) + } + reloadOffsets.each(deleteReqUsedOffset) + + // handleReqResp only parses the body of the response, not the top + // level error code. + // + // The top level error code is related to fetch sessions only, and if + // there was an error, the body was empty (so processing is basically a + // no-op). We process the fetch session error now. + switch err := kerr.ErrorForCode(resp.ErrorCode); err { + case nil: + // success; fall through to session bump below + case kerr.FetchSessionIDNotFound: + if s.session.epoch == 0 { + // If the epoch was zero, the broker did not even + // establish a session for us (and thus is maxed on + // sessions). We stop trying. + s.cl.cfg.logger.Log(LogLevelInfo, "session failed with SessionIDNotFound while trying to establish a session; broker likely maxed on sessions; continuing on without using sessions", "broker", logID(s.nodeID)) + s.session.kill() + } else { + s.cl.cfg.logger.Log(LogLevelInfo, "received SessionIDNotFound from our in use session, our session was likely evicted; resetting session", "broker", logID(s.nodeID)) + s.session.reset() + } + return fetched + case kerr.InvalidFetchSessionEpoch: + s.cl.cfg.logger.Log(LogLevelInfo, "resetting fetch session", "broker", logID(s.nodeID), "err", err) + s.session.reset() + return fetched + + case kerr.FetchSessionTopicIDError, kerr.InconsistentTopicID: + s.cl.cfg.logger.Log(LogLevelInfo, "topic id issues, resetting session and updating metadata", "broker", logID(s.nodeID), "err", err) + s.session.reset() + s.cl.triggerUpdateMetadataNow("topic id issues") + return fetched + + default: + // Any other top-level error is unexpected: current brokers + // only emit session-related codes here. Rather than bumping + // the session epoch against a failed request, reset defensively + // so the next request re-establishes state the broker agrees + // with. + s.cl.cfg.logger.Log(LogLevelWarn, "fetch response has unexpected top-level error, resetting session", "broker", logID(s.nodeID), "err", err) + s.session.reset() + return fetched + } + + // At this point, we have successfully processed the response. Even if + // the response contains no records, we want to keep any offset + // advancements (we could have consumed only control records, we must + // advance past them). + setOffsets = true + + if resp.Version < 7 || resp.SessionID <= 0 { + // If the version is less than 7, we cannot use fetch sessions, + // so we kill them on the first response. + s.session.kill() + } else { + s.session.bumpEpoch(resp.SessionID) + // Commit the request's session-used mutations now that the + // broker has acknowledged the request. + s.session.commitFromReq(req.committedTopics, req.committedForgotten) + } + + // If we have a reason to update (per-partition fetch errors), and the + // reason is not just unknown topic or partition, then we immediately + // update metadata. We avoid updating for unknown because it _likely_ + // means the topic does not exist and reloading is wasteful. We only + // trigger a metadata update if we have no reload offsets. Having + // reload offsets *always* triggers a metadata update. + if updateWhy != nil { + why := updateWhy.reason(fmt.Sprintf("fetch had inner topic errors from broker %d", s.nodeID)) + // loadWithSessionNow triggers a metadata update IF there are + // offsets to reload. If there are no offsets to reload, we + // trigger one here. + if !reloadOffsets.loadWithSessionNow(consumerSession, why) { + if updateWhy.isOnly(kerr.UnknownTopicOrPartition) || updateWhy.isOnly(kerr.UnknownTopicID) { + s.cl.triggerUpdateMetadata(false, why) + } else { + s.cl.triggerUpdateMetadataNow(why) + } + } + } + + if fetch.hasErrorsOrRecords() { + buffered = true + s.buffered = bufferedFetch{ + fetch: fetch, + doneFetch: doneFetch, + usedOffsets: req.usedOffsets, + } + s.sem = make(chan struct{}) + s.hook(&fetch, true, false) // buffered, not polled + s.cl.consumer.addSourceReadyForDraining(s) + } else if allErrsStripped { + // If we stripped all errors from the response, we are likely + // fetching from topics that were deleted. We want to back off + // a bit rather than spin-loop immediately re-requesting + // deleted topics. + backoff("empty fetch response due to all partitions having retryable errors") + } + return fetched +} + +// Parses a fetch response into a Fetch, offsets to reload, and whether +// metadata needs updating. +// +// This function reads cursor fields which may be updated outside of consumer +// sessions, thus, we need to run this only inside a consumer session. +func (s *source) handleReqResp(br *broker, req *fetchRequest, resp *kmsg.FetchResponse) ( + f Fetch, + reloadOffsets listOrEpochLoads, + preferreds cursorPreferreds, + allErrsStripped bool, + updateWhy multiUpdateWhy, +) { + f = Fetch{Topics: make([]FetchTopic, 0, len(resp.Topics))} + var ( + debugWhyStripped multiUpdateWhy + numErrsStripped int + kip320 = s.cl.supportsOffsetForLeaderEpoch() + kmove kip951move + ) + defer kmove.maybeBeginMove(s.cl) + + strip := func(t string, p int32, err error) { + numErrsStripped++ + if s.cl.cfg.logger.Level() < LogLevelDebug { + return + } + debugWhyStripped.add(t, p, err) + } + + for _, rt := range resp.Topics { + topic := rt.Topic + // v13 only uses topic IDs, so we have to map the response + // uuid's to our string topics. + if resp.Version >= 13 { + topic = req.id2topic[rt.TopicID] + } + + // We always include all cursors on this source in the fetch; + // we should not receive any topics or partitions we do not + // expect. + topicOffsets, ok := req.usedOffsets[topic] + if !ok { + s.cl.cfg.logger.Log(LogLevelWarn, "broker returned topic from fetch that we did not ask for", + "broker", logID(s.nodeID), + "topic", topic, + ) + continue + } + + fetchTopic := FetchTopic{ + Topic: topic, + TopicID: rt.TopicID, + Partitions: make([]FetchPartition, 0, len(rt.Partitions)), + } + + for i := range rt.Partitions { + rp := &rt.Partitions[i] + partition := rp.Partition + partOffset, ok := topicOffsets[partition] + if !ok { + s.cl.cfg.logger.Log(LogLevelWarn, "broker returned partition from fetch that we did not ask for", + "broker", logID(s.nodeID), + "topic", topic, + "partition", partition, + ) + continue + } + c := partOffset.from + + // If we are fetching from the replica already, Kafka replies with a -1 + // preferred read replica. If Kafka replies with a preferred replica, + // it sends no records. + if preferred := rp.PreferredReadReplica; resp.Version >= 11 && preferred >= 0 { + preferreds = append(preferreds, cursorOffsetPreferred{ + cursorOffsetNext: *partOffset, + preferredReplica: preferred, + }) + continue + } + + fp := partOffset.processRespPartition(br, rp, s.cl.cfg.decompressor, s.cl.cfg.hooks) + if fp.Err != nil { + if moving := kmove.maybeAddFetchPartition(resp, rp, c); moving { + strip(topic, partition, fp.Err) + continue + } + updateWhy.add(topic, partition, fp.Err) + } + + // We only keep the partition if it has no error, or an + // error we do not internally retry. + var keep bool + switch fp.Err { + default: + if kerr.IsRetriable(fp.Err) && !s.cl.cfg.keepRetryableFetchErrors { + // UnknownLeaderEpoch: our meta is newer than the broker we fetched from + // OffsetNotAvailable: fetched from out of sync replica or a behind in-sync one (KIP-392 case 1 and case 2) + // UnknownTopicID: kafka has not synced the state on all brokers + // And other standard retryable errors. + strip(topic, partition, fp.Err) + } else { + // - bad auth + // - unsupported compression + // - unsupported message version + // - unknown error + // - or, no error + keep = true + } + + case nil: + c.unknownIDFails.Store(0) + keep = true + + case kerr.UnknownTopicID: + // We need to keep UnknownTopicID even though it is + // retryable, because encountering this error means + // the topic has been recreated and we will never + // consume the topic again anymore. This is an error + // worth bubbling up. + // + // Kafka will actually return this error for a brief + // window immediately after creating a topic for the + // first time, meaning the controller has not yet + // propagated to the leader that it is now the leader + // of a new partition. We need to ignore this error + // for a little bit. + if fails := c.unknownIDFails.Add(1); fails > 5 { + c.unknownIDFails.Add(-1) + keep = true + } else if s.cl.cfg.keepRetryableFetchErrors { + keep = true + } else { + strip(topic, partition, fp.Err) + } + + case kerr.OffsetOutOfRange: + // If we are out of range, we reset to what we can. + // With Kafka >= 2.1, we should only get offset out + // of range if we fetch before the start, but a user + // could start past the end and want to reset to + // the end. We respect that. + // + // KIP-392 (case 3) specifies that if we are consuming + // from a follower, then if our offset request is before + // the low watermark, we list offsets from the follower. + // However, Kafka does not actually implement handling + // ListOffsets from anything from the leader, so we + // need to redirect ourselves back to the leader. + // + // KIP-392 (case 4) specifies that if we are consuming + // a follower and our request is larger than the high + // watermark, then we should first check for truncation + // from the leader and then if we still get out of + // range, reset with list offsets. + // + // It further goes on to say that "out of range errors + // due to ISR propagation delays should be extremely + // rare". Rather than falling back to listing offsets, + // we stay in a cycle of validating the leader epoch + // until the follower has caught up. + // + // In all cases except case 4, we also have to check if + // no reset offset was configured. If so, we ignore + // trying to reset and instead keep our failed partition. + addList := func(replica int32, log bool) { + if s.cl.cfg.resetOffset.noReset { + keep = true + } else if !c.lastConsumedTime.IsZero() { + reloadOffsets.addLoad(topic, partition, loadTypeList, offsetLoad{ + replica: replica, + Offset: NewOffset().AfterMilli(c.lastConsumedTime.UnixMilli()), + }) + if log { + s.cl.cfg.logger.Log(LogLevelWarn, "received OFFSET_OUT_OF_RANGE, resetting to the nearest offset; either you were consuming too slowly and the broker has deleted the segment you were in the middle of consuming, or the broker has lost data and has not yet transferred leadership", + "broker", logID(s.nodeID), + "topic", topic, + "partition", partition, + "prior_offset", partOffset.offset, + ) + } + } else { + reloadOffsets.addLoad(topic, partition, loadTypeList, offsetLoad{ + replica: replica, + Offset: s.cl.cfg.resetOffset, + }) + if log { + s.cl.cfg.logger.Log(LogLevelInfo, "received OFFSET_OUT_OF_RANGE on the first fetch, resetting to the configured ConsumeResetOffset", + "broker", logID(s.nodeID), + "topic", topic, + "partition", partition, + "prior_offset", partOffset.offset, + ) + } + } + } + + switch { + case s.nodeID == c.leader: // non KIP-392 case + addList(-1, true) + + case partOffset.offset < fp.LogStartOffset: // KIP-392 case 3 + // KIP-392 specifies that we should list offsets against the follower, + // but that actually is not supported and the Java client redirects + // back to the leader. The leader then does *not* direct the client + // back to the follower because the follower is not an in sync + // replica. If we did not redirect back to the leader, we would spin + // loop receiving offset_out_of_range from the follower for Fetch, and + // then not_leader_or_follower from the follower for ListOffsets + // (even though it is a follower). So, we just set the preferred replica + // back to the follower. We go directly back to fetching with the + // hope that the offset is available on the leader, and if not, we'll + // just get an OOOR error again and fall into case 1 just above. + preferreds = append(preferreds, cursorOffsetPreferred{ + cursorOffsetNext: *partOffset, + preferredReplica: c.leader, + ooor: true, + }) + + default: // partOffset.offset > fp.HighWatermark, KIP-392 case 4 + if kip320 { + reloadOffsets.addLoad(topic, partition, loadTypeEpoch, offsetLoad{ + replica: -1, + Offset: Offset{ + at: partOffset.offset, + epoch: partOffset.lastConsumedEpoch, + }, + }) + } else { + // If the broker does not support offset for leader epoch but + // does support follower fetching for some reason, we have to + // fallback to listing. + addList(-1, true) + } + } + + case kerr.FencedLeaderEpoch: + // With fenced leader epoch, we notify an error only + // if necessary after we find out if loss occurred. + // If we have consumed nothing, then we got unlucky + // by being fenced right after we grabbed metadata. + // We just refresh metadata and try again. + // + // It would be odd for a broker to reply we are fenced + // but not support offset for leader epoch, so we do + // not check KIP-320 support here. + if partOffset.lastConsumedEpoch >= 0 { + reloadOffsets.addLoad(topic, partition, loadTypeEpoch, offsetLoad{ + replica: -1, + Offset: Offset{ + at: partOffset.offset, + epoch: partOffset.lastConsumedEpoch, + }, + }) + } + } + + if keep { + fetchTopic.Partitions = append(fetchTopic.Partitions, fp) + } + } + + if len(fetchTopic.Partitions) > 0 { + f.Topics = append(f.Topics, fetchTopic) + } + } + + if s.cl.cfg.logger.Level() >= LogLevelDebug && len(debugWhyStripped) > 0 { + s.cl.cfg.logger.Log(LogLevelDebug, "fetch stripped partitions", "why", debugWhyStripped.reason("")) + } + + return f, reloadOffsets, preferreds, req.numOffsets == numErrsStripped, updateWhy +} + +func (o *cursorOffsetNext) processRespPartition(br *broker, rp *kmsg.FetchResponseTopicPartition, decompressor Decompressor, hooks hooks) (fp FetchPartition) { + if rp.ErrorCode == 0 { + o.hwm = rp.HighWatermark + } + opts := ProcessFetchPartitionOpts{ + KeepControlRecords: br.cl.cfg.keepControl, + DisableCRCValidation: br.cl.cfg.disableFetchCRCValidation, + Offset: o.offset, + IsolationLevel: IsolationLevel{br.cl.cfg.isolationLevel}, + Topic: o.from.topic, + Partition: o.from.partition, + Pools: br.cl.cfg.pools, + } + fp, o.offset = ProcessFetchPartition(opts, rp, decompressor, func(m FetchBatchMetrics) { + hooks.each(func(h Hook) { + if h, ok := h.(HookFetchBatchRead); ok { + h.OnFetchBatchRead(br.meta, o.from.topic, o.from.partition, m) + } + }) + }) + if len(fp.Records) > 0 { + lastRecord := fp.Records[len(fp.Records)-1] + o.lastConsumedEpoch = lastRecord.LeaderEpoch + o.lastConsumedTime = lastRecord.Timestamp + } + + return fp +} + +// ProcessFetchPartitionOpts contains required inputs for processing a fetch +// partition and options for how records & offsets should be processed. +type ProcessFetchPartitionOpts struct { + // KeepControlRecords sets the parser to keep control messages and + // return them with fetches, overriding the default that discards them. + // + // Generally, control messages are not useful. This field is the same + // as [KeepControlRecords]. + KeepControlRecords bool + + // DisableCRCValidation opts out of validating the CRC prefixing + // every batch. This should only be true if your broker does not + // properly support CRCs. + DisableCRCValidation bool + + // Offset is the minimum offset for which we'll parse records. Records + // with lower offsets will not be parsed or returned. + Offset int64 + + // IsolationLevel controls whether or not to return uncommitted records. + // See [IsolationLevel]. + IsolationLevel IsolationLevel + + // Topic is used to populate the Topic field of each Record. + Topic string + + // Partition is used to populate the Partition field of each Record. + Partition int32 + + // Pools contain potential pools to use for memory pooling. + Pools []Pool + + // shareAckSlab, if non-nil, is called once per decoded batch + // to allocate a slab for share group ack state tracking. The + // slab is injected into each record's context via shareAckKey. + shareAckSlab func(numRecords int, firstRecord *Record) *shareAckSlab +} + +// ProcessFetchPartition processes all records in all batches or message sets +// in a *kmsg.FetchResponseTopicPartition, returning the processed +// FetchPartition and the offset of the last record that was processed. If +// hooks is non-nil, it is called with the metrics from processing this batch. +// +// This function is useful when issuing manual Fetch requests for records or in +// any scenario where you want to process raw fetch responses. +func ProcessFetchPartition(o ProcessFetchPartitionOpts, rp *kmsg.FetchResponseTopicPartition, decompressor Decompressor, hooks func(FetchBatchMetrics)) (FetchPartition, int64) { + fp := FetchPartition{ + Partition: rp.Partition, + Err: kerr.ErrorForCode(rp.ErrorCode), + HighWatermark: rp.HighWatermark, + LastStableOffset: rp.LastStableOffset, + LogStartOffset: rp.LogStartOffset, + } + + var aborter aborter + if o.IsolationLevel.level == 1 { + aborter = buildAborter(rp) + } + + // A response could contain any of message v0, message v1, or record + // batches, and this is solely dictated by the magic byte (not the + // fetch response version). The magic byte is located at byte 17. + // + // 1 thru 8: int64 offset / first offset + // 9 thru 12: int32 length + // 13 thru 16: crc (magic 0 or 1), or partition leader epoch (magic 2) + // 17: magic + // + // We decode and validate similarly for messages and record batches, so + // we "abstract" away the high level stuff into a check function just + // below, and then switch based on the magic for how to process. + var ( + in = rp.RecordBatches + + r readerFrom + kind string + length int32 + lengthField *int32 + crcField *int32 + crcTable *crc32.Table + crcAt int + + check = func() bool { + // If we call into check, we know we have a valid + // length, so we should be at least able to parse our + // top level struct and validate the length and CRC. + if err := r.ReadFrom(in[:length]); err != nil { + fp.Err = fmt.Errorf("unable to read %s, not enough data", kind) + return false + } + if length := int32(len(in[12:length])); length != *lengthField { + fp.Err = fmt.Errorf("encoded length %d does not match read length %d", *lengthField, length) + return false + } + // We have already validated that the slice is at least + // 17 bytes, but our CRC may be later (i.e. RecordBatch + // starts at byte 21). Ensure there is at least space + // for a CRC. + if !o.DisableCRCValidation { + if len(in) < crcAt { + fp.Err = fmt.Errorf("length %d is too short to allow for a crc", len(in)) + return false + } + if crcCalc := int32(crc32.Checksum(in[crcAt:length], crcTable)); crcCalc != *crcField { + fp.Err = fmt.Errorf("encoded crc %x does not match calculated crc %x", *crcField, crcCalc) + return false + } + } + return true + } + ) + + for len(in) > 17 && fp.Err == nil { + offset := int64(binary.BigEndian.Uint64(in)) + length = int32(binary.BigEndian.Uint32(in[8:])) + length += 12 // for the int64 offset we skipped and int32 length field itself + if len(in) < int(length) { + break + } + + switch magic := in[16]; magic { + case 0: + m := new(kmsg.MessageV0) + kind = "message v0" + lengthField = &m.MessageSize + crcField = &m.CRC + crcTable = crc32.IEEETable + crcAt = 16 + r = m + case 1: + m := new(kmsg.MessageV1) + kind = "message v1" + lengthField = &m.MessageSize + crcField = &m.CRC + crcTable = crc32.IEEETable + crcAt = 16 + r = m + case 2: + rb := new(kmsg.RecordBatch) + kind = "record batch" + lengthField = &rb.Length + crcField = &rb.CRC + crcTable = crc32c + crcAt = 21 + r = rb + + default: + fp.Err = fmt.Errorf("unknown magic %d; message offset is %d and length is %d, skipping and setting to next offset", magic, offset, length) + if next := offset + 1; next > o.Offset { + o.Offset = next + } + return fp, o.Offset + } + + if !check() { + break + } + + in = in[length:] + + var m FetchBatchMetrics + + switch t := r.(type) { + case *kmsg.MessageV0: + m.CompressedBytes = int(length) // for message sets, we include the message set overhead in length + m.CompressionType = uint8(t.Attributes) & 0b0000_0111 + m.NumRecords, m.UncompressedBytes = o.processV0OuterMessage(&fp, t, decompressor) + + case *kmsg.MessageV1: + m.CompressedBytes = int(length) + m.CompressionType = uint8(t.Attributes) & 0b0000_0111 + m.NumRecords, m.UncompressedBytes = o.processV1OuterMessage(&fp, t, decompressor) + + case *kmsg.RecordBatch: + m.CompressedBytes = len(t.Records) // for record batches, we only track the record batch length + m.CompressionType = uint8(t.Attributes) & 0b0000_0111 + m.NumRecords, m.UncompressedBytes = o.processRecordBatch(&fp, t, aborter, decompressor) + } + + if m.UncompressedBytes == 0 { + m.UncompressedBytes = m.CompressedBytes + } + if hooks != nil { + hooks(m) + } + + // If we encounter a decompression error BUT we have successfully decompressed + // one batch, it is likely that we have received a partial batch. Kafka returns + // UP TO the requested max partition bytes, sometimes truncating data at the end. + // It returns at least one valid batch, but everything after is copied as is + // (i.e. a quick slab copy). We set the error to nil and return what we have. + // + // If we have a decompression error immediately, we keep it and bubble it up. + // The client cannot progress, and the end user needs visibility. + if isDecompressErr(fp.Err) && len(fp.Records) > 0 { + fp.Err = nil + break + } + } + + return fp, o.Offset +} + +type aborter map[int64][]int64 + +func buildAborter(rp *kmsg.FetchResponseTopicPartition) aborter { + if len(rp.AbortedTransactions) == 0 { + return nil + } + a := make(aborter) + for _, abort := range rp.AbortedTransactions { + a[abort.ProducerID] = append(a[abort.ProducerID], abort.FirstOffset) + } + return a +} + +func (a aborter) shouldAbortBatch(b *kmsg.RecordBatch) bool { + if len(a) == 0 || b.Attributes&0b0001_0000 == 0 { + return false + } + pidAborts := a[b.ProducerID] + if len(pidAborts) == 0 { + return false + } + // If the first offset in this batch is less than the first offset + // aborted, then this batch is not aborted. + if b.FirstOffset < pidAborts[0] { + return false + } + return true +} + +func (a aborter) trackAbortedPID(producerID int64) { + remaining := a[producerID][1:] + if len(remaining) == 0 { + delete(a, producerID) + } else { + a[producerID] = remaining + } +} + +////////////////////////////////////// +// processing records to fetch part // +////////////////////////////////////// + +// readRawRecordsInto reads records from in and returns them, returning early +// if there were partial records. +func readRawRecordsInto(rs []kmsg.Record, in []byte) []kmsg.Record { + for i := range rs { + length, used := kbin.Varint(in) + total := used + int(length) + if used == 0 || length < 0 || len(in) < total { + return rs[:i] + } + if err := (&rs[i]).ReadFrom(in[:total]); err != nil { + rs[i] = kmsg.Record{} // clear any invalid partial data + return rs[:i] + } + in = in[total:] + } + return rs +} + +func (o *ProcessFetchPartitionOpts) processRecordBatch( + fp *FetchPartition, + batch *kmsg.RecordBatch, + aborter aborter, + decompressor Decompressor, +) (int, int) { + if batch.Magic != 2 { + fp.Err = fmt.Errorf("unknown batch magic %d", batch.Magic) + return 0, 0 + } + lastOffset := batch.FirstOffset + int64(batch.LastOffsetDelta) + if lastOffset < o.Offset { + // If the last offset in this batch is less than what we asked + // for, we got a batch that we entirely do not need. We can + // avoid all work (although we should not get this batch). + return 0, 0 + } + + var usesPools bool + + rawRecords := batch.Records + var decompressBytes []byte + if compression := CompressionCodecType(batch.Attributes & 0x0007); compression != 0 { + var err error + if rawRecords, err = decompressor.Decompress(rawRecords, compression); err != nil { + fp.Err = &errDecompress{err} + return 0, 0 // truncated batch + } + // We only put back into the decompress pool IF we decompressed + // AND if a pool implement the interface AND if the batch was + // actually compressed. The default decompressor uses the pool + // if present, and it is expected that users overriding the + // decompressor use the pool as well (if provided to the + // client). Worst case, the use gets data put back into their + // pool that they didn't create. + pools(o.Pools).each(func(p Pool) bool { + if _, ok := p.(PoolDecompressBytes); ok { + decompressBytes = rawRecords + usesPools = true + return true + } + return false + }) + } + + uncompressedBytes := len(rawRecords) + + numRecords := int(batch.NumRecords) + var krecords []kmsg.Record + var krecordsPool PoolKRecords + pools(o.Pools).each(func(p Pool) bool { + if pkrecs, ok := p.(PoolKRecords); ok { + krecords = pkrecs.GetKRecords(numRecords) + krecordsPool = pkrecs + return true + } + return false + }) + if krecordsPool != nil { + defer func() { + krecords = krecords[:cap(krecords)] + krecordsPool.PutKRecords(krecords) + }() + } + krecords = ensureLen(krecords, numRecords) + krecords = readRawRecordsInto(krecords, rawRecords) + + // KAFKA-5443: compacted topics preserve the last offset in a batch, + // even if the last record is removed, meaning that using offsets from + // records alone may not get us to the next offset we need to ask for. + // + // We only perform this logic if we did not consume a truncated batch. + // If we consume a truncated batch, then what was truncated could have + // been an offset we are interested in consuming. Even if our fetch did + // not advance this partition at all, we will eventually fetch from the + // partition and not have a truncated response, at which point we will + // either advance offsets or will set to nextAskOffset. + nextAskOffset := lastOffset + 1 + defer func() { + if numRecords == len(krecords) && o.Offset < nextAskOffset { + o.Offset = nextAskOffset + } + }() + + abortBatch := aborter.shouldAbortBatch(batch) + var rrecords []Record + pools(o.Pools).each(func(p Pool) bool { + if precs, ok := p.(PoolRecords); ok { + rrecords = precs.GetRecords(numRecords) + usesPools = true + return true + } + return false + }) + rrecords = ensureLen(rrecords, numRecords) + + var p *recordPools + var poolsCtx context.Context + if usesPools { + p, poolsCtx = recordPoolsCtx(o.Pools, decompressBytes, rrecords) + } + + recordCtx := poolsCtx + if o.shareAckSlab != nil && len(rrecords) > 0 { + if slab := o.shareAckSlab(numRecords, &rrecords[0]); slab != nil { + parent := poolsCtx + if parent == nil { + parent = context.Background() + } + recordCtx = context.WithValue(parent, shareAckKey, slab) + } + } + var nkept int + defer func() { + if p != nil && nkept > 0 { + p.n.Add(int64(nkept)) + } + }() + + for i := range krecords { + record := &rrecords[i] + recordToRecord( + o.Topic, + fp.Partition, + batch, + &krecords[i], + record, + ) + record.Context = recordCtx //nolint:fatcontext // not a nested context + krecords[i] = kmsg.Record{} // prevent the kmsg.Record from hanging onto anything + if kept := o.maybeKeepRecord(fp, record, abortBatch); kept { + nkept++ + } + + if abortBatch && record.Attrs.IsControl() { + // A control record has a key and a value where the key + // is int16 version and int16 type. Aborted records + // have a type of 0. + if key := record.Key; len(key) >= 4 && key[2] == 0 && key[3] == 0 { + aborter.trackAbortedPID(batch.ProducerID) + } + } + } + + return len(krecords), uncompressedBytes +} + +// Processes an outer v1 message. There could be no inner message, which makes +// this easy, but if not, we decompress and process each inner message as +// either v0 or v1. We only expect the inner message to be v1, but technically +// a crazy pipeline could have v0 anywhere. +func (o *ProcessFetchPartitionOpts) processV1OuterMessage( + fp *FetchPartition, + message *kmsg.MessageV1, + decompressor Decompressor, +) (int, int) { + compression := CompressionCodecType(message.Attributes & 0x0003) + if compression == 0 { + o.processV1Message(fp, message) + return 1, 0 + } + + rawInner, err := decompressor.Decompress(message.Value, compression) + if err != nil { + fp.Err = &errDecompress{err} + return 0, 0 // truncated batch + } + + uncompressedBytes := len(rawInner) + + var innerMessages []readerFrom +out: + for len(rawInner) > 17 { // magic at byte 17 + length := int32(binary.BigEndian.Uint32(rawInner[8:])) + length += 12 // offset and length fields + if len(rawInner) < int(length) { + break + } + + var ( + magic = rawInner[16] + + msg readerFrom + lengthField *int32 + crcField *int32 + ) + + switch magic { + case 0: + m := new(kmsg.MessageV0) + msg = m + lengthField = &m.MessageSize + crcField = &m.CRC + case 1: + m := new(kmsg.MessageV1) + msg = m + lengthField = &m.MessageSize + crcField = &m.CRC + + default: + fp.Err = fmt.Errorf("message set v1 has inner message with invalid magic %d", magic) + break out + } + + if err := msg.ReadFrom(rawInner[:length]); err != nil { + fp.Err = fmt.Errorf("unable to read message v%d, not enough data", magic) + break + } + if length := int32(len(rawInner[12:length])); length != *lengthField { + fp.Err = fmt.Errorf("encoded length %d does not match read length %d", *lengthField, length) + break + } + if !o.DisableCRCValidation { + if crcCalc := int32(crc32.ChecksumIEEE(rawInner[16:length])); crcCalc != *crcField { + fp.Err = fmt.Errorf("encoded crc %x does not match calculated crc %x", *crcField, crcCalc) + break + } + } + innerMessages = append(innerMessages, msg) + rawInner = rawInner[length:] + } + if len(innerMessages) == 0 { + return 0, uncompressedBytes + } + + firstOffset := message.Offset - int64(len(innerMessages)) + 1 + for i := range innerMessages { + innerMessage := innerMessages[i] + switch innerMessage := innerMessage.(type) { + case *kmsg.MessageV0: + innerMessage.Offset = firstOffset + int64(i) + innerMessage.Attributes |= int8(compression) + if !o.processV0Message(fp, innerMessage) { + return i, uncompressedBytes + } + case *kmsg.MessageV1: + innerMessage.Offset = firstOffset + int64(i) + innerMessage.Attributes |= int8(compression) + if !o.processV1Message(fp, innerMessage) { + return i, uncompressedBytes + } + } + } + return len(innerMessages), uncompressedBytes +} + +func (o *ProcessFetchPartitionOpts) processV1Message( + fp *FetchPartition, + message *kmsg.MessageV1, +) bool { + if message.Magic != 1 { + fp.Err = fmt.Errorf("unknown message magic %d", message.Magic) + return false + } + if uint8(message.Attributes)&0b1111_0000 != 0 { + fp.Err = fmt.Errorf("unknown attributes on message %d", message.Attributes) + return false + } + record := v1MessageToRecord(o.Topic, fp.Partition, message) + o.maybeKeepRecord(fp, record, false) + return true +} + +// Processes an outer v0 message. We expect inner messages to be entirely v0 as +// well, so this only tries v0 always. +func (o *ProcessFetchPartitionOpts) processV0OuterMessage( + fp *FetchPartition, + message *kmsg.MessageV0, + decompressor Decompressor, +) (int, int) { + compression := CompressionCodecType(message.Attributes & 0x0003) + if compression == 0 { + o.processV0Message(fp, message) + return 1, 0 // uncompressed bytes is 0; set to compressed bytes on return + } + + rawInner, err := decompressor.Decompress(message.Value, compression) + if err != nil { + fp.Err = &errDecompress{err} + return 0, 0 // truncated batch + } + + uncompressedBytes := len(rawInner) + + var innerMessages []kmsg.MessageV0 + for len(rawInner) > 17 { // magic at byte 17 + length := int32(binary.BigEndian.Uint32(rawInner[8:])) + length += 12 // offset and length fields + if len(rawInner) < int(length) { + break // truncated batch + } + var m kmsg.MessageV0 + if err := m.ReadFrom(rawInner[:length]); err != nil { + fp.Err = fmt.Errorf("unable to read message v0, not enough data") + break + } + if length := int32(len(rawInner[12:length])); length != m.MessageSize { + fp.Err = fmt.Errorf("encoded length %d does not match read length %d", m.MessageSize, length) + break + } + if !o.DisableCRCValidation { + if crcCalc := int32(crc32.ChecksumIEEE(rawInner[16:length])); crcCalc != m.CRC { + fp.Err = fmt.Errorf("encoded crc %x does not match calculated crc %x", m.CRC, crcCalc) + break + } + } + innerMessages = append(innerMessages, m) + rawInner = rawInner[length:] + } + if len(innerMessages) == 0 { + return 0, uncompressedBytes + } + + firstOffset := message.Offset - int64(len(innerMessages)) + 1 + for i := range innerMessages { + innerMessage := &innerMessages[i] + innerMessage.Attributes |= int8(compression) + innerMessage.Offset = firstOffset + int64(i) + if !o.processV0Message(fp, innerMessage) { + return i, uncompressedBytes + } + } + return len(innerMessages), uncompressedBytes +} + +func (o *ProcessFetchPartitionOpts) processV0Message( + fp *FetchPartition, + message *kmsg.MessageV0, +) bool { + if message.Magic != 0 { + fp.Err = fmt.Errorf("unknown message magic %d", message.Magic) + return false + } + if uint8(message.Attributes)&0b1111_1000 != 0 { + fp.Err = fmt.Errorf("unknown attributes on message %d", message.Attributes) + return false + } + record := v0MessageToRecord(o.Topic, fp.Partition, message) + o.maybeKeepRecord(fp, record, false) + return true +} + +// maybeKeepRecord keeps a record if it is within our range of offsets to keep. +// +// If the record is being aborted or the record is a control record and the +// client does not want to keep control records, this does not keep the record. +func (o *ProcessFetchPartitionOpts) maybeKeepRecord(fp *FetchPartition, record *Record, abort bool) (kept bool) { + if record.Offset < o.Offset { + // We asked for offset 5, but that was in the middle of a + // batch; we got offsets 0 thru 4 that we need to skip. + return false + } + + // We only keep control records if specifically requested. + if record.Attrs.IsControl() { + abort = !o.KeepControlRecords + } + if !abort { + fp.Records = append(fp.Records, record) + kept = true + } + + // The record offset may be much larger than our expected offset if the + // topic is compacted. + o.Offset = record.Offset + 1 + return kept +} + +/////////////////////////////// +// kmsg.Record to kgo.Record // +/////////////////////////////// + +func timeFromMillis(millis int64) time.Time { + return time.Unix(0, millis*1e6) +} + +// recordToRecord converts a kmsg.RecordBatch's Record to a kgo Record. +func recordToRecord( + topic string, + partition int32, + batch *kmsg.RecordBatch, + krecord *kmsg.Record, + r *Record, +) { + var h []RecordHeader + if len(krecord.Headers) > 0 { + h = make([]RecordHeader, len(krecord.Headers)) + for i, kv := range krecord.Headers { + h[i] = RecordHeader{ + Key: kv.Key, + Value: kv.Value, + } + } + } + *r = Record{ + Key: krecord.Key, + Value: krecord.Value, + Headers: h, + Topic: topic, + Partition: partition, + Attrs: RecordAttrs{uint8(batch.Attributes)}, + ProducerID: batch.ProducerID, + ProducerEpoch: batch.ProducerEpoch, + LeaderEpoch: batch.PartitionLeaderEpoch, + } + if batch.FirstOffset == -1 { + r.Offset = -1 + } else { + r.Offset = batch.FirstOffset + int64(krecord.OffsetDelta) + } + if r.Attrs.TimestampType() == 0 { + r.Timestamp = timeFromMillis(batch.FirstTimestamp + krecord.TimestampDelta64) + } else { + r.Timestamp = timeFromMillis(batch.MaxTimestamp) + } +} + +func messageAttrsToRecordAttrs(attrs int8, v0 bool) RecordAttrs { + uattrs := uint8(attrs) + if v0 { + uattrs |= 0b1000_0000 + } + return RecordAttrs{uattrs} +} + +func v0MessageToRecord( + topic string, + partition int32, + message *kmsg.MessageV0, +) *Record { + return &Record{ + Key: message.Key, + Value: message.Value, + Topic: topic, + Partition: partition, + Attrs: messageAttrsToRecordAttrs(message.Attributes, true), + ProducerID: -1, + ProducerEpoch: -1, + LeaderEpoch: -1, + Offset: message.Offset, + } +} + +func v1MessageToRecord( + topic string, + partition int32, + message *kmsg.MessageV1, +) *Record { + return &Record{ + Key: message.Key, + Value: message.Value, + Timestamp: timeFromMillis(message.Timestamp), + Topic: topic, + Partition: partition, + Attrs: messageAttrsToRecordAttrs(message.Attributes, false), + ProducerID: -1, + ProducerEpoch: -1, + LeaderEpoch: -1, + Offset: message.Offset, + } +} + +////////////////// +// fetchRequest // +////////////////// + +type fetchRequest struct { + version int16 + maxWait int32 + minBytes int32 + maxBytes int32 + maxPartBytes int32 + rack string + + isolationLevel int8 + preferLagFn PreferLagFn + + numOffsets int + usedOffsets usedOffsets + + torder []string // order of topics to write + porder map[string][]int32 // per topic, order of partitions to write + + // topic2id and id2topic track bidirectional lookup of topics and IDs + // that are being added to *this* specific request. topic2id slightly + // duplicates the map t2id in the fetch session, but t2id is different + // in that t2id tracks IDs in use from all prior requests -- and, + // importantly, t2id is cleared of IDs that are no longer used (see + // ForgottenTopics). + // + // We need to have both a session t2id map and a request t2id map: + // + // * The session t2id is what we use when creating forgotten topics. + // If we are forgetting a topic, the ID is not in the req t2id. + // + // * The req topic2id is used for adding to the session t2id. When + // building a request, if the id is in req.topic2id but not + // session.t2id, we promote the ID into the session map. + // + // Lastly, id2topic is used when handling the response, as our reverse + // lookup from the ID back to the topic (and then we work with the + // topic name only). There is no equivalent in the session because + // there is no need for the id2topic lookup ever in the session. + topic2id map[string][16]byte + id2topic map[[16]byte]string + + disableIDs bool // #295: using an old IBP on new Kafka results in ApiVersions advertising 13+ while the broker does not return IDs + + // Session is a copy of the source session at the time a request is + // built. If the source is reset, the session it has is reset at the + // field level only. Our view of the original session is still valid. + session fetchSession + + // Snapshots of the topic / forgotten-topic slices from the + // serialized kmsg.FetchRequest, captured at AppendTo time so fetch() + // can apply them to s.session.used after a successful response. + // Overwritten by each AppendTo call; the last call wins, which is + // correct because only the last serialization made it onto the + // wire. + committedTopics []kmsg.FetchRequestTopic + committedForgotten []kmsg.FetchRequestForgottenTopic +} + +func (f *fetchRequest) addCursor(c *cursor) { + if f.usedOffsets == nil { + f.usedOffsets = make(usedOffsets) + f.id2topic = make(map[[16]byte]string) + f.topic2id = make(map[string][16]byte) + f.porder = make(map[string][]int32) + } + partitions := f.usedOffsets[c.topic] + if partitions == nil { + partitions = make(map[int32]*cursorOffsetNext) + f.usedOffsets[c.topic] = partitions + f.id2topic[c.topicID] = c.topic + f.topic2id[c.topic] = c.topicID + var noID [16]byte + if c.topicID == noID { + f.disableIDs = true + } + f.torder = append(f.torder, c.topic) + } + partitions[c.partition] = c.use() + f.porder[c.topic] = append(f.porder[c.topic], c.partition) + f.numOffsets++ +} + +// PreferLagFn accepts topic and partition lag, the previously determined topic +// order, and the previously determined per-topic partition order, and returns +// a new topic and per-topic partition order. +// +// Most use cases will not need to look at the prior orders, but they exist if +// you want to get fancy. +// +// You can return partial results: if you only return topics, partitions within +// each topic keep their prior ordering. If you only return some topics but not +// all, the topics you do not return / the partitions you do not return will +// retain their original ordering *after* your given ordering. +// +// NOTE: torderPrior and porderPrior must not be modified. To avoid a bit of +// unnecessary allocations, these arguments are views into data that is used to +// build a fetch request. +type PreferLagFn func(lag map[string]map[int32]int64, torderPrior []string, porderPrior map[string][]int32) ([]string, map[string][]int32) + +// PreferLagAt is a simple PreferLagFn that orders the largest lag first, for +// any topic that is collectively lagging more than preferLagAt, and for any +// partition that is lagging more than preferLagAt. +// +// The function does not prescribe any ordering for topics that have the same +// lag. It is recommended to use a number more than 0 or 1: if you use 0, you +// may just always undo client ordering when there is no actual lag. +func PreferLagAt(preferLagAt int64) PreferLagFn { + if preferLagAt < 0 { + return nil + } + return func(lag map[string]map[int32]int64, _ []string, _ map[string][]int32) ([]string, map[string][]int32) { + type plag struct { + p int32 + lag int64 + } + type tlag struct { + t string + lag int64 + ps []plag + } + + // First, collect all partition lag into per-topic lag. + tlags := make(map[string]tlag, len(lag)) + for t, ps := range lag { + for p, lag := range ps { + prior := tlags[t] + tlags[t] = tlag{ + t: t, + lag: prior.lag + lag, + ps: append(prior.ps, plag{p, lag}), + } + } + } + + // We now remove topics and partitions that are not lagging + // enough. Collectively, the topic could be lagging too much, + // but individually, no partition is lagging that much: we will + // sort the topic first and keep the old partition ordering. + for t, tlag := range tlags { + if tlag.lag < preferLagAt { + delete(tlags, t) + continue + } + for i := 0; i < len(tlag.ps); i++ { + plag := tlag.ps[i] + if plag.lag < preferLagAt { + tlag.ps[i] = tlag.ps[len(tlag.ps)-1] + tlag.ps = tlag.ps[:len(tlag.ps)-1] + i-- + } + } + } + if len(tlags) == 0 { + return nil, nil + } + + var sortedLags []tlag + for _, tlag := range tlags { + sort.Slice(tlag.ps, func(i, j int) bool { return tlag.ps[i].lag > tlag.ps[j].lag }) + sortedLags = append(sortedLags, tlag) + } + sort.Slice(sortedLags, func(i, j int) bool { return sortedLags[i].lag > sortedLags[j].lag }) + + // We now return our laggy topics and partitions, and let the + // caller add back any missing topics / partitions in their + // prior order. + torder := make([]string, 0, len(sortedLags)) + for _, t := range sortedLags { + torder = append(torder, t.t) + } + porder := make(map[string][]int32, len(sortedLags)) + for _, tlag := range sortedLags { + ps := make([]int32, 0, len(tlag.ps)) + for _, p := range tlag.ps { + ps = append(ps, p.p) + } + porder[tlag.t] = ps + } + return torder, porder + } +} + +// If the end user prefers to consume lag, we reorder our previously ordered +// partitions, preferring first the laggiest topics, and then within those, the +// laggiest partitions. +func (f *fetchRequest) adjustPreferringLag() { + if f.preferLagFn == nil { + return + } + + tall := make(map[string]struct{}, len(f.torder)) + for _, t := range f.torder { + tall[t] = struct{}{} + } + pall := make(map[string][]int32, len(f.porder)) + for t, ps := range f.porder { + pall[t] = slices.Clone(ps) + } + + lag := make(map[string]map[int32]int64, len(f.torder)) + for t, ps := range f.usedOffsets { + plag := make(map[int32]int64, len(ps)) + lag[t] = plag + for p, c := range ps { + hwm := max(c.hwm, 0) + lag := hwm - c.offset + if c.offset <= 0 { + lag = hwm + } + if lag < 0 { + lag = 0 + } + plag[p] = lag + } + } + + torder, porder := f.preferLagFn(lag, f.torder, f.porder) + if torder == nil && porder == nil { + return + } + defer func() { f.torder, f.porder = torder, porder }() + + if len(torder) == 0 { + torder = f.torder // user did not modify topic order, keep old order + } else { + // Remove any extra topics the user returned that we were not + // consuming, and add all topics they did not give back. + for i := 0; i < len(torder); i++ { + t := torder[i] + if _, exists := tall[t]; !exists { + torder = slices.Delete(torder, i, i+1) // user gave topic we were not fetching + i-- + } + delete(tall, t) + } + for _, t := range f.torder { + if _, exists := tall[t]; exists { + torder = append(torder, t) // user did not return topic we were fetching + delete(tall, t) + } + } + } + + if len(porder) == 0 { + porder = f.porder // user did not modify partition order, keep old order + return + } + + pused := make(map[int32]struct{}) + for t, ps := range pall { + order, exists := porder[t] + if !exists { + porder[t] = ps // shortcut: user did not define this partition's oorder, keep old order + continue + } + for _, p := range ps { + pused[p] = struct{}{} + } + for i := 0; i < len(order); i++ { + p := order[i] + if _, exists := pused[p]; !exists { + order = slices.Delete(order, i, i+1) + i-- + } + delete(pused, p) + } + for _, p := range f.porder[t] { + if _, exists := pused[p]; exists { + order = append(order, p) + delete(pused, p) + } + } + porder[t] = order + } +} + +func (*fetchRequest) Key() int16 { return 1 } +func (f *fetchRequest) MaxVersion() int16 { + if f.disableIDs || f.session.disableIDs { + return 12 + } + return 18 +} +func (f *fetchRequest) SetVersion(v int16) { f.version = v } +func (f *fetchRequest) GetVersion() int16 { return f.version } +func (f *fetchRequest) IsFlexible() bool { return f.version >= 12 } // version 12+ is flexible +func (f *fetchRequest) AppendTo(dst []byte) []byte { + req := kmsg.NewFetchRequest() + req.Version = f.version + req.ReplicaID = -1 + req.MaxWaitMillis = f.maxWait + req.MinBytes = f.minBytes + req.MaxBytes = f.maxBytes + req.IsolationLevel = f.isolationLevel + req.SessionID = f.session.id + req.SessionEpoch = f.session.epoch + req.Rack = f.rack + + // We track which partitions we add in this request; any partitions + // missing that are already in the session get added to forgotten + // topics at the end. + var sessionUsed map[string]map[int32]struct{} + if !f.session.killed { + sessionUsed = make(map[string]map[int32]struct{}, len(f.usedOffsets)) + } + + f.adjustPreferringLag() + + for _, topic := range f.torder { + partitions := f.usedOffsets[topic] + + var reqTopic *kmsg.FetchRequestTopic + sessionTopic := f.session.lookupTopic(topic, f.topic2id) + + var usedTopic map[int32]struct{} + if sessionUsed != nil { + usedTopic = make(map[int32]struct{}, len(partitions)) + } + + for _, partition := range f.porder[topic] { + cursorOffsetNext := partitions[partition] + + if usedTopic != nil { + usedTopic[partition] = struct{}{} + } + + if !sessionTopic.hasPartitionAt( + partition, + cursorOffsetNext.offset, + cursorOffsetNext.currentLeaderEpoch, + ) { + if reqTopic == nil { + t := kmsg.NewFetchRequestTopic() + t.Topic = topic + t.TopicID = f.topic2id[topic] + req.Topics = append(req.Topics, t) + reqTopic = &req.Topics[len(req.Topics)-1] + } + + reqPartition := kmsg.NewFetchRequestTopicPartition() + reqPartition.Partition = partition + reqPartition.CurrentLeaderEpoch = cursorOffsetNext.currentLeaderEpoch + reqPartition.FetchOffset = cursorOffsetNext.offset + reqPartition.LastFetchedEpoch = -1 + reqPartition.LogStartOffset = -1 + reqPartition.PartitionMaxBytes = f.maxPartBytes + reqTopic.Partitions = append(reqTopic.Partitions, reqPartition) + } + } + + if sessionUsed != nil { + sessionUsed[topic] = usedTopic + } + } + + // Now for everything that we did not use in our session, add it to + // forgotten topics. The resulting ForgottenTopics list is what the + // broker is asked to drop; the actual mutation of f.session.used and + // f.session.t2id happens only after the response is processed, via + // fetchSession.commitFromReq. Doing the mutation here would be + // unsafe under broker.go's zero-bytes-written retry path, which can + // call AppendTo a second time on the same *fetchRequest. + if sessionUsed != nil { + for topic, partitions := range f.session.used { + var forgottenTopic *kmsg.FetchRequestForgottenTopic + topicUsed := sessionUsed[topic] + for partition := range partitions { + if topicUsed != nil { + if _, partitionUsed := topicUsed[partition]; partitionUsed { + continue + } + } + if forgottenTopic == nil { + t := kmsg.NewFetchRequestForgottenTopic() + t.Topic = topic + t.TopicID = f.session.t2id[topic] + req.ForgottenTopics = append(req.ForgottenTopics, t) + forgottenTopic = &req.ForgottenTopics[len(req.ForgottenTopics)-1] + } + forgottenTopic.Partitions = append(forgottenTopic.Partitions, partition) + } + } + } + + // Snapshot the topics / forgotten-topics so fetch() can commit them + // to s.session.used after a successful response. Last AppendTo wins + // (the last serialization is what actually went on the wire). + f.committedTopics = req.Topics + f.committedForgotten = req.ForgottenTopics + + return req.AppendTo(dst) +} + +func (*fetchRequest) ReadFrom([]byte) error { + panic("unreachable -- the client never uses ReadFrom on its internal fetchRequest") +} + +func (f *fetchRequest) ResponseKind() kmsg.Response { + r := kmsg.NewPtrFetchResponse() + r.Version = f.version + return r +} + +// fetchSessions, introduced in KIP-227, allow us to send less information back +// and forth to a Kafka broker. +type fetchSession struct { + id int32 + epoch int32 + + used map[string]map[int32]fetchSessionOffsetEpoch // what we have in the session so far + t2id map[string][16]byte + + disableIDs bool // if anything in t2id has no ID + killed bool // if we cannot use a session anymore +} + +func (s *fetchSession) kill() { + s.epoch = -1 + s.used = nil + s.t2id = nil + s.disableIDs = false + s.killed = true +} + +// reset resets the session by setting the next request to use epoch 0. +// We do not reset the ID; using epoch 0 for an existing ID unregisters the +// prior session. +func (s *fetchSession) reset() { + if s.killed { + return + } + s.epoch = 0 + s.used = nil + s.t2id = nil + s.disableIDs = false +} + +// commitFromReq applies to s the session-used mutations implied by a +// successful fetch request: partitions in req.Topics were TOLD to the +// broker at their FetchOffset/CurrentLeaderEpoch, and partitions in +// req.ForgottenTopics were TOLD to be removed. We call this only after +// the broker has acknowledged the request via a successful (non-error) +// response, guaranteeing the broker's session state matches what we're +// recording here. +func (s *fetchSession) commitFromReq(topics []kmsg.FetchRequestTopic, forgotten []kmsg.FetchRequestForgottenTopic) { + if s.killed { + return + } + if s.used == nil { + s.used = make(map[string]map[int32]fetchSessionOffsetEpoch) + s.t2id = make(map[string][16]byte) + } + var noID [16]byte + for _, rt := range topics { + topic := rt.Topic + if topic == "" { + continue + } + t := s.used[topic] + if t == nil { + t = make(map[int32]fetchSessionOffsetEpoch) + s.used[topic] = t + s.t2id[topic] = rt.TopicID + if rt.TopicID == noID { + s.disableIDs = true + } + } + for _, rp := range rt.Partitions { + t[rp.Partition] = fetchSessionOffsetEpoch{rp.FetchOffset, rp.CurrentLeaderEpoch} + } + } + for _, rft := range forgotten { + topic := rft.Topic + if topic == "" { + continue + } + t, ok := s.used[topic] + if !ok { + continue + } + for _, p := range rft.Partitions { + delete(t, p) + } + if len(t) == 0 { + id := s.t2id[topic] + delete(s.used, topic) + delete(s.t2id, topic) + if id == noID { + // Recompute disableIDs from remaining topics. + s.disableIDs = false + for _, tid := range s.t2id { + if tid == noID { + s.disableIDs = true + break + } + } + } + } + } +} + +// bumpEpoch bumps the epoch and saves the session id. +// +// Kafka replies with the session ID of the session to use. When it does, we +// start from epoch 1, wrapping back to 1 if we go negative. +func (s *fetchSession) bumpEpoch(id int32) { + if s.killed { + return + } + if id != s.id { + s.epoch = 0 // new session: reset to 0 for the increment below + } + s.epoch++ + if s.epoch < 0 { + s.epoch = 1 // we wrapped: reset back to 1 to continue this session + } + s.id = id +} + +func (s *fetchSession) lookupTopic(topic string, t2id map[string][16]byte) fetchSessionTopic { + if s.killed { + return nil + } + if s.used == nil { + s.used = make(map[string]map[int32]fetchSessionOffsetEpoch) + s.t2id = make(map[string][16]byte) + } + t := s.used[topic] + if t == nil { + t = make(map[int32]fetchSessionOffsetEpoch) + s.used[topic] = t + id := t2id[topic] + s.t2id[topic] = id + if id == ([16]byte{}) { + s.disableIDs = true + } + } + return t +} + +type fetchSessionOffsetEpoch struct { + offset int64 + epoch int32 +} + +type fetchSessionTopic map[int32]fetchSessionOffsetEpoch + +// hasPartitionAt is a pure query: does s already track partition at the +// given offset/epoch? It MUST NOT mutate s. Session-used writes happen +// only after a successful response, via fetchSession.commitFromReq. +func (s fetchSessionTopic) hasPartitionAt(partition int32, offset int64, epoch int32) bool { + if s == nil { // if we are nil, the session was killed + return false + } + at, exists := s[partition] + return exists && at == fetchSessionOffsetEpoch{offset, epoch} +} + +/////////// +// SHARE // +/////////// + +type shareBufferedFetch struct { + fetch Fetch + doneFetch chan<- bool +} + +func (s *source) signalShareAcks() { + select { + case s.share.ackCh <- struct{}{}: + default: + } + s.maybeShareConsume() +} + +func (s *source) signalShareAckFlush() { + select { + case s.share.ackFlushCh <- struct{}{}: + default: + } + s.maybeShareConsume() +} + +func (s *source) resetShareSession() { + s.share.mu.Lock() + prev := s.share.sessionEpoch + s.share.sessionEpoch = 0 + clear(s.share.sessionParts) // must also be cleared, else we'll have a corrupted session + s.share.mu.Unlock() + s.cl.cfg.logger.Log(LogLevelDebug, "resetting share session", + "broker", logID(s.nodeID), + "prev_session_epoch", prev, + ) +} + +// bumpShareSessionEpochIfCurrent increments the session epoch only if +// it still matches the caller-supplied epoch. Used after top-level +// errors that are NOT session-destroying: a concurrent manage +// goroutine may have reset our session mid-flight (UnknownMemberID +// path), in which case we must not bump because the reset set the +// epoch to 0. +func (s *source) bumpShareSessionEpochIfCurrent(epoch int32) { + s.share.mu.Lock() + if s.share.sessionEpoch == epoch { + s.share.sessionEpoch++ + } + s.share.mu.Unlock() +} + +func (s *source) removeShareCursor(c *shareCursor) { + s.share.mu.Lock() + if c.cursorsIdx != len(s.share.cursors)-1 { + s.share.cursors[c.cursorsIdx], s.share.cursors[len(s.share.cursors)-1] = s.share.cursors[len(s.share.cursors)-1], nil + s.share.cursors[c.cursorsIdx].cursorsIdx = c.cursorsIdx + } else { + s.share.cursors[c.cursorsIdx] = nil + } + s.share.cursors = s.share.cursors[:len(s.share.cursors)-1] + if s.share.cursorsStart == len(s.share.cursors) { + s.share.cursorsStart = 0 + } + s.share.mu.Unlock() + // We don't ned to wake the source to send this is a forgotten + // partition, but it doesn't hurt. + s.maybeShareConsume() +} + +func (s *source) addShareCursor(add *shareCursor) { + s.share.mu.Lock() + add.cursorsIdx = len(s.share.cursors) + s.share.cursors = append(s.share.cursors, add) + s.share.mu.Unlock() + // A cursor migrating between sources may carry pending acks + // from the old source. Signal the new source's loop so those + // acks are drained rather than stranded. + // + // No lost-signal window with concurrent appendAck on the same + // cursor: appendAck always (a) acquires c.ackMu to append the + // new entry, then (b) releases c.ackMu, then (c) atomically + // loads c.source and signals it. The interleavings: + // + // - addShareCursor reads pending under c.ackMu BEFORE + // appendAck appends: hasPending may be false here, but + // appendAck's later c.source.Load() returns this new + // source (applyMoves stored it before calling + // addShareCursor) and signals us directly. + // - addShareCursor reads pending AFTER appendAck appends: + // hasPending is true and we signal here. appendAck's + // concurrent c.source-read+signal is a harmless duplicate. + // + // Either way, every appended entry is followed by a signal to + // the cursor's current source. The same reasoning covers + // pendingGaps via enqueueGaps' signal. + add.ackMu.Lock() + hasPending := len(add.pendingAcks) > 0 || len(add.pendingGaps) > 0 + add.ackMu.Unlock() + if hasPending { + s.signalShareAcks() + } + s.maybeShareConsume() +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/strftime.go b/vendor/github.com/twmb/franz-go/pkg/kgo/strftime.go new file mode 100644 index 00000000000..6ff862fbf50 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/strftime.go @@ -0,0 +1,205 @@ +package kgo + +import ( + "strconv" + "time" +) + +// NOTE: this code is copied from github.com/twmb/go-strftime, with AppendFormat +// being unexported. + +// appendFormat appends t to dst according to the input strftime format. +// +// this does not take into account locale; some high level differences: +// +// %E and %O are stripped, as well as a single subsequent alpha char +// %x is DD/MM/YY +// %c is time.ANSIC +// +// In normal strftime, %a, %A, %b, %B, %c, %p, %P, %r, %x, and %X are all +// affected by locale. This package hardcodes the implementation to mirror +// LC_TIME=C (minus %x). Every strftime(3) formatter is accounted for. +func strftimeAppendFormat(dst []byte, format string, t time.Time) []byte { + for i := 0; i < len(format); i++ { + c := format[i] + if c != '%' || i == len(format)-1 { + dst = append(dst, c) + continue + } + + i++ + c = format[i] + switch c { + default: + dst = append(dst, '%', c) + case 'a': // abbrev day + dst = t.AppendFormat(dst, "Mon") + case 'A': // full day + dst = t.AppendFormat(dst, "Monday") + case 'b', 'h': // abbrev month, h is equivalent to b + dst = t.AppendFormat(dst, "Jan") + case 'B': // full month + dst = t.AppendFormat(dst, "January") + case 'c': // preferred date and time representation + dst = t.AppendFormat(dst, time.ANSIC) + case 'C': // century (year/100) as two digit num + dst = append0Pad(dst, t.Year()/100, 2) + case 'd': // day of month as two digit num + dst = append0Pad(dst, t.Day(), 2) + case 'D': // %m/%d/%y + dst = append0Pad(dst, int(t.Month()), 2) + dst = append(dst, '/') + dst = append0Pad(dst, t.Day(), 2) + dst = append(dst, '/') + dst = append0Pad(dst, t.Year()%100, 2) + case 'e': // day of month as num like %d, but leading 0 is space instead + dst = appendSpacePad(dst, t.Day()) + case 'E', 'O': // modifier, ignored and skip next (if ascii) + if i+1 < len(format) { + next := format[i+1] + if 'a' <= next && next <= 'z' || 'A' <= next && next <= 'Z' { + i++ + } + } + case 'F': // %Y-%m-%d (iso8601) + dst = strconv.AppendInt(dst, int64(t.Year()), 10) + dst = append(dst, '-') + dst = append0Pad(dst, int(t.Month()), 2) + dst = append(dst, '-') + dst = append0Pad(dst, t.Day(), 2) + case 'G': // iso8601 week-based year + year, _ := t.ISOWeek() + dst = append0Pad(dst, year, 4) + case 'g': // like %G, but two digit year (no century) + year, _ := t.ISOWeek() + dst = append0Pad(dst, year%100, 2) + case 'H': // hour as number on 24hr clock + dst = append0Pad(dst, t.Hour(), 2) + case 'I': // hour as number on 12hr clock + dst = append0Pad(dst, t.Hour()%12, 2) + case 'j': // day of year as decimal number + dst = append0Pad(dst, t.YearDay(), 3) + case 'k': // 24hr as number, space padded + dst = appendSpacePad(dst, t.Hour()) + case 'l': // 12hr as number, space padded + dst = appendSpacePad(dst, t.Hour()%12) + case 'm': // month as number + dst = append0Pad(dst, int(t.Month()), 2) + case 'M': // minute as number + dst = append0Pad(dst, t.Minute(), 2) + case 'n': // newline + dst = append(dst, '\n') + case 'p': // AM or PM + dst = appendAMPM(dst, t.Hour()) + case 'P': // like %p buf lowercase + dst = appendampm(dst, t.Hour()) + case 'r': // %I:%M:%S %p + h := t.Hour() + dst = append0Pad(dst, h%12, 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Minute(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Second(), 2) + dst = append(dst, ' ') + dst = appendAMPM(dst, h) + case 'R': // %H:%M + dst = append0Pad(dst, t.Hour(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Minute(), 2) + case 's': // seconds since epoch + dst = strconv.AppendInt(dst, t.Unix(), 10) + case 'S': // second as number thru 60 for leap second + dst = append0Pad(dst, t.Second(), 2) + case 't': // tab + dst = append(dst, '\t') + case 'T': // %H:%M:%S + dst = append0Pad(dst, t.Hour(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Minute(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Second(), 2) + case 'u': // day of week as num; Monday is 1 + day := byte(t.Weekday()) + if day == 0 { + day = 7 + } + dst = append(dst, '0'+day) + case 'U': // week number of year starting from first Sunday + dst = append0Pad(dst, (t.YearDay()-int(t.Weekday())+7)/7, 2) + case 'V': // iso8601 week number + _, week := t.ISOWeek() + dst = append0Pad(dst, week, 2) + case 'w': // day of week, 0 to 6, Sunday 0 + dst = strconv.AppendInt(dst, int64(t.Weekday()), 10) + case 'W': // week number of year starting from first Monday + dst = append0Pad(dst, (t.YearDay()-(int(t.Weekday())+6)%7+7)/7, 2) + case 'x': // date representation for current locale; we go DD/MM/YY + dst = append0Pad(dst, t.Day(), 2) + dst = append(dst, '/') + dst = append0Pad(dst, int(t.Month()), 2) + dst = append(dst, '/') + dst = append0Pad(dst, t.Year()%100, 2) + case 'X': // time representation for current locale; we go HH:MM:SS + dst = append0Pad(dst, t.Hour(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Minute(), 2) + dst = append(dst, ':') + dst = append0Pad(dst, t.Second(), 2) + case 'y': // year as num without century + dst = append0Pad(dst, t.Year()%100, 2) + case 'Y': // year as a num + dst = append0Pad(dst, t.Year(), 4) + case 'z': // +hhmm or -hhmm offset from utc + dst = t.AppendFormat(dst, "-0700") + case 'Z': // timezone + dst = t.AppendFormat(dst, "MST") + case '+': // date and time in date(1) format + dst = t.AppendFormat(dst, "Mon Jan _2 15:04:05 MST 2006") + case '%': + dst = append(dst, '%') + } + } + return dst +} + +// all space padded numbers are two length +func appendSpacePad(p []byte, n int) []byte { + if n < 10 { + return append(p, ' ', '0'+byte(n)) + } + return strconv.AppendInt(p, int64(n), 10) +} + +func append0Pad(dst []byte, n, size int) []byte { + switch size { + case 4: + if n < 1000 { + dst = append(dst, '0') + } + fallthrough + case 3: + if n < 100 { + dst = append(dst, '0') + } + fallthrough + case 2: + if n < 10 { + dst = append(dst, '0') + } + } + return strconv.AppendInt(dst, int64(n), 10) +} + +func appendampm(p []byte, h int) []byte { + if h < 12 { + return append(p, 'a', 'm') + } + return append(p, 'p', 'm') +} + +func appendAMPM(p []byte, h int) []byte { + if h < 12 { + return append(p, 'A', 'M') + } + return append(p, 'P', 'M') +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/topics_and_partitions.go b/vendor/github.com/twmb/franz-go/pkg/kgo/topics_and_partitions.go new file mode 100644 index 00000000000..1fb5941996d --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/topics_and_partitions.go @@ -0,0 +1,1043 @@ +package kgo + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync/atomic" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +///////////// +// HELPERS // -- ugly types to eliminate the toil of nil maps and lookups +///////////// + +type tidp struct { + id [16]byte + p int32 +} + +func strtid(tid [16]byte) string { + return base64.RawURLEncoding.EncodeToString(tid[:]) +} + +func dupmsi32(m map[string]int32) map[string]int32 { + d := make(map[string]int32, len(m)) + maps.Copy(d, m) + return d +} + +// "Atomic map of topic partitions", for lack of a better name at this point. +type amtps struct { + v atomic.Value +} + +func (a *amtps) read() map[string][]int32 { + v := a.v.Load() + if v == nil { + return nil + } + return v.(map[string][]int32) +} + +func (a *amtps) write(fn func(map[string][]int32)) { + dup := a.clone() + fn(dup) + a.store(dup) +} + +func (a *amtps) clone() map[string][]int32 { + orig := a.read() + dup := make(map[string][]int32, len(orig)) + for t, ps := range orig { + dup[t] = append(dup[t], ps...) + } + return dup +} + +func mapi32sDeepEq(l, r map[string][]int32) bool { + if len(l) != len(r) { + return false + } + if l == nil && r != nil { + return false + } + if r == nil && l != nil { + return false + } + var dupls, duprs []int32 + for k, lvs := range l { + rvs, ok := r[k] + if !ok { + return false + } + if len(lvs) != len(rvs) { + return false + } + dupls = append(dupls[:0], lvs...) + duprs = append(duprs[:0], rvs...) + slices.Sort(dupls) + slices.Sort(duprs) + if !slices.Equal(dupls, duprs) { + return false + } + } + return true +} + +func (a *amtps) store(m map[string][]int32) { a.v.Store(m) } + +type mtps map[string][]int32 + +func (m mtps) String() string { + var sb strings.Builder + var topicsWritten int + ts := make([]string, 0, len(m)) + var ps []int32 + for t := range m { + ts = append(ts, t) + } + sort.Strings(ts) + for _, t := range ts { + ps = append(ps[:0], m[t]...) + slices.Sort(ps) + topicsWritten++ + fmt.Fprintf(&sb, "%s%v", t, ps) + if topicsWritten < len(m) { + sb.WriteString(", ") + } + } + return sb.String() +} + +type mtmps map[string]map[int32]struct{} // map of topics to map of partitions + +func (m *mtmps) add(t string, p int32) { + if *m == nil { + *m = make(mtmps) + } + mps := (*m)[t] + if mps == nil { + mps = make(map[int32]struct{}) + (*m)[t] = mps + } + mps[p] = struct{}{} +} + +func (m *mtmps) addt(t string) { + if *m == nil { + *m = make(mtmps) + } + mps := (*m)[t] + if mps == nil { + mps = make(map[int32]struct{}) + (*m)[t] = mps + } +} + +func (m mtmps) onlyt(t string) bool { + if m == nil { + return false + } + ps, exists := m[t] + return exists && len(ps) == 0 +} + +func (m mtmps) remove(t string, p int32) { + if m == nil { + return + } + mps, exists := m[t] + if !exists { + return + } + delete(mps, p) + if len(mps) == 0 { + delete(m, t) + } +} + +//////////// +// PAUSED // -- types for pausing topics and partitions +//////////// + +type pausedTopics map[string]pausedPartitions + +type pausedPartitions struct { + all bool + m map[int32]struct{} +} + +func (m pausedTopics) t(topic string) (pausedPartitions, bool) { + if len(m) == 0 { // potentially nil + return pausedPartitions{}, false + } + pps, exists := m[topic] + return pps, exists +} + +func (m pausedTopics) has(topic string, partition int32) (paused bool) { + if len(m) == 0 { + return false + } + pps, exists := m[topic] + if !exists { + return false + } + if pps.all { + return true + } + _, exists = pps.m[partition] + return exists +} + +func (m pausedTopics) addTopics(topics ...string) { + for _, topic := range topics { + pps, exists := m[topic] + if !exists { + pps = pausedPartitions{m: make(map[int32]struct{})} + } + pps.all = true + m[topic] = pps + } +} + +func (m pausedTopics) delTopics(topics ...string) { + for _, topic := range topics { + pps, exists := m[topic] + if !exists { + continue + } + pps.all = false + m[topic] = pps + if !pps.all && len(pps.m) == 0 { + delete(m, topic) + } + } +} + +func (m pausedTopics) addPartitions(topicPartitions map[string][]int32) { + for topic, partitions := range topicPartitions { + pps, exists := m[topic] + if !exists { + pps = pausedPartitions{m: make(map[int32]struct{})} + } + for _, partition := range partitions { + pps.m[partition] = struct{}{} + } + m[topic] = pps + } +} + +func (m pausedTopics) delPartitions(topicPartitions map[string][]int32) { + for topic, partitions := range topicPartitions { + pps, exists := m[topic] + if !exists { + continue + } + for _, partition := range partitions { + delete(pps.m, partition) + } + if !pps.all && len(pps.m) == 0 { + delete(m, topic) + } + } +} + +func (m pausedTopics) pausedTopics() []string { + var r []string + for topic, pps := range m { + if pps.all { + r = append(r, topic) + } + } + return r +} + +func (m pausedTopics) pausedPartitions() map[string][]int32 { + r := make(map[string][]int32) + for topic, pps := range m { + ps := make([]int32, 0, len(pps.m)) + for partition := range pps.m { + ps = append(ps, partition) + } + r[topic] = ps + } + return r +} + +func (m pausedTopics) clone() pausedTopics { + dup := make(pausedTopics) + dup.addTopics(m.pausedTopics()...) + dup.addPartitions(m.pausedPartitions()) + return dup +} + +////////// +// GUTS // -- the key types for storing important metadata for topics & partitions +////////// + +func newTopicPartitions() *topicPartitions { + parts := new(topicPartitions) + parts.v.Store(new(topicPartitionsData)) + return parts +} + +// Contains all information about a topic's partitions. +type topicPartitions struct { + v atomic.Value // *topicPartitionsData + + partsMu xsync.Mutex + partitioner TopicPartitioner + lb *leastBackupInput // for partitioning if the partitioner is a LoadTopicPartitioner +} + +func (t *topicPartitions) load() *topicPartitionsData { return t.v.Load().(*topicPartitionsData) } + +func newTopicsPartitions() *topicsPartitions { + var t topicsPartitions + t.v.Store(make(topicsPartitionsData)) + return &t +} + +// A helper type mapping topics to their partitions; +// this is the inner value of topicPartitions.v. +type topicsPartitionsData map[string]*topicPartitions + +func (d topicsPartitionsData) hasTopic(t string) bool { _, exists := d[t]; return exists } +func (d topicsPartitionsData) loadTopic(t string) *topicPartitionsData { + tp, exists := d[t] + if !exists { + return nil + } + return tp.load() +} + +// A helper type mapping topics to their partitions that can be updated +// atomically. +type topicsPartitions struct { + v atomic.Value // topicsPartitionsData (map[string]*topicPartitions) +} + +func (t *topicsPartitions) load() topicsPartitionsData { + if t == nil { + return nil + } + return t.v.Load().(topicsPartitionsData) +} +func (t *topicsPartitions) storeData(d topicsPartitionsData) { t.v.Store(d) } +func (t *topicsPartitions) storeTopics(topics []string) { t.v.Store(t.ensureTopics(topics)) } +func (t *topicsPartitions) clone() topicsPartitionsData { + current := t.load() + clone := make(map[string]*topicPartitions, len(current)) + maps.Copy(clone, current) + return clone +} + +// Ensures that the topics exist in the returned map, but does not store the +// update. This can be used to update the data and store later, rather than +// storing immediately. +func (t *topicsPartitions) ensureTopics(topics []string) topicsPartitionsData { + var cloned bool + current := t.load() + for _, topic := range topics { + if _, exists := current[topic]; !exists { + if !cloned { + current = t.clone() + cloned = true + } + current[topic] = newTopicPartitions() + } + } + return current +} + +// Opposite of ensureTopics, this purges the input topics and *does* store. +func (t *topicsPartitions) purgeTopics(topics []string) { + var cloned bool + current := t.load() + for _, topic := range topics { + if _, exists := current[topic]; exists { + if !cloned { + current = t.clone() + cloned = true + } + delete(current, topic) + } + } + if cloned { + t.storeData(current) + } +} + +// Updates the topic partitions data atomic value. +// +// If this is the first time seeing partitions, we do processing of unknown +// partitions that may be buffered for producing. +func (cl *Client) storePartitionsUpdate(topic string, l *topicPartitions, lv *topicPartitionsData, hadPartitions bool) { + // If the topic already had partitions, then there would be no + // unknown topic waiting and we do not need to notify anything. + if hadPartitions { + l.v.Store(lv) + return + } + + p := &cl.producer + + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + // If the topic did not have partitions, then we need to store the + // partition update BEFORE unlocking the mutex to guard against this + // sequence of events: + // + // - unlock waiters + // - delete waiter + // - new produce recreates waiter + // - we store update + // - we never notify the recreated waiter + // + // By storing before releasing the locks, we ensure that later + // partition loads for this topic under the mu will see our update. + defer l.v.Store(lv) + + // If there are no unknown topics or this topic is not unknown, then we + // have nothing to do. + if len(p.unknownTopics) == 0 { + return + } + unknown, exists := p.unknownTopics[topic] + if !exists { + return + } + + // If we loaded no partitions because of a retryable error, we signal + // the waiting goroutine that a try happened. It is possible the + // goroutine is quitting and will not be draining unknownWait, so we do + // not require the send. + if len(lv.partitions) == 0 && kerr.IsRetriable(lv.loadErr) { + select { + case unknown.wait <- lv.loadErr: + default: + } + return + } + + // Either we have a fatal error or we can successfully partition. + // + // Even with a fatal error, if we loaded any partitions, we partition. + // If we only had a fatal error, we can finish promises in a goroutine. + // If we are partitioning, we have to do it under the unknownMu to + // ensure prior buffered records are produced in order before we + // release the mu. + delete(p.unknownTopics, topic) + close(unknown.wait) // allow waiting goroutine to quit + + if len(lv.partitions) == 0 { + cl.producer.promiseBatch(batchPromise{ + recs: unknown.buffered, + err: lv.loadErr, + }) + } else { + for _, pr := range unknown.buffered { + cl.doPartition(l, lv, pr) + } + } +} + +// If a metadata request fails after retrying (internally retrying, so only a +// few times), or the metadata request does not return topics that we requested +// (which may also happen additionally consuming via regex), then we need to +// bump errors for topics that were previously loaded, and bump errors for +// topics awaiting load. +// +// This has two modes of operation: +// +// 1. if no topics were missing, then the metadata request failed outright, +// and we need to bump errors on all stored topics and unknown topics. +// +// 2. if topics were missing, then the metadata request was successful but +// had missing data, and we need to bump errors on only what was missing. +func (cl *Client) bumpMetadataFailForTopics(requested map[string]*topicPartitions, err error, missingTopics ...string) { + p := &cl.producer + + // mode 1 + if len(missingTopics) == 0 { + for _, topic := range requested { + for _, topicPartition := range topic.load().partitions { + topicPartition.records.bumpRepeatedLoadErr(err) + } + } + } + + // mode 2 + var missing map[string]bool + for _, failTopic := range missingTopics { + if missing == nil { + missing = make(map[string]bool, len(missingTopics)) + } + missing[failTopic] = true + + if topic, exists := requested[failTopic]; exists { + for _, topicPartition := range topic.load().partitions { + topicPartition.records.bumpRepeatedLoadErr(err) + } + } + } + + p.unknownTopicsMu.Lock() + defer p.unknownTopicsMu.Unlock() + + for topic, unknown := range p.unknownTopics { + // if nil, mode 1 (req err), else mode 2 (missing resp) + if missing != nil && !missing[topic] { + continue + } + + select { + case unknown.wait <- err: + default: + } + } +} + +// topicPartitionsData is the data behind a topicPartitions' v. +// +// We keep this in an atomic because it is expected to be extremely read heavy, +// and if it were behind a lock, the lock would need to be held for a while. +type topicPartitionsData struct { + // NOTE if adding anything to this struct, be sure to fix meta merge. + loadErr error // could be auth, unknown, leader not avail, or creation err + isInternal bool + partitions []*topicPartition // partition num => partition + writablePartitions []*topicPartition // subset of above + topic string + id [16]byte + when int64 +} + +type topicID [16]byte + +func (t topicID) String() string { return hex.EncodeToString(t[:]) } + +// topicPartition contains all information from Kafka for a topic's partition, +// as well as what a client is producing to it or info about consuming from it. +type topicPartition struct { + // If we have a load error (leader/listener/replica not available), we + // keep the old topicPartition data and the new error. + loadErr error + + // If, on metadata refresh, the leader epoch for this partition goes + // backwards, we ignore the metadata refresh and signal the metadata + // should be reloaded: the broker we requested is stale. However, the + // broker could get into a bad state through some weird cluster failure + // scenarios. If we see the epoch rewind repeatedly, we eventually keep + // the metadata refresh. This is not detrimental and at worst will lead + // to the broker telling us to update our metadata. + epochRewinds uint8 + + // If we do not have a load error, we determine if the new + // topicPartition is the same or different from the old based on + // whether the data changed (leader or leader epoch, etc.). + topicPartitionData + + // If we do not have a load error, we copy the records, cursor, or + // shareCursor pointer from the old after updating any necessary + // fields in them (see migrate functions below). + // + // Exactly one of records, cursor, or shareCursor is non-nil. The + // records field is for produce, cursor for classic consume, and + // shareCursor for share consume. shareCursor lives forever on the + // topicPartition; its source field updates on leader migration. + records *recBuf + cursor *cursor + shareCursor *shareCursor +} + +// partitionKind is what role a topicPartition plays for this client: +// produce, classic consume, or share consume. Each partition is exactly +// one of these. +type partitionKind uint8 + +const ( + partitionKindProduce partitionKind = iota + partitionKindConsume + partitionKindShare +) + +func (tp *topicPartition) partition() int32 { + if tp.records != nil { + return tp.records.partition + } + if tp.shareCursor != nil { + return tp.shareCursor.partition + } + return tp.cursor.partition +} + +// Contains stuff that changes on metadata update that we copy into a cursor or +// recBuf. +type topicPartitionData struct { + // Our leader; if metadata sees this change, the metadata update + // migrates the cursor to a different source with the session stopped, + // and the recBuf to a different sink under a tight mutex. + leader int32 + + // What we believe to be the epoch of the leader for this partition. + // + // For cursors, for KIP-320, if a broker receives a fetch request where + // the current leader epoch does not match the brokers, either the + // broker is behind and returns UnknownLeaderEpoch, or we are behind + // and the broker returns FencedLeaderEpoch. For the former, we back + // off and retry. For the latter, we update our metadata. + leaderEpoch int32 +} + +// migrateProductionTo is called on metadata update if a topic partition's sink +// has changed. This moves record production from one sink to the other; this +// must be done such that records produced during migration follow those +// already buffered. +func (old *topicPartition) migrateProductionTo(new *topicPartition) { //nolint:revive // old/new naming makes this clearer + // First, remove our record buffer from the old sink. + old.records.sink.removeRecBuf(old.records) + + // Before this next lock, record producing will buffer to the + // in-migration-progress records and may trigger draining to + // the old sink. That is fine, the old sink no longer consumes + // from these records. We just have wasted drain triggers. + + old.records.mu.Lock() // guard setting sink and topic partition data + old.records.sink = new.records.sink + old.records.topicPartitionData = new.topicPartitionData + // okOnSink tracks "the last response on this recBuf's current sink + // was a success", which gates >1 in-flight per #223. After a sink + // change, a stale true from the old sink could allow pipelining two + // unresolved requests on the new sink before its first ack. Reset so + // the new sink re-earns pipelining through its own response. + old.records.okOnSink = false + old.records.mu.Unlock() + + // After the unlock above, record buffering can trigger drains + // on the new sink, which is not yet consuming from these + // records. Again, just more wasted drain triggers. + + old.records.sink.addRecBuf(old.records) // add our record source to the new sink + + // At this point, the new sink will be draining our records. We lastly + // need to copy the records pointer to our new topicPartition. + new.records = old.records +} + +// migrateCursorTo is called on metadata update if a topic partition's leader +// or leader epoch has changed. +// +// This is a little bit different from above, in that we do this logic only +// after stopping a consumer session. With the consumer session stopped, we +// have fewer concurrency issues to worry about. +func (old *topicPartition) migrateCursorTo( //nolint:revive // old/new naming makes this clearer + new *topicPartition, + css *consumerSessionStopper, +) { + css.stop() + + old.cursor.source.removeCursor(old.cursor) + + // With the session stopped, we can update fields on the old cursor + // with no concurrency issue. + old.cursor.source = new.cursor.source + + // KIP-320: if we had consumed some messages, we need to validate the + // leader epoch on the new broker to see if we experienced data loss + // before we can use this cursor. + // + // Metadata ensures that leaderEpoch is non-negative only if the broker + // supports KIP-320. + if new.leaderEpoch != -1 && old.cursor.lastConsumedEpoch >= 0 { + // Since the cursor consumed messages, it is definitely usable. + // We use it so that the epoch load can finish using it + // properly. + old.cursor.use() + css.reloadOffsets.addLoad(old.cursor.topic, old.cursor.partition, loadTypeEpoch, offsetLoad{ + replica: -1, + Offset: Offset{ + at: old.cursor.offset, + epoch: old.cursor.lastConsumedEpoch, + }, + }) + } + + old.cursor.topicPartitionData = new.topicPartitionData + + old.cursor.source.addCursor(old.cursor) + new.cursor = old.cursor +} + +func (tp *topicPartition) migrateShareCursorTo(cl *Client, new *topicPartition) { + c := tp.shareCursor + cl.sinksAndSourcesMu.Lock() + sns := cl.sinksAndSources[new.leader] + cl.sinksAndSourcesMu.Unlock() + if oldSource := c.source.Load(); oldSource != nil { + oldSource.removeShareCursor(c) + } + c.source.Store(sns.source) + sns.source.addShareCursor(c) + new.shareCursor = c +} + +type kip951move struct { + recBufs map[*recBuf]topicPartitionData + cursors map[*cursor]topicPartitionData + brokers []BrokerMetadata +} + +func (k *kip951move) empty() bool { + return len(k.brokers) == 0 +} + +func (k *kip951move) hasRecBuf(rb *recBuf) bool { + if k == nil || k.recBufs == nil { + return false + } + _, ok := k.recBufs[rb] + return ok +} + +func (k *kip951move) maybeAddProducePartition(resp *kmsg.ProduceResponse, p *kmsg.ProduceResponseTopicPartition, rb *recBuf) bool { + if resp.GetVersion() < 10 || + p.ErrorCode != kerr.NotLeaderForPartition.Code || + len(resp.Brokers) == 0 || + p.CurrentLeader.LeaderID < 0 || + p.CurrentLeader.LeaderEpoch < 0 { + return false + } + if len(k.brokers) == 0 { + for _, rb := range resp.Brokers { + b := BrokerMetadata{ + NodeID: rb.NodeID, + Host: rb.Host, + Port: rb.Port, + Rack: rb.Rack, + } + k.brokers = append(k.brokers, b) + } + } + if k.recBufs == nil { + k.recBufs = make(map[*recBuf]topicPartitionData) + } + k.recBufs[rb] = topicPartitionData{ + leader: p.CurrentLeader.LeaderID, + leaderEpoch: p.CurrentLeader.LeaderEpoch, + } + return true +} + +func (k *kip951move) maybeAddFetchPartition(resp *kmsg.FetchResponse, p *kmsg.FetchResponseTopicPartition, c *cursor) bool { + if resp.GetVersion() < 16 || + p.ErrorCode != kerr.NotLeaderForPartition.Code || + len(resp.Brokers) == 0 || + p.CurrentLeader.LeaderID < 0 || + p.CurrentLeader.LeaderEpoch < 0 { + return false + } + + if len(k.brokers) == 0 { + for _, rb := range resp.Brokers { + b := BrokerMetadata{ + NodeID: rb.NodeID, + Host: rb.Host, + Port: rb.Port, + Rack: rb.Rack, + } + k.brokers = append(k.brokers, b) + } + } + if k.cursors == nil { + k.cursors = make(map[*cursor]topicPartitionData) + } + k.cursors[c] = topicPartitionData{ + leader: p.CurrentLeader.LeaderID, + leaderEpoch: p.CurrentLeader.LeaderEpoch, + } + return true +} + +func (k *kip951move) ensureSinksAndSources(cl *Client) { + cl.sinksAndSourcesMu.Lock() + defer cl.sinksAndSourcesMu.Unlock() + + ensure := func(leader int32) { + if _, exists := cl.sinksAndSources[leader]; exists { + return + } + cl.sinksAndSources[leader] = sinkAndSource{ + sink: cl.newSink(leader), + source: cl.newSource(leader), + } + } + + for _, td := range k.recBufs { + ensure(td.leader) + } + for _, td := range k.cursors { + ensure(td.leader) + } +} + +func (k *kip951move) ensureBrokers(cl *Client) { + if len(k.brokers) == 0 { + return + } + + // We only add or replace individual brokers here, never remove + // unrelated ones. updateBrokers does a full merge-sort that + // removes any existing broker not in the input list. The KIP-951 + // response only includes brokers relevant to the move (a subset), + // so calling updateBrokers would destroy all other brokers. Those + // destroyed brokers get recreated on the next metadata refresh as + // new objects with new connections, which breaks the single- + // connection-per-broker ordering guarantee that Kafka requires + // for idempotent/transactional produce. + cl.brokersMu.Lock() + defer cl.brokersMu.Unlock() + + if cl.stopBrokers { + return + } + + var changed bool + for _, b := range k.brokers { + nb := kmsg.MetadataResponseBroker{ + NodeID: b.NodeID, + Host: b.Host, + Port: b.Port, + Rack: b.Rack, + } + var found bool + for i, existing := range cl.brokers { + if existing.meta.NodeID != b.NodeID { + continue + } + found = true + if !existing.meta.equals(nb) { + existing.stopForever() + cl.brokers[i] = cl.newBroker(b.NodeID, b.Host, b.Port, b.Rack) + changed = true + } + break + } + if !found { + cl.brokers = append(cl.brokers, cl.newBroker(b.NodeID, b.Host, b.Port, b.Rack)) + changed = true + } + } + + if changed { + sort.Slice(cl.brokers, func(i, j int) bool { + return cl.brokers[i].meta.NodeID < cl.brokers[j].meta.NodeID + }) + cl.reinitAnyBrokerOrd() + } +} + +func (k *kip951move) maybeBeginMove(cl *Client) { + if k.empty() { + return + } + // We want to do the move independent of whatever is calling us, BUT we + // want to ensure it is not concurrent with a metadata request. + go cl.blockingMetadataFn(func() { + k.ensureBrokers(cl) + k.ensureSinksAndSources(cl) + k.doMove(cl) + }) +} + +func (k *kip951move) doMove(cl *Client) { + // Moving partitions is theoretically simple, but the client is written + // in a confusing way around concurrency. + // + // The problem is that topicPartitionsData is read-only after + // initialization. Updates are done via atomic stores of the containing + // topicPartitionsData struct. Moving a single partition requires some + // deep copying. + + // oldNew pairs what NEEDS to be atomically updated (old; left value) + // with the value that WILL be stored (new; right value). + type oldNew struct { + l *topicPartitions + r *topicPartitionsData + } + topics := make(map[string]oldNew) + + // getT returns the oldNew for the topic, performing a shallow clone of + // the old whole-topic struct. + getT := func(m topicsPartitionsData, topic string) (oldNew, bool) { + lr, ok := topics[topic] + if !ok { + l := m[topic] + if l == nil { + return oldNew{}, false + } + dup := *l.load() + r := &dup + r.writablePartitions = slices.Clone(r.writablePartitions) + r.partitions = slices.Clone(r.partitions) + lr = oldNew{l, r} + topics[topic] = lr + } + return lr, true + } + + // modifyP returns the old topicPartition and a new one that will be + // used in migrateTo. The new topicPartition only contains the sink + // and topicPartitionData that will be copied into old under old's + // mutex. The actual migration is done in the migrate function (see + // below). + // + // A migration is not needed if the old value has a higher leader + // epoch. If the leader epoch is equal, we check if the leader is the + // same (this allows easier injection of failures in local testing). A + // higher epoch can come from a concurrent metadata update that + // actually performed the move first. + modifyP := func(d *topicPartitionsData, partition int32, td topicPartitionData) (old, new *topicPartition, modified bool) { + old = d.partitions[partition] + if old.leaderEpoch > td.leaderEpoch { + return nil, nil, false + } + if old.leaderEpoch == td.leaderEpoch && old.leader == td.leader { + return nil, nil, false + } + + cl.sinksAndSourcesMu.Lock() + sns := cl.sinksAndSources[td.leader] + cl.sinksAndSourcesMu.Unlock() + + dup := *old + new = &dup + new.topicPartitionData = topicPartitionData{ + leader: td.leader, + leaderEpoch: td.leaderEpoch, + } + if new.records != nil { + new.records = &recBuf{ + sink: sns.sink, + topicPartitionData: new.topicPartitionData, + } + } else { + new.cursor = &cursor{ + source: sns.source, + topicPartitionData: new.topicPartitionData, + } + } + + // We now have to mirror the new partition back to the topic + // slice that will be atomically stored. + d.partitions[partition] = new + idxWritable := sort.Search(len(d.writablePartitions), func(i int) bool { return d.writablePartitions[i].partition() >= partition }) + if idxWritable < len(d.writablePartitions) && d.writablePartitions[idxWritable].partition() == partition { + if d.writablePartitions[idxWritable] != old { + panic("invalid invariant -- partition in writablePartitions != partition at expected index in partitions") + } + d.writablePartitions[idxWritable] = new + } + + return old, new, true + } + + if k.recBufs != nil { + tpsProducer := cl.producer.topics.load() // must be non-nil, since we have recBufs to move + for recBuf, td := range k.recBufs { + lr, ok := getT(tpsProducer, recBuf.topic) + if !ok { + continue // perhaps concurrently purged + } + old, new, modified := modifyP(lr.r, recBuf.partition, td) + if modified { + cl.cfg.logger.Log(LogLevelInfo, "moving producing partition due to kip-951 not_leader_for_partition", + "topic", recBuf.topic, + "partition", recBuf.partition, + "new_leader", new.leader, + "new_leader_epoch", new.leaderEpoch, + "old_leader", old.leader, + "old_leader_epoch", old.leaderEpoch, + ) + old.migrateProductionTo(new) + } else { + recBuf.clearFailing() + } + } + } else { + var tpsConsumer topicsPartitionsData + c := &cl.consumer + switch { + case c.g != nil: + tpsConsumer = c.g.tps.load() + case c.d != nil: + tpsConsumer = c.d.tps.load() + } + css := &consumerSessionStopper{cl: cl} + defer css.maybeRestart() + for cursor, td := range k.cursors { + lr, ok := getT(tpsConsumer, cursor.topic) + if !ok { + continue // perhaps concurrently purged + } + old, new, modified := modifyP(lr.r, cursor.partition, td) + if modified { + cl.cfg.logger.Log(LogLevelInfo, "moving consuming partition due to kip-951 not_leader_for_partition", + "topic", cursor.topic, + "partition", cursor.partition, + "new_leader", new.leader, + "new_leader_epoch", new.leaderEpoch, + "old_leader", old.leader, + "old_leader_epoch", old.leaderEpoch, + ) + old.migrateCursorTo(new, css) + } + } + } + + // We can always do a simple store. For producing, we *must* have + // had partitions, so this is not updating an unknown topic. + for _, lr := range topics { + lr.l.v.Store(lr.r) + } +} + +// Migrating a cursor requires stopping any consumer session. If we +// stop a session, we need to eventually re-start any offset listing or +// epoch loading that was stopped. Thus, we simply merge what we +// stopped into what we will reload. +type consumerSessionStopper struct { + cl *Client + stopped bool + reloadOffsets listOrEpochLoads + tpsPrior *topicsPartitions +} + +func (css *consumerSessionStopper) stop() { + if css.stopped { + return + } + css.stopped = true + loads, tps := css.cl.consumer.stopSession() + css.reloadOffsets.mergeFrom(loads) + css.tpsPrior = tps +} + +func (css *consumerSessionStopper) maybeRestart() { + if !css.stopped { + return + } + session := css.cl.consumer.startNewSession(css.tpsPrior) + defer session.decWorker() + css.reloadOffsets.loadWithSession(session, "resuming reload offsets after session stopped for cursor migrating in metadata") +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kgo/txn.go b/vendor/github.com/twmb/franz-go/pkg/kgo/txn.go new file mode 100644 index 00000000000..289777e25b4 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kgo/txn.go @@ -0,0 +1,1165 @@ +package kgo + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo/internal/xsync" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func ctx2fn(ctx context.Context) func() context.Context { return func() context.Context { return ctx } } + +// TransactionEndTry is simply a named bool. +type TransactionEndTry bool + +const ( + // TryAbort attempts to end a transaction with an abort. + TryAbort TransactionEndTry = false + + // TryCommit attempts to end a transaction with a commit. + TryCommit TransactionEndTry = true +) + +// GroupTransactSession abstracts away the proper way to begin and end a +// transaction when consuming in a group, modifying records, and producing +// (EOS). +// +// If you are running Kafka 2.5+, it is strongly recommended that you also use +// RequireStableFetchOffsets. See that config option's documentation for more +// details. +type GroupTransactSession struct { + cl *Client + + failMu xsync.Mutex + + revoked bool + revokedCh chan struct{} // closed once when revoked is set; reset after End + lost bool + lostCh chan struct{} // closed once when lost is set; reset after End +} + +// NewGroupTransactSession is exactly the same as NewClient, but wraps the +// client's OnPartitionsRevoked / OnPartitionsLost to ensure that transactions +// are correctly aborted whenever necessary so as to properly provide EOS. +// +// When ETLing in a group in a transaction, if a rebalance happens before the +// transaction is ended, you either (a) must block the rebalance from finishing +// until you are done producing, and then commit before unblocking, or (b) +// allow the rebalance to happen, but abort any work you did. +// +// The problem with (a) is that if your ETL work loop is slow, you run the risk +// of exceeding the rebalance timeout and being kicked from the group. You will +// try to commit, and depending on the Kafka version, the commit may even be +// erroneously successful (pre Kafka 2.5). This will lead to duplicates. +// +// Instead, for safety, a GroupTransactSession favors (b). If a rebalance +// occurs at any time before ending a transaction with a commit, this will +// abort the transaction. +// +// This leaves the risk that ending the transaction itself exceeds the +// rebalance timeout, but this is just one request with no cpu logic. With a +// proper rebalance timeout, this single request will not fail and the commit +// will succeed properly. +// +// If this client detects you are talking to a pre-2.5 cluster, OR if you have +// not enabled RequireStableFetchOffsets, the client will sleep for 200ms after +// a successful commit to allow Kafka's txn markers to propagate. This is not +// foolproof in the event of some extremely unlikely communication patterns and +// **potentially** could allow duplicates. See this repo's transaction's doc +// for more details. +func NewGroupTransactSession(opts ...Opt) (*GroupTransactSession, error) { + s := &GroupTransactSession{ + revokedCh: make(chan struct{}), + lostCh: make(chan struct{}), + } + + var noGroup error + + // We append one option, which will get applied last. Because it is + // applied last, we can execute some logic and override some existing + // options. + opts = append(opts, groupOpt{func(cfg *cfg) { + if cfg.group == "" { + cfg.seedBrokers = nil // force a validation error + noGroup = errors.New("missing required group") + return + } + + userRevoked := cfg.onRevoked + cfg.onRevoked = func(ctx context.Context, cl *Client, rev map[string][]int32) { + s.failMu.Lock() + defer s.failMu.Unlock() + if s.revoked { + return + } + + if cl.consumer.g.cooperative.Load() && len(rev) == 0 && !s.revoked { + cl.cfg.logger.Log(LogLevelInfo, "transact session in on_revoke with nothing to revoke; allowing next commit") + } else { + cl.cfg.logger.Log(LogLevelInfo, "transact session in on_revoke; aborting next commit if we are currently in a transaction") + s.revoked = true + close(s.revokedCh) + } + + if userRevoked != nil { + userRevoked(ctx, cl, rev) + } + } + + userLost := cfg.onLost + cfg.onLost = func(ctx context.Context, cl *Client, lost map[string][]int32) { + s.failMu.Lock() + defer s.failMu.Unlock() + if s.lost { + return + } + + cl.cfg.logger.Log(LogLevelInfo, "transact session in on_lost; aborting next commit if we are currently in a transaction") + s.lost = true + close(s.lostCh) + + if userLost != nil { + userLost(ctx, cl, lost) + } else if userRevoked != nil { + userRevoked(ctx, cl, lost) + } + } + }}) + + cl, err := NewClient(opts...) + if err != nil { + if noGroup != nil { + err = noGroup + } + return nil, err + } + s.cl = cl + return s, nil +} + +// Client returns the underlying client that this transact session wraps. This +// can be useful for functions that require a client, such as raw requests. The +// returned client should not be used to manage transactions (leave that to the +// GroupTransactSession). +func (s *GroupTransactSession) Client() *Client { + return s.cl +} + +// Close is a wrapper around Client.Close, with the exact same semantics. +// Refer to that function's documentation. +// +// This function must be called to leave the group before shutting down. +func (s *GroupTransactSession) Close() { + s.cl.Close() +} + +// AllowRebalance is a wrapper around Client.AllowRebalance, with the exact +// same semantics. Refer to that function's documentation. +func (s *GroupTransactSession) AllowRebalance() { + s.cl.AllowRebalance() +} + +// CloseAllowingRebalance is a wrapper around Client.CloseAllowingRebalance, +// with the exact same semantics. Refer to that function's documentation. +func (s *GroupTransactSession) CloseAllowingRebalance() { + s.cl.CloseAllowingRebalance() +} + +// PollFetches is a wrapper around Client.PollFetches, with the exact same +// semantics. Refer to that function's documentation. +// +// It is invalid to call PollFetches concurrently with Begin or End. +func (s *GroupTransactSession) PollFetches(ctx context.Context) Fetches { + return s.cl.PollFetches(ctx) +} + +// PollRecords is a wrapper around Client.PollRecords, with the exact same +// semantics. Refer to that function's documentation. +// +// It is invalid to call PollRecords concurrently with Begin or End. +func (s *GroupTransactSession) PollRecords(ctx context.Context, maxPollRecords int) Fetches { + return s.cl.PollRecords(ctx, maxPollRecords) +} + +// ProduceSync is a wrapper around Client.ProduceSync, with the exact same +// semantics. Refer to that function's documentation. +// +// It is invalid to call ProduceSync concurrently with Begin or End. +func (s *GroupTransactSession) ProduceSync(ctx context.Context, rs ...*Record) ProduceResults { + return s.cl.ProduceSync(ctx, rs...) +} + +// Produce is a wrapper around Client.Produce, with the exact same semantics. +// Refer to that function's documentation. +// +// It is invalid to call Produce concurrently with Begin or End. +func (s *GroupTransactSession) Produce(ctx context.Context, r *Record, promise func(*Record, error)) { + s.cl.Produce(ctx, r, promise) +} + +// TryProduce is a wrapper around Client.TryProduce, with the exact same +// semantics. Refer to that function's documentation. +// +// It is invalid to call TryProduce concurrently with Begin or End. +func (s *GroupTransactSession) TryProduce(ctx context.Context, r *Record, promise func(*Record, error)) { + s.cl.TryProduce(ctx, r, promise) +} + +// Begin begins a transaction, returning an error if the client has no +// transactional id or is already in a transaction. Begin must be called +// before producing records in a transaction. +func (s *GroupTransactSession) Begin() error { + s.cl.cfg.logger.Log(LogLevelInfo, "beginning transact session") + return s.cl.BeginTransaction() +} + +func (s *GroupTransactSession) failed() bool { + return s.revoked || s.lost +} + +// End ends a transaction, committing if commit is true, if the group did not +// rebalance since the transaction began, and if committing offsets is +// successful. If any of these conditions are false, this aborts. This flushes +// or aborts depending on `commit`. +// +// This returns whether the transaction committed or any error that occurred. +// No returned error is retryable. Either the transactional ID has entered a +// failed state, or the client retried so much that the retry limit was hit, +// and odds are you should not continue. While a context is allowed, canceling +// it will likely leave the client in an invalid state. Canceling should only +// be done if you want to shut down. +func (s *GroupTransactSession) End(ctx context.Context, commit TransactionEndTry) (committed bool, err error) { + defer func() { + s.failMu.Lock() + s.revoked = false + s.revokedCh = make(chan struct{}) + s.lost = false + s.lostCh = make(chan struct{}) + s.failMu.Unlock() + }() + + switch commit { + case TryCommit: + if err := s.cl.Flush(ctx); err != nil { + return false, err // we do not abort below, because an error here is ctx closing + } + case TryAbort: + if err := s.cl.AbortBufferedRecords(ctx); err != nil { + return false, err // same + } + } + + wantCommit := bool(commit) + + s.failMu.Lock() + failed := s.failed() + + precommit := s.cl.CommittedOffsets() + postcommit := s.cl.UncommittedOffsets() + s.failMu.Unlock() + + var hasAbortableCommitErr bool + var commitErr error + var g *groupConsumer + + kip447 := false + if wantCommit && !failed { + isAbortableCommitErr := func(err error) bool { + // ILLEGAL_GENERATION: rebalance began and completed + // before we committed. + // + // UNKNOWN_MEMBER_ID: coordinator forgot us (session + // timeout under load). Our consumer_group manage loop + // rejoins with a new member id; the txn just needs to + // abort here so the caller retries on a fresh session. + // Semantically equivalent to ILLEGAL_GENERATION. + // + // STALE_MEMBER_EPOCH: 848-side equivalent of + // ILLEGAL_GENERATION -- our epoch drifted (e.g. a + // heartbeat response was lost), the 848 manage loop + // rejoins, same recovery as above. + // + // REBALANCE_IN_PREGRESS: rebalance began, abort. + // + // COORDINATOR_NOT_AVAILABLE, + // COORDINATOR_LOAD_IN_PROGRESS, + // NOT_COORDINATOR: request failed too many times + // + // CONCURRENT_TRANSACTIONS: Kafka not harmonized, + // we can just abort. + // + // UNKNOWN_SERVER_ERROR: technically should not happen, + // but we can just abort. Redpanda returns this in + // certain versions. + switch { + case errors.Is(err, kerr.IllegalGeneration), + errors.Is(err, kerr.UnknownMemberID), + errors.Is(err, kerr.StaleMemberEpoch), + errors.Is(err, kerr.RebalanceInProgress), + errors.Is(err, kerr.CoordinatorNotAvailable), + errors.Is(err, kerr.CoordinatorLoadInProgress), + errors.Is(err, kerr.NotCoordinator), + errors.Is(err, kerr.ConcurrentTransactions), + errors.Is(err, kerr.UnknownServerError), + errors.Is(err, kerr.TransactionAbortable): + return true + } + return false + } + + var commitErrs []error + + committed := make(chan struct{}) + g = s.cl.commitTransactionOffsets(ctx, postcommit, + func(_ *kmsg.TxnOffsetCommitRequest, resp *kmsg.TxnOffsetCommitResponse, err error) { + defer close(committed) + if err != nil { + if isAbortableCommitErr(err) { + hasAbortableCommitErr = true + return + } + commitErrs = append(commitErrs, err) + return + } + kip447 = resp.Version >= 3 + + for _, t := range resp.Topics { + for _, p := range t.Partitions { + if err := kerr.ErrorForCode(p.ErrorCode); err != nil { + if isAbortableCommitErr(err) { + hasAbortableCommitErr = true + } else { + commitErrs = append(commitErrs, fmt.Errorf("topic %s partition %d: %w", t.Topic, p.Partition, err)) + } + } + } + } + }, + ) + <-committed + + if len(commitErrs) > 0 { + // Preserve the kerr chain via errors.Join so callers can + // use errors.Is(err, kerr.SomeError) to classify the + // failure. Without this, a raw string join would erase + // the wrapping and force callers to string-match. + commitErr = fmt.Errorf("unable to commit transaction offsets: %w", errors.Join(commitErrs...)) + } + } + + // Now that we have committed our offsets, before we allow them to be + // used, we force a heartbeat. By forcing a heartbeat, if there is no + // error, then we know we have up to RebalanceTimeout to write our + // EndTxnRequest without a problem. + // + // We should not be booted from the group if we receive an ok + // heartbeat, meaning that, as mentioned, we should be able to end the + // transaction safely. + var okHeartbeat bool + var heartbeatRebalance bool + if g != nil && commitErr == nil { + waitHeartbeat := make(chan struct{}) + var heartbeatErr error + select { + case g.heartbeatForceCh <- func(err error) { + defer close(waitHeartbeat) + heartbeatErr = err + }: + select { + case <-waitHeartbeat: + okHeartbeat = heartbeatErr == nil + heartbeatRebalance = errors.Is(heartbeatErr, kerr.RebalanceInProgress) + case <-s.revokedCh: + case <-s.lostCh: + } + case <-s.revokedCh: + case <-s.lostCh: + } + } + + s.failMu.Lock() + + // If we know the broker supports KIP-447, we can unlock immediately + // because RequireStable (always enabled) causes the broker to block + // any rebalance's OffsetFetch from outstanding transactions. + // + // If the broker is too old for KIP-447, we spin up a goroutine that + // sleeps for 500ms before unlocking to give Kafka a chance to avoid + // some odd race that would permit duplicates. + // + // This 500ms is not perfect but it should be well enough time on a + // stable cluster. On an unstable cluster, I still expect clients to be + // slower than intra-cluster communication, but there is a risk. + if kip447 { + defer s.failMu.Unlock() + } else { + defer func() { + if committed { + s.cl.cfg.logger.Log(LogLevelDebug, "sleeping 500ms before allowing a rebalance to continue to give the brokers a chance to write txn markers and avoid duplicates") + go func() { + time.Sleep(500 * time.Millisecond) + s.failMu.Unlock() + }() + } else { + s.failMu.Unlock() + } + }() + } + + // If we have KIP-447, we can safely commit even if the heartbeat + // returned REBALANCE_IN_PROGRESS. The TxnOffsetCommit already + // succeeded; the offsets are stored as pending transactional offsets + // on the broker. RequireStable (always enabled) causes any new + // consumer's OffsetFetch to return UNSTABLE_OFFSET_COMMIT while our + // transaction is pending, blocking it from consuming until our + // EndTransaction completes. This is safe even if the rebalance + // timeout expires and we are kicked from the group: the blocking is + // based on transaction state, not group membership. + canCommitDespiteRebalance := heartbeatRebalance && kip447 + if canCommitDespiteRebalance { + s.cl.cfg.logger.Log(LogLevelInfo, "heartbeat returned RebalanceInProgress, but TxnOffsetCommit succeeded and RequireStable is always enabled; allowing commit") + } + tryCommit := !s.failed() && commitErr == nil && !hasAbortableCommitErr && (okHeartbeat || canCommitDespiteRebalance) + willTryCommit := wantCommit && tryCommit + + s.cl.cfg.logger.Log(LogLevelInfo, "transaction session ending", + "was_failed", s.failed(), + "want_commit", wantCommit, + "can_try_commit", tryCommit, + "will_try_commit", willTryCommit, + ) + + // We have a few potential retryable errors from EndTransaction. + // OperationNotAttempted will be returned at most once. + // + // UnknownServerError should not be returned, but some brokers do: + // technically this is fatal, but there is no downside to retrying + // (even retrying a commit) and seeing if we are successful or if we + // get a better error. + var tries int +retry: + endTxnErr := s.cl.EndTransaction(ctx, TransactionEndTry(willTryCommit)) + tries++ + if endTxnErr != nil && tries < 10 { + switch { + case errors.Is(endTxnErr, kerr.OperationNotAttempted): + s.cl.cfg.logger.Log(LogLevelInfo, "end transaction with commit not attempted; retrying as abort") + willTryCommit = false + goto retry + + case errors.Is(endTxnErr, kerr.TransactionAbortable): + s.cl.cfg.logger.Log(LogLevelInfo, "end transaction returned TransactionAbortable; retrying as abort") + willTryCommit = false + goto retry + + case errors.Is(endTxnErr, kerr.UnknownServerError): + s.cl.cfg.logger.Log(LogLevelInfo, "end transaction with commit unknown server error; retrying") + after := time.NewTimer(s.cl.cfg.retryBackoff(tries)) + select { + case <-after.C: // context canceled; we will see when we retry + case <-s.cl.ctx.Done(): + after.Stop() + } + goto retry + } + } + + if !willTryCommit || endTxnErr != nil { + currentCommit := s.cl.CommittedOffsets() + s.cl.cfg.logger.Log(LogLevelInfo, "transact session resetting to current committed state (potentially after a rejoin)", + "tried_commit", willTryCommit, + "commit_err", endTxnErr, + "state_precommit", precommit, + "state_currently_committed", currentCommit, + ) + s.cl.setOffsets(currentCommit, false) + } else if willTryCommit && endTxnErr == nil { + s.cl.cfg.logger.Log(LogLevelInfo, "transact session successful, setting to newly committed state", + "tried_commit", willTryCommit, + "postcommit", postcommit, + ) + s.cl.setOffsets(postcommit, false) + } + + switch { + case commitErr != nil && endTxnErr == nil: + return false, commitErr + + case commitErr == nil && endTxnErr != nil: + return false, endTxnErr + + case commitErr != nil && endTxnErr != nil: + return false, endTxnErr + + default: // both errs nil + committed = willTryCommit + return willTryCommit, nil + } +} + +// BeginTransaction sets the client to a transactional state, erroring if there +// is no transactional ID, or if the producer is currently in a fatal +// (unrecoverable) state, or if the client is already in a transaction. +// +// This must not be called concurrently with other client functions. +func (cl *Client) BeginTransaction() error { + if cl.cfg.txnID == nil { + return errNotTransactional + } + + cl.producer.txnMu.Lock() + defer cl.producer.txnMu.Unlock() + + if cl.producer.inTxn { + return errors.New("invalid attempt to begin a transaction while already in a transaction") + } + + needRecover, didRecover, err := cl.maybeRecoverProducerID(context.Background()) + if needRecover && !didRecover { + cl.cfg.logger.Log(LogLevelInfo, "unable to begin transaction due to unrecoverable producer id error", "err", err) + return fmt.Errorf("producer ID has a fatal, unrecoverable error, err: %w", err) + } + + cl.producer.inTxn = true + if !cl.producer.tx890p2.Load() && cl.supportsKIP890p2() { + cl.producer.tx890p2.Store(true) + } + cl.producer.producingTxn.Store(true) // allow produces for txns now + cl.cfg.logger.Log(LogLevelInfo, "beginning transaction", "transactional_id", *cl.cfg.txnID) + + return nil +} + +// EndBeginTxnHow controls the safety of how EndAndBeginTransaction executes. +type EndBeginTxnHow uint8 + +const ( + // EndBeginTxnSafe ensures a safe execution of EndAndBeginTransaction. + // This option blocks all produce requests and only resumes produce + // requests when onEnd finishes. Note that some produce requests may + // have finished successfully and records that were a part of a + // transaction may have their promises waiting to be called. + EndBeginTxnSafe EndBeginTxnHow = iota + + // EndBeginTxnUnsafe is a no-op. + // + // Deprecated: Kafka 3.6 removed support for the hacky behavior that + // this option was abusing. Thus, as of Kafka 3.6, this option does not + // work against Kafka. This option also has never worked for Redpanda + // because Redpanda always strictly validated that partitions were a + // part of a transaction. + EndBeginTxnUnsafe +) + +// EndAndBeginTransaction is a combination of Flush, EndTransaction, and +// BeginTransaction. You cannot concurrently produce during this function. +// +// The onEnd function is called with your input context and the result of +// Flush, or EndTransaction if Flush is successful. If onEnd returns an error, +// BeginTransaction is not called and this function returns the result of +// onEnd. Otherwise, this function returns the result of BeginTransaction. See +// the documentation on EndTransaction and BeginTransaction for further +// details. It is invalid to call this function more than once at a time, and +// it is invalid to call concurrent with EndTransaction or BeginTransaction. +// +// This function used to serve more purpose, allowing you to produce +// concurrently while calling this and avoiding flushing, but the internal +// optimizations are no longer valid as of Kafka 4.0 due to KIP-890 changing +// some internal semantics. +func (cl *Client) EndAndBeginTransaction( + ctx context.Context, + _ EndBeginTxnHow, + commit TransactionEndTry, + onEnd func(context.Context, error) error, +) (rerr error) { + defer func() { + rerr = onEnd(ctx, rerr) + if rerr == nil { + rerr = cl.BeginTransaction() + } + }() + if err := cl.Flush(ctx); err != nil { + return err + } + return cl.EndTransaction(ctx, commit) +} + +// AbortBufferedRecords fails all unflushed records with ErrAborted and waits +// for there to be no buffered records. +// +// This accepts a context to quit the wait early, but quitting the wait may +// lead to an invalid state and should only be used if you are quitting your +// application. This function waits to abort records at safe points: if records +// are known to not be in flight. This function is safe to call multiple times +// concurrently, and safe to call concurrent with Flush. +// +// NOTE: This aborting record waits until all inflight requests have known +// responses. The client must wait to ensure no duplicate sequence number +// issues. For more details, and for an immediate alternative, check the +// documentation on UnsafeAbortBufferedRecords. +func (cl *Client) AbortBufferedRecords(ctx context.Context) error { + cl.producer.aborting.Add(1) + defer cl.producer.aborting.Add(-1) + + cl.cfg.logger.Log(LogLevelInfo, "producer state set to aborting; continuing to wait via flushing") + defer cl.cfg.logger.Log(LogLevelDebug, "aborted buffered records") + + // We must clear unknown topics ourselves, because flush just waits + // like normal. + p := &cl.producer + p.unknownTopicsMu.Lock() + for _, unknown := range p.unknownTopics { + select { + case unknown.fatal <- ErrAborting: + default: + } + } + p.unknownTopicsMu.Unlock() + + // Setting the aborting state allows records to fail before + // or after produce requests; thus, now we just flush. + return cl.Flush(ctx) +} + +// UnsafeAbortBufferedRecords fails all unflushed records with ErrAborted and +// waits for there to be no buffered records. This function does NOT wait for +// any inflight produce requests to finish, meaning topics in the client may be +// in an invalid state and producing to an invalid-state topic may cause the +// client to enter a fatal failed state. If you want to produce to topics that +// were unsafely aborted, it is recommended to use PurgeTopicsFromClient to +// forcefully reset the topics before producing to them again. +// +// When producing with idempotency enabled or with transactions, every record +// has a sequence number. The client must wait for inflight requests to have +// responses before failing a record, otherwise the client cannot know if a +// sequence number was seen by the broker and tracked or not seen by the broker +// and not tracked. By unsafely aborting, the client forcefully abandons all +// records, and producing to the topics again may re-use a sequence number and +// cause internal errors. +func (cl *Client) UnsafeAbortBufferedRecords() { + cl.failBufferedRecords(ErrAborting) +} + +// EndTransaction ends a transaction and resets the client's internal state to +// not be in a transaction. +// +// Flush and CommitOffsetsForTransaction must be called before this function; +// this function does not flush and does not itself ensure that all buffered +// records are flushed. If no record yet has caused a partition to be added to +// the transaction, this function does nothing and returns nil. Alternatively, +// AbortBufferedRecords should be called before aborting a transaction to +// ensure that any buffered records not yet flushed will not be a part of a new +// transaction. +// +// If the producer ID has an error and you are trying to commit, this will +// return with kerr.OperationNotAttempted. If this happened, retry +// EndTransaction with TryAbort. If this returns kerr.TransactionAbortable, you +// can retry with TryAbort. You should not retry this function on any other +// error. +// +// It may be possible for the client to recover in a new transaction via +// BeginTransaction if an error is returned from this function: +// +// - Before Kafka 4.0, InvalidProducerIDMapping and InvalidProducerEpoch +// are recoverable +// - UnknownProducerID is recoverable for Kafka 2.5+ +// - TransactionAbortable is always recoverable (after aborting) +// +// Note that canceling the context will likely leave the client in an +// undesirable state, because canceling the context may cancel the in-flight +// EndTransaction request, making it impossible to know whether the commit or +// abort was successful. It is recommended to not cancel the context. +func (cl *Client) EndTransaction(ctx context.Context, commit TransactionEndTry) error { + cl.producer.txnMu.Lock() + defer cl.producer.txnMu.Unlock() + + if !cl.producer.inTxn { + return nil + } + cl.producer.inTxn = false + + cl.producer.producingTxn.Store(false) // forbid any new produces while ending txn + + // anyAdded tracks if any partitions were added to this txn, because + // any partitions written to triggers AddPartitionToTxn, which triggers + // the txn to actually begin within Kafka. + // + // If we consumed at all but did not produce, the transaction ending + // issues AddOffsetsToTxn, which internally adds a __consumer_offsets + // partition to the transaction. Thus, if we added offsets, then we + // also produced. + var anyAdded bool + if g := cl.consumer.g; g != nil { + // We do not lock because we expect commitTransactionOffsets to + // be called *before* ending a transaction. + if g.offsetsAddedToTxn { + g.offsetsAddedToTxn = false + anyAdded = true + } + } else { + cl.cfg.logger.Log(LogLevelDebug, "transaction ending, no group loaded; this must be a producer-only transaction, not consume-modify-produce EOS") + } + + // After the flush, no records are being produced to, and we can set + // addedToTxn to false outside of any mutex. + for _, parts := range cl.producer.topics.load() { + for _, part := range parts.load().partitions { + anyAdded = part.records.addedToTxn.Swap(false) || anyAdded + } + } + + // If no partition was added to a transaction, then we have nothing to commit. + // + // Note that anyAdded is true if the producer ID was failed, meaning we will + // get to the potential recovery logic below if necessary. + if !anyAdded { + cl.cfg.logger.Log(LogLevelDebug, "no records were produced during the commit; thus no transaction was began; ending without doing anything") + return nil + } + + id, epoch, err := cl.producerID(ctx2fn(ctx)) + if err != nil { + if commit { + return kerr.OperationNotAttempted + } + + // If we recovered the producer ID, we return early, since + // there is no reason to issue an abort now that the id is + // different. Otherwise, we issue our EndTxn which will likely + // fail, but that is ok, we will just return error. + _, didRecover, _ := cl.maybeRecoverProducerID(ctx) + if didRecover { + return nil + } + } + + cl.cfg.logger.Log(LogLevelInfo, "ending transaction", + "transactional_id", *cl.cfg.txnID, + "commit", commit, + "producer_id", id, + "epoch", epoch, + ) + + err = cl.doWithConcurrentTransactions(ctx, "EndTxn", func() error { + req := kmsg.NewPtrEndTxnRequest() + req.TransactionalID = *cl.cfg.txnID + req.ProducerID = id + req.ProducerEpoch = epoch + req.Commit = bool(commit) + ctx := ctx // capture a local ctx variable in case we introduce concurrency above later + if !cl.producer.tx890p2.Load() { + ctx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 4}) // v5 is only supported with KIP-890 part 2 + } + resp, err := req.RequestWith(ctx, cl) + if err != nil { + return err + } + if err = kerr.ErrorForCode(resp.ErrorCode); err != nil { + return err + } + if resp.Version >= 5 && resp.ProducerID >= 0 { + cl.producer.id.Store(&producerID{ + id: resp.ProducerID, + epoch: resp.ProducerEpoch, + }) + cl.resetAllProducerSequences() + cl.cfg.logger.Log(LogLevelInfo, "end transaction response successfully received", + "transactional_id", *cl.cfg.txnID, + "commit", commit, + "prior_id", id, + "prior_epoch", epoch, + "new_id", resp.ProducerID, + "new_epoch", resp.ProducerEpoch, + ) + } else { + cl.cfg.logger.Log(LogLevelInfo, "end transaction response successfully received", + "transactional_id", *cl.cfg.txnID, + "commit", commit, + "producer_id", id, + "epoch", epoch, + ) + if !cl.producer.tx890p2.Load() && cl.supportsKIP890p2() { + cl.cfg.logger.Log(LogLevelInfo, "end transaction noticed the cluster now supports KIP-890p2, reloading the producer ID and opting in", + "transactional_id", *cl.cfg.txnID, + "producer_id", id, + "epoch", epoch, + ) + cl.producer.id.Store(&producerID{ + id: id, + epoch: epoch, + err: errReloadProducerID, + }) + } + } + return nil + }) + + // If the returned error is still a Kafka error, this is fatal and we + // need to fail our producer ID we loaded above. + // + // UNKNOWN_SERVER_ERROR can theoretically be returned (not all brokers + // do). This technically is fatal, but we do not really know whether it + // is. We can just return this error and let the caller decide to + // continue, if the caller does continue, we will try something and + // eventually then receive our proper transactional error, if any. + var ke *kerr.Error + if errors.As(err, &ke) && !ke.Retriable && ke.Code != kerr.UnknownServerError.Code { + cl.failProducerID(id, epoch, err) + } + + return err +} + +// This returns if it is necessary to recover the producer ID (it has an +// error), whether it is possible to recover, and, if not, the error. +// +// We call this when beginning a transaction or when ending with an abort. +func (cl *Client) maybeRecoverProducerID(ctx context.Context) (necessary, did bool, err error) { + cl.producer.mu.Lock() + defer cl.producer.mu.Unlock() + + id, epoch, err := cl.producerID(ctx2fn(ctx)) + if err == nil { + return false, false, nil + } + + var ke *kerr.Error + if ok := errors.As(err, &ke); !ok { + // The stored PID error is not a kerr (broker-side) error -- most + // likely a transient network error wrapped in errProducerIDLoadFail + // (dial refused, EOF, etc.) from a broker restart or transient + // unreachability. Rather than surfacing "producer ID has a fatal, + // unrecoverable error" to the caller, flag the PID for reload so + // the next produce/begin re-runs InitProducerID now that the + // broker is (probably) back. + if isRetryableBrokerErr(err) || isAnyDialErr(err) { + cl.producer.id.Store(&producerID{ + id: id, + epoch: epoch, + err: errReloadProducerID, + }) + return true, true, nil + } + return true, false, err + } + + var recoverable bool + if cl.supportsKeyVersion(int16(kmsg.EndTxn), 5) { + // As of KIP-890 / Kafka 4.0, InvalidProducerIDMapping and + // InvalidProducerEpoch are NOT recoverable. Only + // UnknownProducerID and TransactionAbortable are. + recoverable = errors.Is(ke, kerr.UnknownProducerID) || errors.Is(ke, kerr.TransactionAbortable) + } else { + kip360 := cl.producer.idVersion >= 3 && (errors.Is(ke, kerr.UnknownProducerID) || errors.Is(ke, kerr.InvalidProducerIDMapping)) + kip588 := cl.producer.idVersion >= 4 && errors.Is(ke, kerr.InvalidProducerEpoch /* || err == kerr.TransactionTimedOut when implemented in Kafka */) + recoverable = kip360 || kip588 + } + + if !recoverable { + return true, false, err // fatal, unrecoverable + } + + // Storing errReloadProducerID will reset sequence numbers as appropriate + // when the producer ID is reloaded successfully. + cl.producer.id.Store(&producerID{ + id: id, + epoch: epoch, + err: errReloadProducerID, + }) + return true, true, nil +} + +// If a transaction is begun too quickly after finishing an old transaction, +// Kafka may still be finalizing its commit / abort and will return a +// concurrent transactions error. We handle that by retrying for a bit. +func (cl *Client) doWithConcurrentTransactions(ctx context.Context, name string, fn func() error) error { + start := time.Now() + tries := 0 + backoff := cl.cfg.txnBackoff + +start: + err := fn() + if errors.Is(err, kerr.ConcurrentTransactions) { + // The longer we are stalled, the more we enforce a minimum + // backoff. + since := time.Since(start) + switch { + case since > time.Second: + if backoff < 200*time.Millisecond { + backoff = 200 * time.Millisecond + } + case since > 5*time.Second/2: + if backoff < 500*time.Millisecond { + backoff = 500 * time.Millisecond + } + case since > 5*time.Second: + if backoff < time.Second { + backoff = time.Second + } + } + + tries++ + cl.cfg.logger.Log(LogLevelDebug, fmt.Sprintf("%s failed with CONCURRENT_TRANSACTIONS, which may be because we ended a txn and began producing in a new txn too quickly; backing off and retrying", name), + "backoff", backoff, + "since_request_tries_start", time.Since(start), + "tries", tries, + ) + select { + case <-time.After(backoff): + case <-ctx.Done(): + cl.cfg.logger.Log(LogLevelError, fmt.Sprintf("abandoning %s retry due to request ctx quitting", name)) + return err + case <-cl.ctx.Done(): + cl.cfg.logger.Log(LogLevelError, fmt.Sprintf("abandoning %s retry due to client ctx quitting", name)) + return err + } + goto start + } + return err +} + +//////////////////////////////////////////////////////////////////////////////////////////// +// TRANSACTIONAL COMMITTING // +// MOSTLY DUPLICATED CODE DUE TO NO GENERICS AND BECAUSE THE TYPES ARE SLIGHTLY DIFFERENT // +//////////////////////////////////////////////////////////////////////////////////////////// + +// commitTransactionOffsets is exactly like CommitOffsets, but specifically for +// use with transactional consuming and producing. +// +// Since this function is a gigantic footgun if not done properly, we hide this +// and only allow transaction sessions to commit. +// +// Unlike CommitOffsets, we do not update the group's uncommitted map. We leave +// that to the calling code to do properly with SetOffsets depending on whether +// an eventual abort happens or not. +func (cl *Client) commitTransactionOffsets( + ctx context.Context, + uncommitted map[string]map[int32]EpochOffset, + onDone func(*kmsg.TxnOffsetCommitRequest, *kmsg.TxnOffsetCommitResponse, error), +) *groupConsumer { + cl.cfg.logger.Log(LogLevelDebug, "in commitTransactionOffsets", "with", uncommitted) + defer cl.cfg.logger.Log(LogLevelDebug, "left commitTransactionOffsets") + + if cl.cfg.txnID == nil { + onDone(nil, nil, errNotTransactional) + return nil + } + + // Before committing, ensure we are at least in a transaction. We + // unlock the producer txnMu before committing to allow EndTransaction + // to go through, even though that could cut off our commit. + cl.producer.txnMu.Lock() + var unlockedTxn bool + unlockTxn := func() { + if !unlockedTxn { + cl.producer.txnMu.Unlock() + } + unlockedTxn = true + } + defer unlockTxn() + if !cl.producer.inTxn { + onDone(nil, nil, errNotInTransaction) + return nil + } + + g := cl.consumer.g + if g == nil { + onDone(kmsg.NewPtrTxnOffsetCommitRequest(), kmsg.NewPtrTxnOffsetCommitResponse(), errNotGroup) + return nil + } + + req, err := g.prepareTxnOffsetCommit(ctx, uncommitted) + if err != nil { + onDone(req, kmsg.NewPtrTxnOffsetCommitResponse(), err) + return g + } + if len(req.Topics) == 0 { + onDone(kmsg.NewPtrTxnOffsetCommitRequest(), kmsg.NewPtrTxnOffsetCommitResponse(), nil) + return g + } + + tx890p2 := cl.producer.tx890p2.Load() + + if !g.offsetsAddedToTxn { + // We only need to issue the AddOffsetsToTxn request if we are + // pre-KIP-890p2, or if we are KIP-890p2 but for some reason + // the broker does not support TxnOffsetCommit v5+. + if !tx890p2 || !cl.supportsKeyVersion(int16(kmsg.TxnOffsetCommit), 5) { + if err := cl.addOffsetsToTxn(ctx, g.cfg.group); err != nil { + if onDone != nil { + onDone(nil, nil, err) + } + return g + } + } + g.offsetsAddedToTxn = true + } + + unlockTxn() + + if err := g.waitJoinSyncMu(ctx); err != nil { + onDone(kmsg.NewPtrTxnOffsetCommitRequest(), kmsg.NewPtrTxnOffsetCommitResponse(), err) + return nil + } + unblockJoinSync := func(req *kmsg.TxnOffsetCommitRequest, resp *kmsg.TxnOffsetCommitResponse, err error) { + g.noCommitDuringJoinAndSync.RUnlock() + onDone(req, resp, err) + } + g.mu.Lock() + defer g.mu.Unlock() + + g.commitTxn(ctx, tx890p2, req, unblockJoinSync) + return g +} + +// Ties a transactional producer to a group. Since this requires a producer ID, +// this initializes one if it is not yet initialized. This would only be the +// case if trying to commit before any records have been sent. +func (cl *Client) addOffsetsToTxn(ctx context.Context, group string) error { + id, epoch, err := cl.producerID(ctx2fn(ctx)) + if err != nil { + return err + } + + err = cl.doWithConcurrentTransactions(ctx, "AddOffsetsToTxn", func() error { // committing offsets without producing causes a transaction to begin within Kafka + cl.cfg.logger.Log(LogLevelInfo, "issuing AddOffsetsToTxn", + "txn", *cl.cfg.txnID, + "producerID", id, + "producerEpoch", epoch, + "group", group, + ) + req := kmsg.NewPtrAddOffsetsToTxnRequest() + req.TransactionalID = *cl.cfg.txnID + req.ProducerID = id + req.ProducerEpoch = epoch + req.Group = group + resp, err := req.RequestWith(ctx, cl) + if err != nil { + return err + } + return kerr.ErrorForCode(resp.ErrorCode) + }) + + // If the returned error is still a Kafka error, this is fatal and we + // need to fail our producer ID we created just above. + // + // We special case UNKNOWN_SERVER_ERROR, because we do not really know + // if this is fatal. If it is, we will catch it later on a better + // error. Some brokers send this when things fail internally, we can + // just abort our commit and see if things are still bad in + // EndTransaction. + var ke *kerr.Error + if errors.As(err, &ke) && !ke.Retriable && ke.Code != kerr.UnknownServerError.Code { + cl.failProducerID(id, epoch, err) + } + + return err +} + +// commitTxn is ALMOST EXACTLY THE SAME as commit, but changed for txn types +// and we avoid updateCommitted. We avoid updating because we manually +// SetOffsets when ending the transaction. +func (g *groupConsumer) commitTxn(ctx context.Context, tx890p2 bool, req *kmsg.TxnOffsetCommitRequest, onDone func(*kmsg.TxnOffsetCommitRequest, *kmsg.TxnOffsetCommitResponse, error)) { + if onDone == nil { // note we must always call onDone + onDone = func(_ *kmsg.TxnOffsetCommitRequest, _ *kmsg.TxnOffsetCommitResponse, _ error) {} + } + + priorDone := g.commitDone + + // Unlike the non-txn consumer, we use the group context for + // transaction offset committing. We want to quit when the group is + // left, and we are not committing when leaving. We rely on proper + // usage of the GroupTransactSession API to issue commits, so there is + // no reason not to use the group context here. + commitCtx, commitCancel := context.WithCancel(g.ctx) // enable ours to be canceled and waited for + commitDone := make(chan struct{}) + + g.commitDone = commitDone + + if ctx.Done() != nil { + go func() { + select { + case <-ctx.Done(): + commitCancel() + case <-commitCtx.Done(): + } + }() + } + + go func() { + defer close(commitDone) // allow future commits to continue when we are done + defer commitCancel() + if priorDone != nil { // wait for any prior request to finish + // Same as commit(): we must NOT cancel the prior commit + // because canceling kills the TCP connection, and our + // subsequent request on a new connection can be processed + // out of order at the broker, rewinding committed offsets. + select { + case <-priorDone: + default: + g.cl.cfg.logger.Log(LogLevelDebug, "waiting for prior txn offset commit to finish before issuing another") + <-priorDone + } + } + g.cl.cfg.logger.Log(LogLevelDebug, "issuing txn offset commit", "uncommitted", req) + + start := time.Now() + ctx := ctx // capture a local ctx variable; do not use the shared one that is concurrently read above + if !tx890p2 { + ctx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 4}) // v5 is only supported with KIP-890 part 2 + } + resp, err := req.RequestWith(ctx, g.cl) + if err != nil { + onDone(req, nil, err) + return + } + g.cl.metrics.observeTime(&g.cl.metrics.cCommitLatency, time.Since(start).Milliseconds()) + + onDone(req, resp, nil) + }() +} + +func (g *groupConsumer) prepareTxnOffsetCommit(ctx context.Context, uncommitted map[string]map[int32]EpochOffset) (*kmsg.TxnOffsetCommitRequest, error) { + req := kmsg.NewPtrTxnOffsetCommitRequest() + + // We're now generating the producerID before addOffsetsToTxn. + // We will not make this request until after addOffsetsToTxn, but it's possible to fail here due to a failed producerID. + id, epoch, err := g.cl.producerID(ctx2fn(ctx)) + if err != nil { + return req, err + } + + req.TransactionalID = *g.cl.cfg.txnID + req.Group = g.cfg.group + req.ProducerID = id + req.ProducerEpoch = epoch + memberID, generation := g.memberGen.load() + req.Generation = generation + req.MemberID = memberID + req.InstanceID = g.cfg.instanceID + + for topic, partitions := range uncommitted { + reqTopic := kmsg.NewTxnOffsetCommitRequestTopic() + reqTopic.Topic = topic + for partition, eo := range partitions { + reqPartition := kmsg.NewTxnOffsetCommitRequestTopicPartition() + reqPartition.Partition = partition + reqPartition.Offset = eo.Offset + reqPartition.LeaderEpoch = eo.Epoch + reqPartition.Metadata = &req.MemberID + reqTopic.Partitions = append(reqTopic.Partitions, reqPartition) + } + req.Topics = append(req.Topics, reqTopic) + } + + if fn, ok := ctx.Value(txnCommitContextFn).(func(*kmsg.TxnOffsetCommitRequest) error); ok { + if err := fn(req); err != nil { + return req, err + } + } + return req, nil +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kmsg/LICENSE b/vendor/github.com/twmb/franz-go/pkg/kmsg/LICENSE new file mode 100644 index 00000000000..36e18034325 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kmsg/LICENSE @@ -0,0 +1,24 @@ +Copyright 2020, Travis Bischel. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/twmb/franz-go/pkg/kmsg/api.go b/vendor/github.com/twmb/franz-go/pkg/kmsg/api.go new file mode 100644 index 00000000000..236433b0e66 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kmsg/api.go @@ -0,0 +1,477 @@ +// Package kmsg contains Kafka request and response types and autogenerated +// serialization and deserialization functions. +// +// This package may bump major versions whenever Kafka makes a backwards +// incompatible protocol change, per the types chosen for this package. For +// example, Kafka can change a field from non-nullable to nullable, which would +// require changing a field from a non-pointer to a pointer. We could get +// around this by making everything an opaque struct and having getters, but +// that is more tedious than having a few rare major version bumps. +// +// If you are using this package directly with kgo, you should either always +// use New functions, or Default functions after creating structs, or you +// should pin the max supported version. If you use New functions, you will +// have safe defaults as new fields are added. If you pin versions, you will +// avoid new fields being used. If you do neither of these, you may opt in to +// new fields that do not have safe zero value defaults, and this may lead to +// errors or unexpected results. +// +// Thus, whenever you initialize a struct from this package, do the following: +// +// struct := kmsg.NewFoo() +// struct.Field = "value I want to set" +// +// Most of this package is generated, but a few things are manual. What is +// manual: all interfaces, the RequestFormatter, record / message / record +// batch reading, and sticky member metadata serialization. +package kmsg + +import ( + "context" + "sort" + + "github.com/twmb/franz-go/pkg/kmsg/internal/kbin" +) + +//go:generate cp ../kbin/primitives.go internal/kbin/ + +// Requestor issues requests. Notably, the kgo.Client and kgo.Broker implements +// Requestor. All Requests in this package have a RequestWith function to have +// type-safe requests. +type Requestor interface { + // Request issues a Request and returns either a Response or an error. + Request(context.Context, Request) (Response, error) +} + +// Request represents a type that can be requested to Kafka. +type Request interface { + // Key returns the protocol key for this message kind. + Key() int16 + // MaxVersion returns the maximum protocol version this message + // supports. + // + // This function allows one to implement a client that chooses message + // versions based off of the max of a message's max version in the + // client and the broker's max supported version. + MaxVersion() int16 + // SetVersion sets the version to use for this request and response. + SetVersion(int16) + // GetVersion returns the version currently set to use for the request + // and response. + GetVersion() int16 + // IsFlexible returns whether the request at its current version is + // "flexible" as per the KIP-482. + IsFlexible() bool + // AppendTo appends this message in wire protocol form to a slice and + // returns the slice. + AppendTo([]byte) []byte + // ReadFrom parses all of the input slice into the response type. + // + // This should return an error if too little data is input. + ReadFrom([]byte) error + // ResponseKind returns an empty Response that is expected for + // this message request. + ResponseKind() Response +} + +// AdminRequest represents a request that must be issued to Kafka controllers. +type AdminRequest interface { + // IsAdminRequest is a method attached to requests that must be + // issed to Kafka controllers. + IsAdminRequest() + Request +} + +// GroupCoordinatorRequest represents a request that must be issued to a +// group coordinator. +type GroupCoordinatorRequest interface { + // IsGroupCoordinatorRequest is a method attached to requests that + // must be issued to group coordinators. + IsGroupCoordinatorRequest() + Request +} + +// TxnCoordinatorRequest represents a request that must be issued to a +// transaction coordinator. +type TxnCoordinatorRequest interface { + // IsTxnCoordinatorRequest is a method attached to requests that + // must be issued to transaction coordinators. + IsTxnCoordinatorRequest() + Request +} + +// ShareCoordinatorRequest represents a request that must be issued to a +// share coordinator. +type ShareCoordinatorRequest interface { + // IsShareCoordinatorRequest is a method attached to requests that + // must be issued to share coordinators. + IsShareCoordinatorRequest() + Request +} + +// Response represents a type that Kafka responds with. +type Response interface { + // Key returns the protocol key for this message kind. + Key() int16 + // MaxVersion returns the maximum protocol version this message + // supports. + MaxVersion() int16 + // SetVersion sets the version to use for this request and response. + SetVersion(int16) + // GetVersion returns the version currently set to use for the request + // and response. + GetVersion() int16 + // IsFlexible returns whether the request at its current version is + // "flexible" as per the KIP-482. + IsFlexible() bool + // AppendTo appends this message in wire protocol form to a slice and + // returns the slice. + AppendTo([]byte) []byte + // ReadFrom parses all of the input slice into the response type. + // + // This should return an error if too little data is input. + ReadFrom([]byte) error + // RequestKind returns an empty Request that is expected for + // this message request. + RequestKind() Request +} + +// UnsafeReadFrom is implemented by all requests and responses generated in +// this package, and switches to using unsafe slice-to-string conversions when +// reading. This can be used to avoid a lot of garbage, but it means you have +// to be careful when using any strings in structs: if you hold onto the +// string, the underlying response slice will not be garbage collected. +type UnsafeReadFrom interface { + UnsafeReadFrom([]byte) error +} + +// ThrottleResponse represents a response that could have a throttle applied by +// Kafka. Any response that implements ThrottleResponse also implements +// SetThrottleResponse. +// +// Kafka 2.0.0 switched throttles from being applied before responses to being +// applied after responses. +type ThrottleResponse interface { + // Throttle returns the response's throttle millis value and + // whether Kafka applies the throttle after the response. + Throttle() (int32, bool) +} + +// SetThrottleResponse sets the throttle in a response that can have a throttle +// applied. Any kmsg interface that implements ThrottleResponse also implements +// SetThrottleResponse. +type SetThrottleResponse interface { + // SetThrottle sets the response's throttle millis value. + SetThrottle(int32) +} + +// TimeoutRequest represents a request that has a TimeoutMillis field. +// Any request that implements TimeoutRequest also implements SetTimeoutRequest. +type TimeoutRequest interface { + // Timeout returns the request's timeout millis value. + Timeout() int32 +} + +// SetTimeoutRequest sets the timeout in a request that can have a timeout +// applied. Any kmsg interface that implements ThrottleRequest also implements +// SetThrottleRequest. +type SetTimeoutRequest interface { + // SetTimeout sets the request's timeout millis value. + SetTimeout(timeoutMillis int32) +} + +// RequestFormatter formats requests. +// +// The default empty struct works correctly, but can be extended with the +// NewRequestFormatter function. +type RequestFormatter struct { + clientID *string +} + +// RequestFormatterOpt applys options to a RequestFormatter. +type RequestFormatterOpt interface { + apply(*RequestFormatter) +} + +type formatterOpt struct{ fn func(*RequestFormatter) } + +func (opt formatterOpt) apply(f *RequestFormatter) { opt.fn(f) } + +// FormatterClientID attaches the given client ID to any issued request, +// minus controlled shutdown v0, which uses its own special format. +func FormatterClientID(id string) RequestFormatterOpt { + return formatterOpt{func(f *RequestFormatter) { f.clientID = &id }} +} + +// NewRequestFormatter returns a RequestFormatter with the opts applied. +func NewRequestFormatter(opts ...RequestFormatterOpt) *RequestFormatter { + a := new(RequestFormatter) + for _, opt := range opts { + opt.apply(a) + } + return a +} + +// AppendRequest appends a full message request to dst, returning the updated +// slice. This message is the full body that needs to be written to issue a +// Kafka request. +func (f *RequestFormatter) AppendRequest( + dst []byte, + r Request, + correlationID int32, +) []byte { + dst = append(dst, 0, 0, 0, 0) // reserve length + k := r.Key() + v := r.GetVersion() + dst = kbin.AppendInt16(dst, k) + dst = kbin.AppendInt16(dst, v) + dst = kbin.AppendInt32(dst, correlationID) + if k == 7 && v == 0 { + return dst + } + + // Even with flexible versions, we do not use a compact client id. + // Clients issue ApiVersions immediately before knowing the broker + // version, and old brokers will not be able to understand a compact + // client id. + dst = kbin.AppendNullableString(dst, f.clientID) + + // The flexible tags end the request header, and then begins the + // request body. + if r.IsFlexible() { + var numTags uint8 + dst = append(dst, numTags) + if numTags != 0 { //nolint:revive,staticcheck // TODO when tags are added + } + } + + // Now the request body. + dst = r.AppendTo(dst) + + kbin.AppendInt32(dst[:0], int32(len(dst[4:]))) + return dst +} + +// StringPtr is a helper to return a pointer to a string. +func StringPtr(in string) *string { + return &in +} + +// ReadFrom provides decoding various versions of sticky member metadata. A key +// point of this type is that it does not contain a version number inside it, +// but it is versioned: if decoding v1 fails, this falls back to v0. +func (s *StickyMemberMetadata) ReadFrom(src []byte) error { + return s.readFrom(src, false) +} + +// UnsafeReadFrom is the same as ReadFrom, but uses unsafe slice to string +// conversions to reduce garbage. +func (s *StickyMemberMetadata) UnsafeReadFrom(src []byte) error { + return s.readFrom(src, true) +} + +func (s *StickyMemberMetadata) readFrom(src []byte, unsafe bool) error { + b := kbin.Reader{Src: src} + numAssignments := b.ArrayLen() + if numAssignments < 0 { + numAssignments = 0 + } + need := numAssignments - int32(cap(s.CurrentAssignment)) + if need > 0 { + s.CurrentAssignment = append(s.CurrentAssignment[:cap(s.CurrentAssignment)], make([]StickyMemberMetadataCurrentAssignment, need)...) + } else { + s.CurrentAssignment = s.CurrentAssignment[:numAssignments] + } + for i := int32(0); i < numAssignments; i++ { + var topic string + if unsafe { + topic = b.UnsafeString() + } else { + topic = b.String() + } + numPartitions := b.ArrayLen() + if numPartitions < 0 { + numPartitions = 0 + } + a := &s.CurrentAssignment[i] + a.Topic = topic + need := numPartitions - int32(cap(a.Partitions)) + if need > 0 { + a.Partitions = append(a.Partitions[:cap(a.Partitions)], make([]int32, need)...) + } else { + a.Partitions = a.Partitions[:numPartitions] + } + for i := range a.Partitions { + a.Partitions[i] = b.Int32() + } + } + if len(b.Src) > 0 { + s.Generation = b.Int32() + } else { + s.Generation = -1 + } + return b.Complete() +} + +// AppendTo provides appending various versions of sticky member metadata to dst. +// If generation is not -1 (default for v0), this appends as version 1. +func (s *StickyMemberMetadata) AppendTo(dst []byte) []byte { + dst = kbin.AppendArrayLen(dst, len(s.CurrentAssignment)) + for _, assignment := range s.CurrentAssignment { + dst = kbin.AppendString(dst, assignment.Topic) + dst = kbin.AppendArrayLen(dst, len(assignment.Partitions)) + for _, partition := range assignment.Partitions { + dst = kbin.AppendInt32(dst, partition) + } + } + if s.Generation != -1 { + dst = kbin.AppendInt32(dst, s.Generation) + } + return dst +} + +// TagReader has is a type that has the ability to skip tags. +// +// This is effectively a trimmed version of the kbin.Reader, with the purpose +// being that kmsg cannot depend on an external package. +type TagReader interface { + // Uvarint returns a uint32. If the reader has read too much and has + // exhausted all bytes, this should set the reader's internal state + // to failed and return 0. + Uvarint() uint32 + + // Span returns n bytes from the reader. If the reader has read too + // much and exhausted all bytes this should set the reader's internal + // to failed and return nil. + Span(n int) []byte +} + +// SkipTags skips tags in a TagReader. +func SkipTags(b TagReader) { + for num := b.Uvarint(); num > 0; num-- { + _, size := b.Uvarint(), b.Uvarint() + b.Span(int(size)) + } +} + +// ReadTags reads tags in a TagReader and returns the tags. +func ReadTags(b TagReader) Tags { + var t Tags + for num := b.Uvarint(); num > 0; num-- { + key, size := b.Uvarint(), b.Uvarint() + t.Set(key, b.Span(int(size))) + } + return t +} + +// internalReadTags reads tags in a reader and returns the tags from a +// duplicated inner kbin.Reader. +func internalReadTags(b *kbin.Reader) Tags { + var t Tags + for num := b.Uvarint(); num > 0; num-- { + key, size := b.Uvarint(), b.Uvarint() + t.Set(key, b.Span(int(size))) + } + return t +} + +// Tags is an opaque structure capturing unparsed tags. +type Tags struct { + keyvals map[uint32][]byte +} + +// Len returns the number of keyvals in Tags. +func (t *Tags) Len() int { return len(t.keyvals) } + +// Each calls fn for each key and val in the tags. +func (t *Tags) Each(fn func(uint32, []byte)) { + if len(t.keyvals) == 0 { + return + } + // We must encode keys in order. We expect to have limited (no) unknown + // keys, so for now, we take a lazy approach and allocate an ordered + // slice. + ordered := make([]uint32, 0, len(t.keyvals)) + for key := range t.keyvals { + ordered = append(ordered, key) + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + for _, key := range ordered { + fn(key, t.keyvals[key]) + } +} + +// Set sets a tag's key and val. +// +// Note that serializing tags does NOT check if the set key overlaps with an +// existing used key. It is invalid to set a key used by Kafka itself. +func (t *Tags) Set(key uint32, val []byte) { + if t.keyvals == nil { + t.keyvals = make(map[uint32][]byte) + } + t.keyvals[key] = val +} + +// AppendEach appends each keyval in tags to dst and returns the updated dst. +func (t *Tags) AppendEach(dst []byte) []byte { + t.Each(func(key uint32, val []byte) { + dst = kbin.AppendUvarint(dst, key) + dst = kbin.AppendUvarint(dst, uint32(len(val))) + dst = append(dst, val...) + }) + return dst +} + +//////////////////////// +// DEPRECATED RENAMES // +//////////////////////// + +// ControlRecordKeyTypeLeaderChange was renamed to ControlRecordKeyTypeSnapshotHeader. +// +// Deprecated: Use ControlRecordKeyTypeSnapshotHeader. +const ControlRecordKeyTypeLeaderChange = ControlRecordKeyTypeSnapshotHeader + +type ( + // ListClientMetricsResourcesRequest was renamed to ListConfigResourcesRequest. + // + // Deprecated: Use ListConfigResourcesRequest. + ListClientMetricsResourcesRequest = ListConfigResourcesRequest + // ListClientMetricsResourcesResponse was renamed to ListConfigResourcesResponse. + // + // Deprecated: Use ListConfigResourcesResponse. + ListClientMetricsResourcesResponse = ListConfigResourcesResponse + // ListClientMetricsResourcesResponseClientMetricsResource was renamed to ListConfigResourcesResponseConfigResource. + // + // Deprecated: Use ListConfigResourcesResponseConfigResource. + ListClientMetricsResourcesResponseClientMetricsResource = ListConfigResourcesResponseConfigResource +) + +// ListClientMetricsResources was renamed to ListConfigResources. +// +// Deprecated: Use ListConfigResources. +var ListClientMetricsResources Key = 74 + +// Deprecated: this was renamed to NewPtrListConfigResourcesRequest. +func NewPtrListClientMetricsResourcesRequest() *ListClientMetricsResourcesRequest { + return NewPtrListConfigResourcesRequest() +} + +// Deprecated: this was renamed to NewListConfigResourcesRequest. +func NewListClientMetricsResourcesRequest() ListClientMetricsResourcesRequest { + return NewListConfigResourcesRequest() +} + +// Deprecated: this was renamed to NewListConfigResourcesResponseConfigResource. +func NewListClientMetricsResourcesResponseClientMetricsResource() ListClientMetricsResourcesResponseClientMetricsResource { + return NewListConfigResourcesResponseConfigResource() +} + +// Deprecated: this was renamed to NewPtrListConfigResourcesResponse. +func NewPtrListClientMetricsResourcesResponse() *ListClientMetricsResourcesResponse { + return NewPtrListConfigResourcesResponse() +} + +// Deprecated: this was renamed to NewListConfigResourcesResponse. +func NewListClientMetricsResourcesResponse() ListClientMetricsResourcesResponse { + return NewListConfigResourcesResponse() +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kmsg/generated.go b/vendor/github.com/twmb/franz-go/pkg/kmsg/generated.go new file mode 100644 index 00000000000..751839ab2c2 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kmsg/generated.go @@ -0,0 +1,69191 @@ +package kmsg + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/twmb/franz-go/pkg/kmsg/internal/kbin" +) + +// Code generated by franz-go/generate. DO NOT EDIT. + +// MaxKey is the maximum key used for any messages in this package. +// Note that this value will change as Kafka adds more messages. +const MaxKey = 92 + +type AssignmentTopicPartition struct { + TopicID [16]byte + + Topic string + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignmentTopicPartition. +func (v *AssignmentTopicPartition) Default() { +} + +// NewAssignmentTopicPartition returns a default AssignmentTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignmentTopicPartition() AssignmentTopicPartition { + var v AssignmentTopicPartition + v.Default() + return v +} + +type TopicInfoConfig struct { + // Key is the config key. + Key string + + // Value is the config value. + Value string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TopicInfoConfig. +func (v *TopicInfoConfig) Default() { +} + +// NewTopicInfoConfig returns a default TopicInfoConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewTopicInfoConfig() TopicInfoConfig { + var v TopicInfoConfig + v.Default() + return v +} + +// MessageV0 is the message format Kafka used prior to 0.10. +// +// To produce or fetch messages, Kafka would write many messages contiguously +// as an array without specifying the array length. +type MessageV0 struct { + // Offset is the offset of this record. + // + // If this is the outer message of a recursive message set (i.e. a + // message set has been compressed and this is the outer message), + // then the offset should be the offset of the last inner value. + Offset int64 + + // MessageSize is the size of everything that follows in this message. + MessageSize int32 + + // CRC is the crc of everything that follows this field (NOT using the + // Castagnoli polynomial, as is the case in the 0.11+ RecordBatch). + CRC int32 + + // Magic is 0. + Magic int8 + + // Attributes describe the attributes of this message. + // + // The first three bits correspond to compression: + // - 00 is no compression + // - 01 is gzip compression + // - 10 is snappy compression + // + // The remaining bits are unused and must be 0. + Attributes int8 + + // Key is an blob of data for a record. + // + // Key's are usually used for hashing the record to specific Kafka partitions. + Key []byte + + // Value is a blob of data. This field is the main "message" portion of a + // record. + Value []byte +} + +func (v *MessageV0) AppendTo(dst []byte) []byte { + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.MessageSize + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CRC + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Magic + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Attributes + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Key + dst = kbin.AppendNullableBytes(dst, v) + } + { + v := v.Value + dst = kbin.AppendNullableBytes(dst, v) + } + return dst +} + +func (v *MessageV0) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *MessageV0) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *MessageV0) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + s := v + { + v := b.Int64() + s.Offset = v + } + { + v := b.Int32() + s.MessageSize = v + } + { + v := b.Int32() + s.CRC = v + } + { + v := b.Int8() + s.Magic = v + } + { + v := b.Int8() + s.Attributes = v + } + { + v := b.NullableBytes() + s.Key = v + } + { + v := b.NullableBytes() + s.Value = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MessageV0. +func (v *MessageV0) Default() { +} + +// NewMessageV0 returns a default MessageV0 +// This is a shortcut for creating a struct and calling Default yourself. +func NewMessageV0() MessageV0 { + var v MessageV0 + v.Default() + return v +} + +// MessageV1 is the message format Kafka used prior to 0.11. +// +// To produce or fetch messages, Kafka would write many messages contiguously +// as an array without specifying the array length. +// +// To support compression, an entire message set would be compressed and used +// as the Value in another message set (thus being "recursive"). The key for +// this outer message set must be null. +type MessageV1 struct { + // Offset is the offset of this record. + // + // Different from v0, if this message set is a recursive message set + // (that is, compressed and inside another message set), the offset + // on the inner set is relative to the offset of the outer set. + Offset int64 + + // MessageSize is the size of everything that follows in this message. + MessageSize int32 + + // CRC is the crc of everything that follows this field (NOT using the + // Castagnoli polynomial, as is the case in the 0.11+ RecordBatch). + CRC int32 + + // Magic is 1. + Magic int8 + + // Attributes describe the attributes of this message. + // + // The first three bits correspond to compression: + // - 00 is no compression + // - 01 is gzip compression + // - 10 is snappy compression + // + // Bit 4 is the timestamp type, with 0 meaning CreateTime corresponding + // to the timestamp being from the producer, and 1 meaning LogAppendTime + // corresponding to the timestamp being from the broker. + // Setting this to LogAppendTime will cause batches to be rejected. + // + // The remaining bits are unused and must be 0. + Attributes int8 + + // Timestamp is the millisecond timestamp of this message. + Timestamp int64 + + // Key is an blob of data for a record. + // + // Key's are usually used for hashing the record to specific Kafka partitions. + Key []byte + + // Value is a blob of data. This field is the main "message" portion of a + // record. + Value []byte +} + +func (v *MessageV1) AppendTo(dst []byte) []byte { + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.MessageSize + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CRC + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Magic + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Attributes + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Timestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Key + dst = kbin.AppendNullableBytes(dst, v) + } + { + v := v.Value + dst = kbin.AppendNullableBytes(dst, v) + } + return dst +} + +func (v *MessageV1) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *MessageV1) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *MessageV1) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + s := v + { + v := b.Int64() + s.Offset = v + } + { + v := b.Int32() + s.MessageSize = v + } + { + v := b.Int32() + s.CRC = v + } + { + v := b.Int8() + s.Magic = v + } + { + v := b.Int8() + s.Attributes = v + } + { + v := b.Int64() + s.Timestamp = v + } + { + v := b.NullableBytes() + s.Key = v + } + { + v := b.NullableBytes() + s.Value = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MessageV1. +func (v *MessageV1) Default() { +} + +// NewMessageV1 returns a default MessageV1 +// This is a shortcut for creating a struct and calling Default yourself. +func NewMessageV1() MessageV1 { + var v MessageV1 + v.Default() + return v +} + +// Header is user provided metadata for a record. Kafka does not look at +// headers at all; they are solely for producers and consumers. +type Header struct { + Key string + + Value []byte +} + +func (v *Header) AppendTo(dst []byte) []byte { + { + v := v.Key + dst = kbin.AppendVarintString(dst, v) + } + { + v := v.Value + dst = kbin.AppendVarintBytes(dst, v) + } + return dst +} + +func (v *Header) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *Header) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *Header) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + s := v + { + var v string + if unsafe { + v = b.UnsafeVarintString() + } else { + v = b.VarintString() + } + s.Key = v + } + { + v := b.VarintBytes() + s.Value = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to Header. +func (v *Header) Default() { +} + +// NewHeader returns a default Header +// This is a shortcut for creating a struct and calling Default yourself. +func NewHeader() Header { + var v Header + v.Default() + return v +} + +// RecordBatch is a Kafka concept that groups many individual records together +// in a more optimized format. +type RecordBatch struct { + // FirstOffset is the first offset in a record batch. + // + // For producing, this is usually 0. + FirstOffset int64 + + // Length is the wire length of everything that follows this field. + Length int32 + + // PartitionLeaderEpoch is the leader epoch of the broker at the time + // this batch was written. Kafka uses this for cluster communication, + // but clients can also use this to better aid truncation detection. + // See KIP-320. Producers should set this to -1. + PartitionLeaderEpoch int32 + + // Magic is the current "magic" number of this message format. + // The current magic number is 2. + Magic int8 + + // CRC is the crc of everything that follows this field using the + // Castagnoli polynomial. + CRC int32 + + // Attributes describe the records array of this batch. + // + // The first three bits correspond to compression: + // - 000 is no compression + // - 001 is gzip compression + // - 010 is snappy compression + // - 011 is lz4 compression + // - 100 is zstd compression (produce request version 7+) + // + // Bit 4 is the timestamp type, with 0 meaning CreateTime corresponding + // to the timestamp being from the producer, and 1 meaning LogAppendTime + // corresponding to the timestamp being from the broker. + // Setting this to LogAppendTime will cause batches to be rejected. + // + // Bit 5 indicates whether the batch is part of a transaction (1 is yes). + // + // Bit 6 indicates if the batch includes a control message (1 is yes). + // Control messages are used to enable transactions and are generated from + // the broker. Clients should not return control batches to applications. + Attributes int16 + + // LastOffsetDelta is the offset of the last message in a batch. This is used + // by the broker to ensure correct behavior even with batch compaction. + LastOffsetDelta int32 + + // FirstTimestamp is the timestamp (in milliseconds) of the first record + // in a batch. + FirstTimestamp int64 + + // MaxTimestamp is the timestamp (in milliseconds) of the last record + // in a batch. Similar to LastOffsetDelta, this is used to ensure correct + // behavior with compacting. + MaxTimestamp int64 + + // ProducerID is the broker assigned producerID from an InitProducerID + // request. + // + // Clients that wish to support idempotent messages and transactions must + // set this field. + // + // Note that when not using transactions, any producer here is always + // accepted (and the epoch is always zero). Outside transactions, the ID + // is used only to deduplicate requests (and there must be at max 5 + // concurrent requests). + ProducerID int64 + + // ProducerEpoch is the broker assigned producerEpoch from an InitProducerID + // request. + // + // Clients that wish to support idempotent messages and transactions must + // set this field. + ProducerEpoch int16 + + // FirstSequence is the producer assigned sequence number used by the + // broker to deduplicate messages. + // + // Clients that wish to support idempotent messages and transactions must + // set this field. + // + // The sequence number for each record in a batch is OffsetDelta + FirstSequence. + FirstSequence int32 + + // NumRecords is the number of records in the array below. + // + // This is separate from Records due to the potential for records to be + // compressed. + NumRecords int32 + + // Records contains records, either compressed or uncompressed. + // + // For uncompressed records, this is an array of records ([Record]). + // + // For compressed records, the length of the uncompressed array is kept + // but everything that follows is compressed. + // + // The number of bytes is expected to be the Length field minus 49. + Records []byte +} + +func (v *RecordBatch) AppendTo(dst []byte) []byte { + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Length + dst = kbin.AppendInt32(dst, v) + } + { + v := v.PartitionLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Magic + dst = kbin.AppendInt8(dst, v) + } + { + v := v.CRC + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Attributes + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LastOffsetDelta + dst = kbin.AppendInt32(dst, v) + } + { + v := v.FirstTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.MaxTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.FirstSequence + dst = kbin.AppendInt32(dst, v) + } + { + v := v.NumRecords + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Records + dst = append(dst, v...) + } + return dst +} + +func (v *RecordBatch) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *RecordBatch) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *RecordBatch) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int32() + s.Length = v + } + { + v := b.Int32() + s.PartitionLeaderEpoch = v + } + { + v := b.Int8() + s.Magic = v + } + { + v := b.Int32() + s.CRC = v + } + { + v := b.Int16() + s.Attributes = v + } + { + v := b.Int32() + s.LastOffsetDelta = v + } + { + v := b.Int64() + s.FirstTimestamp = v + } + { + v := b.Int64() + s.MaxTimestamp = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := b.Int32() + s.FirstSequence = v + } + { + v := b.Int32() + s.NumRecords = v + } + { + v := b.Span(int(s.Length) - 49) + s.Records = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to RecordBatch. +func (v *RecordBatch) Default() { +} + +// NewRecordBatch returns a default RecordBatch +// This is a shortcut for creating a struct and calling Default yourself. +func NewRecordBatch() RecordBatch { + var v RecordBatch + v.Default() + return v +} + +// OffsetCommitKey is the key for the Kafka internal __consumer_offsets topic +// if the key starts with an int16 with a value of 0 or 1. +// +// This type was introduced in KAFKA-1012 commit a670537aa3 with release 0.8.2 +// and has been in use ever since. +type OffsetCommitKey struct { + // Version is which encoding version this value is using. + Version int16 + + // Group is the group being committed. + Group string + + // Topic is the topic being committed. + Topic string + + // Partition is the partition being committed. + Partition int32 +} + +func (v *OffsetCommitKey) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Group + dst = kbin.AppendString(dst, v) + } + { + v := v.Topic + dst = kbin.AppendString(dst, v) + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + return dst +} + +func (v *OffsetCommitKey) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetCommitKey) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetCommitKey) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Group = v + } + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitKey. +func (v *OffsetCommitKey) Default() { +} + +// NewOffsetCommitKey returns a default OffsetCommitKey +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitKey() OffsetCommitKey { + var v OffsetCommitKey + v.Default() + return v +} + +// OffsetCommitValue is the value for the Kafka internal __consumer_offsets +// topic if the key is of OffsetCommitKey type. +// +// Version 0 was introduced with the key version 0. +// +// KAFKA-1634 commit c5df2a8e3a in 0.9.0 released version 1. +// +// KAFKA-4682 commit 418a91b5d4, proposed in KIP-211 and included in 2.1.0 +// released version 2. +// +// KAFKA-7437 commit 9f7267dd2f, proposed in KIP-320 and included in 2.1.0 +// released version 3. +// +// KAFKA-19141 commit 199772a included in 4.1.0 released version 4. +type OffsetCommitValue struct { + // Version is which encoding version this value is using. + Version int16 + + // Offset is the committed offset. + Offset int64 + + // LeaderEpoch is the epoch of the leader committing this message. + LeaderEpoch int32 // v3+ + + // Metadata is the metadata included in the commit. + Metadata string + + // CommitTimestamp is when this commit occurred. + CommitTimestamp int64 + + // ExpireTimestamp, introduced in v1 and dropped in v2 with KIP-111, + // is when this commit expires. + ExpireTimestamp int64 // v1-v1 + + // TopicID is the topic id of the committed offset + TopicID [16]byte // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (v *OffsetCommitValue) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if version >= 3 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.CommitTimestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 && version <= 1 { + v := v.ExpireTimestamp + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + var toEncode []uint32 + if v.TopicID != [16]byte{} { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.TopicID + dst = kbin.AppendUvarint(dst, 0) + dst = kbin.AppendUvarint(dst, 16) + dst = kbin.AppendUuid(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetCommitValue) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetCommitValue) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetCommitValue) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + v := b.Int64() + s.Offset = v + } + if version >= 3 { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Metadata = v + } + { + v := b.Int64() + s.CommitTimestamp = v + } + if version >= 1 && version <= 1 { + v := b.Int64() + s.ExpireTimestamp = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Uuid() + s.TopicID = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} +func (v *OffsetCommitValue) IsFlexible() bool { return v.Version >= 4 } + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitValue. +func (v *OffsetCommitValue) Default() { +} + +// NewOffsetCommitValue returns a default OffsetCommitValue +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitValue() OffsetCommitValue { + var v OffsetCommitValue + v.Default() + return v +} + +// GroupMetadataKey is the key for the Kafka internal __consumer_offsets topic +// if the key starts with an int16 with a value of 2. +// +// This type was introduced in KAFKA-2017 commit 7c33475274 with release 0.9.0 +// and has been in use ever since. +type GroupMetadataKey struct { + // Version is which encoding version this value is using. + Version int16 + + // Group is the group this metadata is for. + Group string +} + +func (v *GroupMetadataKey) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Group + dst = kbin.AppendString(dst, v) + } + return dst +} + +func (v *GroupMetadataKey) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *GroupMetadataKey) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *GroupMetadataKey) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Group = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to GroupMetadataKey. +func (v *GroupMetadataKey) Default() { +} + +// NewGroupMetadataKey returns a default GroupMetadataKey +// This is a shortcut for creating a struct and calling Default yourself. +func NewGroupMetadataKey() GroupMetadataKey { + var v GroupMetadataKey + v.Default() + return v +} + +type GroupMetadataValueMember struct { + // MemberID is a group member. + MemberID string + + // InstanceID is the instance ID of this member in the group (KIP-345). + InstanceID *string // v3+ + + // ClientID is the client ID of this group member. + ClientID string + + // ClientHost is the hostname of this group member. + ClientHost string + + // RebalanceTimeoutMillis is the rebalance timeout of this group member. + // + // This field has a default of -1. + RebalanceTimeoutMillis int32 // v1+ + + // SessionTimeoutMillis is the session timeout of this group member. + SessionTimeoutMillis int32 + + // Subscription is the subscription of this group member. + Subscription []byte + + // Assignment is what the leader assigned this group member. + Assignment []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to GroupMetadataValueMember. +func (v *GroupMetadataValueMember) Default() { + v.RebalanceTimeoutMillis = -1 +} + +// NewGroupMetadataValueMember returns a default GroupMetadataValueMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewGroupMetadataValueMember() GroupMetadataValueMember { + var v GroupMetadataValueMember + v.Default() + return v +} + +// GroupMetadataValue is the value for the Kafka internal __consumer_offsets +// topic if the key is of GroupMetadataKey type. +// +// Version 0 was introduced with the key version 0. +// +// KAFKA-3888 commit 40b1dd3f49, proposed in KIP-62 and included in 0.10.1 +// released version 1. +// +// KAFKA-4682 commit 418a91b5d4, proposed in KIP-211 and included in 2.1.0 +// released version 2. +// +// KAFKA-7862 commit 0f995ba6be, proposed in KIP-345 and included in 2.3.0 +// released version 3. +type GroupMetadataValue struct { + // Version is the version of this value. + Version int16 + + // ProtocolType is the type of protocol being used for the group + // (i.e., "consumer"). + ProtocolType string + + // Generation is the generation of this group. + Generation int32 + + // Protocol is the agreed upon protocol all members are using to partition + // (i.e., "sticky"). + Protocol *string + + // Leader is the group leader. + Leader *string + + // CurrentStateTimestamp is the timestamp for this state of the group + // (stable, etc.). + // + // This field has a default of -1. + CurrentStateTimestamp int64 // v2+ + + // Members are the group members. + Members []GroupMetadataValueMember + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (v *GroupMetadataValue) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Protocol + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Leader + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 2 { + v := v.CurrentStateTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ClientID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ClientHost + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.RebalanceTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.SessionTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Subscription + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.Assignment + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *GroupMetadataValue) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *GroupMetadataValue) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *GroupMetadataValue) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ProtocolType = v + } + { + v := b.Int32() + s.Generation = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Protocol = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Leader = v + } + if version >= 2 { + v := b.Int64() + s.CurrentStateTimestamp = v + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]GroupMetadataValueMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientHost = v + } + if version >= 1 { + v := b.Int32() + s.RebalanceTimeoutMillis = v + } + { + v := b.Int32() + s.SessionTimeoutMillis = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Subscription = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Assignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} +func (v *GroupMetadataValue) IsFlexible() bool { return v.Version >= 4 } + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to GroupMetadataValue. +func (v *GroupMetadataValue) Default() { + v.CurrentStateTimestamp = -1 +} + +// NewGroupMetadataValue returns a default GroupMetadataValue +// This is a shortcut for creating a struct and calling Default yourself. +func NewGroupMetadataValue() GroupMetadataValue { + var v GroupMetadataValue + v.Default() + return v +} + +// TxnMetadataKey is the key for the Kafka internal __transaction_state topic +// if the key starts with an int16 with a value of 0. +type TxnMetadataKey struct { + // Version is the version of this type. + Version int16 + + // TransactionalID is the transactional ID this record is for. + TransactionalID string +} + +func (v *TxnMetadataKey) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.TransactionalID + dst = kbin.AppendString(dst, v) + } + return dst +} + +func (v *TxnMetadataKey) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *TxnMetadataKey) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *TxnMetadataKey) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.TransactionalID = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnMetadataKey. +func (v *TxnMetadataKey) Default() { +} + +// NewTxnMetadataKey returns a default TxnMetadataKey +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnMetadataKey() TxnMetadataKey { + var v TxnMetadataKey + v.Default() + return v +} + +type TxnMetadataValueTopic struct { + // Topic is a topic involved in this transaction. + Topic string + + // Partitions are partitions in this topic involved in the transaction. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnMetadataValueTopic. +func (v *TxnMetadataValueTopic) Default() { +} + +// NewTxnMetadataValueTopic returns a default TxnMetadataValueTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnMetadataValueTopic() TxnMetadataValueTopic { + var v TxnMetadataValueTopic + v.Default() + return v +} + +// TxnMetadataValue is the value for the Kafka internal __transaction_state +// topic if the key is of TxnMetadataKey type. +// +// Version 0 was introduced in 0.11.0. +// +// KAFKA-14869 commit 4b6dcf19dc, proposed in KIP-915 and included in 3.5 +// released version 1. +type TxnMetadataValue struct { + // Version is the version of this value. + Version int16 + + // ProducerID is the ID in use by the transactional ID. + ProducerID int64 + + // ProducerEpoch is the epoch associated with the producer ID. + ProducerEpoch int16 + + // TimeoutMillis is the timeout of this transaction in milliseconds. + TimeoutMillis int32 + + // State is the state this transaction is in, + // 0 is Empty, 1 is Ongoing, 2 is PrepareCommit, 3 is PrepareAbort, 4 is + // CompleteCommit, 5 is CompleteAbort, 6 is Dead, and 7 is PrepareEpochFence. + State TransactionState + + // Topics are topics that are involved in this transaction. + Topics []TxnMetadataValueTopic + + // LastUpdateTimestamp is the timestamp in millis of when this transaction + // was last updated. + LastUpdateTimestamp int64 + + // StartTimestamp is the timestamp in millis of when this transaction started. + StartTimestamp int64 + + // PreviousProducerID is the producer ID used by the last committed + // transaction. KAFKA-14562 commit ede0c94aaa, proposed in KIP-890 and + // included in 4.0. + // + // This field has a default of -1. + PreviousProducerID int64 // tag 0 + + // NextProducerID is the latest producer ID sent to the producer for the + // given transactional ID. KAFKA-14562 commit ede0c94aaa, proposed in + // KIP-890 and included in 4.0. + // + // This field has a default of -1. + NextProducerID int64 // tag 1 + + // ClientTransactionVersion is the transaction version used by the client. + // KAFKA-14562 commit ede0c94aaa, proposed in KIP-890 and included in 4.0. + ClientTransactionVersion int16 // tag 2 + + // NextProducerEpoch is the producer epoch associated with NextProducerID. + // KAFKA-15370 commit 247c0f0ba5, proposed in KIP-939 and included in 4.1.0. + // + // This field has a default of -1. + NextProducerEpoch int16 // tag 3 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (v *TxnMetadataValue) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.State + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.LastUpdateTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.StartTimestamp + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + var toEncode []uint32 + if v.PreviousProducerID != -1 { + toEncode = append(toEncode, 0) + } + if v.NextProducerID != -1 { + toEncode = append(toEncode, 1) + } + if v.ClientTransactionVersion != 0 { + toEncode = append(toEncode, 2) + } + if v.NextProducerEpoch != -1 { + toEncode = append(toEncode, 3) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.PreviousProducerID + dst = kbin.AppendUvarint(dst, 0) + dst = kbin.AppendUvarint(dst, 8) + dst = kbin.AppendInt64(dst, v) + } + case 1: + { + v := v.NextProducerID + dst = kbin.AppendUvarint(dst, 1) + dst = kbin.AppendUvarint(dst, 8) + dst = kbin.AppendInt64(dst, v) + } + case 2: + { + v := v.ClientTransactionVersion + dst = kbin.AppendUvarint(dst, 2) + dst = kbin.AppendUvarint(dst, 2) + dst = kbin.AppendInt16(dst, v) + } + case 3: + { + v := v.NextProducerEpoch + dst = kbin.AppendUvarint(dst, 3) + dst = kbin.AppendUvarint(dst, 2) + dst = kbin.AppendInt16(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *TxnMetadataValue) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *TxnMetadataValue) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *TxnMetadataValue) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + { + var t TransactionState + { + v := b.Int8() + t = TransactionState(v) + } + v := t + s.State = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TxnMetadataValueTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TxnMetadataValueTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int64() + s.LastUpdateTimestamp = v + } + { + v := b.Int64() + s.StartTimestamp = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int64() + s.PreviousProducerID = v + if err := b.Complete(); err != nil { + return err + } + case 1: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int64() + s.NextProducerID = v + if err := b.Complete(); err != nil { + return err + } + case 2: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int16() + s.ClientTransactionVersion = v + if err := b.Complete(); err != nil { + return err + } + case 3: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int16() + s.NextProducerEpoch = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} +func (v *TxnMetadataValue) IsFlexible() bool { return v.Version >= 1 } + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnMetadataValue. +func (v *TxnMetadataValue) Default() { + v.PreviousProducerID = -1 + v.NextProducerID = -1 + v.NextProducerEpoch = -1 +} + +// NewTxnMetadataValue returns a default TxnMetadataValue +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnMetadataValue() TxnMetadataValue { + var v TxnMetadataValue + v.Default() + return v +} + +type StickyMemberMetadataCurrentAssignment struct { + // Topic is a topic the group member is currently assigned. + Topic string + + // Partitions are the partitions within a topic that a group member is + // currently assigned. + Partitions []int32 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StickyMemberMetadataCurrentAssignment. +func (v *StickyMemberMetadataCurrentAssignment) Default() { +} + +// NewStickyMemberMetadataCurrentAssignment returns a default StickyMemberMetadataCurrentAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewStickyMemberMetadataCurrentAssignment() StickyMemberMetadataCurrentAssignment { + var v StickyMemberMetadataCurrentAssignment + v.Default() + return v +} + +// StickyMemberMetadata is is what is encoded in UserData for +// ConsumerMemberMetadata in group join requests with the sticky partitioning +// strategy. +// +// V1 added generation, which fixed a bug with flaky group members joining +// repeatedly. See KIP-341 for more details. +// +// Note that clients should always try decoding as v1 and, if that fails, +// fall back to v0. This is necessary due to there being no version number +// anywhere in this type. +type StickyMemberMetadata struct { + // CurrentAssignment is the assignment that a group member has when + // issuing a join. + CurrentAssignment []StickyMemberMetadataCurrentAssignment + + // Generation is the generation of this join. This is incremented every join. + // + // This field has a default of -1. + Generation int32 // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StickyMemberMetadata. +func (v *StickyMemberMetadata) Default() { + v.Generation = -1 +} + +// NewStickyMemberMetadata returns a default StickyMemberMetadata +// This is a shortcut for creating a struct and calling Default yourself. +func NewStickyMemberMetadata() StickyMemberMetadata { + var v StickyMemberMetadata + v.Default() + return v +} + +type ConsumerMemberMetadataOwnedPartition struct { + Topic string + + Partitions []int32 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerMemberMetadataOwnedPartition. +func (v *ConsumerMemberMetadataOwnedPartition) Default() { +} + +// NewConsumerMemberMetadataOwnedPartition returns a default ConsumerMemberMetadataOwnedPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerMemberMetadataOwnedPartition() ConsumerMemberMetadataOwnedPartition { + var v ConsumerMemberMetadataOwnedPartition + v.Default() + return v +} + +// ConsumerMemberMetadata is the metadata that is usually sent with a join group +// request with the "consumer" protocol (normal, non-connect consumers). +type ConsumerMemberMetadata struct { + // Version is 0, 1, 2, or 3. + Version int16 + + // Topics is the list of topics in the group that this member is interested + // in consuming. + Topics []string + + // UserData is arbitrary client data for a given client in the group. + // For sticky assignment, this is StickyMemberMetadata. + UserData []byte + + // OwnedPartitions, introduced for KIP-429, are the partitions that this + // member currently owns. + OwnedPartitions []ConsumerMemberMetadataOwnedPartition // v1+ + + // Generation is the generation of the group. + // + // This field has a default of -1. + Generation int32 // v2+ + + // Rack, if non-nil, opts into rack-aware replica assignment. + Rack *string // v3+ +} + +func (v *ConsumerMemberMetadata) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendString(dst, v) + } + } + { + v := v.UserData + dst = kbin.AppendNullableBytes(dst, v) + } + if version >= 1 { + v := v.OwnedPartitions + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Topic + dst = kbin.AppendString(dst, v) + } + { + v := v.Partitions + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + } + } + if version >= 2 { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + if version >= 3 { + v := v.Rack + dst = kbin.AppendNullableString(dst, v) + } + return dst +} + +func (v *ConsumerMemberMetadata) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerMemberMetadata) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerMemberMetadata) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + v := s.Topics + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + a[i] = v + } + v = a + s.Topics = v + } + { + v := b.NullableBytes() + s.UserData = v + } + if version >= 1 { + v := s.OwnedPartitions + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerMemberMetadataOwnedPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + } + v = a + s.OwnedPartitions = v + } + if version >= 2 { + v := b.Int32() + s.Generation = v + } + if version >= 3 { + var v *string + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + s.Rack = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerMemberMetadata. +func (v *ConsumerMemberMetadata) Default() { + v.Generation = -1 +} + +// NewConsumerMemberMetadata returns a default ConsumerMemberMetadata +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerMemberMetadata() ConsumerMemberMetadata { + var v ConsumerMemberMetadata + v.Default() + return v +} + +type ConsumerMemberAssignmentTopic struct { + // Topic is a topic in the assignment. + Topic string + + // Partitions contains partitions in the assignment. + Partitions []int32 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerMemberAssignmentTopic. +func (v *ConsumerMemberAssignmentTopic) Default() { +} + +// NewConsumerMemberAssignmentTopic returns a default ConsumerMemberAssignmentTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerMemberAssignmentTopic() ConsumerMemberAssignmentTopic { + var v ConsumerMemberAssignmentTopic + v.Default() + return v +} + +// ConsumerMemberAssignment is the assignment data that is usually sent with a +// sync group request with the "consumer" protocol (normal, non-connect +// consumers). +type ConsumerMemberAssignment struct { + // Verson is 0, 1, or 2. + Version int16 + + // Topics contains topics in the assignment. + Topics []ConsumerMemberAssignmentTopic + + // UserData is arbitrary client data for a given client in the group. + UserData []byte +} + +func (v *ConsumerMemberAssignment) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Topic + dst = kbin.AppendString(dst, v) + } + { + v := v.Partitions + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + } + } + { + v := v.UserData + dst = kbin.AppendNullableBytes(dst, v) + } + return dst +} + +func (v *ConsumerMemberAssignment) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerMemberAssignment) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerMemberAssignment) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + v := s.Topics + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerMemberAssignmentTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + } + v = a + s.Topics = v + } + { + v := b.NullableBytes() + s.UserData = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerMemberAssignment. +func (v *ConsumerMemberAssignment) Default() { +} + +// NewConsumerMemberAssignment returns a default ConsumerMemberAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerMemberAssignment() ConsumerMemberAssignment { + var v ConsumerMemberAssignment + v.Default() + return v +} + +// ConnectMemberMetadata is the metadata used in a join group request with the +// "connect" protocol. v1 introduced incremental cooperative rebalancing (akin +// to cooperative-sticky) per KIP-415. +// +// v0 defined in connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocol.java +// v1+ defined in connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeConnectProtocol.java +type ConnectMemberMetadata struct { + Version int16 + + URL string + + ConfigOffset int64 + + CurrentAssignment []byte // v1+ +} + +func (v *ConnectMemberMetadata) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.URL + dst = kbin.AppendString(dst, v) + } + { + v := v.ConfigOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.CurrentAssignment + dst = kbin.AppendNullableBytes(dst, v) + } + return dst +} + +func (v *ConnectMemberMetadata) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConnectMemberMetadata) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConnectMemberMetadata) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.URL = v + } + { + v := b.Int64() + s.ConfigOffset = v + } + if version >= 1 { + v := b.NullableBytes() + s.CurrentAssignment = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConnectMemberMetadata. +func (v *ConnectMemberMetadata) Default() { +} + +// NewConnectMemberMetadata returns a default ConnectMemberMetadata +// This is a shortcut for creating a struct and calling Default yourself. +func NewConnectMemberMetadata() ConnectMemberMetadata { + var v ConnectMemberMetadata + v.Default() + return v +} + +type ConnectMemberAssignmentAssignment struct { + Connector string + + Tasks []int16 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConnectMemberAssignmentAssignment. +func (v *ConnectMemberAssignmentAssignment) Default() { +} + +// NewConnectMemberAssignmentAssignment returns a default ConnectMemberAssignmentAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewConnectMemberAssignmentAssignment() ConnectMemberAssignmentAssignment { + var v ConnectMemberAssignmentAssignment + v.Default() + return v +} + +type ConnectMemberAssignmentRevoked struct { + Connector string + + Tasks []int16 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConnectMemberAssignmentRevoked. +func (v *ConnectMemberAssignmentRevoked) Default() { +} + +// NewConnectMemberAssignmentRevoked returns a default ConnectMemberAssignmentRevoked +// This is a shortcut for creating a struct and calling Default yourself. +func NewConnectMemberAssignmentRevoked() ConnectMemberAssignmentRevoked { + var v ConnectMemberAssignmentRevoked + v.Default() + return v +} + +// ConnectMemberAssignment is the assignment that is used in a sync group +// request with the "connect" protocol. See ConnectMemberMetadata for links to +// the Kafka code where these fields are defined. +type ConnectMemberAssignment struct { + Version int16 + + Error int16 + + Leader string + + LeaderURL string + + ConfigOffset int64 + + Assignment []ConnectMemberAssignmentAssignment + + Revoked []ConnectMemberAssignmentRevoked // v1+ + + ScheduledDelay int32 // v1+ +} + +func (v *ConnectMemberAssignment) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Error + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Leader + dst = kbin.AppendString(dst, v) + } + { + v := v.LeaderURL + dst = kbin.AppendString(dst, v) + } + { + v := v.ConfigOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Assignment + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Connector + dst = kbin.AppendString(dst, v) + } + { + v := v.Tasks + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendInt16(dst, v) + } + } + } + } + if version >= 1 { + v := v.Revoked + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Connector + dst = kbin.AppendString(dst, v) + } + { + v := v.Tasks + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendInt16(dst, v) + } + } + } + } + if version >= 1 { + v := v.ScheduledDelay + dst = kbin.AppendInt32(dst, v) + } + return dst +} + +func (v *ConnectMemberAssignment) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConnectMemberAssignment) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConnectMemberAssignment) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + v := b.Int16() + s.Error = v + } + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Leader = v + } + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.LeaderURL = v + } + { + v := b.Int64() + s.ConfigOffset = v + } + { + v := s.Assignment + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConnectMemberAssignmentAssignment, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Connector = v + } + { + v := s.Tasks + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int16, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int16() + a[i] = v + } + v = a + s.Tasks = v + } + } + v = a + s.Assignment = v + } + if version >= 1 { + v := s.Revoked + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConnectMemberAssignmentRevoked, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Connector = v + } + { + v := s.Tasks + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int16, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int16() + a[i] = v + } + v = a + s.Tasks = v + } + } + v = a + s.Revoked = v + } + if version >= 1 { + v := b.Int32() + s.ScheduledDelay = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConnectMemberAssignment. +func (v *ConnectMemberAssignment) Default() { +} + +// NewConnectMemberAssignment returns a default ConnectMemberAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewConnectMemberAssignment() ConnectMemberAssignment { + var v ConnectMemberAssignment + v.Default() + return v +} + +// DefaultPrincipalData is the encoded principal data. This is used in an +// envelope request from broker to broker. +type DefaultPrincipalData struct { + Version int16 + + // The principal type. + Type string + + // The principal name. + Name string + + // Whether the principal was authenticated by a delegation token on the forwarding broker. + TokenAuthenticated bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (v *DefaultPrincipalData) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Type + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TokenAuthenticated + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DefaultPrincipalData) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DefaultPrincipalData) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DefaultPrincipalData) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Type = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Bool() + s.TokenAuthenticated = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} +func (v *DefaultPrincipalData) IsFlexible() bool { return v.Version >= 0 } + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DefaultPrincipalData. +func (v *DefaultPrincipalData) Default() { +} + +// NewDefaultPrincipalData returns a default DefaultPrincipalData +// This is a shortcut for creating a struct and calling Default yourself. +func NewDefaultPrincipalData() DefaultPrincipalData { + var v DefaultPrincipalData + v.Default() + return v +} + +// ControlRecordKey is the key in a control record. +type ControlRecordKey struct { + Version int16 + + Type ControlRecordKeyType +} + +func (v *ControlRecordKey) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Type + { + v := int16(v) + dst = kbin.AppendInt16(dst, v) + } + } + return dst +} + +func (v *ControlRecordKey) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ControlRecordKey) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ControlRecordKey) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + var t ControlRecordKeyType + { + v := b.Int16() + t = ControlRecordKeyType(v) + } + v := t + s.Type = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControlRecordKey. +func (v *ControlRecordKey) Default() { +} + +// NewControlRecordKey returns a default ControlRecordKey +// This is a shortcut for creating a struct and calling Default yourself. +func NewControlRecordKey() ControlRecordKey { + var v ControlRecordKey + v.Default() + return v +} + +// EndTxnMarker is the value for a control record when the key is type 0 or 1. +type EndTxnMarker struct { + Version int16 + + CoordinatorEpoch int32 +} + +func (v *EndTxnMarker) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.CoordinatorEpoch + dst = kbin.AppendInt32(dst, v) + } + return dst +} + +func (v *EndTxnMarker) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EndTxnMarker) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EndTxnMarker) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + s := v + { + v := b.Int32() + s.CoordinatorEpoch = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndTxnMarker. +func (v *EndTxnMarker) Default() { +} + +// NewEndTxnMarker returns a default EndTxnMarker +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndTxnMarker() EndTxnMarker { + var v EndTxnMarker + v.Default() + return v +} + +type LeaderChangeMessageVoter struct { + VoterID int32 + + // VoterDirectoryID is the directory ID of the voter. + // KAFKA-16915 commit da8fe6355b, proposed in KIP-853 and included in 3.9. + VoterDirectoryID [16]byte // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderChangeMessageVoter. +func (v *LeaderChangeMessageVoter) Default() { +} + +// NewLeaderChangeMessageVoter returns a default LeaderChangeMessageVoter +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderChangeMessageVoter() LeaderChangeMessageVoter { + var v LeaderChangeMessageVoter + v.Default() + return v +} + +// LeaderChangeMessage is the value for a control record when the key is type 2. +type LeaderChangeMessage struct { + Version int16 + + // The ID of the newly elected leader. + LeaderID int32 + + // The set of voters in the quorum for this epoch. + Voters []LeaderChangeMessageVoter + + // The voters who voted for the leader at the time of election. + GrantingVoters []LeaderChangeMessageVoter + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (v *LeaderChangeMessage) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Version + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Voters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.GrantingVoters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *LeaderChangeMessage) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *LeaderChangeMessage) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *LeaderChangeMessage) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + v.Version = b.Int16() + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := s.Voters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderChangeMessageVoter, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.VoterID = v + } + if version >= 1 { + v := b.Uuid() + s.VoterDirectoryID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Voters = v + } + { + v := s.GrantingVoters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderChangeMessageVoter, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.VoterID = v + } + if version >= 1 { + v := b.Uuid() + s.VoterDirectoryID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.GrantingVoters = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} +func (v *LeaderChangeMessage) IsFlexible() bool { return v.Version >= 0 } + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderChangeMessage. +func (v *LeaderChangeMessage) Default() { +} + +// NewLeaderChangeMessage returns a default LeaderChangeMessage +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderChangeMessage() LeaderChangeMessage { + var v LeaderChangeMessage + v.Default() + return v +} + +type ProduceRequestTopicPartition struct { + // Partition is a partition to send a record batch to. + Partition int32 + + // Records is a batch of records to write to a topic's partition. + // + // For Kafka pre 0.11.0, the contents of the byte array is a serialized + // message set. At or after 0.11.0, the contents of the byte array is a + // serialized RecordBatch. + Records []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceRequestTopicPartition. +func (v *ProduceRequestTopicPartition) Default() { +} + +// NewProduceRequestTopicPartition returns a default ProduceRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceRequestTopicPartition() ProduceRequestTopicPartition { + var v ProduceRequestTopicPartition + v.Default() + return v +} + +type ProduceRequestTopic struct { + // Topic is a topic to send record batches to. + Topic string // v0-v12 + + // TopicID is the uuid of the topic to produce records to. + TopicID [16]byte // v13+ + + // Partitions is an array of partitions to send record batches to. + Partitions []ProduceRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceRequestTopic. +func (v *ProduceRequestTopic) Default() { +} + +// NewProduceRequestTopic returns a default ProduceRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceRequestTopic() ProduceRequestTopic { + var v ProduceRequestTopic + v.Default() + return v +} + +// ProduceRequest issues records to be created to Kafka. +// +// Kafka 0.10.0 (v2) changed Records from MessageSet v0 to MessageSet v1. +// Kafka 0.11.0 (v3) again changed Records to RecordBatch. +// +// Note that the special client ID "__admin_client" will allow you to produce +// records to internal topics. This is generally recommended if you want to +// break your Kafka cluster. +type ProduceRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionID is the transaction ID to use for this request, allowing for + // exactly once semantics. + TransactionID *string // v3+ + + // Acks specifies the number of acks that the partition leaders must receive + // from in sync replicas before considering a record batch fully written. + // + // Valid values are -1, 0, or 1 corresponding to all, none, or the leader only. + // + // Note that if no acks are requested, Kafka will close the connection + // if any topic or partition errors to trigger a client metadata refresh. + Acks int16 + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 15000. + TimeoutMillis int32 + + // Topics is an array of topics to send record batches to. + Topics []ProduceRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +func (*ProduceRequest) Key() int16 { return 0 } +func (*ProduceRequest) MaxVersion() int16 { return 13 } +func (v *ProduceRequest) SetVersion(version int16) { v.Version = version } +func (v *ProduceRequest) GetVersion() int16 { return v.Version } +func (v *ProduceRequest) IsFlexible() bool { return v.Version >= 9 } +func (v *ProduceRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *ProduceRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *ProduceRequest) ResponseKind() Response { + r := &ProduceResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ProduceRequest) RequestWith(ctx context.Context, r Requestor) (*ProduceResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ProduceResponse) + return resp, err +} + +func (v *ProduceRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + if version >= 3 { + v := v.TransactionID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Acks + dst = kbin.AppendInt16(dst, v) + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 12 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 13 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Records + if isFlexible { + dst = kbin.AppendCompactNullableBytes(dst, v) + } else { + dst = kbin.AppendNullableBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ProduceRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ProduceRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ProduceRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + s := v + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.TransactionID = v + } + { + v := b.Int16() + s.Acks = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 12 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 13 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + var v []byte + if isFlexible { + v = b.CompactNullableBytes() + } else { + v = b.NullableBytes() + } + s.Records = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrProduceRequest returns a pointer to a default ProduceRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrProduceRequest() *ProduceRequest { + var v ProduceRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceRequest. +func (v *ProduceRequest) Default() { + v.TimeoutMillis = 15000 +} + +// NewProduceRequest returns a default ProduceRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceRequest() ProduceRequest { + var v ProduceRequest + v.Default() + return v +} + +type ProduceResponseTopicPartitionErrorRecord struct { + // RelativeOffset is the offset of the record that caused problems. + RelativeOffset int32 + + // ErrorMessage is the error of this record. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponseTopicPartitionErrorRecord. +func (v *ProduceResponseTopicPartitionErrorRecord) Default() { +} + +// NewProduceResponseTopicPartitionErrorRecord returns a default ProduceResponseTopicPartitionErrorRecord +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponseTopicPartitionErrorRecord() ProduceResponseTopicPartitionErrorRecord { + var v ProduceResponseTopicPartitionErrorRecord + v.Default() + return v +} + +type ProduceResponseTopicPartitionCurrentLeader struct { + // The ID of the current leader, or -1 if unknown. + // + // This field has a default of -1. + LeaderID int32 + + // The latest known leader epoch. + // + // This field has a default of -1. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponseTopicPartitionCurrentLeader. +func (v *ProduceResponseTopicPartitionCurrentLeader) Default() { + v.LeaderID = -1 + v.LeaderEpoch = -1 +} + +// NewProduceResponseTopicPartitionCurrentLeader returns a default ProduceResponseTopicPartitionCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponseTopicPartitionCurrentLeader() ProduceResponseTopicPartitionCurrentLeader { + var v ProduceResponseTopicPartitionCurrentLeader + v.Default() + return v +} + +type ProduceResponseTopicPartition struct { + // Partition is the partition this response pertains to. + Partition int32 + + // ErrorCode is any error for a topic/partition in the request. + // There are many error codes for produce requests. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned for all topics and + // partitions if the request had a transactional ID but the client + // is not authorized for transactions. + // + // CLUSTER_AUTHORIZATION_FAILED is returned for all topics and partitions + // if the request was idempotent but the client is not authorized + // for idempotent requests. + // + // TOPIC_AUTHORIZATION_FAILED is returned for all topics the client + // is not authorized to talk to. + // + // INVALID_REQUIRED_ACKS is returned if the request contained an invalid + // number for "acks". + // + // CORRUPT_MESSAGE is returned for many reasons, generally related to + // problems with messages (invalid magic, size mismatch, etc.). + // + // MESSAGE_TOO_LARGE is returned if a record batch is larger than the + // broker's configured max.message.size. + // + // RECORD_LIST_TOO_LARGE is returned if the record batch is larger than + // the broker's segment.bytes. + // + // INVALID_TIMESTAMP is returned if the record batch uses LogAppendTime + // or if the timestamp delta from when the broker receives the message + // is more than the broker's log.message.timestamp.difference.max.ms. + // + // UNSUPPORTED_FOR_MESSAGE_FORMAT is returned if using a Kafka v2 message + // format (i.e. RecordBatch) feature (idempotence) while sending v1 + // messages (i.e. a MessageSet). + // + // KAFKA_STORAGE_ERROR is returned if the log directory for a partition + // is offline. + // + // NOT_ENOUGH_REPLICAS is returned if all acks are required, but there + // are not enough in sync replicas yet. + // + // NOT_ENOUGH_REPLICAS_AFTER_APPEND is returned on old Kafka versions + // (pre 0.11.0.0) when a message was written to disk and then Kafka + // noticed not enough replicas existed to replicate the message. + // + // DUPLICATE_SEQUENCE_NUMBER is returned for Kafka <1.1.0 when a + // sequence number is detected as a duplicate. After, out of order + // is returned. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic or partition + // is unknown. + // + // NOT_LEADER_FOR_PARTITION is returned if the broker is not a leader + // for this partition. This means that the client has stale metadata. + // + // INVALID_PRODUCER_EPOCH is returned if the produce request was + // attempted with an old epoch. Either there is a newer producer using + // the same transaction ID, or the transaction ID used has expired. + // + // UNKNOWN_PRODUCER_ID, added in Kafka 1.0.0 (message format v5+) is + // returned if the producer used an ID that Kafka does not know about or + // if the request has a larger sequence number than Kafka expects. The + // LogStartOffset must be checked in this case. If the offset is greater + // than the last acknowledged offset, then no data loss has occurred; the + // client just sent data so long ago that Kafka rotated the partition out + // of existence and no longer knows of this producer ID. In this case, + // reset your sequence numbers to 0. If the log start offset is equal to + // or less than what the client sent prior, then data loss has occurred. + // See KAFKA-5793 for more details. NOTE: Unfortunately, even UNKNOWN_PRODUCER_ID + // is unsafe to handle, so this error should likely be treated the same + // as OUT_OF_ORDER_SEQUENCE_NUMER. See KIP-360 for more details. + // + // OUT_OF_ORDER_SEQUENCE_NUMBER is sent if the batch's FirstSequence was + // not what it should be (the last FirstSequence, plus the number of + // records in the last batch, plus one). After 1.0.0, this generally + // means data loss. Before, there could be confusion on if the broker + // actually rotated the partition out of existence (this is why + // UNKNOWN_PRODUCER_ID was introduced). + // + // UNKNOWN_TOPIC_ID is returned if producing to an unknown topic ID. + ErrorCode int16 + + // BaseOffset is the offset that the records in the produce request began + // at in the partition. + BaseOffset int64 + + // LogAppendTime is the millisecond that records were appended to the + // partition inside Kafka. This is only not -1 if records were written + // with the log append time flag (which producers cannot do). + // + // This field has a default of -1. + LogAppendTime int64 // v2+ + + // LogStartOffset, introduced in Kafka 1.0.0, can be used to see if an + // UNKNOWN_PRODUCER_ID means Kafka rotated records containing the used + // producer ID out of existence, or if Kafka lost data. + // + // This field has a default of -1. + LogStartOffset int64 // v5+ + + // ErrorRecords are indices of individual records that caused a batch + // to error. This was added for KIP-467. + ErrorRecords []ProduceResponseTopicPartitionErrorRecord // v8+ + + // ErrorMessage is the global error message of of what caused this batch + // to error. + ErrorMessage *string // v8+ + + CurrentLeader ProduceResponseTopicPartitionCurrentLeader // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponseTopicPartition. +func (v *ProduceResponseTopicPartition) Default() { + v.LogAppendTime = -1 + v.LogStartOffset = -1 + { + v := &v.CurrentLeader + _ = v + v.LeaderID = -1 + v.LeaderEpoch = -1 + } +} + +// NewProduceResponseTopicPartition returns a default ProduceResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponseTopicPartition() ProduceResponseTopicPartition { + var v ProduceResponseTopicPartition + v.Default() + return v +} + +type ProduceResponseTopic struct { + // Topic is the topic this response pertains to. + Topic string // v0-v12 + + // TopicID is the uuid of the topic produced to. + TopicID [16]byte // v13+ + + // Partitions is an array of responses for the partition's that + // batches were sent to. + Partitions []ProduceResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponseTopic. +func (v *ProduceResponseTopic) Default() { +} + +// NewProduceResponseTopic returns a default ProduceResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponseTopic() ProduceResponseTopic { + var v ProduceResponseTopic + v.Default() + return v +} + +type ProduceResponseBroker struct { + // NodeID is the node ID of a Kafka broker. + NodeID int32 + + // Host is the hostname of a Kafka broker. + Host string + + // Port is the port of a Kafka broker. + Port int32 + + // Rack is the rack this Kafka broker is in. + Rack *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponseBroker. +func (v *ProduceResponseBroker) Default() { +} + +// NewProduceResponseBroker returns a default ProduceResponseBroker +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponseBroker() ProduceResponseBroker { + var v ProduceResponseBroker + v.Default() + return v +} + +// ProduceResponse is returned from a ProduceRequest. +type ProduceResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics is an array of responses for the topic's that batches were sent + // to. + Topics []ProduceResponseTopic + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 6. + ThrottleMillis int32 // v1+ + + // Brokers is present if any partition responses contain the error + // NOT_LEADER_OR_FOLLOWER. + Brokers []ProduceResponseBroker // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +func (*ProduceResponse) Key() int16 { return 0 } +func (*ProduceResponse) MaxVersion() int16 { return 13 } +func (v *ProduceResponse) SetVersion(version int16) { v.Version = version } +func (v *ProduceResponse) GetVersion() int16 { return v.Version } +func (v *ProduceResponse) IsFlexible() bool { return v.Version >= 9 } +func (v *ProduceResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 6 } +func (v *ProduceResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ProduceResponse) RequestKind() Request { return &ProduceRequest{Version: v.Version} } + +func (v *ProduceResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 12 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 13 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.BaseOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 2 { + v := v.LogAppendTime + dst = kbin.AppendInt64(dst, v) + } + if version >= 5 { + v := v.LogStartOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 8 { + v := v.ErrorRecords + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.RelativeOffset + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 8 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + var toEncode []uint32 + if !reflect.DeepEqual(v.CurrentLeader, (func() ProduceResponseTopicPartitionCurrentLeader { + var v ProduceResponseTopicPartitionCurrentLeader + v.Default() + return v + })()) { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.CurrentLeader + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fCurrentLeader: + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fCurrentLeader + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + var toEncode []uint32 + if len(v.Brokers) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.Brokers + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fBrokers: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fBrokers + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ProduceResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ProduceResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ProduceResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 12 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 13 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.BaseOffset = v + } + if version >= 2 { + v := b.Int64() + s.LogAppendTime = v + } + if version >= 5 { + v := b.Int64() + s.LogStartOffset = v + } + if version >= 8 { + v := s.ErrorRecords + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceResponseTopicPartitionErrorRecord, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.RelativeOffset = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ErrorRecords = v + } + if version >= 8 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.Brokers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ProduceResponseBroker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Brokers = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrProduceResponse returns a pointer to a default ProduceResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrProduceResponse() *ProduceResponse { + var v ProduceResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ProduceResponse. +func (v *ProduceResponse) Default() { +} + +// NewProduceResponse returns a default ProduceResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewProduceResponse() ProduceResponse { + var v ProduceResponse + v.Default() + return v +} + +type FetchRequestReplicaState struct { + // The replica ID of the follower, or -1 if this request is from a consumer. + // + // This field has a default of -1. + ID int32 + + // The epoch of this follower, or -1 if not available. + // + // This field has a default of -1. + Epoch int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchRequestReplicaState. +func (v *FetchRequestReplicaState) Default() { + v.ID = -1 + v.Epoch = -1 +} + +// NewFetchRequestReplicaState returns a default FetchRequestReplicaState +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchRequestReplicaState() FetchRequestReplicaState { + var v FetchRequestReplicaState + v.Default() + return v +} + +type FetchRequestTopicPartition struct { + // Partition is a partition in a topic to try to fetch records for. + Partition int32 + + // CurrentLeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0, + // allows brokers to check if the client is fenced (has an out of date + // leader) or is using an unknown leader. + // + // The initial leader epoch can be determined from a MetadataResponse. + // To skip log truncation checking, use -1. + // + // This field has a default of -1. + CurrentLeaderEpoch int32 // v9+ + + // FetchOffset is the offset to begin the fetch from. Kafka will + // return records at and after this offset. + FetchOffset int64 + + // The epoch of the last fetched record, or -1 if there is none. + // + // This field has a default of -1. + LastFetchedEpoch int32 // v12+ + + // LogStartOffset is a broker-follower only field added for KIP-107. + // This is the start offset of the partition in a follower. + // + // This field has a default of -1. + LogStartOffset int64 // v5+ + + // PartitionMaxBytes is the maximum bytes to return for this partition. + // This can be used to limit how many bytes an individual partition in + // a request is allotted so that it does not dominate all of MaxBytes. + PartitionMaxBytes int32 + + // The directory ID of the follower fetching. This is not relevant for + // clients; see KIP-853. + ReplicaDirectoryID [16]byte // tag 0 + + // The high-watermark known by the replica. -1 if the high-watermark is + // not known and 9223372036854775807 if the feature is not supported. + // For KIP-1166; not relevant for clients. Should not have forced a + // version bump in the protocol... + // + // This field has a default of 9223372036854775807. + HighWatermark int64 // tag 1 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchRequestTopicPartition. +func (v *FetchRequestTopicPartition) Default() { + v.CurrentLeaderEpoch = -1 + v.LastFetchedEpoch = -1 + v.LogStartOffset = -1 + v.HighWatermark = 9223372036854775807 +} + +// NewFetchRequestTopicPartition returns a default FetchRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchRequestTopicPartition() FetchRequestTopicPartition { + var v FetchRequestTopicPartition + v.Default() + return v +} + +type FetchRequestTopic struct { + // Topic is a topic to try to fetch records for. + Topic string // v0-v12 + + // TopicID is the uuid of the topic to fetch records for. + TopicID [16]byte // v13+ + + // Partitions contains partitions in a topic to try to fetch records for. + Partitions []FetchRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchRequestTopic. +func (v *FetchRequestTopic) Default() { +} + +// NewFetchRequestTopic returns a default FetchRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchRequestTopic() FetchRequestTopic { + var v FetchRequestTopic + v.Default() + return v +} + +type FetchRequestForgottenTopic struct { + // Topic is a topic to remove from being tracked (with the partitions below). + Topic string // v7-v12 + + // TopicID is the uuid of a topic to remove from being tracked (with the + // partitions below). + TopicID [16]byte // v13+ + + // Partitions are partitions to remove from tracking for a topic. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchRequestForgottenTopic. +func (v *FetchRequestForgottenTopic) Default() { +} + +// NewFetchRequestForgottenTopic returns a default FetchRequestForgottenTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchRequestForgottenTopic() FetchRequestForgottenTopic { + var v FetchRequestForgottenTopic + v.Default() + return v +} + +// FetchRequest is a long-poll request of records from Kafka. +// +// Kafka 0.11.0.0 released v4 and changed the returned RecordBatches to contain +// the RecordBatch type. Prior, Kafka used the MessageSet type (and, for v0 and +// v1, Kafka used a different type). +// +// Note that starting in v3, Kafka began processing partitions in order, +// meaning the order of partitions in the fetch request is important due to +// potential size constraints. +// +// Starting in v13, topics must use UUIDs rather than their string name +// identifiers. +// +// Version 15 adds the ReplicaState which includes new field ReplicaEpoch and +// the ReplicaID, and deprecates the old ReplicaID (KIP-903). +type FetchRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The cluster ID, if known. This is used to validate metadata fetches + // prior to broker registration. + // + // This field has a default of null. + ClusterID *string // tag 0 + + // ReplicaID is the broker ID of performing the fetch request. Standard + // clients should use -1. To be a "debug" replica, use -2. The debug + // replica can be used to fetch messages from non-leaders. + // + // This field has a default of -1. + ReplicaID int32 // v0-v14 + + // ReplicaState is a broker-only tag for v15+, see KIP-903 for more details. + ReplicaState FetchRequestReplicaState // tag 1 + + // MaxWaitMillis is how long to wait for MinBytes to be hit before a broker + // responds to a fetch request. + MaxWaitMillis int32 + + // MinBytes is the minimum amount of bytes to attempt to read before a broker + // responds to a fetch request. + MinBytes int32 + + // MaxBytes is the maximum amount of bytes to read in a fetch request. The + // response can exceed MaxBytes if the first record in the first non-empty + // partition is larger than MaxBytes. + // + // This field has a default of 0x7fffffff. + MaxBytes int32 // v3+ + + // IsolationLevel changes which messages are fetched. Follower replica ID's + // (non-negative, non-standard-client) fetch from the end. + // + // Standard clients fetch from the high watermark, which corresponds to + // IsolationLevel 0, READ_UNCOMMITTED. + // + // To only read committed records, use IsolationLevel 1, corresponding to + // READ_COMMITTED. + IsolationLevel int8 // v4+ + + // SessionID is used to potentially reduce the amount of back and forth + // data between a client and a broker. If opting in to sessions, the first + // ID used should be 0, and thereafter (until session resets) the ID should + // be the ID returned in the fetch response. + // + // Read KIP-227 for more details. + SessionID int32 // v7+ + + // SessionEpoch is the session epoch for this request if using sessions. + // + // Read KIP-227 for more details. Use -1 if you are not using sessions. + // + // This field has a default of -1. + SessionEpoch int32 // v7+ + + // Topic contains topics to try to fetch records for. + Topics []FetchRequestTopic + + // ForgottenTopics contains topics and partitions that a fetch session + // wants to remove from its session. + // + // See KIP-227 for more details. + ForgottenTopics []FetchRequestForgottenTopic // v7+ + + // Rack of the consumer making this request (see KIP-392; introduced in + // Kafka 2.2.0). + Rack string // v11+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +func (*FetchRequest) Key() int16 { return 1 } +func (*FetchRequest) MaxVersion() int16 { return 18 } +func (v *FetchRequest) SetVersion(version int16) { v.Version = version } +func (v *FetchRequest) GetVersion() int16 { return v.Version } +func (v *FetchRequest) IsFlexible() bool { return v.Version >= 12 } +func (v *FetchRequest) ResponseKind() Response { + r := &FetchResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *FetchRequest) RequestWith(ctx context.Context, r Requestor) (*FetchResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*FetchResponse) + return resp, err +} + +func (v *FetchRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 12 + _ = isFlexible + if version >= 0 && version <= 14 { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MaxWaitMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MinBytes + dst = kbin.AppendInt32(dst, v) + } + if version >= 3 { + v := v.MaxBytes + dst = kbin.AppendInt32(dst, v) + } + if version >= 4 { + v := v.IsolationLevel + dst = kbin.AppendInt8(dst, v) + } + if version >= 7 { + v := v.SessionID + dst = kbin.AppendInt32(dst, v) + } + if version >= 7 { + v := v.SessionEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 12 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 13 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 9 { + v := v.CurrentLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.FetchOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 12 { + v := v.LastFetchedEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 5 { + v := v.LogStartOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.PartitionMaxBytes + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + var toEncode []uint32 + if v.ReplicaDirectoryID != [16]byte{} { + toEncode = append(toEncode, 0) + } + if v.HighWatermark != 9223372036854775807 { + toEncode = append(toEncode, 1) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.ReplicaDirectoryID + dst = kbin.AppendUvarint(dst, 0) + dst = kbin.AppendUvarint(dst, 16) + dst = kbin.AppendUuid(dst, v) + } + case 1: + { + v := v.HighWatermark + dst = kbin.AppendUvarint(dst, 1) + dst = kbin.AppendUvarint(dst, 8) + dst = kbin.AppendInt64(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 7 { + v := v.ForgottenTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 7 && version <= 12 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 13 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 11 { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + var toEncode []uint32 + if v.ClusterID != nil { + toEncode = append(toEncode, 0) + } + if !reflect.DeepEqual(v.ReplicaState, (func() FetchRequestReplicaState { var v FetchRequestReplicaState; v.Default(); return v })()) { + toEncode = append(toEncode, 1) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.ClusterID + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fClusterID: + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fClusterID + } + } + case 1: + { + v := v.ReplicaState + dst = kbin.AppendUvarint(dst, 1) + sized := false + lenAt := len(dst) + fReplicaState: + { + v := v.ID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Epoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fReplicaState + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FetchRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FetchRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FetchRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 12 + _ = isFlexible + s := v + if version >= 0 && version <= 14 { + v := b.Int32() + s.ReplicaID = v + } + { + v := b.Int32() + s.MaxWaitMillis = v + } + { + v := b.Int32() + s.MinBytes = v + } + if version >= 3 { + v := b.Int32() + s.MaxBytes = v + } + if version >= 4 { + v := b.Int8() + s.IsolationLevel = v + } + if version >= 7 { + v := b.Int32() + s.SessionID = v + } + if version >= 7 { + v := b.Int32() + s.SessionEpoch = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 12 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 13 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if version >= 9 { + v := b.Int32() + s.CurrentLeaderEpoch = v + } + { + v := b.Int64() + s.FetchOffset = v + } + if version >= 12 { + v := b.Int32() + s.LastFetchedEpoch = v + } + if version >= 5 { + v := b.Int64() + s.LogStartOffset = v + } + { + v := b.Int32() + s.PartitionMaxBytes = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Uuid() + s.ReplicaDirectoryID = v + if err := b.Complete(); err != nil { + return err + } + case 1: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int64() + s.HighWatermark = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 7 { + v := s.ForgottenTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchRequestForgottenTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 7 && version <= 12 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 13 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ForgottenTopics = v + } + if version >= 11 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Rack = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + if err := b.Complete(); err != nil { + return err + } + case 1: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.ReplicaState + v.Default() + s := v + { + v := b.Int32() + s.ID = v + } + { + v := b.Int64() + s.Epoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrFetchRequest returns a pointer to a default FetchRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFetchRequest() *FetchRequest { + var v FetchRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchRequest. +func (v *FetchRequest) Default() { + v.ClusterID = nil + v.ReplicaID = -1 + { + v := &v.ReplicaState + _ = v + v.ID = -1 + v.Epoch = -1 + } + v.MaxBytes = 2147483647 + v.SessionEpoch = -1 +} + +// NewFetchRequest returns a default FetchRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchRequest() FetchRequest { + var v FetchRequest + v.Default() + return v +} + +type FetchResponseTopicPartitionDivergingEpoch struct { + // This field has a default of -1. + Epoch int32 + + // This field has a default of -1. + EndOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopicPartitionDivergingEpoch. +func (v *FetchResponseTopicPartitionDivergingEpoch) Default() { + v.Epoch = -1 + v.EndOffset = -1 +} + +// NewFetchResponseTopicPartitionDivergingEpoch returns a default FetchResponseTopicPartitionDivergingEpoch +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopicPartitionDivergingEpoch() FetchResponseTopicPartitionDivergingEpoch { + var v FetchResponseTopicPartitionDivergingEpoch + v.Default() + return v +} + +type FetchResponseTopicPartitionCurrentLeader struct { + // The ID of the current leader, or -1 if unknown. + // + // This field has a default of -1. + LeaderID int32 + + // The latest known leader epoch. + // + // This field has a default of -1. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopicPartitionCurrentLeader. +func (v *FetchResponseTopicPartitionCurrentLeader) Default() { + v.LeaderID = -1 + v.LeaderEpoch = -1 +} + +// NewFetchResponseTopicPartitionCurrentLeader returns a default FetchResponseTopicPartitionCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopicPartitionCurrentLeader() FetchResponseTopicPartitionCurrentLeader { + var v FetchResponseTopicPartitionCurrentLeader + v.Default() + return v +} + +type FetchResponseTopicPartitionSnapshotID struct { + // This field has a default of -1. + EndOffset int64 + + // This field has a default of -1. + Epoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopicPartitionSnapshotID. +func (v *FetchResponseTopicPartitionSnapshotID) Default() { + v.EndOffset = -1 + v.Epoch = -1 +} + +// NewFetchResponseTopicPartitionSnapshotID returns a default FetchResponseTopicPartitionSnapshotID +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopicPartitionSnapshotID() FetchResponseTopicPartitionSnapshotID { + var v FetchResponseTopicPartitionSnapshotID + v.Default() + return v +} + +type FetchResponseTopicPartitionAbortedTransaction struct { + // ProducerID is the producer ID that caused this aborted transaction. + ProducerID int64 + + // FirstOffset is the offset where this aborted transaction began. + FirstOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopicPartitionAbortedTransaction. +func (v *FetchResponseTopicPartitionAbortedTransaction) Default() { +} + +// NewFetchResponseTopicPartitionAbortedTransaction returns a default FetchResponseTopicPartitionAbortedTransaction +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopicPartitionAbortedTransaction() FetchResponseTopicPartitionAbortedTransaction { + var v FetchResponseTopicPartitionAbortedTransaction + v.Default() + return v +} + +type FetchResponseTopicPartition struct { + // Partition is a partition in a topic that records may have been + // received for. + Partition int32 + + // ErrorCode is an error returned for an individual partition in a + // fetch request. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not + // authorized to read the partition. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic or partition + // does not exist on this broker. + // + // UNSUPPORTED_COMPRESSION_TYPE is returned if the request version was + // under 10 and the batch is compressed with zstd. + // + // UNSUPPORTED_VERSION is returned if the broker has records newer than + // the client can support (magic value) and the broker has disabled + // message downconversion. + // + // NOT_LEADER_FOR_PARTITION is returned if requesting data for this + // partition as a follower (non-negative ReplicaID) and the broker + // is not the leader for this partition. + // + // REPLICA_NOT_AVAILABLE is returned if the partition exists but + // the requested broker is not the leader for it. + // + // KAFKA_STORAGE_EXCEPTION is returned if the requested partition is + // offline. + // + // UNKNOWN_LEADER_EPOCH is returned if the request used a larger leader + // epoch than the broker knows of. + // + // FENCED_LEADER_EPOCH is returned if the request used a smaller leader + // epoch than the broker is at (see KIP-320). + // + // OFFSET_OUT_OF_RANGE is returned if requesting an offset past the + // current end offset or before the beginning offset. + // + // UNKNOWN_TOPIC_ID is returned if using uuid's and the uuid is unknown + // (v13+ / Kafka 3.1+). + // + // OFFSET_MOVED_TO_TIERED_STORAGE is returned if a follower is trying to + // fetch from an offset that is now in tiered storage. + ErrorCode int16 + + // HighWatermark is the current high watermark for this partition, + // that is, the current offset that is on all in sync replicas. + HighWatermark int64 + + // LastStableOffset is the offset at which all prior offsets have + // been "decided". Non transactional records are always decided + // immediately, but transactional records are only decided once + // they are committed or aborted. + // + // The LastStableOffset will always be at or under the HighWatermark. + // + // This field has a default of -1. + LastStableOffset int64 // v4+ + + // LogStartOffset is the beginning offset for this partition. + // This field was added for KIP-107. + // + // This field has a default of -1. + LogStartOffset int64 // v5+ + + // In case divergence is detected based on the LastFetchedEpoch and + // FetchOffset in the request, this field indicates the largest epoch and + // its end offset such that subsequent records are known to diverge. + DivergingEpoch FetchResponseTopicPartitionDivergingEpoch // tag 0 + + // CurrentLeader is the currently known leader ID and epoch for this + // partition. + CurrentLeader FetchResponseTopicPartitionCurrentLeader // tag 1 + + // In the case of fetching an offset less than the LogStartOffset, this + // is the end offset and epoch that should be used in the FetchSnapshot + // request. + SnapshotID FetchResponseTopicPartitionSnapshotID // tag 2 + + // AbortedTransactions is an array of aborted transactions within the + // returned offset range. This is only returned if the requested + // isolation level was READ_COMMITTED. + AbortedTransactions []FetchResponseTopicPartitionAbortedTransaction // v4+ + + // PreferredReadReplica is the preferred replica for the consumer + // to use on its next fetch request. See KIP-392. + // + // This field has a default of -1. + PreferredReadReplica int32 // v11+ + + // RecordBatches is an array of record batches for a topic partition. + // + // This is encoded as a raw byte array, with the standard int32 size + // prefix. One important catch to note is that the final element of the + // array may be **partial**. This is an optimization in Kafka that + // clients must deal with by discarding a partial trailing batch. + // + // Starting v2, this transitioned to the MessageSet v1 format (and this + // would contain many MessageV1 structs). + // + // Starting v4, this transitioned to the RecordBatch format (thus this + // contains many RecordBatch structs). + RecordBatches []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopicPartition. +func (v *FetchResponseTopicPartition) Default() { + v.LastStableOffset = -1 + v.LogStartOffset = -1 + { + v := &v.DivergingEpoch + _ = v + v.Epoch = -1 + v.EndOffset = -1 + } + { + v := &v.CurrentLeader + _ = v + v.LeaderID = -1 + v.LeaderEpoch = -1 + } + { + v := &v.SnapshotID + _ = v + v.EndOffset = -1 + v.Epoch = -1 + } + v.PreferredReadReplica = -1 +} + +// NewFetchResponseTopicPartition returns a default FetchResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopicPartition() FetchResponseTopicPartition { + var v FetchResponseTopicPartition + v.Default() + return v +} + +type FetchResponseTopic struct { + // Topic is a topic that records may have been received for. + Topic string // v0-v12 + + // TopicID is the uuid of a topic that records may have been received for. + TopicID [16]byte // v13+ + + // Partitions contains partitions in a topic that records may have + // been received for. + Partitions []FetchResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseTopic. +func (v *FetchResponseTopic) Default() { +} + +// NewFetchResponseTopic returns a default FetchResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseTopic() FetchResponseTopic { + var v FetchResponseTopic + v.Default() + return v +} + +type FetchResponseBroker struct { + // NodeID is the node ID of a Kafka broker. + NodeID int32 + + // Host is the hostname of a Kafka broker. + Host string + + // Port is the port of a Kafka broker. + Port int32 + + // Rack is the rack this Kafka broker is in. + Rack *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponseBroker. +func (v *FetchResponseBroker) Default() { +} + +// NewFetchResponseBroker returns a default FetchResponseBroker +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponseBroker() FetchResponseBroker { + var v FetchResponseBroker + v.Default() + return v +} + +// FetchResponse is returned from a FetchRequest. +type FetchResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 8. + ThrottleMillis int32 // v1+ + + // ErrorCode is a full-response error code for a fetch request. This was + // added in support of KIP-227. This error is only non-zero if using fetch + // sessions. + // + // FETCH_SESSION_ID_NOT_FOUND is returned if the request used a + // session ID that the broker does not know of. + // + // INVALID_FETCH_SESSION_EPOCH is returned if the request used an + // invalid session epoch. + ErrorCode int16 // v7+ + + // SessionID is the id for this session if using sessions. + // + // See KIP-227 for more details. + SessionID int32 // v7+ + + // Topics contains an array of topic partitions and the records received + // for them. + Topics []FetchResponseTopic + + // Brokers is present if any partition responses contain the error + // NOT_LEADER_OR_FOLLOWER. + Brokers []FetchResponseBroker // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v12+ +} + +func (*FetchResponse) Key() int16 { return 1 } +func (*FetchResponse) MaxVersion() int16 { return 18 } +func (v *FetchResponse) SetVersion(version int16) { v.Version = version } +func (v *FetchResponse) GetVersion() int16 { return v.Version } +func (v *FetchResponse) IsFlexible() bool { return v.Version >= 12 } +func (v *FetchResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 8 } +func (v *FetchResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *FetchResponse) RequestKind() Request { return &FetchRequest{Version: v.Version} } + +func (v *FetchResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 12 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 7 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 7 { + v := v.SessionID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 12 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 13 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.HighWatermark + dst = kbin.AppendInt64(dst, v) + } + if version >= 4 { + v := v.LastStableOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 5 { + v := v.LogStartOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 4 { + v := v.AbortedTransactions + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 11 { + v := v.PreferredReadReplica + dst = kbin.AppendInt32(dst, v) + } + { + v := v.RecordBatches + if isFlexible { + dst = kbin.AppendCompactNullableBytes(dst, v) + } else { + dst = kbin.AppendNullableBytes(dst, v) + } + } + if isFlexible { + var toEncode []uint32 + if !reflect.DeepEqual(v.DivergingEpoch, (func() FetchResponseTopicPartitionDivergingEpoch { + var v FetchResponseTopicPartitionDivergingEpoch + v.Default() + return v + })()) { + toEncode = append(toEncode, 0) + } + if !reflect.DeepEqual(v.CurrentLeader, (func() FetchResponseTopicPartitionCurrentLeader { + var v FetchResponseTopicPartitionCurrentLeader + v.Default() + return v + })()) { + toEncode = append(toEncode, 1) + } + if !reflect.DeepEqual(v.SnapshotID, (func() FetchResponseTopicPartitionSnapshotID { + var v FetchResponseTopicPartitionSnapshotID + v.Default() + return v + })()) { + toEncode = append(toEncode, 2) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.DivergingEpoch + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fDivergingEpoch: + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.EndOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fDivergingEpoch + } + } + case 1: + { + v := v.CurrentLeader + dst = kbin.AppendUvarint(dst, 1) + sized := false + lenAt := len(dst) + fCurrentLeader: + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fCurrentLeader + } + } + case 2: + { + v := v.SnapshotID + dst = kbin.AppendUvarint(dst, 2) + sized := false + lenAt := len(dst) + fSnapshotID: + { + v := v.EndOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fSnapshotID + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if len(v.Brokers) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.Brokers + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fBrokers: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fBrokers + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FetchResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FetchResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FetchResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 12 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 7 { + v := b.Int16() + s.ErrorCode = v + } + if version >= 7 { + v := b.Int32() + s.SessionID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 12 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 13 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.HighWatermark = v + } + if version >= 4 { + v := b.Int64() + s.LastStableOffset = v + } + if version >= 5 { + v := b.Int64() + s.LogStartOffset = v + } + if version >= 4 { + v := s.AbortedTransactions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []FetchResponseTopicPartitionAbortedTransaction{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchResponseTopicPartitionAbortedTransaction, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int64() + s.FirstOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.AbortedTransactions = v + } + if version >= 11 { + v := b.Int32() + s.PreferredReadReplica = v + } + { + var v []byte + if isFlexible { + v = b.CompactNullableBytes() + } else { + v = b.NullableBytes() + } + s.RecordBatches = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.DivergingEpoch + v.Default() + s := v + { + v := b.Int32() + s.Epoch = v + } + { + v := b.Int64() + s.EndOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + case 1: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + case 2: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.SnapshotID + v.Default() + s := v + { + v := b.Int64() + s.EndOffset = v + } + { + v := b.Int32() + s.Epoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.Brokers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchResponseBroker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Brokers = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrFetchResponse returns a pointer to a default FetchResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFetchResponse() *FetchResponse { + var v FetchResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchResponse. +func (v *FetchResponse) Default() { +} + +// NewFetchResponse returns a default FetchResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchResponse() FetchResponse { + var v FetchResponse + v.Default() + return v +} + +type ListOffsetsRequestTopicPartition struct { + // Partition is a partition of a topic to get offsets for. + Partition int32 + + // CurrentLeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0, + // allows brokers to check if the client is fenced (has an out of date + // leader) or is using an unknown leader. + // + // The initial leader epoch can be determined from a MetadataResponse. + // To skip log truncation checking, use -1. + // + // This field has a default of -1. + CurrentLeaderEpoch int32 // v4+ + + // Timestamp controls which offset to return in a response for this + // partition. + // + // The offset returned will be the one of the message whose timestamp is + // the first timestamp greater than or equal to this requested timestamp. + // + // If no such message is found, then no offset is returned (-1). + // + // There exist two special timestamps: -2 corresponds to the earliest + // timestamp, and -1 corresponds to the latest. + // + // If you are talking to Kafka 3.0+, there exists an additional special + // timestamp -3 that returns the latest timestamp produced so far and its + // corresponding offset. This is subtly different from the latest offset, + // because timestamps are client-side generated. More importantly though, + // because this returns the latest produced timestamp, this can be used + // to determine topic "liveness" (when was the last produce?). + // Previously, this was not easy to determine. See KIP-734 for more + // detail. + // + // If you are talking to Kafka 3.4+ and using request version 8+ (for + // KIP-405), the new special timestamp -4 returns the local log start + // offset. In the context of tiered storage, the earliest local log start + // offset is the offset actually available on disk on the broker. + // + // If you are talking to Kafka 3.9+ and using request version 9+ (for KIP-1005), + // the special timestamp -5 returns the latest offset in remote storage. + Timestamp int64 + + // MaxNumOffsets is the maximum number of offsets to report. + // This was removed after v0. + // + // This field has a default of 1. + MaxNumOffsets int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsRequestTopicPartition. +func (v *ListOffsetsRequestTopicPartition) Default() { + v.CurrentLeaderEpoch = -1 + v.MaxNumOffsets = 1 +} + +// NewListOffsetsRequestTopicPartition returns a default ListOffsetsRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsRequestTopicPartition() ListOffsetsRequestTopicPartition { + var v ListOffsetsRequestTopicPartition + v.Default() + return v +} + +type ListOffsetsRequestTopic struct { + // Topic is a topic to get offsets for. + Topic string + + // Partitions is an array of partitions in a topic to get offsets for. + Partitions []ListOffsetsRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsRequestTopic. +func (v *ListOffsetsRequestTopic) Default() { +} + +// NewListOffsetsRequestTopic returns a default ListOffsetsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsRequestTopic() ListOffsetsRequestTopic { + var v ListOffsetsRequestTopic + v.Default() + return v +} + +// ListOffsetsRequest requests partition offsets from Kafka for use in +// consuming records. +// +// Version 5, introduced in Kafka 2.2.0, is the same as version 4. Using +// version 5 implies you support Kafka's OffsetNotAvailableException +// See KIP-207 for details. +// +// Version 7, introduced in Kafka 3.0, supports -3 as a timestamp to return +// the timestamp and offset for the record with the largest timestamp. +// +// Version 8, introduced in Kafka 3.4, supports -4 as a timestamp to return +// the local log start offset (in the context of tiered storage, see KIP-405). +// +// Version 9, introduced in Kafka 3.9, supports -5 as a timestamp to return +// the latest offset in remote storage. See KIP-1005. +// +// Version 10, introduced in Kafka 4.0, adds TimeoutMillis, allowing you to set +// a timeout when the ListOffsets request triggers a lookup from remote storage. +// See KIP-1075. +// +// Version 11, introduced in Kafka 4.2, supports -6 as a timestamp to return +// the earliest pending upload offset. See KIP-1023. +type ListOffsetsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ReplicaID is the broker ID to get offsets from. As a Kafka client, use -1. + // The consumer replica ID (-1) causes requests to only succeed if issued + // against the leader broker. + // + // This field has a default of -1. + ReplicaID int32 + + // IsolationLevel configures which record offsets are visible in the + // response. READ_UNCOMMITTED (0) makes all records visible. READ_COMMITTED + // (1) makes non-transactional and committed transactional records visible. + // READ_COMMITTED means all offsets smaller than the last stable offset and + // includes aborted transactions (allowing consumers to discard aborted + // records). + IsolationLevel int8 // v2+ + + // Topics is an array of topics to get offsets for. + Topics []ListOffsetsRequestTopic + + // TimeoutMillis is how long to wait for a lookup the offset being looked up + // is in remote storage. + // + // This field has a default of 30000. + TimeoutMillis int32 // v10+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*ListOffsetsRequest) Key() int16 { return 2 } +func (*ListOffsetsRequest) MaxVersion() int16 { return 11 } +func (v *ListOffsetsRequest) SetVersion(version int16) { v.Version = version } +func (v *ListOffsetsRequest) GetVersion() int16 { return v.Version } +func (v *ListOffsetsRequest) IsFlexible() bool { return v.Version >= 6 } +func (v *ListOffsetsRequest) ResponseKind() Response { + r := &ListOffsetsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ListOffsetsRequest) RequestWith(ctx context.Context, r Requestor) (*ListOffsetsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ListOffsetsResponse) + return resp, err +} + +func (v *ListOffsetsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.IsolationLevel + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 4 { + v := v.CurrentLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Timestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 0 && version <= 0 { + v := v.MaxNumOffsets + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 10 { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListOffsetsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListOffsetsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListOffsetsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + { + v := b.Int32() + s.ReplicaID = v + } + if version >= 2 { + v := b.Int8() + s.IsolationLevel = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListOffsetsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListOffsetsRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if version >= 4 { + v := b.Int32() + s.CurrentLeaderEpoch = v + } + { + v := b.Int64() + s.Timestamp = v + } + if version >= 0 && version <= 0 { + v := b.Int32() + s.MaxNumOffsets = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 10 { + v := b.Int32() + s.TimeoutMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListOffsetsRequest returns a pointer to a default ListOffsetsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListOffsetsRequest() *ListOffsetsRequest { + var v ListOffsetsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsRequest. +func (v *ListOffsetsRequest) Default() { + v.ReplicaID = -1 + v.TimeoutMillis = 30000 +} + +// NewListOffsetsRequest returns a default ListOffsetsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsRequest() ListOffsetsRequest { + var v ListOffsetsRequest + v.Default() + return v +} + +type ListOffsetsResponseTopicPartition struct { + // Partition is the partition this array slot is for. + Partition int32 + + // ErrorCode is any error for a topic partition in a ListOffsets request. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // to describe the topic. + // + // INVALID_REQUEST is returned if the requested topic partitions had + // contained duplicates. + // + // KAFKA_STORAGE_EXCEPTION is returned if the topic / partition is in + // an offline log directory. + // + // UNSUPPORTED_FOR_MESSAGE_FORMAT is returned if the broker is using + // Kafka 0.10.0 messages and the requested timestamp was not -1 nor -2. + // + // NOT_LEADER_FOR_PARTITION is returned if the broker is not a leader + // for this partition. This means that the client has stale metadata. + // If the request used the debug replica ID, the returned error will + // be REPLICA_NOT_AVAILABLE. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know + // of the requested topic or partition. + // + // FENCED_LEADER_EPOCH is returned if the broker has a higher leader + // epoch than what the request sent. + // + // UNKNOWN_LEADER_EPOCH is returned if the request used a leader epoch + // that the broker does not know about. + // + // OFFSET_NOT_AVAILABLE, introduced in Kafka 2.2.0 with produce request + // v5+, is returned when talking to a broker that is a new leader while + // that broker's high water mark catches up. This avoids situations where + // the old broker returned higher offsets than the new broker would. Note + // that if unclean leader election is allowed, you could still run into + // the situation where offsets returned from list offsets requests are + // not monotonically increasing. This error is only returned if the + // request used the consumer replica ID (-1). If the client did not use + // a v5+ list offsets request, LEADER_NOT_AVAILABLE is returned. + // See KIP-207 for more details. + ErrorCode int16 + + // OldStyleOffsets is a list of offsets. This was removed after + // version 0 and, since it is so historic, is undocumented. + OldStyleOffsets []int64 + + // If the request was for the earliest or latest timestamp (-2 or -1), or + // if an offset could not be found after the requested one, this will be -1. + // + // This field has a default of -1. + Timestamp int64 // v1+ + + // Offset is the offset corresponding to the record on or after the + // requested timestamp. If one could not be found, this will be -1. + // + // This field has a default of -1. + Offset int64 // v1+ + + // LeaderEpoch is the leader epoch of the record at this offset, + // or -1 if there was no leader epoch. + // + // This field has a default of -1. + LeaderEpoch int32 // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsResponseTopicPartition. +func (v *ListOffsetsResponseTopicPartition) Default() { + v.Timestamp = -1 + v.Offset = -1 + v.LeaderEpoch = -1 +} + +// NewListOffsetsResponseTopicPartition returns a default ListOffsetsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsResponseTopicPartition() ListOffsetsResponseTopicPartition { + var v ListOffsetsResponseTopicPartition + v.Default() + return v +} + +type ListOffsetsResponseTopic struct { + // Topic is the topic this array slot is for. + Topic string + + // Partitions is an array of partition responses corresponding to + // the requested partitions for a topic. + Partitions []ListOffsetsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsResponseTopic. +func (v *ListOffsetsResponseTopic) Default() { +} + +// NewListOffsetsResponseTopic returns a default ListOffsetsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsResponseTopic() ListOffsetsResponseTopic { + var v ListOffsetsResponseTopic + v.Default() + return v +} + +// ListOffsetsResponse is returned from a ListOffsetsRequest. +type ListOffsetsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 3. + ThrottleMillis int32 // v2+ + + // Topics is an array of topic / partition responses corresponding to + // the requested topics and partitions. + Topics []ListOffsetsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*ListOffsetsResponse) Key() int16 { return 2 } +func (*ListOffsetsResponse) MaxVersion() int16 { return 11 } +func (v *ListOffsetsResponse) SetVersion(version int16) { v.Version = version } +func (v *ListOffsetsResponse) GetVersion() int16 { return v.Version } +func (v *ListOffsetsResponse) IsFlexible() bool { return v.Version >= 6 } +func (v *ListOffsetsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 3 } +func (v *ListOffsetsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ListOffsetsResponse) RequestKind() Request { return &ListOffsetsRequest{Version: v.Version} } + +func (v *ListOffsetsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + if version >= 2 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 0 && version <= 0 { + v := v.OldStyleOffsets + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt64(dst, v) + } + } + if version >= 1 { + v := v.Timestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if version >= 4 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListOffsetsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListOffsetsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListOffsetsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + if version >= 2 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListOffsetsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListOffsetsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 0 && version <= 0 { + v := s.OldStyleOffsets + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int64, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int64() + a[i] = v + } + v = a + s.OldStyleOffsets = v + } + if version >= 1 { + v := b.Int64() + s.Timestamp = v + } + if version >= 1 { + v := b.Int64() + s.Offset = v + } + if version >= 4 { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListOffsetsResponse returns a pointer to a default ListOffsetsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListOffsetsResponse() *ListOffsetsResponse { + var v ListOffsetsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListOffsetsResponse. +func (v *ListOffsetsResponse) Default() { +} + +// NewListOffsetsResponse returns a default ListOffsetsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewListOffsetsResponse() ListOffsetsResponse { + var v ListOffsetsResponse + v.Default() + return v +} + +type MetadataRequestTopic struct { + // The topic ID. Only one of either topic ID or topic name should be used. + // If using the topic name, this should just be the default empty value. + TopicID [16]byte // v10+ + + // Topic is the topic to request metadata for. Version 10 switched this + // from a string to a nullable string; if using a topic ID, this field + // should be null. + Topic *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataRequestTopic. +func (v *MetadataRequestTopic) Default() { +} + +// NewMetadataRequestTopic returns a default MetadataRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataRequestTopic() MetadataRequestTopic { + var v MetadataRequestTopic + v.Default() + return v +} + +// MetadataRequest requests metadata from Kafka. +type MetadataRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics is a list of topics to return metadata about. If this is null + // in v1+, all topics are included. If this is empty, no topics are. + // For v0 (= 9 } +func (v *MetadataRequest) ResponseKind() Response { + r := &MetadataResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *MetadataRequest) RequestWith(ctx context.Context, r Requestor) (*MetadataResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*MetadataResponse) + return resp, err +} + +func (v *MetadataRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + { + v := v.Topics + if version >= 1 { + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + } else { + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + } + for i := range v { + v := &v[i] + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topic + if version < 10 { + var vv string + if v != nil { + vv = *v + } + { + v := vv + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } else { + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 4 { + v := v.AllowAutoTopicCreation + dst = kbin.AppendBool(dst, v) + } + if version >= 8 && version <= 10 { + v := v.IncludeClusterAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if version >= 8 { + v := v.IncludeTopicAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *MetadataRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *MetadataRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *MetadataRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 1 || l == 0 { + a = []MetadataRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]MetadataRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + { + var v *string + if version < 10 { + var vv string + if isFlexible { + if unsafe { + vv = b.UnsafeCompactString() + } else { + vv = b.CompactString() + } + } else { + if unsafe { + vv = b.UnsafeString() + } else { + vv = b.String() + } + } + v = &vv + } else { + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + } + s.Topic = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 4 { + v := b.Bool() + s.AllowAutoTopicCreation = v + } + if version >= 8 && version <= 10 { + v := b.Bool() + s.IncludeClusterAuthorizedOperations = v + } + if version >= 8 { + v := b.Bool() + s.IncludeTopicAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrMetadataRequest returns a pointer to a default MetadataRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrMetadataRequest() *MetadataRequest { + var v MetadataRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataRequest. +func (v *MetadataRequest) Default() { +} + +// NewMetadataRequest returns a default MetadataRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataRequest() MetadataRequest { + var v MetadataRequest + v.Default() + return v +} + +type MetadataResponseBroker struct { + // NodeID is the node ID of a Kafka broker. + NodeID int32 + + // Host is the hostname of a Kafka broker. + Host string + + // Port is the port of a Kafka broker. + Port int32 + + // Rack is the rack this Kafka broker is in. + Rack *string // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataResponseBroker. +func (v *MetadataResponseBroker) Default() { +} + +// NewMetadataResponseBroker returns a default MetadataResponseBroker +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataResponseBroker() MetadataResponseBroker { + var v MetadataResponseBroker + v.Default() + return v +} + +type MetadataResponseTopicPartition struct { + // ErrorCode is any error for a partition in topic metadata. + // + // LEADER_NOT_AVAILABLE is returned if a leader is unavailable for this + // partition. For v0 metadata responses, this is also returned if a + // partition leader's listener does not exist. + // + // LISTENER_NOT_FOUND is returned if a leader ID is known but the + // listener for it is not (v1+). + // + // REPLICA_NOT_AVAILABLE is returned in v0 responses if any replica is + // unavailable. + // + // UNKNOWN_TOPIC_ID is returned if using a topic ID and the ID does not + // exist. + ErrorCode int16 + + // Partition is a partition number for a topic. + Partition int32 + + // Leader is the broker leader for this partition. This will be -1 + // on leader / listener error. + Leader int32 + + // LeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0 is the + // epoch of the broker leader. + // + // This field has a default of -1. + LeaderEpoch int32 // v7+ + + // Replicas returns all broker IDs containing replicas of this partition. + Replicas []int32 + + // ISR returns all broker IDs of in-sync replicas of this partition. + ISR []int32 + + // OfflineReplicas, proposed in KIP-112 and introduced in Kafka 1.0, + // returns all offline broker IDs that should be replicating this partition. + OfflineReplicas []int32 // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataResponseTopicPartition. +func (v *MetadataResponseTopicPartition) Default() { + v.LeaderEpoch = -1 +} + +// NewMetadataResponseTopicPartition returns a default MetadataResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataResponseTopicPartition() MetadataResponseTopicPartition { + var v MetadataResponseTopicPartition + v.Default() + return v +} + +type MetadataResponseTopic struct { + // ErrorCode is any error for a topic in a metadata request. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // to describe the topic, or if the metadata request specified topic auto + // creation, the topic did not exist, and the user lacks permission to create. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if a topic does not exist and + // the request did not specify autocreation. + // + // LEADER_NOT_AVAILABLE is returned if a new topic is created successfully + // (since there is no leader on an immediately new topic). + // + // There can be a myriad of other errors for unsuccessful topic creation. + ErrorCode int16 + + // Topic is the topic this metadata corresponds to. + Topic *string + + // The topic ID. + TopicID [16]byte // v10+ + + // IsInternal signifies whether this topic is a Kafka internal topic. + IsInternal bool // v1+ + + // Partitions contains metadata about partitions for a topic. + Partitions []MetadataResponseTopicPartition + + // AuthorizedOperations, proposed in KIP-430 and introduced in Kafka 2.3.0, + // is a bitfield (corresponding to AclOperation) containing which operations + // the client is allowed to perform on this topic. + // This is only returned if requested. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 // v8+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataResponseTopic. +func (v *MetadataResponseTopic) Default() { + v.AuthorizedOperations = -2147483648 +} + +// NewMetadataResponseTopic returns a default MetadataResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataResponseTopic() MetadataResponseTopic { + var v MetadataResponseTopic + v.Default() + return v +} + +// MetadataResponse is returned from a MetdataRequest. +type MetadataResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 6. + ThrottleMillis int32 // v3+ + + // Brokers is a set of alive Kafka brokers. + Brokers []MetadataResponseBroker + + // ClusterID, proposed in KIP-78 and introduced in Kafka 0.10.1.0, is a + // unique string specifying the cluster that the replying Kafka belongs to. + ClusterID *string // v2+ + + // ControllerID is the ID of the controller broker (the admin broker). + // + // This field has a default of -1. + ControllerID int32 // v1+ + + // Topics contains metadata about each topic requested in the + // MetadataRequest. + Topics []MetadataResponseTopic + + // AuthorizedOperations is a bitfield containing which operations the client + // is allowed to perform on this cluster. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 // v8-v10 + + // ErrorCode indicates any error. Kafka 4.0 introduced this via KIP-1102 + // to signal to clients that rebootstrapping is required. + ErrorCode int16 // v13+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v9+ +} + +func (*MetadataResponse) Key() int16 { return 3 } +func (*MetadataResponse) MaxVersion() int16 { return 13 } +func (v *MetadataResponse) SetVersion(version int16) { v.Version = version } +func (v *MetadataResponse) GetVersion() int16 { return v.Version } +func (v *MetadataResponse) IsFlexible() bool { return v.Version >= 9 } +func (v *MetadataResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 6 } +func (v *MetadataResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *MetadataResponse) RequestKind() Request { return &MetadataRequest{Version: v.Version} } + +func (v *MetadataResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + if version >= 3 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Brokers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 2 { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topic + if version < 12 { + var vv string + if v != nil { + vv = *v + } + { + v := vv + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } else { + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + } + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + if version >= 1 { + v := v.IsInternal + dst = kbin.AppendBool(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Leader + dst = kbin.AppendInt32(dst, v) + } + if version >= 7 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 5 { + v := v.OfflineReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 8 { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 8 && version <= 10 { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if version >= 13 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *MetadataResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *MetadataResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *MetadataResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 9 + _ = isFlexible + s := v + if version >= 3 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Brokers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]MetadataResponseBroker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + if version >= 1 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Brokers = v + } + if version >= 2 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + if version >= 1 { + v := b.Int32() + s.ControllerID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]MetadataResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if version < 12 { + var vv string + if isFlexible { + if unsafe { + vv = b.UnsafeCompactString() + } else { + vv = b.CompactString() + } + } else { + if unsafe { + vv = b.UnsafeString() + } else { + vv = b.String() + } + } + v = &vv + } else { + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + } + s.Topic = v + } + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + if version >= 1 { + v := b.Bool() + s.IsInternal = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]MetadataResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.Leader = v + } + if version >= 7 { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + if version >= 5 { + v := s.OfflineReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.OfflineReplicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if version >= 8 { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 8 && version <= 10 { + v := b.Int32() + s.AuthorizedOperations = v + } + if version >= 13 { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrMetadataResponse returns a pointer to a default MetadataResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrMetadataResponse() *MetadataResponse { + var v MetadataResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to MetadataResponse. +func (v *MetadataResponse) Default() { + v.ControllerID = -1 + v.AuthorizedOperations = -2147483648 +} + +// NewMetadataResponse returns a default MetadataResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewMetadataResponse() MetadataResponse { + var v MetadataResponse + v.Default() + return v +} + +// LeaderAndISRRequestTopicPartition is a common struct that is used across +// different versions of LeaderAndISRRequest. +type LeaderAndISRRequestTopicPartition struct { + Topic string // v0-v1 + + Partition int32 + + ControllerEpoch int32 + + Leader int32 + + LeaderEpoch int32 + + ISR []int32 + + ZKVersion int32 + + Replicas []int32 + + AddingReplicas []int32 // v3+ + + RemovingReplicas []int32 // v3+ + + IsNew bool // v1+ + + LeaderRecoveryState int8 // v6+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRRequestTopicPartition. +func (v *LeaderAndISRRequestTopicPartition) Default() { +} + +// NewLeaderAndISRRequestTopicPartition returns a default LeaderAndISRRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRRequestTopicPartition() LeaderAndISRRequestTopicPartition { + var v LeaderAndISRRequestTopicPartition + v.Default() + return v +} + +// LeaderAndISRResponseTopicPartition is a common struct that is used across +// different versions of LeaderAndISRResponse. +type LeaderAndISRResponseTopicPartition struct { + Topic string // v0-v4 + + Partition int32 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRResponseTopicPartition. +func (v *LeaderAndISRResponseTopicPartition) Default() { +} + +// NewLeaderAndISRResponseTopicPartition returns a default LeaderAndISRResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRResponseTopicPartition() LeaderAndISRResponseTopicPartition { + var v LeaderAndISRResponseTopicPartition + v.Default() + return v +} + +type LeaderAndISRRequestTopicState struct { + Topic string + + TopicID [16]byte // v5+ + + PartitionStates []LeaderAndISRRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRRequestTopicState. +func (v *LeaderAndISRRequestTopicState) Default() { +} + +// NewLeaderAndISRRequestTopicState returns a default LeaderAndISRRequestTopicState +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRRequestTopicState() LeaderAndISRRequestTopicState { + var v LeaderAndISRRequestTopicState + v.Default() + return v +} + +type LeaderAndISRRequestLiveLeader struct { + BrokerID int32 + + Host string + + Port int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRRequestLiveLeader. +func (v *LeaderAndISRRequestLiveLeader) Default() { +} + +// NewLeaderAndISRRequestLiveLeader returns a default LeaderAndISRRequestLiveLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRRequestLiveLeader() LeaderAndISRRequestLiveLeader { + var v LeaderAndISRRequestLiveLeader + v.Default() + return v +} + +// LeaderAndISRRequest is an advanced request that controller brokers use +// to broadcast state to other brokers. Manually using this request is a +// great way to break your cluster. +// +// As this is an advanced request and there is little reason to issue it as a +// client, this request is undocumented. +// +// Kafka 1.0 introduced version 1. Kafka 2.2 introduced version 2, proposed +// in KIP-380, which changed the layout of the struct to be more memory +// efficient. Kafka 2.4.0 introduced version 3 with KIP-455. +// Kafka 3.4 introduced version 7 with KIP-866. +type LeaderAndISRRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ControllerID int32 + + // If KRaft controller id is used during migration. See KIP-866. + IsKRaftController bool // v7+ + + ControllerEpoch int32 + + // This field has a default of -1. + BrokerEpoch int64 // v2+ + + Type int8 // v5+ + + PartitionStates []LeaderAndISRRequestTopicPartition // v0-v1 + + TopicStates []LeaderAndISRRequestTopicState // v2+ + + LiveLeaders []LeaderAndISRRequestLiveLeader + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*LeaderAndISRRequest) Key() int16 { return 4 } +func (*LeaderAndISRRequest) MaxVersion() int16 { return 7 } +func (v *LeaderAndISRRequest) SetVersion(version int16) { v.Version = version } +func (v *LeaderAndISRRequest) GetVersion() int16 { return v.Version } +func (v *LeaderAndISRRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *LeaderAndISRRequest) ResponseKind() Response { + r := &LeaderAndISRResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *LeaderAndISRRequest) RequestWith(ctx context.Context, r Requestor) (*LeaderAndISRResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*LeaderAndISRResponse) + return resp, err +} + +func (v *LeaderAndISRRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + if version >= 7 { + v := v.IsKRaftController + dst = kbin.AppendBool(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if version >= 5 { + v := v.Type + dst = kbin.AppendInt8(dst, v) + } + if version >= 0 && version <= 1 { + v := v.PartitionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 1 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Leader + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ZKVersion + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.AddingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.RemovingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 1 { + v := v.IsNew + dst = kbin.AppendBool(dst, v) + } + if version >= 6 { + v := v.LeaderRecoveryState + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 2 { + v := v.TopicStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 5 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.PartitionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 1 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Leader + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ZKVersion + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.AddingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.RemovingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 1 { + v := v.IsNew + dst = kbin.AppendBool(dst, v) + } + if version >= 6 { + v := v.LeaderRecoveryState + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.LiveLeaders + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *LeaderAndISRRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *LeaderAndISRRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *LeaderAndISRRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + v := b.Int32() + s.ControllerID = v + } + if version >= 7 { + v := b.Bool() + s.IsKRaftController = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + if version >= 2 { + v := b.Int64() + s.BrokerEpoch = v + } + if version >= 5 { + v := b.Int8() + s.Type = v + } + if version >= 0 && version <= 1 { + v := s.PartitionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + { + v := b.Int32() + s.Leader = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + { + v := b.Int32() + s.ZKVersion = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if version >= 3 { + v := s.AddingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.AddingReplicas = v + } + if version >= 3 { + v := s.RemovingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.RemovingReplicas = v + } + if version >= 1 { + v := b.Bool() + s.IsNew = v + } + if version >= 6 { + v := b.Int8() + s.LeaderRecoveryState = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionStates = v + } + if version >= 2 { + v := s.TopicStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRRequestTopicState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 5 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.PartitionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + { + v := b.Int32() + s.Leader = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + { + v := b.Int32() + s.ZKVersion = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if version >= 3 { + v := s.AddingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.AddingReplicas = v + } + if version >= 3 { + v := s.RemovingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.RemovingReplicas = v + } + if version >= 1 { + v := b.Bool() + s.IsNew = v + } + if version >= 6 { + v := b.Int8() + s.LeaderRecoveryState = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionStates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicStates = v + } + { + v := s.LiveLeaders + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRRequestLiveLeader, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.LiveLeaders = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrLeaderAndISRRequest returns a pointer to a default LeaderAndISRRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrLeaderAndISRRequest() *LeaderAndISRRequest { + var v LeaderAndISRRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRRequest. +func (v *LeaderAndISRRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewLeaderAndISRRequest returns a default LeaderAndISRRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRRequest() LeaderAndISRRequest { + var v LeaderAndISRRequest + v.Default() + return v +} + +type LeaderAndISRResponseTopic struct { + TopicID [16]byte + + Partitions []LeaderAndISRResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRResponseTopic. +func (v *LeaderAndISRResponseTopic) Default() { +} + +// NewLeaderAndISRResponseTopic returns a default LeaderAndISRResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRResponseTopic() LeaderAndISRResponseTopic { + var v LeaderAndISRResponseTopic + v.Default() + return v +} + +// LeaderAndISRResponse is returned from a LeaderAndISRRequest. +type LeaderAndISRResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + Partitions []LeaderAndISRResponseTopicPartition // v0-v4 + + Topics []LeaderAndISRResponseTopic // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*LeaderAndISRResponse) Key() int16 { return 4 } +func (*LeaderAndISRResponse) MaxVersion() int16 { return 7 } +func (v *LeaderAndISRResponse) SetVersion(version int16) { v.Version = version } +func (v *LeaderAndISRResponse) GetVersion() int16 { return v.Version } +func (v *LeaderAndISRResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *LeaderAndISRResponse) RequestKind() Request { return &LeaderAndISRRequest{Version: v.Version} } + +func (v *LeaderAndISRResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 0 && version <= 4 { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 4 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 5 { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 4 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *LeaderAndISRResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *LeaderAndISRResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *LeaderAndISRResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 0 && version <= 4 { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 4 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if version >= 5 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaderAndISRResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 4 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrLeaderAndISRResponse returns a pointer to a default LeaderAndISRResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrLeaderAndISRResponse() *LeaderAndISRResponse { + var v LeaderAndISRResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaderAndISRResponse. +func (v *LeaderAndISRResponse) Default() { +} + +// NewLeaderAndISRResponse returns a default LeaderAndISRResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaderAndISRResponse() LeaderAndISRResponse { + var v LeaderAndISRResponse + v.Default() + return v +} + +type StopReplicaRequestTopicPartitionState struct { + Partition int32 + + // This field has a default of -1. + LeaderEpoch int32 + + Delete bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StopReplicaRequestTopicPartitionState. +func (v *StopReplicaRequestTopicPartitionState) Default() { + v.LeaderEpoch = -1 +} + +// NewStopReplicaRequestTopicPartitionState returns a default StopReplicaRequestTopicPartitionState +// This is a shortcut for creating a struct and calling Default yourself. +func NewStopReplicaRequestTopicPartitionState() StopReplicaRequestTopicPartitionState { + var v StopReplicaRequestTopicPartitionState + v.Default() + return v +} + +type StopReplicaRequestTopic struct { + Topic string + + Partition int32 + + Partitions []int32 // v1-v2 + + PartitionStates []StopReplicaRequestTopicPartitionState // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StopReplicaRequestTopic. +func (v *StopReplicaRequestTopic) Default() { +} + +// NewStopReplicaRequestTopic returns a default StopReplicaRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewStopReplicaRequestTopic() StopReplicaRequestTopic { + var v StopReplicaRequestTopic + v.Default() + return v +} + +// StopReplicaRequest is an advanced request that brokers use to stop replicas. +// +// As this is an advanced request and there is little reason to issue it as a +// client, this request is undocumented. +// +// Kafka 2.2 introduced version 1, proposed in KIP-380, which changed the +// layout of the struct to be more memory efficient. +// +// Kafka 2.6 introduced version 3, proposed in KIP-570, reorganizes partitions +// to be stored and adds the leader epoch and delete partition fields per partition. +// Kafka 3.4 introduced version 4 with KIP-866. +type StopReplicaRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ControllerID int32 + + ControllerEpoch int32 + + // If KRaft controller id is used during migration. See KIP-866. + IsKRaftController bool // v4+ + + // This field has a default of -1. + BrokerEpoch int64 // v1+ + + DeletePartitions bool // v0-v2 + + Topics []StopReplicaRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*StopReplicaRequest) Key() int16 { return 5 } +func (*StopReplicaRequest) MaxVersion() int16 { return 4 } +func (v *StopReplicaRequest) SetVersion(version int16) { v.Version = version } +func (v *StopReplicaRequest) GetVersion() int16 { return v.Version } +func (v *StopReplicaRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *StopReplicaRequest) ResponseKind() Response { + r := &StopReplicaResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *StopReplicaRequest) RequestWith(ctx context.Context, r Requestor) (*StopReplicaResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*StopReplicaResponse) + return resp, err +} + +func (v *StopReplicaRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 4 { + v := v.IsKRaftController + dst = kbin.AppendBool(dst, v) + } + if version >= 1 { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if version >= 0 && version <= 2 { + v := v.DeletePartitions + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 0 { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 && version <= 2 { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.PartitionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Delete + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StopReplicaRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StopReplicaRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StopReplicaRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ControllerID = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + if version >= 4 { + v := b.Bool() + s.IsKRaftController = v + } + if version >= 1 { + v := b.Int64() + s.BrokerEpoch = v + } + if version >= 0 && version <= 2 { + v := b.Bool() + s.DeletePartitions = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StopReplicaRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 0 && version <= 0 { + v := b.Int32() + s.Partition = v + } + if version >= 1 && version <= 2 { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if version >= 3 { + v := s.PartitionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StopReplicaRequestTopicPartitionState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Bool() + s.Delete = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionStates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStopReplicaRequest returns a pointer to a default StopReplicaRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStopReplicaRequest() *StopReplicaRequest { + var v StopReplicaRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StopReplicaRequest. +func (v *StopReplicaRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewStopReplicaRequest returns a default StopReplicaRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewStopReplicaRequest() StopReplicaRequest { + var v StopReplicaRequest + v.Default() + return v +} + +type StopReplicaResponsePartition struct { + Topic string + + Partition int32 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StopReplicaResponsePartition. +func (v *StopReplicaResponsePartition) Default() { +} + +// NewStopReplicaResponsePartition returns a default StopReplicaResponsePartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewStopReplicaResponsePartition() StopReplicaResponsePartition { + var v StopReplicaResponsePartition + v.Default() + return v +} + +// StopReplicasResponse is returned from a StopReplicasRequest. +type StopReplicaResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Version 3 returns FENCED_LEADER_EPOCH if the leader is stale (KIP-570). + ErrorCode int16 + + Partitions []StopReplicaResponsePartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*StopReplicaResponse) Key() int16 { return 5 } +func (*StopReplicaResponse) MaxVersion() int16 { return 4 } +func (v *StopReplicaResponse) SetVersion(version int16) { v.Version = version } +func (v *StopReplicaResponse) GetVersion() int16 { return v.Version } +func (v *StopReplicaResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *StopReplicaResponse) RequestKind() Request { return &StopReplicaRequest{Version: v.Version} } + +func (v *StopReplicaResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StopReplicaResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StopReplicaResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StopReplicaResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StopReplicaResponsePartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStopReplicaResponse returns a pointer to a default StopReplicaResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStopReplicaResponse() *StopReplicaResponse { + var v StopReplicaResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StopReplicaResponse. +func (v *StopReplicaResponse) Default() { +} + +// NewStopReplicaResponse returns a default StopReplicaResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewStopReplicaResponse() StopReplicaResponse { + var v StopReplicaResponse + v.Default() + return v +} + +type UpdateMetadataRequestTopicPartition struct { + Topic string // v0-v4 + + Partition int32 + + ControllerEpoch int32 + + Leader int32 + + LeaderEpoch int32 + + ISR []int32 + + ZKVersion int32 + + Replicas []int32 + + OfflineReplicas []int32 // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataRequestTopicPartition. +func (v *UpdateMetadataRequestTopicPartition) Default() { +} + +// NewUpdateMetadataRequestTopicPartition returns a default UpdateMetadataRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataRequestTopicPartition() UpdateMetadataRequestTopicPartition { + var v UpdateMetadataRequestTopicPartition + v.Default() + return v +} + +type UpdateMetadataRequestTopicState struct { + Topic string + + TopicID [16]byte // v7+ + + PartitionStates []UpdateMetadataRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataRequestTopicState. +func (v *UpdateMetadataRequestTopicState) Default() { +} + +// NewUpdateMetadataRequestTopicState returns a default UpdateMetadataRequestTopicState +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataRequestTopicState() UpdateMetadataRequestTopicState { + var v UpdateMetadataRequestTopicState + v.Default() + return v +} + +type UpdateMetadataRequestLiveBrokerEndpoint struct { + Port int32 + + Host string + + ListenerName string // v3+ + + SecurityProtocol int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataRequestLiveBrokerEndpoint. +func (v *UpdateMetadataRequestLiveBrokerEndpoint) Default() { +} + +// NewUpdateMetadataRequestLiveBrokerEndpoint returns a default UpdateMetadataRequestLiveBrokerEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataRequestLiveBrokerEndpoint() UpdateMetadataRequestLiveBrokerEndpoint { + var v UpdateMetadataRequestLiveBrokerEndpoint + v.Default() + return v +} + +type UpdateMetadataRequestLiveBroker struct { + ID int32 + + Host string + + Port int32 + + Endpoints []UpdateMetadataRequestLiveBrokerEndpoint // v1+ + + Rack *string // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataRequestLiveBroker. +func (v *UpdateMetadataRequestLiveBroker) Default() { +} + +// NewUpdateMetadataRequestLiveBroker returns a default UpdateMetadataRequestLiveBroker +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataRequestLiveBroker() UpdateMetadataRequestLiveBroker { + var v UpdateMetadataRequestLiveBroker + v.Default() + return v +} + +// UpdateMetadataRequest is an advanced request that brokers use to +// issue metadata updates to each other. +// +// As this is an advanced request and there is little reason to issue it as a +// client, this request is undocumented. +// +// Version 1 changed the layout of the live brokers. +// +// Kafka 2.2 introduced version 5, proposed in KIP-380, which changed the +// layout of the struct to be more memory efficient. +// Kafka 3.4 introduced version 8 with KIP-866. +type UpdateMetadataRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ControllerID int32 + + // If KRaft controller id is used during migration. See KIP-866. + IsKRaftController bool // v8+ + + ControllerEpoch int32 + + // This field has a default of -1. + BrokerEpoch int64 // v5+ + + PartitionStates []UpdateMetadataRequestTopicPartition // v0-v4 + + TopicStates []UpdateMetadataRequestTopicState // v5+ + + LiveBrokers []UpdateMetadataRequestLiveBroker + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*UpdateMetadataRequest) Key() int16 { return 6 } +func (*UpdateMetadataRequest) MaxVersion() int16 { return 8 } +func (v *UpdateMetadataRequest) SetVersion(version int16) { v.Version = version } +func (v *UpdateMetadataRequest) GetVersion() int16 { return v.Version } +func (v *UpdateMetadataRequest) IsFlexible() bool { return v.Version >= 6 } +func (v *UpdateMetadataRequest) ResponseKind() Response { + r := &UpdateMetadataResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *UpdateMetadataRequest) RequestWith(ctx context.Context, r Requestor) (*UpdateMetadataResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*UpdateMetadataResponse) + return resp, err +} + +func (v *UpdateMetadataRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + if version >= 8 { + v := v.IsKRaftController + dst = kbin.AppendBool(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 5 { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if version >= 0 && version <= 4 { + v := v.PartitionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 4 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Leader + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ZKVersion + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 4 { + v := v.OfflineReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 5 { + v := v.TopicStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 7 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.PartitionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 4 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ControllerEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Leader + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ZKVersion + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 4 { + v := v.OfflineReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.LiveBrokers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ID + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 0 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 0 { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Endpoints + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.ListenerName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SecurityProtocol + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 2 { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateMetadataRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateMetadataRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateMetadataRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + { + v := b.Int32() + s.ControllerID = v + } + if version >= 8 { + v := b.Bool() + s.IsKRaftController = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + if version >= 5 { + v := b.Int64() + s.BrokerEpoch = v + } + if version >= 0 && version <= 4 { + v := s.PartitionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateMetadataRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 4 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + { + v := b.Int32() + s.Leader = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + { + v := b.Int32() + s.ZKVersion = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if version >= 4 { + v := s.OfflineReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.OfflineReplicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionStates = v + } + if version >= 5 { + v := s.TopicStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateMetadataRequestTopicState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 7 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.PartitionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateMetadataRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 4 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.ControllerEpoch = v + } + { + v := b.Int32() + s.Leader = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + { + v := b.Int32() + s.ZKVersion = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if version >= 4 { + v := s.OfflineReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.OfflineReplicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionStates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicStates = v + } + { + v := s.LiveBrokers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateMetadataRequestLiveBroker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.ID = v + } + if version >= 0 && version <= 0 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 0 && version <= 0 { + v := b.Int32() + s.Port = v + } + if version >= 1 { + v := s.Endpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateMetadataRequestLiveBrokerEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Port = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ListenerName = v + } + { + v := b.Int16() + s.SecurityProtocol = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Endpoints = v + } + if version >= 2 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.LiveBrokers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUpdateMetadataRequest returns a pointer to a default UpdateMetadataRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateMetadataRequest() *UpdateMetadataRequest { + var v UpdateMetadataRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataRequest. +func (v *UpdateMetadataRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewUpdateMetadataRequest returns a default UpdateMetadataRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataRequest() UpdateMetadataRequest { + var v UpdateMetadataRequest + v.Default() + return v +} + +// UpdateMetadataResponses is returned from an UpdateMetadataRequest. +type UpdateMetadataResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*UpdateMetadataResponse) Key() int16 { return 6 } +func (*UpdateMetadataResponse) MaxVersion() int16 { return 8 } +func (v *UpdateMetadataResponse) SetVersion(version int16) { v.Version = version } +func (v *UpdateMetadataResponse) GetVersion() int16 { return v.Version } +func (v *UpdateMetadataResponse) IsFlexible() bool { return v.Version >= 6 } +func (v *UpdateMetadataResponse) RequestKind() Request { + return &UpdateMetadataRequest{Version: v.Version} +} + +func (v *UpdateMetadataResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateMetadataResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateMetadataResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateMetadataResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUpdateMetadataResponse returns a pointer to a default UpdateMetadataResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateMetadataResponse() *UpdateMetadataResponse { + var v UpdateMetadataResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateMetadataResponse. +func (v *UpdateMetadataResponse) Default() { +} + +// NewUpdateMetadataResponse returns a default UpdateMetadataResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateMetadataResponse() UpdateMetadataResponse { + var v UpdateMetadataResponse + v.Default() + return v +} + +// ControlledShutdownRequest is an advanced request that can be used to +// sthudown a broker in a controlled manner. +// +// As this is an advanced request and there is little reason to issue it as a +// client, this request is undocumented. However, the minimal amount of fields +// here makes the usage rather obvious. +// +// Kafka 2.2.0 introduced version 2, proposed in KIP-380. +// +// Note that version 0 of this request uses a special encoding format +// where the request does not include the client ID. +type ControlledShutdownRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + BrokerID int32 + + // This field has a default of -1. + BrokerEpoch int64 // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ControlledShutdownRequest) Key() int16 { return 7 } +func (*ControlledShutdownRequest) MaxVersion() int16 { return 3 } +func (v *ControlledShutdownRequest) SetVersion(version int16) { v.Version = version } +func (v *ControlledShutdownRequest) GetVersion() int16 { return v.Version } +func (v *ControlledShutdownRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *ControlledShutdownRequest) ResponseKind() Response { + r := &ControlledShutdownResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ControlledShutdownRequest) RequestWith(ctx context.Context, r Requestor) (*ControlledShutdownResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ControlledShutdownResponse) + return resp, err +} + +func (v *ControlledShutdownRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ControlledShutdownRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ControlledShutdownRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ControlledShutdownRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + if version >= 2 { + v := b.Int64() + s.BrokerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrControlledShutdownRequest returns a pointer to a default ControlledShutdownRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrControlledShutdownRequest() *ControlledShutdownRequest { + var v ControlledShutdownRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControlledShutdownRequest. +func (v *ControlledShutdownRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewControlledShutdownRequest returns a default ControlledShutdownRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewControlledShutdownRequest() ControlledShutdownRequest { + var v ControlledShutdownRequest + v.Default() + return v +} + +type ControlledShutdownResponsePartitionsRemaining struct { + Topic string + + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControlledShutdownResponsePartitionsRemaining. +func (v *ControlledShutdownResponsePartitionsRemaining) Default() { +} + +// NewControlledShutdownResponsePartitionsRemaining returns a default ControlledShutdownResponsePartitionsRemaining +// This is a shortcut for creating a struct and calling Default yourself. +func NewControlledShutdownResponsePartitionsRemaining() ControlledShutdownResponsePartitionsRemaining { + var v ControlledShutdownResponsePartitionsRemaining + v.Default() + return v +} + +// ControlledShutdownResponse is returned from a ControlledShutdownRequest. +type ControlledShutdownResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + PartitionsRemaining []ControlledShutdownResponsePartitionsRemaining + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ControlledShutdownResponse) Key() int16 { return 7 } +func (*ControlledShutdownResponse) MaxVersion() int16 { return 3 } +func (v *ControlledShutdownResponse) SetVersion(version int16) { v.Version = version } +func (v *ControlledShutdownResponse) GetVersion() int16 { return v.Version } +func (v *ControlledShutdownResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *ControlledShutdownResponse) RequestKind() Request { + return &ControlledShutdownRequest{Version: v.Version} +} + +func (v *ControlledShutdownResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.PartitionsRemaining + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ControlledShutdownResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ControlledShutdownResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ControlledShutdownResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.PartitionsRemaining + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ControlledShutdownResponsePartitionsRemaining, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionsRemaining = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrControlledShutdownResponse returns a pointer to a default ControlledShutdownResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrControlledShutdownResponse() *ControlledShutdownResponse { + var v ControlledShutdownResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControlledShutdownResponse. +func (v *ControlledShutdownResponse) Default() { +} + +// NewControlledShutdownResponse returns a default ControlledShutdownResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewControlledShutdownResponse() ControlledShutdownResponse { + var v ControlledShutdownResponse + v.Default() + return v +} + +type OffsetCommitRequestTopicPartition struct { + // Partition if a partition to commit offsets for. + Partition int32 + + // Offset is an offset to commit. + Offset int64 + + // Timestamp is the first iteration of tracking how long offset commits + // should persist in Kafka. This field only existed for v1. + // The expiration would be timestamp + offset.retention.minutes, or, if + // timestamp was zero, current time + offset.retention.minutes. + // + // This field has a default of -1. + Timestamp int64 // v1-v1 + + // LeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0, + // is the leader epoch of the record this request is committing. + // + // The initial leader epoch can be determined from a MetadataResponse. + // To skip log truncation checking, use -1. + // + // This field has a default of -1. + LeaderEpoch int32 // v6+ + + // Metadata is optional data to include with committing the offset. This + // can contain information such as which node is doing the committing, etc. + Metadata *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitRequestTopicPartition. +func (v *OffsetCommitRequestTopicPartition) Default() { + v.Timestamp = -1 + v.LeaderEpoch = -1 +} + +// NewOffsetCommitRequestTopicPartition returns a default OffsetCommitRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitRequestTopicPartition() OffsetCommitRequestTopicPartition { + var v OffsetCommitRequestTopicPartition + v.Default() + return v +} + +type OffsetCommitRequestTopic struct { + // Topic is a topic to commit offsets for. + Topic string // v0-v9 + + // TopicID is the topic ID, replacing Topic in v10+. + TopicID [16]byte // v10+ + + // Partitions contains partitions in a topic for which to commit offsets. + Partitions []OffsetCommitRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitRequestTopic. +func (v *OffsetCommitRequestTopic) Default() { +} + +// NewOffsetCommitRequestTopic returns a default OffsetCommitRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitRequestTopic() OffsetCommitRequestTopic { + var v OffsetCommitRequestTopic + v.Default() + return v +} + +// OffsetCommitRequest commits offsets for consumed topics / partitions in +// a group. +// Version 10, introduced in Kafka 4.2, replaces topic names with topic IDs +// (KIP-848). +type OffsetCommitRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group this request is committing offsets to. + Group string + + // Generation being -1 and group being empty means the group is being used + // to store offsets only. No generation validation, no rebalancing. + // + // This field has a default of -1. + Generation int32 // v1+ + + // MemberID is the ID of the client issuing this request in the group. + MemberID string // v1+ + + // InstanceID is the instance ID of this member in the group (KIP-345). + InstanceID *string // v7+ + + // RetentionTimeMillis is how long this commit will persist in Kafka. + // + // This was introduced in v2, replacing an individual topic/partition's + // Timestamp from v1, and was removed in v5 with Kafka 2.1.0. + // + // This was removed because rarely committing consumers could have their + // offsets expired before committing, even though the consumer was still + // active. After restarting or rebalancing, the consumer would now not know + // the last committed offset and would have to start at the beginning or end, + // leading to duplicates or log loss. + // + // Post 2.1.0, if this field is empty, offsets are only deleted once the + // group is empty. Read KIP-211 for more details. + // + // This field has a default of -1. + RetentionTimeMillis int64 // v2-v4 + + // Topics is contains topics and partitions for which to commit offsets. + Topics []OffsetCommitRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +func (*OffsetCommitRequest) Key() int16 { return 8 } +func (*OffsetCommitRequest) MaxVersion() int16 { return 10 } +func (v *OffsetCommitRequest) SetVersion(version int16) { v.Version = version } +func (v *OffsetCommitRequest) GetVersion() int16 { return v.Version } +func (v *OffsetCommitRequest) IsFlexible() bool { return v.Version >= 8 } +func (v *OffsetCommitRequest) IsGroupCoordinatorRequest() {} +func (v *OffsetCommitRequest) ResponseKind() Response { + r := &OffsetCommitResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *OffsetCommitRequest) RequestWith(ctx context.Context, r Requestor) (*OffsetCommitResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*OffsetCommitResponse) + return resp, err +} + +func (v *OffsetCommitRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 8 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 7 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 2 && version <= 4 { + v := v.RetentionTimeMillis + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 9 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 && version <= 1 { + v := v.Timestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 6 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetCommitRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetCommitRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetCommitRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 8 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + if version >= 1 { + v := b.Int32() + s.Generation = v + } + if version >= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 7 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + if version >= 2 && version <= 4 { + v := b.Int64() + s.RetentionTimeMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetCommitRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 9 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetCommitRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if version >= 1 && version <= 1 { + v := b.Int64() + s.Timestamp = v + } + if version >= 6 { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Metadata = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetCommitRequest returns a pointer to a default OffsetCommitRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetCommitRequest() *OffsetCommitRequest { + var v OffsetCommitRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitRequest. +func (v *OffsetCommitRequest) Default() { + v.Generation = -1 + v.RetentionTimeMillis = -1 +} + +// NewOffsetCommitRequest returns a default OffsetCommitRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitRequest() OffsetCommitRequest { + var v OffsetCommitRequest + v.Default() + return v +} + +type OffsetCommitResponseTopicPartition struct { + // Partition is the partition in a topic this array slot corresponds to. + Partition int32 + + // ErrorCode is the error for this partition response. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // for the group. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // for the topic / partition. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic / partition does + // not exist. + // + // OFFSET_METADATA_TOO_LARGE is returned if the request metadata is + // larger than the brokers offset.metadata.max.bytes. + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // (due to the requested broker shutting down or it has not completed startup). + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // ILLEGAL_GENERATION is returned if the request's generation ID is invalid. + // + // UNKNOWN_MEMBER_ID is returned if the group is dead or the group does not + // know of the request's member ID. + // + // REBALANCE_IN_PROGRESS is returned if the group is finishing a rebalance. + // + // INVALID_COMMIT_OFFSET_SIZE is returned if the offset commit results in + // a record batch that is too large (likely due to large metadata). + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitResponseTopicPartition. +func (v *OffsetCommitResponseTopicPartition) Default() { +} + +// NewOffsetCommitResponseTopicPartition returns a default OffsetCommitResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitResponseTopicPartition() OffsetCommitResponseTopicPartition { + var v OffsetCommitResponseTopicPartition + v.Default() + return v +} + +type OffsetCommitResponseTopic struct { + // Topic is the topic this offset commit response corresponds to. + Topic string // v0-v9 + + // TopicID is the topic ID, replacing Topic in v10+. + TopicID [16]byte // v10+ + + // Partitions contains responses for each requested partition in + // a topic. + Partitions []OffsetCommitResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitResponseTopic. +func (v *OffsetCommitResponseTopic) Default() { +} + +// NewOffsetCommitResponseTopic returns a default OffsetCommitResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitResponseTopic() OffsetCommitResponseTopic { + var v OffsetCommitResponseTopic + v.Default() + return v +} + +// OffsetCommitResponse is returned from an OffsetCommitRequest. +type OffsetCommitResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 4. + ThrottleMillis int32 // v3+ + + // Topics contains responses for each topic / partition in the commit request. + Topics []OffsetCommitResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v8+ +} + +func (*OffsetCommitResponse) Key() int16 { return 8 } +func (*OffsetCommitResponse) MaxVersion() int16 { return 10 } +func (v *OffsetCommitResponse) SetVersion(version int16) { v.Version = version } +func (v *OffsetCommitResponse) GetVersion() int16 { return v.Version } +func (v *OffsetCommitResponse) IsFlexible() bool { return v.Version >= 8 } +func (v *OffsetCommitResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 4 } +func (v *OffsetCommitResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *OffsetCommitResponse) RequestKind() Request { return &OffsetCommitRequest{Version: v.Version} } + +func (v *OffsetCommitResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 8 + _ = isFlexible + if version >= 3 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 9 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetCommitResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetCommitResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetCommitResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 8 + _ = isFlexible + s := v + if version >= 3 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetCommitResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 9 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetCommitResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetCommitResponse returns a pointer to a default OffsetCommitResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetCommitResponse() *OffsetCommitResponse { + var v OffsetCommitResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetCommitResponse. +func (v *OffsetCommitResponse) Default() { +} + +// NewOffsetCommitResponse returns a default OffsetCommitResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetCommitResponse() OffsetCommitResponse { + var v OffsetCommitResponse + v.Default() + return v +} + +type OffsetFetchRequestTopic struct { + // Topic is a topic to fetch offsets for. + Topic string + + // Partitions in a list of partitions in a group to fetch offsets for. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchRequestTopic. +func (v *OffsetFetchRequestTopic) Default() { +} + +// NewOffsetFetchRequestTopic returns a default OffsetFetchRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchRequestTopic() OffsetFetchRequestTopic { + var v OffsetFetchRequestTopic + v.Default() + return v +} + +type OffsetFetchRequestGroupTopic struct { + Topic string // v8-v9 + + // TopicID is the topic ID, replacing Topic in v10+. + TopicID [16]byte // v10+ + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchRequestGroupTopic. +func (v *OffsetFetchRequestGroupTopic) Default() { +} + +// NewOffsetFetchRequestGroupTopic returns a default OffsetFetchRequestGroupTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchRequestGroupTopic() OffsetFetchRequestGroupTopic { + var v OffsetFetchRequestGroupTopic + v.Default() + return v +} + +type OffsetFetchRequestGroup struct { + Group string + + // The member ID assigned by the group coordinator if using the new consumer protocol (KIP-848). + MemberID *string // v9+ + + // The member epoch if using the new consumer protocol (KIP-848). + // + // This field has a default of -1. + MemberEpoch int32 // v9+ + + Topics []OffsetFetchRequestGroupTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchRequestGroup. +func (v *OffsetFetchRequestGroup) Default() { + v.MemberEpoch = -1 +} + +// NewOffsetFetchRequestGroup returns a default OffsetFetchRequestGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchRequestGroup() OffsetFetchRequestGroup { + var v OffsetFetchRequestGroup + v.Default() + return v +} + +// OffsetFetchRequest requests the most recent committed offsets for topic +// partitions in a group. +// Version 10, introduced in Kafka 4.2, replaces topic names with topic IDs +// in the Groups field (KIP-848). +type OffsetFetchRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group to fetch offsets for. + Group string // v0-v7 + + // Topics contains topics to fetch offets for. Version 2+ allows this to be + // null to return all topics the client is authorized to describe in the group. + Topics []OffsetFetchRequestTopic // v0-v7 + + // Groups, introduced in v8 (Kafka 3.0), allows for fetching offsets for + // multiple groups at a time. + // + // The fields here mirror the old top level fields on the request, thus they + // are left undocumented. Refer to the top level documentation if necessary. + Groups []OffsetFetchRequestGroup // v8+ + + // RequireStable signifies whether the broker should wait on returning + // unstable offsets, instead setting a retryable error on the relevant + // unstable partitions (UNSTABLE_OFFSET_COMMIT). See KIP-447 for more + // details. + RequireStable bool // v7+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*OffsetFetchRequest) Key() int16 { return 9 } +func (*OffsetFetchRequest) MaxVersion() int16 { return 10 } +func (v *OffsetFetchRequest) SetVersion(version int16) { v.Version = version } +func (v *OffsetFetchRequest) GetVersion() int16 { return v.Version } +func (v *OffsetFetchRequest) IsFlexible() bool { return v.Version >= 6 } +func (v *OffsetFetchRequest) IsGroupCoordinatorRequest() {} +func (v *OffsetFetchRequest) ResponseKind() Response { + r := &OffsetFetchResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *OffsetFetchRequest) RequestWith(ctx context.Context, r Requestor) (*OffsetFetchResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*OffsetFetchResponse) + return resp, err +} + +func (v *OffsetFetchRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + if version >= 0 && version <= 7 { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 7 { + v := v.Topics + if version >= 2 { + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + } else { + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 8 { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 9 { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 9 { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + if version >= 8 && version <= 9 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 7 { + v := v.RequireStable + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetFetchRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetFetchRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetFetchRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + if version >= 0 && version <= 7 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + if version >= 0 && version <= 7 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 2 || l == 0 { + a = []OffsetFetchRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 8 { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchRequestGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + if version >= 9 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.MemberID = v + } + if version >= 9 { + v := b.Int32() + s.MemberEpoch = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []OffsetFetchRequestGroupTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchRequestGroupTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 8 && version <= 9 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if version >= 7 { + v := b.Bool() + s.RequireStable = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetFetchRequest returns a pointer to a default OffsetFetchRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetFetchRequest() *OffsetFetchRequest { + var v OffsetFetchRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchRequest. +func (v *OffsetFetchRequest) Default() { +} + +// NewOffsetFetchRequest returns a default OffsetFetchRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchRequest() OffsetFetchRequest { + var v OffsetFetchRequest + v.Default() + return v +} + +type OffsetFetchResponseTopicPartition struct { + // Partition is the partition in a topic this array slot corresponds to. + Partition int32 + + // Offset is the most recently committed offset for this topic partition + // in a group. + Offset int64 + + // LeaderEpoch is the leader epoch of the last consumed record. + // + // This was proposed in KIP-320 and introduced in Kafka 2.1.0 and allows + // clients to detect log truncation. See the KIP for more details. + // + // This field has a default of -1. + LeaderEpoch int32 // v5+ + + // Metadata is client provided metadata corresponding to the offset commit. + // This can be useful for adding who made the commit, etc. + Metadata *string + + // ErrorCode is the error for this partition response. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to the group. + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // (due to the requested broker shutting down or it has not completed startup). + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the requested topic or partition + // is unknown. + // + // UNSTABLE_OFFSET_COMMIT is returned for v7+ if the request set RequireStable. + // See KIP-447 for more details. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponseTopicPartition. +func (v *OffsetFetchResponseTopicPartition) Default() { + v.LeaderEpoch = -1 +} + +// NewOffsetFetchResponseTopicPartition returns a default OffsetFetchResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponseTopicPartition() OffsetFetchResponseTopicPartition { + var v OffsetFetchResponseTopicPartition + v.Default() + return v +} + +type OffsetFetchResponseTopic struct { + // Topic is the topic this offset fetch response corresponds to. + Topic string + + // Partitions contains responses for each requested partition in + // a topic. + Partitions []OffsetFetchResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponseTopic. +func (v *OffsetFetchResponseTopic) Default() { +} + +// NewOffsetFetchResponseTopic returns a default OffsetFetchResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponseTopic() OffsetFetchResponseTopic { + var v OffsetFetchResponseTopic + v.Default() + return v +} + +type OffsetFetchResponseGroupTopicPartition struct { + Partition int32 + + Offset int64 + + // This field has a default of -1. + LeaderEpoch int32 + + Metadata *string + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponseGroupTopicPartition. +func (v *OffsetFetchResponseGroupTopicPartition) Default() { + v.LeaderEpoch = -1 +} + +// NewOffsetFetchResponseGroupTopicPartition returns a default OffsetFetchResponseGroupTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponseGroupTopicPartition() OffsetFetchResponseGroupTopicPartition { + var v OffsetFetchResponseGroupTopicPartition + v.Default() + return v +} + +type OffsetFetchResponseGroupTopic struct { + Topic string // v8-v9 + + // TopicID is the topic ID, replacing Topic in v10+. + TopicID [16]byte // v10+ + + Partitions []OffsetFetchResponseGroupTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponseGroupTopic. +func (v *OffsetFetchResponseGroupTopic) Default() { +} + +// NewOffsetFetchResponseGroupTopic returns a default OffsetFetchResponseGroupTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponseGroupTopic() OffsetFetchResponseGroupTopic { + var v OffsetFetchResponseGroupTopic + v.Default() + return v +} + +type OffsetFetchResponseGroup struct { + Group string + + Topics []OffsetFetchResponseGroupTopic + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponseGroup. +func (v *OffsetFetchResponseGroup) Default() { +} + +// NewOffsetFetchResponseGroup returns a default OffsetFetchResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponseGroup() OffsetFetchResponseGroup { + var v OffsetFetchResponseGroup + v.Default() + return v +} + +// OffsetFetchResponse is returned from an OffsetFetchRequest. +type OffsetFetchResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 4. + ThrottleMillis int32 // v3+ + + // Topics contains responses for each requested topic/partition. + Topics []OffsetFetchResponseTopic // v0-v7 + + // ErrorCode is a top level error code that applies to all topic/partitions. + // This will be any group error. + ErrorCode int16 // v2-v7 + + // Groups is the response for all groups. Each field mirrors the fields in the + // top level request, thus they are left undocumented. Refer to the top level + // documentation if necessary. + Groups []OffsetFetchResponseGroup // v8+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*OffsetFetchResponse) Key() int16 { return 9 } +func (*OffsetFetchResponse) MaxVersion() int16 { return 10 } +func (v *OffsetFetchResponse) SetVersion(version int16) { v.Version = version } +func (v *OffsetFetchResponse) GetVersion() int16 { return v.Version } +func (v *OffsetFetchResponse) IsFlexible() bool { return v.Version >= 6 } +func (v *OffsetFetchResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 4 } +func (v *OffsetFetchResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *OffsetFetchResponse) RequestKind() Request { return &OffsetFetchRequest{Version: v.Version} } + +func (v *OffsetFetchResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + if version >= 3 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 7 { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if version >= 5 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 2 && version <= 7 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 8 { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 8 && version <= 9 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 10 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetFetchResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetFetchResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetFetchResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + if version >= 3 { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 0 && version <= 7 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if version >= 5 { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Metadata = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 2 && version <= 7 { + v := b.Int16() + s.ErrorCode = v + } + if version >= 8 { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchResponseGroupTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 8 && version <= 9 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 10 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetFetchResponseGroupTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Metadata = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetFetchResponse returns a pointer to a default OffsetFetchResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetFetchResponse() *OffsetFetchResponse { + var v OffsetFetchResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetFetchResponse. +func (v *OffsetFetchResponse) Default() { +} + +// NewOffsetFetchResponse returns a default OffsetFetchResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetFetchResponse() OffsetFetchResponse { + var v OffsetFetchResponse + v.Default() + return v +} + +// FindCoordinatorRequest requests the coordinator for a group or transaction. +// +// This coordinator is different from the broker leader coordinator. This +// coordinator is the partition leader for the partition that is storing +// the group or transaction ID. +// +// Kafka 3.9, introducing request version 6, adds support for share groups (see KIP-932). +// The format for key type SHARE(2) is "groupId:topicId:partition". +type FindCoordinatorRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // CoordinatorKey is the ID to use for finding the coordinator. For groups, + // this is the group name, for transactional producer, this is the + // transactional ID. + CoordinatorKey string // v0-v3 + + // CoordinatorType is the type that key is. Groups are type 0, + // transactional IDs are type 1, share groups are 2. + CoordinatorType int8 // v1+ + + // CoordinatorKeys contains all keys to find the coordinator for. + CoordinatorKeys []string // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*FindCoordinatorRequest) Key() int16 { return 10 } +func (*FindCoordinatorRequest) MaxVersion() int16 { return 6 } +func (v *FindCoordinatorRequest) SetVersion(version int16) { v.Version = version } +func (v *FindCoordinatorRequest) GetVersion() int16 { return v.Version } +func (v *FindCoordinatorRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *FindCoordinatorRequest) ResponseKind() Response { + r := &FindCoordinatorResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *FindCoordinatorRequest) RequestWith(ctx context.Context, r Requestor) (*FindCoordinatorResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*FindCoordinatorResponse) + return resp, err +} + +func (v *FindCoordinatorRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 0 && version <= 3 { + v := v.CoordinatorKey + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.CoordinatorType + dst = kbin.AppendInt8(dst, v) + } + if version >= 4 { + v := v.CoordinatorKeys + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FindCoordinatorRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FindCoordinatorRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FindCoordinatorRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 0 && version <= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.CoordinatorKey = v + } + if version >= 1 { + v := b.Int8() + s.CoordinatorType = v + } + if version >= 4 { + v := s.CoordinatorKeys + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.CoordinatorKeys = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrFindCoordinatorRequest returns a pointer to a default FindCoordinatorRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFindCoordinatorRequest() *FindCoordinatorRequest { + var v FindCoordinatorRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FindCoordinatorRequest. +func (v *FindCoordinatorRequest) Default() { +} + +// NewFindCoordinatorRequest returns a default FindCoordinatorRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewFindCoordinatorRequest() FindCoordinatorRequest { + var v FindCoordinatorRequest + v.Default() + return v +} + +type FindCoordinatorResponseCoordinator struct { + Key string + + NodeID int32 + + Host string + + Port int32 + + ErrorCode int16 + + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FindCoordinatorResponseCoordinator. +func (v *FindCoordinatorResponseCoordinator) Default() { +} + +// NewFindCoordinatorResponseCoordinator returns a default FindCoordinatorResponseCoordinator +// This is a shortcut for creating a struct and calling Default yourself. +func NewFindCoordinatorResponseCoordinator() FindCoordinatorResponseCoordinator { + var v FindCoordinatorResponseCoordinator + v.Default() + return v +} + +// FindCoordinatorResponse is returned from a FindCoordinatorRequest. +type FindCoordinatorResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // ErrorCode is the error returned for the request. + // + // GROUP_AUTHORIZATION_FAILED is returned if for a group ID request and the + // client is not authorized to describe groups. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned for a transactional ID + // request and the client is not authorized to describe transactional IDs. + // + // INVALID_REQUEST is returned if not asking for a known type (group, + // or transaction). + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // for the requested ID, which would be if the group or transactional topic + // does not exist or the partition the requested key maps to is not available. + ErrorCode int16 // v0-v3 + + // ErrorMessage is an informative message if the request errored. + ErrorMessage *string // v1-v3 + + // NodeID is the broker ID of the coordinator. + NodeID int32 // v0-v3 + + // Host is the host of the coordinator. + Host string // v0-v3 + + // Port is the port of the coordinator. + Port int32 // v0-v3 + + // Coordinators, introduced for KIP-699, is the bulk response for + // coordinators. The fields in the struct exactly match the original fields + // in the FindCoordinatorResponse, thus they are left undocumented. + Coordinators []FindCoordinatorResponseCoordinator // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*FindCoordinatorResponse) Key() int16 { return 10 } +func (*FindCoordinatorResponse) MaxVersion() int16 { return 6 } +func (v *FindCoordinatorResponse) SetVersion(version int16) { v.Version = version } +func (v *FindCoordinatorResponse) GetVersion() int16 { return v.Version } +func (v *FindCoordinatorResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *FindCoordinatorResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *FindCoordinatorResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *FindCoordinatorResponse) RequestKind() Request { + return &FindCoordinatorRequest{Version: v.Version} +} + +func (v *FindCoordinatorResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 3 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 1 && version <= 3 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 0 && version <= 3 { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 3 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 3 { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + if version >= 4 { + v := v.Coordinators + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FindCoordinatorResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FindCoordinatorResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FindCoordinatorResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 0 && version <= 3 { + v := b.Int16() + s.ErrorCode = v + } + if version >= 1 && version <= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 0 && version <= 3 { + v := b.Int32() + s.NodeID = v + } + if version >= 0 && version <= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 0 && version <= 3 { + v := b.Int32() + s.Port = v + } + if version >= 4 { + v := s.Coordinators + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FindCoordinatorResponseCoordinator, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Coordinators = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrFindCoordinatorResponse returns a pointer to a default FindCoordinatorResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFindCoordinatorResponse() *FindCoordinatorResponse { + var v FindCoordinatorResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FindCoordinatorResponse. +func (v *FindCoordinatorResponse) Default() { +} + +// NewFindCoordinatorResponse returns a default FindCoordinatorResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewFindCoordinatorResponse() FindCoordinatorResponse { + var v FindCoordinatorResponse + v.Default() + return v +} + +type JoinGroupRequestProtocol struct { + // Name is a name of a protocol. This is arbitrary, but is used + // in the official client to agree on a partition balancing strategy. + // + // The official client uses range, roundrobin, or sticky (which was + // introduced in KIP-54). + Name string + + // Metadata is arbitrary information to pass along with this + // protocol name for this member. + // + // Note that while this is not documented in any protocol page, + // this is usually a serialized GroupMemberMetadata as described in + // https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Client-side+Assignment+Proposal. + // + // The protocol metadata is where group members will communicate which + // topics they collectively as a group want to consume. + Metadata []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to JoinGroupRequestProtocol. +func (v *JoinGroupRequestProtocol) Default() { +} + +// NewJoinGroupRequestProtocol returns a default JoinGroupRequestProtocol +// This is a shortcut for creating a struct and calling Default yourself. +func NewJoinGroupRequestProtocol() JoinGroupRequestProtocol { + var v JoinGroupRequestProtocol + v.Default() + return v +} + +// JoinGroupRequest issues a request to join a Kafka group. This will create a +// group if one does not exist. If joining an existing group, this may trigger +// a group rebalance. +// +// This will trigger a group rebalance if the request is from the group leader, +// or if the request is from a group member with different metadata, or if the +// request is with a new group member. +// +// Version 4 introduced replying to joins of existing groups with +// MEMBER_ID_REQUIRED, which requires re-issuing the join group with the +// returned member ID. See KIP-394 for more details. +// +// Version 5 introduced InstanceID, allowing for more "static" membership. +// See KIP-345 for more details. +type JoinGroupRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group to join. + Group string + + // SessionTimeoutMillis is how long a member in the group can go between + // heartbeats. If a member does not send a heartbeat within this timeout, + // the broker will remove the member from the group and initiate a rebalance. + SessionTimeoutMillis int32 + + // RebalanceTimeoutMillis is how long the broker waits for members to join a group + // once a rebalance begins. Kafka waits for the longest rebalance of all + // members in the group. Member sessions are still alive; heartbeats will be + // replied to with REBALANCE_IN_PROGRESS. Those members must transition to + // joining within this rebalance timeout. Members that do not rejoin within + // this timeout will be removed from the group. Members must commit offsets + // within this timeout. + // + // The first join for a new group has a 3 second grace period for other + // members to join; this grace period is extended until the RebalanceTimeoutMillis + // is up or until 3 seconds lapse with no new members. + // + // This field has a default of -1. + RebalanceTimeoutMillis int32 // v1+ + + // MemberID is the member ID to join the group with. When joining a group for + // the first time, use the empty string. The response will contain the member + // ID that should be used going forward. + MemberID string + + // InstanceID is a user configured ID that is used for making a group + // member "static", allowing many rebalances to be avoided. + InstanceID *string // v5+ + + // ProtocolType is the "type" of protocol being used for the join group. + // The initial group creation sets the type; all additional members must + // have the same type or they will be rejected. + // + // This is completely arbitrary, but the Java client and everything else + // uses "consumer" as the protocol type. + ProtocolType string + + // Protocols contains arbitrary information that group members use + // for rebalancing. All group members must agree on at least one protocol + // name. + Protocols []JoinGroupRequestProtocol + + // Reason is an optional reason the member is joining (or rejoining) the + // group (KIP-800, Kafka 3.2+). + Reason *string // v8+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*JoinGroupRequest) Key() int16 { return 11 } +func (*JoinGroupRequest) MaxVersion() int16 { return 9 } +func (v *JoinGroupRequest) SetVersion(version int16) { v.Version = version } +func (v *JoinGroupRequest) GetVersion() int16 { return v.Version } +func (v *JoinGroupRequest) IsFlexible() bool { return v.Version >= 6 } +func (v *JoinGroupRequest) IsGroupCoordinatorRequest() {} +func (v *JoinGroupRequest) ResponseKind() Response { + r := &JoinGroupResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *JoinGroupRequest) RequestWith(ctx context.Context, r Requestor) (*JoinGroupResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*JoinGroupResponse) + return resp, err +} + +func (v *JoinGroupRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SessionTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.RebalanceTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 5 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Protocols + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 8 { + v := v.Reason + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *JoinGroupRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *JoinGroupRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *JoinGroupRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := b.Int32() + s.SessionTimeoutMillis = v + } + if version >= 1 { + v := b.Int32() + s.RebalanceTimeoutMillis = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ProtocolType = v + } + { + v := s.Protocols + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]JoinGroupRequestProtocol, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Metadata = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Protocols = v + } + if version >= 8 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Reason = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrJoinGroupRequest returns a pointer to a default JoinGroupRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrJoinGroupRequest() *JoinGroupRequest { + var v JoinGroupRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to JoinGroupRequest. +func (v *JoinGroupRequest) Default() { + v.RebalanceTimeoutMillis = -1 +} + +// NewJoinGroupRequest returns a default JoinGroupRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewJoinGroupRequest() JoinGroupRequest { + var v JoinGroupRequest + v.Default() + return v +} + +type JoinGroupResponseMember struct { + // MemberID is a member in this group. + MemberID string + + // InstanceID is an instance ID of a member in this group (KIP-345). + InstanceID *string // v5+ + + // ProtocolMetadata is the metadata for this member for this protocol. + // This is usually of type GroupMemberMetadata. + ProtocolMetadata []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to JoinGroupResponseMember. +func (v *JoinGroupResponseMember) Default() { +} + +// NewJoinGroupResponseMember returns a default JoinGroupResponseMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewJoinGroupResponseMember() JoinGroupResponseMember { + var v JoinGroupResponseMember + v.Default() + return v +} + +// JoinGroupResponse is returned from a JoinGroupRequest. +type JoinGroupResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 3. + ThrottleMillis int32 // v2+ + + // ErrorCode is the error for the join group request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to the group (no read perms). + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // (due to the requested broker shutting down or it has not completed startup). + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // INVALID_SESSION_TIMEOUT is returned if the requested SessionTimeout is + // not within the broker's group.{min,max}.session.timeout.ms. + // + // INCONSISTENT_GROUP_PROTOCOL is returned if the requested protocols are + // incompatible with the existing group member's protocols, or if the join + // was for a new group but contained no protocols. + // + // UNKNOWN_MEMBER_ID is returned is the requested group is dead (likely + // just migrated to another coordinator or the group is temporarily unstable), + // or if the request was for a new group but contained a non-empty member ID, + // or if the group does not have the requested member ID (and the client must + // do the new-join-group dance). + // + // MEMBER_ID_REQUIRED is returned on the initial join of an existing group. + // This error was proposed in KIP-394 and introduced in Kafka 2.2.0 to + // prevent flaky clients from continually triggering rebalances and prevent + // these clients from consuming RAM with metadata. If a client sees + // this error, it should re-issue the join with the MemberID in the response. + // Non-flaky clients will join with this new member ID, but flaky clients + // will not join quickly enough before the pending member ID is rotated out + // due to hitting the session.timeout.ms. + // + // GROUP_MAX_SIZE_REACHED is returned as of Kafka 2.2.0 if the group has + // reached a broker's group.max.size. + ErrorCode int16 + + // Generation is the current "generation" of this group. + // + // This field has a default of -1. + Generation int32 + + // ProtocolType is the "type" of protocol being used for this group. + ProtocolType *string // v7+ + + // Protocol is the agreed upon protocol name (i.e. "sticky", "range"). + // + // v7 of this response changed this field to be nullable. + Protocol *string + + // LeaderID is the leader member. + LeaderID string + + // True if the leader must skip running the assignment (KIP-814, Kafka 3.2+). + SkipAssignment bool // v9+ + + // MemberID is the member of the receiving client. + MemberID string + + // Members contains all other members of this group. Only the group leader + // receives the members. The leader is responsible for balancing subscribed + // topic partitions and replying appropriately in a SyncGroup request. + Members []JoinGroupResponseMember + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v6+ +} + +func (*JoinGroupResponse) Key() int16 { return 11 } +func (*JoinGroupResponse) MaxVersion() int16 { return 9 } +func (v *JoinGroupResponse) SetVersion(version int16) { v.Version = version } +func (v *JoinGroupResponse) GetVersion() int16 { return v.Version } +func (v *JoinGroupResponse) IsFlexible() bool { return v.Version >= 6 } +func (v *JoinGroupResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 3 } +func (v *JoinGroupResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *JoinGroupResponse) RequestKind() Request { return &JoinGroupRequest{Version: v.Version} } + +func (v *JoinGroupResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + if version >= 2 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + if version >= 7 { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Protocol + if version < 7 { + var vv string + if v != nil { + vv = *v + } + { + v := vv + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } else { + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + } + { + v := v.LeaderID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 9 { + v := v.SkipAssignment + dst = kbin.AppendBool(dst, v) + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 5 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ProtocolMetadata + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *JoinGroupResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *JoinGroupResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *JoinGroupResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 6 + _ = isFlexible + s := v + if version >= 2 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.Generation = v + } + if version >= 7 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ProtocolType = v + } + { + var v *string + if version < 7 { + var vv string + if isFlexible { + if unsafe { + vv = b.UnsafeCompactString() + } else { + vv = b.CompactString() + } + } else { + if unsafe { + vv = b.UnsafeString() + } else { + vv = b.String() + } + } + v = &vv + } else { + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + } + s.Protocol = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.LeaderID = v + } + if version >= 9 { + v := b.Bool() + s.SkipAssignment = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]JoinGroupResponseMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.ProtocolMetadata = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrJoinGroupResponse returns a pointer to a default JoinGroupResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrJoinGroupResponse() *JoinGroupResponse { + var v JoinGroupResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to JoinGroupResponse. +func (v *JoinGroupResponse) Default() { + v.Generation = -1 +} + +// NewJoinGroupResponse returns a default JoinGroupResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewJoinGroupResponse() JoinGroupResponse { + var v JoinGroupResponse + v.Default() + return v +} + +// HeartbeatRequest issues a heartbeat for a member in a group, ensuring that +// Kafka does not expire the member from the group. +type HeartbeatRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group ID this heartbeat is for. + Group string + + // Generation is the group generation this heartbeat is for. + Generation int32 + + // MemberID is the member ID this member is for. + MemberID string + + // InstanceID is the instance ID of this member in the group (KIP-345). + InstanceID *string // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*HeartbeatRequest) Key() int16 { return 12 } +func (*HeartbeatRequest) MaxVersion() int16 { return 4 } +func (v *HeartbeatRequest) SetVersion(version int16) { v.Version = version } +func (v *HeartbeatRequest) GetVersion() int16 { return v.Version } +func (v *HeartbeatRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *HeartbeatRequest) IsGroupCoordinatorRequest() {} +func (v *HeartbeatRequest) ResponseKind() Response { + r := &HeartbeatResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *HeartbeatRequest) RequestWith(ctx context.Context, r Requestor) (*HeartbeatResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*HeartbeatResponse) + return resp, err +} + +func (v *HeartbeatRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *HeartbeatRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *HeartbeatRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *HeartbeatRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := b.Int32() + s.Generation = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrHeartbeatRequest returns a pointer to a default HeartbeatRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrHeartbeatRequest() *HeartbeatRequest { + var v HeartbeatRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to HeartbeatRequest. +func (v *HeartbeatRequest) Default() { +} + +// NewHeartbeatRequest returns a default HeartbeatRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewHeartbeatRequest() HeartbeatRequest { + var v HeartbeatRequest + v.Default() + return v +} + +// HeartbeatResponse is returned from a HeartbeatRequest. +type HeartbeatResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // ErrorCode is the error for the heartbeat request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to the group (no read perms). + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // (due to the requested broker shutting down or it has not completed startup). + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // UNKNOWN_MEMBER_ID is returned if the member ID is not a part of the group, + // or if the group is empty or dead. + // + // ILLEGAL_GENERATION is returned if the request's generation ID is invalid. + // + // REBALANCE_IN_PROGRESS is returned if the group is currently rebalancing. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*HeartbeatResponse) Key() int16 { return 12 } +func (*HeartbeatResponse) MaxVersion() int16 { return 4 } +func (v *HeartbeatResponse) SetVersion(version int16) { v.Version = version } +func (v *HeartbeatResponse) GetVersion() int16 { return v.Version } +func (v *HeartbeatResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *HeartbeatResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *HeartbeatResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *HeartbeatResponse) RequestKind() Request { return &HeartbeatRequest{Version: v.Version} } + +func (v *HeartbeatResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *HeartbeatResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *HeartbeatResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *HeartbeatResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrHeartbeatResponse returns a pointer to a default HeartbeatResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrHeartbeatResponse() *HeartbeatResponse { + var v HeartbeatResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to HeartbeatResponse. +func (v *HeartbeatResponse) Default() { +} + +// NewHeartbeatResponse returns a default HeartbeatResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewHeartbeatResponse() HeartbeatResponse { + var v HeartbeatResponse + v.Default() + return v +} + +type LeaveGroupRequestMember struct { + MemberID string + + InstanceID *string + + // Reason is an optional reason why this member is leaving the group + // (KIP-800, Kafka 3.2+). + Reason *string // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaveGroupRequestMember. +func (v *LeaveGroupRequestMember) Default() { +} + +// NewLeaveGroupRequestMember returns a default LeaveGroupRequestMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaveGroupRequestMember() LeaveGroupRequestMember { + var v LeaveGroupRequestMember + v.Default() + return v +} + +// LeaveGroupRequest issues a request for a group member to leave the group, +// triggering a group rebalance. +// +// Version 3 changed removed MemberID and added a batch instance+member ID +// way of leaving a group. +type LeaveGroupRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group to leave. + Group string + + // MemberID is the member that is leaving. + MemberID string // v0-v2 + + // Members are member and group instance IDs to cause to leave a group. + Members []LeaveGroupRequestMember // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*LeaveGroupRequest) Key() int16 { return 13 } +func (*LeaveGroupRequest) MaxVersion() int16 { return 5 } +func (v *LeaveGroupRequest) SetVersion(version int16) { v.Version = version } +func (v *LeaveGroupRequest) GetVersion() int16 { return v.Version } +func (v *LeaveGroupRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *LeaveGroupRequest) IsGroupCoordinatorRequest() {} +func (v *LeaveGroupRequest) ResponseKind() Response { + r := &LeaveGroupResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *LeaveGroupRequest) RequestWith(ctx context.Context, r Requestor) (*LeaveGroupResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*LeaveGroupResponse) + return resp, err +} + +func (v *LeaveGroupRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 2 { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 5 { + v := v.Reason + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *LeaveGroupRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *LeaveGroupRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *LeaveGroupRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + if version >= 0 && version <= 2 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 3 { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaveGroupRequestMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Reason = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrLeaveGroupRequest returns a pointer to a default LeaveGroupRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrLeaveGroupRequest() *LeaveGroupRequest { + var v LeaveGroupRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaveGroupRequest. +func (v *LeaveGroupRequest) Default() { +} + +// NewLeaveGroupRequest returns a default LeaveGroupRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaveGroupRequest() LeaveGroupRequest { + var v LeaveGroupRequest + v.Default() + return v +} + +type LeaveGroupResponseMember struct { + MemberID string + + InstanceID *string + + // An individual member's leave error code. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaveGroupResponseMember. +func (v *LeaveGroupResponseMember) Default() { +} + +// NewLeaveGroupResponseMember returns a default LeaveGroupResponseMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaveGroupResponseMember() LeaveGroupResponseMember { + var v LeaveGroupResponseMember + v.Default() + return v +} + +// LeaveGroupResponse is returned from a LeaveGroupRequest. +type LeaveGroupResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // ErrorCode is the error for the leave group request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to the group (no read perms). + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available + // (due to the requested broker shutting down or it has not completed startup). + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // UNKNOWN_MEMBER_ID is returned if the member ID is not a part of the group, + // or if the group is empty or dead. + ErrorCode int16 + + // Members are the list of members and group instance IDs that left the group. + Members []LeaveGroupResponseMember // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*LeaveGroupResponse) Key() int16 { return 13 } +func (*LeaveGroupResponse) MaxVersion() int16 { return 5 } +func (v *LeaveGroupResponse) SetVersion(version int16) { v.Version = version } +func (v *LeaveGroupResponse) GetVersion() int16 { return v.Version } +func (v *LeaveGroupResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *LeaveGroupResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *LeaveGroupResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *LeaveGroupResponse) RequestKind() Request { return &LeaveGroupRequest{Version: v.Version} } + +func (v *LeaveGroupResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 3 { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *LeaveGroupResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *LeaveGroupResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *LeaveGroupResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 3 { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]LeaveGroupResponseMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrLeaveGroupResponse returns a pointer to a default LeaveGroupResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrLeaveGroupResponse() *LeaveGroupResponse { + var v LeaveGroupResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to LeaveGroupResponse. +func (v *LeaveGroupResponse) Default() { +} + +// NewLeaveGroupResponse returns a default LeaveGroupResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewLeaveGroupResponse() LeaveGroupResponse { + var v LeaveGroupResponse + v.Default() + return v +} + +type SyncGroupRequestGroupAssignment struct { + // MemberID is the member this assignment is for. + MemberID string + + // MemberAssignment is the assignment for this member. This is typically + // of type GroupMemberAssignment. + MemberAssignment []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SyncGroupRequestGroupAssignment. +func (v *SyncGroupRequestGroupAssignment) Default() { +} + +// NewSyncGroupRequestGroupAssignment returns a default SyncGroupRequestGroupAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewSyncGroupRequestGroupAssignment() SyncGroupRequestGroupAssignment { + var v SyncGroupRequestGroupAssignment + v.Default() + return v +} + +// SyncGroupRequest is issued by all group members after they receive a a +// response for JoinGroup. The group leader is responsible for sending member +// assignments with the request; all other members do not. +// +// Once the leader sends the group assignment, all members will be replied to. +type SyncGroupRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group ID this sync group is for. + Group string + + // Generation is the group generation this sync is for. + Generation int32 + + // MemberID is the member ID this member is. + MemberID string + + // InstanceID is the instance ID of this member in the group (KIP-345). + InstanceID *string // v3+ + + // ProtocolType is the "type" of protocol being used for this group. + ProtocolType *string // v5+ + + // Protocol is the agreed upon protocol name (i.e. "sticky", "range"). + Protocol *string // v5+ + + // GroupAssignment, sent only from the group leader, is the topic partition + // assignment it has decided on for all members. + GroupAssignment []SyncGroupRequestGroupAssignment + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*SyncGroupRequest) Key() int16 { return 14 } +func (*SyncGroupRequest) MaxVersion() int16 { return 5 } +func (v *SyncGroupRequest) SetVersion(version int16) { v.Version = version } +func (v *SyncGroupRequest) GetVersion() int16 { return v.Version } +func (v *SyncGroupRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *SyncGroupRequest) IsGroupCoordinatorRequest() {} +func (v *SyncGroupRequest) ResponseKind() Response { + r := &SyncGroupResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *SyncGroupRequest) RequestWith(ctx context.Context, r Requestor) (*SyncGroupResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*SyncGroupResponse) + return resp, err +} + +func (v *SyncGroupRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 5 { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 5 { + v := v.Protocol + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.GroupAssignment + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberAssignment + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *SyncGroupRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SyncGroupRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SyncGroupRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := b.Int32() + s.Generation = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ProtocolType = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Protocol = v + } + { + v := s.GroupAssignment + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]SyncGroupRequestGroupAssignment, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.MemberAssignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.GroupAssignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrSyncGroupRequest returns a pointer to a default SyncGroupRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSyncGroupRequest() *SyncGroupRequest { + var v SyncGroupRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SyncGroupRequest. +func (v *SyncGroupRequest) Default() { +} + +// NewSyncGroupRequest returns a default SyncGroupRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewSyncGroupRequest() SyncGroupRequest { + var v SyncGroupRequest + v.Default() + return v +} + +// SyncGroupResponse is returned from a SyncGroupRequest. +type SyncGroupResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // ErrorCode is the error for the sync group request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to the group (no read perms). + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // UNKNOWN_MEMBER_ID is returned if the member ID is not a part of the group, + // or if the group is empty or dead. + // + // ILLEGAL_GENERATION is returned if the request's generation ID is invalid. + // + // REBALANCE_IN_PROGRESS is returned if the group switched back to rebalancing. + // + // UNKNOWN_SERVER_ERROR is returned if the store of the group assignment + // resulted in a too large message. + ErrorCode int16 + + // ProtocolType is the "type" of protocol being used for this group. + ProtocolType *string // v5+ + + // Protocol is the agreed upon protocol name (i.e. "sticky", "range"). + Protocol *string // v5+ + + // MemberAssignment is the assignment for this member that the leader + // determined. + MemberAssignment []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*SyncGroupResponse) Key() int16 { return 14 } +func (*SyncGroupResponse) MaxVersion() int16 { return 5 } +func (v *SyncGroupResponse) SetVersion(version int16) { v.Version = version } +func (v *SyncGroupResponse) GetVersion() int16 { return v.Version } +func (v *SyncGroupResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *SyncGroupResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *SyncGroupResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *SyncGroupResponse) RequestKind() Request { return &SyncGroupRequest{Version: v.Version} } + +func (v *SyncGroupResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 5 { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 5 { + v := v.Protocol + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberAssignment + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *SyncGroupResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SyncGroupResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SyncGroupResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ProtocolType = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Protocol = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.MemberAssignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrSyncGroupResponse returns a pointer to a default SyncGroupResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSyncGroupResponse() *SyncGroupResponse { + var v SyncGroupResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SyncGroupResponse. +func (v *SyncGroupResponse) Default() { +} + +// NewSyncGroupResponse returns a default SyncGroupResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewSyncGroupResponse() SyncGroupResponse { + var v SyncGroupResponse + v.Default() + return v +} + +// DescribeGroupsRequest requests metadata for group IDs. +type DescribeGroupsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Groups is an array of group IDs to request metadata for. + Groups []string + + // IncludeAuthorizedOperations, introduced in Kafka 2.3.0, specifies + // whether to include a bitfield of AclOperations this client can perform + // on the groups. See KIP-430 for more details. + IncludeAuthorizedOperations bool // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +func (*DescribeGroupsRequest) Key() int16 { return 15 } +func (*DescribeGroupsRequest) MaxVersion() int16 { return 6 } +func (v *DescribeGroupsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeGroupsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeGroupsRequest) IsFlexible() bool { return v.Version >= 5 } +func (v *DescribeGroupsRequest) IsGroupCoordinatorRequest() {} +func (v *DescribeGroupsRequest) ResponseKind() Response { + r := &DescribeGroupsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeGroupsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeGroupsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeGroupsResponse) + return resp, err +} + +func (v *DescribeGroupsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if version >= 3 { + v := v.IncludeAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeGroupsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeGroupsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeGroupsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + s := v + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.Groups = v + } + if version >= 3 { + v := b.Bool() + s.IncludeAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeGroupsRequest returns a pointer to a default DescribeGroupsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeGroupsRequest() *DescribeGroupsRequest { + var v DescribeGroupsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeGroupsRequest. +func (v *DescribeGroupsRequest) Default() { +} + +// NewDescribeGroupsRequest returns a default DescribeGroupsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeGroupsRequest() DescribeGroupsRequest { + var v DescribeGroupsRequest + v.Default() + return v +} + +type DescribeGroupsResponseGroupMember struct { + // MemberID is the member ID of a member in this group. + MemberID string + + // InstanceID is the instance ID of this member in the group (KIP-345). + InstanceID *string // v4+ + + // ClientID is the client ID used by this member. + ClientID string + + // ClientHost is the host this client is running on. + ClientHost string + + // ProtocolMetadata is the metadata this member included when joining + // the group. If using normal (Java-like) consumers, this will be of + // type GroupMemberMetadata. + ProtocolMetadata []byte + + // MemberAssignment is the assignment for this member in the group. + // If using normal (Java-like) consumers, this will be of type + // GroupMemberAssignment. + MemberAssignment []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeGroupsResponseGroupMember. +func (v *DescribeGroupsResponseGroupMember) Default() { +} + +// NewDescribeGroupsResponseGroupMember returns a default DescribeGroupsResponseGroupMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeGroupsResponseGroupMember() DescribeGroupsResponseGroupMember { + var v DescribeGroupsResponseGroupMember + v.Default() + return v +} + +type DescribeGroupsResponseGroup struct { + // ErrorCode is the error code for an individual group in a request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to describe a group. + // + // INVALID_GROUP_ID is returned if the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator for this + // group is not yet active. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the + // coordinator for this group. + // + // GROUP_ID_NOT_FOUND is returned on v6+ if the group ID is not found (KIP-1043). + ErrorCode int16 + + // ErrorMessage is an optional message with more detail for the error code. + ErrorMessage *string // v6+ + + // Group is the id of this group. + Group string + + // State is the state this group is in. + State string + + // ProtocolType is the "type" of protocol being used for this group. + ProtocolType string + + // Protocol is the agreed upon protocol for all members in this group. + Protocol string + + // Members contains members in this group. + Members []DescribeGroupsResponseGroupMember + + // AuthorizedOperations is a bitfield containing which operations the + // the client is allowed to perform on this group. + // This is only returned if requested. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeGroupsResponseGroup. +func (v *DescribeGroupsResponseGroup) Default() { + v.AuthorizedOperations = -2147483648 +} + +// NewDescribeGroupsResponseGroup returns a default DescribeGroupsResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeGroupsResponseGroup() DescribeGroupsResponseGroup { + var v DescribeGroupsResponseGroup + v.Default() + return v +} + +// DescribeGroupsResponse is returned from a DescribeGroupsRequest. +type DescribeGroupsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // Groups is an array of group metadata. + Groups []DescribeGroupsResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +func (*DescribeGroupsResponse) Key() int16 { return 15 } +func (*DescribeGroupsResponse) MaxVersion() int16 { return 6 } +func (v *DescribeGroupsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeGroupsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeGroupsResponse) IsFlexible() bool { return v.Version >= 5 } +func (v *DescribeGroupsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *DescribeGroupsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DescribeGroupsResponse) RequestKind() Request { + return &DescribeGroupsRequest{Version: v.Version} +} + +func (v *DescribeGroupsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 6 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.State + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Protocol + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 4 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ClientID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ClientHost + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProtocolMetadata + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.MemberAssignment + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 3 { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeGroupsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeGroupsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeGroupsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeGroupsResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 6 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.State = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ProtocolType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Protocol = v + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeGroupsResponseGroupMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 4 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientHost = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.ProtocolMetadata = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.MemberAssignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + if version >= 3 { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeGroupsResponse returns a pointer to a default DescribeGroupsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeGroupsResponse() *DescribeGroupsResponse { + var v DescribeGroupsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeGroupsResponse. +func (v *DescribeGroupsResponse) Default() { +} + +// NewDescribeGroupsResponse returns a default DescribeGroupsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeGroupsResponse() DescribeGroupsResponse { + var v DescribeGroupsResponse + v.Default() + return v +} + +// ListGroupsRequest issues a request to list all groups. +// +// To list all groups in a cluster, this must be issued to every broker. +type ListGroupsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // StatesFilter, proposed in KIP-518 and introduced in Kafka 2.6.0, + // allows filtering groups by state, where a state is any of + // "Preparing", "PreparingRebalance", "CompletingRebalance", "Stable", + // "Dead", "Empty", "Assigning", "Reconciling", or "NotReady". + // If empty, all groups are returned. + StatesFilter []string // v4+ + + // TypesFilter, part of KIP-848, filters the types of groups we want + // to list. If empty, all groups are returned. + TypesFilter []string // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ListGroupsRequest) Key() int16 { return 16 } +func (*ListGroupsRequest) MaxVersion() int16 { return 5 } +func (v *ListGroupsRequest) SetVersion(version int16) { v.Version = version } +func (v *ListGroupsRequest) GetVersion() int16 { return v.Version } +func (v *ListGroupsRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *ListGroupsRequest) ResponseKind() Response { + r := &ListGroupsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ListGroupsRequest) RequestWith(ctx context.Context, r Requestor) (*ListGroupsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ListGroupsResponse) + return resp, err +} + +func (v *ListGroupsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 4 { + v := v.StatesFilter + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if version >= 5 { + v := v.TypesFilter + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListGroupsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListGroupsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListGroupsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 4 { + v := s.StatesFilter + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.StatesFilter = v + } + if version >= 5 { + v := s.TypesFilter + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.TypesFilter = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListGroupsRequest returns a pointer to a default ListGroupsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListGroupsRequest() *ListGroupsRequest { + var v ListGroupsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListGroupsRequest. +func (v *ListGroupsRequest) Default() { +} + +// NewListGroupsRequest returns a default ListGroupsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewListGroupsRequest() ListGroupsRequest { + var v ListGroupsRequest + v.Default() + return v +} + +type ListGroupsResponseGroup struct { + // Group is a Kafka group. + Group string + + // ProtocolType is the protocol type in use by the group. + ProtocolType string + + // The group state. + GroupState string // v4+ + + // The group type. + GroupType string // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListGroupsResponseGroup. +func (v *ListGroupsResponseGroup) Default() { +} + +// NewListGroupsResponseGroup returns a default ListGroupsResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewListGroupsResponseGroup() ListGroupsResponseGroup { + var v ListGroupsResponseGroup + v.Default() + return v +} + +// ListGroupsResponse is returned from a ListGroupsRequest. +type ListGroupsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // ErrorCode is the error returned for the list groups request. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not yet active. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group manager is loading. + ErrorCode int16 + + // Groups is the list of groups Kafka knows of. + Groups []ListGroupsResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ListGroupsResponse) Key() int16 { return 16 } +func (*ListGroupsResponse) MaxVersion() int16 { return 5 } +func (v *ListGroupsResponse) SetVersion(version int16) { v.Version = version } +func (v *ListGroupsResponse) GetVersion() int16 { return v.Version } +func (v *ListGroupsResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *ListGroupsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *ListGroupsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ListGroupsResponse) RequestKind() Request { return &ListGroupsRequest{Version: v.Version} } + +func (v *ListGroupsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProtocolType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 4 { + v := v.GroupState + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 5 { + v := v.GroupType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListGroupsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListGroupsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListGroupsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListGroupsResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ProtocolType = v + } + if version >= 4 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupState = v + } + if version >= 5 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListGroupsResponse returns a pointer to a default ListGroupsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListGroupsResponse() *ListGroupsResponse { + var v ListGroupsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListGroupsResponse. +func (v *ListGroupsResponse) Default() { +} + +// NewListGroupsResponse returns a default ListGroupsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewListGroupsResponse() ListGroupsResponse { + var v ListGroupsResponse + v.Default() + return v +} + +// SASLHandshakeRequest begins the sasl authentication flow. Note that Kerberos +// GSSAPI authentication has its own unique flow. +type SASLHandshakeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Mechanism is the mechanism to use for the sasl handshake (e.g., "PLAIN"). + // + // For version 0, if this mechanism is supported, it is expected that the + // client immediately authenticates using this mechanism. Note that the + // only mechanism exclusive to v0 is PLAIN. + // + // For version 1, if the mechanism is supported, the next request to issue + // is SASLHandshakeRequest. + Mechanism string +} + +func (*SASLHandshakeRequest) Key() int16 { return 17 } +func (*SASLHandshakeRequest) MaxVersion() int16 { return 1 } +func (v *SASLHandshakeRequest) SetVersion(version int16) { v.Version = version } +func (v *SASLHandshakeRequest) GetVersion() int16 { return v.Version } +func (v *SASLHandshakeRequest) IsFlexible() bool { return false } +func (v *SASLHandshakeRequest) ResponseKind() Response { + r := &SASLHandshakeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *SASLHandshakeRequest) RequestWith(ctx context.Context, r Requestor) (*SASLHandshakeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*SASLHandshakeResponse) + return resp, err +} + +func (v *SASLHandshakeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Mechanism + dst = kbin.AppendString(dst, v) + } + return dst +} + +func (v *SASLHandshakeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SASLHandshakeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SASLHandshakeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Mechanism = v + } + return b.Complete() +} + +// NewPtrSASLHandshakeRequest returns a pointer to a default SASLHandshakeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSASLHandshakeRequest() *SASLHandshakeRequest { + var v SASLHandshakeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SASLHandshakeRequest. +func (v *SASLHandshakeRequest) Default() { +} + +// NewSASLHandshakeRequest returns a default SASLHandshakeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewSASLHandshakeRequest() SASLHandshakeRequest { + var v SASLHandshakeRequest + v.Default() + return v +} + +// SASLHandshakeResponse is returned for a SASLHandshakeRequest. +type SASLHandshakeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is non-zero for ILLEGAL_SASL_STATE, meaning a sasl handshake + // is not expected at this point in the connection, or UNSUPPORTED_SASL_MECHANISM, + // meaning the requested mechanism is not supported. + ErrorCode int16 + + // SupportedMechanisms is the list of mechanisms supported if this request + // errored. + SupportedMechanisms []string +} + +func (*SASLHandshakeResponse) Key() int16 { return 17 } +func (*SASLHandshakeResponse) MaxVersion() int16 { return 1 } +func (v *SASLHandshakeResponse) SetVersion(version int16) { v.Version = version } +func (v *SASLHandshakeResponse) GetVersion() int16 { return v.Version } +func (v *SASLHandshakeResponse) IsFlexible() bool { return false } +func (v *SASLHandshakeResponse) RequestKind() Request { + return &SASLHandshakeRequest{Version: v.Version} +} + +func (v *SASLHandshakeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.SupportedMechanisms + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := v[i] + dst = kbin.AppendString(dst, v) + } + } + return dst +} + +func (v *SASLHandshakeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SASLHandshakeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SASLHandshakeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.SupportedMechanisms + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + a[i] = v + } + v = a + s.SupportedMechanisms = v + } + return b.Complete() +} + +// NewPtrSASLHandshakeResponse returns a pointer to a default SASLHandshakeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSASLHandshakeResponse() *SASLHandshakeResponse { + var v SASLHandshakeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SASLHandshakeResponse. +func (v *SASLHandshakeResponse) Default() { +} + +// NewSASLHandshakeResponse returns a default SASLHandshakeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewSASLHandshakeResponse() SASLHandshakeResponse { + var v SASLHandshakeResponse + v.Default() + return v +} + +// ApiVersionsRequest requests what API versions a Kafka broker supports. +// +// Note that the client does not know the version a broker supports before +// sending this request. +// +// Before Kafka 2.4.0, if the client used a version larger than the broker +// understands, the broker would reply with an UNSUPPORTED_VERSION error using +// the version 0 message format (i.e., 6 bytes long!). The client should retry +// with a lower version. +// +// After Kafka 2.4.0, if the client uses a version larger than the broker +// understands, the broker replies with UNSUPPORTED_VERSIONS using the version +// 0 message format but additionally includes the api versions the broker does +// support. +type ApiVersionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ClientSoftwareName, added for KIP-511 with Kafka 2.4.0, is the name of the + // client issuing this request. The broker can use this to enrich its own + // debugging information of which version of what clients are connected. + // + // If using v3, this field is required and must match the following pattern: + // + // [a-zA-Z0-9](?:[a-zA-Z0-9\\-.]*[a-zA-Z0-9])? + ClientSoftwareName string // v3+ + + // ClientSoftwareVersion is the version of the software name in the prior + // field. It must match the same regex (thus, this is also required). + ClientSoftwareVersion string // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ApiVersionsRequest) Key() int16 { return 18 } +func (*ApiVersionsRequest) MaxVersion() int16 { return 4 } +func (v *ApiVersionsRequest) SetVersion(version int16) { v.Version = version } +func (v *ApiVersionsRequest) GetVersion() int16 { return v.Version } +func (v *ApiVersionsRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *ApiVersionsRequest) ResponseKind() Response { + r := &ApiVersionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ApiVersionsRequest) RequestWith(ctx context.Context, r Requestor) (*ApiVersionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ApiVersionsResponse) + return resp, err +} + +func (v *ApiVersionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 3 { + v := v.ClientSoftwareName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.ClientSoftwareVersion + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ApiVersionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ApiVersionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ApiVersionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientSoftwareName = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientSoftwareVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrApiVersionsRequest returns a pointer to a default ApiVersionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrApiVersionsRequest() *ApiVersionsRequest { + var v ApiVersionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ApiVersionsRequest. +func (v *ApiVersionsRequest) Default() { +} + +// NewApiVersionsRequest returns a default ApiVersionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewApiVersionsRequest() ApiVersionsRequest { + var v ApiVersionsRequest + v.Default() + return v +} + +type ApiVersionsResponseApiKey struct { + // ApiKey is the key of a message request. + ApiKey int16 + + // MinVersion is the min version a broker supports for an API key. + MinVersion int16 + + // MaxVersion is the max version a broker supports for an API key. + MaxVersion int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ApiVersionsResponseApiKey. +func (v *ApiVersionsResponseApiKey) Default() { +} + +// NewApiVersionsResponseApiKey returns a default ApiVersionsResponseApiKey +// This is a shortcut for creating a struct and calling Default yourself. +func NewApiVersionsResponseApiKey() ApiVersionsResponseApiKey { + var v ApiVersionsResponseApiKey + v.Default() + return v +} + +type ApiVersionsResponseSupportedFeature struct { + // The name of the feature. + Name string + + // The minimum supported version for the feature. + MinVersion int16 + + // The maximum supported version for the feature. + MaxVersion int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ApiVersionsResponseSupportedFeature. +func (v *ApiVersionsResponseSupportedFeature) Default() { +} + +// NewApiVersionsResponseSupportedFeature returns a default ApiVersionsResponseSupportedFeature +// This is a shortcut for creating a struct and calling Default yourself. +func NewApiVersionsResponseSupportedFeature() ApiVersionsResponseSupportedFeature { + var v ApiVersionsResponseSupportedFeature + v.Default() + return v +} + +type ApiVersionsResponseFinalizedFeature struct { + // The name of the feature. + Name string + + // The cluster-wide finalized max version level for the feature. + MaxVersionLevel int16 + + // The cluster-wide finalized min version level for the feature. + MinVersionLevel int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ApiVersionsResponseFinalizedFeature. +func (v *ApiVersionsResponseFinalizedFeature) Default() { +} + +// NewApiVersionsResponseFinalizedFeature returns a default ApiVersionsResponseFinalizedFeature +// This is a shortcut for creating a struct and calling Default yourself. +func NewApiVersionsResponseFinalizedFeature() ApiVersionsResponseFinalizedFeature { + var v ApiVersionsResponseFinalizedFeature + v.Default() + return v +} + +// ApiVersionsResponse is returned from an ApiVersionsRequest. +type ApiVersionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is UNSUPPORTED_VERSION if the request was issued with a higher + // version than the broker supports. Before Kafka 2.4.0, if this error is + // returned, the rest of this struct will be empty. + // + // Starting in Kafka 2.4.0 (with version 3), even with an UNSUPPORTED_VERSION + // error, the broker still replies with the ApiKeys it supports. + ErrorCode int16 + + // ApiKeys is an array corresponding to API keys the broker supports + // and the range of supported versions for each key. + ApiKeys []ApiVersionsResponseApiKey + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // Features supported by the broker (see KIP-584). + SupportedFeatures []ApiVersionsResponseSupportedFeature // tag 0 + + // The monotonically increasing epoch for the finalized features information, + // where -1 indicates an unknown epoch. + // + // This field has a default of -1. + FinalizedFeaturesEpoch int64 // tag 1 + + // The list of cluster-wide finalized features (only valid if + // FinalizedFeaturesEpoch is >= 0). + FinalizedFeatures []ApiVersionsResponseFinalizedFeature // tag 2 + + // Set by a KRaft controller if the required configurations for ZK migration + // are present + ZkMigrationReady bool // tag 3 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*ApiVersionsResponse) Key() int16 { return 18 } +func (*ApiVersionsResponse) MaxVersion() int16 { return 4 } +func (v *ApiVersionsResponse) SetVersion(version int16) { v.Version = version } +func (v *ApiVersionsResponse) GetVersion() int16 { return v.Version } +func (v *ApiVersionsResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *ApiVersionsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *ApiVersionsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ApiVersionsResponse) RequestKind() Request { return &ApiVersionsRequest{Version: v.Version} } + +func (v *ApiVersionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ApiKeys + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ApiKey + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MinVersion + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MaxVersion + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + var toEncode []uint32 + if len(v.SupportedFeatures) > 0 { + toEncode = append(toEncode, 0) + } + if v.FinalizedFeaturesEpoch != -1 { + toEncode = append(toEncode, 1) + } + if len(v.FinalizedFeatures) > 0 { + toEncode = append(toEncode, 2) + } + if v.ZkMigrationReady != false { + toEncode = append(toEncode, 3) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.SupportedFeatures + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fSupportedFeatures: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MinVersion + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MaxVersion + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fSupportedFeatures + } + } + case 1: + { + v := v.FinalizedFeaturesEpoch + dst = kbin.AppendUvarint(dst, 1) + dst = kbin.AppendUvarint(dst, 8) + dst = kbin.AppendInt64(dst, v) + } + case 2: + { + v := v.FinalizedFeatures + dst = kbin.AppendUvarint(dst, 2) + sized := false + lenAt := len(dst) + fFinalizedFeatures: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MaxVersionLevel + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MinVersionLevel + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fFinalizedFeatures + } + } + case 3: + { + v := v.ZkMigrationReady + dst = kbin.AppendUvarint(dst, 3) + dst = kbin.AppendUvarint(dst, 1) + dst = kbin.AppendBool(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ApiVersionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ApiVersionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ApiVersionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.ApiKeys + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ApiVersionsResponseApiKey, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ApiKey = v + } + { + v := b.Int16() + s.MinVersion = v + } + { + v := b.Int16() + s.MaxVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ApiKeys = v + } + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.SupportedFeatures + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ApiVersionsResponseSupportedFeature, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int16() + s.MinVersion = v + } + { + v := b.Int16() + s.MaxVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.SupportedFeatures = v + if err := b.Complete(); err != nil { + return err + } + case 1: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int64() + s.FinalizedFeaturesEpoch = v + if err := b.Complete(); err != nil { + return err + } + case 2: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.FinalizedFeatures + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ApiVersionsResponseFinalizedFeature, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int16() + s.MaxVersionLevel = v + } + { + v := b.Int16() + s.MinVersionLevel = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.FinalizedFeatures = v + if err := b.Complete(); err != nil { + return err + } + case 3: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Bool() + s.ZkMigrationReady = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrApiVersionsResponse returns a pointer to a default ApiVersionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrApiVersionsResponse() *ApiVersionsResponse { + var v ApiVersionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ApiVersionsResponse. +func (v *ApiVersionsResponse) Default() { + v.FinalizedFeaturesEpoch = -1 +} + +// NewApiVersionsResponse returns a default ApiVersionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewApiVersionsResponse() ApiVersionsResponse { + var v ApiVersionsResponse + v.Default() + return v +} + +type CreateTopicsRequestTopicReplicaAssignment struct { + // Partition is a partition to create. + Partition int32 + + // Replicas are broker IDs the partition must exist on. + Replicas []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsRequestTopicReplicaAssignment. +func (v *CreateTopicsRequestTopicReplicaAssignment) Default() { +} + +// NewCreateTopicsRequestTopicReplicaAssignment returns a default CreateTopicsRequestTopicReplicaAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsRequestTopicReplicaAssignment() CreateTopicsRequestTopicReplicaAssignment { + var v CreateTopicsRequestTopicReplicaAssignment + v.Default() + return v +} + +type CreateTopicsRequestTopicConfig struct { + // Name is a topic level config key (e.g. segment.bytes). + Name string + + // Value is a topic level config value (e.g. 1073741824) + Value *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsRequestTopicConfig. +func (v *CreateTopicsRequestTopicConfig) Default() { +} + +// NewCreateTopicsRequestTopicConfig returns a default CreateTopicsRequestTopicConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsRequestTopicConfig() CreateTopicsRequestTopicConfig { + var v CreateTopicsRequestTopicConfig + v.Default() + return v +} + +type CreateTopicsRequestTopic struct { + // Topic is a topic to create. + Topic string + + // NumPartitions is how many partitions to give a topic. This must + // be -1 if specifying partitions manually (see ReplicaAssignment) + // or, starting v4+, to use the broker default partitions. + NumPartitions int32 + + // ReplicationFactor is how many replicas every partition must have. + // This must be -1 if specifying partitions manually (see ReplicaAssignment) + // or, starting v4+, to use the broker default replication factor. + ReplicationFactor int16 + + // ReplicaAssignment is an array to manually dicate replicas and their + // partitions for a topic. If using this, both ReplicationFactor and + // NumPartitions must be -1. + ReplicaAssignment []CreateTopicsRequestTopicReplicaAssignment + + // Configs is an array of key value config pairs for a topic. + // These correspond to Kafka Topic-Level Configs: http://kafka.apache.org/documentation/#topicconfigs. + Configs []CreateTopicsRequestTopicConfig + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsRequestTopic. +func (v *CreateTopicsRequestTopic) Default() { +} + +// NewCreateTopicsRequestTopic returns a default CreateTopicsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsRequestTopic() CreateTopicsRequestTopic { + var v CreateTopicsRequestTopic + v.Default() + return v +} + +// CreateTopicsRequest creates Kafka topics. +// +// Version 4, introduced in Kafka 2.4.0, implies client support for +// creation defaults. See KIP-464. +// +// Version 5, also in 2.4.0, returns topic configs in the response (KIP-525). +type CreateTopicsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics is an array of topics to attempt to create. + Topics []CreateTopicsRequestTopic + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 60000. + TimeoutMillis int32 + + // ValidateOnly is makes this request a dry-run; everything is validated but + // no topics are actually created. + ValidateOnly bool // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +func (*CreateTopicsRequest) Key() int16 { return 19 } +func (*CreateTopicsRequest) MaxVersion() int16 { return 7 } +func (v *CreateTopicsRequest) SetVersion(version int16) { v.Version = version } +func (v *CreateTopicsRequest) GetVersion() int16 { return v.Version } +func (v *CreateTopicsRequest) IsFlexible() bool { return v.Version >= 5 } +func (v *CreateTopicsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *CreateTopicsRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *CreateTopicsRequest) IsAdminRequest() {} +func (v *CreateTopicsRequest) ResponseKind() Response { + r := &CreateTopicsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *CreateTopicsRequest) RequestWith(ctx context.Context, r Requestor) (*CreateTopicsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*CreateTopicsResponse) + return resp, err +} + +func (v *CreateTopicsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ReplicaAssignment + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateTopicsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateTopicsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateTopicsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateTopicsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.NumPartitions = v + } + { + v := b.Int16() + s.ReplicationFactor = v + } + { + v := s.ReplicaAssignment + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateTopicsRequestTopicReplicaAssignment, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ReplicaAssignment = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateTopicsRequestTopicConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + if version >= 1 { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateTopicsRequest returns a pointer to a default CreateTopicsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateTopicsRequest() *CreateTopicsRequest { + var v CreateTopicsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsRequest. +func (v *CreateTopicsRequest) Default() { + v.TimeoutMillis = 60000 +} + +// NewCreateTopicsRequest returns a default CreateTopicsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsRequest() CreateTopicsRequest { + var v CreateTopicsRequest + v.Default() + return v +} + +type CreateTopicsResponseTopicConfig struct { + // Name is the configuration name (e.g. segment.bytes). + Name string + + // Value is the value for this config key. If the key is sensitive, + // the value will be null. + Value *string + + // ReadOnly signifies whether this is not a dynamic config option. + ReadOnly bool + + // Source is where this config entry is from. See the documentation + // on DescribeConfigsRequest's Source for more details. + // + // This field has a default of -1. + Source int8 + + // IsSensitive signifies whether this is a sensitive config key, which + // is either a password or an unknown type. + IsSensitive bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsResponseTopicConfig. +func (v *CreateTopicsResponseTopicConfig) Default() { + v.Source = -1 +} + +// NewCreateTopicsResponseTopicConfig returns a default CreateTopicsResponseTopicConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsResponseTopicConfig() CreateTopicsResponseTopicConfig { + var v CreateTopicsResponseTopicConfig + v.Default() + return v +} + +type CreateTopicsResponseTopic struct { + // Topic is the topic this response corresponds to. + Topic string + + // The unique topic ID. + TopicID [16]byte // v7+ + + // ErrorCode is the error code for an individual topic creation. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized. + // + // INVALID_REQUEST is returned if the same topic occurred multiple times + // in the request. + // + // POLICY_VIOLATION is returned if the broker is using a + // create.topic.policy.class.name that returns a policy violation. + // + // INVALID_TOPIC_EXCEPTION if the topic collides with another topic when + // both topic's names' periods are replaced with underscores (e.g. + // topic.foo and topic_foo collide). + // + // TOPIC_ALREADY_EXISTS is returned if the topic already exists. + // + // INVALID_PARTITIONS is returned if the requested number of partitions is + // <= 0. + // + // INVALID_REPLICATION_FACTOR is returned if the requested replication + // factor is <= 0. + // + // INVALID_REPLICA_ASSIGNMENT is returned if not all partitions have the same + // number of replicas, or duplica replicas are assigned, or the partitions + // are not consecutive starting from 0. + // + // INVALID_CONFIG is returned if the requested topic config is invalid. + // to create a topic. + ErrorCode int16 + + // ErrorMessage is an informative message if the topic creation failed. + ErrorMessage *string // v1+ + + // ConfigErrorCode is non-zero if configs are unable to be returned. + // + // This is the first tagged field, introduced in version 5. As such, it is + // only possible to be present in v5+. + ConfigErrorCode int16 // tag 0 + + // NumPartitions is how many partitions were created for this topic. + // + // This field has a default of -1. + NumPartitions int32 // v5+ + + // ReplicationFactor is how many replicas every partition has for this topic. + // + // This field has a default of -1. + ReplicationFactor int16 // v5+ + + // Configs contains this topic's configuration. + Configs []CreateTopicsResponseTopicConfig // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsResponseTopic. +func (v *CreateTopicsResponseTopic) Default() { + v.NumPartitions = -1 + v.ReplicationFactor = -1 +} + +// NewCreateTopicsResponseTopic returns a default CreateTopicsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsResponseTopic() CreateTopicsResponseTopic { + var v CreateTopicsResponseTopic + v.Default() + return v +} + +// CreateTopicsResponse is returned from a CreateTopicsRequest. +type CreateTopicsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 3. + ThrottleMillis int32 // v2+ + + // Topics contains responses to the requested topic creations. + Topics []CreateTopicsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v5+ +} + +func (*CreateTopicsResponse) Key() int16 { return 19 } +func (*CreateTopicsResponse) MaxVersion() int16 { return 7 } +func (v *CreateTopicsResponse) SetVersion(version int16) { v.Version = version } +func (v *CreateTopicsResponse) GetVersion() int16 { return v.Version } +func (v *CreateTopicsResponse) IsFlexible() bool { return v.Version >= 5 } +func (v *CreateTopicsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 3 } +func (v *CreateTopicsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *CreateTopicsResponse) RequestKind() Request { return &CreateTopicsRequest{Version: v.Version} } + +func (v *CreateTopicsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + if version >= 2 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 7 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 1 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 5 { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + if version >= 5 { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + if version >= 5 { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ReadOnly + dst = kbin.AppendBool(dst, v) + } + { + v := v.Source + dst = kbin.AppendInt8(dst, v) + } + { + v := v.IsSensitive + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if v.ConfigErrorCode != 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.ConfigErrorCode + dst = kbin.AppendUvarint(dst, 0) + dst = kbin.AppendUvarint(dst, 2) + dst = kbin.AppendInt16(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateTopicsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateTopicsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateTopicsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 5 + _ = isFlexible + s := v + if version >= 2 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateTopicsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 7 { + v := b.Uuid() + s.TopicID = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 1 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 5 { + v := b.Int32() + s.NumPartitions = v + } + if version >= 5 { + v := b.Int16() + s.ReplicationFactor = v + } + if version >= 5 { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []CreateTopicsResponseTopicConfig{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateTopicsResponseTopicConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + { + v := b.Bool() + s.ReadOnly = v + } + { + v := b.Int8() + s.Source = v + } + { + v := b.Bool() + s.IsSensitive = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Int16() + s.ConfigErrorCode = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateTopicsResponse returns a pointer to a default CreateTopicsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateTopicsResponse() *CreateTopicsResponse { + var v CreateTopicsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateTopicsResponse. +func (v *CreateTopicsResponse) Default() { +} + +// NewCreateTopicsResponse returns a default CreateTopicsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateTopicsResponse() CreateTopicsResponse { + var v CreateTopicsResponse + v.Default() + return v +} + +type DeleteTopicsRequestTopic struct { + Topic *string + + TopicID [16]byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteTopicsRequestTopic. +func (v *DeleteTopicsRequestTopic) Default() { +} + +// NewDeleteTopicsRequestTopic returns a default DeleteTopicsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteTopicsRequestTopic() DeleteTopicsRequestTopic { + var v DeleteTopicsRequestTopic + v.Default() + return v +} + +// DeleteTopicsRequest deletes Kafka topics. +type DeleteTopicsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics is an array of topics to delete. + TopicNames []string // v0-v5 + + // The name or topic ID of topics to delete. + Topics []DeleteTopicsRequestTopic // v6+ + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 15000. + TimeoutMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*DeleteTopicsRequest) Key() int16 { return 20 } +func (*DeleteTopicsRequest) MaxVersion() int16 { return 6 } +func (v *DeleteTopicsRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteTopicsRequest) GetVersion() int16 { return v.Version } +func (v *DeleteTopicsRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *DeleteTopicsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *DeleteTopicsRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *DeleteTopicsRequest) IsAdminRequest() {} +func (v *DeleteTopicsRequest) ResponseKind() Response { + r := &DeleteTopicsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteTopicsRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteTopicsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteTopicsResponse) + return resp, err +} + +func (v *DeleteTopicsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 0 && version <= 5 { + v := v.TopicNames + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if version >= 6 { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteTopicsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteTopicsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteTopicsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 0 && version <= 5 { + v := s.TopicNames + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.TopicNames = v + } + if version >= 6 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteTopicsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Topic = v + } + { + v := b.Uuid() + s.TopicID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteTopicsRequest returns a pointer to a default DeleteTopicsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteTopicsRequest() *DeleteTopicsRequest { + var v DeleteTopicsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteTopicsRequest. +func (v *DeleteTopicsRequest) Default() { + v.TimeoutMillis = 15000 +} + +// NewDeleteTopicsRequest returns a default DeleteTopicsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteTopicsRequest() DeleteTopicsRequest { + var v DeleteTopicsRequest + v.Default() + return v +} + +type DeleteTopicsResponseTopic struct { + // Topic is the topic requested for deletion. + Topic *string + + // The topic ID requested for deletion. + TopicID [16]byte // v6+ + + // ErrorCode is the error code returned for an individual topic in + // deletion request. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // to delete a topic. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the topic. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // TOPIC_DELETION_DISABLED is returned for deletion requests version 3+ + // and brokers >= 2.1.0. INVALID_REQUEST is issued for request versions + // 0-2 against brokers >= 2.1.0. Otherwise, the request hangs until it + // times out. + // + // UNSUPPORTED_VERSION is returned when using topic IDs with a cluster + // that is not yet Kafka v2.8+. + // + // UNKNOWN_TOPIC_ID is returned when using topic IDs to a Kafka cluster + // v2.8+ and the topic ID is not found. + ErrorCode int16 + + // ErrorMessage is a message for an error. + ErrorMessage *string // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteTopicsResponseTopic. +func (v *DeleteTopicsResponseTopic) Default() { +} + +// NewDeleteTopicsResponseTopic returns a default DeleteTopicsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteTopicsResponseTopic() DeleteTopicsResponseTopic { + var v DeleteTopicsResponseTopic + v.Default() + return v +} + +// DeleteTopicsResponse is returned from a DeleteTopicsRequest. +// Version 3 added the TOPIC_DELETION_DISABLED error proposed in KIP-322 +// and introduced in Kafka 2.1.0. Prior, the request timed out. +type DeleteTopicsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 // v1+ + + // Topics contains responses for each topic requested for deletion. + Topics []DeleteTopicsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*DeleteTopicsResponse) Key() int16 { return 20 } +func (*DeleteTopicsResponse) MaxVersion() int16 { return 6 } +func (v *DeleteTopicsResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteTopicsResponse) GetVersion() int16 { return v.Version } +func (v *DeleteTopicsResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *DeleteTopicsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *DeleteTopicsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DeleteTopicsResponse) RequestKind() Request { return &DeleteTopicsRequest{Version: v.Version} } + +func (v *DeleteTopicsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 1 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if version < 6 { + var vv string + if v != nil { + vv = *v + } + { + v := vv + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } else { + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + } + if version >= 6 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 5 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteTopicsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteTopicsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteTopicsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteTopicsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v *string + if version < 6 { + var vv string + if isFlexible { + if unsafe { + vv = b.UnsafeCompactString() + } else { + vv = b.CompactString() + } + } else { + if unsafe { + vv = b.UnsafeString() + } else { + vv = b.String() + } + } + v = &vv + } else { + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + } + s.Topic = v + } + if version >= 6 { + v := b.Uuid() + s.TopicID = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 5 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteTopicsResponse returns a pointer to a default DeleteTopicsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteTopicsResponse() *DeleteTopicsResponse { + var v DeleteTopicsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteTopicsResponse. +func (v *DeleteTopicsResponse) Default() { +} + +// NewDeleteTopicsResponse returns a default DeleteTopicsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteTopicsResponse() DeleteTopicsResponse { + var v DeleteTopicsResponse + v.Default() + return v +} + +type DeleteRecordsRequestTopicPartition struct { + // Partition is a partition to delete records from. + Partition int32 + + // Offset is the offset to set the partition's low watermark (start + // offset) to. After a successful response, all records before this + // offset are considered deleted and are no longer readable. + // + // To delete all records, use -1, which is mapped to the partition's + // current high watermark. + Offset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsRequestTopicPartition. +func (v *DeleteRecordsRequestTopicPartition) Default() { +} + +// NewDeleteRecordsRequestTopicPartition returns a default DeleteRecordsRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsRequestTopicPartition() DeleteRecordsRequestTopicPartition { + var v DeleteRecordsRequestTopicPartition + v.Default() + return v +} + +type DeleteRecordsRequestTopic struct { + // Topic is a topic to delete records from. + Topic string + + // Partitions contains partitions to delete records from. + Partitions []DeleteRecordsRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsRequestTopic. +func (v *DeleteRecordsRequestTopic) Default() { +} + +// NewDeleteRecordsRequestTopic returns a default DeleteRecordsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsRequestTopic() DeleteRecordsRequestTopic { + var v DeleteRecordsRequestTopic + v.Default() + return v +} + +// DeleteRecordsRequest is an admin request to delete records from Kafka. +// This was added for KIP-107. +// +// To delete records, Kafka sets the LogStartOffset for partitions to +// the requested offset. All segments whose max partition is before the +// requested offset are deleted, and any records within the segment before +// the requested offset can no longer be read. +// +// This request must be issued to the correct brokers that own the partitions +// you intend to delete records for. +type DeleteRecordsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics contains topics for which to delete records from. + Topics []DeleteRecordsRequestTopic + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 15000. + TimeoutMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteRecordsRequest) Key() int16 { return 21 } +func (*DeleteRecordsRequest) MaxVersion() int16 { return 2 } +func (v *DeleteRecordsRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteRecordsRequest) GetVersion() int16 { return v.Version } +func (v *DeleteRecordsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteRecordsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *DeleteRecordsRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *DeleteRecordsRequest) ResponseKind() Response { + r := &DeleteRecordsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteRecordsRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteRecordsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteRecordsResponse) + return resp, err +} + +func (v *DeleteRecordsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteRecordsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteRecordsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteRecordsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteRecordsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteRecordsRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteRecordsRequest returns a pointer to a default DeleteRecordsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteRecordsRequest() *DeleteRecordsRequest { + var v DeleteRecordsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsRequest. +func (v *DeleteRecordsRequest) Default() { + v.TimeoutMillis = 15000 +} + +// NewDeleteRecordsRequest returns a default DeleteRecordsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsRequest() DeleteRecordsRequest { + var v DeleteRecordsRequest + v.Default() + return v +} + +type DeleteRecordsResponseTopicPartition struct { + // Partition is the partition this response corresponds to. + Partition int32 + + // LowWatermark is the new earliest offset for this partition. + LowWatermark int64 + + // ErrorCode is the error code returned for a given partition in + // the delete request. + // + // TOPIC_AUTHORIZATION_FAILED is returned for all partitions if the + // client is not authorized to delete records. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned for all partitions that + // the requested broker does not know of. + // + // NOT_LEADER_FOR_PARTITION is returned for partitions that the + // requested broker is not a leader of. + // + // OFFSET_OUT_OF_RANGE is returned if the requested offset is + // negative or higher than the current high watermark. + // + // POLICY_VIOLATION is returned if records cannot be deleted due to + // broker configuration. + // + // KAFKA_STORAGE_EXCEPTION is returned if the partition is in an + // offline log directory. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsResponseTopicPartition. +func (v *DeleteRecordsResponseTopicPartition) Default() { +} + +// NewDeleteRecordsResponseTopicPartition returns a default DeleteRecordsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsResponseTopicPartition() DeleteRecordsResponseTopicPartition { + var v DeleteRecordsResponseTopicPartition + v.Default() + return v +} + +type DeleteRecordsResponseTopic struct { + // Topic is the topic this response corresponds to. + Topic string + + // Partitions contains responses for each partition in a requested topic + // in the delete records request. + Partitions []DeleteRecordsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsResponseTopic. +func (v *DeleteRecordsResponseTopic) Default() { +} + +// NewDeleteRecordsResponseTopic returns a default DeleteRecordsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsResponseTopic() DeleteRecordsResponseTopic { + var v DeleteRecordsResponseTopic + v.Default() + return v +} + +// DeleteRecordsResponse is returned from a DeleteRecordsRequest. +type DeleteRecordsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Topics contains responses for each topic in the delete records request. + Topics []DeleteRecordsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteRecordsResponse) Key() int16 { return 21 } +func (*DeleteRecordsResponse) MaxVersion() int16 { return 2 } +func (v *DeleteRecordsResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteRecordsResponse) GetVersion() int16 { return v.Version } +func (v *DeleteRecordsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteRecordsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *DeleteRecordsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DeleteRecordsResponse) RequestKind() Request { + return &DeleteRecordsRequest{Version: v.Version} +} + +func (v *DeleteRecordsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LowWatermark + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteRecordsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteRecordsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteRecordsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteRecordsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteRecordsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.LowWatermark = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteRecordsResponse returns a pointer to a default DeleteRecordsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteRecordsResponse() *DeleteRecordsResponse { + var v DeleteRecordsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteRecordsResponse. +func (v *DeleteRecordsResponse) Default() { +} + +// NewDeleteRecordsResponse returns a default DeleteRecordsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteRecordsResponse() DeleteRecordsResponse { + var v DeleteRecordsResponse + v.Default() + return v +} + +// InitProducerIDRequest initializes a producer ID for idempotent transactions, +// and if using transactions, a producer epoch. This is the first request +// necessary to begin idempotent producing or transactions. +// +// Note that you do not need to go to a txn coordinator if you are initializing +// a producer id without a transactional id. +type InitProducerIDRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionalID is the ID to use for transactions if using transactions. + TransactionalID *string + + // TransactionTimeoutMillis is how long a transaction is allowed before + // EndTxn is required. + // + // Note that this timeout only begins on the first AddPartitionsToTxn + // request. + TransactionTimeoutMillis int32 + + // ProducerID, added for KIP-360, is the current producer ID. This allows + // the client to potentially recover on UNKNOWN_PRODUCER_ID errors. + // + // This field has a default of -1. + ProducerID int64 // v3+ + + // The producer's current epoch. This will be checked against the producer + // epoch on the broker, and the request will return an error if they do not + // match. Also added for KIP-360. + // + // This field has a default of -1. + ProducerEpoch int16 // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*InitProducerIDRequest) Key() int16 { return 22 } +func (*InitProducerIDRequest) MaxVersion() int16 { return 5 } +func (v *InitProducerIDRequest) SetVersion(version int16) { v.Version = version } +func (v *InitProducerIDRequest) GetVersion() int16 { return v.Version } +func (v *InitProducerIDRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *InitProducerIDRequest) IsTxnCoordinatorRequest() {} +func (v *InitProducerIDRequest) ResponseKind() Response { + r := &InitProducerIDResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *InitProducerIDRequest) RequestWith(ctx context.Context, r Requestor) (*InitProducerIDResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*InitProducerIDResponse) + return resp, err +} + +func (v *InitProducerIDRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.TransactionTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 3 { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + if version >= 3 { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *InitProducerIDRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *InitProducerIDRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *InitProducerIDRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.TransactionalID = v + } + { + v := b.Int32() + s.TransactionTimeoutMillis = v + } + if version >= 3 { + v := b.Int64() + s.ProducerID = v + } + if version >= 3 { + v := b.Int16() + s.ProducerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrInitProducerIDRequest returns a pointer to a default InitProducerIDRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrInitProducerIDRequest() *InitProducerIDRequest { + var v InitProducerIDRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitProducerIDRequest. +func (v *InitProducerIDRequest) Default() { + v.ProducerID = -1 + v.ProducerEpoch = -1 +} + +// NewInitProducerIDRequest returns a default InitProducerIDRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitProducerIDRequest() InitProducerIDRequest { + var v InitProducerIDRequest + v.Default() + return v +} + +// InitProducerIDResponse is returned for an InitProducerIDRequest. +type InitProducerIDResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // CLUSTER_AUTHORIZATION_FAILED is returned when not using transactions if + // the client is not authorized for idempotent_write on cluster. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned when using transactions + // if the client is not authorized to write on transactional_id. + // + // INVALID_REQUEST is returned if using transactions and the transactional id + // is an empty, non-null string + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the coordinator for this + // transactional ID is still loading. + // + // NOT_COORDINATOR is returned if the broker is not the coordinator for + // this transactional ID. + // + // INVALID_TRANSACTION_TIMEOUT is returned if using transactions and the timeout + // is equal to over over transaction.max.timeout.ms or under 0. + // + // CONCURRENT_TRANSACTIONS is returned if there is an ongoing transaction + // that is completing at the time this init is called. + ErrorCode int16 + + // ProducerID is the next producer ID that Kafka generated. This ID is used + // to ensure repeated produce requests do not result in duplicate records. + // + // This field has a default of -1. + ProducerID int64 + + // ProducerEpoch is the producer epoch to use for transactions. + ProducerEpoch int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*InitProducerIDResponse) Key() int16 { return 22 } +func (*InitProducerIDResponse) MaxVersion() int16 { return 5 } +func (v *InitProducerIDResponse) SetVersion(version int16) { v.Version = version } +func (v *InitProducerIDResponse) GetVersion() int16 { return v.Version } +func (v *InitProducerIDResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *InitProducerIDResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *InitProducerIDResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *InitProducerIDResponse) RequestKind() Request { + return &InitProducerIDRequest{Version: v.Version} +} + +func (v *InitProducerIDResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *InitProducerIDResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *InitProducerIDResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *InitProducerIDResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrInitProducerIDResponse returns a pointer to a default InitProducerIDResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrInitProducerIDResponse() *InitProducerIDResponse { + var v InitProducerIDResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitProducerIDResponse. +func (v *InitProducerIDResponse) Default() { + v.ProducerID = -1 +} + +// NewInitProducerIDResponse returns a default InitProducerIDResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitProducerIDResponse() InitProducerIDResponse { + var v InitProducerIDResponse + v.Default() + return v +} + +type OffsetForLeaderEpochRequestTopicPartition struct { + // Partition is the number of a partition. + Partition int32 + + // CurrentLeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0, + // allows brokers to check if the client is fenced (has an out of date + // leader) or if the client is ahead of the broker. + // + // The initial leader epoch can be determined from a MetadataResponse. + // + // This field has a default of -1. + CurrentLeaderEpoch int32 // v2+ + + // LeaderEpoch is the epoch to fetch the end offset for. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochRequestTopicPartition. +func (v *OffsetForLeaderEpochRequestTopicPartition) Default() { + v.CurrentLeaderEpoch = -1 +} + +// NewOffsetForLeaderEpochRequestTopicPartition returns a default OffsetForLeaderEpochRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochRequestTopicPartition() OffsetForLeaderEpochRequestTopicPartition { + var v OffsetForLeaderEpochRequestTopicPartition + v.Default() + return v +} + +type OffsetForLeaderEpochRequestTopic struct { + // Topic is the name of a topic. + Topic string + + // Partitions are partitions within a topic to fetch leader epoch offsets for. + Partitions []OffsetForLeaderEpochRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochRequestTopic. +func (v *OffsetForLeaderEpochRequestTopic) Default() { +} + +// NewOffsetForLeaderEpochRequestTopic returns a default OffsetForLeaderEpochRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochRequestTopic() OffsetForLeaderEpochRequestTopic { + var v OffsetForLeaderEpochRequestTopic + v.Default() + return v +} + +// OffsetForLeaderEpochRequest requests log end offsets for partitions. +// +// Version 2, proposed in KIP-320 and introduced in Kafka 2.1.0, can be used by +// consumers to perform more accurate offset resetting in the case of data loss. +// +// In support of version 2, this requires DESCRIBE on TOPIC. +type OffsetForLeaderEpochRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ReplicaID, added in support of KIP-392, is the broker ID of the follower, + // or -1 if this request is from a consumer. + // + // This field has a default of -2. + ReplicaID int32 // v3+ + + // Topics are topics to fetch leader epoch offsets for. + Topics []OffsetForLeaderEpochRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*OffsetForLeaderEpochRequest) Key() int16 { return 23 } +func (*OffsetForLeaderEpochRequest) MaxVersion() int16 { return 4 } +func (v *OffsetForLeaderEpochRequest) SetVersion(version int16) { v.Version = version } +func (v *OffsetForLeaderEpochRequest) GetVersion() int16 { return v.Version } +func (v *OffsetForLeaderEpochRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *OffsetForLeaderEpochRequest) ResponseKind() Response { + r := &OffsetForLeaderEpochResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *OffsetForLeaderEpochRequest) RequestWith(ctx context.Context, r Requestor) (*OffsetForLeaderEpochResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*OffsetForLeaderEpochResponse) + return resp, err +} + +func (v *OffsetForLeaderEpochRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 3 { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.CurrentLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetForLeaderEpochRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetForLeaderEpochRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetForLeaderEpochRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 3 { + v := b.Int32() + s.ReplicaID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetForLeaderEpochRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetForLeaderEpochRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if version >= 2 { + v := b.Int32() + s.CurrentLeaderEpoch = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetForLeaderEpochRequest returns a pointer to a default OffsetForLeaderEpochRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetForLeaderEpochRequest() *OffsetForLeaderEpochRequest { + var v OffsetForLeaderEpochRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochRequest. +func (v *OffsetForLeaderEpochRequest) Default() { + v.ReplicaID = -2 +} + +// NewOffsetForLeaderEpochRequest returns a default OffsetForLeaderEpochRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochRequest() OffsetForLeaderEpochRequest { + var v OffsetForLeaderEpochRequest + v.Default() + return v +} + +type OffsetForLeaderEpochResponseTopicPartition struct { + // ErrorCode is the error code returned on request failure. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client does not have + // the necessary permissions to issue this request. + // + // KAFKA_STORAGE_ERROR is returned if the partition is offline. + // + // NOT_LEADER_FOR_PARTITION is returned if the broker knows of the partition + // but does not own it. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of the + // partition. + // + // FENCED_LEADER_EPOCH is returned if the client is using a current leader epoch + // older than the actual leader epoch. + // + // UNKNOWN_LEADER_EPOCH if returned if the client is using a current leader epoch + // that the actual leader does not know of. This could occur when the client + // has newer metadata than the broker when the broker just became the leader for + // a replica. + ErrorCode int16 + + // Partition is the partition this response is for. + Partition int32 + + // LeaderEpoch is similar to the requested leader epoch, but pairs with the + // next field. If the requested leader epoch is unknown, this is -1. If the + // requested epoch had no records produced during the requested epoch, this + // is the first prior epoch that had records. + // + // This field has a default of -1. + LeaderEpoch int32 // v1+ + + // EndOffset is either (1) just past the last recorded offset in the + // current partition if the broker leader has the same epoch as the + // leader epoch in the request, or (2) the beginning offset of the next + // epoch if the leader is past the requested epoch. The second scenario + // can be seen as equivalent to the first: the beginning offset of the + // next epoch is just past the final offset of the prior epoch. + // + // (2) allows consumers to detect data loss: if the consumer consumed + // past the end offset that is returned, then the consumer should reset + // to the returned offset and the consumer knows everything past the end + // offset was lost. + // + // With the prior field, consumers know that at this offset, the broker + // either has no more records (consumer is caught up), or the broker + // transitioned to a new epoch. + // + // This field has a default of -1. + EndOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochResponseTopicPartition. +func (v *OffsetForLeaderEpochResponseTopicPartition) Default() { + v.LeaderEpoch = -1 + v.EndOffset = -1 +} + +// NewOffsetForLeaderEpochResponseTopicPartition returns a default OffsetForLeaderEpochResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochResponseTopicPartition() OffsetForLeaderEpochResponseTopicPartition { + var v OffsetForLeaderEpochResponseTopicPartition + v.Default() + return v +} + +type OffsetForLeaderEpochResponseTopic struct { + // Topic is the topic this response corresponds to. + Topic string + + // Partitions are responses to partitions in a topic in the request. + Partitions []OffsetForLeaderEpochResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochResponseTopic. +func (v *OffsetForLeaderEpochResponseTopic) Default() { +} + +// NewOffsetForLeaderEpochResponseTopic returns a default OffsetForLeaderEpochResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochResponseTopic() OffsetForLeaderEpochResponseTopic { + var v OffsetForLeaderEpochResponseTopic + v.Default() + return v +} + +// OffsetForLeaderEpochResponse is returned from an OffsetForLeaderEpochRequest. +type OffsetForLeaderEpochResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 // v2+ + + // Topics are responses to topics in the request. + Topics []OffsetForLeaderEpochResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*OffsetForLeaderEpochResponse) Key() int16 { return 23 } +func (*OffsetForLeaderEpochResponse) MaxVersion() int16 { return 4 } +func (v *OffsetForLeaderEpochResponse) SetVersion(version int16) { v.Version = version } +func (v *OffsetForLeaderEpochResponse) GetVersion() int16 { return v.Version } +func (v *OffsetForLeaderEpochResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *OffsetForLeaderEpochResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *OffsetForLeaderEpochResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *OffsetForLeaderEpochResponse) RequestKind() Request { + return &OffsetForLeaderEpochRequest{Version: v.Version} +} + +func (v *OffsetForLeaderEpochResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + if version >= 2 { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.EndOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *OffsetForLeaderEpochResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetForLeaderEpochResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetForLeaderEpochResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + if version >= 2 { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetForLeaderEpochResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetForLeaderEpochResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.Partition = v + } + if version >= 1 { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Int64() + s.EndOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrOffsetForLeaderEpochResponse returns a pointer to a default OffsetForLeaderEpochResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetForLeaderEpochResponse() *OffsetForLeaderEpochResponse { + var v OffsetForLeaderEpochResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetForLeaderEpochResponse. +func (v *OffsetForLeaderEpochResponse) Default() { +} + +// NewOffsetForLeaderEpochResponse returns a default OffsetForLeaderEpochResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetForLeaderEpochResponse() OffsetForLeaderEpochResponse { + var v OffsetForLeaderEpochResponse + v.Default() + return v +} + +type AddPartitionsToTxnRequestTopic struct { + // Topic is a topic name. + Topic string + + // Partitions are partitions within a topic to add as part of the producer + // side of a transaction. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnRequestTopic. +func (v *AddPartitionsToTxnRequestTopic) Default() { +} + +// NewAddPartitionsToTxnRequestTopic returns a default AddPartitionsToTxnRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnRequestTopic() AddPartitionsToTxnRequestTopic { + var v AddPartitionsToTxnRequestTopic + v.Default() + return v +} + +type AddPartitionsToTxnRequestTransactionTopic struct { + Topic string + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnRequestTransactionTopic. +func (v *AddPartitionsToTxnRequestTransactionTopic) Default() { +} + +// NewAddPartitionsToTxnRequestTransactionTopic returns a default AddPartitionsToTxnRequestTransactionTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnRequestTransactionTopic() AddPartitionsToTxnRequestTransactionTopic { + var v AddPartitionsToTxnRequestTransactionTopic + v.Default() + return v +} + +type AddPartitionsToTxnRequestTransaction struct { + TransactionalID string + + ProducerID int64 + + ProducerEpoch int16 + + // VerifyOnly signifies if we want to check if the partition is in the + // transaction rather than add it. + VerifyOnly bool + + Topics []AddPartitionsToTxnRequestTransactionTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnRequestTransaction. +func (v *AddPartitionsToTxnRequestTransaction) Default() { +} + +// NewAddPartitionsToTxnRequestTransaction returns a default AddPartitionsToTxnRequestTransaction +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnRequestTransaction() AddPartitionsToTxnRequestTransaction { + var v AddPartitionsToTxnRequestTransaction + v.Default() + return v +} + +// AddPartitionsToTxnRequest begins the producer side of a transaction for all +// partitions in the request. Before producing any records to a partition in +// the transaction, that partition must have been added to the transaction with +// this request. +// +// Versions 3 and below are exclusively used by clients and versions 4 and +// above are used by brokers. +// +// Version 4 adds VerifyOnly field to check if partitions are already in +// transaction and adds support to batch multiple transactions. +type AddPartitionsToTxnRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionalID is the transactional ID to use for this request. + TransactionalID string // v0-v3 + + // ProducerID is the producer ID of the client for this transactional ID + // as received from InitProducerID. + ProducerID int64 // v0-v3 + + // ProducerEpoch is the producer epoch of the client for this transactional ID + // as received from InitProducerID. + ProducerEpoch int16 // v0-v3 + + // Topics are topics to add as part of the producer side of a transaction. + Topics []AddPartitionsToTxnRequestTopic // v0-v3 + + // The list of transactions to add partitions to, for v4+, for brokers only. + // The fields in this are batch broker requests that duplicate the above fields + // and thus are undocumented (except VerifyOnly, which is new). + Transactions []AddPartitionsToTxnRequestTransaction // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*AddPartitionsToTxnRequest) Key() int16 { return 24 } +func (*AddPartitionsToTxnRequest) MaxVersion() int16 { return 5 } +func (v *AddPartitionsToTxnRequest) SetVersion(version int16) { v.Version = version } +func (v *AddPartitionsToTxnRequest) GetVersion() int16 { return v.Version } +func (v *AddPartitionsToTxnRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *AddPartitionsToTxnRequest) IsTxnCoordinatorRequest() {} +func (v *AddPartitionsToTxnRequest) ResponseKind() Response { + r := &AddPartitionsToTxnResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AddPartitionsToTxnRequest) RequestWith(ctx context.Context, r Requestor) (*AddPartitionsToTxnResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AddPartitionsToTxnResponse) + return resp, err +} + +func (v *AddPartitionsToTxnRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + if version >= 0 && version <= 3 { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 0 && version <= 3 { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + if version >= 0 && version <= 3 { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + if version >= 0 && version <= 3 { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 4 { + v := v.Transactions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.VerifyOnly + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddPartitionsToTxnRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddPartitionsToTxnRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddPartitionsToTxnRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + if version >= 0 && version <= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + if version >= 0 && version <= 3 { + v := b.Int64() + s.ProducerID = v + } + if version >= 0 && version <= 3 { + v := b.Int16() + s.ProducerEpoch = v + } + if version >= 0 && version <= 3 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 4 { + v := s.Transactions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnRequestTransaction, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := b.Bool() + s.VerifyOnly = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnRequestTransactionTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Transactions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddPartitionsToTxnRequest returns a pointer to a default AddPartitionsToTxnRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddPartitionsToTxnRequest() *AddPartitionsToTxnRequest { + var v AddPartitionsToTxnRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnRequest. +func (v *AddPartitionsToTxnRequest) Default() { +} + +// NewAddPartitionsToTxnRequest returns a default AddPartitionsToTxnRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnRequest() AddPartitionsToTxnRequest { + var v AddPartitionsToTxnRequest + v.Default() + return v +} + +type AddPartitionsToTxnResponseTransactionTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponseTransactionTopicPartition. +func (v *AddPartitionsToTxnResponseTransactionTopicPartition) Default() { +} + +// NewAddPartitionsToTxnResponseTransactionTopicPartition returns a default AddPartitionsToTxnResponseTransactionTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponseTransactionTopicPartition() AddPartitionsToTxnResponseTransactionTopicPartition { + var v AddPartitionsToTxnResponseTransactionTopicPartition + v.Default() + return v +} + +type AddPartitionsToTxnResponseTransactionTopic struct { + Topic string + + Partitions []AddPartitionsToTxnResponseTransactionTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponseTransactionTopic. +func (v *AddPartitionsToTxnResponseTransactionTopic) Default() { +} + +// NewAddPartitionsToTxnResponseTransactionTopic returns a default AddPartitionsToTxnResponseTransactionTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponseTransactionTopic() AddPartitionsToTxnResponseTransactionTopic { + var v AddPartitionsToTxnResponseTransactionTopic + v.Default() + return v +} + +type AddPartitionsToTxnResponseTransaction struct { + // The transactional id corresponding to the transaction. + TransactionalID string + + Topics []AddPartitionsToTxnResponseTransactionTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponseTransaction. +func (v *AddPartitionsToTxnResponseTransaction) Default() { +} + +// NewAddPartitionsToTxnResponseTransaction returns a default AddPartitionsToTxnResponseTransaction +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponseTransaction() AddPartitionsToTxnResponseTransaction { + var v AddPartitionsToTxnResponseTransaction + v.Default() + return v +} + +type AddPartitionsToTxnResponseTopicPartition struct { + // Partition is a partition being responded to. + Partition int32 + + // ErrorCode is any error for this topic/partition commit. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned if the client is + // not authorized for write with transactional IDs with the requested + // transactional ID. + // + // TOPIC_AUTHORIZATION_FAILED is returned for all topics that the client + // is not authorized to write to. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned for all topics or partitions + // that the broker does not know of. + // + // OPERATION_NOT_ATTEMPTED is returned if any of the above errors occur + // for all partitions that did not have the above errors. + // + // INVALID_REQUEST is returned if the transactional ID is invalid. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the coordinator for this + // transactional ID is still loading. + // + // NOT_COORDINATOR is returned if the broker is not the coordinator for + // this transactional ID. + // + // INVALID_PRODUCER_ID_MAPPING is returned if the produce request used + // a producer ID that is not tied to the transactional ID (i.e., mismatch + // from what was returned from InitProducerID). + // + // INVALID_PRODUCER_EPOCH is returned if the requested epoch does not match + // the broker epoch for this transactional ID. + // + // CONCURRENT_TRANSACTIONS is returned if there is an ongoing transaction for + // this transactional ID, if the producer ID and epoch matches the broker's. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponseTopicPartition. +func (v *AddPartitionsToTxnResponseTopicPartition) Default() { +} + +// NewAddPartitionsToTxnResponseTopicPartition returns a default AddPartitionsToTxnResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponseTopicPartition() AddPartitionsToTxnResponseTopicPartition { + var v AddPartitionsToTxnResponseTopicPartition + v.Default() + return v +} + +type AddPartitionsToTxnResponseTopic struct { + // Topic is a topic being responded to. + Topic string + + // Partitions are responses to partitions in the request. + Partitions []AddPartitionsToTxnResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponseTopic. +func (v *AddPartitionsToTxnResponseTopic) Default() { +} + +// NewAddPartitionsToTxnResponseTopic returns a default AddPartitionsToTxnResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponseTopic() AddPartitionsToTxnResponseTopic { + var v AddPartitionsToTxnResponseTopic + v.Default() + return v +} + +// AddPartitionsToTxnResponse is a response to an AddPartitionsToTxnRequest. +type AddPartitionsToTxnResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // The response top level error code. + ErrorCode int16 // v4+ + + // Results categorized by transactional ID, v4+ only, for brokers only. + // The fields duplicate v3 and below fields (except TransactionalID) and + // are left undocumented. + Transactions []AddPartitionsToTxnResponseTransaction // v4+ + + // Topics are responses to topics in the request. + Topics []AddPartitionsToTxnResponseTopic // v0-v3 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*AddPartitionsToTxnResponse) Key() int16 { return 24 } +func (*AddPartitionsToTxnResponse) MaxVersion() int16 { return 5 } +func (v *AddPartitionsToTxnResponse) SetVersion(version int16) { v.Version = version } +func (v *AddPartitionsToTxnResponse) GetVersion() int16 { return v.Version } +func (v *AddPartitionsToTxnResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *AddPartitionsToTxnResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *AddPartitionsToTxnResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AddPartitionsToTxnResponse) RequestKind() Request { + return &AddPartitionsToTxnRequest{Version: v.Version} +} + +func (v *AddPartitionsToTxnResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 4 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 4 { + v := v.Transactions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 0 && version <= 3 { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddPartitionsToTxnResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddPartitionsToTxnResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddPartitionsToTxnResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 4 { + v := b.Int16() + s.ErrorCode = v + } + if version >= 4 { + v := s.Transactions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnResponseTransaction, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnResponseTransactionTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnResponseTransactionTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Transactions = v + } + if version >= 0 && version <= 3 { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddPartitionsToTxnResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddPartitionsToTxnResponse returns a pointer to a default AddPartitionsToTxnResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddPartitionsToTxnResponse() *AddPartitionsToTxnResponse { + var v AddPartitionsToTxnResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddPartitionsToTxnResponse. +func (v *AddPartitionsToTxnResponse) Default() { +} + +// NewAddPartitionsToTxnResponse returns a default AddPartitionsToTxnResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddPartitionsToTxnResponse() AddPartitionsToTxnResponse { + var v AddPartitionsToTxnResponse + v.Default() + return v +} + +// AddOffsetsToTxnRequest is a request that ties produced records to what group +// is being consumed for the transaction. +// +// This request must be called before TxnOffsetCommitRequest. +// +// Internally, this request simply adds the __consumer_offsets topic as a +// partition for this transaction with AddPartitionsToTxn for the partition +// in that topic that contains the group. +type AddOffsetsToTxnRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionalID is the transactional ID to use for this request. + TransactionalID string + + // ProducerID is the producer ID of the client for this transactional ID + // as received from InitProducerID. + ProducerID int64 + + // ProducerEpoch is the producer epoch of the client for this transactional ID + // as received from InitProducerID. + ProducerEpoch int16 + + // Group is the group to tie this transaction to. + Group string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*AddOffsetsToTxnRequest) Key() int16 { return 25 } +func (*AddOffsetsToTxnRequest) MaxVersion() int16 { return 4 } +func (v *AddOffsetsToTxnRequest) SetVersion(version int16) { v.Version = version } +func (v *AddOffsetsToTxnRequest) GetVersion() int16 { return v.Version } +func (v *AddOffsetsToTxnRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *AddOffsetsToTxnRequest) IsTxnCoordinatorRequest() {} +func (v *AddOffsetsToTxnRequest) ResponseKind() Response { + r := &AddOffsetsToTxnResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AddOffsetsToTxnRequest) RequestWith(ctx context.Context, r Requestor) (*AddOffsetsToTxnResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AddOffsetsToTxnResponse) + return resp, err +} + +func (v *AddOffsetsToTxnRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddOffsetsToTxnRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddOffsetsToTxnRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddOffsetsToTxnRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddOffsetsToTxnRequest returns a pointer to a default AddOffsetsToTxnRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddOffsetsToTxnRequest() *AddOffsetsToTxnRequest { + var v AddOffsetsToTxnRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddOffsetsToTxnRequest. +func (v *AddOffsetsToTxnRequest) Default() { +} + +// NewAddOffsetsToTxnRequest returns a default AddOffsetsToTxnRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddOffsetsToTxnRequest() AddOffsetsToTxnRequest { + var v AddOffsetsToTxnRequest + v.Default() + return v +} + +// AddOffsetsToTxnResponse is a response to an AddOffsetsToTxnRequest. +type AddOffsetsToTxnResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // ErrorCode is any error for this topic/partition commit. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned if the client is + // not authorized for write with transactional IDs with the requested + // transactional ID. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to read group with the requested group id. + // + // This also can return any error that AddPartitionsToTxn returns. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*AddOffsetsToTxnResponse) Key() int16 { return 25 } +func (*AddOffsetsToTxnResponse) MaxVersion() int16 { return 4 } +func (v *AddOffsetsToTxnResponse) SetVersion(version int16) { v.Version = version } +func (v *AddOffsetsToTxnResponse) GetVersion() int16 { return v.Version } +func (v *AddOffsetsToTxnResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *AddOffsetsToTxnResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *AddOffsetsToTxnResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AddOffsetsToTxnResponse) RequestKind() Request { + return &AddOffsetsToTxnRequest{Version: v.Version} +} + +func (v *AddOffsetsToTxnResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddOffsetsToTxnResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddOffsetsToTxnResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddOffsetsToTxnResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddOffsetsToTxnResponse returns a pointer to a default AddOffsetsToTxnResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddOffsetsToTxnResponse() *AddOffsetsToTxnResponse { + var v AddOffsetsToTxnResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddOffsetsToTxnResponse. +func (v *AddOffsetsToTxnResponse) Default() { +} + +// NewAddOffsetsToTxnResponse returns a default AddOffsetsToTxnResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddOffsetsToTxnResponse() AddOffsetsToTxnResponse { + var v AddOffsetsToTxnResponse + v.Default() + return v +} + +// EndTxnRequest ends a transaction. This should be called after +// TxnOffsetCommitRequest. +type EndTxnRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionalID is the transactional ID to use for this request. + TransactionalID string + + // ProducerID is the producer ID of the client for this transactional ID + // as received from InitProducerID. + ProducerID int64 + + // ProducerEpoch is the producer epoch of the client for this transactional ID + // as received from InitProducerID. + ProducerEpoch int16 + + // Commit is whether to commit this transaction: true for yes, false for abort. + Commit bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*EndTxnRequest) Key() int16 { return 26 } +func (*EndTxnRequest) MaxVersion() int16 { return 5 } +func (v *EndTxnRequest) SetVersion(version int16) { v.Version = version } +func (v *EndTxnRequest) GetVersion() int16 { return v.Version } +func (v *EndTxnRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *EndTxnRequest) IsTxnCoordinatorRequest() {} +func (v *EndTxnRequest) ResponseKind() Response { + r := &EndTxnResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *EndTxnRequest) RequestWith(ctx context.Context, r Requestor) (*EndTxnResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*EndTxnResponse) + return resp, err +} + +func (v *EndTxnRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Commit + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EndTxnRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EndTxnRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EndTxnRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := b.Bool() + s.Commit = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrEndTxnRequest returns a pointer to a default EndTxnRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEndTxnRequest() *EndTxnRequest { + var v EndTxnRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndTxnRequest. +func (v *EndTxnRequest) Default() { +} + +// NewEndTxnRequest returns a default EndTxnRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndTxnRequest() EndTxnRequest { + var v EndTxnRequest + v.Default() + return v +} + +// EndTxnResponse is a response for an EndTxnRequest. +type EndTxnResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // ErrorCode is any error for this topic/partition commit. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned if the client is + // not authorized for write with transactional IDs with the requested + // transactional ID. + // + // INVALID_REQUEST is returned if the transactional ID is invalid. + // + // INVALID_PRODUCER_ID_MAPPING is returned if the produce request used + // a producer ID that is not tied to the transactional ID (i.e., mismatch + // from what was returned from InitProducerID). + // + // INVALID_PRODUCER_EPOCH is returned if the requested epoch does not match + // the broker epoch for this transactional ID. + // + // CONCURRENT_TRANSACTIONS is returned if there is an ongoing transaction for + // this transactional ID, if the producer ID and epoch matches the broker's. + // + // INVALID_TXN_STATE is returned if this request is attempted at the wrong + // time (given the order of how transaction requests should go). + ErrorCode int16 + + // Kafka 4.0+ returns the producer ID that the producer should use on the + // *next* transaction. This is the same as the ID used in the request, but + // is bumped if the epoch overflows. See KIP-890. + // + // This field has a default of -1. + ProducerID int64 // v5+ + + // Kafka 4.0+ returns the producer epoch that the producer should use on the + // *next* transaction. See KIP-890. + // + // This field has a default of -1. + ProducerEpoch int16 // v5+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*EndTxnResponse) Key() int16 { return 26 } +func (*EndTxnResponse) MaxVersion() int16 { return 5 } +func (v *EndTxnResponse) SetVersion(version int16) { v.Version = version } +func (v *EndTxnResponse) GetVersion() int16 { return v.Version } +func (v *EndTxnResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *EndTxnResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *EndTxnResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *EndTxnResponse) RequestKind() Request { return &EndTxnRequest{Version: v.Version} } + +func (v *EndTxnResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 5 { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + if version >= 5 { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EndTxnResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EndTxnResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EndTxnResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 5 { + v := b.Int64() + s.ProducerID = v + } + if version >= 5 { + v := b.Int16() + s.ProducerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrEndTxnResponse returns a pointer to a default EndTxnResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEndTxnResponse() *EndTxnResponse { + var v EndTxnResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndTxnResponse. +func (v *EndTxnResponse) Default() { + v.ProducerID = -1 + v.ProducerEpoch = -1 +} + +// NewEndTxnResponse returns a default EndTxnResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndTxnResponse() EndTxnResponse { + var v EndTxnResponse + v.Default() + return v +} + +type WriteTxnMarkersRequestMarkerTopic struct { + // Topic is the name of the topic to write markers for. + Topic string + + // Partitions contains partitions to write markers for. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersRequestMarkerTopic. +func (v *WriteTxnMarkersRequestMarkerTopic) Default() { +} + +// NewWriteTxnMarkersRequestMarkerTopic returns a default WriteTxnMarkersRequestMarkerTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersRequestMarkerTopic() WriteTxnMarkersRequestMarkerTopic { + var v WriteTxnMarkersRequestMarkerTopic + v.Default() + return v +} + +type WriteTxnMarkersRequestMarker struct { + // ProducerID is the current producer ID to use when writing a marker. + ProducerID int64 + + // ProducerEpoch is the current producer epoch to use when writing a + // marker. + ProducerEpoch int16 + + // Committed is true if this marker is for a committed transaction, + // otherwise false if this is for an aborted transaction. + Committed bool + + // Topics contains the topics we are writing markers for. + Topics []WriteTxnMarkersRequestMarkerTopic + + // CoordinatorEpoch is the current epoch of the transaction coordinator we + // are writing a marker to. This is used to detect fenced writers. + CoordinatorEpoch int32 + + // TransactionVersion is the transaction version of the marker. + // 0/1 = legacy (TV0/TV1), 2 = TV2 etc. + TransactionVersion int8 // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersRequestMarker. +func (v *WriteTxnMarkersRequestMarker) Default() { +} + +// NewWriteTxnMarkersRequestMarker returns a default WriteTxnMarkersRequestMarker +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersRequestMarker() WriteTxnMarkersRequestMarker { + var v WriteTxnMarkersRequestMarker + v.Default() + return v +} + +// WriteTxnMarkersRequest is a broker-to-broker request that Kafka uses to +// finish transactions. +// Version 2, introduced in Kafka 4.2, adds TransactionVersion (KIP-1228). +type WriteTxnMarkersRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Markers contains transactional markers to be written. + Markers []WriteTxnMarkersRequestMarker + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*WriteTxnMarkersRequest) Key() int16 { return 27 } +func (*WriteTxnMarkersRequest) MaxVersion() int16 { return 2 } +func (v *WriteTxnMarkersRequest) SetVersion(version int16) { v.Version = version } +func (v *WriteTxnMarkersRequest) GetVersion() int16 { return v.Version } +func (v *WriteTxnMarkersRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *WriteTxnMarkersRequest) ResponseKind() Response { + r := &WriteTxnMarkersResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *WriteTxnMarkersRequest) RequestWith(ctx context.Context, r Requestor) (*WriteTxnMarkersResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*WriteTxnMarkersResponse) + return resp, err +} + +func (v *WriteTxnMarkersRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Markers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Committed + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.CoordinatorEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.TransactionVersion + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *WriteTxnMarkersRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *WriteTxnMarkersRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *WriteTxnMarkersRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := s.Markers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteTxnMarkersRequestMarker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := b.Bool() + s.Committed = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteTxnMarkersRequestMarkerTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.CoordinatorEpoch = v + } + if version >= 2 { + v := b.Int8() + s.TransactionVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Markers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrWriteTxnMarkersRequest returns a pointer to a default WriteTxnMarkersRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrWriteTxnMarkersRequest() *WriteTxnMarkersRequest { + var v WriteTxnMarkersRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersRequest. +func (v *WriteTxnMarkersRequest) Default() { +} + +// NewWriteTxnMarkersRequest returns a default WriteTxnMarkersRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersRequest() WriteTxnMarkersRequest { + var v WriteTxnMarkersRequest + v.Default() + return v +} + +type WriteTxnMarkersResponseMarkerTopicPartition struct { + // Partition is the partition this result is for. + Partition int32 + + // ErrorCode is non-nil if writing the transansactional marker for this + // partition errored. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if the user does not have + // CLUSTER_ACTION on CLUSTER. + // + // NOT_LEADER_OR_FOLLOWER is returned if the broker receiving this + // request is not the leader of the partition. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic or partition is + // not known to exist. + // + // INVALID_PRODUCER_EPOCH is returned if the cluster epoch is provided + // and the provided epoch does not match. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersResponseMarkerTopicPartition. +func (v *WriteTxnMarkersResponseMarkerTopicPartition) Default() { +} + +// NewWriteTxnMarkersResponseMarkerTopicPartition returns a default WriteTxnMarkersResponseMarkerTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersResponseMarkerTopicPartition() WriteTxnMarkersResponseMarkerTopicPartition { + var v WriteTxnMarkersResponseMarkerTopicPartition + v.Default() + return v +} + +type WriteTxnMarkersResponseMarkerTopic struct { + // Topic is the topic these results are for. + Topic string + + // Partitions contains per-partition results for the write markers + // request. + Partitions []WriteTxnMarkersResponseMarkerTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersResponseMarkerTopic. +func (v *WriteTxnMarkersResponseMarkerTopic) Default() { +} + +// NewWriteTxnMarkersResponseMarkerTopic returns a default WriteTxnMarkersResponseMarkerTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersResponseMarkerTopic() WriteTxnMarkersResponseMarkerTopic { + var v WriteTxnMarkersResponseMarkerTopic + v.Default() + return v +} + +type WriteTxnMarkersResponseMarker struct { + // ProducerID is the producer ID these results are for (from the input + // request). + ProducerID int64 + + // Topics contains the results for the write markers request. + Topics []WriteTxnMarkersResponseMarkerTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersResponseMarker. +func (v *WriteTxnMarkersResponseMarker) Default() { +} + +// NewWriteTxnMarkersResponseMarker returns a default WriteTxnMarkersResponseMarker +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersResponseMarker() WriteTxnMarkersResponseMarker { + var v WriteTxnMarkersResponseMarker + v.Default() + return v +} + +// WriteTxnMarkersResponse is a response to a WriteTxnMarkersRequest. +type WriteTxnMarkersResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Markers contains results for writing transactional markers. + Markers []WriteTxnMarkersResponseMarker + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*WriteTxnMarkersResponse) Key() int16 { return 27 } +func (*WriteTxnMarkersResponse) MaxVersion() int16 { return 2 } +func (v *WriteTxnMarkersResponse) SetVersion(version int16) { v.Version = version } +func (v *WriteTxnMarkersResponse) GetVersion() int16 { return v.Version } +func (v *WriteTxnMarkersResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *WriteTxnMarkersResponse) RequestKind() Request { + return &WriteTxnMarkersRequest{Version: v.Version} +} + +func (v *WriteTxnMarkersResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Markers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *WriteTxnMarkersResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *WriteTxnMarkersResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *WriteTxnMarkersResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := s.Markers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteTxnMarkersResponseMarker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.ProducerID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteTxnMarkersResponseMarkerTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteTxnMarkersResponseMarkerTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Markers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrWriteTxnMarkersResponse returns a pointer to a default WriteTxnMarkersResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrWriteTxnMarkersResponse() *WriteTxnMarkersResponse { + var v WriteTxnMarkersResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteTxnMarkersResponse. +func (v *WriteTxnMarkersResponse) Default() { +} + +// NewWriteTxnMarkersResponse returns a default WriteTxnMarkersResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteTxnMarkersResponse() WriteTxnMarkersResponse { + var v WriteTxnMarkersResponse + v.Default() + return v +} + +type TxnOffsetCommitRequestTopicPartition struct { + // Partition is a partition to add for a pending commit. + Partition int32 + + // Offset is the offset within partition to commit once EndTxnRequest is + // called (with commit; abort obviously aborts). + Offset int64 + + // LeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0, + // allows brokers to check if the client is fenced (has an out of date + // leader) or is using an unknown leader. + // + // The initial leader epoch can be determined from a MetadataResponse. + // To skip log truncation checking, use -1. + // + // This field has a default of -1. + LeaderEpoch int32 // v2+ + + // Metadata is optional metadata the client wants to include with this + // commit. + Metadata *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitRequestTopicPartition. +func (v *TxnOffsetCommitRequestTopicPartition) Default() { + v.LeaderEpoch = -1 +} + +// NewTxnOffsetCommitRequestTopicPartition returns a default TxnOffsetCommitRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitRequestTopicPartition() TxnOffsetCommitRequestTopicPartition { + var v TxnOffsetCommitRequestTopicPartition + v.Default() + return v +} + +type TxnOffsetCommitRequestTopic struct { + // Topic is a topic to add for a pending commit. + Topic string + + // Partitions are partitions to add for pending commits. + Partitions []TxnOffsetCommitRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitRequestTopic. +func (v *TxnOffsetCommitRequestTopic) Default() { +} + +// NewTxnOffsetCommitRequestTopic returns a default TxnOffsetCommitRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitRequestTopic() TxnOffsetCommitRequestTopic { + var v TxnOffsetCommitRequestTopic + v.Default() + return v +} + +// TxnOffsetCommitRequest sends offsets that are a part of this transaction +// to be committed once the transaction itself finishes. This effectively +// replaces OffsetCommitRequest for when using transactions. +type TxnOffsetCommitRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TransactionalID is the transactional ID to use for this request. + TransactionalID string + + // Group is the group consumed in this transaction and to be used for + // committing. + Group string + + // ProducerID is the producer ID of the client for this transactional ID + // as received from InitProducerID. + ProducerID int64 + + // ProducerEpoch is the producer epoch of the client for this transactional ID + // as received from InitProducerID. + ProducerEpoch int16 + + // Generation is the group generation this transactional offset commit request is for. + // + // This field has a default of -1. + Generation int32 // v3+ + + // MemberID is the member ID this member is for. + MemberID string // v3+ + + // InstanceID is the instance ID of this member in the group (KIP-345, KIP-447). + InstanceID *string // v3+ + + // Topics are topics to add for pending commits. + Topics []TxnOffsetCommitRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*TxnOffsetCommitRequest) Key() int16 { return 28 } +func (*TxnOffsetCommitRequest) MaxVersion() int16 { return 5 } +func (v *TxnOffsetCommitRequest) SetVersion(version int16) { v.Version = version } +func (v *TxnOffsetCommitRequest) GetVersion() int16 { return v.Version } +func (v *TxnOffsetCommitRequest) IsFlexible() bool { return v.Version >= 3 } +func (v *TxnOffsetCommitRequest) IsGroupCoordinatorRequest() {} +func (v *TxnOffsetCommitRequest) ResponseKind() Response { + r := &TxnOffsetCommitResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *TxnOffsetCommitRequest) RequestWith(ctx context.Context, r Requestor) (*TxnOffsetCommitResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*TxnOffsetCommitResponse) + return resp, err +} + +func (v *TxnOffsetCommitRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + if version >= 3 { + v := v.Generation + dst = kbin.AppendInt32(dst, v) + } + if version >= 3 { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if version >= 2 { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Metadata + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *TxnOffsetCommitRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *TxnOffsetCommitRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *TxnOffsetCommitRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + if version >= 3 { + v := b.Int32() + s.Generation = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TxnOffsetCommitRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TxnOffsetCommitRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if version >= 2 { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Metadata = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrTxnOffsetCommitRequest returns a pointer to a default TxnOffsetCommitRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrTxnOffsetCommitRequest() *TxnOffsetCommitRequest { + var v TxnOffsetCommitRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitRequest. +func (v *TxnOffsetCommitRequest) Default() { + v.Generation = -1 +} + +// NewTxnOffsetCommitRequest returns a default TxnOffsetCommitRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitRequest() TxnOffsetCommitRequest { + var v TxnOffsetCommitRequest + v.Default() + return v +} + +type TxnOffsetCommitResponseTopicPartition struct { + // Partition is the partition this response is for. + Partition int32 + + // ErrorCode is any error for this topic/partition commit. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned if the client is + // not authorized for write with transactional IDs with the requested + // transactional ID. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to read group with the requested group id. + // + // TOPIC_AUTHORIZATION_FAILED is returned for all topics that the client + // is not authorized to read. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned for all topics or partitions + // that the broker does not know of. + // + // INVALID_GROUP_ID is returned if the requested group does not exist. + // + // COORDINATOR_NOT_AVAILABLE is returned if the broker is not yet fully + // started or is shutting down, or if the group was just deleted or is + // migrating to another broker. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is still loading. + // + // NOT_COORDINATOR is returned if the broker is not the coordinator for + // the group. + // + // FENCED_INSTANCE_ID is returned if the member is fenced (another newer + // transactional member is using the same instance ID). + // + // UNKNOWN_MEMBER_ID is returned if the consumer group does not know of + // this member. + // + // ILLEGAL_GENERATION is returned if the consumer group's generation is + // different than the requested generation. + // + // OFFSET_METADATA_TOO_LARGE is returned if the commit metadata is too + // large. + // + // REBALANCE_IN_PROGRESS is returned if the group is completing a rebalance. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitResponseTopicPartition. +func (v *TxnOffsetCommitResponseTopicPartition) Default() { +} + +// NewTxnOffsetCommitResponseTopicPartition returns a default TxnOffsetCommitResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitResponseTopicPartition() TxnOffsetCommitResponseTopicPartition { + var v TxnOffsetCommitResponseTopicPartition + v.Default() + return v +} + +type TxnOffsetCommitResponseTopic struct { + // Topic is the topic this response is for. + Topic string + + // Partitions contains responses to the partitions in this topic. + Partitions []TxnOffsetCommitResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitResponseTopic. +func (v *TxnOffsetCommitResponseTopic) Default() { +} + +// NewTxnOffsetCommitResponseTopic returns a default TxnOffsetCommitResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitResponseTopic() TxnOffsetCommitResponseTopic { + var v TxnOffsetCommitResponseTopic + v.Default() + return v +} + +// TxnOffsetCommitResponse is a response to a TxnOffsetCommitRequest. +type TxnOffsetCommitResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Topics contains responses to the topics in the request. + Topics []TxnOffsetCommitResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v3+ +} + +func (*TxnOffsetCommitResponse) Key() int16 { return 28 } +func (*TxnOffsetCommitResponse) MaxVersion() int16 { return 5 } +func (v *TxnOffsetCommitResponse) SetVersion(version int16) { v.Version = version } +func (v *TxnOffsetCommitResponse) GetVersion() int16 { return v.Version } +func (v *TxnOffsetCommitResponse) IsFlexible() bool { return v.Version >= 3 } +func (v *TxnOffsetCommitResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *TxnOffsetCommitResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *TxnOffsetCommitResponse) RequestKind() Request { + return &TxnOffsetCommitRequest{Version: v.Version} +} + +func (v *TxnOffsetCommitResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *TxnOffsetCommitResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *TxnOffsetCommitResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *TxnOffsetCommitResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 3 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TxnOffsetCommitResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TxnOffsetCommitResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrTxnOffsetCommitResponse returns a pointer to a default TxnOffsetCommitResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrTxnOffsetCommitResponse() *TxnOffsetCommitResponse { + var v TxnOffsetCommitResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TxnOffsetCommitResponse. +func (v *TxnOffsetCommitResponse) Default() { +} + +// NewTxnOffsetCommitResponse returns a default TxnOffsetCommitResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewTxnOffsetCommitResponse() TxnOffsetCommitResponse { + var v TxnOffsetCommitResponse + v.Default() + return v +} + +// DescribeACLsRequest describes ACLs. Describing ACLs works on a filter basis: +// anything that matches the filter is described. Note that there are two +// "types" of filters in this request: the resource filter and the entry +// filter, with entries corresponding to users. The first three fields form the +// resource filter, the last four the entry filter. +type DescribeACLsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ResourceType is the type of resource to describe. + ResourceType ACLResourceType + + // ResourceName is the name to filter out. For the CLUSTER resource type, + // this must be "kafka-cluster". + ResourceName *string + + // ResourcePatternType is how ResourceName is understood. + // + // This field has a default of 3. + ResourcePatternType ACLResourcePatternType // v1+ + + // Principal is the user to filter for. In Kafka with the simple authorizor, + // all principals begin with "User:". Pluggable authorizors are allowed, but + // Kafka still expects principals to lead with a principal type ("User") and + // have a colon separating the principal name ("bob" in "User:bob"). + Principal *string + + // Host is a host to filter for. + Host *string + + // Operation is an operation to filter for. + // + // Note that READ, WRITE, DELETE, and ALTER imply DESCRIBE, and ALTER_CONFIGS + // implies DESCRIBE_CONFIGS. + Operation ACLOperation + + // PermissionType is the permission type to filter for. UNKNOWN is 0. + PermissionType ACLPermissionType + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeACLsRequest) Key() int16 { return 29 } +func (*DescribeACLsRequest) MaxVersion() int16 { return 3 } +func (v *DescribeACLsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeACLsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeACLsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeACLsRequest) ResponseKind() Response { + r := &DescribeACLsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeACLsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeACLsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeACLsResponse) + return resp, err +} + +func (v *DescribeACLsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.ResourcePatternType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Principal + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Operation + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PermissionType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeACLsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeACLsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeACLsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + var t ACLResourceType + { + v := b.Int8() + t = ACLResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ResourceName = v + } + if version >= 1 { + var t ACLResourcePatternType + { + v := b.Int8() + t = ACLResourcePatternType(v) + } + v := t + s.ResourcePatternType = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Principal = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Host = v + } + { + var t ACLOperation + { + v := b.Int8() + t = ACLOperation(v) + } + v := t + s.Operation = v + } + { + var t ACLPermissionType + { + v := b.Int8() + t = ACLPermissionType(v) + } + v := t + s.PermissionType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeACLsRequest returns a pointer to a default DescribeACLsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeACLsRequest() *DescribeACLsRequest { + var v DescribeACLsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeACLsRequest. +func (v *DescribeACLsRequest) Default() { + v.ResourcePatternType = 3 +} + +// NewDescribeACLsRequest returns a default DescribeACLsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeACLsRequest() DescribeACLsRequest { + var v DescribeACLsRequest + v.Default() + return v +} + +type DescribeACLsResponseResourceACL struct { + // Principal is who this ACL applies to. + Principal string + + // Host is on which host this ACL applies. + Host string + + // Operation is the operation being described. + Operation ACLOperation + + // PermissionType is the permission being described. + PermissionType ACLPermissionType + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeACLsResponseResourceACL. +func (v *DescribeACLsResponseResourceACL) Default() { +} + +// NewDescribeACLsResponseResourceACL returns a default DescribeACLsResponseResourceACL +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeACLsResponseResourceACL() DescribeACLsResponseResourceACL { + var v DescribeACLsResponseResourceACL + v.Default() + return v +} + +type DescribeACLsResponseResource struct { + // ResourceType is the resource type being described. + ResourceType ACLResourceType + + // ResourceName is the resource name being described. + ResourceName string + + // ResourcePatternType is the pattern type being described. + // + // This field has a default of 3. + ResourcePatternType ACLResourcePatternType // v1+ + + // ACLs contains users / entries being described. + ACLs []DescribeACLsResponseResourceACL + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeACLsResponseResource. +func (v *DescribeACLsResponseResource) Default() { + v.ResourcePatternType = 3 +} + +// NewDescribeACLsResponseResource returns a default DescribeACLsResponseResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeACLsResponseResource() DescribeACLsResponseResource { + var v DescribeACLsResponseResource + v.Default() + return v +} + +// DescribeACLsResponse is a response to a describe acls request. +type DescribeACLsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // ErrorCode is the error code returned on request failure. + // + // SECURITY_DISABLED is returned if there is no authorizer configured on the + // broker. + // + // There can be other authorization failures. + ErrorCode int16 + + // ErrorMessage is a message for an error. + ErrorMessage *string + + // Resources are the describe resources. + Resources []DescribeACLsResponseResource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeACLsResponse) Key() int16 { return 29 } +func (*DescribeACLsResponse) MaxVersion() int16 { return 3 } +func (v *DescribeACLsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeACLsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeACLsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeACLsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *DescribeACLsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DescribeACLsResponse) RequestKind() Request { return &DescribeACLsRequest{Version: v.Version} } + +func (v *DescribeACLsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.ResourcePatternType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ACLs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Principal + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Operation + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PermissionType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeACLsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeACLsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeACLsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeACLsResponseResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ACLResourceType + { + v := b.Int8() + t = ACLResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + if version >= 1 { + var t ACLResourcePatternType + { + v := b.Int8() + t = ACLResourcePatternType(v) + } + v := t + s.ResourcePatternType = v + } + { + v := s.ACLs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeACLsResponseResourceACL, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Principal = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + var t ACLOperation + { + v := b.Int8() + t = ACLOperation(v) + } + v := t + s.Operation = v + } + { + var t ACLPermissionType + { + v := b.Int8() + t = ACLPermissionType(v) + } + v := t + s.PermissionType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ACLs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeACLsResponse returns a pointer to a default DescribeACLsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeACLsResponse() *DescribeACLsResponse { + var v DescribeACLsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeACLsResponse. +func (v *DescribeACLsResponse) Default() { +} + +// NewDescribeACLsResponse returns a default DescribeACLsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeACLsResponse() DescribeACLsResponse { + var v DescribeACLsResponse + v.Default() + return v +} + +type CreateACLsRequestCreation struct { + // ResourceType is the type of resource this acl entry will be on. + // It is invalid to use UNKNOWN or ANY. + ResourceType ACLResourceType + + // ResourceName is the name of the resource this acl entry will be on. + // For CLUSTER, this must be "kafka-cluster". + ResourceName string + + // ResourcePatternType is the pattern type to use for the resource name. + // This cannot be UNKNOWN or MATCH (i.e. this must be LITERAL or PREFIXED). + // The default for pre-Kafka 2.0.0 is effectively LITERAL. + // + // This field has a default of 3. + ResourcePatternType ACLResourcePatternType // v1+ + + // Principal is the user to apply this acl for. With the Kafka simple + // authorizer, this must begin with "User:". + Principal string + + // Host is the host address to use for this acl. Each host to allow + // the principal access from must be specified as a new creation. KIP-252 + // might solve this someday. The special wildcard host "*" allows all hosts. + Host string + + // Operation is the operation this acl is for. This must not be UNKNOWN or + // ANY. + Operation ACLOperation + + // PermissionType is the permission of this acl. This must be either ALLOW + // or DENY. + PermissionType ACLPermissionType + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateACLsRequestCreation. +func (v *CreateACLsRequestCreation) Default() { + v.ResourcePatternType = 3 +} + +// NewCreateACLsRequestCreation returns a default CreateACLsRequestCreation +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateACLsRequestCreation() CreateACLsRequestCreation { + var v CreateACLsRequestCreation + v.Default() + return v +} + +// CreateACLsRequest creates acls. Creating acls can be done as a batch; each +// "creation" will be an acl entry. +// +// See the DescribeACLsRequest documentation for more descriptions of what +// valid values for the fields in this request are. +type CreateACLsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + Creations []CreateACLsRequestCreation + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreateACLsRequest) Key() int16 { return 30 } +func (*CreateACLsRequest) MaxVersion() int16 { return 3 } +func (v *CreateACLsRequest) SetVersion(version int16) { v.Version = version } +func (v *CreateACLsRequest) GetVersion() int16 { return v.Version } +func (v *CreateACLsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *CreateACLsRequest) ResponseKind() Response { + r := &CreateACLsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *CreateACLsRequest) RequestWith(ctx context.Context, r Requestor) (*CreateACLsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*CreateACLsResponse) + return resp, err +} + +func (v *CreateACLsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Creations + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.ResourcePatternType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Principal + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Operation + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PermissionType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateACLsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateACLsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateACLsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Creations + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateACLsRequestCreation, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ACLResourceType + { + v := b.Int8() + t = ACLResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + if version >= 1 { + var t ACLResourcePatternType + { + v := b.Int8() + t = ACLResourcePatternType(v) + } + v := t + s.ResourcePatternType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Principal = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + var t ACLOperation + { + v := b.Int8() + t = ACLOperation(v) + } + v := t + s.Operation = v + } + { + var t ACLPermissionType + { + v := b.Int8() + t = ACLPermissionType(v) + } + v := t + s.PermissionType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Creations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateACLsRequest returns a pointer to a default CreateACLsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateACLsRequest() *CreateACLsRequest { + var v CreateACLsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateACLsRequest. +func (v *CreateACLsRequest) Default() { +} + +// NewCreateACLsRequest returns a default CreateACLsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateACLsRequest() CreateACLsRequest { + var v CreateACLsRequest + v.Default() + return v +} + +type CreateACLsResponseResult struct { + // ErrorCode is an error for this particular creation (index wise). + ErrorCode int16 + + // ErrorMessage is a message for this error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateACLsResponseResult. +func (v *CreateACLsResponseResult) Default() { +} + +// NewCreateACLsResponseResult returns a default CreateACLsResponseResult +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateACLsResponseResult() CreateACLsResponseResult { + var v CreateACLsResponseResult + v.Default() + return v +} + +// CreateACLsResponse is a response for a CreateACLsRequest. +type CreateACLsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Results contains responses to each creation request. + Results []CreateACLsResponseResult + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreateACLsResponse) Key() int16 { return 30 } +func (*CreateACLsResponse) MaxVersion() int16 { return 3 } +func (v *CreateACLsResponse) SetVersion(version int16) { v.Version = version } +func (v *CreateACLsResponse) GetVersion() int16 { return v.Version } +func (v *CreateACLsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *CreateACLsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *CreateACLsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *CreateACLsResponse) RequestKind() Request { return &CreateACLsRequest{Version: v.Version} } + +func (v *CreateACLsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Results + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateACLsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateACLsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateACLsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Results + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateACLsResponseResult, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Results = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateACLsResponse returns a pointer to a default CreateACLsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateACLsResponse() *CreateACLsResponse { + var v CreateACLsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateACLsResponse. +func (v *CreateACLsResponse) Default() { +} + +// NewCreateACLsResponse returns a default CreateACLsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateACLsResponse() CreateACLsResponse { + var v CreateACLsResponse + v.Default() + return v +} + +type DeleteACLsRequestFilter struct { + ResourceType ACLResourceType + + ResourceName *string + + // This field has a default of 3. + ResourcePatternType ACLResourcePatternType // v1+ + + Principal *string + + Host *string + + Operation ACLOperation + + PermissionType ACLPermissionType + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteACLsRequestFilter. +func (v *DeleteACLsRequestFilter) Default() { + v.ResourcePatternType = 3 +} + +// NewDeleteACLsRequestFilter returns a default DeleteACLsRequestFilter +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteACLsRequestFilter() DeleteACLsRequestFilter { + var v DeleteACLsRequestFilter + v.Default() + return v +} + +// DeleteACLsRequest deletes acls. This request works on filters the same way +// that DescribeACLsRequest does. See DescribeACLsRequest for documentation of +// the fields. +type DeleteACLsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Filters are filters for acls to delete. + Filters []DeleteACLsRequestFilter + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteACLsRequest) Key() int16 { return 31 } +func (*DeleteACLsRequest) MaxVersion() int16 { return 3 } +func (v *DeleteACLsRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteACLsRequest) GetVersion() int16 { return v.Version } +func (v *DeleteACLsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteACLsRequest) ResponseKind() Response { + r := &DeleteACLsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteACLsRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteACLsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteACLsResponse) + return resp, err +} + +func (v *DeleteACLsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Filters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.ResourcePatternType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Principal + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Operation + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PermissionType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteACLsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteACLsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteACLsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Filters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteACLsRequestFilter, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ACLResourceType + { + v := b.Int8() + t = ACLResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ResourceName = v + } + if version >= 1 { + var t ACLResourcePatternType + { + v := b.Int8() + t = ACLResourcePatternType(v) + } + v := t + s.ResourcePatternType = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Principal = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Host = v + } + { + var t ACLOperation + { + v := b.Int8() + t = ACLOperation(v) + } + v := t + s.Operation = v + } + { + var t ACLPermissionType + { + v := b.Int8() + t = ACLPermissionType(v) + } + v := t + s.PermissionType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Filters = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteACLsRequest returns a pointer to a default DeleteACLsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteACLsRequest() *DeleteACLsRequest { + var v DeleteACLsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteACLsRequest. +func (v *DeleteACLsRequest) Default() { +} + +// NewDeleteACLsRequest returns a default DeleteACLsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteACLsRequest() DeleteACLsRequest { + var v DeleteACLsRequest + v.Default() + return v +} + +type DeleteACLsResponseResultMatchingACL struct { + // ErrorCode contains an error for this individual acl for this filter. + ErrorCode int16 + + // ErrorMessage is a message for this error. + ErrorMessage *string + + ResourceType ACLResourceType + + ResourceName string + + // This field has a default of 3. + ResourcePatternType ACLResourcePatternType // v1+ + + Principal string + + Host string + + Operation ACLOperation + + PermissionType ACLPermissionType + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteACLsResponseResultMatchingACL. +func (v *DeleteACLsResponseResultMatchingACL) Default() { + v.ResourcePatternType = 3 +} + +// NewDeleteACLsResponseResultMatchingACL returns a default DeleteACLsResponseResultMatchingACL +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteACLsResponseResultMatchingACL() DeleteACLsResponseResultMatchingACL { + var v DeleteACLsResponseResultMatchingACL + v.Default() + return v +} + +type DeleteACLsResponseResult struct { + // ErrorCode is the overall error code for this individual filter. + ErrorCode int16 + + // ErrorMessage is a message for this error. + ErrorMessage *string + + // MatchingACLs contains all acls that were matched for this filter. + MatchingACLs []DeleteACLsResponseResultMatchingACL + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteACLsResponseResult. +func (v *DeleteACLsResponseResult) Default() { +} + +// NewDeleteACLsResponseResult returns a default DeleteACLsResponseResult +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteACLsResponseResult() DeleteACLsResponseResult { + var v DeleteACLsResponseResult + v.Default() + return v +} + +// DeleteACLsResponse is a response for a DeleteACLsRequest. +type DeleteACLsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Results contains a response to each requested filter. + Results []DeleteACLsResponseResult + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteACLsResponse) Key() int16 { return 31 } +func (*DeleteACLsResponse) MaxVersion() int16 { return 3 } +func (v *DeleteACLsResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteACLsResponse) GetVersion() int16 { return v.Version } +func (v *DeleteACLsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteACLsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *DeleteACLsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DeleteACLsResponse) RequestKind() Request { return &DeleteACLsRequest{Version: v.Version} } + +func (v *DeleteACLsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Results + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MatchingACLs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.ResourcePatternType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Principal + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Operation + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PermissionType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteACLsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteACLsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteACLsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Results + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteACLsResponseResult, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.MatchingACLs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteACLsResponseResultMatchingACL, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var t ACLResourceType + { + v := b.Int8() + t = ACLResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + if version >= 1 { + var t ACLResourcePatternType + { + v := b.Int8() + t = ACLResourcePatternType(v) + } + v := t + s.ResourcePatternType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Principal = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + var t ACLOperation + { + v := b.Int8() + t = ACLOperation(v) + } + v := t + s.Operation = v + } + { + var t ACLPermissionType + { + v := b.Int8() + t = ACLPermissionType(v) + } + v := t + s.PermissionType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.MatchingACLs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Results = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteACLsResponse returns a pointer to a default DeleteACLsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteACLsResponse() *DeleteACLsResponse { + var v DeleteACLsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteACLsResponse. +func (v *DeleteACLsResponse) Default() { +} + +// NewDeleteACLsResponse returns a default DeleteACLsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteACLsResponse() DeleteACLsResponse { + var v DeleteACLsResponse + v.Default() + return v +} + +type DescribeConfigsRequestResource struct { + // ResourceType is an enum corresponding to the type of config to describe. + ResourceType ConfigResourceType + + // ResourceName is the name of config to describe. + // + // If the requested type is a topic, this corresponds to a topic name. + // + // If the requested type if a broker, this should either be empty or be + // the ID of the broker this request is issued to. If it is empty, this + // returns all broker configs, but only the dynamic configuration values. + // If a specific ID, this returns all broker config values. + ResourceName string + + // ConfigNames is a list of config entries to return. Null requests all. + ConfigNames []string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsRequestResource. +func (v *DescribeConfigsRequestResource) Default() { +} + +// NewDescribeConfigsRequestResource returns a default DescribeConfigsRequestResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsRequestResource() DescribeConfigsRequestResource { + var v DescribeConfigsRequestResource + v.Default() + return v +} + +// DescribeConfigsRequest issues a request to describe configs that Kafka +// currently has. These are the key/value pairs that one uses to configure +// brokers and topics. +type DescribeConfigsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Resources is a list of resources to describe. + Resources []DescribeConfigsRequestResource + + // IncludeSynonyms signifies whether to return config entry synonyms for + // all config entries. + IncludeSynonyms bool // v1+ + + // IncludeDocumentation signifies whether to return documentation for + // config entries. + IncludeDocumentation bool // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*DescribeConfigsRequest) Key() int16 { return 32 } +func (*DescribeConfigsRequest) MaxVersion() int16 { return 4 } +func (v *DescribeConfigsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeConfigsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeConfigsRequest) IsFlexible() bool { return v.Version >= 4 } +func (v *DescribeConfigsRequest) ResponseKind() Response { + r := &DescribeConfigsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeConfigsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeConfigsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeConfigsResponse) + return resp, err +} + +func (v *DescribeConfigsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ConfigNames + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.IncludeSynonyms + dst = kbin.AppendBool(dst, v) + } + if version >= 3 { + v := v.IncludeDocumentation + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeConfigsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeConfigsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeConfigsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeConfigsRequestResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + { + v := s.ConfigNames + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []string{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.ConfigNames = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + if version >= 1 { + v := b.Bool() + s.IncludeSynonyms = v + } + if version >= 3 { + v := b.Bool() + s.IncludeDocumentation = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeConfigsRequest returns a pointer to a default DescribeConfigsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeConfigsRequest() *DescribeConfigsRequest { + var v DescribeConfigsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsRequest. +func (v *DescribeConfigsRequest) Default() { +} + +// NewDescribeConfigsRequest returns a default DescribeConfigsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsRequest() DescribeConfigsRequest { + var v DescribeConfigsRequest + v.Default() + return v +} + +type DescribeConfigsResponseResourceConfigConfigSynonym struct { + Name string + + Value *string + + Source ConfigSource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsResponseResourceConfigConfigSynonym. +func (v *DescribeConfigsResponseResourceConfigConfigSynonym) Default() { +} + +// NewDescribeConfigsResponseResourceConfigConfigSynonym returns a default DescribeConfigsResponseResourceConfigConfigSynonym +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsResponseResourceConfigConfigSynonym() DescribeConfigsResponseResourceConfigConfigSynonym { + var v DescribeConfigsResponseResourceConfigConfigSynonym + v.Default() + return v +} + +type DescribeConfigsResponseResourceConfig struct { + // Name is a key this entry corresponds to (e.g. segment.bytes). + Name string + + // Value is the value for this config key. If the key is sensitive, + // the value will be null. + Value *string + + // ReadOnly signifies whether this is not a dynamic config option. + // + // Note that this field is not always correct, and you may need to check + // whether the Source is any dynamic enum. See franz-go#91 for more details. + ReadOnly bool + + // IsDefault is whether this is a default config option. This has been + // replaced in favor of Source. + IsDefault bool + + // Source is where this config entry is from. + // + // This field has a default of -1. + Source ConfigSource // v1+ + + // IsSensitive signifies whether this is a sensitive config key, which + // is either a password or an unknown type. + IsSensitive bool + + // ConfigSynonyms contains fallback key/value pairs for this config + // entry, in order of preference. That is, if a config entry is both + // dynamically configured and has a default, the top level return will be + // the dynamic configuration, while its "synonym" will be the default. + ConfigSynonyms []DescribeConfigsResponseResourceConfigConfigSynonym // v1+ + + // ConfigType specifies the configuration data type. + ConfigType ConfigType // v3+ + + // Documentation is optional documentation for the config entry. + Documentation *string // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsResponseResourceConfig. +func (v *DescribeConfigsResponseResourceConfig) Default() { + v.Source = -1 +} + +// NewDescribeConfigsResponseResourceConfig returns a default DescribeConfigsResponseResourceConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsResponseResourceConfig() DescribeConfigsResponseResourceConfig { + var v DescribeConfigsResponseResourceConfig + v.Default() + return v +} + +type DescribeConfigsResponseResource struct { + // ErrorCode is the error code returned for describing configs. + // + // INVALID_REQUEST is returned if asking to descibe an invalid resource + // type. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if asking to describe broker + // configs but the client is not authorized to do so. + // + // TOPIC_AUTHORIZATION_FAILED is returned if asking to describe topic + // configs but the client is not authorized to do so. + // + // INVALID_TOPIC_EXCEPTION is returned if the requested topic was invalid. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the requested topic. + ErrorCode int16 + + // ErrorMessage is an informative message if the describe config failed. + ErrorMessage *string + + // ResourceType is the enum corresponding to the type of described config. + ResourceType ConfigResourceType + + // ResourceName is the name corresponding to the describe config request. + ResourceName string + + // Configs contains information about key/value config pairs for + // the requested resource. + Configs []DescribeConfigsResponseResourceConfig + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsResponseResource. +func (v *DescribeConfigsResponseResource) Default() { +} + +// NewDescribeConfigsResponseResource returns a default DescribeConfigsResponseResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsResponseResource() DescribeConfigsResponseResource { + var v DescribeConfigsResponseResource + v.Default() + return v +} + +// DescribeConfigsResponse is returned from a DescribeConfigsRequest. +type DescribeConfigsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 2. + ThrottleMillis int32 + + // Resources are responses for each resource in the describe config request. + Resources []DescribeConfigsResponseResource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v4+ +} + +func (*DescribeConfigsResponse) Key() int16 { return 32 } +func (*DescribeConfigsResponse) MaxVersion() int16 { return 4 } +func (v *DescribeConfigsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeConfigsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeConfigsResponse) IsFlexible() bool { return v.Version >= 4 } +func (v *DescribeConfigsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 2 } +func (v *DescribeConfigsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeConfigsResponse) RequestKind() Request { + return &DescribeConfigsRequest{Version: v.Version} +} + +func (v *DescribeConfigsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ReadOnly + dst = kbin.AppendBool(dst, v) + } + if version >= 0 && version <= 0 { + v := v.IsDefault + dst = kbin.AppendBool(dst, v) + } + if version >= 1 { + v := v.Source + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.IsSensitive + dst = kbin.AppendBool(dst, v) + } + if version >= 1 { + v := v.ConfigSynonyms + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Source + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 3 { + v := v.ConfigType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + if version >= 3 { + v := v.Documentation + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeConfigsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeConfigsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeConfigsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 4 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeConfigsResponseResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeConfigsResponseResourceConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + { + v := b.Bool() + s.ReadOnly = v + } + if version >= 0 && version <= 0 { + v := b.Bool() + s.IsDefault = v + } + if version >= 1 { + var t ConfigSource + { + v := b.Int8() + t = ConfigSource(v) + } + v := t + s.Source = v + } + { + v := b.Bool() + s.IsSensitive = v + } + if version >= 1 { + v := s.ConfigSynonyms + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeConfigsResponseResourceConfigConfigSynonym, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + { + var t ConfigSource + { + v := b.Int8() + t = ConfigSource(v) + } + v := t + s.Source = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ConfigSynonyms = v + } + if version >= 3 { + var t ConfigType + { + v := b.Int8() + t = ConfigType(v) + } + v := t + s.ConfigType = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Documentation = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeConfigsResponse returns a pointer to a default DescribeConfigsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeConfigsResponse() *DescribeConfigsResponse { + var v DescribeConfigsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeConfigsResponse. +func (v *DescribeConfigsResponse) Default() { +} + +// NewDescribeConfigsResponse returns a default DescribeConfigsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeConfigsResponse() DescribeConfigsResponse { + var v DescribeConfigsResponse + v.Default() + return v +} + +type AlterConfigsRequestResourceConfig struct { + // Name is a key to set (e.g. segment.bytes). + Name string + + // Value is a value to set for the key (e.g. 10). + Value *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterConfigsRequestResourceConfig. +func (v *AlterConfigsRequestResourceConfig) Default() { +} + +// NewAlterConfigsRequestResourceConfig returns a default AlterConfigsRequestResourceConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterConfigsRequestResourceConfig() AlterConfigsRequestResourceConfig { + var v AlterConfigsRequestResourceConfig + v.Default() + return v +} + +type AlterConfigsRequestResource struct { + // ResourceType is an enum corresponding to the type of config to alter. + // The only two valid values are 2 (for topic) and 4 (for broker). + ResourceType ConfigResourceType + + // ResourceName is the name of config to alter. + // + // If the requested type is a topic, this corresponds to a topic name. + // + // If the requested type if a broker, this should either be empty or be + // the ID of the broker this request is issued to. If it is empty, this + // updates all broker configs. If a specific ID, this updates just the + // broker. Using a specific ID also ensures that brokers reload config + // or secret files even if the file path has not changed. Lastly, password + // config options can only be defined on a per broker basis. + // + // If the type is broker logger, this must be a broker ID. + ResourceName string + + // Configs contains key/value config pairs to set on the resource. + Configs []AlterConfigsRequestResourceConfig + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterConfigsRequestResource. +func (v *AlterConfigsRequestResource) Default() { +} + +// NewAlterConfigsRequestResource returns a default AlterConfigsRequestResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterConfigsRequestResource() AlterConfigsRequestResource { + var v AlterConfigsRequestResource + v.Default() + return v +} + +// AlterConfigsRequest issues a request to alter either topic or broker +// configs. +// +// Note that to alter configs, you must specify the whole config on every +// request. All existing non-static values will be removed. This means that +// to add one key/value to a config, you must describe the config and then +// issue an alter request with the current config with the new key value. +// This also means that dynamic sensitive values, which are not returned +// in describe configs, will be lost. +// +// To fix this problem, the AlterConfigs request / response was deprecated +// in Kafka 2.3.0 in favor of the new IncrementalAlterConfigs request / response. +// See KIP-339 for more details. +type AlterConfigsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Resources is an array of configs to alter. + Resources []AlterConfigsRequestResource + + // ValidateOnly validates the request but does not apply it. + ValidateOnly bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*AlterConfigsRequest) Key() int16 { return 33 } +func (*AlterConfigsRequest) MaxVersion() int16 { return 2 } +func (v *AlterConfigsRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterConfigsRequest) GetVersion() int16 { return v.Version } +func (v *AlterConfigsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *AlterConfigsRequest) ResponseKind() Response { + r := &AlterConfigsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterConfigsRequest) RequestWith(ctx context.Context, r Requestor) (*AlterConfigsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterConfigsResponse) + return resp, err +} + +func (v *AlterConfigsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterConfigsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterConfigsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterConfigsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterConfigsRequestResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterConfigsRequestResourceConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterConfigsRequest returns a pointer to a default AlterConfigsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterConfigsRequest() *AlterConfigsRequest { + var v AlterConfigsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterConfigsRequest. +func (v *AlterConfigsRequest) Default() { +} + +// NewAlterConfigsRequest returns a default AlterConfigsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterConfigsRequest() AlterConfigsRequest { + var v AlterConfigsRequest + v.Default() + return v +} + +type AlterConfigsResponseResource struct { + // ErrorCode is the error code returned for altering configs. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if asking to alter broker + // configs but the client is not authorized to do so. + // + // TOPIC_AUTHORIZATION_FAILED is returned if asking to alter topic + // configs but the client is not authorized to do so. + // + // INVALID_TOPIC_EXCEPTION is returned if the requested topic was invalid. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the requested topic. + // + // INVALID_REQUEST is returned if the requested config is invalid or if + // asking Kafka to alter an invalid resource. + ErrorCode int16 + + // ErrorMessage is an informative message if the alter config failed. + ErrorMessage *string + + // ResourceType is the enum corresponding to the type of altered config. + ResourceType ConfigResourceType + + // ResourceName is the name corresponding to the alter config request. + ResourceName string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterConfigsResponseResource. +func (v *AlterConfigsResponseResource) Default() { +} + +// NewAlterConfigsResponseResource returns a default AlterConfigsResponseResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterConfigsResponseResource() AlterConfigsResponseResource { + var v AlterConfigsResponseResource + v.Default() + return v +} + +// AlterConfigsResponse is returned from an AlterConfigsRequest. +type AlterConfigsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Resources are responses for each resource in the alter request. + Resources []AlterConfigsResponseResource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*AlterConfigsResponse) Key() int16 { return 33 } +func (*AlterConfigsResponse) MaxVersion() int16 { return 2 } +func (v *AlterConfigsResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterConfigsResponse) GetVersion() int16 { return v.Version } +func (v *AlterConfigsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *AlterConfigsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *AlterConfigsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *AlterConfigsResponse) RequestKind() Request { return &AlterConfigsRequest{Version: v.Version} } + +func (v *AlterConfigsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterConfigsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterConfigsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterConfigsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterConfigsResponseResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterConfigsResponse returns a pointer to a default AlterConfigsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterConfigsResponse() *AlterConfigsResponse { + var v AlterConfigsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterConfigsResponse. +func (v *AlterConfigsResponse) Default() { +} + +// NewAlterConfigsResponse returns a default AlterConfigsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterConfigsResponse() AlterConfigsResponse { + var v AlterConfigsResponse + v.Default() + return v +} + +type AlterReplicaLogDirsRequestDirTopic struct { + // Topic is a topic to move. + Topic string + + // Partitions contains partitions for the topic to move. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsRequestDirTopic. +func (v *AlterReplicaLogDirsRequestDirTopic) Default() { +} + +// NewAlterReplicaLogDirsRequestDirTopic returns a default AlterReplicaLogDirsRequestDirTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsRequestDirTopic() AlterReplicaLogDirsRequestDirTopic { + var v AlterReplicaLogDirsRequestDirTopic + v.Default() + return v +} + +type AlterReplicaLogDirsRequestDir struct { + // Dir is an absolute path where everything listed below should + // end up. + Dir string + + // Topics contains topics to move to the above log directory. + Topics []AlterReplicaLogDirsRequestDirTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsRequestDir. +func (v *AlterReplicaLogDirsRequestDir) Default() { +} + +// NewAlterReplicaLogDirsRequestDir returns a default AlterReplicaLogDirsRequestDir +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsRequestDir() AlterReplicaLogDirsRequestDir { + var v AlterReplicaLogDirsRequestDir + v.Default() + return v +} + +// AlterReplicaLogDirsRequest requests for log directories to be moved +// within Kafka. +// +// This is primarily useful for moving directories between disks. +type AlterReplicaLogDirsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Dirs contains absolute paths of where you want things to end up. + Dirs []AlterReplicaLogDirsRequestDir + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*AlterReplicaLogDirsRequest) Key() int16 { return 34 } +func (*AlterReplicaLogDirsRequest) MaxVersion() int16 { return 2 } +func (v *AlterReplicaLogDirsRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterReplicaLogDirsRequest) GetVersion() int16 { return v.Version } +func (v *AlterReplicaLogDirsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *AlterReplicaLogDirsRequest) ResponseKind() Response { + r := &AlterReplicaLogDirsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterReplicaLogDirsRequest) RequestWith(ctx context.Context, r Requestor) (*AlterReplicaLogDirsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterReplicaLogDirsResponse) + return resp, err +} + +func (v *AlterReplicaLogDirsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Dirs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Dir + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterReplicaLogDirsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterReplicaLogDirsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterReplicaLogDirsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Dirs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterReplicaLogDirsRequestDir, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Dir = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterReplicaLogDirsRequestDirTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Dirs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterReplicaLogDirsRequest returns a pointer to a default AlterReplicaLogDirsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterReplicaLogDirsRequest() *AlterReplicaLogDirsRequest { + var v AlterReplicaLogDirsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsRequest. +func (v *AlterReplicaLogDirsRequest) Default() { +} + +// NewAlterReplicaLogDirsRequest returns a default AlterReplicaLogDirsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsRequest() AlterReplicaLogDirsRequest { + var v AlterReplicaLogDirsRequest + v.Default() + return v +} + +type AlterReplicaLogDirsResponseTopicPartition struct { + // Partition is the partition this array slot corresponds to. + Partition int32 + + // CLUSTER_AUTHORIZATION_FAILED is returned if the client is not + // authorized to alter replica dirs. + // + // LOG_DIR_NOT_FOUND is returned when the requested log directory + // is not in the broker config. + // + // KAFKA_STORAGE_EXCEPTION is returned when destination directory or + // requested replica is offline. + // + // REPLICA_NOT_AVAILABLE is returned if the replica does not exist + // yet. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsResponseTopicPartition. +func (v *AlterReplicaLogDirsResponseTopicPartition) Default() { +} + +// NewAlterReplicaLogDirsResponseTopicPartition returns a default AlterReplicaLogDirsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsResponseTopicPartition() AlterReplicaLogDirsResponseTopicPartition { + var v AlterReplicaLogDirsResponseTopicPartition + v.Default() + return v +} + +type AlterReplicaLogDirsResponseTopic struct { + // Topic is the topic this array slot corresponds to. + Topic string + + // Partitions contains responses to each partition that was requested + // to move. + Partitions []AlterReplicaLogDirsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsResponseTopic. +func (v *AlterReplicaLogDirsResponseTopic) Default() { +} + +// NewAlterReplicaLogDirsResponseTopic returns a default AlterReplicaLogDirsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsResponseTopic() AlterReplicaLogDirsResponseTopic { + var v AlterReplicaLogDirsResponseTopic + v.Default() + return v +} + +// AlterReplicaLogDirsResponse is returned from an AlterReplicaLogDirsRequest. +type AlterReplicaLogDirsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Topics contains responses to each topic that had partitions requested + // for moving. + Topics []AlterReplicaLogDirsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*AlterReplicaLogDirsResponse) Key() int16 { return 34 } +func (*AlterReplicaLogDirsResponse) MaxVersion() int16 { return 2 } +func (v *AlterReplicaLogDirsResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterReplicaLogDirsResponse) GetVersion() int16 { return v.Version } +func (v *AlterReplicaLogDirsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *AlterReplicaLogDirsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *AlterReplicaLogDirsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AlterReplicaLogDirsResponse) RequestKind() Request { + return &AlterReplicaLogDirsRequest{Version: v.Version} +} + +func (v *AlterReplicaLogDirsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterReplicaLogDirsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterReplicaLogDirsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterReplicaLogDirsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterReplicaLogDirsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterReplicaLogDirsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterReplicaLogDirsResponse returns a pointer to a default AlterReplicaLogDirsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterReplicaLogDirsResponse() *AlterReplicaLogDirsResponse { + var v AlterReplicaLogDirsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterReplicaLogDirsResponse. +func (v *AlterReplicaLogDirsResponse) Default() { +} + +// NewAlterReplicaLogDirsResponse returns a default AlterReplicaLogDirsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterReplicaLogDirsResponse() AlterReplicaLogDirsResponse { + var v AlterReplicaLogDirsResponse + v.Default() + return v +} + +type DescribeLogDirsRequestTopic struct { + // Topic is a topic to describe the log dir of. + Topic string + + // Partitions contains topic partitions to describe the log dirs of. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsRequestTopic. +func (v *DescribeLogDirsRequestTopic) Default() { +} + +// NewDescribeLogDirsRequestTopic returns a default DescribeLogDirsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsRequestTopic() DescribeLogDirsRequestTopic { + var v DescribeLogDirsRequestTopic + v.Default() + return v +} + +// DescribeLogDirsRequest requests directory information for topic partitions. +// This request was added in support of KIP-113. +type DescribeLogDirsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics is an array of topics to describe the log dirs of. If this is + // null, the response includes all topics and all of their partitions. + Topics []DescribeLogDirsRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeLogDirsRequest) Key() int16 { return 35 } +func (*DescribeLogDirsRequest) MaxVersion() int16 { return 4 } +func (v *DescribeLogDirsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeLogDirsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeLogDirsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeLogDirsRequest) ResponseKind() Response { + r := &DescribeLogDirsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeLogDirsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeLogDirsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeLogDirsResponse) + return resp, err +} + +func (v *DescribeLogDirsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeLogDirsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeLogDirsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeLogDirsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []DescribeLogDirsRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeLogDirsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeLogDirsRequest returns a pointer to a default DescribeLogDirsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeLogDirsRequest() *DescribeLogDirsRequest { + var v DescribeLogDirsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsRequest. +func (v *DescribeLogDirsRequest) Default() { +} + +// NewDescribeLogDirsRequest returns a default DescribeLogDirsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsRequest() DescribeLogDirsRequest { + var v DescribeLogDirsRequest + v.Default() + return v +} + +type DescribeLogDirsResponseDirTopicPartition struct { + // Partition is a partition ID. + Partition int32 + + // Size is the total size of the log sements of this partition, in bytes. + Size int64 + + // OffsetLag is how far behind the log end offset is compared to + // the partition's high watermark (if this is the current log for + // the partition) or compared to the current replica's log end + // offset (if this is the future log for the patition). + // + // The math is, + // + // if IsFuture, localLogEndOffset - futurelogEndOffset. + // + // otherwise, max(localHighWatermark - logEndOffset, 0). + OffsetLag int64 + + // IsFuture is true if this replica was created by an + // AlterReplicaLogDirsRequest and will replace the current log of the + // replica in the future. + IsFuture bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsResponseDirTopicPartition. +func (v *DescribeLogDirsResponseDirTopicPartition) Default() { +} + +// NewDescribeLogDirsResponseDirTopicPartition returns a default DescribeLogDirsResponseDirTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsResponseDirTopicPartition() DescribeLogDirsResponseDirTopicPartition { + var v DescribeLogDirsResponseDirTopicPartition + v.Default() + return v +} + +type DescribeLogDirsResponseDirTopic struct { + // Topic is the name of a Kafka topic. + Topic string + + // Partitions is the set of queried partitions for a topic that are + // within a log directory. + Partitions []DescribeLogDirsResponseDirTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsResponseDirTopic. +func (v *DescribeLogDirsResponseDirTopic) Default() { +} + +// NewDescribeLogDirsResponseDirTopic returns a default DescribeLogDirsResponseDirTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsResponseDirTopic() DescribeLogDirsResponseDirTopic { + var v DescribeLogDirsResponseDirTopic + v.Default() + return v +} + +type DescribeLogDirsResponseDir struct { + // ErrorCode is the error code returned for describing log dirs. + // + // KAFKA_STORAGE_ERROR is returned if the log directory is offline. + ErrorCode int16 + + // Dir is the absolute path of a log directory. + Dir string + + // Topics is an array of topics within a log directory. + Topics []DescribeLogDirsResponseDirTopic + + // TotalBytes is the total size in bytes of the volume the log directory is + // in. + // + // This field has a default of -1. + TotalBytes int64 // v4+ + + // UsableBytes is the usable size in bytes of the volume the log directory + // is in. + // + // This field has a default of -1. + UsableBytes int64 // v4+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsResponseDir. +func (v *DescribeLogDirsResponseDir) Default() { + v.TotalBytes = -1 + v.UsableBytes = -1 +} + +// NewDescribeLogDirsResponseDir returns a default DescribeLogDirsResponseDir +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsResponseDir() DescribeLogDirsResponseDir { + var v DescribeLogDirsResponseDir + v.Default() + return v +} + +// DescribeLogDirsResponse is returned from a DescribeLogDirsRequest. +type DescribeLogDirsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // The error code, or 0 if there was no error. + ErrorCode int16 // v3+ + + // Dirs pairs log directories with the topics and partitions that are + // stored in those directories. + Dirs []DescribeLogDirsResponseDir + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeLogDirsResponse) Key() int16 { return 35 } +func (*DescribeLogDirsResponse) MaxVersion() int16 { return 4 } +func (v *DescribeLogDirsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeLogDirsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeLogDirsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeLogDirsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *DescribeLogDirsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeLogDirsResponse) RequestKind() Request { + return &DescribeLogDirsRequest{Version: v.Version} +} + +func (v *DescribeLogDirsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 3 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Dirs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Dir + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Size + dst = kbin.AppendInt64(dst, v) + } + { + v := v.OffsetLag + dst = kbin.AppendInt64(dst, v) + } + { + v := v.IsFuture + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 4 { + v := v.TotalBytes + dst = kbin.AppendInt64(dst, v) + } + if version >= 4 { + v := v.UsableBytes + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeLogDirsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeLogDirsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeLogDirsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 3 { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Dirs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeLogDirsResponseDir, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Dir = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeLogDirsResponseDirTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeLogDirsResponseDirTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Size = v + } + { + v := b.Int64() + s.OffsetLag = v + } + { + v := b.Bool() + s.IsFuture = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 4 { + v := b.Int64() + s.TotalBytes = v + } + if version >= 4 { + v := b.Int64() + s.UsableBytes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Dirs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeLogDirsResponse returns a pointer to a default DescribeLogDirsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeLogDirsResponse() *DescribeLogDirsResponse { + var v DescribeLogDirsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeLogDirsResponse. +func (v *DescribeLogDirsResponse) Default() { +} + +// NewDescribeLogDirsResponse returns a default DescribeLogDirsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeLogDirsResponse() DescribeLogDirsResponse { + var v DescribeLogDirsResponse + v.Default() + return v +} + +// SASLAuthenticate continues a sasl authentication flow. Prior to Kafka 1.0.0, +// authenticating with sasl involved sending raw blobs of data back and forth. +// After, those blobs are wrapped in a SASLAuthenticateRequest The benefit of +// this wrapping is that Kafka can indicate errors in the response, rather than +// just closing the connection. Additionally, the response allows for further +// extension fields. +type SASLAuthenticateRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // SASLAuthBytes contains bytes for a SASL client request. + SASLAuthBytes []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*SASLAuthenticateRequest) Key() int16 { return 36 } +func (*SASLAuthenticateRequest) MaxVersion() int16 { return 2 } +func (v *SASLAuthenticateRequest) SetVersion(version int16) { v.Version = version } +func (v *SASLAuthenticateRequest) GetVersion() int16 { return v.Version } +func (v *SASLAuthenticateRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *SASLAuthenticateRequest) ResponseKind() Response { + r := &SASLAuthenticateResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *SASLAuthenticateRequest) RequestWith(ctx context.Context, r Requestor) (*SASLAuthenticateResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*SASLAuthenticateResponse) + return resp, err +} + +func (v *SASLAuthenticateRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.SASLAuthBytes + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *SASLAuthenticateRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SASLAuthenticateRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SASLAuthenticateRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.SASLAuthBytes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrSASLAuthenticateRequest returns a pointer to a default SASLAuthenticateRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSASLAuthenticateRequest() *SASLAuthenticateRequest { + var v SASLAuthenticateRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SASLAuthenticateRequest. +func (v *SASLAuthenticateRequest) Default() { +} + +// NewSASLAuthenticateRequest returns a default SASLAuthenticateRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewSASLAuthenticateRequest() SASLAuthenticateRequest { + var v SASLAuthenticateRequest + v.Default() + return v +} + +// SASLAuthenticateResponse is returned for a SASLAuthenticateRequest. +type SASLAuthenticateResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is a potential error. + ErrorCode int16 + + // ErrorMessage can contain a message for an error. + ErrorMessage *string + + // SASLAuthBytes is the server challenge continuing SASL flow. + SASLAuthBytes []byte + + // SessionLifetimeMillis, added in Kafka 2.2.0, is how long the SASL + // authentication is valid for. This timeout is only enforced if the request + // was v1. After this timeout, Kafka expects the next bytes on the wire to + // begin reauthentication. Otherwise, Kafka closes the connection. + SessionLifetimeMillis int64 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*SASLAuthenticateResponse) Key() int16 { return 36 } +func (*SASLAuthenticateResponse) MaxVersion() int16 { return 2 } +func (v *SASLAuthenticateResponse) SetVersion(version int16) { v.Version = version } +func (v *SASLAuthenticateResponse) GetVersion() int16 { return v.Version } +func (v *SASLAuthenticateResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *SASLAuthenticateResponse) RequestKind() Request { + return &SASLAuthenticateRequest{Version: v.Version} +} + +func (v *SASLAuthenticateResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.SASLAuthBytes + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if version >= 1 { + v := v.SessionLifetimeMillis + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *SASLAuthenticateResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *SASLAuthenticateResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *SASLAuthenticateResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.SASLAuthBytes = v + } + if version >= 1 { + v := b.Int64() + s.SessionLifetimeMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrSASLAuthenticateResponse returns a pointer to a default SASLAuthenticateResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrSASLAuthenticateResponse() *SASLAuthenticateResponse { + var v SASLAuthenticateResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to SASLAuthenticateResponse. +func (v *SASLAuthenticateResponse) Default() { +} + +// NewSASLAuthenticateResponse returns a default SASLAuthenticateResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewSASLAuthenticateResponse() SASLAuthenticateResponse { + var v SASLAuthenticateResponse + v.Default() + return v +} + +type CreatePartitionsRequestTopicAssignment struct { + // Replicas are replicas to assign a new partition to. + Replicas []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreatePartitionsRequestTopicAssignment. +func (v *CreatePartitionsRequestTopicAssignment) Default() { +} + +// NewCreatePartitionsRequestTopicAssignment returns a default CreatePartitionsRequestTopicAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreatePartitionsRequestTopicAssignment() CreatePartitionsRequestTopicAssignment { + var v CreatePartitionsRequestTopicAssignment + v.Default() + return v +} + +type CreatePartitionsRequestTopic struct { + // Topic is a topic for which to create additional partitions for. + Topic string + + // Count is the final count of partitions this topic must have after this + // request. This must be greater than the current number of partitions. + Count int32 + + // Assignment is a two-level array, the first corresponding to new + // partitions, the second contining broker IDs for where new partition + // replicas should live. + // + // The second level, the replicas, cannot have duplicate broker IDs (i.e. + // you cannot replicate a single partition twice on the same broker). + // Additionally, the number of replicas must match the current number of + // replicas per partition on the topic. + // + // The first level's length must be equal to the delta of Count and the + // current number of partitions. + Assignment []CreatePartitionsRequestTopicAssignment + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreatePartitionsRequestTopic. +func (v *CreatePartitionsRequestTopic) Default() { +} + +// NewCreatePartitionsRequestTopic returns a default CreatePartitionsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreatePartitionsRequestTopic() CreatePartitionsRequestTopic { + var v CreatePartitionsRequestTopic + v.Default() + return v +} + +// CreatePartitionsRequest creates additional partitions for topics. +type CreatePartitionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics contains topics to create partitions for. + Topics []CreatePartitionsRequestTopic + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 15000. + TimeoutMillis int32 + + // ValidateOnly is makes this request a dry-run; everything is validated but + // no partitions are actually created. + ValidateOnly bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreatePartitionsRequest) Key() int16 { return 37 } +func (*CreatePartitionsRequest) MaxVersion() int16 { return 3 } +func (v *CreatePartitionsRequest) SetVersion(version int16) { v.Version = version } +func (v *CreatePartitionsRequest) GetVersion() int16 { return v.Version } +func (v *CreatePartitionsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *CreatePartitionsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *CreatePartitionsRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *CreatePartitionsRequest) IsAdminRequest() {} +func (v *CreatePartitionsRequest) ResponseKind() Response { + r := &CreatePartitionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *CreatePartitionsRequest) RequestWith(ctx context.Context, r Requestor) (*CreatePartitionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*CreatePartitionsResponse) + return resp, err +} + +func (v *CreatePartitionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Count + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Assignment + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreatePartitionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreatePartitionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreatePartitionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreatePartitionsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Count = v + } + { + v := s.Assignment + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []CreatePartitionsRequestTopicAssignment{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreatePartitionsRequestTopicAssignment, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Assignment = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreatePartitionsRequest returns a pointer to a default CreatePartitionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreatePartitionsRequest() *CreatePartitionsRequest { + var v CreatePartitionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreatePartitionsRequest. +func (v *CreatePartitionsRequest) Default() { + v.TimeoutMillis = 15000 +} + +// NewCreatePartitionsRequest returns a default CreatePartitionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreatePartitionsRequest() CreatePartitionsRequest { + var v CreatePartitionsRequest + v.Default() + return v +} + +type CreatePartitionsResponseTopic struct { + // Topic is the topic that partitions were requested to be made for. + Topic string + + // ErrorCode is the error code returned for each topic in the request. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // to create partitions for a topic. + // + // INVALID_REQUEST is returned for duplicate topics in the request. + // + // INVALID_TOPIC_EXCEPTION is returned if the topic is queued for deletion. + // + // REASSIGNMENT_IN_PROGRESS is returned if the request was issued while + // partitions were being reassigned. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the topic for which to create partitions. + // + // INVALID_PARTITIONS is returned if the request would drop the total + // count of partitions down, or if the request would not add any more + // partitions, or if the request uses unknown brokers, or if the request + // assigns a different number of brokers than the increase in the + // partition count. + ErrorCode int16 + + // ErrorMessage is an informative message if the topic creation failed. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreatePartitionsResponseTopic. +func (v *CreatePartitionsResponseTopic) Default() { +} + +// NewCreatePartitionsResponseTopic returns a default CreatePartitionsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreatePartitionsResponseTopic() CreatePartitionsResponseTopic { + var v CreatePartitionsResponseTopic + v.Default() + return v +} + +// CreatePartitionsResponse is returned from a CreatePartitionsRequest. +type CreatePartitionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Topics is a response to each topic in the creation request. + Topics []CreatePartitionsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreatePartitionsResponse) Key() int16 { return 37 } +func (*CreatePartitionsResponse) MaxVersion() int16 { return 3 } +func (v *CreatePartitionsResponse) SetVersion(version int16) { v.Version = version } +func (v *CreatePartitionsResponse) GetVersion() int16 { return v.Version } +func (v *CreatePartitionsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *CreatePartitionsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *CreatePartitionsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *CreatePartitionsResponse) RequestKind() Request { + return &CreatePartitionsRequest{Version: v.Version} +} + +func (v *CreatePartitionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreatePartitionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreatePartitionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreatePartitionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreatePartitionsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreatePartitionsResponse returns a pointer to a default CreatePartitionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreatePartitionsResponse() *CreatePartitionsResponse { + var v CreatePartitionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreatePartitionsResponse. +func (v *CreatePartitionsResponse) Default() { +} + +// NewCreatePartitionsResponse returns a default CreatePartitionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreatePartitionsResponse() CreatePartitionsResponse { + var v CreatePartitionsResponse + v.Default() + return v +} + +type CreateDelegationTokenRequestRenewer struct { + // PrincipalType is the "type" this principal is. This must be "User". + PrincipalType string + + // PrincipalName is the user name allowed to renew the returned token. + PrincipalName string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateDelegationTokenRequestRenewer. +func (v *CreateDelegationTokenRequestRenewer) Default() { +} + +// NewCreateDelegationTokenRequestRenewer returns a default CreateDelegationTokenRequestRenewer +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateDelegationTokenRequestRenewer() CreateDelegationTokenRequestRenewer { + var v CreateDelegationTokenRequestRenewer + v.Default() + return v +} + +// CreateDelegationTokenRequest issues a request to create a delegation token. +// +// Creating delegation tokens allows for an (ideally) quicker and easier method +// of enabling authorization for a wide array of clients. Rather than having to +// manage many passwords external to Kafka, you only need to manage a few +// accounts and use those to create delegation tokens per client. +// +// Note that delegation tokens inherit the same ACLs as the user creating the +// token. Thus, if you want to properly scope ACLs, you should not create +// delegation tokens with admin accounts. +// +// Delegation tokens live inside of Kafka and use SASL SCRAM-SHA-256 for +// authorization. +type CreateDelegationTokenRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The principal type of the owner of the token. If null, this defaults + // to the token request principal. + OwnerPrincipalType *string // v3+ + + // Principal name of the owner of the token. If null, this defaults to + // the token request principal. + OwnerPrincipalName *string // v3+ + + // Renewers is a list of who can renew this delegation token. If empty, the + // default is the principal (user) who created the token. + Renewers []CreateDelegationTokenRequestRenewer + + // MaxLifetimeMillis is how long this delegation token will be valid for. + // If -1, the default will be the server's delegation.token.max.lifetime.ms. + MaxLifetimeMillis int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreateDelegationTokenRequest) Key() int16 { return 38 } +func (*CreateDelegationTokenRequest) MaxVersion() int16 { return 3 } +func (v *CreateDelegationTokenRequest) SetVersion(version int16) { v.Version = version } +func (v *CreateDelegationTokenRequest) GetVersion() int16 { return v.Version } +func (v *CreateDelegationTokenRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *CreateDelegationTokenRequest) ResponseKind() Response { + r := &CreateDelegationTokenResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *CreateDelegationTokenRequest) RequestWith(ctx context.Context, r Requestor) (*CreateDelegationTokenResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*CreateDelegationTokenResponse) + return resp, err +} + +func (v *CreateDelegationTokenRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + if version >= 3 { + v := v.OwnerPrincipalType + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 3 { + v := v.OwnerPrincipalName + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Renewers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.PrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.PrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.MaxLifetimeMillis + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateDelegationTokenRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateDelegationTokenRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateDelegationTokenRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.OwnerPrincipalType = v + } + if version >= 3 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.OwnerPrincipalName = v + } + { + v := s.Renewers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]CreateDelegationTokenRequestRenewer, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalName = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Renewers = v + } + { + v := b.Int64() + s.MaxLifetimeMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateDelegationTokenRequest returns a pointer to a default CreateDelegationTokenRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateDelegationTokenRequest() *CreateDelegationTokenRequest { + var v CreateDelegationTokenRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateDelegationTokenRequest. +func (v *CreateDelegationTokenRequest) Default() { +} + +// NewCreateDelegationTokenRequest returns a default CreateDelegationTokenRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateDelegationTokenRequest() CreateDelegationTokenRequest { + var v CreateDelegationTokenRequest + v.Default() + return v +} + +// CreateDelegationTokenResponse is a response to a CreateDelegationTokenRequest. +type CreateDelegationTokenResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is any error that caused the request to fail. + ErrorCode int16 + + // PrincipalType is the type of principal that granted this delegation token. + // This will always be "User" with the simple authorizer. + PrincipalType string + + // PrincipalName is the name of the principal that granted this delegation + // token. + PrincipalName string + + // The principal type of the requester of the token. + TokenRequesterPrincipalType string // v3+ + + // The principal name of the requester token. + TokenRequesterPrincipalName string // v3+ + + // IssueTimestamp is the millisecond timestamp this delegation token was + // issued. + IssueTimestamp int64 + + // ExpiryTimestamp is the millisecond timestamp this token will expire. The + // token can be renewed up to MaxTimestamp, past which point, it will be + // invalid. The Kafka default is 24h. + ExpiryTimestamp int64 + + // MaxTimestamp is the millisecond timestamp past which this token cannot + // be renewed. + MaxTimestamp int64 + + // TokenID is the ID of this token; this will be used as the username for + // scram authentication. + TokenID string + + // HMAC is the password of this token; this will be used as the password for + // scram authentication. + HMAC []byte + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*CreateDelegationTokenResponse) Key() int16 { return 38 } +func (*CreateDelegationTokenResponse) MaxVersion() int16 { return 3 } +func (v *CreateDelegationTokenResponse) SetVersion(version int16) { v.Version = version } +func (v *CreateDelegationTokenResponse) GetVersion() int16 { return v.Version } +func (v *CreateDelegationTokenResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *CreateDelegationTokenResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *CreateDelegationTokenResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *CreateDelegationTokenResponse) RequestKind() Request { + return &CreateDelegationTokenRequest{Version: v.Version} +} + +func (v *CreateDelegationTokenResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.PrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.PrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.TokenRequesterPrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.TokenRequesterPrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.IssueTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ExpiryTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.MaxTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.TokenID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.HMAC + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *CreateDelegationTokenResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *CreateDelegationTokenResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *CreateDelegationTokenResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalName = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenRequesterPrincipalType = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenRequesterPrincipalName = v + } + { + v := b.Int64() + s.IssueTimestamp = v + } + { + v := b.Int64() + s.ExpiryTimestamp = v + } + { + v := b.Int64() + s.MaxTimestamp = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenID = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.HMAC = v + } + { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrCreateDelegationTokenResponse returns a pointer to a default CreateDelegationTokenResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrCreateDelegationTokenResponse() *CreateDelegationTokenResponse { + var v CreateDelegationTokenResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to CreateDelegationTokenResponse. +func (v *CreateDelegationTokenResponse) Default() { +} + +// NewCreateDelegationTokenResponse returns a default CreateDelegationTokenResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewCreateDelegationTokenResponse() CreateDelegationTokenResponse { + var v CreateDelegationTokenResponse + v.Default() + return v +} + +// RenewDelegationTokenRequest is a request to renew a delegation token that +// has not yet hit its max timestamp. Note that a client using a token cannot +// renew its own token. +type RenewDelegationTokenRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // HMAC is the HMAC of the token to be renewed. + HMAC []byte + + // RenewTimeMillis is how long to renew the token for. If -1, Kafka uses its + // delegation.token.max.lifetime.ms. + RenewTimeMillis int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*RenewDelegationTokenRequest) Key() int16 { return 39 } +func (*RenewDelegationTokenRequest) MaxVersion() int16 { return 2 } +func (v *RenewDelegationTokenRequest) SetVersion(version int16) { v.Version = version } +func (v *RenewDelegationTokenRequest) GetVersion() int16 { return v.Version } +func (v *RenewDelegationTokenRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *RenewDelegationTokenRequest) ResponseKind() Response { + r := &RenewDelegationTokenResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *RenewDelegationTokenRequest) RequestWith(ctx context.Context, r Requestor) (*RenewDelegationTokenResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*RenewDelegationTokenResponse) + return resp, err +} + +func (v *RenewDelegationTokenRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.HMAC + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.RenewTimeMillis + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *RenewDelegationTokenRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *RenewDelegationTokenRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *RenewDelegationTokenRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.HMAC = v + } + { + v := b.Int64() + s.RenewTimeMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrRenewDelegationTokenRequest returns a pointer to a default RenewDelegationTokenRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrRenewDelegationTokenRequest() *RenewDelegationTokenRequest { + var v RenewDelegationTokenRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to RenewDelegationTokenRequest. +func (v *RenewDelegationTokenRequest) Default() { +} + +// NewRenewDelegationTokenRequest returns a default RenewDelegationTokenRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewRenewDelegationTokenRequest() RenewDelegationTokenRequest { + var v RenewDelegationTokenRequest + v.Default() + return v +} + +// RenewDelegationTokenResponse is a response to a RenewDelegationTokenRequest. +type RenewDelegationTokenResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is any error that caused the request to fail. + ErrorCode int16 + + // ExpiryTimestamp is the millisecond timestamp this token will expire. The + // token can be renewed up to MaxTimestamp, past which point, it will be + // invalid. The Kafka default is 24h. + ExpiryTimestamp int64 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*RenewDelegationTokenResponse) Key() int16 { return 39 } +func (*RenewDelegationTokenResponse) MaxVersion() int16 { return 2 } +func (v *RenewDelegationTokenResponse) SetVersion(version int16) { v.Version = version } +func (v *RenewDelegationTokenResponse) GetVersion() int16 { return v.Version } +func (v *RenewDelegationTokenResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *RenewDelegationTokenResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *RenewDelegationTokenResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *RenewDelegationTokenResponse) RequestKind() Request { + return &RenewDelegationTokenRequest{Version: v.Version} +} + +func (v *RenewDelegationTokenResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ExpiryTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *RenewDelegationTokenResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *RenewDelegationTokenResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *RenewDelegationTokenResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.ExpiryTimestamp = v + } + { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrRenewDelegationTokenResponse returns a pointer to a default RenewDelegationTokenResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrRenewDelegationTokenResponse() *RenewDelegationTokenResponse { + var v RenewDelegationTokenResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to RenewDelegationTokenResponse. +func (v *RenewDelegationTokenResponse) Default() { +} + +// NewRenewDelegationTokenResponse returns a default RenewDelegationTokenResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewRenewDelegationTokenResponse() RenewDelegationTokenResponse { + var v RenewDelegationTokenResponse + v.Default() + return v +} + +// ExpireDelegationTokenRequest is a request to change the expiry timestamp +// of a delegation token. Note that a client using a token cannot expire its +// own token. +type ExpireDelegationTokenRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // HMAC is the HMAC of the token to change the expiry timestamp of. + HMAC []byte + + // ExpiryPeriodMillis changes the delegation token's expiry timestamp to + // now + expiry time millis. This can be used to force tokens to expire + // quickly, or to allow tokens a grace period before expiry. You cannot + // add enough expiry that exceeds the original max timestamp. + ExpiryPeriodMillis int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*ExpireDelegationTokenRequest) Key() int16 { return 40 } +func (*ExpireDelegationTokenRequest) MaxVersion() int16 { return 2 } +func (v *ExpireDelegationTokenRequest) SetVersion(version int16) { v.Version = version } +func (v *ExpireDelegationTokenRequest) GetVersion() int16 { return v.Version } +func (v *ExpireDelegationTokenRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *ExpireDelegationTokenRequest) ResponseKind() Response { + r := &ExpireDelegationTokenResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ExpireDelegationTokenRequest) RequestWith(ctx context.Context, r Requestor) (*ExpireDelegationTokenResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ExpireDelegationTokenResponse) + return resp, err +} + +func (v *ExpireDelegationTokenRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.HMAC + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.ExpiryPeriodMillis + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ExpireDelegationTokenRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ExpireDelegationTokenRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ExpireDelegationTokenRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.HMAC = v + } + { + v := b.Int64() + s.ExpiryPeriodMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrExpireDelegationTokenRequest returns a pointer to a default ExpireDelegationTokenRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrExpireDelegationTokenRequest() *ExpireDelegationTokenRequest { + var v ExpireDelegationTokenRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ExpireDelegationTokenRequest. +func (v *ExpireDelegationTokenRequest) Default() { +} + +// NewExpireDelegationTokenRequest returns a default ExpireDelegationTokenRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewExpireDelegationTokenRequest() ExpireDelegationTokenRequest { + var v ExpireDelegationTokenRequest + v.Default() + return v +} + +// ExpireDelegationTokenResponse is a response to an ExpireDelegationTokenRequest. +type ExpireDelegationTokenResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is any error that caused the request to fail. + ErrorCode int16 + + // ExpiryTimestamp is the new timestamp at which the delegation token will + // expire. + ExpiryTimestamp int64 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*ExpireDelegationTokenResponse) Key() int16 { return 40 } +func (*ExpireDelegationTokenResponse) MaxVersion() int16 { return 2 } +func (v *ExpireDelegationTokenResponse) SetVersion(version int16) { v.Version = version } +func (v *ExpireDelegationTokenResponse) GetVersion() int16 { return v.Version } +func (v *ExpireDelegationTokenResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *ExpireDelegationTokenResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *ExpireDelegationTokenResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ExpireDelegationTokenResponse) RequestKind() Request { + return &ExpireDelegationTokenRequest{Version: v.Version} +} + +func (v *ExpireDelegationTokenResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ExpiryTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ExpireDelegationTokenResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ExpireDelegationTokenResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ExpireDelegationTokenResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.ExpiryTimestamp = v + } + { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrExpireDelegationTokenResponse returns a pointer to a default ExpireDelegationTokenResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrExpireDelegationTokenResponse() *ExpireDelegationTokenResponse { + var v ExpireDelegationTokenResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ExpireDelegationTokenResponse. +func (v *ExpireDelegationTokenResponse) Default() { +} + +// NewExpireDelegationTokenResponse returns a default ExpireDelegationTokenResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewExpireDelegationTokenResponse() ExpireDelegationTokenResponse { + var v ExpireDelegationTokenResponse + v.Default() + return v +} + +type DescribeDelegationTokenRequestOwner struct { + // PrincipalType is a type to match to describe delegation tokens created + // with this principal. This would be "User" with the simple authorizer. + PrincipalType string + + // PrincipalName is the name to match to describe delegation tokens created + // with this principal. + PrincipalName string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeDelegationTokenRequestOwner. +func (v *DescribeDelegationTokenRequestOwner) Default() { +} + +// NewDescribeDelegationTokenRequestOwner returns a default DescribeDelegationTokenRequestOwner +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeDelegationTokenRequestOwner() DescribeDelegationTokenRequestOwner { + var v DescribeDelegationTokenRequestOwner + v.Default() + return v +} + +// DescribeDelegationTokenRequest is a request to describe delegation tokens. +type DescribeDelegationTokenRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Owners contains owners to describe delegation tokens for, or null for all. + // If non-null, only tokens created from a matching principal type, name + // combination are printed. + Owners []DescribeDelegationTokenRequestOwner + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeDelegationTokenRequest) Key() int16 { return 41 } +func (*DescribeDelegationTokenRequest) MaxVersion() int16 { return 3 } +func (v *DescribeDelegationTokenRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeDelegationTokenRequest) GetVersion() int16 { return v.Version } +func (v *DescribeDelegationTokenRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeDelegationTokenRequest) ResponseKind() Response { + r := &DescribeDelegationTokenResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeDelegationTokenRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeDelegationTokenResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeDelegationTokenResponse) + return resp, err +} + +func (v *DescribeDelegationTokenRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Owners + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.PrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.PrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeDelegationTokenRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeDelegationTokenRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeDelegationTokenRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Owners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []DescribeDelegationTokenRequestOwner{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeDelegationTokenRequestOwner, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalName = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Owners = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeDelegationTokenRequest returns a pointer to a default DescribeDelegationTokenRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeDelegationTokenRequest() *DescribeDelegationTokenRequest { + var v DescribeDelegationTokenRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeDelegationTokenRequest. +func (v *DescribeDelegationTokenRequest) Default() { +} + +// NewDescribeDelegationTokenRequest returns a default DescribeDelegationTokenRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeDelegationTokenRequest() DescribeDelegationTokenRequest { + var v DescribeDelegationTokenRequest + v.Default() + return v +} + +type DescribeDelegationTokenResponseTokenDetailRenewer struct { + PrincipalType string + + PrincipalName string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeDelegationTokenResponseTokenDetailRenewer. +func (v *DescribeDelegationTokenResponseTokenDetailRenewer) Default() { +} + +// NewDescribeDelegationTokenResponseTokenDetailRenewer returns a default DescribeDelegationTokenResponseTokenDetailRenewer +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeDelegationTokenResponseTokenDetailRenewer() DescribeDelegationTokenResponseTokenDetailRenewer { + var v DescribeDelegationTokenResponseTokenDetailRenewer + v.Default() + return v +} + +type DescribeDelegationTokenResponseTokenDetail struct { + // PrincipalType is the principal type of who created this token. + PrincipalType string + + // PrincipalName is the principal name of who created this token. + PrincipalName string + + // The principal type of the requester of the token. + TokenRequesterPrincipalType string // v3+ + + // The principal name of the requester token. + TokenRequesterPrincipalName string // v3+ + + // IssueTimestamp is the millisecond timestamp of when this token was issued. + IssueTimestamp int64 + + // ExpiryTimestamp is the millisecond timestamp of when this token will expire. + ExpiryTimestamp int64 + + // MaxTimestamp is the millisecond timestamp past which whis token cannot + // be renewed. + MaxTimestamp int64 + + // TokenID is the ID (scram username) of this token. + TokenID string + + // HMAC is the password of this token. + HMAC []byte + + // Renewers is a list of users that can renew this token. + Renewers []DescribeDelegationTokenResponseTokenDetailRenewer + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeDelegationTokenResponseTokenDetail. +func (v *DescribeDelegationTokenResponseTokenDetail) Default() { +} + +// NewDescribeDelegationTokenResponseTokenDetail returns a default DescribeDelegationTokenResponseTokenDetail +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeDelegationTokenResponseTokenDetail() DescribeDelegationTokenResponseTokenDetail { + var v DescribeDelegationTokenResponseTokenDetail + v.Default() + return v +} + +// DescribeDelegationTokenResponsee is a response to a DescribeDelegationTokenRequest. +type DescribeDelegationTokenResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is any error that caused the request to fail. + ErrorCode int16 + + // TokenDetails shows information about each token created from any principal + // in the request. + TokenDetails []DescribeDelegationTokenResponseTokenDetail + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DescribeDelegationTokenResponse) Key() int16 { return 41 } +func (*DescribeDelegationTokenResponse) MaxVersion() int16 { return 3 } +func (v *DescribeDelegationTokenResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeDelegationTokenResponse) GetVersion() int16 { return v.Version } +func (v *DescribeDelegationTokenResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DescribeDelegationTokenResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 1 +} + +func (v *DescribeDelegationTokenResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeDelegationTokenResponse) RequestKind() Request { + return &DescribeDelegationTokenRequest{Version: v.Version} +} + +func (v *DescribeDelegationTokenResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.TokenDetails + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.PrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.PrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.TokenRequesterPrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 3 { + v := v.TokenRequesterPrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.IssueTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ExpiryTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.MaxTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.TokenID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.HMAC + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.Renewers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.PrincipalType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.PrincipalName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeDelegationTokenResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeDelegationTokenResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeDelegationTokenResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.TokenDetails + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeDelegationTokenResponseTokenDetail, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalName = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenRequesterPrincipalType = v + } + if version >= 3 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenRequesterPrincipalName = v + } + { + v := b.Int64() + s.IssueTimestamp = v + } + { + v := b.Int64() + s.ExpiryTimestamp = v + } + { + v := b.Int64() + s.MaxTimestamp = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TokenID = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.HMAC = v + } + { + v := s.Renewers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeDelegationTokenResponseTokenDetailRenewer, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.PrincipalName = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Renewers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TokenDetails = v + } + { + v := b.Int32() + s.ThrottleMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeDelegationTokenResponse returns a pointer to a default DescribeDelegationTokenResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeDelegationTokenResponse() *DescribeDelegationTokenResponse { + var v DescribeDelegationTokenResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeDelegationTokenResponse. +func (v *DescribeDelegationTokenResponse) Default() { +} + +// NewDescribeDelegationTokenResponse returns a default DescribeDelegationTokenResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeDelegationTokenResponse() DescribeDelegationTokenResponse { + var v DescribeDelegationTokenResponse + v.Default() + return v +} + +// DeleteGroupsRequest deletes consumer groups. This request was added for +// Kafka 1.1.0 corresponding to the removal of RetentionTimeMillis from +// OffsetCommitRequest. See KIP-229 for more details. +type DeleteGroupsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Groups is a list of groups to delete. + Groups []string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteGroupsRequest) Key() int16 { return 42 } +func (*DeleteGroupsRequest) MaxVersion() int16 { return 2 } +func (v *DeleteGroupsRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteGroupsRequest) GetVersion() int16 { return v.Version } +func (v *DeleteGroupsRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteGroupsRequest) IsGroupCoordinatorRequest() {} +func (v *DeleteGroupsRequest) ResponseKind() Response { + r := &DeleteGroupsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteGroupsRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteGroupsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteGroupsResponse) + return resp, err +} + +func (v *DeleteGroupsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteGroupsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteGroupsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteGroupsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteGroupsRequest returns a pointer to a default DeleteGroupsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteGroupsRequest() *DeleteGroupsRequest { + var v DeleteGroupsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteGroupsRequest. +func (v *DeleteGroupsRequest) Default() { +} + +// NewDeleteGroupsRequest returns a default DeleteGroupsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteGroupsRequest() DeleteGroupsRequest { + var v DeleteGroupsRequest + v.Default() + return v +} + +type DeleteGroupsResponseGroup struct { + // Group is a group ID requested for deletion. + Group string + + // ErrorCode is the error code returned for this group's deletion request. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // to delete a group. + // + // INVALID_GROUP_ID is returned if the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator for this + // group is not yet active. + // + // GROUP_ID_NOT_FOUND is returned if the group ID does not exist. + // + // NON_EMPTY_GROUP is returned if attempting to delete a group that is + // not in the empty state. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteGroupsResponseGroup. +func (v *DeleteGroupsResponseGroup) Default() { +} + +// NewDeleteGroupsResponseGroup returns a default DeleteGroupsResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteGroupsResponseGroup() DeleteGroupsResponseGroup { + var v DeleteGroupsResponseGroup + v.Default() + return v +} + +// DeleteGroupsResponse is returned from a DeleteGroupsRequest. +type DeleteGroupsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after this request. + // For Kafka < 2.0.0, the throttle is applied before issuing a response. + // For Kafka >= 2.0.0, the throttle is applied after issuing a response. + // + // This request switched at version 1. + ThrottleMillis int32 + + // Groups are the responses to each group requested for deletion. + Groups []DeleteGroupsResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*DeleteGroupsResponse) Key() int16 { return 42 } +func (*DeleteGroupsResponse) MaxVersion() int16 { return 2 } +func (v *DeleteGroupsResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteGroupsResponse) GetVersion() int16 { return v.Version } +func (v *DeleteGroupsResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *DeleteGroupsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 1 } +func (v *DeleteGroupsResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *DeleteGroupsResponse) RequestKind() Request { return &DeleteGroupsRequest{Version: v.Version} } + +func (v *DeleteGroupsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteGroupsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteGroupsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteGroupsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteGroupsResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteGroupsResponse returns a pointer to a default DeleteGroupsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteGroupsResponse() *DeleteGroupsResponse { + var v DeleteGroupsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteGroupsResponse. +func (v *DeleteGroupsResponse) Default() { +} + +// NewDeleteGroupsResponse returns a default DeleteGroupsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteGroupsResponse() DeleteGroupsResponse { + var v DeleteGroupsResponse + v.Default() + return v +} + +type ElectLeadersRequestTopic struct { + // Topic is a topic to trigger leader elections for (but only for the + // partitions below). + Topic string + + // Partitions is an array of partitions in a topic to trigger leader + // elections for. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ElectLeadersRequestTopic. +func (v *ElectLeadersRequestTopic) Default() { +} + +// NewElectLeadersRequestTopic returns a default ElectLeadersRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewElectLeadersRequestTopic() ElectLeadersRequestTopic { + var v ElectLeadersRequestTopic + v.Default() + return v +} + +// ElectLeadersRequest begins a leader election for all given topic +// partitions. This request was added in Kafka 2.2.0 to replace the zookeeper +// only option of triggering leader elections before. See KIP-183 for more +// details. KIP-460 introduced the ElectionType field with Kafka 2.4.0. +type ElectLeadersRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ElectionType is the type of election to conduct. 0 elects the preferred + // replica, 1 elects the first live replica if there are no in-sync replicas + // (i.e., unclean leader election). + ElectionType int8 // v1+ + + // Topics is an array of topics and corresponding partitions to + // trigger leader elections for, or null for all. + Topics []ElectLeadersRequestTopic + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 60000. + TimeoutMillis int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*ElectLeadersRequest) Key() int16 { return 43 } +func (*ElectLeadersRequest) MaxVersion() int16 { return 2 } +func (v *ElectLeadersRequest) SetVersion(version int16) { v.Version = version } +func (v *ElectLeadersRequest) GetVersion() int16 { return v.Version } +func (v *ElectLeadersRequest) IsFlexible() bool { return v.Version >= 2 } +func (v *ElectLeadersRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *ElectLeadersRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *ElectLeadersRequest) IsAdminRequest() {} +func (v *ElectLeadersRequest) ResponseKind() Response { + r := &ElectLeadersResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ElectLeadersRequest) RequestWith(ctx context.Context, r Requestor) (*ElectLeadersResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ElectLeadersResponse) + return resp, err +} + +func (v *ElectLeadersRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + if version >= 1 { + v := v.ElectionType + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ElectLeadersRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ElectLeadersRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ElectLeadersRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + if version >= 1 { + v := b.Int8() + s.ElectionType = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []ElectLeadersRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ElectLeadersRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrElectLeadersRequest returns a pointer to a default ElectLeadersRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrElectLeadersRequest() *ElectLeadersRequest { + var v ElectLeadersRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ElectLeadersRequest. +func (v *ElectLeadersRequest) Default() { + v.TimeoutMillis = 60000 +} + +// NewElectLeadersRequest returns a default ElectLeadersRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewElectLeadersRequest() ElectLeadersRequest { + var v ElectLeadersRequest + v.Default() + return v +} + +type ElectLeadersResponseTopicPartition struct { + // Partition is the partition for this result. + Partition int32 + + // ErrorCode is the error code returned for this topic/partition leader + // election. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if the client is not + // authorized to trigger leader elections. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic/partition does + // not exist on any broker in the cluster (this is slightly different + // from the usual meaning of a single broker not knowing of the topic + // partition). + // + // PREFERRED_LEADER_NOT_AVAILABLE is returned if the preferred leader + // could not be elected (for example, the preferred leader was not in + // the ISR). + ErrorCode int16 + + // ErrorMessage is an informative message if the leader election failed. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ElectLeadersResponseTopicPartition. +func (v *ElectLeadersResponseTopicPartition) Default() { +} + +// NewElectLeadersResponseTopicPartition returns a default ElectLeadersResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewElectLeadersResponseTopicPartition() ElectLeadersResponseTopicPartition { + var v ElectLeadersResponseTopicPartition + v.Default() + return v +} + +type ElectLeadersResponseTopic struct { + // Topic is topic for the given partition results below. + Topic string + + // Partitions contains election results for a topic's partitions. + Partitions []ElectLeadersResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ElectLeadersResponseTopic. +func (v *ElectLeadersResponseTopic) Default() { +} + +// NewElectLeadersResponseTopic returns a default ElectLeadersResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewElectLeadersResponseTopic() ElectLeadersResponseTopic { + var v ElectLeadersResponseTopic + v.Default() + return v +} + +// ElectLeadersResponse is a response for an ElectLeadersRequest. +type ElectLeadersResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is any error that applies to all partitions. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if the client is not + // authorized to reassign partitions. + ErrorCode int16 // v1+ + + // Topics contains leader election results for each requested topic. + Topics []ElectLeadersResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v2+ +} + +func (*ElectLeadersResponse) Key() int16 { return 43 } +func (*ElectLeadersResponse) MaxVersion() int16 { return 2 } +func (v *ElectLeadersResponse) SetVersion(version int16) { v.Version = version } +func (v *ElectLeadersResponse) GetVersion() int16 { return v.Version } +func (v *ElectLeadersResponse) IsFlexible() bool { return v.Version >= 2 } +func (v *ElectLeadersResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *ElectLeadersResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ElectLeadersResponse) RequestKind() Request { return &ElectLeadersRequest{Version: v.Version} } + +func (v *ElectLeadersResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ElectLeadersResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ElectLeadersResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ElectLeadersResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 2 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 1 { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ElectLeadersResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ElectLeadersResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrElectLeadersResponse returns a pointer to a default ElectLeadersResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrElectLeadersResponse() *ElectLeadersResponse { + var v ElectLeadersResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ElectLeadersResponse. +func (v *ElectLeadersResponse) Default() { +} + +// NewElectLeadersResponse returns a default ElectLeadersResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewElectLeadersResponse() ElectLeadersResponse { + var v ElectLeadersResponse + v.Default() + return v +} + +type IncrementalAlterConfigsRequestResourceConfig struct { + // Name is a key to modify (e.g. segment.bytes). + // + // For broker loggers, see KIP-412 section "Request/Response Overview" + // for details on how to change per logger log levels. + Name string + + // Op is the type of operation to perform for this config name. + // + // SET (0) is to set a configuration value; the value must not be null. + // + // DELETE (1) is to delete a configuration key. + // + // APPEND (2) is to add a value to the list of values for a key (if the + // key is for a list of values). + // + // SUBTRACT (3) is to remove a value from a list of values (if the key + // is for a list of values). + Op IncrementalAlterConfigOp + + // Value is a value to set for the key (e.g. 10). + Value *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to IncrementalAlterConfigsRequestResourceConfig. +func (v *IncrementalAlterConfigsRequestResourceConfig) Default() { +} + +// NewIncrementalAlterConfigsRequestResourceConfig returns a default IncrementalAlterConfigsRequestResourceConfig +// This is a shortcut for creating a struct and calling Default yourself. +func NewIncrementalAlterConfigsRequestResourceConfig() IncrementalAlterConfigsRequestResourceConfig { + var v IncrementalAlterConfigsRequestResourceConfig + v.Default() + return v +} + +type IncrementalAlterConfigsRequestResource struct { + // ResourceType is an enum corresponding to the type of config to alter. + ResourceType ConfigResourceType + + // ResourceName is the name of config to alter. + // + // If the requested type is a topic, this corresponds to a topic name. + // + // If the requested type if a broker, this should either be empty or be + // the ID of the broker this request is issued to. If it is empty, this + // updates all broker configs. If a specific ID, this updates just the + // broker. Using a specific ID also ensures that brokers reload config + // or secret files even if the file path has not changed. Lastly, password + // config options can only be defined on a per broker basis. + // + // If the type is broker logger, this must be a broker ID. + ResourceName string + + // Configs contains key/value config pairs to set on the resource. + Configs []IncrementalAlterConfigsRequestResourceConfig + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to IncrementalAlterConfigsRequestResource. +func (v *IncrementalAlterConfigsRequestResource) Default() { +} + +// NewIncrementalAlterConfigsRequestResource returns a default IncrementalAlterConfigsRequestResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewIncrementalAlterConfigsRequestResource() IncrementalAlterConfigsRequestResource { + var v IncrementalAlterConfigsRequestResource + v.Default() + return v +} + +// IncrementalAlterConfigsRequest issues ar equest to alter either topic or +// broker configs. +// +// This API was added in Kafka 2.3.0 to replace AlterConfigs. The key benefit +// of this API is that consumers do not need to know the full config state +// to add or remove new config options. See KIP-339 for more details. +type IncrementalAlterConfigsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Resources is an array of configs to alter. + Resources []IncrementalAlterConfigsRequestResource + + // ValidateOnly validates the request but does not apply it. + ValidateOnly bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*IncrementalAlterConfigsRequest) Key() int16 { return 44 } +func (*IncrementalAlterConfigsRequest) MaxVersion() int16 { return 1 } +func (v *IncrementalAlterConfigsRequest) SetVersion(version int16) { v.Version = version } +func (v *IncrementalAlterConfigsRequest) GetVersion() int16 { return v.Version } +func (v *IncrementalAlterConfigsRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *IncrementalAlterConfigsRequest) ResponseKind() Response { + r := &IncrementalAlterConfigsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *IncrementalAlterConfigsRequest) RequestWith(ctx context.Context, r Requestor) (*IncrementalAlterConfigsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*IncrementalAlterConfigsResponse) + return resp, err +} + +func (v *IncrementalAlterConfigsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Op + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *IncrementalAlterConfigsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *IncrementalAlterConfigsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *IncrementalAlterConfigsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]IncrementalAlterConfigsRequestResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]IncrementalAlterConfigsRequestResourceConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var t IncrementalAlterConfigOp + { + v := b.Int8() + t = IncrementalAlterConfigOp(v) + } + v := t + s.Op = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrIncrementalAlterConfigsRequest returns a pointer to a default IncrementalAlterConfigsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrIncrementalAlterConfigsRequest() *IncrementalAlterConfigsRequest { + var v IncrementalAlterConfigsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to IncrementalAlterConfigsRequest. +func (v *IncrementalAlterConfigsRequest) Default() { +} + +// NewIncrementalAlterConfigsRequest returns a default IncrementalAlterConfigsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewIncrementalAlterConfigsRequest() IncrementalAlterConfigsRequest { + var v IncrementalAlterConfigsRequest + v.Default() + return v +} + +type IncrementalAlterConfigsResponseResource struct { + // ErrorCode is the error code returned for incrementally altering configs. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if asking to alter broker + // configs but the client is not authorized to do so. + // + // TOPIC_AUTHORIZATION_FAILED is returned if asking to alter topic + // configs but the client is not authorized to do so. + // + // INVALID_TOPIC_EXCEPTION is returned if the requested topic was invalid. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the requested topic. + // + // INVALID_REQUEST is returned if the requested config is invalid or if + // asking Kafka to alter an invalid resource. + ErrorCode int16 + + // ErrorMessage is an informative message if the incremental alter config failed. + ErrorMessage *string + + // ResourceType is the enum corresponding to the type of altered config. + ResourceType ConfigResourceType + + // ResourceName is the name corresponding to the incremental alter config + // request. + ResourceName string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to IncrementalAlterConfigsResponseResource. +func (v *IncrementalAlterConfigsResponseResource) Default() { +} + +// NewIncrementalAlterConfigsResponseResource returns a default IncrementalAlterConfigsResponseResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewIncrementalAlterConfigsResponseResource() IncrementalAlterConfigsResponseResource { + var v IncrementalAlterConfigsResponseResource + v.Default() + return v +} + +// IncrementalAlterConfigsResponse is returned from an IncrementalAlterConfigsRequest. +type IncrementalAlterConfigsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Resources are responses for each resources in the alter request. + Resources []IncrementalAlterConfigsResponseResource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*IncrementalAlterConfigsResponse) Key() int16 { return 44 } +func (*IncrementalAlterConfigsResponse) MaxVersion() int16 { return 1 } +func (v *IncrementalAlterConfigsResponse) SetVersion(version int16) { v.Version = version } +func (v *IncrementalAlterConfigsResponse) GetVersion() int16 { return v.Version } +func (v *IncrementalAlterConfigsResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *IncrementalAlterConfigsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *IncrementalAlterConfigsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *IncrementalAlterConfigsResponse) RequestKind() Request { + return &IncrementalAlterConfigsRequest{Version: v.Version} +} + +func (v *IncrementalAlterConfigsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Resources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ResourceType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.ResourceName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *IncrementalAlterConfigsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *IncrementalAlterConfigsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *IncrementalAlterConfigsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Resources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]IncrementalAlterConfigsResponseResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var t ConfigResourceType + { + v := b.Int8() + t = ConfigResourceType(v) + } + v := t + s.ResourceType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ResourceName = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Resources = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrIncrementalAlterConfigsResponse returns a pointer to a default IncrementalAlterConfigsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrIncrementalAlterConfigsResponse() *IncrementalAlterConfigsResponse { + var v IncrementalAlterConfigsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to IncrementalAlterConfigsResponse. +func (v *IncrementalAlterConfigsResponse) Default() { +} + +// NewIncrementalAlterConfigsResponse returns a default IncrementalAlterConfigsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewIncrementalAlterConfigsResponse() IncrementalAlterConfigsResponse { + var v IncrementalAlterConfigsResponse + v.Default() + return v +} + +type AlterPartitionAssignmentsRequestTopicPartition struct { + // Partition is a partition to reassign. + Partition int32 + + // Replicas are replicas to place the partition on, or null to + // cancel a pending reassignment of this partition. + Replicas []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsRequestTopicPartition. +func (v *AlterPartitionAssignmentsRequestTopicPartition) Default() { +} + +// NewAlterPartitionAssignmentsRequestTopicPartition returns a default AlterPartitionAssignmentsRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsRequestTopicPartition() AlterPartitionAssignmentsRequestTopicPartition { + var v AlterPartitionAssignmentsRequestTopicPartition + v.Default() + return v +} + +type AlterPartitionAssignmentsRequestTopic struct { + // Topic is a topic to reassign the partitions of. + Topic string + + // Partitions contains partitions to reassign. + Partitions []AlterPartitionAssignmentsRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsRequestTopic. +func (v *AlterPartitionAssignmentsRequestTopic) Default() { +} + +// NewAlterPartitionAssignmentsRequestTopic returns a default AlterPartitionAssignmentsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsRequestTopic() AlterPartitionAssignmentsRequestTopic { + var v AlterPartitionAssignmentsRequestTopic + v.Default() + return v +} + +// AlterPartitionAssignmentsRequest, proposed in KIP-455 and implemented in +// Kafka 2.4.0, is a request to reassign partitions to certain brokers. +// +// ACL wise, this requires ALTER on CLUSTER. +type AlterPartitionAssignmentsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 60000. + TimeoutMillis int32 + + // The option indicating whether changing the replication factor of any given + // partition as part of this request is a valid move. + // + // This field has a default of true. + AllowReplicationFactorChange bool // v1+ + + // Topics are topics for which to reassign partitions of. + Topics []AlterPartitionAssignmentsRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterPartitionAssignmentsRequest) Key() int16 { return 45 } +func (*AlterPartitionAssignmentsRequest) MaxVersion() int16 { return 1 } +func (v *AlterPartitionAssignmentsRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterPartitionAssignmentsRequest) GetVersion() int16 { return v.Version } +func (v *AlterPartitionAssignmentsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterPartitionAssignmentsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *AlterPartitionAssignmentsRequest) SetTimeout(timeoutMillis int32) { + v.TimeoutMillis = timeoutMillis +} +func (v *AlterPartitionAssignmentsRequest) IsAdminRequest() {} +func (v *AlterPartitionAssignmentsRequest) ResponseKind() Response { + r := &AlterPartitionAssignmentsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterPartitionAssignmentsRequest) RequestWith(ctx context.Context, r Requestor) (*AlterPartitionAssignmentsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterPartitionAssignmentsResponse) + return resp, err +} + +func (v *AlterPartitionAssignmentsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.AllowReplicationFactorChange + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterPartitionAssignmentsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterPartitionAssignmentsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterPartitionAssignmentsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.TimeoutMillis = v + } + if version >= 1 { + v := b.Bool() + s.AllowReplicationFactorChange = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionAssignmentsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionAssignmentsRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []int32{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterPartitionAssignmentsRequest returns a pointer to a default AlterPartitionAssignmentsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterPartitionAssignmentsRequest() *AlterPartitionAssignmentsRequest { + var v AlterPartitionAssignmentsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsRequest. +func (v *AlterPartitionAssignmentsRequest) Default() { + v.TimeoutMillis = 60000 + v.AllowReplicationFactorChange = true +} + +// NewAlterPartitionAssignmentsRequest returns a default AlterPartitionAssignmentsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsRequest() AlterPartitionAssignmentsRequest { + var v AlterPartitionAssignmentsRequest + v.Default() + return v +} + +type AlterPartitionAssignmentsResponseTopicPartition struct { + // Partition is the partition being responded to. + Partition int32 + + // ErrorCode is the error code returned for partition reassignments. + // + // REQUEST_TIMED_OUT is returned if the request timed out. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if the client is not + // authorized to reassign partitions. + // + // NO_REASSIGNMENT_IN_PROGRESS is returned for partition reassignment + // cancellations when the partition was not being reassigned. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the requested topic or the topic is being deleted. + ErrorCode int16 + + // ErrorMessage is an informative message if the partition reassignment failed. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsResponseTopicPartition. +func (v *AlterPartitionAssignmentsResponseTopicPartition) Default() { +} + +// NewAlterPartitionAssignmentsResponseTopicPartition returns a default AlterPartitionAssignmentsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsResponseTopicPartition() AlterPartitionAssignmentsResponseTopicPartition { + var v AlterPartitionAssignmentsResponseTopicPartition + v.Default() + return v +} + +type AlterPartitionAssignmentsResponseTopic struct { + // Topic is the topic being responded to. + Topic string + + // Partitions contains responses for partitions. + Partitions []AlterPartitionAssignmentsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsResponseTopic. +func (v *AlterPartitionAssignmentsResponseTopic) Default() { +} + +// NewAlterPartitionAssignmentsResponseTopic returns a default AlterPartitionAssignmentsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsResponseTopic() AlterPartitionAssignmentsResponseTopic { + var v AlterPartitionAssignmentsResponseTopic + v.Default() + return v +} + +// AlterPartitionAssignmentsResponse is returned for an AlterPartitionAssignmentsRequest. +type AlterPartitionAssignmentsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The option indicating whether changing the replication factor of any given + // partition as part of the request was allowed. + // + // This field has a default of true. + AllowReplicationFactorChange bool // v1+ + + // ErrorCode is any global (applied to all partitions) error code. + ErrorCode int16 + + // ErrorMessage is any global (applied to all partitions) error message. + ErrorMessage *string + + // Topics contains responses for each topic requested. + Topics []AlterPartitionAssignmentsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterPartitionAssignmentsResponse) Key() int16 { return 45 } +func (*AlterPartitionAssignmentsResponse) MaxVersion() int16 { return 1 } +func (v *AlterPartitionAssignmentsResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterPartitionAssignmentsResponse) GetVersion() int16 { return v.Version } +func (v *AlterPartitionAssignmentsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterPartitionAssignmentsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *AlterPartitionAssignmentsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AlterPartitionAssignmentsResponse) RequestKind() Request { + return &AlterPartitionAssignmentsRequest{Version: v.Version} +} + +func (v *AlterPartitionAssignmentsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.AllowReplicationFactorChange + dst = kbin.AppendBool(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterPartitionAssignmentsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterPartitionAssignmentsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterPartitionAssignmentsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + if version >= 1 { + v := b.Bool() + s.AllowReplicationFactorChange = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionAssignmentsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionAssignmentsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterPartitionAssignmentsResponse returns a pointer to a default AlterPartitionAssignmentsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterPartitionAssignmentsResponse() *AlterPartitionAssignmentsResponse { + var v AlterPartitionAssignmentsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionAssignmentsResponse. +func (v *AlterPartitionAssignmentsResponse) Default() { + v.AllowReplicationFactorChange = true +} + +// NewAlterPartitionAssignmentsResponse returns a default AlterPartitionAssignmentsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionAssignmentsResponse() AlterPartitionAssignmentsResponse { + var v AlterPartitionAssignmentsResponse + v.Default() + return v +} + +type ListPartitionReassignmentsRequestTopic struct { + // Topic is a topic to list in progress partition reassingments of. + Topic string + + // Partitions are partitions to list in progress reassignments of. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListPartitionReassignmentsRequestTopic. +func (v *ListPartitionReassignmentsRequestTopic) Default() { +} + +// NewListPartitionReassignmentsRequestTopic returns a default ListPartitionReassignmentsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewListPartitionReassignmentsRequestTopic() ListPartitionReassignmentsRequestTopic { + var v ListPartitionReassignmentsRequestTopic + v.Default() + return v +} + +// ListPartitionReassignmentsRequest, proposed in KIP-455 and implemented in +// Kafka 2.4.0, is a request to list in progress partition reassignments. +// +// ACL wise, this requires DESCRIBE on CLUSTER. +type ListPartitionReassignmentsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 60000. + TimeoutMillis int32 + + // Topics are topics to list in progress partition reassignments of, or null + // to list everything. + Topics []ListPartitionReassignmentsRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListPartitionReassignmentsRequest) Key() int16 { return 46 } +func (*ListPartitionReassignmentsRequest) MaxVersion() int16 { return 0 } +func (v *ListPartitionReassignmentsRequest) SetVersion(version int16) { v.Version = version } +func (v *ListPartitionReassignmentsRequest) GetVersion() int16 { return v.Version } +func (v *ListPartitionReassignmentsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ListPartitionReassignmentsRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *ListPartitionReassignmentsRequest) SetTimeout(timeoutMillis int32) { + v.TimeoutMillis = timeoutMillis +} +func (v *ListPartitionReassignmentsRequest) IsAdminRequest() {} +func (v *ListPartitionReassignmentsRequest) ResponseKind() Response { + r := &ListPartitionReassignmentsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ListPartitionReassignmentsRequest) RequestWith(ctx context.Context, r Requestor) (*ListPartitionReassignmentsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ListPartitionReassignmentsResponse) + return resp, err +} + +func (v *ListPartitionReassignmentsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListPartitionReassignmentsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListPartitionReassignmentsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListPartitionReassignmentsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []ListPartitionReassignmentsRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListPartitionReassignmentsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListPartitionReassignmentsRequest returns a pointer to a default ListPartitionReassignmentsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListPartitionReassignmentsRequest() *ListPartitionReassignmentsRequest { + var v ListPartitionReassignmentsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListPartitionReassignmentsRequest. +func (v *ListPartitionReassignmentsRequest) Default() { + v.TimeoutMillis = 60000 +} + +// NewListPartitionReassignmentsRequest returns a default ListPartitionReassignmentsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewListPartitionReassignmentsRequest() ListPartitionReassignmentsRequest { + var v ListPartitionReassignmentsRequest + v.Default() + return v +} + +type ListPartitionReassignmentsResponseTopicPartition struct { + // Partition is the partition being responded to. + Partition int32 + + // Replicas is the partition's current replicas. + Replicas []int32 + + // AddingReplicas are replicas currently being added to the partition. + AddingReplicas []int32 + + // RemovingReplicas are replicas currently being removed from the partition. + RemovingReplicas []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListPartitionReassignmentsResponseTopicPartition. +func (v *ListPartitionReassignmentsResponseTopicPartition) Default() { +} + +// NewListPartitionReassignmentsResponseTopicPartition returns a default ListPartitionReassignmentsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewListPartitionReassignmentsResponseTopicPartition() ListPartitionReassignmentsResponseTopicPartition { + var v ListPartitionReassignmentsResponseTopicPartition + v.Default() + return v +} + +type ListPartitionReassignmentsResponseTopic struct { + // Topic is the topic being responded to. + Topic string + + // Partitions contains responses for partitions. + Partitions []ListPartitionReassignmentsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListPartitionReassignmentsResponseTopic. +func (v *ListPartitionReassignmentsResponseTopic) Default() { +} + +// NewListPartitionReassignmentsResponseTopic returns a default ListPartitionReassignmentsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewListPartitionReassignmentsResponseTopic() ListPartitionReassignmentsResponseTopic { + var v ListPartitionReassignmentsResponseTopic + v.Default() + return v +} + +// ListPartitionReassignmentsResponse is returned for a ListPartitionReassignmentsRequest. +type ListPartitionReassignmentsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the error code returned for listing reassignments. + // + // REQUEST_TIMED_OUT is returned if the request timed out. + // + // NOT_CONTROLLER is returned if the request was not issued to a Kafka + // controller. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if the client is not + // authorized to reassign partitions. + ErrorCode int16 + + // ErrorMessage is any global (applied to all partitions) error message. + ErrorMessage *string + + // Topics contains responses for each topic requested. + Topics []ListPartitionReassignmentsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListPartitionReassignmentsResponse) Key() int16 { return 46 } +func (*ListPartitionReassignmentsResponse) MaxVersion() int16 { return 0 } +func (v *ListPartitionReassignmentsResponse) SetVersion(version int16) { v.Version = version } +func (v *ListPartitionReassignmentsResponse) GetVersion() int16 { return v.Version } +func (v *ListPartitionReassignmentsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ListPartitionReassignmentsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ListPartitionReassignmentsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ListPartitionReassignmentsResponse) RequestKind() Request { + return &ListPartitionReassignmentsRequest{Version: v.Version} +} + +func (v *ListPartitionReassignmentsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.AddingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.RemovingReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListPartitionReassignmentsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListPartitionReassignmentsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListPartitionReassignmentsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListPartitionReassignmentsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListPartitionReassignmentsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + { + v := s.AddingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.AddingReplicas = v + } + { + v := s.RemovingReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.RemovingReplicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListPartitionReassignmentsResponse returns a pointer to a default ListPartitionReassignmentsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListPartitionReassignmentsResponse() *ListPartitionReassignmentsResponse { + var v ListPartitionReassignmentsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListPartitionReassignmentsResponse. +func (v *ListPartitionReassignmentsResponse) Default() { +} + +// NewListPartitionReassignmentsResponse returns a default ListPartitionReassignmentsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewListPartitionReassignmentsResponse() ListPartitionReassignmentsResponse { + var v ListPartitionReassignmentsResponse + v.Default() + return v +} + +type OffsetDeleteRequestTopicPartition struct { + // Partition is a partition to delete offsets for. + Partition int32 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteRequestTopicPartition. +func (v *OffsetDeleteRequestTopicPartition) Default() { +} + +// NewOffsetDeleteRequestTopicPartition returns a default OffsetDeleteRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteRequestTopicPartition() OffsetDeleteRequestTopicPartition { + var v OffsetDeleteRequestTopicPartition + v.Default() + return v +} + +type OffsetDeleteRequestTopic struct { + // Topic is a topic to delete offsets in. + Topic string + + // Partitions are partitions to delete offsets for. + Partitions []OffsetDeleteRequestTopicPartition +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteRequestTopic. +func (v *OffsetDeleteRequestTopic) Default() { +} + +// NewOffsetDeleteRequestTopic returns a default OffsetDeleteRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteRequestTopic() OffsetDeleteRequestTopic { + var v OffsetDeleteRequestTopic + v.Default() + return v +} + +// OffsetDeleteRequest, proposed in KIP-496 and implemented in Kafka 2.4.0, is +// a request to delete group offsets. +// +// ACL wise, this requires DELETE on GROUP for the group and READ on TOPIC for +// each topic. +type OffsetDeleteRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group to delete offsets in. + Group string + + // Topics are topics to delete offsets in. + Topics []OffsetDeleteRequestTopic +} + +func (*OffsetDeleteRequest) Key() int16 { return 47 } +func (*OffsetDeleteRequest) MaxVersion() int16 { return 0 } +func (v *OffsetDeleteRequest) SetVersion(version int16) { v.Version = version } +func (v *OffsetDeleteRequest) GetVersion() int16 { return v.Version } +func (v *OffsetDeleteRequest) IsFlexible() bool { return false } +func (v *OffsetDeleteRequest) IsGroupCoordinatorRequest() {} +func (v *OffsetDeleteRequest) ResponseKind() Response { + r := &OffsetDeleteResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *OffsetDeleteRequest) RequestWith(ctx context.Context, r Requestor) (*OffsetDeleteResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*OffsetDeleteResponse) + return resp, err +} + +func (v *OffsetDeleteRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.Group + dst = kbin.AppendString(dst, v) + } + { + v := v.Topics + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Topic + dst = kbin.AppendString(dst, v) + } + { + v := v.Partitions + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + } + } + } + } + return dst +} + +func (v *OffsetDeleteRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetDeleteRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetDeleteRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Group = v + } + { + v := s.Topics + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetDeleteRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetDeleteRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + } + v = a + s.Partitions = v + } + } + v = a + s.Topics = v + } + return b.Complete() +} + +// NewPtrOffsetDeleteRequest returns a pointer to a default OffsetDeleteRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetDeleteRequest() *OffsetDeleteRequest { + var v OffsetDeleteRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteRequest. +func (v *OffsetDeleteRequest) Default() { +} + +// NewOffsetDeleteRequest returns a default OffsetDeleteRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteRequest() OffsetDeleteRequest { + var v OffsetDeleteRequest + v.Default() + return v +} + +type OffsetDeleteResponseTopicPartition struct { + // Partition is the partition being responded to. + Partition int32 + + // ErrorCode is any per partition error code. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the client is not authorized + // for the topic / partition. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the broker does not know of + // the requested topic. + // + // GROUP_SUBSCRIBED_TO_TOPIC is returned if the topic is still subscribed to. + ErrorCode int16 +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteResponseTopicPartition. +func (v *OffsetDeleteResponseTopicPartition) Default() { +} + +// NewOffsetDeleteResponseTopicPartition returns a default OffsetDeleteResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteResponseTopicPartition() OffsetDeleteResponseTopicPartition { + var v OffsetDeleteResponseTopicPartition + v.Default() + return v +} + +type OffsetDeleteResponseTopic struct { + // Topic is the topic being responded to. + Topic string + + // Partitions are partitions being responded to. + Partitions []OffsetDeleteResponseTopicPartition +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteResponseTopic. +func (v *OffsetDeleteResponseTopic) Default() { +} + +// NewOffsetDeleteResponseTopic returns a default OffsetDeleteResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteResponseTopic() OffsetDeleteResponseTopic { + var v OffsetDeleteResponseTopic + v.Default() + return v +} + +// OffsetDeleteResponse is a response to an offset delete request. +type OffsetDeleteResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ErrorCode is any group wide error. + // + // GROUP_AUTHORIZATION_FAILED is returned if the client is not authorized + // for the group. + // + // INVALID_GROUP_ID is returned in the requested group ID is invalid. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is not available. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the group is loading. + // + // NOT_COORDINATOR is returned if the requested broker is not the coordinator + // for the requested group. + // + // GROUP_ID_NOT_FOUND is returned if the group ID does not exist. + ErrorCode int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Topics are responses to requested topics. + Topics []OffsetDeleteResponseTopic +} + +func (*OffsetDeleteResponse) Key() int16 { return 47 } +func (*OffsetDeleteResponse) MaxVersion() int16 { return 0 } +func (v *OffsetDeleteResponse) SetVersion(version int16) { v.Version = version } +func (v *OffsetDeleteResponse) GetVersion() int16 { return v.Version } +func (v *OffsetDeleteResponse) IsFlexible() bool { return false } +func (v *OffsetDeleteResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *OffsetDeleteResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *OffsetDeleteResponse) RequestKind() Request { return &OffsetDeleteRequest{Version: v.Version} } + +func (v *OffsetDeleteResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Topic + dst = kbin.AppendString(dst, v) + } + { + v := v.Partitions + dst = kbin.AppendArrayLen(dst, len(v)) + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + } + } + } + } + return dst +} + +func (v *OffsetDeleteResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *OffsetDeleteResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *OffsetDeleteResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetDeleteResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeString() + } else { + v = b.String() + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + l = b.ArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]OffsetDeleteResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + } + v = a + s.Partitions = v + } + } + v = a + s.Topics = v + } + return b.Complete() +} + +// NewPtrOffsetDeleteResponse returns a pointer to a default OffsetDeleteResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrOffsetDeleteResponse() *OffsetDeleteResponse { + var v OffsetDeleteResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to OffsetDeleteResponse. +func (v *OffsetDeleteResponse) Default() { +} + +// NewOffsetDeleteResponse returns a default OffsetDeleteResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewOffsetDeleteResponse() OffsetDeleteResponse { + var v OffsetDeleteResponse + v.Default() + return v +} + +type DescribeClientQuotasRequestComponent struct { + // EntityType is the entity component type that this filter component + // applies to; some possible values are "user" or "client-id". + EntityType string + + // MatchType specifies how to match an entity, + // with 0 meaning match on the name exactly, + // 1 meaning match on the default name, + // and 2 meaning any specified name. + MatchType QuotasMatchType + + // Match is the string to match against, or null if unused for the given + // match type. + Match *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasRequestComponent. +func (v *DescribeClientQuotasRequestComponent) Default() { +} + +// NewDescribeClientQuotasRequestComponent returns a default DescribeClientQuotasRequestComponent +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasRequestComponent() DescribeClientQuotasRequestComponent { + var v DescribeClientQuotasRequestComponent + v.Default() + return v +} + +// DescribeClientQuotasRequest, proposed in KIP-546 and introduced with Kafka 2.6.0, +// provides a way to describe client quotas. +type DescribeClientQuotasRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Components is a list of match filters to apply for describing quota entities. + Components []DescribeClientQuotasRequestComponent + + // Strict signifies whether matches are strict; if true, the response + // excludes entities with unspecified entity types. + Strict bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*DescribeClientQuotasRequest) Key() int16 { return 48 } +func (*DescribeClientQuotasRequest) MaxVersion() int16 { return 1 } +func (v *DescribeClientQuotasRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeClientQuotasRequest) GetVersion() int16 { return v.Version } +func (v *DescribeClientQuotasRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *DescribeClientQuotasRequest) ResponseKind() Response { + r := &DescribeClientQuotasResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeClientQuotasRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeClientQuotasResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeClientQuotasResponse) + return resp, err +} + +func (v *DescribeClientQuotasRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Components + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.EntityType + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MatchType + { + v := int8(v) + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.Match + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Strict + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeClientQuotasRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeClientQuotasRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeClientQuotasRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := s.Components + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeClientQuotasRequestComponent, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.EntityType = v + } + { + var t QuotasMatchType + { + v := b.Int8() + t = QuotasMatchType(v) + } + v := t + s.MatchType = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Match = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Components = v + } + { + v := b.Bool() + s.Strict = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeClientQuotasRequest returns a pointer to a default DescribeClientQuotasRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeClientQuotasRequest() *DescribeClientQuotasRequest { + var v DescribeClientQuotasRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasRequest. +func (v *DescribeClientQuotasRequest) Default() { +} + +// NewDescribeClientQuotasRequest returns a default DescribeClientQuotasRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasRequest() DescribeClientQuotasRequest { + var v DescribeClientQuotasRequest + v.Default() + return v +} + +type DescribeClientQuotasResponseEntryEntity struct { + // Type is the entity type. + Type string + + // Name is the entity name, or null if the default. + Name *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasResponseEntryEntity. +func (v *DescribeClientQuotasResponseEntryEntity) Default() { +} + +// NewDescribeClientQuotasResponseEntryEntity returns a default DescribeClientQuotasResponseEntryEntity +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasResponseEntryEntity() DescribeClientQuotasResponseEntryEntity { + var v DescribeClientQuotasResponseEntryEntity + v.Default() + return v +} + +type DescribeClientQuotasResponseEntryValue struct { + // Key is the quota configuration key. + Key string + + // Value is the quota configuration value. + Value float64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasResponseEntryValue. +func (v *DescribeClientQuotasResponseEntryValue) Default() { +} + +// NewDescribeClientQuotasResponseEntryValue returns a default DescribeClientQuotasResponseEntryValue +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasResponseEntryValue() DescribeClientQuotasResponseEntryValue { + var v DescribeClientQuotasResponseEntryValue + v.Default() + return v +} + +type DescribeClientQuotasResponseEntry struct { + // Entity contains the quota entity components being described. + Entity []DescribeClientQuotasResponseEntryEntity + + // Values are quota values for the entity. + Values []DescribeClientQuotasResponseEntryValue + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasResponseEntry. +func (v *DescribeClientQuotasResponseEntry) Default() { +} + +// NewDescribeClientQuotasResponseEntry returns a default DescribeClientQuotasResponseEntry +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasResponseEntry() DescribeClientQuotasResponseEntry { + var v DescribeClientQuotasResponseEntry + v.Default() + return v +} + +// DescribeClientQuotasResponse is a response for a DescribeClientQuotasRequest. +type DescribeClientQuotasResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is any error for the request. + ErrorCode int16 + + // ErrorMessage is an error message for the request, or null if the request succeeded. + ErrorMessage *string + + // Entries contains entities that were matched. + Entries []DescribeClientQuotasResponseEntry + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*DescribeClientQuotasResponse) Key() int16 { return 48 } +func (*DescribeClientQuotasResponse) MaxVersion() int16 { return 1 } +func (v *DescribeClientQuotasResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeClientQuotasResponse) GetVersion() int16 { return v.Version } +func (v *DescribeClientQuotasResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *DescribeClientQuotasResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DescribeClientQuotasResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeClientQuotasResponse) RequestKind() Request { + return &DescribeClientQuotasRequest{Version: v.Version} +} + +func (v *DescribeClientQuotasResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Entries + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Entity + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Type + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Values + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + dst = kbin.AppendFloat64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeClientQuotasResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeClientQuotasResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeClientQuotasResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Entries + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []DescribeClientQuotasResponseEntry{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeClientQuotasResponseEntry, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := s.Entity + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeClientQuotasResponseEntryEntity, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Type = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Name = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entity = v + } + { + v := s.Values + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeClientQuotasResponseEntryValue, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + v := b.Float64() + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Values = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entries = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeClientQuotasResponse returns a pointer to a default DescribeClientQuotasResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeClientQuotasResponse() *DescribeClientQuotasResponse { + var v DescribeClientQuotasResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClientQuotasResponse. +func (v *DescribeClientQuotasResponse) Default() { +} + +// NewDescribeClientQuotasResponse returns a default DescribeClientQuotasResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClientQuotasResponse() DescribeClientQuotasResponse { + var v DescribeClientQuotasResponse + v.Default() + return v +} + +type AlterClientQuotasRequestEntryEntity struct { + // Type is the entity component's type; e.g. "client-id", "user" or "ip". + Type string + + // Name is the name of the entity, or null for the default. + Name *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasRequestEntryEntity. +func (v *AlterClientQuotasRequestEntryEntity) Default() { +} + +// NewAlterClientQuotasRequestEntryEntity returns a default AlterClientQuotasRequestEntryEntity +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasRequestEntryEntity() AlterClientQuotasRequestEntryEntity { + var v AlterClientQuotasRequestEntryEntity + v.Default() + return v +} + +type AlterClientQuotasRequestEntryOp struct { + // Key is the quota configuration key to alter. + Key string + + // Value is the value to set; ignored if remove is true. + Value float64 + + // Remove is whether the quota configuration value should be removed or set. + Remove bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasRequestEntryOp. +func (v *AlterClientQuotasRequestEntryOp) Default() { +} + +// NewAlterClientQuotasRequestEntryOp returns a default AlterClientQuotasRequestEntryOp +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasRequestEntryOp() AlterClientQuotasRequestEntryOp { + var v AlterClientQuotasRequestEntryOp + v.Default() + return v +} + +type AlterClientQuotasRequestEntry struct { + // Entity contains the components of a quota entity to alter. + Entity []AlterClientQuotasRequestEntryEntity + + // Ops contains quota configuration entries to alter. + Ops []AlterClientQuotasRequestEntryOp + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasRequestEntry. +func (v *AlterClientQuotasRequestEntry) Default() { +} + +// NewAlterClientQuotasRequestEntry returns a default AlterClientQuotasRequestEntry +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasRequestEntry() AlterClientQuotasRequestEntry { + var v AlterClientQuotasRequestEntry + v.Default() + return v +} + +// AlterClientQuotaRequest, proposed in KIP-546 and introduced with Kafka 2.6.0, +// provides a way to alter client quotas. +type AlterClientQuotasRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Entries are quota configuration entries to alter. + Entries []AlterClientQuotasRequestEntry + + // ValidateOnly is makes this request a dry-run; the alteration is validated + // but not performed. + ValidateOnly bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*AlterClientQuotasRequest) Key() int16 { return 49 } +func (*AlterClientQuotasRequest) MaxVersion() int16 { return 1 } +func (v *AlterClientQuotasRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterClientQuotasRequest) GetVersion() int16 { return v.Version } +func (v *AlterClientQuotasRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *AlterClientQuotasRequest) ResponseKind() Response { + r := &AlterClientQuotasResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterClientQuotasRequest) RequestWith(ctx context.Context, r Requestor) (*AlterClientQuotasResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterClientQuotasResponse) + return resp, err +} + +func (v *AlterClientQuotasRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.Entries + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Entity + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Type + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Ops + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + dst = kbin.AppendFloat64(dst, v) + } + { + v := v.Remove + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterClientQuotasRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterClientQuotasRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterClientQuotasRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := s.Entries + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterClientQuotasRequestEntry, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := s.Entity + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterClientQuotasRequestEntryEntity, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Type = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Name = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entity = v + } + { + v := s.Ops + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterClientQuotasRequestEntryOp, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + v := b.Float64() + s.Value = v + } + { + v := b.Bool() + s.Remove = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Ops = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entries = v + } + { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterClientQuotasRequest returns a pointer to a default AlterClientQuotasRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterClientQuotasRequest() *AlterClientQuotasRequest { + var v AlterClientQuotasRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasRequest. +func (v *AlterClientQuotasRequest) Default() { +} + +// NewAlterClientQuotasRequest returns a default AlterClientQuotasRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasRequest() AlterClientQuotasRequest { + var v AlterClientQuotasRequest + v.Default() + return v +} + +type AlterClientQuotasResponseEntryEntity struct { + // Type is the entity component's type; e.g. "client-id" or "user". + Type string + + // Name is the name of the entity, or null for the default. + Name *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasResponseEntryEntity. +func (v *AlterClientQuotasResponseEntryEntity) Default() { +} + +// NewAlterClientQuotasResponseEntryEntity returns a default AlterClientQuotasResponseEntryEntity +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasResponseEntryEntity() AlterClientQuotasResponseEntryEntity { + var v AlterClientQuotasResponseEntryEntity + v.Default() + return v +} + +type AlterClientQuotasResponseEntry struct { + // ErrorCode is the error code for an alter on a matched entity. + ErrorCode int16 + + // ErrorMessage is an informative message if the alter on this entity failed. + ErrorMessage *string + + // Entity contains the components of a matched entity. + Entity []AlterClientQuotasResponseEntryEntity + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasResponseEntry. +func (v *AlterClientQuotasResponseEntry) Default() { +} + +// NewAlterClientQuotasResponseEntry returns a default AlterClientQuotasResponseEntry +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasResponseEntry() AlterClientQuotasResponseEntry { + var v AlterClientQuotasResponseEntry + v.Default() + return v +} + +// AlterClientQuotasResponse is a response to an AlterClientQuotasRequest. +type AlterClientQuotasResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Entries contains results for the alter request. + Entries []AlterClientQuotasResponseEntry + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*AlterClientQuotasResponse) Key() int16 { return 49 } +func (*AlterClientQuotasResponse) MaxVersion() int16 { return 1 } +func (v *AlterClientQuotasResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterClientQuotasResponse) GetVersion() int16 { return v.Version } +func (v *AlterClientQuotasResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *AlterClientQuotasResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *AlterClientQuotasResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AlterClientQuotasResponse) RequestKind() Request { + return &AlterClientQuotasRequest{Version: v.Version} +} + +func (v *AlterClientQuotasResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Entries + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Entity + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Type + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterClientQuotasResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterClientQuotasResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterClientQuotasResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Entries + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterClientQuotasResponseEntry, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Entity + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterClientQuotasResponseEntryEntity, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Type = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Name = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entity = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Entries = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterClientQuotasResponse returns a pointer to a default AlterClientQuotasResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterClientQuotasResponse() *AlterClientQuotasResponse { + var v AlterClientQuotasResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterClientQuotasResponse. +func (v *AlterClientQuotasResponse) Default() { +} + +// NewAlterClientQuotasResponse returns a default AlterClientQuotasResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterClientQuotasResponse() AlterClientQuotasResponse { + var v AlterClientQuotasResponse + v.Default() + return v +} + +type DescribeUserSCRAMCredentialsRequestUser struct { + // The user name. + Name string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeUserSCRAMCredentialsRequestUser. +func (v *DescribeUserSCRAMCredentialsRequestUser) Default() { +} + +// NewDescribeUserSCRAMCredentialsRequestUser returns a default DescribeUserSCRAMCredentialsRequestUser +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeUserSCRAMCredentialsRequestUser() DescribeUserSCRAMCredentialsRequestUser { + var v DescribeUserSCRAMCredentialsRequestUser + v.Default() + return v +} + +// DescribeUserSCRAMCredentialsRequest, proposed in KIP-554 and introduced +// with Kafka 2.7.0, describes user SCRAM credentials. +// +// This request was introduced as part of the overarching KIP-500 initiative, +// which is to remove Zookeeper as a dependency. +// +// This request requires DESCRIBE on CLUSTER. +type DescribeUserSCRAMCredentialsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The users to describe, or null to describe all. + Users []DescribeUserSCRAMCredentialsRequestUser + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeUserSCRAMCredentialsRequest) Key() int16 { return 50 } +func (*DescribeUserSCRAMCredentialsRequest) MaxVersion() int16 { return 0 } +func (v *DescribeUserSCRAMCredentialsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeUserSCRAMCredentialsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeUserSCRAMCredentialsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeUserSCRAMCredentialsRequest) ResponseKind() Response { + r := &DescribeUserSCRAMCredentialsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeUserSCRAMCredentialsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeUserSCRAMCredentialsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeUserSCRAMCredentialsResponse) + return resp, err +} + +func (v *DescribeUserSCRAMCredentialsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Users + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeUserSCRAMCredentialsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeUserSCRAMCredentialsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeUserSCRAMCredentialsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Users + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []DescribeUserSCRAMCredentialsRequestUser{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeUserSCRAMCredentialsRequestUser, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Users = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeUserSCRAMCredentialsRequest returns a pointer to a default DescribeUserSCRAMCredentialsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeUserSCRAMCredentialsRequest() *DescribeUserSCRAMCredentialsRequest { + var v DescribeUserSCRAMCredentialsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeUserSCRAMCredentialsRequest. +func (v *DescribeUserSCRAMCredentialsRequest) Default() { +} + +// NewDescribeUserSCRAMCredentialsRequest returns a default DescribeUserSCRAMCredentialsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeUserSCRAMCredentialsRequest() DescribeUserSCRAMCredentialsRequest { + var v DescribeUserSCRAMCredentialsRequest + v.Default() + return v +} + +type DescribeUserSCRAMCredentialsResponseResultCredentialInfo struct { + // The SCRAM mechanism for this user, where 0 is UNKNOWN, 1 is SCRAM-SHA-256, + // and 2 is SCRAM-SHA-512. + Mechanism int8 + + // The number of iterations used in the SCRAM credential. + Iterations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeUserSCRAMCredentialsResponseResultCredentialInfo. +func (v *DescribeUserSCRAMCredentialsResponseResultCredentialInfo) Default() { +} + +// NewDescribeUserSCRAMCredentialsResponseResultCredentialInfo returns a default DescribeUserSCRAMCredentialsResponseResultCredentialInfo +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeUserSCRAMCredentialsResponseResultCredentialInfo() DescribeUserSCRAMCredentialsResponseResultCredentialInfo { + var v DescribeUserSCRAMCredentialsResponseResultCredentialInfo + v.Default() + return v +} + +type DescribeUserSCRAMCredentialsResponseResult struct { + // The name this result corresponds to. + User string + + // The user-level error code. + // + // RESOURCE_NOT_FOUND if the user does not exist or has no credentials. + // + // DUPLICATE_RESOURCE if the user is requested twice+. + ErrorCode int16 + + // The user-level error message, if any. + ErrorMessage *string + + // Information about the SCRAM credentials for this user. + CredentialInfos []DescribeUserSCRAMCredentialsResponseResultCredentialInfo + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeUserSCRAMCredentialsResponseResult. +func (v *DescribeUserSCRAMCredentialsResponseResult) Default() { +} + +// NewDescribeUserSCRAMCredentialsResponseResult returns a default DescribeUserSCRAMCredentialsResponseResult +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeUserSCRAMCredentialsResponseResult() DescribeUserSCRAMCredentialsResponseResult { + var v DescribeUserSCRAMCredentialsResponseResult + v.Default() + return v +} + +// DescribeUserSCRAMCredentialsResponse is a response for a +// DescribeUserSCRAMCredentialsRequest. +type DescribeUserSCRAMCredentialsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The request-level error code. This is 0 except for auth or infra issues. + // + // CLUSTER_AUTHORIZATION_FAILED if you do not have DESCRIBE on CLUSTER. + ErrorCode int16 + + // The request-level error message, if any. + ErrorMessage *string + + // Results for descriptions, one per user. + Results []DescribeUserSCRAMCredentialsResponseResult + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeUserSCRAMCredentialsResponse) Key() int16 { return 50 } +func (*DescribeUserSCRAMCredentialsResponse) MaxVersion() int16 { return 0 } +func (v *DescribeUserSCRAMCredentialsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeUserSCRAMCredentialsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeUserSCRAMCredentialsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeUserSCRAMCredentialsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DescribeUserSCRAMCredentialsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeUserSCRAMCredentialsResponse) RequestKind() Request { + return &DescribeUserSCRAMCredentialsRequest{Version: v.Version} +} + +func (v *DescribeUserSCRAMCredentialsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Results + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.User + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.CredentialInfos + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Mechanism + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Iterations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeUserSCRAMCredentialsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeUserSCRAMCredentialsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeUserSCRAMCredentialsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Results + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeUserSCRAMCredentialsResponseResult, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.User = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.CredentialInfos + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeUserSCRAMCredentialsResponseResultCredentialInfo, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int8() + s.Mechanism = v + } + { + v := b.Int32() + s.Iterations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.CredentialInfos = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Results = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeUserSCRAMCredentialsResponse returns a pointer to a default DescribeUserSCRAMCredentialsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeUserSCRAMCredentialsResponse() *DescribeUserSCRAMCredentialsResponse { + var v DescribeUserSCRAMCredentialsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeUserSCRAMCredentialsResponse. +func (v *DescribeUserSCRAMCredentialsResponse) Default() { +} + +// NewDescribeUserSCRAMCredentialsResponse returns a default DescribeUserSCRAMCredentialsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeUserSCRAMCredentialsResponse() DescribeUserSCRAMCredentialsResponse { + var v DescribeUserSCRAMCredentialsResponse + v.Default() + return v +} + +type AlterUserSCRAMCredentialsRequestDeletion struct { + // The user name to match for removal. + Name string + + // The mechanism for the user name to remove. + Mechanism int8 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterUserSCRAMCredentialsRequestDeletion. +func (v *AlterUserSCRAMCredentialsRequestDeletion) Default() { +} + +// NewAlterUserSCRAMCredentialsRequestDeletion returns a default AlterUserSCRAMCredentialsRequestDeletion +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterUserSCRAMCredentialsRequestDeletion() AlterUserSCRAMCredentialsRequestDeletion { + var v AlterUserSCRAMCredentialsRequestDeletion + v.Default() + return v +} + +type AlterUserSCRAMCredentialsRequestUpsertion struct { + // The user name to use. + Name string + + // The mechanism to use for creating, where 1 is SCRAM-SHA-256 and 2 is + // SCRAM-SHA-512. + Mechanism int8 + + // The number of iterations to use. This must be more than the minimum for + // the mechanism and cannot be more than 16384. + Iterations int32 + + // A random salt generated by the client. + Salt []byte + + // The salted password to use. + SaltedPassword []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterUserSCRAMCredentialsRequestUpsertion. +func (v *AlterUserSCRAMCredentialsRequestUpsertion) Default() { +} + +// NewAlterUserSCRAMCredentialsRequestUpsertion returns a default AlterUserSCRAMCredentialsRequestUpsertion +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterUserSCRAMCredentialsRequestUpsertion() AlterUserSCRAMCredentialsRequestUpsertion { + var v AlterUserSCRAMCredentialsRequestUpsertion + v.Default() + return v +} + +// AlterUserSCRAMCredentialsRequest, proposed in KIP-554 and introduced +// with Kafka 2.7.0, alters or deletes user SCRAM credentials. +// +// This request was introduced as part of the overarching KIP-500 initiative, +// which is to remove Zookeeper as a dependency. +// +// This request requires ALTER on CLUSTER. +type AlterUserSCRAMCredentialsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The SCRAM credentials to remove. + Deletions []AlterUserSCRAMCredentialsRequestDeletion + + // The SCRAM credentials to update or insert. + Upsertions []AlterUserSCRAMCredentialsRequestUpsertion + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterUserSCRAMCredentialsRequest) Key() int16 { return 51 } +func (*AlterUserSCRAMCredentialsRequest) MaxVersion() int16 { return 0 } +func (v *AlterUserSCRAMCredentialsRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterUserSCRAMCredentialsRequest) GetVersion() int16 { return v.Version } +func (v *AlterUserSCRAMCredentialsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterUserSCRAMCredentialsRequest) IsAdminRequest() {} +func (v *AlterUserSCRAMCredentialsRequest) ResponseKind() Response { + r := &AlterUserSCRAMCredentialsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterUserSCRAMCredentialsRequest) RequestWith(ctx context.Context, r Requestor) (*AlterUserSCRAMCredentialsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterUserSCRAMCredentialsResponse) + return resp, err +} + +func (v *AlterUserSCRAMCredentialsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Deletions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Mechanism + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Upsertions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Mechanism + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Iterations + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Salt + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.SaltedPassword + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterUserSCRAMCredentialsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterUserSCRAMCredentialsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterUserSCRAMCredentialsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Deletions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterUserSCRAMCredentialsRequestDeletion, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int8() + s.Mechanism = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Deletions = v + } + { + v := s.Upsertions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterUserSCRAMCredentialsRequestUpsertion, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int8() + s.Mechanism = v + } + { + v := b.Int32() + s.Iterations = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Salt = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.SaltedPassword = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Upsertions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterUserSCRAMCredentialsRequest returns a pointer to a default AlterUserSCRAMCredentialsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterUserSCRAMCredentialsRequest() *AlterUserSCRAMCredentialsRequest { + var v AlterUserSCRAMCredentialsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterUserSCRAMCredentialsRequest. +func (v *AlterUserSCRAMCredentialsRequest) Default() { +} + +// NewAlterUserSCRAMCredentialsRequest returns a default AlterUserSCRAMCredentialsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterUserSCRAMCredentialsRequest() AlterUserSCRAMCredentialsRequest { + var v AlterUserSCRAMCredentialsRequest + v.Default() + return v +} + +type AlterUserSCRAMCredentialsResponseResult struct { + // The name this result corresponds to. + User string + + // The user-level error code. + ErrorCode int16 + + // The user-level error message, if any. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterUserSCRAMCredentialsResponseResult. +func (v *AlterUserSCRAMCredentialsResponseResult) Default() { +} + +// NewAlterUserSCRAMCredentialsResponseResult returns a default AlterUserSCRAMCredentialsResponseResult +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterUserSCRAMCredentialsResponseResult() AlterUserSCRAMCredentialsResponseResult { + var v AlterUserSCRAMCredentialsResponseResult + v.Default() + return v +} + +// AlterUserSCRAMCredentialsResponse is a response for an +// AlterUserSCRAMCredentialsRequest. +type AlterUserSCRAMCredentialsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The results for deletions and upsertions. + Results []AlterUserSCRAMCredentialsResponseResult + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterUserSCRAMCredentialsResponse) Key() int16 { return 51 } +func (*AlterUserSCRAMCredentialsResponse) MaxVersion() int16 { return 0 } +func (v *AlterUserSCRAMCredentialsResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterUserSCRAMCredentialsResponse) GetVersion() int16 { return v.Version } +func (v *AlterUserSCRAMCredentialsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterUserSCRAMCredentialsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *AlterUserSCRAMCredentialsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AlterUserSCRAMCredentialsResponse) RequestKind() Request { + return &AlterUserSCRAMCredentialsRequest{Version: v.Version} +} + +func (v *AlterUserSCRAMCredentialsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Results + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.User + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterUserSCRAMCredentialsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterUserSCRAMCredentialsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterUserSCRAMCredentialsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Results + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterUserSCRAMCredentialsResponseResult, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.User = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Results = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterUserSCRAMCredentialsResponse returns a pointer to a default AlterUserSCRAMCredentialsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterUserSCRAMCredentialsResponse() *AlterUserSCRAMCredentialsResponse { + var v AlterUserSCRAMCredentialsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterUserSCRAMCredentialsResponse. +func (v *AlterUserSCRAMCredentialsResponse) Default() { +} + +// NewAlterUserSCRAMCredentialsResponse returns a default AlterUserSCRAMCredentialsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterUserSCRAMCredentialsResponse() AlterUserSCRAMCredentialsResponse { + var v AlterUserSCRAMCredentialsResponse + v.Default() + return v +} + +type VoteRequestTopicPartition struct { + Partition int32 + + // The bumped epoch of the voter sending the request. + CandidateEpoch int32 + + // The ID of the voter sending the request. + CandidateID int32 + + // The directory ID of the voter sending the request. + CandidateDirectoryID [16]byte // v1+ + + // The directory ID of the voter receiving the request. + VoterDirectoryID [16]byte // v1+ + + // The epoch of the last record written to the metadata log. + LastOffsetEpoch int32 + + // The offset of the last record written to the metadata log. + LastOffset int64 + + // Whether the request is a PreVote request (not persisted) or not. + PreVote bool // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteRequestTopicPartition. +func (v *VoteRequestTopicPartition) Default() { +} + +// NewVoteRequestTopicPartition returns a default VoteRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteRequestTopicPartition() VoteRequestTopicPartition { + var v VoteRequestTopicPartition + v.Default() + return v +} + +type VoteRequestTopic struct { + Topic string + + Partitions []VoteRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteRequestTopic. +func (v *VoteRequestTopic) Default() { +} + +// NewVoteRequestTopic returns a default VoteRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteRequestTopic() VoteRequestTopic { + var v VoteRequestTopic + v.Default() + return v +} + +// Part of KIP-595 to replace Kafka's dependence on Zookeeper with a +// Kafka-only raft protocol, +// VoteRequest is used by voters to hold a leader election. +// +// Since this is relatively Kafka internal, most fields are left undocumented. +type VoteRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ClusterID *string + + // This field has a default of -1. + VoterID int32 // v1+ + + Topics []VoteRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*VoteRequest) Key() int16 { return 52 } +func (*VoteRequest) MaxVersion() int16 { return 2 } +func (v *VoteRequest) SetVersion(version int16) { v.Version = version } +func (v *VoteRequest) GetVersion() int16 { return v.Version } +func (v *VoteRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *VoteRequest) IsAdminRequest() {} +func (v *VoteRequest) ResponseKind() Response { + r := &VoteResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *VoteRequest) RequestWith(ctx context.Context, r Requestor) (*VoteResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*VoteResponse) + return resp, err +} + +func (v *VoteRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CandidateEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CandidateID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.CandidateDirectoryID + dst = kbin.AppendUuid(dst, v) + } + if version >= 1 { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.LastOffsetEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 2 { + v := v.PreVote + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *VoteRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *VoteRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *VoteRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + if version >= 1 { + v := b.Int32() + s.VoterID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]VoteRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]VoteRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.CandidateEpoch = v + } + { + v := b.Int32() + s.CandidateID = v + } + if version >= 1 { + v := b.Uuid() + s.CandidateDirectoryID = v + } + if version >= 1 { + v := b.Uuid() + s.VoterDirectoryID = v + } + { + v := b.Int32() + s.LastOffsetEpoch = v + } + { + v := b.Int64() + s.LastOffset = v + } + if version >= 2 { + v := b.Bool() + s.PreVote = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrVoteRequest returns a pointer to a default VoteRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrVoteRequest() *VoteRequest { + var v VoteRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteRequest. +func (v *VoteRequest) Default() { + v.VoterID = -1 +} + +// NewVoteRequest returns a default VoteRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteRequest() VoteRequest { + var v VoteRequest + v.Default() + return v +} + +type VoteResponseTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // The ID of the current leader, or -1 if the leader is unknown. + LeaderID int32 + + // The latest known leader epoch. + LeaderEpoch int32 + + // Whether the vote was granted. + VoteGranted bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteResponseTopicPartition. +func (v *VoteResponseTopicPartition) Default() { +} + +// NewVoteResponseTopicPartition returns a default VoteResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteResponseTopicPartition() VoteResponseTopicPartition { + var v VoteResponseTopicPartition + v.Default() + return v +} + +type VoteResponseTopic struct { + Topic string + + Partitions []VoteResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteResponseTopic. +func (v *VoteResponseTopic) Default() { +} + +// NewVoteResponseTopic returns a default VoteResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteResponseTopic() VoteResponseTopic { + var v VoteResponseTopic + v.Default() + return v +} + +type VoteResponseNodeEndpoint struct { + NodeID int32 // v1+ + + Host string // v1+ + + Port uint16 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteResponseNodeEndpoint. +func (v *VoteResponseNodeEndpoint) Default() { +} + +// NewVoteResponseNodeEndpoint returns a default VoteResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteResponseNodeEndpoint() VoteResponseNodeEndpoint { + var v VoteResponseNodeEndpoint + v.Default() + return v +} + +type VoteResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + Topics []VoteResponseTopic + + // Endpoints for all leaders enumerated in PartitionData. + NodeEndpoints []VoteResponseNodeEndpoint // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*VoteResponse) Key() int16 { return 52 } +func (*VoteResponse) MaxVersion() int16 { return 2 } +func (v *VoteResponse) SetVersion(version int16) { v.Version = version } +func (v *VoteResponse) GetVersion() int16 { return v.Version } +func (v *VoteResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *VoteResponse) RequestKind() Request { return &VoteRequest{Version: v.Version} } + +func (v *VoteResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoteGranted + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if len(v.NodeEndpoints) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.NodeEndpoints + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fNodeEndpoints: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 1 { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fNodeEndpoints + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *VoteResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *VoteResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *VoteResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]VoteResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]VoteResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Bool() + s.VoteGranted = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]VoteResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 1 { + v := b.Int32() + s.NodeID = v + } + if version >= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 1 { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrVoteResponse returns a pointer to a default VoteResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrVoteResponse() *VoteResponse { + var v VoteResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to VoteResponse. +func (v *VoteResponse) Default() { +} + +// NewVoteResponse returns a default VoteResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewVoteResponse() VoteResponse { + var v VoteResponse + v.Default() + return v +} + +type BeginQuorumEpochRequestTopicPartition struct { + Partition int32 + + // Directory ID of the receiving replica. + VoterDirectoryID [16]byte // v1+ + + // The ID of the newly elected leader. + LeaderID int32 + + // The epoch of the newly elected leader. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochRequestTopicPartition. +func (v *BeginQuorumEpochRequestTopicPartition) Default() { +} + +// NewBeginQuorumEpochRequestTopicPartition returns a default BeginQuorumEpochRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochRequestTopicPartition() BeginQuorumEpochRequestTopicPartition { + var v BeginQuorumEpochRequestTopicPartition + v.Default() + return v +} + +type BeginQuorumEpochRequestTopic struct { + Topic string + + Partitions []BeginQuorumEpochRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochRequestTopic. +func (v *BeginQuorumEpochRequestTopic) Default() { +} + +// NewBeginQuorumEpochRequestTopic returns a default BeginQuorumEpochRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochRequestTopic() BeginQuorumEpochRequestTopic { + var v BeginQuorumEpochRequestTopic + v.Default() + return v +} + +type BeginQuorumEpochRequestLeaderEndpoint struct { + Name string + + Host string + + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochRequestLeaderEndpoint. +func (v *BeginQuorumEpochRequestLeaderEndpoint) Default() { +} + +// NewBeginQuorumEpochRequestLeaderEndpoint returns a default BeginQuorumEpochRequestLeaderEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochRequestLeaderEndpoint() BeginQuorumEpochRequestLeaderEndpoint { + var v BeginQuorumEpochRequestLeaderEndpoint + v.Default() + return v +} + +// Part of KIP-595 to replace Kafka's dependence on Zookeeper with a +// Kafka-only raft protocol, +// BeginQuorumEpochRequest is sent by a leader (once it has enough votes) +// to all voters in the election. +// +// Since this is relatively Kafka internal, most fields are left undocumented. +type BeginQuorumEpochRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ClusterID *string + + // Replica ID of the voter receiving the request. + // + // This field has a default of -1. + VoterID int32 // v1+ + + Topics []BeginQuorumEpochRequestTopic + + // Endpoints for the leader. + LeaderEndpoints []BeginQuorumEpochRequestLeaderEndpoint // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*BeginQuorumEpochRequest) Key() int16 { return 53 } +func (*BeginQuorumEpochRequest) MaxVersion() int16 { return 1 } +func (v *BeginQuorumEpochRequest) SetVersion(version int16) { v.Version = version } +func (v *BeginQuorumEpochRequest) GetVersion() int16 { return v.Version } +func (v *BeginQuorumEpochRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *BeginQuorumEpochRequest) IsAdminRequest() {} +func (v *BeginQuorumEpochRequest) ResponseKind() Response { + r := &BeginQuorumEpochResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *BeginQuorumEpochRequest) RequestWith(ctx context.Context, r Requestor) (*BeginQuorumEpochResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*BeginQuorumEpochResponse) + return resp, err +} + +func (v *BeginQuorumEpochRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.LeaderEndpoints + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BeginQuorumEpochRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BeginQuorumEpochRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BeginQuorumEpochRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + if version >= 1 { + v := b.Int32() + s.VoterID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if version >= 1 { + v := b.Uuid() + s.VoterDirectoryID = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 1 { + v := s.LeaderEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochRequestLeaderEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.LeaderEndpoints = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrBeginQuorumEpochRequest returns a pointer to a default BeginQuorumEpochRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBeginQuorumEpochRequest() *BeginQuorumEpochRequest { + var v BeginQuorumEpochRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochRequest. +func (v *BeginQuorumEpochRequest) Default() { + v.VoterID = -1 +} + +// NewBeginQuorumEpochRequest returns a default BeginQuorumEpochRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochRequest() BeginQuorumEpochRequest { + var v BeginQuorumEpochRequest + v.Default() + return v +} + +type BeginQuorumEpochResponseTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // The ID of the current leader, or -1 if the leader is unknown. + LeaderID int32 + + // The latest known leader epoch. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochResponseTopicPartition. +func (v *BeginQuorumEpochResponseTopicPartition) Default() { +} + +// NewBeginQuorumEpochResponseTopicPartition returns a default BeginQuorumEpochResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochResponseTopicPartition() BeginQuorumEpochResponseTopicPartition { + var v BeginQuorumEpochResponseTopicPartition + v.Default() + return v +} + +type BeginQuorumEpochResponseTopic struct { + Topic string + + Partitions []BeginQuorumEpochResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochResponseTopic. +func (v *BeginQuorumEpochResponseTopic) Default() { +} + +// NewBeginQuorumEpochResponseTopic returns a default BeginQuorumEpochResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochResponseTopic() BeginQuorumEpochResponseTopic { + var v BeginQuorumEpochResponseTopic + v.Default() + return v +} + +type BeginQuorumEpochResponseNodeEndpoint struct { + NodeID int32 // v1+ + + Host string // v1+ + + Port uint16 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochResponseNodeEndpoint. +func (v *BeginQuorumEpochResponseNodeEndpoint) Default() { +} + +// NewBeginQuorumEpochResponseNodeEndpoint returns a default BeginQuorumEpochResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochResponseNodeEndpoint() BeginQuorumEpochResponseNodeEndpoint { + var v BeginQuorumEpochResponseNodeEndpoint + v.Default() + return v +} + +type BeginQuorumEpochResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + Topics []BeginQuorumEpochResponseTopic + + // Endpoints for all leaders enumerated in PartitionData. + NodeEndpoints []BeginQuorumEpochResponseNodeEndpoint // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*BeginQuorumEpochResponse) Key() int16 { return 53 } +func (*BeginQuorumEpochResponse) MaxVersion() int16 { return 1 } +func (v *BeginQuorumEpochResponse) SetVersion(version int16) { v.Version = version } +func (v *BeginQuorumEpochResponse) GetVersion() int16 { return v.Version } +func (v *BeginQuorumEpochResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *BeginQuorumEpochResponse) RequestKind() Request { + return &BeginQuorumEpochRequest{Version: v.Version} +} + +func (v *BeginQuorumEpochResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if len(v.NodeEndpoints) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.NodeEndpoints + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fNodeEndpoints: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 1 { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fNodeEndpoints + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BeginQuorumEpochResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BeginQuorumEpochResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BeginQuorumEpochResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BeginQuorumEpochResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 1 { + v := b.Int32() + s.NodeID = v + } + if version >= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 1 { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrBeginQuorumEpochResponse returns a pointer to a default BeginQuorumEpochResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBeginQuorumEpochResponse() *BeginQuorumEpochResponse { + var v BeginQuorumEpochResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BeginQuorumEpochResponse. +func (v *BeginQuorumEpochResponse) Default() { +} + +// NewBeginQuorumEpochResponse returns a default BeginQuorumEpochResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewBeginQuorumEpochResponse() BeginQuorumEpochResponse { + var v BeginQuorumEpochResponse + v.Default() + return v +} + +type EndQuorumEpochRequestTopicPartitionPreferredCandidate struct { + CandidateID int32 + + CandidateDirectoryID [16]byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochRequestTopicPartitionPreferredCandidate. +func (v *EndQuorumEpochRequestTopicPartitionPreferredCandidate) Default() { +} + +// NewEndQuorumEpochRequestTopicPartitionPreferredCandidate returns a default EndQuorumEpochRequestTopicPartitionPreferredCandidate +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochRequestTopicPartitionPreferredCandidate() EndQuorumEpochRequestTopicPartitionPreferredCandidate { + var v EndQuorumEpochRequestTopicPartitionPreferredCandidate + v.Default() + return v +} + +type EndQuorumEpochRequestTopicPartition struct { + Partition int32 + + // The current leader ID that is resigning. + LeaderID int32 + + // The current epoch. + LeaderEpoch int32 + + // A sorted list of preferred successors to start the election. + PreferredSuccessors []int32 + + // A sorted list of preferred candidates to start the election. + PreferredCandidates []EndQuorumEpochRequestTopicPartitionPreferredCandidate // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochRequestTopicPartition. +func (v *EndQuorumEpochRequestTopicPartition) Default() { +} + +// NewEndQuorumEpochRequestTopicPartition returns a default EndQuorumEpochRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochRequestTopicPartition() EndQuorumEpochRequestTopicPartition { + var v EndQuorumEpochRequestTopicPartition + v.Default() + return v +} + +type EndQuorumEpochRequestTopic struct { + Topic string + + Partitions []EndQuorumEpochRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochRequestTopic. +func (v *EndQuorumEpochRequestTopic) Default() { +} + +// NewEndQuorumEpochRequestTopic returns a default EndQuorumEpochRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochRequestTopic() EndQuorumEpochRequestTopic { + var v EndQuorumEpochRequestTopic + v.Default() + return v +} + +type EndQuorumEpochRequestLeaderEndpoint struct { + Name string + + Host string + + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochRequestLeaderEndpoint. +func (v *EndQuorumEpochRequestLeaderEndpoint) Default() { +} + +// NewEndQuorumEpochRequestLeaderEndpoint returns a default EndQuorumEpochRequestLeaderEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochRequestLeaderEndpoint() EndQuorumEpochRequestLeaderEndpoint { + var v EndQuorumEpochRequestLeaderEndpoint + v.Default() + return v +} + +// Part of KIP-595 to replace Kafka's dependence on Zookeeper with a +// Kafka-only raft protocol, +// EndQuorumEpochRequest is sent by a leader to gracefully step down as leader +// (i.e. on shutdown). Stepping down begins a new election. +// +// Since this is relatively Kafka internal, most fields are left undocumented. +type EndQuorumEpochRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ClusterID *string + + Topics []EndQuorumEpochRequestTopic + + // Endpoints for the leader. + LeaderEndpoints []EndQuorumEpochRequestLeaderEndpoint // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*EndQuorumEpochRequest) Key() int16 { return 54 } +func (*EndQuorumEpochRequest) MaxVersion() int16 { return 1 } +func (v *EndQuorumEpochRequest) SetVersion(version int16) { v.Version = version } +func (v *EndQuorumEpochRequest) GetVersion() int16 { return v.Version } +func (v *EndQuorumEpochRequest) IsFlexible() bool { return v.Version >= 1 } +func (v *EndQuorumEpochRequest) IsAdminRequest() {} +func (v *EndQuorumEpochRequest) ResponseKind() Response { + r := &EndQuorumEpochResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *EndQuorumEpochRequest) RequestWith(ctx context.Context, r Requestor) (*EndQuorumEpochResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*EndQuorumEpochResponse) + return resp, err +} + +func (v *EndQuorumEpochRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 0 { + v := v.PreferredSuccessors + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 1 { + v := v.PreferredCandidates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.CandidateID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CandidateDirectoryID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.LeaderEndpoints + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EndQuorumEpochRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EndQuorumEpochRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EndQuorumEpochRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if version >= 0 && version <= 0 { + v := s.PreferredSuccessors + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.PreferredSuccessors = v + } + if version >= 1 { + v := s.PreferredCandidates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochRequestTopicPartitionPreferredCandidate, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.CandidateID = v + } + { + v := b.Uuid() + s.CandidateDirectoryID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PreferredCandidates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 1 { + v := s.LeaderEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochRequestLeaderEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.LeaderEndpoints = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrEndQuorumEpochRequest returns a pointer to a default EndQuorumEpochRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEndQuorumEpochRequest() *EndQuorumEpochRequest { + var v EndQuorumEpochRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochRequest. +func (v *EndQuorumEpochRequest) Default() { +} + +// NewEndQuorumEpochRequest returns a default EndQuorumEpochRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochRequest() EndQuorumEpochRequest { + var v EndQuorumEpochRequest + v.Default() + return v +} + +type EndQuorumEpochResponseTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // The ID of the current leader, or -1 if the leader is unknown. + LeaderID int32 + + // The latest known leader epoch. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochResponseTopicPartition. +func (v *EndQuorumEpochResponseTopicPartition) Default() { +} + +// NewEndQuorumEpochResponseTopicPartition returns a default EndQuorumEpochResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochResponseTopicPartition() EndQuorumEpochResponseTopicPartition { + var v EndQuorumEpochResponseTopicPartition + v.Default() + return v +} + +type EndQuorumEpochResponseTopic struct { + Topic string + + Partitions []EndQuorumEpochResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochResponseTopic. +func (v *EndQuorumEpochResponseTopic) Default() { +} + +// NewEndQuorumEpochResponseTopic returns a default EndQuorumEpochResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochResponseTopic() EndQuorumEpochResponseTopic { + var v EndQuorumEpochResponseTopic + v.Default() + return v +} + +type EndQuorumEpochResponseNodeEndpoint struct { + NodeID int32 // v1+ + + Host string // v1+ + + Port uint16 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochResponseNodeEndpoint. +func (v *EndQuorumEpochResponseNodeEndpoint) Default() { +} + +// NewEndQuorumEpochResponseNodeEndpoint returns a default EndQuorumEpochResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochResponseNodeEndpoint() EndQuorumEpochResponseNodeEndpoint { + var v EndQuorumEpochResponseNodeEndpoint + v.Default() + return v +} + +type EndQuorumEpochResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + Topics []EndQuorumEpochResponseTopic + + // Endpoints for all leaders enumerated in PartitionData. + NodeEndpoints []EndQuorumEpochResponseNodeEndpoint // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags // v1+ +} + +func (*EndQuorumEpochResponse) Key() int16 { return 54 } +func (*EndQuorumEpochResponse) MaxVersion() int16 { return 1 } +func (v *EndQuorumEpochResponse) SetVersion(version int16) { v.Version = version } +func (v *EndQuorumEpochResponse) GetVersion() int16 { return v.Version } +func (v *EndQuorumEpochResponse) IsFlexible() bool { return v.Version >= 1 } +func (v *EndQuorumEpochResponse) RequestKind() Request { + return &EndQuorumEpochRequest{Version: v.Version} +} + +func (v *EndQuorumEpochResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if len(v.NodeEndpoints) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.NodeEndpoints + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fNodeEndpoints: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 1 { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fNodeEndpoints + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EndQuorumEpochResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EndQuorumEpochResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EndQuorumEpochResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 1 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]EndQuorumEpochResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 1 { + v := b.Int32() + s.NodeID = v + } + if version >= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 1 { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrEndQuorumEpochResponse returns a pointer to a default EndQuorumEpochResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEndQuorumEpochResponse() *EndQuorumEpochResponse { + var v EndQuorumEpochResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EndQuorumEpochResponse. +func (v *EndQuorumEpochResponse) Default() { +} + +// NewEndQuorumEpochResponse returns a default EndQuorumEpochResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndQuorumEpochResponse() EndQuorumEpochResponse { + var v EndQuorumEpochResponse + v.Default() + return v +} + +// A common struct used in DescribeQuorumResponse. +type DescribeQuorumResponseTopicPartitionReplicaState struct { + ReplicaID int32 + + ReplicaDirectoryID [16]byte // v2+ + + // The last known log end offset of the follower, or -1 if it is unknown. + LogEndOffset int64 + + // The last known leader wall clock time when a follower fetched from the + // leader, or -1 for the current leader or if unknown for a voter. + // + // This field has a default of -1. + LastFetchTimestamp int64 // v1+ + + // The leader wall clock append time of the offset for which the follower + // made the most recent fetch request, or -1 for the current leader or if + // unknown for a voter. + // + // This field has a default of -1. + LastCaughtUpTimestamp int64 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponseTopicPartitionReplicaState. +func (v *DescribeQuorumResponseTopicPartitionReplicaState) Default() { + v.LastFetchTimestamp = -1 + v.LastCaughtUpTimestamp = -1 +} + +// NewDescribeQuorumResponseTopicPartitionReplicaState returns a default DescribeQuorumResponseTopicPartitionReplicaState +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponseTopicPartitionReplicaState() DescribeQuorumResponseTopicPartitionReplicaState { + var v DescribeQuorumResponseTopicPartitionReplicaState + v.Default() + return v +} + +type DescribeQuorumRequestTopicPartition struct { + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumRequestTopicPartition. +func (v *DescribeQuorumRequestTopicPartition) Default() { +} + +// NewDescribeQuorumRequestTopicPartition returns a default DescribeQuorumRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumRequestTopicPartition() DescribeQuorumRequestTopicPartition { + var v DescribeQuorumRequestTopicPartition + v.Default() + return v +} + +type DescribeQuorumRequestTopic struct { + Topic string + + Partitions []DescribeQuorumRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumRequestTopic. +func (v *DescribeQuorumRequestTopic) Default() { +} + +// NewDescribeQuorumRequestTopic returns a default DescribeQuorumRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumRequestTopic() DescribeQuorumRequestTopic { + var v DescribeQuorumRequestTopic + v.Default() + return v +} + +// Part of KIP-642 (and KIP-595) to replace Kafka's dependence on Zookeeper with a +// Kafka-only raft protocol, +// DescribeQuorumRequest is sent by a leader to describe the quorum. +type DescribeQuorumRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + Topics []DescribeQuorumRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeQuorumRequest) Key() int16 { return 55 } +func (*DescribeQuorumRequest) MaxVersion() int16 { return 2 } +func (v *DescribeQuorumRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeQuorumRequest) GetVersion() int16 { return v.Version } +func (v *DescribeQuorumRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeQuorumRequest) IsAdminRequest() {} +func (v *DescribeQuorumRequest) ResponseKind() Response { + r := &DescribeQuorumResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeQuorumRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeQuorumResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeQuorumResponse) + return resp, err +} + +func (v *DescribeQuorumRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeQuorumRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeQuorumRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeQuorumRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeQuorumRequest returns a pointer to a default DescribeQuorumRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeQuorumRequest() *DescribeQuorumRequest { + var v DescribeQuorumRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumRequest. +func (v *DescribeQuorumRequest) Default() { +} + +// NewDescribeQuorumRequest returns a default DescribeQuorumRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumRequest() DescribeQuorumRequest { + var v DescribeQuorumRequest + v.Default() + return v +} + +type DescribeQuorumResponseTopicPartition struct { + Partition int32 + + ErrorCode int16 + + ErrorMessage *string // v2+ + + // The ID of the current leader, or -1 if the leader is unknown. + LeaderID int32 + + // The latest known leader epoch. + LeaderEpoch int32 + + HighWatermark int64 + + CurrentVoters []DescribeQuorumResponseTopicPartitionReplicaState + + Observers []DescribeQuorumResponseTopicPartitionReplicaState + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponseTopicPartition. +func (v *DescribeQuorumResponseTopicPartition) Default() { +} + +// NewDescribeQuorumResponseTopicPartition returns a default DescribeQuorumResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponseTopicPartition() DescribeQuorumResponseTopicPartition { + var v DescribeQuorumResponseTopicPartition + v.Default() + return v +} + +type DescribeQuorumResponseTopic struct { + Topic string + + Partitions []DescribeQuorumResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponseTopic. +func (v *DescribeQuorumResponseTopic) Default() { +} + +// NewDescribeQuorumResponseTopic returns a default DescribeQuorumResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponseTopic() DescribeQuorumResponseTopic { + var v DescribeQuorumResponseTopic + v.Default() + return v +} + +type DescribeQuorumResponseNodeListener struct { + Name string + + Host string + + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponseNodeListener. +func (v *DescribeQuorumResponseNodeListener) Default() { +} + +// NewDescribeQuorumResponseNodeListener returns a default DescribeQuorumResponseNodeListener +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponseNodeListener() DescribeQuorumResponseNodeListener { + var v DescribeQuorumResponseNodeListener + v.Default() + return v +} + +type DescribeQuorumResponseNode struct { + NodeID int32 + + Listeners []DescribeQuorumResponseNodeListener + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponseNode. +func (v *DescribeQuorumResponseNode) Default() { +} + +// NewDescribeQuorumResponseNode returns a default DescribeQuorumResponseNode +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponseNode() DescribeQuorumResponseNode { + var v DescribeQuorumResponseNode + v.Default() + return v +} + +type DescribeQuorumResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + ErrorCode int16 + + ErrorMessage *string // v2+ + + Topics []DescribeQuorumResponseTopic + + Nodes []DescribeQuorumResponseNode // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeQuorumResponse) Key() int16 { return 55 } +func (*DescribeQuorumResponse) MaxVersion() int16 { return 2 } +func (v *DescribeQuorumResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeQuorumResponse) GetVersion() int16 { return v.Version } +func (v *DescribeQuorumResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeQuorumResponse) RequestKind() Request { + return &DescribeQuorumRequest{Version: v.Version} +} + +func (v *DescribeQuorumResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 2 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if version >= 2 { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.HighWatermark + dst = kbin.AppendInt64(dst, v) + } + { + v := v.CurrentVoters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.ReplicaDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.LogEndOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.LastFetchTimestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.LastCaughtUpTimestamp + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Observers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.ReplicaDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.LogEndOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.LastFetchTimestamp + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.LastCaughtUpTimestamp + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 2 { + v := v.Nodes + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Listeners + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeQuorumResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeQuorumResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeQuorumResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 2 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if version >= 2 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Int64() + s.HighWatermark = v + } + { + v := s.CurrentVoters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseTopicPartitionReplicaState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.ReplicaID = v + } + if version >= 2 { + v := b.Uuid() + s.ReplicaDirectoryID = v + } + { + v := b.Int64() + s.LogEndOffset = v + } + if version >= 1 { + v := b.Int64() + s.LastFetchTimestamp = v + } + if version >= 1 { + v := b.Int64() + s.LastCaughtUpTimestamp = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.CurrentVoters = v + } + { + v := s.Observers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseTopicPartitionReplicaState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.ReplicaID = v + } + if version >= 2 { + v := b.Uuid() + s.ReplicaDirectoryID = v + } + { + v := b.Int64() + s.LogEndOffset = v + } + if version >= 1 { + v := b.Int64() + s.LastFetchTimestamp = v + } + if version >= 1 { + v := b.Int64() + s.LastCaughtUpTimestamp = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Observers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if version >= 2 { + v := s.Nodes + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseNode, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + v := s.Listeners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeQuorumResponseNodeListener, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Listeners = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Nodes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeQuorumResponse returns a pointer to a default DescribeQuorumResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeQuorumResponse() *DescribeQuorumResponse { + var v DescribeQuorumResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeQuorumResponse. +func (v *DescribeQuorumResponse) Default() { +} + +// NewDescribeQuorumResponse returns a default DescribeQuorumResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeQuorumResponse() DescribeQuorumResponse { + var v DescribeQuorumResponse + v.Default() + return v +} + +type AlterPartitionRequestTopicPartitionNewEpochISR struct { + // The broker ID . + BrokerID int32 + + // The broker's epoch; -1 if the epoch check is not supported. + // + // This field has a default of -1. + BrokerEpoch int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionRequestTopicPartitionNewEpochISR. +func (v *AlterPartitionRequestTopicPartitionNewEpochISR) Default() { + v.BrokerEpoch = -1 +} + +// NewAlterPartitionRequestTopicPartitionNewEpochISR returns a default AlterPartitionRequestTopicPartitionNewEpochISR +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionRequestTopicPartitionNewEpochISR() AlterPartitionRequestTopicPartitionNewEpochISR { + var v AlterPartitionRequestTopicPartitionNewEpochISR + v.Default() + return v +} + +type AlterPartitionRequestTopicPartition struct { + Partition int32 + + // The leader epoch of this partition. + LeaderEpoch int32 + + // The ISR for this partition. + NewISR []int32 // v0-v2 + + NewEpochISR []AlterPartitionRequestTopicPartitionNewEpochISR // v3+ + + // 1 if the partition is recovering from unclean leader election; 0 otherwise + LeaderRecoveryState int8 // v1+ + + // The expected epoch of the partition which is being updated. + // For a legacy cluster, this is the ZkVersion in the LeaderAndISR request. + PartitionEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionRequestTopicPartition. +func (v *AlterPartitionRequestTopicPartition) Default() { +} + +// NewAlterPartitionRequestTopicPartition returns a default AlterPartitionRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionRequestTopicPartition() AlterPartitionRequestTopicPartition { + var v AlterPartitionRequestTopicPartition + v.Default() + return v +} + +type AlterPartitionRequestTopic struct { + Topic string // v0-v1 + + TopicID [16]byte // v2+ + + Partitions []AlterPartitionRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionRequestTopic. +func (v *AlterPartitionRequestTopic) Default() { +} + +// NewAlterPartitionRequestTopic returns a default AlterPartitionRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionRequestTopic() AlterPartitionRequestTopic { + var v AlterPartitionRequestTopic + v.Default() + return v +} + +// AlterPartitionRequest, proposed in KIP-497 and introduced in Kafka 2.7.0, +// is an admin request to modify ISR. +// +// Version 3 was added for KIP-903 and replaced NewISR. +type AlterPartitionRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The ID of the requesting broker. + BrokerID int32 + + // The epoch of the requesting broker. + // + // This field has a default of -1. + BrokerEpoch int64 + + Topics []AlterPartitionRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterPartitionRequest) Key() int16 { return 56 } +func (*AlterPartitionRequest) MaxVersion() int16 { return 3 } +func (v *AlterPartitionRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterPartitionRequest) GetVersion() int16 { return v.Version } +func (v *AlterPartitionRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterPartitionRequest) IsAdminRequest() {} +func (v *AlterPartitionRequest) ResponseKind() Response { + r := &AlterPartitionResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterPartitionRequest) RequestWith(ctx context.Context, r Requestor) (*AlterPartitionResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterPartitionResponse) + return resp, err +} + +func (v *AlterPartitionRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 1 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 2 { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 2 { + v := v.NewISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 3 { + v := v.NewEpochISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.LeaderRecoveryState + dst = kbin.AppendInt8(dst, v) + } + { + v := v.PartitionEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterPartitionRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterPartitionRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterPartitionRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 2 { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if version >= 0 && version <= 2 { + v := s.NewISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.NewISR = v + } + if version >= 3 { + v := s.NewEpochISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionRequestTopicPartitionNewEpochISR, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NewEpochISR = v + } + if version >= 1 { + v := b.Int8() + s.LeaderRecoveryState = v + } + { + v := b.Int32() + s.PartitionEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterPartitionRequest returns a pointer to a default AlterPartitionRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterPartitionRequest() *AlterPartitionRequest { + var v AlterPartitionRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionRequest. +func (v *AlterPartitionRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewAlterPartitionRequest returns a default AlterPartitionRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionRequest() AlterPartitionRequest { + var v AlterPartitionRequest + v.Default() + return v +} + +type AlterPartitionResponseTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // The broker ID of the leader. + LeaderID int32 + + // The leader epoch of this partition. + LeaderEpoch int32 + + // The in-sync replica ids. + ISR []int32 + + // 1 if the partition is recovering from unclean leader election; 0 otherwise + LeaderRecoveryState int8 // v1+ + + // The current epoch of the partition for KRaft controllers. + // The current ZK version for legacy controllers. + PartitionEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionResponseTopicPartition. +func (v *AlterPartitionResponseTopicPartition) Default() { +} + +// NewAlterPartitionResponseTopicPartition returns a default AlterPartitionResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionResponseTopicPartition() AlterPartitionResponseTopicPartition { + var v AlterPartitionResponseTopicPartition + v.Default() + return v +} + +type AlterPartitionResponseTopic struct { + Topic string // v0-v1 + + TopidID [16]byte // v2+ + + Partitions []AlterPartitionResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionResponseTopic. +func (v *AlterPartitionResponseTopic) Default() { +} + +// NewAlterPartitionResponseTopic returns a default AlterPartitionResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionResponseTopic() AlterPartitionResponseTopic { + var v AlterPartitionResponseTopic + v.Default() + return v +} + +type AlterPartitionResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + Topics []AlterPartitionResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterPartitionResponse) Key() int16 { return 56 } +func (*AlterPartitionResponse) MaxVersion() int16 { return 3 } +func (v *AlterPartitionResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterPartitionResponse) GetVersion() int16 { return v.Version } +func (v *AlterPartitionResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterPartitionResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *AlterPartitionResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *AlterPartitionResponse) RequestKind() Request { + return &AlterPartitionRequest{Version: v.Version} +} + +func (v *AlterPartitionResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 0 && version <= 1 { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 2 { + v := v.TopidID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if version >= 1 { + v := v.LeaderRecoveryState + dst = kbin.AppendInt8(dst, v) + } + { + v := v.PartitionEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterPartitionResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterPartitionResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterPartitionResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 0 && version <= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if version >= 2 { + v := b.Uuid() + s.TopidID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterPartitionResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + if version >= 1 { + v := b.Int8() + s.LeaderRecoveryState = v + } + { + v := b.Int32() + s.PartitionEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterPartitionResponse returns a pointer to a default AlterPartitionResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterPartitionResponse() *AlterPartitionResponse { + var v AlterPartitionResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterPartitionResponse. +func (v *AlterPartitionResponse) Default() { +} + +// NewAlterPartitionResponse returns a default AlterPartitionResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterPartitionResponse() AlterPartitionResponse { + var v AlterPartitionResponse + v.Default() + return v +} + +type UpdateFeaturesRequestFeatureUpdate struct { + // The name of the finalized feature to update. + Feature string + + // The new maximum version level for the finalized feature. A value >= 1 is + // valid. A value < 1, is special, and can be used to request the deletion + // of the finalized feature. + MaxVersionLevel int16 + + // When set to true, the finalized feature version level is allowed to be + // downgraded/deleted. The downgrade request will fail if the new maximum + // version level is a value that's not lower than the existing maximum + // finalized version level. + // + // Replaced in v1 with ValidateOnly. + AllowDowngrade bool + + // Determine which type of upgrade will be performed: 1 will perform an + // upgrade only (default), 2 is safe downgrades only (lossless), 3 is + // unsafe downgrades (lossy). + UpgradeType int8 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateFeaturesRequestFeatureUpdate. +func (v *UpdateFeaturesRequestFeatureUpdate) Default() { +} + +// NewUpdateFeaturesRequestFeatureUpdate returns a default UpdateFeaturesRequestFeatureUpdate +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateFeaturesRequestFeatureUpdate() UpdateFeaturesRequestFeatureUpdate { + var v UpdateFeaturesRequestFeatureUpdate + v.Default() + return v +} + +// From KIP-584 and introduced in 2.7.0, this request updates broker-wide features. +type UpdateFeaturesRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 60000. + TimeoutMillis int32 + + // The list of updates to finalized features. + FeatureUpdates []UpdateFeaturesRequestFeatureUpdate + + // True if we should validate the request, but not perform the upgrade or + // downgrade. + ValidateOnly bool // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UpdateFeaturesRequest) Key() int16 { return 57 } +func (*UpdateFeaturesRequest) MaxVersion() int16 { return 2 } +func (v *UpdateFeaturesRequest) SetVersion(version int16) { v.Version = version } +func (v *UpdateFeaturesRequest) GetVersion() int16 { return v.Version } +func (v *UpdateFeaturesRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *UpdateFeaturesRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *UpdateFeaturesRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *UpdateFeaturesRequest) IsAdminRequest() {} +func (v *UpdateFeaturesRequest) ResponseKind() Response { + r := &UpdateFeaturesResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *UpdateFeaturesRequest) RequestWith(ctx context.Context, r Requestor) (*UpdateFeaturesResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*UpdateFeaturesResponse) + return resp, err +} + +func (v *UpdateFeaturesRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.FeatureUpdates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Feature + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MaxVersionLevel + dst = kbin.AppendInt16(dst, v) + } + if version >= 0 && version <= 0 { + v := v.AllowDowngrade + dst = kbin.AppendBool(dst, v) + } + if version >= 1 { + v := v.UpgradeType + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.ValidateOnly + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateFeaturesRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateFeaturesRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateFeaturesRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := s.FeatureUpdates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateFeaturesRequestFeatureUpdate, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Feature = v + } + { + v := b.Int16() + s.MaxVersionLevel = v + } + if version >= 0 && version <= 0 { + v := b.Bool() + s.AllowDowngrade = v + } + if version >= 1 { + v := b.Int8() + s.UpgradeType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.FeatureUpdates = v + } + if version >= 1 { + v := b.Bool() + s.ValidateOnly = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUpdateFeaturesRequest returns a pointer to a default UpdateFeaturesRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateFeaturesRequest() *UpdateFeaturesRequest { + var v UpdateFeaturesRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateFeaturesRequest. +func (v *UpdateFeaturesRequest) Default() { + v.TimeoutMillis = 60000 +} + +// NewUpdateFeaturesRequest returns a default UpdateFeaturesRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateFeaturesRequest() UpdateFeaturesRequest { + var v UpdateFeaturesRequest + v.Default() + return v +} + +type UpdateFeaturesResponseResult struct { + // The name of the finalized feature. + Feature string + + // The feature update error code, if any. + ErrorCode int16 + + // The feature update error, if any. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateFeaturesResponseResult. +func (v *UpdateFeaturesResponseResult) Default() { +} + +// NewUpdateFeaturesResponseResult returns a default UpdateFeaturesResponseResult +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateFeaturesResponseResult() UpdateFeaturesResponseResult { + var v UpdateFeaturesResponseResult + v.Default() + return v +} + +type UpdateFeaturesResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The top level error code, if any. + ErrorCode int16 + + // An informative message if the request errored, if any. + ErrorMessage *string + + // The results for each feature update request. + Results []UpdateFeaturesResponseResult // v0-v1 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UpdateFeaturesResponse) Key() int16 { return 57 } +func (*UpdateFeaturesResponse) MaxVersion() int16 { return 2 } +func (v *UpdateFeaturesResponse) SetVersion(version int16) { v.Version = version } +func (v *UpdateFeaturesResponse) GetVersion() int16 { return v.Version } +func (v *UpdateFeaturesResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *UpdateFeaturesResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *UpdateFeaturesResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *UpdateFeaturesResponse) RequestKind() Request { + return &UpdateFeaturesRequest{Version: v.Version} +} + +func (v *UpdateFeaturesResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 0 && version <= 1 { + v := v.Results + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Feature + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateFeaturesResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateFeaturesResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateFeaturesResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 0 && version <= 1 { + v := s.Results + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateFeaturesResponseResult, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Feature = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Results = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUpdateFeaturesResponse returns a pointer to a default UpdateFeaturesResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateFeaturesResponse() *UpdateFeaturesResponse { + var v UpdateFeaturesResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateFeaturesResponse. +func (v *UpdateFeaturesResponse) Default() { +} + +// NewUpdateFeaturesResponse returns a default UpdateFeaturesResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateFeaturesResponse() UpdateFeaturesResponse { + var v UpdateFeaturesResponse + v.Default() + return v +} + +// Introduced for KIP-590, EnvelopeRequest is what brokers use to wrap an +// incoming request before forwarding it to another broker. +type EnvelopeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The embedded request header and data. + RequestData []byte + + // Value of the initial client principal when the request is redirected by a broker. + RequestPrincipal []byte + + // The original client's address in bytes. + ClientHostAddress []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*EnvelopeRequest) Key() int16 { return 58 } +func (*EnvelopeRequest) MaxVersion() int16 { return 0 } +func (v *EnvelopeRequest) SetVersion(version int16) { v.Version = version } +func (v *EnvelopeRequest) GetVersion() int16 { return v.Version } +func (v *EnvelopeRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *EnvelopeRequest) IsAdminRequest() {} +func (v *EnvelopeRequest) ResponseKind() Response { + r := &EnvelopeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *EnvelopeRequest) RequestWith(ctx context.Context, r Requestor) (*EnvelopeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*EnvelopeResponse) + return resp, err +} + +func (v *EnvelopeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.RequestData + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + { + v := v.RequestPrincipal + if isFlexible { + dst = kbin.AppendCompactNullableBytes(dst, v) + } else { + dst = kbin.AppendNullableBytes(dst, v) + } + } + { + v := v.ClientHostAddress + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EnvelopeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EnvelopeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EnvelopeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.RequestData = v + } + { + var v []byte + if isFlexible { + v = b.CompactNullableBytes() + } else { + v = b.NullableBytes() + } + s.RequestPrincipal = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.ClientHostAddress = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrEnvelopeRequest returns a pointer to a default EnvelopeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEnvelopeRequest() *EnvelopeRequest { + var v EnvelopeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EnvelopeRequest. +func (v *EnvelopeRequest) Default() { +} + +// NewEnvelopeRequest returns a default EnvelopeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewEnvelopeRequest() EnvelopeRequest { + var v EnvelopeRequest + v.Default() + return v +} + +type EnvelopeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The embedded response header and data. + ResponseData []byte + + // The error code, or 0 if there was no error. + // + // NOT_CONTROLLER is returned when the request is not sent to the controller. + // + // CLUSTER_AUTHORIZATION_FAILED is returned if inter-broker authorization failed. + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*EnvelopeResponse) Key() int16 { return 58 } +func (*EnvelopeResponse) MaxVersion() int16 { return 0 } +func (v *EnvelopeResponse) SetVersion(version int16) { v.Version = version } +func (v *EnvelopeResponse) GetVersion() int16 { return v.Version } +func (v *EnvelopeResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *EnvelopeResponse) RequestKind() Request { return &EnvelopeRequest{Version: v.Version} } + +func (v *EnvelopeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ResponseData + if isFlexible { + dst = kbin.AppendCompactNullableBytes(dst, v) + } else { + dst = kbin.AppendNullableBytes(dst, v) + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *EnvelopeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *EnvelopeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *EnvelopeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v []byte + if isFlexible { + v = b.CompactNullableBytes() + } else { + v = b.NullableBytes() + } + s.ResponseData = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrEnvelopeResponse returns a pointer to a default EnvelopeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrEnvelopeResponse() *EnvelopeResponse { + var v EnvelopeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to EnvelopeResponse. +func (v *EnvelopeResponse) Default() { +} + +// NewEnvelopeResponse returns a default EnvelopeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewEnvelopeResponse() EnvelopeResponse { + var v EnvelopeResponse + v.Default() + return v +} + +type FetchSnapshotRequestTopicPartitionSnapshotID struct { + EndOffset int64 + + Epoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotRequestTopicPartitionSnapshotID. +func (v *FetchSnapshotRequestTopicPartitionSnapshotID) Default() { +} + +// NewFetchSnapshotRequestTopicPartitionSnapshotID returns a default FetchSnapshotRequestTopicPartitionSnapshotID +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotRequestTopicPartitionSnapshotID() FetchSnapshotRequestTopicPartitionSnapshotID { + var v FetchSnapshotRequestTopicPartitionSnapshotID + v.Default() + return v +} + +type FetchSnapshotRequestTopicPartition struct { + // The partition to fetch. + Partition int32 + + // The current leader epoch of the partition, or -1 for an unknown leader epoch. + CurrentLeaderEpoch int32 + + // The snapshot end offset and epoch to fetch. + SnapshotID FetchSnapshotRequestTopicPartitionSnapshotID + + // The byte position within the snapshot to start fetching from. + Position int64 + + // The directory id of the follower fetching. + ReplicaDirectoryID [16]byte // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotRequestTopicPartition. +func (v *FetchSnapshotRequestTopicPartition) Default() { + { + v := &v.SnapshotID + _ = v + } +} + +// NewFetchSnapshotRequestTopicPartition returns a default FetchSnapshotRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotRequestTopicPartition() FetchSnapshotRequestTopicPartition { + var v FetchSnapshotRequestTopicPartition + v.Default() + return v +} + +type FetchSnapshotRequestTopic struct { + // The name of the topic to fetch. + Topic string + + // The partitions to fetch. + Partitions []FetchSnapshotRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotRequestTopic. +func (v *FetchSnapshotRequestTopic) Default() { +} + +// NewFetchSnapshotRequestTopic returns a default FetchSnapshotRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotRequestTopic() FetchSnapshotRequestTopic { + var v FetchSnapshotRequestTopic + v.Default() + return v +} + +// Introduced for KIP-630, FetchSnapshotRequest is a part of the inter-Kafka +// raft protocol to remove the dependency on Zookeeper. +type FetchSnapshotRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The ClusterID if known, this is used to validate metadata fetches prior to + // broker registration. + ClusterID *string // tag 0 + + // The broker ID of the follower. + // + // This field has a default of -1. + ReplicaID int32 + + // The maximum bytes to fetch from all of the snapshots. + // + // This field has a default of 0x7fffffff. + MaxBytes int32 + + // The topics to fetch. + Topics []FetchSnapshotRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*FetchSnapshotRequest) Key() int16 { return 59 } +func (*FetchSnapshotRequest) MaxVersion() int16 { return 1 } +func (v *FetchSnapshotRequest) SetVersion(version int16) { v.Version = version } +func (v *FetchSnapshotRequest) GetVersion() int16 { return v.Version } +func (v *FetchSnapshotRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *FetchSnapshotRequest) ResponseKind() Response { + r := &FetchSnapshotResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *FetchSnapshotRequest) RequestWith(ctx context.Context, r Requestor) (*FetchSnapshotResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*FetchSnapshotResponse) + return resp, err +} + +func (v *FetchSnapshotRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ReplicaID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MaxBytes + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CurrentLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := &v.SnapshotID + { + v := v.EndOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := v.Position + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + var toEncode []uint32 + if v.ReplicaDirectoryID != [16]byte{} { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.ReplicaDirectoryID + dst = kbin.AppendUvarint(dst, 0) + dst = kbin.AppendUvarint(dst, 16) + dst = kbin.AppendUuid(dst, v) + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if v.ClusterID != nil { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.ClusterID + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fClusterID: + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fClusterID + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FetchSnapshotRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FetchSnapshotRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FetchSnapshotRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ReplicaID = v + } + { + v := b.Int32() + s.MaxBytes = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchSnapshotRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchSnapshotRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.CurrentLeaderEpoch = v + } + { + v := &s.SnapshotID + v.Default() + s := v + { + v := b.Int64() + s.EndOffset = v + } + { + v := b.Int32() + s.Epoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := b.Int64() + s.Position = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := b.Uuid() + s.ReplicaDirectoryID = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrFetchSnapshotRequest returns a pointer to a default FetchSnapshotRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFetchSnapshotRequest() *FetchSnapshotRequest { + var v FetchSnapshotRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotRequest. +func (v *FetchSnapshotRequest) Default() { + v.ReplicaID = -1 + v.MaxBytes = 2147483647 +} + +// NewFetchSnapshotRequest returns a default FetchSnapshotRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotRequest() FetchSnapshotRequest { + var v FetchSnapshotRequest + v.Default() + return v +} + +type FetchSnapshotResponseTopicPartitionSnapshotID struct { + EndOffset int64 + + Epoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponseTopicPartitionSnapshotID. +func (v *FetchSnapshotResponseTopicPartitionSnapshotID) Default() { +} + +// NewFetchSnapshotResponseTopicPartitionSnapshotID returns a default FetchSnapshotResponseTopicPartitionSnapshotID +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponseTopicPartitionSnapshotID() FetchSnapshotResponseTopicPartitionSnapshotID { + var v FetchSnapshotResponseTopicPartitionSnapshotID + v.Default() + return v +} + +type FetchSnapshotResponseTopicPartitionCurrentLeader struct { + LeaderID int32 + + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponseTopicPartitionCurrentLeader. +func (v *FetchSnapshotResponseTopicPartitionCurrentLeader) Default() { +} + +// NewFetchSnapshotResponseTopicPartitionCurrentLeader returns a default FetchSnapshotResponseTopicPartitionCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponseTopicPartitionCurrentLeader() FetchSnapshotResponseTopicPartitionCurrentLeader { + var v FetchSnapshotResponseTopicPartitionCurrentLeader + v.Default() + return v +} + +type FetchSnapshotResponseTopicPartition struct { + // The partition. + Partition int32 + + // An error code, or 0 if there was no fetch error. + ErrorCode int16 + + // The snapshot end offset and epoch to fetch. + SnapshotID FetchSnapshotResponseTopicPartitionSnapshotID + + // The ID of the current leader (or -1 if unknown) and the latest known + // leader epoch. + CurrentLeader FetchSnapshotResponseTopicPartitionCurrentLeader // tag 0 + + // The total size of the snapshot. + Size int64 + + // The starting byte position within the snapshot included in the Bytes + // field. + Position int64 + + // Snapshot data. + Bytes []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponseTopicPartition. +func (v *FetchSnapshotResponseTopicPartition) Default() { + { + v := &v.SnapshotID + _ = v + } + { + v := &v.CurrentLeader + _ = v + } +} + +// NewFetchSnapshotResponseTopicPartition returns a default FetchSnapshotResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponseTopicPartition() FetchSnapshotResponseTopicPartition { + var v FetchSnapshotResponseTopicPartition + v.Default() + return v +} + +type FetchSnapshotResponseTopic struct { + // The name of the topic to fetch. + Topic string + + // The partitions to fetch. + Partitions []FetchSnapshotResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponseTopic. +func (v *FetchSnapshotResponseTopic) Default() { +} + +// NewFetchSnapshotResponseTopic returns a default FetchSnapshotResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponseTopic() FetchSnapshotResponseTopic { + var v FetchSnapshotResponseTopic + v.Default() + return v +} + +type FetchSnapshotResponseNodeEndpoint struct { + NodeID int32 // v1+ + + Host string // v1+ + + Port uint16 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponseNodeEndpoint. +func (v *FetchSnapshotResponseNodeEndpoint) Default() { +} + +// NewFetchSnapshotResponseNodeEndpoint returns a default FetchSnapshotResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponseNodeEndpoint() FetchSnapshotResponseNodeEndpoint { + var v FetchSnapshotResponseNodeEndpoint + v.Default() + return v +} + +// FetchSnapshotResponse is a response for a FetchSnapshotRequest. +type FetchSnapshotResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The top level response error code. + ErrorCode int16 + + // The topics to fetch. + Topics []FetchSnapshotResponseTopic + + // Endpoints for all leaders enumerated in PartitionData. + NodeEndpoints []FetchSnapshotResponseNodeEndpoint // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*FetchSnapshotResponse) Key() int16 { return 59 } +func (*FetchSnapshotResponse) MaxVersion() int16 { return 1 } +func (v *FetchSnapshotResponse) SetVersion(version int16) { v.Version = version } +func (v *FetchSnapshotResponse) GetVersion() int16 { return v.Version } +func (v *FetchSnapshotResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *FetchSnapshotResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *FetchSnapshotResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *FetchSnapshotResponse) RequestKind() Request { + return &FetchSnapshotRequest{Version: v.Version} +} + +func (v *FetchSnapshotResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := &v.SnapshotID + { + v := v.EndOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := v.Size + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Position + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Bytes + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + var toEncode []uint32 + if !reflect.DeepEqual(v.CurrentLeader, (func() FetchSnapshotResponseTopicPartitionCurrentLeader { + var v FetchSnapshotResponseTopicPartitionCurrentLeader + v.Default() + return v + })()) { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.CurrentLeader + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fCurrentLeader: + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fCurrentLeader + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + var toEncode []uint32 + if len(v.NodeEndpoints) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.NodeEndpoints + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fNodeEndpoints: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + if version >= 1 { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fNodeEndpoints + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *FetchSnapshotResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *FetchSnapshotResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *FetchSnapshotResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchSnapshotResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchSnapshotResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := &s.SnapshotID + v.Default() + s := v + { + v := b.Int64() + s.EndOffset = v + } + { + v := b.Int32() + s.Epoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := b.Int64() + s.Size = v + } + { + v := b.Int64() + s.Position = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Bytes = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + } + } + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]FetchSnapshotResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + if version >= 1 { + v := b.Int32() + s.NodeID = v + } + if version >= 1 { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + if version >= 1 { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrFetchSnapshotResponse returns a pointer to a default FetchSnapshotResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrFetchSnapshotResponse() *FetchSnapshotResponse { + var v FetchSnapshotResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to FetchSnapshotResponse. +func (v *FetchSnapshotResponse) Default() { +} + +// NewFetchSnapshotResponse returns a default FetchSnapshotResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewFetchSnapshotResponse() FetchSnapshotResponse { + var v FetchSnapshotResponse + v.Default() + return v +} + +// Introduced for KIP-700, DescribeClusterRequest is effectively an "admin" +// type metadata request for information that producers or consumers do not +// need to care about. +type DescribeClusterRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Whether to include cluster authorized operations. This requires DESCRIBE + // on CLUSTER. + IncludeClusterAuthorizedOperations bool + + // The endpoint type to describe. 1=brokers, 2=controllers. + // + // This field has a default of 1. + EndpointType int8 // v1+ + + // Whether to include fenced brokers when listing brokers. + IncludeFencedBrokers bool // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeClusterRequest) Key() int16 { return 60 } +func (*DescribeClusterRequest) MaxVersion() int16 { return 2 } +func (v *DescribeClusterRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeClusterRequest) GetVersion() int16 { return v.Version } +func (v *DescribeClusterRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeClusterRequest) ResponseKind() Response { + r := &DescribeClusterResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeClusterRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeClusterResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeClusterResponse) + return resp, err +} + +func (v *DescribeClusterRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.IncludeClusterAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if version >= 1 { + v := v.EndpointType + dst = kbin.AppendInt8(dst, v) + } + if version >= 2 { + v := v.IncludeFencedBrokers + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeClusterRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeClusterRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeClusterRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Bool() + s.IncludeClusterAuthorizedOperations = v + } + if version >= 1 { + v := b.Int8() + s.EndpointType = v + } + if version >= 2 { + v := b.Bool() + s.IncludeFencedBrokers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeClusterRequest returns a pointer to a default DescribeClusterRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeClusterRequest() *DescribeClusterRequest { + var v DescribeClusterRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClusterRequest. +func (v *DescribeClusterRequest) Default() { + v.EndpointType = 1 +} + +// NewDescribeClusterRequest returns a default DescribeClusterRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClusterRequest() DescribeClusterRequest { + var v DescribeClusterRequest + v.Default() + return v +} + +type DescribeClusterResponseBroker struct { + // NodeID is the node ID of a Kafka broker. + NodeID int32 + + // Host is the hostname of a Kafka broker. + Host string + + // Port is the port of a Kafka broker. + Port int32 + + // Rack is the rack this Kafka broker is in, if any. + Rack *string + + // Whether the broker is fenced. + IsFenced bool // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClusterResponseBroker. +func (v *DescribeClusterResponseBroker) Default() { +} + +// NewDescribeClusterResponseBroker returns a default DescribeClusterResponseBroker +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClusterResponseBroker() DescribeClusterResponseBroker { + var v DescribeClusterResponseBroker + v.Default() + return v +} + +// DescribeClusterResponse is a response to a DescribeClusterRequest. +type DescribeClusterResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // The top level response error code. + ErrorCode int16 + + // The top level error message, if any. + ErrorMessage *string + + // The endpoint type that was described. 1=brokers, 2=controllers. + // + // This field has a default of 1. + EndpointType int8 // v1+ + + // The cluster ID that responding broker belongs to. + ClusterID string + + // The ID of the controller broker. + // + // This field has a default of -1. + ControllerID int32 + + // Brokers is a set of alive Kafka brokers (this mirrors MetadataResponse.Brokers). + Brokers []DescribeClusterResponseBroker + + // 32-bit bitfield to represent authorized operations for this cluster. + // + // This field has a default of -2147483648. + ClusterAuthorizedOperations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeClusterResponse) Key() int16 { return 60 } +func (*DescribeClusterResponse) MaxVersion() int16 { return 2 } +func (v *DescribeClusterResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeClusterResponse) GetVersion() int16 { return v.Version } +func (v *DescribeClusterResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeClusterResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *DescribeClusterResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeClusterResponse) RequestKind() Request { + return &DescribeClusterRequest{Version: v.Version} +} + +func (v *DescribeClusterResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.EndpointType + dst = kbin.AppendInt8(dst, v) + } + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Brokers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 2 { + v := v.IsFenced + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ClusterAuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeClusterResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeClusterResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeClusterResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 1 { + v := b.Int8() + s.EndpointType = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClusterID = v + } + { + v := b.Int32() + s.ControllerID = v + } + { + v := s.Brokers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeClusterResponseBroker, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if version >= 2 { + v := b.Bool() + s.IsFenced = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Brokers = v + } + { + v := b.Int32() + s.ClusterAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeClusterResponse returns a pointer to a default DescribeClusterResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeClusterResponse() *DescribeClusterResponse { + var v DescribeClusterResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeClusterResponse. +func (v *DescribeClusterResponse) Default() { + v.EndpointType = 1 + v.ControllerID = -1 + v.ClusterAuthorizedOperations = -2147483648 +} + +// NewDescribeClusterResponse returns a default DescribeClusterResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeClusterResponse() DescribeClusterResponse { + var v DescribeClusterResponse + v.Default() + return v +} + +type DescribeProducersRequestTopic struct { + Topic string + + // The partitions to list producers for for the given topic. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersRequestTopic. +func (v *DescribeProducersRequestTopic) Default() { +} + +// NewDescribeProducersRequestTopic returns a default DescribeProducersRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersRequestTopic() DescribeProducersRequestTopic { + var v DescribeProducersRequestTopic + v.Default() + return v +} + +// Introduced for KIP-664, DescribeProducersRequest allows for introspecting +// the state of the transaction coordinator. This request can be used to detect +// hanging transactions or other EOS-related problems. +// +// This request allows for describing the state of the active +// idempotent/transactional producers. +type DescribeProducersRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The topics to describe producers for. + Topics []DescribeProducersRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeProducersRequest) Key() int16 { return 61 } +func (*DescribeProducersRequest) MaxVersion() int16 { return 0 } +func (v *DescribeProducersRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeProducersRequest) GetVersion() int16 { return v.Version } +func (v *DescribeProducersRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeProducersRequest) ResponseKind() Response { + r := &DescribeProducersResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeProducersRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeProducersResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeProducersResponse) + return resp, err +} + +func (v *DescribeProducersRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeProducersRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeProducersRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeProducersRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeProducersRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeProducersRequest returns a pointer to a default DescribeProducersRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeProducersRequest() *DescribeProducersRequest { + var v DescribeProducersRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersRequest. +func (v *DescribeProducersRequest) Default() { +} + +// NewDescribeProducersRequest returns a default DescribeProducersRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersRequest() DescribeProducersRequest { + var v DescribeProducersRequest + v.Default() + return v +} + +type DescribeProducersResponseTopicPartitionActiveProducer struct { + ProducerID int64 + + ProducerEpoch int32 + + // The last sequence produced. + // + // This field has a default of -1. + LastSequence int32 + + // The last timestamp produced. + // + // This field has a default of -1. + LastTimestamp int64 + + // The epoch of the transactional coordinator for this last produce. + CoordinatorEpoch int32 + + // The first offset of the transaction. + // + // This field has a default of -1. + CurrentTxnStartOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersResponseTopicPartitionActiveProducer. +func (v *DescribeProducersResponseTopicPartitionActiveProducer) Default() { + v.LastSequence = -1 + v.LastTimestamp = -1 + v.CurrentTxnStartOffset = -1 +} + +// NewDescribeProducersResponseTopicPartitionActiveProducer returns a default DescribeProducersResponseTopicPartitionActiveProducer +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersResponseTopicPartitionActiveProducer() DescribeProducersResponseTopicPartitionActiveProducer { + var v DescribeProducersResponseTopicPartitionActiveProducer + v.Default() + return v +} + +type DescribeProducersResponseTopicPartition struct { + Partition int32 + + // The partition error code, or 0 if there was no error. + // + // NOT_LEADER_OR_FOLLOWER is returned if the broker receiving this request + // is not the leader of the partition. + // + // TOPIC_AUTHORIZATION_FAILED is returned if the user does not have Describe + // permissions on the topic. + // + // UNKNOWN_TOPIC_OR_PARTITION is returned if the partition is not known to exist. + // + // Other errors may be returned corresponding to the partition being offline, etc. + ErrorCode int16 + + // The partition error message, which may be null if no additional details are available. + ErrorMessage *string + + // The current idempotent or transactional producers producing to this partition, + // and the metadata related to their produce requests. + ActiveProducers []DescribeProducersResponseTopicPartitionActiveProducer + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersResponseTopicPartition. +func (v *DescribeProducersResponseTopicPartition) Default() { +} + +// NewDescribeProducersResponseTopicPartition returns a default DescribeProducersResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersResponseTopicPartition() DescribeProducersResponseTopicPartition { + var v DescribeProducersResponseTopicPartition + v.Default() + return v +} + +type DescribeProducersResponseTopic struct { + Topic string + + Partitions []DescribeProducersResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersResponseTopic. +func (v *DescribeProducersResponseTopic) Default() { +} + +// NewDescribeProducersResponseTopic returns a default DescribeProducersResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersResponseTopic() DescribeProducersResponseTopic { + var v DescribeProducersResponseTopic + v.Default() + return v +} + +// DescribeProducersResponse is a response to a DescribeProducersRequest. +type DescribeProducersResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + Topics []DescribeProducersResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeProducersResponse) Key() int16 { return 61 } +func (*DescribeProducersResponse) MaxVersion() int16 { return 0 } +func (v *DescribeProducersResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeProducersResponse) GetVersion() int16 { return v.Version } +func (v *DescribeProducersResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeProducersResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *DescribeProducersResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeProducersResponse) RequestKind() Request { + return &DescribeProducersRequest{Version: v.Version} +} + +func (v *DescribeProducersResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ActiveProducers + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LastSequence + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LastTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.CoordinatorEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.CurrentTxnStartOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeProducersResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeProducersResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeProducersResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeProducersResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeProducersResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.ActiveProducers + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeProducersResponseTopicPartitionActiveProducer, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int32() + s.ProducerEpoch = v + } + { + v := b.Int32() + s.LastSequence = v + } + { + v := b.Int64() + s.LastTimestamp = v + } + { + v := b.Int32() + s.CoordinatorEpoch = v + } + { + v := b.Int64() + s.CurrentTxnStartOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActiveProducers = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeProducersResponse returns a pointer to a default DescribeProducersResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeProducersResponse() *DescribeProducersResponse { + var v DescribeProducersResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeProducersResponse. +func (v *DescribeProducersResponse) Default() { +} + +// NewDescribeProducersResponse returns a default DescribeProducersResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeProducersResponse() DescribeProducersResponse { + var v DescribeProducersResponse + v.Default() + return v +} + +type BrokerRegistrationRequestListener struct { + // The name of this endpoint. + Name string + + // The hostname. + Host string + + // The port. + Port uint16 + + // The security protocol. + SecurityProtocol int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerRegistrationRequestListener. +func (v *BrokerRegistrationRequestListener) Default() { +} + +// NewBrokerRegistrationRequestListener returns a default BrokerRegistrationRequestListener +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerRegistrationRequestListener() BrokerRegistrationRequestListener { + var v BrokerRegistrationRequestListener + v.Default() + return v +} + +type BrokerRegistrationRequestFeature struct { + // The name of the feature. + Name string + + // The minimum supported feature level. + MinSupportedVersion int16 + + // The maximum supported feature level. + MaxSupportedVersion int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerRegistrationRequestFeature. +func (v *BrokerRegistrationRequestFeature) Default() { +} + +// NewBrokerRegistrationRequestFeature returns a default BrokerRegistrationRequestFeature +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerRegistrationRequestFeature() BrokerRegistrationRequestFeature { + var v BrokerRegistrationRequestFeature + v.Default() + return v +} + +// For KIP-500 / KIP-631, BrokerRegistrationRequest is an internal +// broker-to-broker only request. +type BrokerRegistrationRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The broker ID. + BrokerID int32 + + // The cluster ID of the broker process. + ClusterID string + + // The incarnation ID of the broker process. + IncarnationID [16]byte + + // The listeners for this broker. + Listeners []BrokerRegistrationRequestListener + + // Features on this broker. + Features []BrokerRegistrationRequestFeature + + // The rack that this broker is in, if any. + Rack *string + + // If the required configurations for ZK migration are present, this value is + // set to true. + IsMigratingZkBroker bool // v1+ + + // Log directories configured in this broker which are available., + LogDirs [][16]byte // v2+ + + // The epoch before a clean shutdown. + // + // This field has a default of -1. + PreviousBrokerEpoch int64 // v3+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*BrokerRegistrationRequest) Key() int16 { return 62 } +func (*BrokerRegistrationRequest) MaxVersion() int16 { return 4 } +func (v *BrokerRegistrationRequest) SetVersion(version int16) { v.Version = version } +func (v *BrokerRegistrationRequest) GetVersion() int16 { return v.Version } +func (v *BrokerRegistrationRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *BrokerRegistrationRequest) ResponseKind() Response { + r := &BrokerRegistrationResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *BrokerRegistrationRequest) RequestWith(ctx context.Context, r Requestor) (*BrokerRegistrationResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*BrokerRegistrationResponse) + return resp, err +} + +func (v *BrokerRegistrationRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.IncarnationID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Listeners + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + { + v := v.SecurityProtocol + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Features + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MinSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MaxSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.IsMigratingZkBroker + dst = kbin.AppendBool(dst, v) + } + if version >= 2 { + v := v.LogDirs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendUuid(dst, v) + } + } + if version >= 3 { + v := v.PreviousBrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BrokerRegistrationRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BrokerRegistrationRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BrokerRegistrationRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClusterID = v + } + { + v := b.Uuid() + s.IncarnationID = v + } + { + v := s.Listeners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BrokerRegistrationRequestListener, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + { + v := b.Int16() + s.SecurityProtocol = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Listeners = v + } + { + v := s.Features + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]BrokerRegistrationRequestFeature, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int16() + s.MinSupportedVersion = v + } + { + v := b.Int16() + s.MaxSupportedVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Features = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if version >= 1 { + v := b.Bool() + s.IsMigratingZkBroker = v + } + if version >= 2 { + v := s.LogDirs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([][16]byte, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Uuid() + a[i] = v + } + v = a + s.LogDirs = v + } + if version >= 3 { + v := b.Int64() + s.PreviousBrokerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrBrokerRegistrationRequest returns a pointer to a default BrokerRegistrationRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBrokerRegistrationRequest() *BrokerRegistrationRequest { + var v BrokerRegistrationRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerRegistrationRequest. +func (v *BrokerRegistrationRequest) Default() { + v.PreviousBrokerEpoch = -1 +} + +// NewBrokerRegistrationRequest returns a default BrokerRegistrationRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerRegistrationRequest() BrokerRegistrationRequest { + var v BrokerRegistrationRequest + v.Default() + return v +} + +// BrokerRegistrationResponse is a response to a BrokerRegistrationRequest. +type BrokerRegistrationResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Any error code, or 0. + ErrorCode int16 + + // The broker's assigned epoch, or -1 if none was assigned. + // + // This field has a default of -1. + BrokerEpoch int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*BrokerRegistrationResponse) Key() int16 { return 62 } +func (*BrokerRegistrationResponse) MaxVersion() int16 { return 4 } +func (v *BrokerRegistrationResponse) SetVersion(version int16) { v.Version = version } +func (v *BrokerRegistrationResponse) GetVersion() int16 { return v.Version } +func (v *BrokerRegistrationResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *BrokerRegistrationResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *BrokerRegistrationResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *BrokerRegistrationResponse) RequestKind() Request { + return &BrokerRegistrationRequest{Version: v.Version} +} + +func (v *BrokerRegistrationResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BrokerRegistrationResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BrokerRegistrationResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BrokerRegistrationResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrBrokerRegistrationResponse returns a pointer to a default BrokerRegistrationResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBrokerRegistrationResponse() *BrokerRegistrationResponse { + var v BrokerRegistrationResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerRegistrationResponse. +func (v *BrokerRegistrationResponse) Default() { + v.BrokerEpoch = -1 +} + +// NewBrokerRegistrationResponse returns a default BrokerRegistrationResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerRegistrationResponse() BrokerRegistrationResponse { + var v BrokerRegistrationResponse + v.Default() + return v +} + +// For KIP-500 / KIP-631, BrokerHeartbeatRequest is an internal +// broker-to-broker only request. +type BrokerHeartbeatRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The broker ID. + BrokerID int32 + + // The broker's epoch. + // + // This field has a default of -1. + BrokerEpoch int64 + + // The highest metadata offset that the broker has reached. + CurrentMetadataOffset int64 + + // True if the broker wants to be fenced. + WantFence bool + + // True if the broker wants to be shutdown. + WantShutdown bool + + // Log directories that failed and went offline. + OfflineLogDirs [][16]byte // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*BrokerHeartbeatRequest) Key() int16 { return 63 } +func (*BrokerHeartbeatRequest) MaxVersion() int16 { return 1 } +func (v *BrokerHeartbeatRequest) SetVersion(version int16) { v.Version = version } +func (v *BrokerHeartbeatRequest) GetVersion() int16 { return v.Version } +func (v *BrokerHeartbeatRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *BrokerHeartbeatRequest) ResponseKind() Response { + r := &BrokerHeartbeatResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *BrokerHeartbeatRequest) RequestWith(ctx context.Context, r Requestor) (*BrokerHeartbeatResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*BrokerHeartbeatResponse) + return resp, err +} + +func (v *BrokerHeartbeatRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + { + v := v.CurrentMetadataOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.WantFence + dst = kbin.AppendBool(dst, v) + } + { + v := v.WantShutdown + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + var toEncode []uint32 + if len(v.OfflineLogDirs) > 0 { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.OfflineLogDirs + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fOfflineLogDirs: + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendUuid(dst, v) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fOfflineLogDirs + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BrokerHeartbeatRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BrokerHeartbeatRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BrokerHeartbeatRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + { + v := b.Int64() + s.CurrentMetadataOffset = v + } + { + v := b.Bool() + s.WantFence = v + } + { + v := b.Bool() + s.WantShutdown = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := s.OfflineLogDirs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([][16]byte, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Uuid() + a[i] = v + } + v = a + s.OfflineLogDirs = v + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrBrokerHeartbeatRequest returns a pointer to a default BrokerHeartbeatRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBrokerHeartbeatRequest() *BrokerHeartbeatRequest { + var v BrokerHeartbeatRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerHeartbeatRequest. +func (v *BrokerHeartbeatRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewBrokerHeartbeatRequest returns a default BrokerHeartbeatRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerHeartbeatRequest() BrokerHeartbeatRequest { + var v BrokerHeartbeatRequest + v.Default() + return v +} + +// BrokerHeartbeatResponse is a response to a BrokerHeartbeatRequest. +type BrokerHeartbeatResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Any error code, or 0. + ErrorCode int16 + + // True if the broker has approximately caught up with the latest metadata. + IsCaughtUp bool + + // True if the broker is fenced. + // + // This field has a default of true. + IsFenced bool + + // True if the broker should proceed with its shutdown. + ShouldShutdown bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*BrokerHeartbeatResponse) Key() int16 { return 63 } +func (*BrokerHeartbeatResponse) MaxVersion() int16 { return 1 } +func (v *BrokerHeartbeatResponse) SetVersion(version int16) { v.Version = version } +func (v *BrokerHeartbeatResponse) GetVersion() int16 { return v.Version } +func (v *BrokerHeartbeatResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *BrokerHeartbeatResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *BrokerHeartbeatResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *BrokerHeartbeatResponse) RequestKind() Request { + return &BrokerHeartbeatRequest{Version: v.Version} +} + +func (v *BrokerHeartbeatResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.IsCaughtUp + dst = kbin.AppendBool(dst, v) + } + { + v := v.IsFenced + dst = kbin.AppendBool(dst, v) + } + { + v := v.ShouldShutdown + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *BrokerHeartbeatResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *BrokerHeartbeatResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *BrokerHeartbeatResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Bool() + s.IsCaughtUp = v + } + { + v := b.Bool() + s.IsFenced = v + } + { + v := b.Bool() + s.ShouldShutdown = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrBrokerHeartbeatResponse returns a pointer to a default BrokerHeartbeatResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrBrokerHeartbeatResponse() *BrokerHeartbeatResponse { + var v BrokerHeartbeatResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to BrokerHeartbeatResponse. +func (v *BrokerHeartbeatResponse) Default() { + v.IsFenced = true +} + +// NewBrokerHeartbeatResponse returns a default BrokerHeartbeatResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewBrokerHeartbeatResponse() BrokerHeartbeatResponse { + var v BrokerHeartbeatResponse + v.Default() + return v +} + +// For KIP-500 / KIP-631, UnregisterBrokerRequest is an admin request to +// remove registration of a broker from the cluster. +type UnregisterBrokerRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The broker ID to unregister. + BrokerID int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UnregisterBrokerRequest) Key() int16 { return 64 } +func (*UnregisterBrokerRequest) MaxVersion() int16 { return 0 } +func (v *UnregisterBrokerRequest) SetVersion(version int16) { v.Version = version } +func (v *UnregisterBrokerRequest) GetVersion() int16 { return v.Version } +func (v *UnregisterBrokerRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *UnregisterBrokerRequest) ResponseKind() Response { + r := &UnregisterBrokerResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *UnregisterBrokerRequest) RequestWith(ctx context.Context, r Requestor) (*UnregisterBrokerResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*UnregisterBrokerResponse) + return resp, err +} + +func (v *UnregisterBrokerRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UnregisterBrokerRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UnregisterBrokerRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UnregisterBrokerRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUnregisterBrokerRequest returns a pointer to a default UnregisterBrokerRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUnregisterBrokerRequest() *UnregisterBrokerRequest { + var v UnregisterBrokerRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UnregisterBrokerRequest. +func (v *UnregisterBrokerRequest) Default() { +} + +// NewUnregisterBrokerRequest returns a default UnregisterBrokerRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewUnregisterBrokerRequest() UnregisterBrokerRequest { + var v UnregisterBrokerRequest + v.Default() + return v +} + +// UnregisterBrokerResponse is a response to a UnregisterBrokerRequest. +type UnregisterBrokerResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Any error code, or 0. + ErrorCode int16 + + // The error message, if any. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UnregisterBrokerResponse) Key() int16 { return 64 } +func (*UnregisterBrokerResponse) MaxVersion() int16 { return 0 } +func (v *UnregisterBrokerResponse) SetVersion(version int16) { v.Version = version } +func (v *UnregisterBrokerResponse) GetVersion() int16 { return v.Version } +func (v *UnregisterBrokerResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *UnregisterBrokerResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *UnregisterBrokerResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *UnregisterBrokerResponse) RequestKind() Request { + return &UnregisterBrokerRequest{Version: v.Version} +} + +func (v *UnregisterBrokerResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UnregisterBrokerResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UnregisterBrokerResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UnregisterBrokerResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUnregisterBrokerResponse returns a pointer to a default UnregisterBrokerResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUnregisterBrokerResponse() *UnregisterBrokerResponse { + var v UnregisterBrokerResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UnregisterBrokerResponse. +func (v *UnregisterBrokerResponse) Default() { +} + +// NewUnregisterBrokerResponse returns a default UnregisterBrokerResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewUnregisterBrokerResponse() UnregisterBrokerResponse { + var v UnregisterBrokerResponse + v.Default() + return v +} + +// For KIP-664, DescribeTransactionsRequest describes the state of transactions. +type DescribeTransactionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Array of transactionalIds to include in describe results. If empty, then + // no results will be returned. + TransactionalIDs []string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeTransactionsRequest) Key() int16 { return 65 } +func (*DescribeTransactionsRequest) MaxVersion() int16 { return 0 } +func (v *DescribeTransactionsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeTransactionsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeTransactionsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeTransactionsRequest) ResponseKind() Response { + r := &DescribeTransactionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeTransactionsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeTransactionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeTransactionsResponse) + return resp, err +} + +func (v *DescribeTransactionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.TransactionalIDs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeTransactionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeTransactionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeTransactionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.TransactionalIDs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.TransactionalIDs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeTransactionsRequest returns a pointer to a default DescribeTransactionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeTransactionsRequest() *DescribeTransactionsRequest { + var v DescribeTransactionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTransactionsRequest. +func (v *DescribeTransactionsRequest) Default() { +} + +// NewDescribeTransactionsRequest returns a default DescribeTransactionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTransactionsRequest() DescribeTransactionsRequest { + var v DescribeTransactionsRequest + v.Default() + return v +} + +type DescribeTransactionsResponseTransactionStateTopic struct { + Topic string + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTransactionsResponseTransactionStateTopic. +func (v *DescribeTransactionsResponseTransactionStateTopic) Default() { +} + +// NewDescribeTransactionsResponseTransactionStateTopic returns a default DescribeTransactionsResponseTransactionStateTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTransactionsResponseTransactionStateTopic() DescribeTransactionsResponseTransactionStateTopic { + var v DescribeTransactionsResponseTransactionStateTopic + v.Default() + return v +} + +type DescribeTransactionsResponseTransactionState struct { + // A potential error code for describing this transaction. + // + // NOT_COORDINATOR is returned if the broker receiving this transactional + // ID does not own the ID. + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the coordinator is loading. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator is being shutdown. + // + // TRANSACTIONAL_ID_NOT_FOUND is returned if the transactional ID could not be found. + // + // TRANSACTIONAL_ID_AUTHORIZATION_FAILED is returned if the user does not have + // Describe permissions on the transactional ID. + ErrorCode int16 + + // TransactionalID is the transactional ID this record is for. + TransactionalID string + + // State is the state the transaction is in. + State string + + // TimeoutMillis is the timeout of this transaction in milliseconds. + TimeoutMillis int32 + + // StartTimestamp is the timestamp in millis of when this transaction started. + StartTimestamp int64 + + // ProducerID is the ID in use by the transactional ID. + ProducerID int64 + + // ProducerEpoch is the epoch associated with the producer ID. + ProducerEpoch int16 + + // The set of partitions included in the current transaction (if active). + // When a transaction is preparing to commit or abort, this will include + // only partitions which do not have markers. + // + // This does not include topics the user is not authorized to describe. + Topics []DescribeTransactionsResponseTransactionStateTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTransactionsResponseTransactionState. +func (v *DescribeTransactionsResponseTransactionState) Default() { +} + +// NewDescribeTransactionsResponseTransactionState returns a default DescribeTransactionsResponseTransactionState +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTransactionsResponseTransactionState() DescribeTransactionsResponseTransactionState { + var v DescribeTransactionsResponseTransactionState + v.Default() + return v +} + +// DescribeTransactionsResponse is a response to a DescribeTransactionsRequest. +type DescribeTransactionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + TransactionStates []DescribeTransactionsResponseTransactionState + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeTransactionsResponse) Key() int16 { return 65 } +func (*DescribeTransactionsResponse) MaxVersion() int16 { return 0 } +func (v *DescribeTransactionsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeTransactionsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeTransactionsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeTransactionsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DescribeTransactionsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeTransactionsResponse) RequestKind() Request { + return &DescribeTransactionsRequest{Version: v.Version} +} + +func (v *DescribeTransactionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.TransactionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.State + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartTimestamp + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerEpoch + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeTransactionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeTransactionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeTransactionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.TransactionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeTransactionsResponseTransactionState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.State = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := b.Int64() + s.StartTimestamp = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + v := b.Int16() + s.ProducerEpoch = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeTransactionsResponseTransactionStateTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TransactionStates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeTransactionsResponse returns a pointer to a default DescribeTransactionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeTransactionsResponse() *DescribeTransactionsResponse { + var v DescribeTransactionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTransactionsResponse. +func (v *DescribeTransactionsResponse) Default() { +} + +// NewDescribeTransactionsResponse returns a default DescribeTransactionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTransactionsResponse() DescribeTransactionsResponse { + var v DescribeTransactionsResponse + v.Default() + return v +} + +// For KIP-664, ListTransactionsRequest lists transactions. +type ListTransactionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The transaction states to filter by: if empty, all transactions are + // returned; if non-empty, then only transactions matching one of the + // filtered states will be returned. + // + // For a list of valid states, see the TransactionState enum. + StateFilters []string + + // The producer IDs to filter by: if empty, all transactions will be + // returned; if non-empty, only transactions which match one of the filtered + // producer IDs will be returned. + ProducerIDFilters []int64 + + // Duration (in millis) to filter by: if < 0, all transactions will be + // returned; otherwise, only transactions running longer than this duration + // will be returned. + // + // This field has a default of -1. + DurationFilterMillis int64 // v1+ + + // The transactional ID regular expression pattern to filter by: if it is + // empty or null, all transactions are returned; otherwise then only the + // transactions matching the given regular expression will be returned. + // Uses re2 syntax. + TransactionalIDPattern *string // v2+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListTransactionsRequest) Key() int16 { return 66 } +func (*ListTransactionsRequest) MaxVersion() int16 { return 2 } +func (v *ListTransactionsRequest) SetVersion(version int16) { v.Version = version } +func (v *ListTransactionsRequest) GetVersion() int16 { return v.Version } +func (v *ListTransactionsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ListTransactionsRequest) ResponseKind() Response { + r := &ListTransactionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ListTransactionsRequest) RequestWith(ctx context.Context, r Requestor) (*ListTransactionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ListTransactionsResponse) + return resp, err +} + +func (v *ListTransactionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.StateFilters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.ProducerIDFilters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt64(dst, v) + } + } + if version >= 1 { + v := v.DurationFilterMillis + dst = kbin.AppendInt64(dst, v) + } + if version >= 2 { + v := v.TransactionalIDPattern + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListTransactionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListTransactionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListTransactionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.StateFilters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.StateFilters = v + } + { + v := s.ProducerIDFilters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int64, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int64() + a[i] = v + } + v = a + s.ProducerIDFilters = v + } + if version >= 1 { + v := b.Int64() + s.DurationFilterMillis = v + } + if version >= 2 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.TransactionalIDPattern = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListTransactionsRequest returns a pointer to a default ListTransactionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListTransactionsRequest() *ListTransactionsRequest { + var v ListTransactionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListTransactionsRequest. +func (v *ListTransactionsRequest) Default() { + v.DurationFilterMillis = -1 +} + +// NewListTransactionsRequest returns a default ListTransactionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewListTransactionsRequest() ListTransactionsRequest { + var v ListTransactionsRequest + v.Default() + return v +} + +type ListTransactionsResponseTransactionState struct { + // The transactional ID being used. + TransactionalID string + + // The producer ID of the producer. + ProducerID int64 + + // The current transaction state of the producer. + TransactionState string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListTransactionsResponseTransactionState. +func (v *ListTransactionsResponseTransactionState) Default() { +} + +// NewListTransactionsResponseTransactionState returns a default ListTransactionsResponseTransactionState +// This is a shortcut for creating a struct and calling Default yourself. +func NewListTransactionsResponseTransactionState() ListTransactionsResponseTransactionState { + var v ListTransactionsResponseTransactionState + v.Default() + return v +} + +// ListTransactionsResponse is a response to a ListTransactionsRequest. +type ListTransactionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // A potential error code for the listing, + // + // COORDINATOR_LOAD_IN_PROGRESS is returned if the coordinator is loading. + // + // COORDINATOR_NOT_AVAILABLE is returned if the coordinator receiving this + // request is shutting down. + // + // INVALID_REGULAR_EXPRESSION is returned if using an invalid regex in + // the request's transactional id pattern field. + ErrorCode int16 + + // Set of state filters provided in the request which were unknown to the + // transaction coordinator. + UnknownStateFilters []string + + // TransactionStates contains all transactions that were matched for listing + // in the request. The response elides transactions that the user does not have + // permission to describe (DESCRIBE on TRANSACTIONAL_ID for the transaction). + TransactionStates []ListTransactionsResponseTransactionState + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListTransactionsResponse) Key() int16 { return 66 } +func (*ListTransactionsResponse) MaxVersion() int16 { return 2 } +func (v *ListTransactionsResponse) SetVersion(version int16) { v.Version = version } +func (v *ListTransactionsResponse) GetVersion() int16 { return v.Version } +func (v *ListTransactionsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ListTransactionsResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *ListTransactionsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ListTransactionsResponse) RequestKind() Request { + return &ListTransactionsRequest{Version: v.Version} +} + +func (v *ListTransactionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.UnknownStateFilters + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.TransactionStates + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TransactionalID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ProducerID + dst = kbin.AppendInt64(dst, v) + } + { + v := v.TransactionState + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListTransactionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListTransactionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListTransactionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.UnknownStateFilters + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.UnknownStateFilters = v + } + { + v := s.TransactionStates + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListTransactionsResponseTransactionState, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionalID = v + } + { + v := b.Int64() + s.ProducerID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.TransactionState = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TransactionStates = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListTransactionsResponse returns a pointer to a default ListTransactionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListTransactionsResponse() *ListTransactionsResponse { + var v ListTransactionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListTransactionsResponse. +func (v *ListTransactionsResponse) Default() { +} + +// NewListTransactionsResponse returns a default ListTransactionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewListTransactionsResponse() ListTransactionsResponse { + var v ListTransactionsResponse + v.Default() + return v +} + +// For KIP-730, AllocateProducerIDsRequest is a broker-to-broker request that +// requests a block of producer IDs from the controller broker. This is more +// specifically introduced for raft, but allows for one more request to avoid +// zookeeper in the non-raft world as well. +type AllocateProducerIDsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The ID of the requesting broker. + BrokerID int32 + + // The epoch of the requesting broker. + // + // This field has a default of -1. + BrokerEpoch int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AllocateProducerIDsRequest) Key() int16 { return 67 } +func (*AllocateProducerIDsRequest) MaxVersion() int16 { return 0 } +func (v *AllocateProducerIDsRequest) SetVersion(version int16) { v.Version = version } +func (v *AllocateProducerIDsRequest) GetVersion() int16 { return v.Version } +func (v *AllocateProducerIDsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AllocateProducerIDsRequest) ResponseKind() Response { + r := &AllocateProducerIDsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AllocateProducerIDsRequest) RequestWith(ctx context.Context, r Requestor) (*AllocateProducerIDsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AllocateProducerIDsResponse) + return resp, err +} + +func (v *AllocateProducerIDsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AllocateProducerIDsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AllocateProducerIDsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AllocateProducerIDsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAllocateProducerIDsRequest returns a pointer to a default AllocateProducerIDsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAllocateProducerIDsRequest() *AllocateProducerIDsRequest { + var v AllocateProducerIDsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AllocateProducerIDsRequest. +func (v *AllocateProducerIDsRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewAllocateProducerIDsRequest returns a default AllocateProducerIDsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAllocateProducerIDsRequest() AllocateProducerIDsRequest { + var v AllocateProducerIDsRequest + v.Default() + return v +} + +// AllocateProducerIDsResponse is a response to an AllocateProducerIDsRequest. +type AllocateProducerIDsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // An error code, if any. + ErrorCode int16 + + // The first producer ID in this range, inclusive. + ProducerIDStart int64 + + // The number of producer IDs in this range. + ProducerIDLen int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AllocateProducerIDsResponse) Key() int16 { return 67 } +func (*AllocateProducerIDsResponse) MaxVersion() int16 { return 0 } +func (v *AllocateProducerIDsResponse) SetVersion(version int16) { v.Version = version } +func (v *AllocateProducerIDsResponse) GetVersion() int16 { return v.Version } +func (v *AllocateProducerIDsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AllocateProducerIDsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *AllocateProducerIDsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AllocateProducerIDsResponse) RequestKind() Request { + return &AllocateProducerIDsRequest{Version: v.Version} +} + +func (v *AllocateProducerIDsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ProducerIDStart + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ProducerIDLen + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AllocateProducerIDsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AllocateProducerIDsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AllocateProducerIDsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int64() + s.ProducerIDStart = v + } + { + v := b.Int32() + s.ProducerIDLen = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAllocateProducerIDsResponse returns a pointer to a default AllocateProducerIDsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAllocateProducerIDsResponse() *AllocateProducerIDsResponse { + var v AllocateProducerIDsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AllocateProducerIDsResponse. +func (v *AllocateProducerIDsResponse) Default() { +} + +// NewAllocateProducerIDsResponse returns a default AllocateProducerIDsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAllocateProducerIDsResponse() AllocateProducerIDsResponse { + var v AllocateProducerIDsResponse + v.Default() + return v +} + +type ConsumerGroupHeartbeatRequestTopic struct { + // The topic ID. + TopicID [16]byte + + // The partitions. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupHeartbeatRequestTopic. +func (v *ConsumerGroupHeartbeatRequestTopic) Default() { +} + +// NewConsumerGroupHeartbeatRequestTopic returns a default ConsumerGroupHeartbeatRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupHeartbeatRequestTopic() ConsumerGroupHeartbeatRequestTopic { + var v ConsumerGroupHeartbeatRequestTopic + v.Default() + return v +} + +// ConsumerGroupHeartbeat is a part of KIP-848; there are a lot of details +// to this request so documentation is left to the KIP itself. +type ConsumerGroupHeartbeatRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The group ID. + Group string + + // The member ID generated by the coordinator. This must be kept during + // the entire lifetime of the member. + MemberID string + + // The current member epoch; 0 to join the group, -1 to leave, -2 to + // indicate that the static member will rejoin. + MemberEpoch int32 + + // Instance ID of the member; null if not provided or if unchanging. + InstanceID *string + + // The rack ID of the member; null if not provided or if unchanging. + RackID *string + + // RebalanceTimeoutMillis is how long the coordinator will wait on a member + // to revoke its partitions. -1 if unchanging. + // + // This field has a default of -1. + RebalanceTimeoutMillis int32 + + // Subscribed topics; null if unchanging. + SubscribedTopicNames []string + + // Subscribed topic regex; null if unchanging. + SubscribedTopicRegex *string // v1+ + + // The server side assignor to use; null if unchanging. + ServerAssignor *string + + // Topic partitions owned by the member; null if unchanging. + Topics []ConsumerGroupHeartbeatRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ConsumerGroupHeartbeatRequest) Key() int16 { return 68 } +func (*ConsumerGroupHeartbeatRequest) MaxVersion() int16 { return 1 } +func (v *ConsumerGroupHeartbeatRequest) SetVersion(version int16) { v.Version = version } +func (v *ConsumerGroupHeartbeatRequest) GetVersion() int16 { return v.Version } +func (v *ConsumerGroupHeartbeatRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ConsumerGroupHeartbeatRequest) IsGroupCoordinatorRequest() {} +func (v *ConsumerGroupHeartbeatRequest) ResponseKind() Response { + r := &ConsumerGroupHeartbeatResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ConsumerGroupHeartbeatRequest) RequestWith(ctx context.Context, r Requestor) (*ConsumerGroupHeartbeatResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ConsumerGroupHeartbeatResponse) + return resp, err +} + +func (v *ConsumerGroupHeartbeatRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RebalanceTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.SubscribedTopicNames + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if version >= 1 { + v := v.SubscribedTopicRegex + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ServerAssignor + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ConsumerGroupHeartbeatRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerGroupHeartbeatRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerGroupHeartbeatRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + v := b.Int32() + s.RebalanceTimeoutMillis = v + } + { + v := s.SubscribedTopicNames + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []string{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SubscribedTopicNames = v + } + if version >= 1 { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.SubscribedTopicRegex = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ServerAssignor = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []ConsumerGroupHeartbeatRequestTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerGroupHeartbeatRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrConsumerGroupHeartbeatRequest returns a pointer to a default ConsumerGroupHeartbeatRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrConsumerGroupHeartbeatRequest() *ConsumerGroupHeartbeatRequest { + var v ConsumerGroupHeartbeatRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupHeartbeatRequest. +func (v *ConsumerGroupHeartbeatRequest) Default() { + v.RebalanceTimeoutMillis = -1 +} + +// NewConsumerGroupHeartbeatRequest returns a default ConsumerGroupHeartbeatRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupHeartbeatRequest() ConsumerGroupHeartbeatRequest { + var v ConsumerGroupHeartbeatRequest + v.Default() + return v +} + +type ConsumerGroupHeartbeatResponseAssignmentTopic struct { + TopicID [16]byte + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupHeartbeatResponseAssignmentTopic. +func (v *ConsumerGroupHeartbeatResponseAssignmentTopic) Default() { +} + +// NewConsumerGroupHeartbeatResponseAssignmentTopic returns a default ConsumerGroupHeartbeatResponseAssignmentTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupHeartbeatResponseAssignmentTopic() ConsumerGroupHeartbeatResponseAssignmentTopic { + var v ConsumerGroupHeartbeatResponseAssignmentTopic + v.Default() + return v +} + +type ConsumerGroupHeartbeatResponseAssignment struct { + // The topics partitions that can be used immediately. + Topics []ConsumerGroupHeartbeatResponseAssignmentTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupHeartbeatResponseAssignment. +func (v *ConsumerGroupHeartbeatResponseAssignment) Default() { +} + +// NewConsumerGroupHeartbeatResponseAssignment returns a default ConsumerGroupHeartbeatResponseAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupHeartbeatResponseAssignment() ConsumerGroupHeartbeatResponseAssignment { + var v ConsumerGroupHeartbeatResponseAssignment + v.Default() + return v +} + +// ConsumerGroupHeartbeatResponse is returned from a ConsumerGroupHeartbeatRequest. +type ConsumerGroupHeartbeatResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the error for this response. + // + // Supported errors: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - NOT_COORDINATOR (version 0+) + // - COORDINATOR_NOT_AVAILABLE (version 0+) + // - COORDINATOR_LOAD_IN_PROGRESS (version 0+) + // - INVALID_REQUEST (version 0+) + // - UNKNOWN_MEMBER_ID (version 0+) + // - FENCED_MEMBER_EPOCH (version 0+) + // - UNSUPPORTED_ASSIGNOR (version 0+) + // - UNRELEASED_INSTANCE_ID (version 0+) + // - GROUP_MAX_SIZE_REACHED (version 0+) + // - INVALID_REGULAR_EXPRESSION (version 1+) + ErrorCode int16 + + // A supplementary message if this errored. + ErrorMessage *string + + // The member ID generated by the coordinator; provided when joining + // with MemberEpoch=0. + MemberID *string + + // The member epoch. + MemberEpoch int32 + + // The heartbeat interval, in milliseconds. + HeartbeatIntervalMillis int32 + + // The assignment; null if not provided. + Assignment *ConsumerGroupHeartbeatResponseAssignment + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ConsumerGroupHeartbeatResponse) Key() int16 { return 68 } +func (*ConsumerGroupHeartbeatResponse) MaxVersion() int16 { return 1 } +func (v *ConsumerGroupHeartbeatResponse) SetVersion(version int16) { v.Version = version } +func (v *ConsumerGroupHeartbeatResponse) GetVersion() int16 { return v.Version } +func (v *ConsumerGroupHeartbeatResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ConsumerGroupHeartbeatResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ConsumerGroupHeartbeatResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ConsumerGroupHeartbeatResponse) RequestKind() Request { + return &ConsumerGroupHeartbeatRequest{Version: v.Version} +} + +func (v *ConsumerGroupHeartbeatResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.HeartbeatIntervalMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Assignment + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ConsumerGroupHeartbeatResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerGroupHeartbeatResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerGroupHeartbeatResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + v := b.Int32() + s.HeartbeatIntervalMillis = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.Assignment = new(ConsumerGroupHeartbeatResponseAssignment) + v := s.Assignment + v.Default() + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerGroupHeartbeatResponseAssignmentTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrConsumerGroupHeartbeatResponse returns a pointer to a default ConsumerGroupHeartbeatResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrConsumerGroupHeartbeatResponse() *ConsumerGroupHeartbeatResponse { + var v ConsumerGroupHeartbeatResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupHeartbeatResponse. +func (v *ConsumerGroupHeartbeatResponse) Default() { + { + v := &v.Assignment + _ = v + } +} + +// NewConsumerGroupHeartbeatResponse returns a default ConsumerGroupHeartbeatResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupHeartbeatResponse() ConsumerGroupHeartbeatResponse { + var v ConsumerGroupHeartbeatResponse + v.Default() + return v +} + +// Assignment contains consumer group assignments. +type Assignment struct { + // The topics & partitions assigned to the member. + TopicPartitions []AssignmentTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to Assignment. +func (v *Assignment) Default() { +} + +// NewAssignment returns a default Assignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignment() Assignment { + var v Assignment + v.Default() + return v +} + +// ConsumerGroupDescribe is a part of KIP-848; this is the +// "next generation" equivalent of DescribeGroups. +type ConsumerGroupDescribeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The IDs of the groups to describe. + Groups []string + + // Whether to include authorized operations. + IncludeAuthorizedOperations bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ConsumerGroupDescribeRequest) Key() int16 { return 69 } +func (*ConsumerGroupDescribeRequest) MaxVersion() int16 { return 1 } +func (v *ConsumerGroupDescribeRequest) SetVersion(version int16) { v.Version = version } +func (v *ConsumerGroupDescribeRequest) GetVersion() int16 { return v.Version } +func (v *ConsumerGroupDescribeRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ConsumerGroupDescribeRequest) ResponseKind() Response { + r := &ConsumerGroupDescribeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ConsumerGroupDescribeRequest) RequestWith(ctx context.Context, r Requestor) (*ConsumerGroupDescribeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ConsumerGroupDescribeResponse) + return resp, err +} + +func (v *ConsumerGroupDescribeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.IncludeAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ConsumerGroupDescribeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerGroupDescribeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerGroupDescribeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.Groups = v + } + { + v := b.Bool() + s.IncludeAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrConsumerGroupDescribeRequest returns a pointer to a default ConsumerGroupDescribeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrConsumerGroupDescribeRequest() *ConsumerGroupDescribeRequest { + var v ConsumerGroupDescribeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupDescribeRequest. +func (v *ConsumerGroupDescribeRequest) Default() { +} + +// NewConsumerGroupDescribeRequest returns a default ConsumerGroupDescribeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupDescribeRequest() ConsumerGroupDescribeRequest { + var v ConsumerGroupDescribeRequest + v.Default() + return v +} + +type ConsumerGroupDescribeResponseGroupMember struct { + // The member ID. + MemberID string + + // The member instance ID, if any. + InstanceID *string + + // The member rack ID, if any. + RackID *string + + // The current member epoch. + MemberEpoch int32 + + // The client ID. + ClientID string + + // The client host. + ClientHost string + + // The subscribed topic names. + SubscribedTopics []string + + // The subscribed topic regex, if any. + SubscribedTopicRegex *string + + // The current assignment. + Assignment Assignment + + // The target assignment. + TargetAssignment Assignment + + // The member type; -1 for unknown. 0 for classic member. +1 for consumer member. + // + // This field has a default of -1. + MemberType int8 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupDescribeResponseGroupMember. +func (v *ConsumerGroupDescribeResponseGroupMember) Default() { + { + v := &v.Assignment + _ = v + } + { + v := &v.TargetAssignment + _ = v + } + v.MemberType = -1 +} + +// NewConsumerGroupDescribeResponseGroupMember returns a default ConsumerGroupDescribeResponseGroupMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupDescribeResponseGroupMember() ConsumerGroupDescribeResponseGroupMember { + var v ConsumerGroupDescribeResponseGroupMember + v.Default() + return v +} + +type ConsumerGroupDescribeResponseGroup struct { + // ErrorCode is the error for this response. + // + // Supported errors: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - NOT_COORDINATOR (version 0+) + // - COORDINATOR_NOT_AVAILABLE (version 0+) + // - COORDINATOR_LOAD_IN_PROGRESS (version 0+) + // - INVALID_REQUEST (version 0+) + // - INVALID_GROUP_ID (version 0+) + // - GROUP_ID_NOT_FOUND (version 0+) + ErrorCode int16 + + // A supplementary message if this errored. + ErrorMessage *string + + // The group ID. + Group string + + // The group state. + State string + + // The group epoch. + Epoch int32 + + // The assignment epoch. + AssignmentEpoch int32 + + // The selected assignor. + AssignorName string + + // Members of the group. + Members []ConsumerGroupDescribeResponseGroupMember + + // 32 bit bitfield representing authorized operations for the group. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupDescribeResponseGroup. +func (v *ConsumerGroupDescribeResponseGroup) Default() { + v.AuthorizedOperations = -2147483648 +} + +// NewConsumerGroupDescribeResponseGroup returns a default ConsumerGroupDescribeResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupDescribeResponseGroup() ConsumerGroupDescribeResponseGroup { + var v ConsumerGroupDescribeResponseGroup + v.Default() + return v +} + +// ConsumerGroupDescribeResponse is returned from a ConsumerGroupDescribeRequest. +type ConsumerGroupDescribeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + Groups []ConsumerGroupDescribeResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ConsumerGroupDescribeResponse) Key() int16 { return 69 } +func (*ConsumerGroupDescribeResponse) MaxVersion() int16 { return 1 } +func (v *ConsumerGroupDescribeResponse) SetVersion(version int16) { v.Version = version } +func (v *ConsumerGroupDescribeResponse) GetVersion() int16 { return v.Version } +func (v *ConsumerGroupDescribeResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ConsumerGroupDescribeResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ConsumerGroupDescribeResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ConsumerGroupDescribeResponse) RequestKind() Request { + return &ConsumerGroupDescribeRequest{Version: v.Version} +} + +func (v *ConsumerGroupDescribeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.State + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AssignmentEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AssignorName + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ClientID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ClientHost + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SubscribedTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.SubscribedTopicRegex + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := &v.Assignment + { + v := v.TopicPartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := &v.TargetAssignment + { + v := v.TopicPartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if version >= 1 { + v := v.MemberType + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ConsumerGroupDescribeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ConsumerGroupDescribeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ConsumerGroupDescribeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerGroupDescribeResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.State = v + } + { + v := b.Int32() + s.Epoch = v + } + { + v := b.Int32() + s.AssignmentEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.AssignorName = v + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ConsumerGroupDescribeResponseGroupMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientHost = v + } + { + v := s.SubscribedTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SubscribedTopics = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.SubscribedTopicRegex = v + } + { + v := &s.Assignment + v.Default() + s := v + { + v := s.TopicPartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignmentTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicPartitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := &s.TargetAssignment + v.Default() + s := v + { + v := s.TopicPartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignmentTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicPartitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + if version >= 1 { + v := b.Int8() + s.MemberType = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrConsumerGroupDescribeResponse returns a pointer to a default ConsumerGroupDescribeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrConsumerGroupDescribeResponse() *ConsumerGroupDescribeResponse { + var v ConsumerGroupDescribeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ConsumerGroupDescribeResponse. +func (v *ConsumerGroupDescribeResponse) Default() { +} + +// NewConsumerGroupDescribeResponse returns a default ConsumerGroupDescribeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewConsumerGroupDescribeResponse() ConsumerGroupDescribeResponse { + var v ConsumerGroupDescribeResponse + v.Default() + return v +} + +type ControllerRegistrationRequestListener struct { + Name string + + Host string + + Port uint16 + + SecurityProtocol int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControllerRegistrationRequestListener. +func (v *ControllerRegistrationRequestListener) Default() { +} + +// NewControllerRegistrationRequestListener returns a default ControllerRegistrationRequestListener +// This is a shortcut for creating a struct and calling Default yourself. +func NewControllerRegistrationRequestListener() ControllerRegistrationRequestListener { + var v ControllerRegistrationRequestListener + v.Default() + return v +} + +type ControllerRegistrationRequestFeature struct { + Name string + + MinSupportedVersion int16 + + MaxSupportedVersion int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControllerRegistrationRequestFeature. +func (v *ControllerRegistrationRequestFeature) Default() { +} + +// NewControllerRegistrationRequestFeature returns a default ControllerRegistrationRequestFeature +// This is a shortcut for creating a struct and calling Default yourself. +func NewControllerRegistrationRequestFeature() ControllerRegistrationRequestFeature { + var v ControllerRegistrationRequestFeature + v.Default() + return v +} + +// ControllerRegistrationRequest, was added for KIP-919, is what controllers +// send to the active controller to register themselves. +// +// This requires CLUSTER_ACTION on CLUSTER. +type ControllerRegistrationRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The ID of the controller to register. + ControllerID int32 + + // The controller incarnation ID, which is unique to each process run. + IncarnationID [16]byte + + // Set if the required configurations for ZK migration are present. + ZkMigrationReady bool + + // The listeners of this controller. + Listeners []ControllerRegistrationRequestListener + + // The features on this controller. + Features []ControllerRegistrationRequestFeature + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ControllerRegistrationRequest) Key() int16 { return 70 } +func (*ControllerRegistrationRequest) MaxVersion() int16 { return 0 } +func (v *ControllerRegistrationRequest) SetVersion(version int16) { v.Version = version } +func (v *ControllerRegistrationRequest) GetVersion() int16 { return v.Version } +func (v *ControllerRegistrationRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ControllerRegistrationRequest) ResponseKind() Response { + r := &ControllerRegistrationResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ControllerRegistrationRequest) RequestWith(ctx context.Context, r Requestor) (*ControllerRegistrationResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ControllerRegistrationResponse) + return resp, err +} + +func (v *ControllerRegistrationRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ControllerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.IncarnationID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.ZkMigrationReady + dst = kbin.AppendBool(dst, v) + } + { + v := v.Listeners + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + { + v := v.SecurityProtocol + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Features + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MinSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MaxSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ControllerRegistrationRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ControllerRegistrationRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ControllerRegistrationRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ControllerID = v + } + { + v := b.Uuid() + s.IncarnationID = v + } + { + v := b.Bool() + s.ZkMigrationReady = v + } + { + v := s.Listeners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ControllerRegistrationRequestListener, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + { + v := b.Int16() + s.SecurityProtocol = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Listeners = v + } + { + v := s.Features + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ControllerRegistrationRequestFeature, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + v := b.Int16() + s.MinSupportedVersion = v + } + { + v := b.Int16() + s.MaxSupportedVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Features = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrControllerRegistrationRequest returns a pointer to a default ControllerRegistrationRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrControllerRegistrationRequest() *ControllerRegistrationRequest { + var v ControllerRegistrationRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControllerRegistrationRequest. +func (v *ControllerRegistrationRequest) Default() { +} + +// NewControllerRegistrationRequest returns a default ControllerRegistrationRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewControllerRegistrationRequest() ControllerRegistrationRequest { + var v ControllerRegistrationRequest + v.Default() + return v +} + +type ControllerRegistrationResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ControllerRegistrationResponse) Key() int16 { return 70 } +func (*ControllerRegistrationResponse) MaxVersion() int16 { return 0 } +func (v *ControllerRegistrationResponse) SetVersion(version int16) { v.Version = version } +func (v *ControllerRegistrationResponse) GetVersion() int16 { return v.Version } +func (v *ControllerRegistrationResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ControllerRegistrationResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ControllerRegistrationResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ControllerRegistrationResponse) RequestKind() Request { + return &ControllerRegistrationRequest{Version: v.Version} +} + +func (v *ControllerRegistrationResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ControllerRegistrationResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ControllerRegistrationResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ControllerRegistrationResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrControllerRegistrationResponse returns a pointer to a default ControllerRegistrationResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrControllerRegistrationResponse() *ControllerRegistrationResponse { + var v ControllerRegistrationResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ControllerRegistrationResponse. +func (v *ControllerRegistrationResponse) Default() { +} + +// NewControllerRegistrationResponse returns a default ControllerRegistrationResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewControllerRegistrationResponse() ControllerRegistrationResponse { + var v ControllerRegistrationResponse + v.Default() + return v +} + +// GetTelemetrySubscriptionsRequest is the initial request from the +// client asking the broker which metrics the client should send. +// This request is periodically reissued to check for updates on what +// or how the client should send. See KIP-714 for more details. +type GetTelemetrySubscriptionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Unique ID for this client instance; must be set to 0 on the first request. + ClientInstanceID [16]byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*GetTelemetrySubscriptionsRequest) Key() int16 { return 71 } +func (*GetTelemetrySubscriptionsRequest) MaxVersion() int16 { return 0 } +func (v *GetTelemetrySubscriptionsRequest) SetVersion(version int16) { v.Version = version } +func (v *GetTelemetrySubscriptionsRequest) GetVersion() int16 { return v.Version } +func (v *GetTelemetrySubscriptionsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *GetTelemetrySubscriptionsRequest) ResponseKind() Response { + r := &GetTelemetrySubscriptionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *GetTelemetrySubscriptionsRequest) RequestWith(ctx context.Context, r Requestor) (*GetTelemetrySubscriptionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*GetTelemetrySubscriptionsResponse) + return resp, err +} + +func (v *GetTelemetrySubscriptionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClientInstanceID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *GetTelemetrySubscriptionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *GetTelemetrySubscriptionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *GetTelemetrySubscriptionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Uuid() + s.ClientInstanceID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrGetTelemetrySubscriptionsRequest returns a pointer to a default GetTelemetrySubscriptionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrGetTelemetrySubscriptionsRequest() *GetTelemetrySubscriptionsRequest { + var v GetTelemetrySubscriptionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to GetTelemetrySubscriptionsRequest. +func (v *GetTelemetrySubscriptionsRequest) Default() { +} + +// NewGetTelemetrySubscriptionsRequest returns a default GetTelemetrySubscriptionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewGetTelemetrySubscriptionsRequest() GetTelemetrySubscriptionsRequest { + var v GetTelemetrySubscriptionsRequest + v.Default() + return v +} + +type GetTelemetrySubscriptionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the error, if any. + ErrorCode int16 + + // Assigned client instance id if ClientInstanceID was 0 in the request, else 0. + ClientInstanceID [16]byte + + // Unique identifier for the current subscription set for this client instance. + SubscriptionID int32 + + // Compression types that broker accepts for the PushTelemetryRequest. + AcceptedCompressionTypes []int8 + + // Configured push interval, which is the lowest configured interval in the current subscription set. + PushIntervalMillis int32 + + // The maximum bytes of binary data the broker accepts in PushTelemetryRequest. + TelemetryMaxBytes int32 + + // Flag to indicate monotonic/counter metrics are to be emitted as deltas or cumulative values. + DeltaTemporality bool + + // Requested metrics prefix string match. Empty array: No metrics subscribed, Array[0] empty string: All metrics subscribed. + RequestedMetrics []string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*GetTelemetrySubscriptionsResponse) Key() int16 { return 71 } +func (*GetTelemetrySubscriptionsResponse) MaxVersion() int16 { return 0 } +func (v *GetTelemetrySubscriptionsResponse) SetVersion(version int16) { v.Version = version } +func (v *GetTelemetrySubscriptionsResponse) GetVersion() int16 { return v.Version } +func (v *GetTelemetrySubscriptionsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *GetTelemetrySubscriptionsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *GetTelemetrySubscriptionsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *GetTelemetrySubscriptionsResponse) RequestKind() Request { + return &GetTelemetrySubscriptionsRequest{Version: v.Version} +} + +func (v *GetTelemetrySubscriptionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ClientInstanceID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.SubscriptionID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AcceptedCompressionTypes + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt8(dst, v) + } + } + { + v := v.PushIntervalMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.TelemetryMaxBytes + dst = kbin.AppendInt32(dst, v) + } + { + v := v.DeltaTemporality + dst = kbin.AppendBool(dst, v) + } + { + v := v.RequestedMetrics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *GetTelemetrySubscriptionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *GetTelemetrySubscriptionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *GetTelemetrySubscriptionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Uuid() + s.ClientInstanceID = v + } + { + v := b.Int32() + s.SubscriptionID = v + } + { + v := s.AcceptedCompressionTypes + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int8, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int8() + a[i] = v + } + v = a + s.AcceptedCompressionTypes = v + } + { + v := b.Int32() + s.PushIntervalMillis = v + } + { + v := b.Int32() + s.TelemetryMaxBytes = v + } + { + v := b.Bool() + s.DeltaTemporality = v + } + { + v := s.RequestedMetrics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.RequestedMetrics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrGetTelemetrySubscriptionsResponse returns a pointer to a default GetTelemetrySubscriptionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrGetTelemetrySubscriptionsResponse() *GetTelemetrySubscriptionsResponse { + var v GetTelemetrySubscriptionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to GetTelemetrySubscriptionsResponse. +func (v *GetTelemetrySubscriptionsResponse) Default() { +} + +// NewGetTelemetrySubscriptionsResponse returns a default GetTelemetrySubscriptionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewGetTelemetrySubscriptionsResponse() GetTelemetrySubscriptionsResponse { + var v GetTelemetrySubscriptionsResponse + v.Default() + return v +} + +// PushTelemetryRequest, part of KIP-714, is a client side push of metrics. +type PushTelemetryRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Unique ID for this client instance. + ClientInstanceID [16]byte + + // The unique identifier for the current subscription. + SubscriptionID int32 + + // If the client is terminating the connection. + Terminating bool + + // Compression codec used to compress the metrics. + CompressionType int8 + + // Metrics encoded in OpenTelemetry MetricsData v1 protobuf format. + Metrics []byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*PushTelemetryRequest) Key() int16 { return 72 } +func (*PushTelemetryRequest) MaxVersion() int16 { return 0 } +func (v *PushTelemetryRequest) SetVersion(version int16) { v.Version = version } +func (v *PushTelemetryRequest) GetVersion() int16 { return v.Version } +func (v *PushTelemetryRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *PushTelemetryRequest) ResponseKind() Response { + r := &PushTelemetryResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *PushTelemetryRequest) RequestWith(ctx context.Context, r Requestor) (*PushTelemetryResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*PushTelemetryResponse) + return resp, err +} + +func (v *PushTelemetryRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClientInstanceID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.SubscriptionID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Terminating + dst = kbin.AppendBool(dst, v) + } + { + v := v.CompressionType + dst = kbin.AppendInt8(dst, v) + } + { + v := v.Metrics + if isFlexible { + dst = kbin.AppendCompactBytes(dst, v) + } else { + dst = kbin.AppendBytes(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *PushTelemetryRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *PushTelemetryRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *PushTelemetryRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Uuid() + s.ClientInstanceID = v + } + { + v := b.Int32() + s.SubscriptionID = v + } + { + v := b.Bool() + s.Terminating = v + } + { + v := b.Int8() + s.CompressionType = v + } + { + var v []byte + if isFlexible { + v = b.CompactBytes() + } else { + v = b.Bytes() + } + s.Metrics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrPushTelemetryRequest returns a pointer to a default PushTelemetryRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrPushTelemetryRequest() *PushTelemetryRequest { + var v PushTelemetryRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to PushTelemetryRequest. +func (v *PushTelemetryRequest) Default() { +} + +// NewPushTelemetryRequest returns a default PushTelemetryRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewPushTelemetryRequest() PushTelemetryRequest { + var v PushTelemetryRequest + v.Default() + return v +} + +type PushTelemetryResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*PushTelemetryResponse) Key() int16 { return 72 } +func (*PushTelemetryResponse) MaxVersion() int16 { return 0 } +func (v *PushTelemetryResponse) SetVersion(version int16) { v.Version = version } +func (v *PushTelemetryResponse) GetVersion() int16 { return v.Version } +func (v *PushTelemetryResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *PushTelemetryResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *PushTelemetryResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *PushTelemetryResponse) RequestKind() Request { + return &PushTelemetryRequest{Version: v.Version} +} + +func (v *PushTelemetryResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *PushTelemetryResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *PushTelemetryResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *PushTelemetryResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrPushTelemetryResponse returns a pointer to a default PushTelemetryResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrPushTelemetryResponse() *PushTelemetryResponse { + var v PushTelemetryResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to PushTelemetryResponse. +func (v *PushTelemetryResponse) Default() { +} + +// NewPushTelemetryResponse returns a default PushTelemetryResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewPushTelemetryResponse() PushTelemetryResponse { + var v PushTelemetryResponse + v.Default() + return v +} + +type AssignReplicasToDirsRequestDirectoryTopicPartition struct { + // The partition index. + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsRequestDirectoryTopicPartition. +func (v *AssignReplicasToDirsRequestDirectoryTopicPartition) Default() { +} + +// NewAssignReplicasToDirsRequestDirectoryTopicPartition returns a default AssignReplicasToDirsRequestDirectoryTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsRequestDirectoryTopicPartition() AssignReplicasToDirsRequestDirectoryTopicPartition { + var v AssignReplicasToDirsRequestDirectoryTopicPartition + v.Default() + return v +} + +type AssignReplicasToDirsRequestDirectoryTopic struct { + // The ID of the assigned topic. + TopicID [16]byte + + // The partitions assigned to the directory. + Partitions []AssignReplicasToDirsRequestDirectoryTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsRequestDirectoryTopic. +func (v *AssignReplicasToDirsRequestDirectoryTopic) Default() { +} + +// NewAssignReplicasToDirsRequestDirectoryTopic returns a default AssignReplicasToDirsRequestDirectoryTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsRequestDirectoryTopic() AssignReplicasToDirsRequestDirectoryTopic { + var v AssignReplicasToDirsRequestDirectoryTopic + v.Default() + return v +} + +type AssignReplicasToDirsRequestDirectory struct { + // The ID of the directory. + ID [16]byte + + // The topics assigned to the directory. + Topics []AssignReplicasToDirsRequestDirectoryTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsRequestDirectory. +func (v *AssignReplicasToDirsRequestDirectory) Default() { +} + +// NewAssignReplicasToDirsRequestDirectory returns a default AssignReplicasToDirsRequestDirectory +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsRequestDirectory() AssignReplicasToDirsRequestDirectory { + var v AssignReplicasToDirsRequestDirectory + v.Default() + return v +} + +// AssignReplicasToDirs, introduced in Kafka 3.7, is a part of KIP-858. +// The KIP is about handling JBOD failures in KRaft, and this request, +// as the name implies, allows you to assign replicas to specific directories. +type AssignReplicasToDirsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The ID of the requesting broker. + BrokerID int32 + + // The epoch of the requesting broker. + // + // This field has a default of -1. + BrokerEpoch int64 + + // The directories to which replicas should be assigned. + Directories []AssignReplicasToDirsRequestDirectory + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AssignReplicasToDirsRequest) Key() int16 { return 73 } +func (*AssignReplicasToDirsRequest) MaxVersion() int16 { return 0 } +func (v *AssignReplicasToDirsRequest) SetVersion(version int16) { v.Version = version } +func (v *AssignReplicasToDirsRequest) GetVersion() int16 { return v.Version } +func (v *AssignReplicasToDirsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AssignReplicasToDirsRequest) ResponseKind() Response { + r := &AssignReplicasToDirsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AssignReplicasToDirsRequest) RequestWith(ctx context.Context, r Requestor) (*AssignReplicasToDirsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AssignReplicasToDirsResponse) + return resp, err +} + +func (v *AssignReplicasToDirsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.BrokerID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.BrokerEpoch + dst = kbin.AppendInt64(dst, v) + } + { + v := v.Directories + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AssignReplicasToDirsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AssignReplicasToDirsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AssignReplicasToDirsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.BrokerID = v + } + { + v := b.Int64() + s.BrokerEpoch = v + } + { + v := s.Directories + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsRequestDirectory, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.ID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsRequestDirectoryTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsRequestDirectoryTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Directories = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAssignReplicasToDirsRequest returns a pointer to a default AssignReplicasToDirsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAssignReplicasToDirsRequest() *AssignReplicasToDirsRequest { + var v AssignReplicasToDirsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsRequest. +func (v *AssignReplicasToDirsRequest) Default() { + v.BrokerEpoch = -1 +} + +// NewAssignReplicasToDirsRequest returns a default AssignReplicasToDirsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsRequest() AssignReplicasToDirsRequest { + var v AssignReplicasToDirsRequest + v.Default() + return v +} + +type AssignReplicasToDirsResponseDirectoryTopicPartition struct { + Partition int32 + + ErrorCode int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsResponseDirectoryTopicPartition. +func (v *AssignReplicasToDirsResponseDirectoryTopicPartition) Default() { +} + +// NewAssignReplicasToDirsResponseDirectoryTopicPartition returns a default AssignReplicasToDirsResponseDirectoryTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsResponseDirectoryTopicPartition() AssignReplicasToDirsResponseDirectoryTopicPartition { + var v AssignReplicasToDirsResponseDirectoryTopicPartition + v.Default() + return v +} + +type AssignReplicasToDirsResponseDirectoryTopic struct { + TopicID [16]byte + + Partitions []AssignReplicasToDirsResponseDirectoryTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsResponseDirectoryTopic. +func (v *AssignReplicasToDirsResponseDirectoryTopic) Default() { +} + +// NewAssignReplicasToDirsResponseDirectoryTopic returns a default AssignReplicasToDirsResponseDirectoryTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsResponseDirectoryTopic() AssignReplicasToDirsResponseDirectoryTopic { + var v AssignReplicasToDirsResponseDirectoryTopic + v.Default() + return v +} + +type AssignReplicasToDirsResponseDirectory struct { + ID [16]byte + + Topics []AssignReplicasToDirsResponseDirectoryTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsResponseDirectory. +func (v *AssignReplicasToDirsResponseDirectory) Default() { +} + +// NewAssignReplicasToDirsResponseDirectory returns a default AssignReplicasToDirsResponseDirectory +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsResponseDirectory() AssignReplicasToDirsResponseDirectory { + var v AssignReplicasToDirsResponseDirectory + v.Default() + return v +} + +// AssignReplicasToDirsResponse mirrors the request fields and shape, +// with an added ErrorCode alongside each partition. +type AssignReplicasToDirsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + Directories []AssignReplicasToDirsResponseDirectory + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AssignReplicasToDirsResponse) Key() int16 { return 73 } +func (*AssignReplicasToDirsResponse) MaxVersion() int16 { return 0 } +func (v *AssignReplicasToDirsResponse) SetVersion(version int16) { v.Version = version } +func (v *AssignReplicasToDirsResponse) GetVersion() int16 { return v.Version } +func (v *AssignReplicasToDirsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AssignReplicasToDirsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *AssignReplicasToDirsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AssignReplicasToDirsResponse) RequestKind() Request { + return &AssignReplicasToDirsRequest{Version: v.Version} +} + +func (v *AssignReplicasToDirsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Directories + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AssignReplicasToDirsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AssignReplicasToDirsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AssignReplicasToDirsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.Directories + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsResponseDirectory, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.ID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsResponseDirectoryTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AssignReplicasToDirsResponseDirectoryTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Directories = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAssignReplicasToDirsResponse returns a pointer to a default AssignReplicasToDirsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAssignReplicasToDirsResponse() *AssignReplicasToDirsResponse { + var v AssignReplicasToDirsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AssignReplicasToDirsResponse. +func (v *AssignReplicasToDirsResponse) Default() { +} + +// NewAssignReplicasToDirsResponse returns a default AssignReplicasToDirsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAssignReplicasToDirsResponse() AssignReplicasToDirsResponse { + var v AssignReplicasToDirsResponse + v.Default() + return v +} + +// ListConfigResourcesRequest, introduced in KIP-1000 and renamed +// in KIP-1142 allows you to list config resources. +// +// This was renamed from ListClientMetricsResources in Kafka 4.1. +type ListConfigResourcesRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The list of resource type. If the list is empty, it uses default supported + // config resource types. + ResourceTypes []int8 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListConfigResourcesRequest) Key() int16 { return 74 } +func (*ListConfigResourcesRequest) MaxVersion() int16 { return 1 } +func (v *ListConfigResourcesRequest) SetVersion(version int16) { v.Version = version } +func (v *ListConfigResourcesRequest) GetVersion() int16 { return v.Version } +func (v *ListConfigResourcesRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ListConfigResourcesRequest) ResponseKind() Response { + r := &ListConfigResourcesResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ListConfigResourcesRequest) RequestWith(ctx context.Context, r Requestor) (*ListConfigResourcesResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ListConfigResourcesResponse) + return resp, err +} + +func (v *ListConfigResourcesRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + if version >= 1 { + v := v.ResourceTypes + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListConfigResourcesRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListConfigResourcesRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListConfigResourcesRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + if version >= 1 { + v := s.ResourceTypes + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int8, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int8() + a[i] = v + } + v = a + s.ResourceTypes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListConfigResourcesRequest returns a pointer to a default ListConfigResourcesRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListConfigResourcesRequest() *ListConfigResourcesRequest { + var v ListConfigResourcesRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListConfigResourcesRequest. +func (v *ListConfigResourcesRequest) Default() { +} + +// NewListConfigResourcesRequest returns a default ListConfigResourcesRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewListConfigResourcesRequest() ListConfigResourcesRequest { + var v ListConfigResourcesRequest + v.Default() + return v +} + +type ListConfigResourcesResponseConfigResource struct { + // The resource name. + Name string + + // The resource type. + // + // This field has a default of 16. + Type int8 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListConfigResourcesResponseConfigResource. +func (v *ListConfigResourcesResponseConfigResource) Default() { + v.Type = 16 +} + +// NewListConfigResourcesResponseConfigResource returns a default ListConfigResourcesResponseConfigResource +// This is a shortcut for creating a struct and calling Default yourself. +func NewListConfigResourcesResponseConfigResource() ListConfigResourcesResponseConfigResource { + var v ListConfigResourcesResponseConfigResource + v.Default() + return v +} + +type ListConfigResourcesResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + // Each client metrics resource in the response. + ConfigResources []ListConfigResourcesResponseConfigResource + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ListConfigResourcesResponse) Key() int16 { return 74 } +func (*ListConfigResourcesResponse) MaxVersion() int16 { return 1 } +func (v *ListConfigResourcesResponse) SetVersion(version int16) { v.Version = version } +func (v *ListConfigResourcesResponse) GetVersion() int16 { return v.Version } +func (v *ListConfigResourcesResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ListConfigResourcesResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ListConfigResourcesResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ListConfigResourcesResponse) RequestKind() Request { + return &ListConfigResourcesRequest{Version: v.Version} +} + +func (v *ListConfigResourcesResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ConfigResources + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if version >= 1 { + v := v.Type + dst = kbin.AppendInt8(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ListConfigResourcesResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ListConfigResourcesResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ListConfigResourcesResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + v := s.ConfigResources + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ListConfigResourcesResponseConfigResource, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + if version >= 1 { + v := b.Int8() + s.Type = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ConfigResources = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrListConfigResourcesResponse returns a pointer to a default ListConfigResourcesResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrListConfigResourcesResponse() *ListConfigResourcesResponse { + var v ListConfigResourcesResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ListConfigResourcesResponse. +func (v *ListConfigResourcesResponse) Default() { +} + +// NewListConfigResourcesResponse returns a default ListConfigResourcesResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewListConfigResourcesResponse() ListConfigResourcesResponse { + var v ListConfigResourcesResponse + v.Default() + return v +} + +type DescribeTopicPartitionsRequestTopic struct { + // Topic is a topic name. + Topic string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsRequestTopic. +func (v *DescribeTopicPartitionsRequestTopic) Default() { +} + +// NewDescribeTopicPartitionsRequestTopic returns a default DescribeTopicPartitionsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsRequestTopic() DescribeTopicPartitionsRequestTopic { + var v DescribeTopicPartitionsRequestTopic + v.Default() + return v +} + +type DescribeTopicPartitionsRequestCursor struct { + // Topic is the topic to start with. + Topic string + + // Partition is the partition to start with. + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsRequestCursor. +func (v *DescribeTopicPartitionsRequestCursor) Default() { +} + +// NewDescribeTopicPartitionsRequestCursor returns a default DescribeTopicPartitionsRequestCursor +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsRequestCursor() DescribeTopicPartitionsRequestCursor { + var v DescribeTopicPartitionsRequestCursor + v.Default() + return v +} + +// DescribeTopicPartitionsRequest, introduced for KIP-966, allows you to fetch +// all details about a topic (particularly, eligible leader replicas). +// Note this contains more information in the response than Metadata does, +// so this request can also help avoid expensive full-cluster queries of +// topic information. +type DescribeTopicPartitionsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics to fetch details for. + Topics []DescribeTopicPartitionsRequestTopic + + // The maximum nuber of partitions included in the response. + // + // This field has a default of 200. + ResponsePartitionLimit int32 + + // If non-nil, cursor is the first topic and partition to fetch details for. + Cursor *DescribeTopicPartitionsRequestCursor + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeTopicPartitionsRequest) Key() int16 { return 75 } +func (*DescribeTopicPartitionsRequest) MaxVersion() int16 { return 0 } +func (v *DescribeTopicPartitionsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeTopicPartitionsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeTopicPartitionsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeTopicPartitionsRequest) ResponseKind() Response { + r := &DescribeTopicPartitionsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeTopicPartitionsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeTopicPartitionsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeTopicPartitionsResponse) + return resp, err +} + +func (v *DescribeTopicPartitionsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ResponsePartitionLimit + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Cursor + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeTopicPartitionsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeTopicPartitionsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeTopicPartitionsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeTopicPartitionsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int32() + s.ResponsePartitionLimit = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.Cursor = new(DescribeTopicPartitionsRequestCursor) + v := s.Cursor + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeTopicPartitionsRequest returns a pointer to a default DescribeTopicPartitionsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeTopicPartitionsRequest() *DescribeTopicPartitionsRequest { + var v DescribeTopicPartitionsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsRequest. +func (v *DescribeTopicPartitionsRequest) Default() { + v.ResponsePartitionLimit = 200 + { + v := &v.Cursor + _ = v + } +} + +// NewDescribeTopicPartitionsRequest returns a default DescribeTopicPartitionsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsRequest() DescribeTopicPartitionsRequest { + var v DescribeTopicPartitionsRequest + v.Default() + return v +} + +type DescribeTopicPartitionsResponseTopicPartition struct { + // The partition error, or 0 if there is no error. + ErrorCode int16 + + // The partition this is a response for. + Partition int32 + + // The ID of the leader broker for this partition. + LeaderID int32 + + // The leader epoch of this partition. + // + // This field has a default of -1. + LeaderEpoch int32 + + // The set of all nodes that host this partition. + Replicas []int32 + + // The set of all nodes that are in sync with the leader for this partition. + ISR []int32 + + // The eligible leader replicas for this partition. + EligibleLeaderReplicas []int32 + + // The last known eligible leader replicas. + LastKnownELR []int32 + + // The set of offline replicas for this partition. + OfflineReplicas []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsResponseTopicPartition. +func (v *DescribeTopicPartitionsResponseTopicPartition) Default() { + v.LeaderEpoch = -1 +} + +// NewDescribeTopicPartitionsResponseTopicPartition returns a default DescribeTopicPartitionsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsResponseTopicPartition() DescribeTopicPartitionsResponseTopicPartition { + var v DescribeTopicPartitionsResponseTopicPartition + v.Default() + return v +} + +type DescribeTopicPartitionsResponseTopic struct { + // The topic error, or 0 if there is no error. + ErrorCode int16 + + // The topic name. + Topic *string + + // The topic ID. + TopicID [16]byte + + // True if the topic is internal. + IsInternal bool + + Partitions []DescribeTopicPartitionsResponseTopicPartition + + // 32-bit bitfield representing authorized operations for this topic. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsResponseTopic. +func (v *DescribeTopicPartitionsResponseTopic) Default() { + v.AuthorizedOperations = -2147483648 +} + +// NewDescribeTopicPartitionsResponseTopic returns a default DescribeTopicPartitionsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsResponseTopic() DescribeTopicPartitionsResponseTopic { + var v DescribeTopicPartitionsResponseTopic + v.Default() + return v +} + +type DescribeTopicPartitionsResponseNextCursor struct { + // Topic is the topic to start with. + Topic string + + // Partition is the partition to start with. + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsResponseNextCursor. +func (v *DescribeTopicPartitionsResponseNextCursor) Default() { +} + +// NewDescribeTopicPartitionsResponseNextCursor returns a default DescribeTopicPartitionsResponseNextCursor +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsResponseNextCursor() DescribeTopicPartitionsResponseNextCursor { + var v DescribeTopicPartitionsResponseNextCursor + v.Default() + return v +} + +type DescribeTopicPartitionsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Each topic in the response. + Topics []DescribeTopicPartitionsResponseTopic + + // The cursor to use in the next request to iterate through topic/partition details. + NextCursor *DescribeTopicPartitionsResponseNextCursor + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeTopicPartitionsResponse) Key() int16 { return 75 } +func (*DescribeTopicPartitionsResponse) MaxVersion() int16 { return 0 } +func (v *DescribeTopicPartitionsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeTopicPartitionsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeTopicPartitionsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeTopicPartitionsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DescribeTopicPartitionsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeTopicPartitionsResponse) RequestKind() Request { + return &DescribeTopicPartitionsRequest{Version: v.Version} +} + +func (v *DescribeTopicPartitionsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.IsInternal + dst = kbin.AppendBool(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Replicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.ISR + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.EligibleLeaderReplicas + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.LastKnownELR + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + { + v := v.OfflineReplicas + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.NextCursor + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeTopicPartitionsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeTopicPartitionsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeTopicPartitionsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeTopicPartitionsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Topic = v + } + { + v := b.Uuid() + s.TopicID = v + } + { + v := b.Bool() + s.IsInternal = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeTopicPartitionsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := s.Replicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Replicas = v + } + { + v := s.ISR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.ISR = v + } + { + v := s.EligibleLeaderReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []int32{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.EligibleLeaderReplicas = v + } + { + v := s.LastKnownELR + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []int32{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.LastKnownELR = v + } + { + v := s.OfflineReplicas + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.OfflineReplicas = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.NextCursor = new(DescribeTopicPartitionsResponseNextCursor) + v := s.NextCursor + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeTopicPartitionsResponse returns a pointer to a default DescribeTopicPartitionsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeTopicPartitionsResponse() *DescribeTopicPartitionsResponse { + var v DescribeTopicPartitionsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeTopicPartitionsResponse. +func (v *DescribeTopicPartitionsResponse) Default() { + { + v := &v.NextCursor + _ = v + } +} + +// NewDescribeTopicPartitionsResponse returns a default DescribeTopicPartitionsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeTopicPartitionsResponse() DescribeTopicPartitionsResponse { + var v DescribeTopicPartitionsResponse + v.Default() + return v +} + +// ShareGroupHeartbeatRequest is a request for share groups. +type ShareGroupHeartbeatRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group ID. + GroupID string + + // MemberID is generated by the consumer. The member id must be kept during + // the entire lifetime of the consumer process. + MemberID string + + // MemberEpoch is the current member epoch; 0 to join the group; -1 to leave + // the group. + MemberEpoch int32 + + // RackID is the rack ID; null if not provided or unchanging since the last + // heartbeat. + RackID *string + + // SubscribedTopicNames are the subscribed topics; null if unchanging since + // the last heartbeat. + SubscribedTopicNames []string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareGroupHeartbeatRequest) Key() int16 { return 76 } +func (*ShareGroupHeartbeatRequest) MaxVersion() int16 { return 1 } +func (v *ShareGroupHeartbeatRequest) SetVersion(version int16) { v.Version = version } +func (v *ShareGroupHeartbeatRequest) GetVersion() int16 { return v.Version } +func (v *ShareGroupHeartbeatRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareGroupHeartbeatRequest) IsGroupCoordinatorRequest() {} +func (v *ShareGroupHeartbeatRequest) ResponseKind() Response { + r := &ShareGroupHeartbeatResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ShareGroupHeartbeatRequest) RequestWith(ctx context.Context, r Requestor) (*ShareGroupHeartbeatResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ShareGroupHeartbeatResponse) + return resp, err +} + +func (v *ShareGroupHeartbeatRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.SubscribedTopicNames + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareGroupHeartbeatRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareGroupHeartbeatRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareGroupHeartbeatRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + v := s.SubscribedTopicNames + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []string{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SubscribedTopicNames = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareGroupHeartbeatRequest returns a pointer to a default ShareGroupHeartbeatRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareGroupHeartbeatRequest() *ShareGroupHeartbeatRequest { + var v ShareGroupHeartbeatRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupHeartbeatRequest. +func (v *ShareGroupHeartbeatRequest) Default() { +} + +// NewShareGroupHeartbeatRequest returns a default ShareGroupHeartbeatRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupHeartbeatRequest() ShareGroupHeartbeatRequest { + var v ShareGroupHeartbeatRequest + v.Default() + return v +} + +type ShareGroupHeartbeatResponseAssignmentTopicPartition struct { + TopicID [16]byte + + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupHeartbeatResponseAssignmentTopicPartition. +func (v *ShareGroupHeartbeatResponseAssignmentTopicPartition) Default() { +} + +// NewShareGroupHeartbeatResponseAssignmentTopicPartition returns a default ShareGroupHeartbeatResponseAssignmentTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupHeartbeatResponseAssignmentTopicPartition() ShareGroupHeartbeatResponseAssignmentTopicPartition { + var v ShareGroupHeartbeatResponseAssignmentTopicPartition + v.Default() + return v +} + +type ShareGroupHeartbeatResponseAssignment struct { + // TopicPartitions contains the partitions assigned to the member. + TopicPartitions []ShareGroupHeartbeatResponseAssignmentTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupHeartbeatResponseAssignment. +func (v *ShareGroupHeartbeatResponseAssignment) Default() { +} + +// NewShareGroupHeartbeatResponseAssignment returns a default ShareGroupHeartbeatResponseAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupHeartbeatResponseAssignment() ShareGroupHeartbeatResponseAssignment { + var v ShareGroupHeartbeatResponseAssignment + v.Default() + return v +} + +// ShareGroupHeartbeatResponse is a response for a ShareGroupHeartbeatRequest. +// +// Version 0 was used for early access of KIP-932 in Apache Kafka 4.0 but +// removed in Apacke Kafka 4.1. +// +// Version 1 is the initial stable version (KIP-932). +type ShareGroupHeartbeatResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is any error for the response. + // + // Supported errors: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - TOPIC_AUTHORIZATION_FAILED (version 1+) + // - NOT_COORDINATOR (version 0+) + // - COORDINATOR_NOT_AVAILABLE (version 0+) + // - COORDINATOR_LOAD_IN_PROGRESS (version 0+) + // - UNKNOWN_MEMBER_ID (version 0+) + // - GROUP_MAX_SIZE_REACHED (version 0+) + // - INVALID_REQUEST (version 0+) + ErrorCode int16 + + // ErrorMessage is an optional message if there is an error. + ErrorMessage *string + + // MemberID is the ID generated by the consumer and provided by the consumer + // for all requests. + MemberID *string + + // MemberEpoch is the member epoch. + MemberEpoch int32 + + // HeartbeatIntervalMillis is the heartbeat interval in milliseconds. + HeartbeatIntervalMillis int32 + + // Assignment is the assignment, if provided. + Assignment *ShareGroupHeartbeatResponseAssignment + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareGroupHeartbeatResponse) Key() int16 { return 76 } +func (*ShareGroupHeartbeatResponse) MaxVersion() int16 { return 1 } +func (v *ShareGroupHeartbeatResponse) SetVersion(version int16) { v.Version = version } +func (v *ShareGroupHeartbeatResponse) GetVersion() int16 { return v.Version } +func (v *ShareGroupHeartbeatResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareGroupHeartbeatResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ShareGroupHeartbeatResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ShareGroupHeartbeatResponse) RequestKind() Request { + return &ShareGroupHeartbeatRequest{Version: v.Version} +} + +func (v *ShareGroupHeartbeatResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.HeartbeatIntervalMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Assignment + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.TopicPartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareGroupHeartbeatResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareGroupHeartbeatResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareGroupHeartbeatResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + v := b.Int32() + s.HeartbeatIntervalMillis = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.Assignment = new(ShareGroupHeartbeatResponseAssignment) + v := s.Assignment + v.Default() + s := v + { + v := s.TopicPartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareGroupHeartbeatResponseAssignmentTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicPartitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareGroupHeartbeatResponse returns a pointer to a default ShareGroupHeartbeatResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareGroupHeartbeatResponse() *ShareGroupHeartbeatResponse { + var v ShareGroupHeartbeatResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupHeartbeatResponse. +func (v *ShareGroupHeartbeatResponse) Default() { + { + v := &v.Assignment + _ = v + } +} + +// NewShareGroupHeartbeatResponse returns a default ShareGroupHeartbeatResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupHeartbeatResponse() ShareGroupHeartbeatResponse { + var v ShareGroupHeartbeatResponse + v.Default() + return v +} + +// ShareGroupDescribeRequest is a request to describe share groups. +// +// Version 0 was used for early access of KIP-932 in Apache Kafka 4.0 but +// removed in Apacke Kafka 4.1. +// +// Version 1 is the initial stable version (KIP-932). +type ShareGroupDescribeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupIDs are the IDs of the groups to describe. + GroupIDs []string + + // IncludeAuthorizedOperations is whether to include authorized operations. + IncludeAuthorizedOperations bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareGroupDescribeRequest) Key() int16 { return 77 } +func (*ShareGroupDescribeRequest) MaxVersion() int16 { return 1 } +func (v *ShareGroupDescribeRequest) SetVersion(version int16) { v.Version = version } +func (v *ShareGroupDescribeRequest) GetVersion() int16 { return v.Version } +func (v *ShareGroupDescribeRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareGroupDescribeRequest) IsGroupCoordinatorRequest() {} +func (v *ShareGroupDescribeRequest) ResponseKind() Response { + r := &ShareGroupDescribeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ShareGroupDescribeRequest) RequestWith(ctx context.Context, r Requestor) (*ShareGroupDescribeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ShareGroupDescribeResponse) + return resp, err +} + +func (v *ShareGroupDescribeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupIDs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.IncludeAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareGroupDescribeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareGroupDescribeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareGroupDescribeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.GroupIDs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.GroupIDs = v + } + { + v := b.Bool() + s.IncludeAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareGroupDescribeRequest returns a pointer to a default ShareGroupDescribeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareGroupDescribeRequest() *ShareGroupDescribeRequest { + var v ShareGroupDescribeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeRequest. +func (v *ShareGroupDescribeRequest) Default() { +} + +// NewShareGroupDescribeRequest returns a default ShareGroupDescribeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeRequest() ShareGroupDescribeRequest { + var v ShareGroupDescribeRequest + v.Default() + return v +} + +type ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition struct { + // TopicID is the topic ID. + TopicID [16]byte + + // Topic is the topic name. + Topic string + + // Partitions are the partitions. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition. +func (v *ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition) Default() { +} + +// NewShareGroupDescribeResponseGroupMemberAssignmentTopicPartition returns a default ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeResponseGroupMemberAssignmentTopicPartition() ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition { + var v ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition + v.Default() + return v +} + +type ShareGroupDescribeResponseGroupMemberAssignment struct { + // TopicPartitions are the assigned topic-partitions to the member. + TopicPartitions []ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeResponseGroupMemberAssignment. +func (v *ShareGroupDescribeResponseGroupMemberAssignment) Default() { +} + +// NewShareGroupDescribeResponseGroupMemberAssignment returns a default ShareGroupDescribeResponseGroupMemberAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeResponseGroupMemberAssignment() ShareGroupDescribeResponseGroupMemberAssignment { + var v ShareGroupDescribeResponseGroupMemberAssignment + v.Default() + return v +} + +type ShareGroupDescribeResponseGroupMember struct { + // MemberID is the member ID. + MemberID string + + // RackID is the member rack ID. + RackID *string + + // MemberEpoch is the current member epoch. + MemberEpoch int32 + + // ClientID is the client ID. + ClientID string + + // ClientHost is the client host. + ClientHost string + + // SubscribedTopicNames are the subscribed topic names. + SubscribedTopicNames []string + + // Assignment is the current assignment. + Assignment ShareGroupDescribeResponseGroupMemberAssignment + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeResponseGroupMember. +func (v *ShareGroupDescribeResponseGroupMember) Default() { + { + v := &v.Assignment + _ = v + } +} + +// NewShareGroupDescribeResponseGroupMember returns a default ShareGroupDescribeResponseGroupMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeResponseGroupMember() ShareGroupDescribeResponseGroupMember { + var v ShareGroupDescribeResponseGroupMember + v.Default() + return v +} + +type ShareGroupDescribeResponseGroup struct { + // ErrorCode is the describe error, or 0 if there was no error. + // + // Supported errors: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - TOPIC_AUTHORIZATION_FAILED (version 1+) + // - NOT_COORDINATOR (version 0+) + // - COORDINATOR_NOT_AVAILABLE (version 0+) + // - COORDINATOR_LOAD_IN_PROGRESS (version 0+) + // - INVALID_GROUP_ID (version 0+) + // - GROUP_ID_NOT_FOUND (version 0+) + // - INVALID_REQUEST (version 0+) + ErrorCode int16 + + // ErrorMessage is an optional message if there is an error. + ErrorMessage *string + + // GroupID is the group ID string. + GroupID string + + // GroupState is the group state string, or the empty string. + GroupState string + + // GroupEpoch is the group epoch. + GroupEpoch int32 + + // AssignmentEpoch is the assignment epoch. + AssignmentEpoch int32 + + // Assignor is the selected assignor. + Assignor string + + // Members are the members. + Members []ShareGroupDescribeResponseGroupMember + + // AuthorizedOperations is a 32-bit bitfield to represent authorized + // operations for this group. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeResponseGroup. +func (v *ShareGroupDescribeResponseGroup) Default() { + v.AuthorizedOperations = -2147483648 +} + +// NewShareGroupDescribeResponseGroup returns a default ShareGroupDescribeResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeResponseGroup() ShareGroupDescribeResponseGroup { + var v ShareGroupDescribeResponseGroup + v.Default() + return v +} + +// ShareGroupDescribeResponse is a response for a ShareGroupDescribeRequest. +type ShareGroupDescribeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Groups contains each described group. + Groups []ShareGroupDescribeResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareGroupDescribeResponse) Key() int16 { return 77 } +func (*ShareGroupDescribeResponse) MaxVersion() int16 { return 1 } +func (v *ShareGroupDescribeResponse) SetVersion(version int16) { v.Version = version } +func (v *ShareGroupDescribeResponse) GetVersion() int16 { return v.Version } +func (v *ShareGroupDescribeResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareGroupDescribeResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *ShareGroupDescribeResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ShareGroupDescribeResponse) RequestKind() Request { + return &ShareGroupDescribeRequest{Version: v.Version} +} + +func (v *ShareGroupDescribeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.GroupState + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.GroupEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AssignmentEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Assignor + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ClientID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ClientHost + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SubscribedTopicNames + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := &v.Assignment + { + v := v.TopicPartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareGroupDescribeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareGroupDescribeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareGroupDescribeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareGroupDescribeResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupState = v + } + { + v := b.Int32() + s.GroupEpoch = v + } + { + v := b.Int32() + s.AssignmentEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Assignor = v + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareGroupDescribeResponseGroupMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientHost = v + } + { + v := s.SubscribedTopicNames + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SubscribedTopicNames = v + } + { + v := &s.Assignment + v.Default() + s := v + { + v := s.TopicPartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareGroupDescribeResponseGroupMemberAssignmentTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TopicPartitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareGroupDescribeResponse returns a pointer to a default ShareGroupDescribeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareGroupDescribeResponse() *ShareGroupDescribeResponse { + var v ShareGroupDescribeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareGroupDescribeResponse. +func (v *ShareGroupDescribeResponse) Default() { +} + +// NewShareGroupDescribeResponse returns a default ShareGroupDescribeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareGroupDescribeResponse() ShareGroupDescribeResponse { + var v ShareGroupDescribeResponse + v.Default() + return v +} + +type ShareFetchRequestTopicPartitionAcknowledgementBatch struct { + // FirstOffset is the first offset of batch of records to acknowledge. + FirstOffset int64 + + // LastOffset is the last offset (inclusive) of batch of records to + // acknowledge. + LastOffset int64 + + // AcknowledgeTypes is an array of acknowledge types - + // 0:Gap,1:Accept,2:Release,3:Reject. + AcknowledgeTypes []int8 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchRequestTopicPartitionAcknowledgementBatch. +func (v *ShareFetchRequestTopicPartitionAcknowledgementBatch) Default() { +} + +// NewShareFetchRequestTopicPartitionAcknowledgementBatch returns a default ShareFetchRequestTopicPartitionAcknowledgementBatch +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchRequestTopicPartitionAcknowledgementBatch() ShareFetchRequestTopicPartitionAcknowledgementBatch { + var v ShareFetchRequestTopicPartitionAcknowledgementBatch + v.Default() + return v +} + +type ShareFetchRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // PartitionMaxBytes is the maximum bytes to fetch from this partition. + // 0 when only acknowledgement with no fetching is required. See KIP-74 + // for cases where this limit may not be honored. + PartitionMaxBytes int32 + + // AcknowledgementBatches are record batches to acknowledge. + AcknowledgementBatches []ShareFetchRequestTopicPartitionAcknowledgementBatch + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchRequestTopicPartition. +func (v *ShareFetchRequestTopicPartition) Default() { +} + +// NewShareFetchRequestTopicPartition returns a default ShareFetchRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchRequestTopicPartition() ShareFetchRequestTopicPartition { + var v ShareFetchRequestTopicPartition + v.Default() + return v +} + +type ShareFetchRequestTopic struct { + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the partitions to fetch. + Partitions []ShareFetchRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchRequestTopic. +func (v *ShareFetchRequestTopic) Default() { +} + +// NewShareFetchRequestTopic returns a default ShareFetchRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchRequestTopic() ShareFetchRequestTopic { + var v ShareFetchRequestTopic + v.Default() + return v +} + +type ShareFetchRequestForgottenTopicsData struct { + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the partitions indexes to forget. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchRequestForgottenTopicsData. +func (v *ShareFetchRequestForgottenTopicsData) Default() { +} + +// NewShareFetchRequestForgottenTopicsData returns a default ShareFetchRequestForgottenTopicsData +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchRequestForgottenTopicsData() ShareFetchRequestForgottenTopicsData { + var v ShareFetchRequestForgottenTopicsData + v.Default() + return v +} + +// ShareFetchRequest is a request to fetch records from share groups. +// +// Version 0 was used for early access of KIP-932 in Apache Kafka 4.0 but +// removed in Apacke Kafka 4.1. +// +// Version 1 is the initial stable version (KIP-932). +// +// Version 2, introduced in Kafka 4.2, adds ShareAcquireMode (KIP-1206) and +// IsRenewAck for renew acknowledgements (KIP-1222). +type ShareFetchRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID *string + + // MemberID is the member ID. + MemberID *string + + // ShareSessionEpoch is the current share session epoch: 0 to open a share + // session; -1 to close it; otherwise increments for consecutive requests. + ShareSessionEpoch int32 + + // MaxWaitMillis is the maximum time in milliseconds to wait for the + // response. + MaxWaitMillis int32 + + // MinBytes is the minimum bytes to accumulate in the response. + MinBytes int32 + + // MaxBytes is the maximum bytes to fetch. See KIP-74 for cases where this + // limit may not be honored. + // + // This field has a default of 0x7fffffff. + MaxBytes int32 + + // MaxRecords is the maximum number of records to fetch. This limit can be + // exceeded for alignment of batch boundaries. + MaxRecords int32 // v1+ + + // BatchSize is the optimal number of records for batches of acquired + // records and acknowledgements. + BatchSize int32 // v1+ + + // ShareAcquireMode controls the fetch behavior: + // 0 = batch-optimized, 1 = record-limit. + ShareAcquireMode int8 // v2+ + + // IsRenewAck indicates whether Renew type acknowledgements are present + // in AcknowledgementBatches. + IsRenewAck bool // v2+ + + // Topics are the topics to fetch. + Topics []ShareFetchRequestTopic + + // ForgottenTopicsData are the partitions to remove from this share session. + ForgottenTopicsData []ShareFetchRequestForgottenTopicsData + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareFetchRequest) Key() int16 { return 78 } +func (*ShareFetchRequest) MaxVersion() int16 { return 2 } +func (v *ShareFetchRequest) SetVersion(version int16) { v.Version = version } +func (v *ShareFetchRequest) GetVersion() int16 { return v.Version } +func (v *ShareFetchRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareFetchRequest) ResponseKind() Response { + r := &ShareFetchResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ShareFetchRequest) RequestWith(ctx context.Context, r Requestor) (*ShareFetchResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ShareFetchResponse) + return resp, err +} + +func (v *ShareFetchRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ShareSessionEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MaxWaitMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MinBytes + dst = kbin.AppendInt32(dst, v) + } + { + v := v.MaxBytes + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.MaxRecords + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.BatchSize + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.ShareAcquireMode + dst = kbin.AppendInt8(dst, v) + } + if version >= 2 { + v := v.IsRenewAck + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if version >= 0 && version <= 0 { + v := v.PartitionMaxBytes + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AcknowledgementBatches + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.AcknowledgeTypes + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ForgottenTopicsData + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareFetchRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareFetchRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareFetchRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.GroupID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.ShareSessionEpoch = v + } + { + v := b.Int32() + s.MaxWaitMillis = v + } + { + v := b.Int32() + s.MinBytes = v + } + { + v := b.Int32() + s.MaxBytes = v + } + if version >= 1 { + v := b.Int32() + s.MaxRecords = v + } + if version >= 1 { + v := b.Int32() + s.BatchSize = v + } + if version >= 2 { + v := b.Int8() + s.ShareAcquireMode = v + } + if version >= 2 { + v := b.Bool() + s.IsRenewAck = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if version >= 0 && version <= 0 { + v := b.Int32() + s.PartitionMaxBytes = v + } + { + v := s.AcknowledgementBatches + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchRequestTopicPartitionAcknowledgementBatch, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int64() + s.LastOffset = v + } + { + v := s.AcknowledgeTypes + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int8, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int8() + a[i] = v + } + v = a + s.AcknowledgeTypes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.AcknowledgementBatches = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := s.ForgottenTopicsData + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchRequestForgottenTopicsData, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ForgottenTopicsData = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareFetchRequest returns a pointer to a default ShareFetchRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareFetchRequest() *ShareFetchRequest { + var v ShareFetchRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchRequest. +func (v *ShareFetchRequest) Default() { + v.MaxBytes = 2147483647 +} + +// NewShareFetchRequest returns a default ShareFetchRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchRequest() ShareFetchRequest { + var v ShareFetchRequest + v.Default() + return v +} + +type ShareFetchResponseTopicPartitionCurrentLeader struct { + // LeaderID is the ID of the current leader or -1 if the leader is + // unknown. + LeaderID int32 + + // LeaderEpoch is the latest known leader epoch. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponseTopicPartitionCurrentLeader. +func (v *ShareFetchResponseTopicPartitionCurrentLeader) Default() { +} + +// NewShareFetchResponseTopicPartitionCurrentLeader returns a default ShareFetchResponseTopicPartitionCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponseTopicPartitionCurrentLeader() ShareFetchResponseTopicPartitionCurrentLeader { + var v ShareFetchResponseTopicPartitionCurrentLeader + v.Default() + return v +} + +type ShareFetchResponseTopicPartitionAcquiredRecord struct { + // FirstOffset is the earliest offset in this batch of acquired + // records. + FirstOffset int64 + + // LastOffset is the last offset of this batch of acquired records. + LastOffset int64 + + // DeliveryCount is the delivery count of this batch of acquired + // records. + DeliveryCount int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponseTopicPartitionAcquiredRecord. +func (v *ShareFetchResponseTopicPartitionAcquiredRecord) Default() { +} + +// NewShareFetchResponseTopicPartitionAcquiredRecord returns a default ShareFetchResponseTopicPartitionAcquiredRecord +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponseTopicPartitionAcquiredRecord() ShareFetchResponseTopicPartitionAcquiredRecord { + var v ShareFetchResponseTopicPartitionAcquiredRecord + v.Default() + return v +} + +type ShareFetchResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the fetch error code, or 0 if there was no fetch error. + ErrorCode int16 + + // ErrorMessage is the fetch error message, or null if there was no + // fetch error. + ErrorMessage *string + + // AcknowledgeErrorCode is the acknowledge error code, or 0 if there was + // no acknowledge error. + AcknowledgeErrorCode int16 + + // AcknowledgeErrorMessage is the acknowledge error message, or null if + // there was no acknowledge error. + AcknowledgeErrorMessage *string + + // CurrentLeader is the current leader of the partition. + CurrentLeader ShareFetchResponseTopicPartitionCurrentLeader + + // Records is the record data. + Records []byte + + // AcquiredRecords are the acquired records. + AcquiredRecords []ShareFetchResponseTopicPartitionAcquiredRecord + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponseTopicPartition. +func (v *ShareFetchResponseTopicPartition) Default() { + { + v := &v.CurrentLeader + _ = v + } +} + +// NewShareFetchResponseTopicPartition returns a default ShareFetchResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponseTopicPartition() ShareFetchResponseTopicPartition { + var v ShareFetchResponseTopicPartition + v.Default() + return v +} + +type ShareFetchResponseTopic struct { + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the topic partitions. + Partitions []ShareFetchResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponseTopic. +func (v *ShareFetchResponseTopic) Default() { +} + +// NewShareFetchResponseTopic returns a default ShareFetchResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponseTopic() ShareFetchResponseTopic { + var v ShareFetchResponseTopic + v.Default() + return v +} + +type ShareFetchResponseNodeEndpoint struct { + // NodeID is the ID of the associated node. + NodeID int32 + + // Host is the node's hostname. + Host string + + // Port is the node's port. + Port int32 + + // Rack is the rack of the node, or null if it has not been assigned to a + // rack. + Rack *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponseNodeEndpoint. +func (v *ShareFetchResponseNodeEndpoint) Default() { +} + +// NewShareFetchResponseNodeEndpoint returns a default ShareFetchResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponseNodeEndpoint() ShareFetchResponseNodeEndpoint { + var v ShareFetchResponseNodeEndpoint + v.Default() + return v +} + +// ShareFetchResponse is a response for a ShareFetchRequest. +type ShareFetchResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the top-level response error code. + // + // Supported errors for ErrorCode and AcknowledgeErrorCode: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - TOPIC_AUTHORIZATION_FAILED (version 0+) + // - SHARE_SESSION_NOT_FOUND (version 0+) + // - INVALID_SHARE_SESSION_EPOCH (version 0+) + // - UNKNOWN_TOPIC_OR_PARTITION (version 0+) + // - NOT_LEADER_OR_FOLLOWER (version 0+) + // - UNKNOWN_TOPIC_ID (version 0+) + // - INVALID_RECORD_STATE (version 0+) - only for AcknowledgeErrorCode + // - KAFKA_STORAGE_ERROR (version 0+) + // - CORRUPT_MESSAGE (version 0+) + // - INVALID_REQUEST (version 0+) + // - UNKNOWN_SERVER_ERROR (version 0+) + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // AcquisitionLockTimeoutMillis is the time in milliseconds for which the + // acquired records are locked. + AcquisitionLockTimeoutMillis int32 // v1+ + + // Topics are the response topics. + Topics []ShareFetchResponseTopic + + // NodeEndpoints are endpoints for all current leaders enumerated in + // PartitionData with error NOT_LEADER_OR_FOLLOWER. + NodeEndpoints []ShareFetchResponseNodeEndpoint + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareFetchResponse) Key() int16 { return 78 } +func (*ShareFetchResponse) MaxVersion() int16 { return 2 } +func (v *ShareFetchResponse) SetVersion(version int16) { v.Version = version } +func (v *ShareFetchResponse) GetVersion() int16 { return v.Version } +func (v *ShareFetchResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareFetchResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *ShareFetchResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *ShareFetchResponse) RequestKind() Request { return &ShareFetchRequest{Version: v.Version} } + +func (v *ShareFetchResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 1 { + v := v.AcquisitionLockTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.AcknowledgeErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.AcknowledgeErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := &v.CurrentLeader + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := v.Records + if isFlexible { + dst = kbin.AppendCompactNullableBytes(dst, v) + } else { + dst = kbin.AppendNullableBytes(dst, v) + } + } + { + v := v.AcquiredRecords + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.DeliveryCount + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.NodeEndpoints + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareFetchResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareFetchResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareFetchResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 1 { + v := b.Int32() + s.AcquisitionLockTimeoutMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := b.Int16() + s.AcknowledgeErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.AcknowledgeErrorMessage = v + } + { + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + var v []byte + if isFlexible { + v = b.CompactNullableBytes() + } else { + v = b.NullableBytes() + } + s.Records = v + } + { + v := s.AcquiredRecords + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchResponseTopicPartitionAcquiredRecord, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int64() + s.LastOffset = v + } + { + v := b.Int16() + s.DeliveryCount = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.AcquiredRecords = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareFetchResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareFetchResponse returns a pointer to a default ShareFetchResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareFetchResponse() *ShareFetchResponse { + var v ShareFetchResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareFetchResponse. +func (v *ShareFetchResponse) Default() { +} + +// NewShareFetchResponse returns a default ShareFetchResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareFetchResponse() ShareFetchResponse { + var v ShareFetchResponse + v.Default() + return v +} + +type ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch struct { + // FirstOffset is the first offset of batch of records to acknowledge. + FirstOffset int64 + + // LastOffset is the last offset (inclusive) of batch of records to + // acknowledge. + LastOffset int64 + + // AcknowledgeTypes is an array of acknowledge types - + // 0:Gap,1:Accept,2:Release,3:Reject. + AcknowledgeTypes []int8 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch. +func (v *ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch) Default() { +} + +// NewShareAcknowledgeRequestTopicPartitionAcknowledgementBatch returns a default ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeRequestTopicPartitionAcknowledgementBatch() ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch { + var v ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch + v.Default() + return v +} + +type ShareAcknowledgeRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // AcknowledgementBatches are record batches to acknowledge. + AcknowledgementBatches []ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeRequestTopicPartition. +func (v *ShareAcknowledgeRequestTopicPartition) Default() { +} + +// NewShareAcknowledgeRequestTopicPartition returns a default ShareAcknowledgeRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeRequestTopicPartition() ShareAcknowledgeRequestTopicPartition { + var v ShareAcknowledgeRequestTopicPartition + v.Default() + return v +} + +type ShareAcknowledgeRequestTopic struct { + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the partitions containing records to acknowledge. + Partitions []ShareAcknowledgeRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeRequestTopic. +func (v *ShareAcknowledgeRequestTopic) Default() { +} + +// NewShareAcknowledgeRequestTopic returns a default ShareAcknowledgeRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeRequestTopic() ShareAcknowledgeRequestTopic { + var v ShareAcknowledgeRequestTopic + v.Default() + return v +} + +// ShareAcknowledgeRequest is a request to acknowledge records in share groups. +// +// Version 0 was used for early access of KIP-932 in Apache Kafka 4.0 but +// removed in Apacke Kafka 4.1. +// +// Version 1 is the initial stable version (KIP-932). +// +// Version 2, introduced in Kafka 4.2, adds IsRenewAck (KIP-1222) and +// AcquisitionLockTimeoutMillis in the response. +type ShareAcknowledgeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID *string + + // MemberID is the member ID. + MemberID *string + + // ShareSessionEpoch is the current share session epoch: 0 to open a share + // session; -1 to close it; otherwise increments for consecutive requests. + ShareSessionEpoch int32 + + // IsRenewAck indicates whether Renew type acknowledgements are present + // in AcknowledgementBatches. + IsRenewAck bool // v2+ + + // Topics are the topics containing records to acknowledge. + Topics []ShareAcknowledgeRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareAcknowledgeRequest) Key() int16 { return 79 } +func (*ShareAcknowledgeRequest) MaxVersion() int16 { return 2 } +func (v *ShareAcknowledgeRequest) SetVersion(version int16) { v.Version = version } +func (v *ShareAcknowledgeRequest) GetVersion() int16 { return v.Version } +func (v *ShareAcknowledgeRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareAcknowledgeRequest) ResponseKind() Response { + r := &ShareAcknowledgeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ShareAcknowledgeRequest) RequestWith(ctx context.Context, r Requestor) (*ShareAcknowledgeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ShareAcknowledgeResponse) + return resp, err +} + +func (v *ShareAcknowledgeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ShareSessionEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 2 { + v := v.IsRenewAck + dst = kbin.AppendBool(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AcknowledgementBatches + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.AcknowledgeTypes + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt8(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareAcknowledgeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareAcknowledgeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareAcknowledgeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.GroupID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.ShareSessionEpoch = v + } + if version >= 2 { + v := b.Bool() + s.IsRenewAck = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := s.AcknowledgementBatches + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeRequestTopicPartitionAcknowledgementBatch, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int64() + s.LastOffset = v + } + { + v := s.AcknowledgeTypes + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int8, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int8() + a[i] = v + } + v = a + s.AcknowledgeTypes = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.AcknowledgementBatches = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareAcknowledgeRequest returns a pointer to a default ShareAcknowledgeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareAcknowledgeRequest() *ShareAcknowledgeRequest { + var v ShareAcknowledgeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeRequest. +func (v *ShareAcknowledgeRequest) Default() { +} + +// NewShareAcknowledgeRequest returns a default ShareAcknowledgeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeRequest() ShareAcknowledgeRequest { + var v ShareAcknowledgeRequest + v.Default() + return v +} + +type ShareAcknowledgeResponseTopicPartitionCurrentLeader struct { + // LeaderID is the ID of the current leader or -1 if the leader is + // unknown. + LeaderID int32 + + // LeaderEpoch is the latest known leader epoch. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeResponseTopicPartitionCurrentLeader. +func (v *ShareAcknowledgeResponseTopicPartitionCurrentLeader) Default() { +} + +// NewShareAcknowledgeResponseTopicPartitionCurrentLeader returns a default ShareAcknowledgeResponseTopicPartitionCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeResponseTopicPartitionCurrentLeader() ShareAcknowledgeResponseTopicPartitionCurrentLeader { + var v ShareAcknowledgeResponseTopicPartitionCurrentLeader + v.Default() + return v +} + +type ShareAcknowledgeResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // CurrentLeader is the current leader of the partition. + CurrentLeader ShareAcknowledgeResponseTopicPartitionCurrentLeader + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeResponseTopicPartition. +func (v *ShareAcknowledgeResponseTopicPartition) Default() { + { + v := &v.CurrentLeader + _ = v + } +} + +// NewShareAcknowledgeResponseTopicPartition returns a default ShareAcknowledgeResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeResponseTopicPartition() ShareAcknowledgeResponseTopicPartition { + var v ShareAcknowledgeResponseTopicPartition + v.Default() + return v +} + +type ShareAcknowledgeResponseTopic struct { + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the topic partitions. + Partitions []ShareAcknowledgeResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeResponseTopic. +func (v *ShareAcknowledgeResponseTopic) Default() { +} + +// NewShareAcknowledgeResponseTopic returns a default ShareAcknowledgeResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeResponseTopic() ShareAcknowledgeResponseTopic { + var v ShareAcknowledgeResponseTopic + v.Default() + return v +} + +type ShareAcknowledgeResponseNodeEndpoint struct { + // NodeID is the ID of the associated node. + NodeID int32 + + // Host is the node's hostname. + Host string + + // Port is the node's port. + Port int32 + + // Rack is the rack of the node, or null if it has not been assigned to a + // rack. + Rack *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeResponseNodeEndpoint. +func (v *ShareAcknowledgeResponseNodeEndpoint) Default() { +} + +// NewShareAcknowledgeResponseNodeEndpoint returns a default ShareAcknowledgeResponseNodeEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeResponseNodeEndpoint() ShareAcknowledgeResponseNodeEndpoint { + var v ShareAcknowledgeResponseNodeEndpoint + v.Default() + return v +} + +// ShareAcknowledgeResponse is a response for a ShareAcknowledgeRequest. +type ShareAcknowledgeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the top level response error code. + // + // Supported errors: + // - GROUP_AUTHORIZATION_FAILED (version 0+) + // - TOPIC_AUTHORIZATION_FAILED (version 0+) + // - UNKNOWN_TOPIC_OR_PARTITION (version 0+) + // - SHARE_SESSION_NOT_FOUND (version 0+) + // - INVALID_SHARE_SESSION_EPOCH (version 0+) + // - NOT_LEADER_OR_FOLLOWER (version 0+) + // - UNKNOWN_TOPIC_ID (version 0+) + // - INVALID_RECORD_STATE (version 0+) + // - KAFKA_STORAGE_ERROR (version 0+) + // - INVALID_REQUEST (version 0+) + // - UNKNOWN_SERVER_ERROR (version 0+) + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // AcquisitionLockTimeoutMillis is the time in milliseconds for which the + // acquired records are locked. + AcquisitionLockTimeoutMillis int32 // v2+ + + // Topics are the response topics. + Topics []ShareAcknowledgeResponseTopic + + // NodeEndpoints are endpoints for all current leaders enumerated in + // PartitionData with error NOT_LEADER_OR_FOLLOWER. + NodeEndpoints []ShareAcknowledgeResponseNodeEndpoint + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ShareAcknowledgeResponse) Key() int16 { return 79 } +func (*ShareAcknowledgeResponse) MaxVersion() int16 { return 2 } +func (v *ShareAcknowledgeResponse) SetVersion(version int16) { v.Version = version } +func (v *ShareAcknowledgeResponse) GetVersion() int16 { return v.Version } +func (v *ShareAcknowledgeResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ShareAcknowledgeResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *ShareAcknowledgeResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *ShareAcknowledgeResponse) RequestKind() Request { + return &ShareAcknowledgeRequest{Version: v.Version} +} + +func (v *ShareAcknowledgeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if version >= 2 { + v := v.AcquisitionLockTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := &v.CurrentLeader + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.NodeEndpoints + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.NodeID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Rack + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ShareAcknowledgeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ShareAcknowledgeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ShareAcknowledgeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if version >= 2 { + v := b.Int32() + s.AcquisitionLockTimeoutMillis = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := s.NodeEndpoints + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ShareAcknowledgeResponseNodeEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.NodeID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.Rack = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.NodeEndpoints = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrShareAcknowledgeResponse returns a pointer to a default ShareAcknowledgeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrShareAcknowledgeResponse() *ShareAcknowledgeResponse { + var v ShareAcknowledgeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ShareAcknowledgeResponse. +func (v *ShareAcknowledgeResponse) Default() { +} + +// NewShareAcknowledgeResponse returns a default ShareAcknowledgeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewShareAcknowledgeResponse() ShareAcknowledgeResponse { + var v ShareAcknowledgeResponse + v.Default() + return v +} + +type AddRaftVoterRequestListener struct { + // The name of the endpoint. + Name string + + // The hostname. + Host string + + // The port. + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddRaftVoterRequestListener. +func (v *AddRaftVoterRequestListener) Default() { +} + +// NewAddRaftVoterRequestListener returns a default AddRaftVoterRequestListener +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddRaftVoterRequestListener() AddRaftVoterRequestListener { + var v AddRaftVoterRequestListener + v.Default() + return v +} + +// AddRaftVoter, added for KIP-853, allows you to manage your KRaft +// controllers. +// Version 1, introduced in Kafka 4.2, adds AckWhenCommitted (KIP-1186). +type AddRaftVoterRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The cluster ID of the request. + ClusterID *string + + // TimeoutMillis is how long Kafka can wait before responding to this request. + // This field has no effect on Kafka's processing of the request; the request + // will continue to be processed if the timeout is reached. If the timeout is + // reached, Kafka will reply with a REQUEST_TIMED_OUT error. + // + // This field has a default of 15000. + TimeoutMillis int32 + + // The replica ID of the voter getting added to the topic partition. + VoterID int32 + + // The directory ID of the voter getting added to the topic partition. + VoterDirectoryID [16]byte + + // The endpoints that can be used to communicate with the voter. + Listeners []AddRaftVoterRequestListener + + // When true, return a response after the new voter set is committed. + // Otherwise, return after the leader writes the changes locally. + // + // This field has a default of true. + AckWhenCommitted bool // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AddRaftVoterRequest) Key() int16 { return 80 } +func (*AddRaftVoterRequest) MaxVersion() int16 { return 1 } +func (v *AddRaftVoterRequest) SetVersion(version int16) { v.Version = version } +func (v *AddRaftVoterRequest) GetVersion() int16 { return v.Version } +func (v *AddRaftVoterRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AddRaftVoterRequest) Timeout() int32 { return v.TimeoutMillis } +func (v *AddRaftVoterRequest) SetTimeout(timeoutMillis int32) { v.TimeoutMillis = timeoutMillis } +func (v *AddRaftVoterRequest) ResponseKind() Response { + r := &AddRaftVoterResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AddRaftVoterRequest) RequestWith(ctx context.Context, r Requestor) (*AddRaftVoterResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AddRaftVoterResponse) + return resp, err +} + +func (v *AddRaftVoterRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.TimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Listeners + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if version >= 1 { + v := v.AckWhenCommitted + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddRaftVoterRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddRaftVoterRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddRaftVoterRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + { + v := b.Int32() + s.TimeoutMillis = v + } + { + v := b.Int32() + s.VoterID = v + } + { + v := b.Uuid() + s.VoterDirectoryID = v + } + { + v := s.Listeners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AddRaftVoterRequestListener, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Listeners = v + } + if version >= 1 { + v := b.Bool() + s.AckWhenCommitted = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddRaftVoterRequest returns a pointer to a default AddRaftVoterRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddRaftVoterRequest() *AddRaftVoterRequest { + var v AddRaftVoterRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddRaftVoterRequest. +func (v *AddRaftVoterRequest) Default() { + v.TimeoutMillis = 15000 + v.AckWhenCommitted = true +} + +// NewAddRaftVoterRequest returns a default AddRaftVoterRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddRaftVoterRequest() AddRaftVoterRequest { + var v AddRaftVoterRequest + v.Default() + return v +} + +type AddRaftVoterResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AddRaftVoterResponse) Key() int16 { return 80 } +func (*AddRaftVoterResponse) MaxVersion() int16 { return 1 } +func (v *AddRaftVoterResponse) SetVersion(version int16) { v.Version = version } +func (v *AddRaftVoterResponse) GetVersion() int16 { return v.Version } +func (v *AddRaftVoterResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AddRaftVoterResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *AddRaftVoterResponse) SetThrottle(throttleMillis int32) { v.ThrottleMillis = throttleMillis } +func (v *AddRaftVoterResponse) RequestKind() Request { return &AddRaftVoterRequest{Version: v.Version} } + +func (v *AddRaftVoterResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AddRaftVoterResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AddRaftVoterResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AddRaftVoterResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAddRaftVoterResponse returns a pointer to a default AddRaftVoterResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAddRaftVoterResponse() *AddRaftVoterResponse { + var v AddRaftVoterResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AddRaftVoterResponse. +func (v *AddRaftVoterResponse) Default() { +} + +// NewAddRaftVoterResponse returns a default AddRaftVoterResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAddRaftVoterResponse() AddRaftVoterResponse { + var v AddRaftVoterResponse + v.Default() + return v +} + +// RemoveRaftVoter, added for KIP-853, allows you to manage your KRaft +// controllers. +type RemoveRaftVoterRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The cluster ID of the request. + ClusterID *string + + // The replica ID of the voter getting added to the topic partition. + VoterID int32 + + // The directory ID of the voter getting added to the topic partition. + VoterDirectoryID [16]byte + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*RemoveRaftVoterRequest) Key() int16 { return 81 } +func (*RemoveRaftVoterRequest) MaxVersion() int16 { return 0 } +func (v *RemoveRaftVoterRequest) SetVersion(version int16) { v.Version = version } +func (v *RemoveRaftVoterRequest) GetVersion() int16 { return v.Version } +func (v *RemoveRaftVoterRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *RemoveRaftVoterRequest) ResponseKind() Response { + r := &RemoveRaftVoterResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *RemoveRaftVoterRequest) RequestWith(ctx context.Context, r Requestor) (*RemoveRaftVoterResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*RemoveRaftVoterResponse) + return resp, err +} + +func (v *RemoveRaftVoterRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *RemoveRaftVoterRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *RemoveRaftVoterRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *RemoveRaftVoterRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + { + v := b.Int32() + s.VoterID = v + } + { + v := b.Uuid() + s.VoterDirectoryID = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrRemoveRaftVoterRequest returns a pointer to a default RemoveRaftVoterRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrRemoveRaftVoterRequest() *RemoveRaftVoterRequest { + var v RemoveRaftVoterRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to RemoveRaftVoterRequest. +func (v *RemoveRaftVoterRequest) Default() { +} + +// NewRemoveRaftVoterRequest returns a default RemoveRaftVoterRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewRemoveRaftVoterRequest() RemoveRaftVoterRequest { + var v RemoveRaftVoterRequest + v.Default() + return v +} + +type RemoveRaftVoterResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*RemoveRaftVoterResponse) Key() int16 { return 81 } +func (*RemoveRaftVoterResponse) MaxVersion() int16 { return 0 } +func (v *RemoveRaftVoterResponse) SetVersion(version int16) { v.Version = version } +func (v *RemoveRaftVoterResponse) GetVersion() int16 { return v.Version } +func (v *RemoveRaftVoterResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *RemoveRaftVoterResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *RemoveRaftVoterResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *RemoveRaftVoterResponse) RequestKind() Request { + return &RemoveRaftVoterRequest{Version: v.Version} +} + +func (v *RemoveRaftVoterResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *RemoveRaftVoterResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *RemoveRaftVoterResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *RemoveRaftVoterResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrRemoveRaftVoterResponse returns a pointer to a default RemoveRaftVoterResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrRemoveRaftVoterResponse() *RemoveRaftVoterResponse { + var v RemoveRaftVoterResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to RemoveRaftVoterResponse. +func (v *RemoveRaftVoterResponse) Default() { +} + +// NewRemoveRaftVoterResponse returns a default RemoveRaftVoterResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewRemoveRaftVoterResponse() RemoveRaftVoterResponse { + var v RemoveRaftVoterResponse + v.Default() + return v +} + +type UpdateRaftVoterRequestListener struct { + // The name of the endpoint. + Name string + + // The hostname. + Host string + + // The port. + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateRaftVoterRequestListener. +func (v *UpdateRaftVoterRequestListener) Default() { +} + +// NewUpdateRaftVoterRequestListener returns a default UpdateRaftVoterRequestListener +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateRaftVoterRequestListener() UpdateRaftVoterRequestListener { + var v UpdateRaftVoterRequestListener + v.Default() + return v +} + +type UpdateRaftVoterRequestKRaftVersionFeature struct { + // The min supported KRaft protocol version. + MinSupportedVersion int16 + + // The max supported KRaft protocol version. + MaxSupportedVersion int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateRaftVoterRequestKRaftVersionFeature. +func (v *UpdateRaftVoterRequestKRaftVersionFeature) Default() { +} + +// NewUpdateRaftVoterRequestKRaftVersionFeature returns a default UpdateRaftVoterRequestKRaftVersionFeature +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateRaftVoterRequestKRaftVersionFeature() UpdateRaftVoterRequestKRaftVersionFeature { + var v UpdateRaftVoterRequestKRaftVersionFeature + v.Default() + return v +} + +// UpdateRaftVoterRequest, added for KIP-853, allows you to manage your KRaft +// controllers. +type UpdateRaftVoterRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // The cluster ID of the request. + ClusterID *string + + // The current leader epoch of the partition; -1 if unknown. + CurrentLeaderEpoch int32 + + // The replica ID of the voter getting added to the topic partition. + VoterID int32 + + // The directory ID of the voter getting added to the topic partition. + VoterDirectoryID [16]byte + + // The endpoints that can be used to communicate with the leader. + Listeners []UpdateRaftVoterRequestListener + + // The range of versions of the protocol that the replica supports. + KRaftVersionFeature UpdateRaftVoterRequestKRaftVersionFeature + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UpdateRaftVoterRequest) Key() int16 { return 82 } +func (*UpdateRaftVoterRequest) MaxVersion() int16 { return 0 } +func (v *UpdateRaftVoterRequest) SetVersion(version int16) { v.Version = version } +func (v *UpdateRaftVoterRequest) GetVersion() int16 { return v.Version } +func (v *UpdateRaftVoterRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *UpdateRaftVoterRequest) ResponseKind() Response { + r := &UpdateRaftVoterResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *UpdateRaftVoterRequest) RequestWith(ctx context.Context, r Requestor) (*UpdateRaftVoterResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*UpdateRaftVoterResponse) + return resp, err +} + +func (v *UpdateRaftVoterRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ClusterID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.CurrentLeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoterID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.VoterDirectoryID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Listeners + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Name + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := &v.KRaftVersionFeature + { + v := v.MinSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + { + v := v.MaxSupportedVersion + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateRaftVoterRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateRaftVoterRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateRaftVoterRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ClusterID = v + } + { + v := b.Int32() + s.CurrentLeaderEpoch = v + } + { + v := b.Int32() + s.VoterID = v + } + { + v := b.Uuid() + s.VoterDirectoryID = v + } + { + v := s.Listeners + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]UpdateRaftVoterRequestListener, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Name = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Listeners = v + } + { + v := &s.KRaftVersionFeature + v.Default() + s := v + { + v := b.Int16() + s.MinSupportedVersion = v + } + { + v := b.Int16() + s.MaxSupportedVersion = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrUpdateRaftVoterRequest returns a pointer to a default UpdateRaftVoterRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateRaftVoterRequest() *UpdateRaftVoterRequest { + var v UpdateRaftVoterRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateRaftVoterRequest. +func (v *UpdateRaftVoterRequest) Default() { + { + v := &v.KRaftVersionFeature + _ = v + } +} + +// NewUpdateRaftVoterRequest returns a default UpdateRaftVoterRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateRaftVoterRequest() UpdateRaftVoterRequest { + var v UpdateRaftVoterRequest + v.Default() + return v +} + +type UpdateRaftVoterResponseCurrentLeader struct { + // The replica ID of the current leader, or -1 if unknown. + // + // This field has a default of -1. + LeaderID int32 + + // The latest known leader epoch. + // + // This field has a default of -1. + LeaderEpoch int32 + + // The node's hostname. + Host string + + // The node's port. + Port int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateRaftVoterResponseCurrentLeader. +func (v *UpdateRaftVoterResponseCurrentLeader) Default() { + v.LeaderID = -1 + v.LeaderEpoch = -1 +} + +// NewUpdateRaftVoterResponseCurrentLeader returns a default UpdateRaftVoterResponseCurrentLeader +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateRaftVoterResponseCurrentLeader() UpdateRaftVoterResponseCurrentLeader { + var v UpdateRaftVoterResponseCurrentLeader + v.Default() + return v +} + +type UpdateRaftVoterResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + ErrorCode int16 + + // Defaults of the current Raft leader. + CurrentLeader UpdateRaftVoterResponseCurrentLeader // tag 0 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*UpdateRaftVoterResponse) Key() int16 { return 82 } +func (*UpdateRaftVoterResponse) MaxVersion() int16 { return 0 } +func (v *UpdateRaftVoterResponse) SetVersion(version int16) { v.Version = version } +func (v *UpdateRaftVoterResponse) GetVersion() int16 { return v.Version } +func (v *UpdateRaftVoterResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *UpdateRaftVoterResponse) Throttle() (int32, bool) { return v.ThrottleMillis, v.Version >= 0 } +func (v *UpdateRaftVoterResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *UpdateRaftVoterResponse) RequestKind() Request { + return &UpdateRaftVoterRequest{Version: v.Version} +} + +func (v *UpdateRaftVoterResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + var toEncode []uint32 + if !reflect.DeepEqual(v.CurrentLeader, (func() UpdateRaftVoterResponseCurrentLeader { + var v UpdateRaftVoterResponseCurrentLeader + v.Default() + return v + })()) { + toEncode = append(toEncode, 0) + } + dst = kbin.AppendUvarint(dst, uint32(len(toEncode)+v.UnknownTags.Len())) + for _, tag := range toEncode { + switch tag { + case 0: + { + v := v.CurrentLeader + dst = kbin.AppendUvarint(dst, 0) + sized := false + lenAt := len(dst) + fCurrentLeader: + { + v := v.LeaderID + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + if !sized { + dst = kbin.AppendUvarint(dst[:lenAt], uint32(len(dst[lenAt:]))) + sized = true + goto fCurrentLeader + } + } + } + } + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *UpdateRaftVoterResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *UpdateRaftVoterResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *UpdateRaftVoterResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + if isFlexible { + for i := b.Uvarint(); i > 0; i-- { + switch key := b.Uvarint(); key { + default: + s.UnknownTags.Set(key, b.Span(int(b.Uvarint()))) + case 0: + b := kbin.Reader{Src: b.Span(int(b.Uvarint()))} + v := &s.CurrentLeader + v.Default() + s := v + { + v := b.Int32() + s.LeaderID = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Int32() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + if err := b.Complete(); err != nil { + return err + } + } + } + } + return b.Complete() +} + +// NewPtrUpdateRaftVoterResponse returns a pointer to a default UpdateRaftVoterResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrUpdateRaftVoterResponse() *UpdateRaftVoterResponse { + var v UpdateRaftVoterResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to UpdateRaftVoterResponse. +func (v *UpdateRaftVoterResponse) Default() { + { + v := &v.CurrentLeader + _ = v + v.LeaderID = -1 + v.LeaderEpoch = -1 + } +} + +// NewUpdateRaftVoterResponse returns a default UpdateRaftVoterResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewUpdateRaftVoterResponse() UpdateRaftVoterResponse { + var v UpdateRaftVoterResponse + v.Default() + return v +} + +type InitializeShareGroupStateRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // StateEpoch is the state epoch for this share-partition. + StateEpoch int32 + + // StartOffset is the share-partition start offset, or -1 if the start + // offset is not being initialized. + StartOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateRequestTopicPartition. +func (v *InitializeShareGroupStateRequestTopicPartition) Default() { +} + +// NewInitializeShareGroupStateRequestTopicPartition returns a default InitializeShareGroupStateRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateRequestTopicPartition() InitializeShareGroupStateRequestTopicPartition { + var v InitializeShareGroupStateRequestTopicPartition + v.Default() + return v +} + +type InitializeShareGroupStateRequestTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the data for the partitions. + Partitions []InitializeShareGroupStateRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateRequestTopic. +func (v *InitializeShareGroupStateRequestTopic) Default() { +} + +// NewInitializeShareGroupStateRequestTopic returns a default InitializeShareGroupStateRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateRequestTopic() InitializeShareGroupStateRequestTopic { + var v InitializeShareGroupStateRequestTopic + v.Default() + return v +} + +// InitializeShareGroupStateRequest is a request to initialize share group state. +type InitializeShareGroupStateRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the data for the topics. + Topics []InitializeShareGroupStateRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*InitializeShareGroupStateRequest) Key() int16 { return 83 } +func (*InitializeShareGroupStateRequest) MaxVersion() int16 { return 0 } +func (v *InitializeShareGroupStateRequest) SetVersion(version int16) { v.Version = version } +func (v *InitializeShareGroupStateRequest) GetVersion() int16 { return v.Version } +func (v *InitializeShareGroupStateRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *InitializeShareGroupStateRequest) IsShareCoordinatorRequest() {} +func (v *InitializeShareGroupStateRequest) ResponseKind() Response { + r := &InitializeShareGroupStateResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *InitializeShareGroupStateRequest) RequestWith(ctx context.Context, r Requestor) (*InitializeShareGroupStateResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*InitializeShareGroupStateResponse) + return resp, err +} + +func (v *InitializeShareGroupStateRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StateEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *InitializeShareGroupStateRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *InitializeShareGroupStateRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *InitializeShareGroupStateRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]InitializeShareGroupStateRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]InitializeShareGroupStateRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.StateEpoch = v + } + { + v := b.Int64() + s.StartOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrInitializeShareGroupStateRequest returns a pointer to a default InitializeShareGroupStateRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrInitializeShareGroupStateRequest() *InitializeShareGroupStateRequest { + var v InitializeShareGroupStateRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateRequest. +func (v *InitializeShareGroupStateRequest) Default() { +} + +// NewInitializeShareGroupStateRequest returns a default InitializeShareGroupStateRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateRequest() InitializeShareGroupStateRequest { + var v InitializeShareGroupStateRequest + v.Default() + return v +} + +type InitializeShareGroupStateResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateResponseTopicPartition. +func (v *InitializeShareGroupStateResponseTopicPartition) Default() { +} + +// NewInitializeShareGroupStateResponseTopicPartition returns a default InitializeShareGroupStateResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateResponseTopicPartition() InitializeShareGroupStateResponseTopicPartition { + var v InitializeShareGroupStateResponseTopicPartition + v.Default() + return v +} + +type InitializeShareGroupStateResponseTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the results for the partitions. + Partitions []InitializeShareGroupStateResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateResponseTopic. +func (v *InitializeShareGroupStateResponseTopic) Default() { +} + +// NewInitializeShareGroupStateResponseTopic returns a default InitializeShareGroupStateResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateResponseTopic() InitializeShareGroupStateResponseTopic { + var v InitializeShareGroupStateResponseTopic + v.Default() + return v +} + +// InitializeShareGroupStateResponse is a response for an InitializeShareGroupStateRequest. +// +// Supported errors: +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - FENCED_STATE_EPOCH (version 0+) +// - INVALID_REQUEST (version 0+) +type InitializeShareGroupStateResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics are the initialization results. + Topics []InitializeShareGroupStateResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*InitializeShareGroupStateResponse) Key() int16 { return 83 } +func (*InitializeShareGroupStateResponse) MaxVersion() int16 { return 0 } +func (v *InitializeShareGroupStateResponse) SetVersion(version int16) { v.Version = version } +func (v *InitializeShareGroupStateResponse) GetVersion() int16 { return v.Version } +func (v *InitializeShareGroupStateResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *InitializeShareGroupStateResponse) RequestKind() Request { + return &InitializeShareGroupStateRequest{Version: v.Version} +} + +func (v *InitializeShareGroupStateResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *InitializeShareGroupStateResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *InitializeShareGroupStateResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *InitializeShareGroupStateResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]InitializeShareGroupStateResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]InitializeShareGroupStateResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrInitializeShareGroupStateResponse returns a pointer to a default InitializeShareGroupStateResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrInitializeShareGroupStateResponse() *InitializeShareGroupStateResponse { + var v InitializeShareGroupStateResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to InitializeShareGroupStateResponse. +func (v *InitializeShareGroupStateResponse) Default() { +} + +// NewInitializeShareGroupStateResponse returns a default InitializeShareGroupStateResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewInitializeShareGroupStateResponse() InitializeShareGroupStateResponse { + var v InitializeShareGroupStateResponse + v.Default() + return v +} + +type ReadShareGroupStateRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // LeaderEpoch is the leader epoch of the share-partition. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateRequestTopicPartition. +func (v *ReadShareGroupStateRequestTopicPartition) Default() { +} + +// NewReadShareGroupStateRequestTopicPartition returns a default ReadShareGroupStateRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateRequestTopicPartition() ReadShareGroupStateRequestTopicPartition { + var v ReadShareGroupStateRequestTopicPartition + v.Default() + return v +} + +type ReadShareGroupStateRequestTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the data for the partitions. + Partitions []ReadShareGroupStateRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateRequestTopic. +func (v *ReadShareGroupStateRequestTopic) Default() { +} + +// NewReadShareGroupStateRequestTopic returns a default ReadShareGroupStateRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateRequestTopic() ReadShareGroupStateRequestTopic { + var v ReadShareGroupStateRequestTopic + v.Default() + return v +} + +// ReadShareGroupStateRequest is a request to read share group state. +type ReadShareGroupStateRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the data for the topics. + Topics []ReadShareGroupStateRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ReadShareGroupStateRequest) Key() int16 { return 84 } +func (*ReadShareGroupStateRequest) MaxVersion() int16 { return 0 } +func (v *ReadShareGroupStateRequest) SetVersion(version int16) { v.Version = version } +func (v *ReadShareGroupStateRequest) GetVersion() int16 { return v.Version } +func (v *ReadShareGroupStateRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ReadShareGroupStateRequest) IsShareCoordinatorRequest() {} +func (v *ReadShareGroupStateRequest) ResponseKind() Response { + r := &ReadShareGroupStateResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ReadShareGroupStateRequest) RequestWith(ctx context.Context, r Requestor) (*ReadShareGroupStateResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ReadShareGroupStateResponse) + return resp, err +} + +func (v *ReadShareGroupStateRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ReadShareGroupStateRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ReadShareGroupStateRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ReadShareGroupStateRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrReadShareGroupStateRequest returns a pointer to a default ReadShareGroupStateRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrReadShareGroupStateRequest() *ReadShareGroupStateRequest { + var v ReadShareGroupStateRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateRequest. +func (v *ReadShareGroupStateRequest) Default() { +} + +// NewReadShareGroupStateRequest returns a default ReadShareGroupStateRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateRequest() ReadShareGroupStateRequest { + var v ReadShareGroupStateRequest + v.Default() + return v +} + +type ReadShareGroupStateResponseTopicPartitionStateBatch struct { + // FirstOffset is the first offset of this state batch. + FirstOffset int64 + + // LastOffset is the last offset of this state batch. + LastOffset int64 + + // DeliveryState is the delivery state - + // 0:Available,2:Acked,4:Archived. + DeliveryState int8 + + // DeliveryCount is the delivery count. + DeliveryCount int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateResponseTopicPartitionStateBatch. +func (v *ReadShareGroupStateResponseTopicPartitionStateBatch) Default() { +} + +// NewReadShareGroupStateResponseTopicPartitionStateBatch returns a default ReadShareGroupStateResponseTopicPartitionStateBatch +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateResponseTopicPartitionStateBatch() ReadShareGroupStateResponseTopicPartitionStateBatch { + var v ReadShareGroupStateResponseTopicPartitionStateBatch + v.Default() + return v +} + +type ReadShareGroupStateResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // StateEpoch is the state epoch of the share-partition. + StateEpoch int32 + + // StartOffset is the share-partition start offset, which can be -1 if + // it is not yet initialized. + StartOffset int64 + + // StateBatches are the state batches for this share-partition. + StateBatches []ReadShareGroupStateResponseTopicPartitionStateBatch + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateResponseTopicPartition. +func (v *ReadShareGroupStateResponseTopicPartition) Default() { +} + +// NewReadShareGroupStateResponseTopicPartition returns a default ReadShareGroupStateResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateResponseTopicPartition() ReadShareGroupStateResponseTopicPartition { + var v ReadShareGroupStateResponseTopicPartition + v.Default() + return v +} + +type ReadShareGroupStateResponseTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the results for the partitions. + Partitions []ReadShareGroupStateResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateResponseTopic. +func (v *ReadShareGroupStateResponseTopic) Default() { +} + +// NewReadShareGroupStateResponseTopic returns a default ReadShareGroupStateResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateResponseTopic() ReadShareGroupStateResponseTopic { + var v ReadShareGroupStateResponseTopic + v.Default() + return v +} + +// ReadShareGroupStateResponse is a response for a ReadShareGroupStateRequest. +// +// Supported errors: +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - UNKNOWN_TOPIC_OR_PARTITION (version 0+) +// - FENCED_LEADER_EPOCH (version 0+) +// - INVALID_REQUEST (version 0+) +type ReadShareGroupStateResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics are the read results. + Topics []ReadShareGroupStateResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ReadShareGroupStateResponse) Key() int16 { return 84 } +func (*ReadShareGroupStateResponse) MaxVersion() int16 { return 0 } +func (v *ReadShareGroupStateResponse) SetVersion(version int16) { v.Version = version } +func (v *ReadShareGroupStateResponse) GetVersion() int16 { return v.Version } +func (v *ReadShareGroupStateResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ReadShareGroupStateResponse) RequestKind() Request { + return &ReadShareGroupStateRequest{Version: v.Version} +} + +func (v *ReadShareGroupStateResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.StateEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.StateBatches + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.DeliveryState + dst = kbin.AppendInt8(dst, v) + } + { + v := v.DeliveryCount + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ReadShareGroupStateResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ReadShareGroupStateResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ReadShareGroupStateResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := b.Int32() + s.StateEpoch = v + } + { + v := b.Int64() + s.StartOffset = v + } + { + v := s.StateBatches + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateResponseTopicPartitionStateBatch, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int64() + s.LastOffset = v + } + { + v := b.Int8() + s.DeliveryState = v + } + { + v := b.Int16() + s.DeliveryCount = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StateBatches = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrReadShareGroupStateResponse returns a pointer to a default ReadShareGroupStateResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrReadShareGroupStateResponse() *ReadShareGroupStateResponse { + var v ReadShareGroupStateResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateResponse. +func (v *ReadShareGroupStateResponse) Default() { +} + +// NewReadShareGroupStateResponse returns a default ReadShareGroupStateResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateResponse() ReadShareGroupStateResponse { + var v ReadShareGroupStateResponse + v.Default() + return v +} + +type WriteShareGroupStateRequestTopicPartitionStateBatch struct { + // FirstOffset is the first offset of this state batch. + FirstOffset int64 + + // LastOffset is the last offset of this state batch. + LastOffset int64 + + // DeliveryState is the delivery state - + // 0:Available,2:Acked,4:Archived. + DeliveryState int8 + + // DeliveryCount is the delivery count. + DeliveryCount int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateRequestTopicPartitionStateBatch. +func (v *WriteShareGroupStateRequestTopicPartitionStateBatch) Default() { +} + +// NewWriteShareGroupStateRequestTopicPartitionStateBatch returns a default WriteShareGroupStateRequestTopicPartitionStateBatch +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateRequestTopicPartitionStateBatch() WriteShareGroupStateRequestTopicPartitionStateBatch { + var v WriteShareGroupStateRequestTopicPartitionStateBatch + v.Default() + return v +} + +type WriteShareGroupStateRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // StateEpoch is the state epoch of the share-partition. + StateEpoch int32 + + // LeaderEpoch is the leader epoch of the share-partition. + LeaderEpoch int32 + + // StartOffset is the share-partition start offset, or -1 if the start + // offset is not being written. + StartOffset int64 + + // DeliveryCompleteCount is the number of offsets >= share-partition start + // offset for which delivery has been completed. -1 if not set. + // + // This field has a default of -1. + DeliveryCompleteCount int32 // v1+ + + // StateBatches are the state batches for the share-partition. + StateBatches []WriteShareGroupStateRequestTopicPartitionStateBatch + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateRequestTopicPartition. +func (v *WriteShareGroupStateRequestTopicPartition) Default() { + v.DeliveryCompleteCount = -1 +} + +// NewWriteShareGroupStateRequestTopicPartition returns a default WriteShareGroupStateRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateRequestTopicPartition() WriteShareGroupStateRequestTopicPartition { + var v WriteShareGroupStateRequestTopicPartition + v.Default() + return v +} + +type WriteShareGroupStateRequestTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the data for the partitions. + Partitions []WriteShareGroupStateRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateRequestTopic. +func (v *WriteShareGroupStateRequestTopic) Default() { +} + +// NewWriteShareGroupStateRequestTopic returns a default WriteShareGroupStateRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateRequestTopic() WriteShareGroupStateRequestTopic { + var v WriteShareGroupStateRequestTopic + v.Default() + return v +} + +// WriteShareGroupStateRequest is a request to write share group state. +// Version 1, introduced in Kafka 4.2, adds DeliveryCompleteCount (KIP-1226). +type WriteShareGroupStateRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the data for the topics. + Topics []WriteShareGroupStateRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*WriteShareGroupStateRequest) Key() int16 { return 85 } +func (*WriteShareGroupStateRequest) MaxVersion() int16 { return 1 } +func (v *WriteShareGroupStateRequest) SetVersion(version int16) { v.Version = version } +func (v *WriteShareGroupStateRequest) GetVersion() int16 { return v.Version } +func (v *WriteShareGroupStateRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *WriteShareGroupStateRequest) IsShareCoordinatorRequest() {} +func (v *WriteShareGroupStateRequest) ResponseKind() Response { + r := &WriteShareGroupStateResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *WriteShareGroupStateRequest) RequestWith(ctx context.Context, r Requestor) (*WriteShareGroupStateResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*WriteShareGroupStateResponse) + return resp, err +} + +func (v *WriteShareGroupStateRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StateEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.DeliveryCompleteCount + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StateBatches + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.FirstOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LastOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.DeliveryState + dst = kbin.AppendInt8(dst, v) + } + { + v := v.DeliveryCount + dst = kbin.AppendInt16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *WriteShareGroupStateRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *WriteShareGroupStateRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *WriteShareGroupStateRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteShareGroupStateRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteShareGroupStateRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.StateEpoch = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Int64() + s.StartOffset = v + } + if version >= 1 { + v := b.Int32() + s.DeliveryCompleteCount = v + } + { + v := s.StateBatches + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteShareGroupStateRequestTopicPartitionStateBatch, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int64() + s.FirstOffset = v + } + { + v := b.Int64() + s.LastOffset = v + } + { + v := b.Int8() + s.DeliveryState = v + } + { + v := b.Int16() + s.DeliveryCount = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StateBatches = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrWriteShareGroupStateRequest returns a pointer to a default WriteShareGroupStateRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrWriteShareGroupStateRequest() *WriteShareGroupStateRequest { + var v WriteShareGroupStateRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateRequest. +func (v *WriteShareGroupStateRequest) Default() { +} + +// NewWriteShareGroupStateRequest returns a default WriteShareGroupStateRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateRequest() WriteShareGroupStateRequest { + var v WriteShareGroupStateRequest + v.Default() + return v +} + +type WriteShareGroupStateResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateResponseTopicPartition. +func (v *WriteShareGroupStateResponseTopicPartition) Default() { +} + +// NewWriteShareGroupStateResponseTopicPartition returns a default WriteShareGroupStateResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateResponseTopicPartition() WriteShareGroupStateResponseTopicPartition { + var v WriteShareGroupStateResponseTopicPartition + v.Default() + return v +} + +type WriteShareGroupStateResponseTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the results for the partitions. + Partitions []WriteShareGroupStateResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateResponseTopic. +func (v *WriteShareGroupStateResponseTopic) Default() { +} + +// NewWriteShareGroupStateResponseTopic returns a default WriteShareGroupStateResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateResponseTopic() WriteShareGroupStateResponseTopic { + var v WriteShareGroupStateResponseTopic + v.Default() + return v +} + +// WriteShareGroupStateResponse is a response for a WriteShareGroupStateRequest. +// +// Supported errors: +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - UNKNOWN_TOPIC_OR_PARTITION (version 0+) +// - FENCED_LEADER_EPOCH (version 0+) +// - FENCED_STATE_EPOCH (version 0+) +// - INVALID_REQUEST (version 0+) +type WriteShareGroupStateResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics are the write results. + Topics []WriteShareGroupStateResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*WriteShareGroupStateResponse) Key() int16 { return 85 } +func (*WriteShareGroupStateResponse) MaxVersion() int16 { return 1 } +func (v *WriteShareGroupStateResponse) SetVersion(version int16) { v.Version = version } +func (v *WriteShareGroupStateResponse) GetVersion() int16 { return v.Version } +func (v *WriteShareGroupStateResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *WriteShareGroupStateResponse) RequestKind() Request { + return &WriteShareGroupStateRequest{Version: v.Version} +} + +func (v *WriteShareGroupStateResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *WriteShareGroupStateResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *WriteShareGroupStateResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *WriteShareGroupStateResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteShareGroupStateResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]WriteShareGroupStateResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrWriteShareGroupStateResponse returns a pointer to a default WriteShareGroupStateResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrWriteShareGroupStateResponse() *WriteShareGroupStateResponse { + var v WriteShareGroupStateResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to WriteShareGroupStateResponse. +func (v *WriteShareGroupStateResponse) Default() { +} + +// NewWriteShareGroupStateResponse returns a default WriteShareGroupStateResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewWriteShareGroupStateResponse() WriteShareGroupStateResponse { + var v WriteShareGroupStateResponse + v.Default() + return v +} + +type DeleteShareGroupStateRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateRequestTopicPartition. +func (v *DeleteShareGroupStateRequestTopicPartition) Default() { +} + +// NewDeleteShareGroupStateRequestTopicPartition returns a default DeleteShareGroupStateRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateRequestTopicPartition() DeleteShareGroupStateRequestTopicPartition { + var v DeleteShareGroupStateRequestTopicPartition + v.Default() + return v +} + +type DeleteShareGroupStateRequestTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the data for the partitions. + Partitions []DeleteShareGroupStateRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateRequestTopic. +func (v *DeleteShareGroupStateRequestTopic) Default() { +} + +// NewDeleteShareGroupStateRequestTopic returns a default DeleteShareGroupStateRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateRequestTopic() DeleteShareGroupStateRequestTopic { + var v DeleteShareGroupStateRequestTopic + v.Default() + return v +} + +// DeleteShareGroupStateRequest is a request to delete share group state. +type DeleteShareGroupStateRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the data for the topics. + Topics []DeleteShareGroupStateRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DeleteShareGroupStateRequest) Key() int16 { return 86 } +func (*DeleteShareGroupStateRequest) MaxVersion() int16 { return 0 } +func (v *DeleteShareGroupStateRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteShareGroupStateRequest) GetVersion() int16 { return v.Version } +func (v *DeleteShareGroupStateRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DeleteShareGroupStateRequest) IsShareCoordinatorRequest() {} +func (v *DeleteShareGroupStateRequest) ResponseKind() Response { + r := &DeleteShareGroupStateResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteShareGroupStateRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteShareGroupStateResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteShareGroupStateResponse) + return resp, err +} + +func (v *DeleteShareGroupStateRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteShareGroupStateRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteShareGroupStateRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteShareGroupStateRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupStateRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupStateRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteShareGroupStateRequest returns a pointer to a default DeleteShareGroupStateRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteShareGroupStateRequest() *DeleteShareGroupStateRequest { + var v DeleteShareGroupStateRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateRequest. +func (v *DeleteShareGroupStateRequest) Default() { +} + +// NewDeleteShareGroupStateRequest returns a default DeleteShareGroupStateRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateRequest() DeleteShareGroupStateRequest { + var v DeleteShareGroupStateRequest + v.Default() + return v +} + +type DeleteShareGroupStateResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateResponseTopicPartition. +func (v *DeleteShareGroupStateResponseTopicPartition) Default() { +} + +// NewDeleteShareGroupStateResponseTopicPartition returns a default DeleteShareGroupStateResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateResponseTopicPartition() DeleteShareGroupStateResponseTopicPartition { + var v DeleteShareGroupStateResponseTopicPartition + v.Default() + return v +} + +type DeleteShareGroupStateResponseTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the results for the partitions. + Partitions []DeleteShareGroupStateResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateResponseTopic. +func (v *DeleteShareGroupStateResponseTopic) Default() { +} + +// NewDeleteShareGroupStateResponseTopic returns a default DeleteShareGroupStateResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateResponseTopic() DeleteShareGroupStateResponseTopic { + var v DeleteShareGroupStateResponseTopic + v.Default() + return v +} + +// DeleteShareGroupStateResponse is a response for a DeleteShareGroupStateRequest. +// +// Supported errors: +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - UNKNOWN_TOPIC_OR_PARTITION (version 0+) +// - FENCED_STATE_EPOCH (version 0+) +// - INVALID_REQUEST (version 0+) +type DeleteShareGroupStateResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics are the delete results. + Topics []DeleteShareGroupStateResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DeleteShareGroupStateResponse) Key() int16 { return 86 } +func (*DeleteShareGroupStateResponse) MaxVersion() int16 { return 0 } +func (v *DeleteShareGroupStateResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteShareGroupStateResponse) GetVersion() int16 { return v.Version } +func (v *DeleteShareGroupStateResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DeleteShareGroupStateResponse) RequestKind() Request { + return &DeleteShareGroupStateRequest{Version: v.Version} +} + +func (v *DeleteShareGroupStateResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteShareGroupStateResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteShareGroupStateResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteShareGroupStateResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupStateResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupStateResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteShareGroupStateResponse returns a pointer to a default DeleteShareGroupStateResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteShareGroupStateResponse() *DeleteShareGroupStateResponse { + var v DeleteShareGroupStateResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupStateResponse. +func (v *DeleteShareGroupStateResponse) Default() { +} + +// NewDeleteShareGroupStateResponse returns a default DeleteShareGroupStateResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupStateResponse() DeleteShareGroupStateResponse { + var v DeleteShareGroupStateResponse + v.Default() + return v +} + +type ReadShareGroupStateSummaryRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // LeaderEpoch is the leader epoch of the share-partition. + LeaderEpoch int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryRequestTopicPartition. +func (v *ReadShareGroupStateSummaryRequestTopicPartition) Default() { +} + +// NewReadShareGroupStateSummaryRequestTopicPartition returns a default ReadShareGroupStateSummaryRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryRequestTopicPartition() ReadShareGroupStateSummaryRequestTopicPartition { + var v ReadShareGroupStateSummaryRequestTopicPartition + v.Default() + return v +} + +type ReadShareGroupStateSummaryRequestTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the data for the partitions. + Partitions []ReadShareGroupStateSummaryRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryRequestTopic. +func (v *ReadShareGroupStateSummaryRequestTopic) Default() { +} + +// NewReadShareGroupStateSummaryRequestTopic returns a default ReadShareGroupStateSummaryRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryRequestTopic() ReadShareGroupStateSummaryRequestTopic { + var v ReadShareGroupStateSummaryRequestTopic + v.Default() + return v +} + +// ReadShareGroupStateSummaryRequest is a request to read share group state summary. +// Version 1, introduced in Kafka 4.2, adds DeliveryCompleteCount to the +// response (KIP-1226). +type ReadShareGroupStateSummaryRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the data for the topics. + Topics []ReadShareGroupStateSummaryRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ReadShareGroupStateSummaryRequest) Key() int16 { return 87 } +func (*ReadShareGroupStateSummaryRequest) MaxVersion() int16 { return 1 } +func (v *ReadShareGroupStateSummaryRequest) SetVersion(version int16) { v.Version = version } +func (v *ReadShareGroupStateSummaryRequest) GetVersion() int16 { return v.Version } +func (v *ReadShareGroupStateSummaryRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *ReadShareGroupStateSummaryRequest) IsShareCoordinatorRequest() {} +func (v *ReadShareGroupStateSummaryRequest) ResponseKind() Response { + r := &ReadShareGroupStateSummaryResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *ReadShareGroupStateSummaryRequest) RequestWith(ctx context.Context, r Requestor) (*ReadShareGroupStateSummaryResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*ReadShareGroupStateSummaryResponse) + return resp, err +} + +func (v *ReadShareGroupStateSummaryRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ReadShareGroupStateSummaryRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ReadShareGroupStateSummaryRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ReadShareGroupStateSummaryRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateSummaryRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateSummaryRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrReadShareGroupStateSummaryRequest returns a pointer to a default ReadShareGroupStateSummaryRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrReadShareGroupStateSummaryRequest() *ReadShareGroupStateSummaryRequest { + var v ReadShareGroupStateSummaryRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryRequest. +func (v *ReadShareGroupStateSummaryRequest) Default() { +} + +// NewReadShareGroupStateSummaryRequest returns a default ReadShareGroupStateSummaryRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryRequest() ReadShareGroupStateSummaryRequest { + var v ReadShareGroupStateSummaryRequest + v.Default() + return v +} + +type ReadShareGroupStateSummaryResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // StateEpoch is the state epoch of the share-partition. + StateEpoch int32 + + // LeaderEpoch is the leader epoch of the share-partition. + LeaderEpoch int32 + + // StartOffset is the share-partition start offset. + StartOffset int64 + + // DeliveryCompleteCount is the number of offsets >= share-partition start + // offset for which delivery has been completed. -1 if not set. + // + // This field has a default of -1. + DeliveryCompleteCount int32 // v1+ + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryResponseTopicPartition. +func (v *ReadShareGroupStateSummaryResponseTopicPartition) Default() { + v.DeliveryCompleteCount = -1 +} + +// NewReadShareGroupStateSummaryResponseTopicPartition returns a default ReadShareGroupStateSummaryResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryResponseTopicPartition() ReadShareGroupStateSummaryResponseTopicPartition { + var v ReadShareGroupStateSummaryResponseTopicPartition + v.Default() + return v +} + +type ReadShareGroupStateSummaryResponseTopic struct { + // TopicID is the topic identifier. + TopicID [16]byte + + // Partitions are the results for the partitions. + Partitions []ReadShareGroupStateSummaryResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryResponseTopic. +func (v *ReadShareGroupStateSummaryResponseTopic) Default() { +} + +// NewReadShareGroupStateSummaryResponseTopic returns a default ReadShareGroupStateSummaryResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryResponseTopic() ReadShareGroupStateSummaryResponseTopic { + var v ReadShareGroupStateSummaryResponseTopic + v.Default() + return v +} + +// ReadShareGroupStateSummaryResponse is a response for a ReadShareGroupStateSummaryRequest. +// +// Supported errors: +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - UNKNOWN_TOPIC_OR_PARTITION (version 0+) +// - FENCED_LEADER_EPOCH (version 0+) +// - INVALID_REQUEST (version 0+) +type ReadShareGroupStateSummaryResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Topics are the read results. + Topics []ReadShareGroupStateSummaryResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*ReadShareGroupStateSummaryResponse) Key() int16 { return 87 } +func (*ReadShareGroupStateSummaryResponse) MaxVersion() int16 { return 1 } +func (v *ReadShareGroupStateSummaryResponse) SetVersion(version int16) { v.Version = version } +func (v *ReadShareGroupStateSummaryResponse) GetVersion() int16 { return v.Version } +func (v *ReadShareGroupStateSummaryResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *ReadShareGroupStateSummaryResponse) RequestKind() Request { + return &ReadShareGroupStateSummaryRequest{Version: v.Version} +} + +func (v *ReadShareGroupStateSummaryResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.StateEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + if version >= 1 { + v := v.DeliveryCompleteCount + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *ReadShareGroupStateSummaryResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *ReadShareGroupStateSummaryResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *ReadShareGroupStateSummaryResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateSummaryResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]ReadShareGroupStateSummaryResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := b.Int32() + s.StateEpoch = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + { + v := b.Int64() + s.StartOffset = v + } + if version >= 1 { + v := b.Int32() + s.DeliveryCompleteCount = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrReadShareGroupStateSummaryResponse returns a pointer to a default ReadShareGroupStateSummaryResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrReadShareGroupStateSummaryResponse() *ReadShareGroupStateSummaryResponse { + var v ReadShareGroupStateSummaryResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to ReadShareGroupStateSummaryResponse. +func (v *ReadShareGroupStateSummaryResponse) Default() { +} + +// NewReadShareGroupStateSummaryResponse returns a default ReadShareGroupStateSummaryResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewReadShareGroupStateSummaryResponse() ReadShareGroupStateSummaryResponse { + var v ReadShareGroupStateSummaryResponse + v.Default() + return v +} + +// TaskIDs contains a subtopology and its partitions. +type TaskIDs struct { + // SubtopologyID is a string that uniquely identifies the subtopology. + SubtopologyID string + + // Partitions are the partitions of the input topics processed by this + // member. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TaskIDs. +func (v *TaskIDs) Default() { +} + +// NewTaskIDs returns a default TaskIDs +// This is a shortcut for creating a struct and calling Default yourself. +func NewTaskIDs() TaskIDs { + var v TaskIDs + v.Default() + return v +} + +// TopicInfo describes a topic's configuration for streams. +type TopicInfo struct { + // Topic is the name of the topic. + Topic string + + // NumPartitions is the number of partitions in the topic. 0 if no specific + // number is enforced; always 0 for changelog topics. + NumPartitions int32 + + // ReplicationFactor is the replication factor of the topic. 0 if the + // default replication factor should be used. + ReplicationFactor int16 + + // Configs are topic-level configurations as key-value pairs. + Configs []TopicInfoConfig + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TopicInfo. +func (v *TopicInfo) Default() { +} + +// NewTopicInfo returns a default TopicInfo +// This is a shortcut for creating a struct and calling Default yourself. +func NewTopicInfo() TopicInfo { + var v TopicInfo + v.Default() + return v +} + +// Endpoint describes a host:port endpoint. +type Endpoint struct { + // Host is the hostname. + Host string + + // Port is the port. + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to Endpoint. +func (v *Endpoint) Default() { +} + +// NewEndpoint returns a default Endpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewEndpoint() Endpoint { + var v Endpoint + v.Default() + return v +} + +// TaskOffset describes a changelog offset for a task. +type TaskOffset struct { + // SubtopologyID is the subtopology identifier. + SubtopologyID string + + // Partition is the partition. + Partition int32 + + // Offset is the offset. + Offset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to TaskOffset. +func (v *TaskOffset) Default() { +} + +// NewTaskOffset returns a default TaskOffset +// This is a shortcut for creating a struct and calling Default yourself. +func NewTaskOffset() TaskOffset { + var v TaskOffset + v.Default() + return v +} + +type StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup struct { + // SourceTopics are indexes into the subtopology's SourceTopics array. + SourceTopics []int16 + + // SourceTopicRegex are indexes into the subtopology's SourceTopicRegex + // array. + SourceTopicRegex []int16 + + // RepartitionSourceTopics are indexes into the subtopology's + // RepartitionSourceTopics array. + RepartitionSourceTopics []int16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup. +func (v *StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup) Default() { +} + +// NewStreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup returns a default StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup() StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup { + var v StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup + v.Default() + return v +} + +type StreamsGroupHeartbeatRequestTopologySubtopology struct { + // SubtopologyID uniquely identifies the subtopology. + SubtopologyID string + + // SourceTopics are the topics the topology reads from. + SourceTopics []string + + // SourceTopicRegex are regular expressions identifying topics the + // subtopology reads from. + SourceTopicRegex []string + + // StateChangelogTopics are changelog topics associated with this + // subtopology, created automatically. + StateChangelogTopics []TopicInfo + + // RepartitionSinkTopics are the repartition topics the subtopology + // writes to. + RepartitionSinkTopics []string + + // RepartitionSourceTopics are source topics that are internally created + // repartition topics, created automatically. + RepartitionSourceTopics []TopicInfo + + // CopartitionGroups are subsets of source topics that must be + // copartitioned. + CopartitionGroups []StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequestTopologySubtopology. +func (v *StreamsGroupHeartbeatRequestTopologySubtopology) Default() { +} + +// NewStreamsGroupHeartbeatRequestTopologySubtopology returns a default StreamsGroupHeartbeatRequestTopologySubtopology +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequestTopologySubtopology() StreamsGroupHeartbeatRequestTopologySubtopology { + var v StreamsGroupHeartbeatRequestTopologySubtopology + v.Default() + return v +} + +type StreamsGroupHeartbeatRequestTopology struct { + // Epoch is the epoch of the topology. + Epoch int32 + + // Subtopologies are the sub-topologies of the streams application. + Subtopologies []StreamsGroupHeartbeatRequestTopologySubtopology + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequestTopology. +func (v *StreamsGroupHeartbeatRequestTopology) Default() { +} + +// NewStreamsGroupHeartbeatRequestTopology returns a default StreamsGroupHeartbeatRequestTopology +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequestTopology() StreamsGroupHeartbeatRequestTopology { + var v StreamsGroupHeartbeatRequestTopology + v.Default() + return v +} + +type StreamsGroupHeartbeatRequestUserEndpoint struct { + // Host is the hostname. + Host string + + // Port is the port. + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequestUserEndpoint. +func (v *StreamsGroupHeartbeatRequestUserEndpoint) Default() { +} + +// NewStreamsGroupHeartbeatRequestUserEndpoint returns a default StreamsGroupHeartbeatRequestUserEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequestUserEndpoint() StreamsGroupHeartbeatRequestUserEndpoint { + var v StreamsGroupHeartbeatRequestUserEndpoint + v.Default() + return v +} + +type StreamsGroupHeartbeatRequestClientTag struct { + // Key is the tag key. + Key string + + // Value is the tag value. + Value string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequestClientTag. +func (v *StreamsGroupHeartbeatRequestClientTag) Default() { +} + +// NewStreamsGroupHeartbeatRequestClientTag returns a default StreamsGroupHeartbeatRequestClientTag +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequestClientTag() StreamsGroupHeartbeatRequestClientTag { + var v StreamsGroupHeartbeatRequestClientTag + v.Default() + return v +} + +// StreamsGroupHeartbeatRequest is a part of KIP-1071; documentation is left +// to the KIP itself for brevity. +type StreamsGroupHeartbeatRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Group is the group identifier. + Group string + + // MemberID is the member ID generated by the streams consumer. Must be + // kept during the entire lifetime of the streams consumer process. + MemberID string + + // MemberEpoch is the current member epoch; 0 to join the group; -1 to + // leave the group; -2 to indicate that the static member will rejoin. + MemberEpoch int32 + + // EndpointInformationEpoch is the current endpoint epoch of this client, + // represents the latest endpoint epoch this client received. + EndpointInformationEpoch int32 + + // InstanceID is the instance ID for static membership; null if not provided + // or if unchanged since the last heartbeat. + InstanceID *string + + // RackID is the rack ID of the member; null if not provided or if unchanged + // since the last heartbeat. + RackID *string + + // RebalanceTimeoutMillis is the maximum time in milliseconds that the + // coordinator will wait on the member to revoke its tasks. -1 if unchanged + // since last heartbeat. + // + // This field has a default of -1. + RebalanceTimeoutMillis int32 + + // Topology is the topology metadata of the streams application. Only sent + // when memberEpoch = 0. Null otherwise. + Topology *StreamsGroupHeartbeatRequestTopology + + // ActiveTasks are the currently owned active tasks. Null if unchanged since + // last heartbeat. + ActiveTasks []TaskIDs + + // StandbyTasks are the currently owned standby tasks. Null if unchanged + // since last heartbeat. + StandbyTasks []TaskIDs + + // WarmupTasks are the currently owned warm-up tasks. Null if unchanged + // since last heartbeat. + WarmupTasks []TaskIDs + + // ProcessID is the identity of the streams instance that may have multiple + // consumers. Null if unchanged since last heartbeat. + ProcessID *string + + // UserEndpoint is the user-defined endpoint for Interactive Queries. Null + // if unchanged since last heartbeat or if not defined on the client. + UserEndpoint *StreamsGroupHeartbeatRequestUserEndpoint + + // ClientTags are used for rack-aware assignment. Null if unchanged since + // last heartbeat. + ClientTags []StreamsGroupHeartbeatRequestClientTag + + // TaskOffsets are cumulative changelog offsets for tasks. Null if unchanged + // since last heartbeat. + TaskOffsets []TaskOffset + + // TaskEndOffsets are cumulative changelog end-offsets for tasks. Null if + // unchanged since last heartbeat. + TaskEndOffsets []TaskOffset + + // ShutdownApplication indicates whether all Streams clients in the group + // should shut down. + ShutdownApplication bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*StreamsGroupHeartbeatRequest) Key() int16 { return 88 } +func (*StreamsGroupHeartbeatRequest) MaxVersion() int16 { return 0 } +func (v *StreamsGroupHeartbeatRequest) SetVersion(version int16) { v.Version = version } +func (v *StreamsGroupHeartbeatRequest) GetVersion() int16 { return v.Version } +func (v *StreamsGroupHeartbeatRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *StreamsGroupHeartbeatRequest) IsGroupCoordinatorRequest() {} +func (v *StreamsGroupHeartbeatRequest) ResponseKind() Response { + r := &StreamsGroupHeartbeatResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *StreamsGroupHeartbeatRequest) RequestWith(ctx context.Context, r Requestor) (*StreamsGroupHeartbeatResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*StreamsGroupHeartbeatResponse) + return resp, err +} + +func (v *StreamsGroupHeartbeatRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.EndpointInformationEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RebalanceTimeoutMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topology + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Subtopologies + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.SourceTopicRegex + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.StateChangelogTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.RepartitionSinkTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.RepartitionSourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.CopartitionGroups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt16(dst, v) + } + } + { + v := v.SourceTopicRegex + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt16(dst, v) + } + } + { + v := v.RepartitionSourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt16(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ActiveTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.StandbyTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.WarmupTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ProcessID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.UserEndpoint + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ClientTags + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TaskOffsets + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TaskEndOffsets + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ShutdownApplication + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StreamsGroupHeartbeatRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StreamsGroupHeartbeatRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StreamsGroupHeartbeatRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + v := b.Int32() + s.EndpointInformationEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + v := b.Int32() + s.RebalanceTimeoutMillis = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.Topology = new(StreamsGroupHeartbeatRequestTopology) + v := s.Topology + v.Default() + s := v + { + v := b.Int32() + s.Epoch = v + } + { + v := s.Subtopologies + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatRequestTopologySubtopology, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.SourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SourceTopics = v + } + { + v := s.SourceTopicRegex + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SourceTopicRegex = v + } + { + v := s.StateChangelogTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfo, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.NumPartitions = v + } + { + v := b.Int16() + s.ReplicationFactor = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfoConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StateChangelogTopics = v + } + { + v := s.RepartitionSinkTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.RepartitionSinkTopics = v + } + { + v := s.RepartitionSourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfo, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.NumPartitions = v + } + { + v := b.Int16() + s.ReplicationFactor = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfoConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.RepartitionSourceTopics = v + } + { + v := s.CopartitionGroups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatRequestTopologySubtopologyCopartitionGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := s.SourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int16, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int16() + a[i] = v + } + v = a + s.SourceTopics = v + } + { + v := s.SourceTopicRegex + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int16, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int16() + a[i] = v + } + v = a + s.SourceTopicRegex = v + } + { + v := s.RepartitionSourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int16, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int16() + a[i] = v + } + v = a + s.RepartitionSourceTopics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.CopartitionGroups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Subtopologies = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + { + v := s.ActiveTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActiveTasks = v + } + { + v := s.StandbyTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StandbyTasks = v + } + { + v := s.WarmupTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.WarmupTasks = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ProcessID = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.UserEndpoint = new(StreamsGroupHeartbeatRequestUserEndpoint) + v := s.UserEndpoint + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + { + v := s.ClientTags + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []StreamsGroupHeartbeatRequestClientTag{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatRequestClientTag, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ClientTags = v + } + { + v := s.TaskOffsets + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskOffset{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskOffset, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TaskOffsets = v + } + { + v := s.TaskEndOffsets + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskOffset{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskOffset, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TaskEndOffsets = v + } + { + v := b.Bool() + s.ShutdownApplication = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStreamsGroupHeartbeatRequest returns a pointer to a default StreamsGroupHeartbeatRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStreamsGroupHeartbeatRequest() *StreamsGroupHeartbeatRequest { + var v StreamsGroupHeartbeatRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatRequest. +func (v *StreamsGroupHeartbeatRequest) Default() { + v.RebalanceTimeoutMillis = -1 + { + v := &v.Topology + _ = v + } + { + v := &v.UserEndpoint + _ = v + } +} + +// NewStreamsGroupHeartbeatRequest returns a default StreamsGroupHeartbeatRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatRequest() StreamsGroupHeartbeatRequest { + var v StreamsGroupHeartbeatRequest + v.Default() + return v +} + +type StreamsGroupHeartbeatResponseStatus struct { + // StatusCode indicates a particular status; 0:STALE_TOPOLOGY, + // 1:MISSING_SOURCE_TOPICS, 2:INCORRECTLY_PARTITIONED_TOPICS, + // 3:MISSING_INTERNAL_TOPICS, 4:SHUTDOWN_APPLICATION, + // 5:ASSIGNMENT_DELAYED. + StatusCode int8 + + // StatusDetail is a string representation of the status. + StatusDetail string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatResponseStatus. +func (v *StreamsGroupHeartbeatResponseStatus) Default() { +} + +// NewStreamsGroupHeartbeatResponseStatus returns a default StreamsGroupHeartbeatResponseStatus +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatResponseStatus() StreamsGroupHeartbeatResponseStatus { + var v StreamsGroupHeartbeatResponseStatus + v.Default() + return v +} + +type StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition struct { + // Topic is the topic name. + Topic string + + // Partitions are the partitions. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition. +func (v *StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition) Default() { +} + +// NewStreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition returns a default StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition() StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition { + var v StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition + v.Default() + return v +} + +type StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition struct { + // Topic is the topic name. + Topic string + + // Partitions are the partitions. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition. +func (v *StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition) Default() { +} + +// NewStreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition returns a default StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition() StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition { + var v StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition + v.Default() + return v +} + +type StreamsGroupHeartbeatResponsePartitionsByUserEndpoint struct { + // UserEndpoint is the user-defined endpoint to connect to the node. + UserEndpoint Endpoint + + // ActivePartitions are all topic partitions materialized by active tasks + // on the node. + ActivePartitions []StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition + + // StandbyPartitions are all topic partitions materialized by standby + // tasks on the node. + StandbyPartitions []StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatResponsePartitionsByUserEndpoint. +func (v *StreamsGroupHeartbeatResponsePartitionsByUserEndpoint) Default() { + { + v := &v.UserEndpoint + _ = v + } +} + +// NewStreamsGroupHeartbeatResponsePartitionsByUserEndpoint returns a default StreamsGroupHeartbeatResponsePartitionsByUserEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatResponsePartitionsByUserEndpoint() StreamsGroupHeartbeatResponsePartitionsByUserEndpoint { + var v StreamsGroupHeartbeatResponsePartitionsByUserEndpoint + v.Default() + return v +} + +// StreamsGroupHeartbeatResponse is returned from a StreamsGroupHeartbeatRequest. +type StreamsGroupHeartbeatResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the top-level error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // MemberID is the member ID. + MemberID string + + // MemberEpoch is the member epoch. + MemberEpoch int32 + + // HeartbeatIntervalMillis is the heartbeat interval in milliseconds. + HeartbeatIntervalMillis int32 + + // AcceptableRecoveryLag is the maximal lag a warm-up task can have to be + // considered caught-up. + AcceptableRecoveryLag int32 + + // TaskOffsetIntervalMillis is the interval in which the task changelog + // offsets on a client are updated on the broker. + TaskOffsetIntervalMillis int32 + + // Status indicates zero or more statuses for the group membership. + Status []StreamsGroupHeartbeatResponseStatus + + // ActiveTasks are assigned active tasks. Null if unchanged since last + // heartbeat. + ActiveTasks []TaskIDs + + // StandbyTasks are assigned standby tasks. Null if unchanged since last + // heartbeat. + StandbyTasks []TaskIDs + + // WarmupTasks are assigned warm-up tasks. Null if unchanged since last + // heartbeat. + WarmupTasks []TaskIDs + + // EndpointInformationEpoch is the endpoint epoch set in the response. + EndpointInformationEpoch int32 + + // PartitionsByUserEndpoint is global assignment information used for + // Interactive Queries. Null if unchanged since last heartbeat. + PartitionsByUserEndpoint []StreamsGroupHeartbeatResponsePartitionsByUserEndpoint + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*StreamsGroupHeartbeatResponse) Key() int16 { return 88 } +func (*StreamsGroupHeartbeatResponse) MaxVersion() int16 { return 0 } +func (v *StreamsGroupHeartbeatResponse) SetVersion(version int16) { v.Version = version } +func (v *StreamsGroupHeartbeatResponse) GetVersion() int16 { return v.Version } +func (v *StreamsGroupHeartbeatResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *StreamsGroupHeartbeatResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *StreamsGroupHeartbeatResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *StreamsGroupHeartbeatResponse) RequestKind() Request { + return &StreamsGroupHeartbeatRequest{Version: v.Version} +} + +func (v *StreamsGroupHeartbeatResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.HeartbeatIntervalMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AcceptableRecoveryLag + dst = kbin.AppendInt32(dst, v) + } + { + v := v.TaskOffsetIntervalMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Status + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.StatusCode + dst = kbin.AppendInt8(dst, v) + } + { + v := v.StatusDetail + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ActiveTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.StandbyTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.WarmupTasks + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.EndpointInformationEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.PartitionsByUserEndpoint + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := &v.UserEndpoint + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := v.ActivePartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.StandbyPartitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StreamsGroupHeartbeatResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StreamsGroupHeartbeatResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StreamsGroupHeartbeatResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + v := b.Int32() + s.HeartbeatIntervalMillis = v + } + { + v := b.Int32() + s.AcceptableRecoveryLag = v + } + { + v := b.Int32() + s.TaskOffsetIntervalMillis = v + } + { + v := s.Status + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []StreamsGroupHeartbeatResponseStatus{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatResponseStatus, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int8() + s.StatusCode = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.StatusDetail = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Status = v + } + { + v := s.ActiveTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActiveTasks = v + } + { + v := s.StandbyTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StandbyTasks = v + } + { + v := s.WarmupTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []TaskIDs{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.WarmupTasks = v + } + { + v := b.Int32() + s.EndpointInformationEpoch = v + } + { + v := s.PartitionsByUserEndpoint + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []StreamsGroupHeartbeatResponsePartitionsByUserEndpoint{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatResponsePartitionsByUserEndpoint, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := &s.UserEndpoint + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := s.ActivePartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatResponsePartitionsByUserEndpointActivePartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActivePartitions = v + } + { + v := s.StandbyPartitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupHeartbeatResponsePartitionsByUserEndpointStandbyPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StandbyPartitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.PartitionsByUserEndpoint = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStreamsGroupHeartbeatResponse returns a pointer to a default StreamsGroupHeartbeatResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStreamsGroupHeartbeatResponse() *StreamsGroupHeartbeatResponse { + var v StreamsGroupHeartbeatResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupHeartbeatResponse. +func (v *StreamsGroupHeartbeatResponse) Default() { +} + +// NewStreamsGroupHeartbeatResponse returns a default StreamsGroupHeartbeatResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupHeartbeatResponse() StreamsGroupHeartbeatResponse { + var v StreamsGroupHeartbeatResponse + v.Default() + return v +} + +// StreamsAssignment contains streams group task assignments. +type StreamsAssignment struct { + // ActiveTasks are active tasks for this client. + ActiveTasks []TaskIDs + + // StandbyTasks are standby tasks for this client. + StandbyTasks []TaskIDs + + // WarmupTasks are warm-up tasks for this client. + WarmupTasks []TaskIDs + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsAssignment. +func (v *StreamsAssignment) Default() { +} + +// NewStreamsAssignment returns a default StreamsAssignment +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsAssignment() StreamsAssignment { + var v StreamsAssignment + v.Default() + return v +} + +// StreamsGroupDescribeRequest describes one or more streams groups. This is +// part of KIP-1071. +type StreamsGroupDescribeRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Groups are the IDs of the groups to describe. + Groups []string + + // IncludeAuthorizedOperations controls whether to include authorized + // operations. + IncludeAuthorizedOperations bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*StreamsGroupDescribeRequest) Key() int16 { return 89 } +func (*StreamsGroupDescribeRequest) MaxVersion() int16 { return 0 } +func (v *StreamsGroupDescribeRequest) SetVersion(version int16) { v.Version = version } +func (v *StreamsGroupDescribeRequest) GetVersion() int16 { return v.Version } +func (v *StreamsGroupDescribeRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *StreamsGroupDescribeRequest) ResponseKind() Response { + r := &StreamsGroupDescribeResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *StreamsGroupDescribeRequest) RequestWith(ctx context.Context, r Requestor) (*StreamsGroupDescribeResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*StreamsGroupDescribeResponse) + return resp, err +} + +func (v *StreamsGroupDescribeRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.IncludeAuthorizedOperations + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StreamsGroupDescribeRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StreamsGroupDescribeRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StreamsGroupDescribeRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.Groups = v + } + { + v := b.Bool() + s.IncludeAuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStreamsGroupDescribeRequest returns a pointer to a default StreamsGroupDescribeRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStreamsGroupDescribeRequest() *StreamsGroupDescribeRequest { + var v StreamsGroupDescribeRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeRequest. +func (v *StreamsGroupDescribeRequest) Default() { +} + +// NewStreamsGroupDescribeRequest returns a default StreamsGroupDescribeRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeRequest() StreamsGroupDescribeRequest { + var v StreamsGroupDescribeRequest + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroupTopologySubtopology struct { + // SubtopologyID uniquely identifies the subtopology. + SubtopologyID string + + // SourceTopics are the topics the subtopology reads from. + SourceTopics []string + + // RepartitionSinkTopics are the repartition topics the subtopology + // writes to. + RepartitionSinkTopics []string + + // StateChangelogTopics are changelog topics associated with this + // subtopology. + StateChangelogTopics []TopicInfo + + // RepartitionSourceTopics are source topics that are internally + // created repartition topics. + RepartitionSourceTopics []TopicInfo + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroupTopologySubtopology. +func (v *StreamsGroupDescribeResponseGroupTopologySubtopology) Default() { +} + +// NewStreamsGroupDescribeResponseGroupTopologySubtopology returns a default StreamsGroupDescribeResponseGroupTopologySubtopology +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroupTopologySubtopology() StreamsGroupDescribeResponseGroupTopologySubtopology { + var v StreamsGroupDescribeResponseGroupTopologySubtopology + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroupTopology struct { + // Epoch is the epoch of the currently initialized topology. + Epoch int32 + + // Subtopologies are the subtopologies of the streams application. Null + // if the group is uninitialized, source topics are missing, or + // incorrectly partitioned. + Subtopologies []StreamsGroupDescribeResponseGroupTopologySubtopology + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroupTopology. +func (v *StreamsGroupDescribeResponseGroupTopology) Default() { +} + +// NewStreamsGroupDescribeResponseGroupTopology returns a default StreamsGroupDescribeResponseGroupTopology +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroupTopology() StreamsGroupDescribeResponseGroupTopology { + var v StreamsGroupDescribeResponseGroupTopology + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroupMemberUserEndpoint struct { + // Host is the hostname. + Host string + + // Port is the port. + Port uint16 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroupMemberUserEndpoint. +func (v *StreamsGroupDescribeResponseGroupMemberUserEndpoint) Default() { +} + +// NewStreamsGroupDescribeResponseGroupMemberUserEndpoint returns a default StreamsGroupDescribeResponseGroupMemberUserEndpoint +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroupMemberUserEndpoint() StreamsGroupDescribeResponseGroupMemberUserEndpoint { + var v StreamsGroupDescribeResponseGroupMemberUserEndpoint + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroupMemberClientTag struct { + // Key is the tag key. + Key string + + // Value is the tag value. + Value string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroupMemberClientTag. +func (v *StreamsGroupDescribeResponseGroupMemberClientTag) Default() { +} + +// NewStreamsGroupDescribeResponseGroupMemberClientTag returns a default StreamsGroupDescribeResponseGroupMemberClientTag +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroupMemberClientTag() StreamsGroupDescribeResponseGroupMemberClientTag { + var v StreamsGroupDescribeResponseGroupMemberClientTag + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroupMember struct { + // MemberID is the member ID. + MemberID string + + // MemberEpoch is the member epoch. + MemberEpoch int32 + + // InstanceID is the member instance ID for static membership. + InstanceID *string + + // RackID is the rack ID. + RackID *string + + // ClientID is the client ID. + ClientID string + + // ClientHost is the client host. + ClientHost string + + // TopologyEpoch is the epoch of the topology on the client. + TopologyEpoch int32 + + // ProcessID is the identity of the streams instance that may have + // multiple clients. + ProcessID string + + // UserEndpoint is the user-defined endpoint for Interactive Queries. + // Null if not defined for this client. + UserEndpoint *StreamsGroupDescribeResponseGroupMemberUserEndpoint + + // ClientTags are used for rack-aware assignment. + ClientTags []StreamsGroupDescribeResponseGroupMemberClientTag + + // TaskOffsets are cumulative changelog offsets for tasks. + TaskOffsets []TaskOffset + + // TaskEndOffsets are cumulative changelog end offsets for tasks. + TaskEndOffsets []TaskOffset + + // Assignment is the current assignment. + Assignment StreamsAssignment + + // TargetAssignment is the target assignment. + TargetAssignment StreamsAssignment + + // IsClassic is true for classic members that have not been upgraded yet. + IsClassic bool + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroupMember. +func (v *StreamsGroupDescribeResponseGroupMember) Default() { + { + v := &v.UserEndpoint + _ = v + } + { + v := &v.Assignment + _ = v + } + { + v := &v.TargetAssignment + _ = v + } +} + +// NewStreamsGroupDescribeResponseGroupMember returns a default StreamsGroupDescribeResponseGroupMember +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroupMember() StreamsGroupDescribeResponseGroupMember { + var v StreamsGroupDescribeResponseGroupMember + v.Default() + return v +} + +type StreamsGroupDescribeResponseGroup struct { + // ErrorCode is the describe error, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // Group is the group ID. + Group string + + // State is the group state. + State string + + // Epoch is the group epoch. + Epoch int32 + + // AssignmentEpoch is the assignment epoch. + AssignmentEpoch int32 + + // Topology is the topology metadata currently initialized for the streams + // application. Null in case of a describe error. + Topology *StreamsGroupDescribeResponseGroupTopology + + // Members are the members of the group. + Members []StreamsGroupDescribeResponseGroupMember + + // AuthorizedOperations is a 32-bit bitfield representing authorized + // operations for the group. + // + // This field has a default of -2147483648. + AuthorizedOperations int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponseGroup. +func (v *StreamsGroupDescribeResponseGroup) Default() { + { + v := &v.Topology + _ = v + } + v.AuthorizedOperations = -2147483648 +} + +// NewStreamsGroupDescribeResponseGroup returns a default StreamsGroupDescribeResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponseGroup() StreamsGroupDescribeResponseGroup { + var v StreamsGroupDescribeResponseGroup + v.Default() + return v +} + +// StreamsGroupDescribeResponse is returned from a StreamsGroupDescribeRequest. +type StreamsGroupDescribeResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Groups contains each described group. + Groups []StreamsGroupDescribeResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*StreamsGroupDescribeResponse) Key() int16 { return 89 } +func (*StreamsGroupDescribeResponse) MaxVersion() int16 { return 0 } +func (v *StreamsGroupDescribeResponse) SetVersion(version int16) { v.Version = version } +func (v *StreamsGroupDescribeResponse) GetVersion() int16 { return v.Version } +func (v *StreamsGroupDescribeResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *StreamsGroupDescribeResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *StreamsGroupDescribeResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *StreamsGroupDescribeResponse) RequestKind() Request { + return &StreamsGroupDescribeRequest{Version: v.Version} +} + +func (v *StreamsGroupDescribeResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Group + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.State + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.AssignmentEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Topology + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Epoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Subtopologies + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.SourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.RepartitionSinkTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + } + { + v := v.StateChangelogTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.RepartitionSourceTopics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.NumPartitions + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ReplicationFactor + dst = kbin.AppendInt16(dst, v) + } + { + v := v.Configs + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.Members + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.MemberID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.MemberEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.InstanceID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.RackID + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.ClientID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.ClientHost + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TopologyEpoch + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ProcessID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.UserEndpoint + if v == nil { + dst = append(dst, 255) + } else { + dst = append(dst, 1) + { + v := v.Host + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Port + dst = kbin.AppendUint16(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ClientTags + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Key + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Value + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TaskOffsets + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.TaskEndOffsets + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Offset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := &v.Assignment + { + v := v.ActiveTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.StandbyTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.WarmupTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := &v.TargetAssignment + { + v := v.ActiveTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.StandbyTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.WarmupTasks + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.SubtopologyID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + { + v := v.IsClassic + dst = kbin.AppendBool(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.AuthorizedOperations + dst = kbin.AppendInt32(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *StreamsGroupDescribeResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *StreamsGroupDescribeResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *StreamsGroupDescribeResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupDescribeResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Group = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.State = v + } + { + v := b.Int32() + s.Epoch = v + } + { + v := b.Int32() + s.AssignmentEpoch = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.Topology = new(StreamsGroupDescribeResponseGroupTopology) + v := s.Topology + v.Default() + s := v + { + v := b.Int32() + s.Epoch = v + } + { + v := s.Subtopologies + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []StreamsGroupDescribeResponseGroupTopologySubtopology{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupDescribeResponseGroupTopologySubtopology, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.SourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.SourceTopics = v + } + { + v := s.RepartitionSinkTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]string, l)...) + } + for i := int32(0); i < l; i++ { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + a[i] = v + } + v = a + s.RepartitionSinkTopics = v + } + { + v := s.StateChangelogTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfo, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.NumPartitions = v + } + { + v := b.Int16() + s.ReplicationFactor = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfoConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StateChangelogTopics = v + } + { + v := s.RepartitionSourceTopics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfo, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Int32() + s.NumPartitions = v + } + { + v := b.Int16() + s.ReplicationFactor = v + } + { + v := s.Configs + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TopicInfoConfig, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Configs = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.RepartitionSourceTopics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Subtopologies = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + { + v := s.Members + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupDescribeResponseGroupMember, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.MemberID = v + } + { + v := b.Int32() + s.MemberEpoch = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.InstanceID = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.RackID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientID = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ClientHost = v + } + { + v := b.Int32() + s.TopologyEpoch = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.ProcessID = v + } + { + if present := b.Int8(); present != -1 && b.Ok() { + s.UserEndpoint = new(StreamsGroupDescribeResponseGroupMemberUserEndpoint) + v := s.UserEndpoint + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Host = v + } + { + v := b.Uint16() + s.Port = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + } + { + v := s.ClientTags + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]StreamsGroupDescribeResponseGroupMemberClientTag, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Key = v + } + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Value = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ClientTags = v + } + { + v := s.TaskOffsets + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskOffset, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TaskOffsets = v + } + { + v := s.TaskEndOffsets + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskOffset, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.Offset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.TaskEndOffsets = v + } + { + v := &s.Assignment + v.Default() + s := v + { + v := s.ActiveTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActiveTasks = v + } + { + v := s.StandbyTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StandbyTasks = v + } + { + v := s.WarmupTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.WarmupTasks = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := &s.TargetAssignment + v.Default() + s := v + { + v := s.ActiveTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.ActiveTasks = v + } + { + v := s.StandbyTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.StandbyTasks = v + } + { + v := s.WarmupTasks + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]TaskIDs, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.SubtopologyID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.WarmupTasks = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + { + v := b.Bool() + s.IsClassic = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Members = v + } + { + v := b.Int32() + s.AuthorizedOperations = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrStreamsGroupDescribeResponse returns a pointer to a default StreamsGroupDescribeResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrStreamsGroupDescribeResponse() *StreamsGroupDescribeResponse { + var v StreamsGroupDescribeResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to StreamsGroupDescribeResponse. +func (v *StreamsGroupDescribeResponse) Default() { +} + +// NewStreamsGroupDescribeResponse returns a default StreamsGroupDescribeResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewStreamsGroupDescribeResponse() StreamsGroupDescribeResponse { + var v StreamsGroupDescribeResponse + v.Default() + return v +} + +type DescribeShareGroupOffsetsRequestGroupTopic struct { + // Topic is the topic name. + Topic string + + // Partitions are the partitions. + Partitions []int32 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsRequestGroupTopic. +func (v *DescribeShareGroupOffsetsRequestGroupTopic) Default() { +} + +// NewDescribeShareGroupOffsetsRequestGroupTopic returns a default DescribeShareGroupOffsetsRequestGroupTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsRequestGroupTopic() DescribeShareGroupOffsetsRequestGroupTopic { + var v DescribeShareGroupOffsetsRequestGroupTopic + v.Default() + return v +} + +type DescribeShareGroupOffsetsRequestGroup struct { + // GroupID is the group identifier. + GroupID string + + // Topics are the topics to describe offsets for, or null for all + // topic-partitions. + Topics []DescribeShareGroupOffsetsRequestGroupTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsRequestGroup. +func (v *DescribeShareGroupOffsetsRequestGroup) Default() { +} + +// NewDescribeShareGroupOffsetsRequestGroup returns a default DescribeShareGroupOffsetsRequestGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsRequestGroup() DescribeShareGroupOffsetsRequestGroup { + var v DescribeShareGroupOffsetsRequestGroup + v.Default() + return v +} + +// DescribeShareGroupOffsetsRequest is a request to describe share group offsets. +// Version 1, introduced in Kafka 4.2, adds Lag to the response (KIP-1226). +type DescribeShareGroupOffsetsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // Groups are the groups to describe offsets for. + Groups []DescribeShareGroupOffsetsRequestGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeShareGroupOffsetsRequest) Key() int16 { return 90 } +func (*DescribeShareGroupOffsetsRequest) MaxVersion() int16 { return 1 } +func (v *DescribeShareGroupOffsetsRequest) SetVersion(version int16) { v.Version = version } +func (v *DescribeShareGroupOffsetsRequest) GetVersion() int16 { return v.Version } +func (v *DescribeShareGroupOffsetsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeShareGroupOffsetsRequest) IsGroupCoordinatorRequest() {} +func (v *DescribeShareGroupOffsetsRequest) ResponseKind() Response { + r := &DescribeShareGroupOffsetsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DescribeShareGroupOffsetsRequest) RequestWith(ctx context.Context, r Requestor) (*DescribeShareGroupOffsetsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DescribeShareGroupOffsetsResponse) + return resp, err +} + +func (v *DescribeShareGroupOffsetsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactNullableArrayLen(dst, len(v), v == nil) + } else { + dst = kbin.AppendNullableArrayLen(dst, len(v), v == nil) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := v[i] + dst = kbin.AppendInt32(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeShareGroupOffsetsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeShareGroupOffsetsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeShareGroupOffsetsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeShareGroupOffsetsRequestGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if version < 0 || l == 0 { + a = []DescribeShareGroupOffsetsRequestGroupTopic{} + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeShareGroupOffsetsRequestGroupTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]int32, l)...) + } + for i := int32(0); i < l; i++ { + v := b.Int32() + a[i] = v + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeShareGroupOffsetsRequest returns a pointer to a default DescribeShareGroupOffsetsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeShareGroupOffsetsRequest() *DescribeShareGroupOffsetsRequest { + var v DescribeShareGroupOffsetsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsRequest. +func (v *DescribeShareGroupOffsetsRequest) Default() { +} + +// NewDescribeShareGroupOffsetsRequest returns a default DescribeShareGroupOffsetsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsRequest() DescribeShareGroupOffsetsRequest { + var v DescribeShareGroupOffsetsRequest + v.Default() + return v +} + +type DescribeShareGroupOffsetsResponseGroupTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // StartOffset is the share-partition start offset. + StartOffset int64 + + // LeaderEpoch is the leader epoch of the partition. + LeaderEpoch int32 + + // Lag is the share-partition lag. -1 if not available. + // + // This field has a default of -1. + Lag int64 // v1+ + + // ErrorCode is the partition-level error code, or 0 if there was no + // error. + ErrorCode int16 + + // ErrorMessage is the partition-level error message, or null if there + // was no error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsResponseGroupTopicPartition. +func (v *DescribeShareGroupOffsetsResponseGroupTopicPartition) Default() { + v.Lag = -1 +} + +// NewDescribeShareGroupOffsetsResponseGroupTopicPartition returns a default DescribeShareGroupOffsetsResponseGroupTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsResponseGroupTopicPartition() DescribeShareGroupOffsetsResponseGroupTopicPartition { + var v DescribeShareGroupOffsetsResponseGroupTopicPartition + v.Default() + return v +} + +type DescribeShareGroupOffsetsResponseGroupTopic struct { + // Topic is the topic name. + Topic string + + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the partition results. + Partitions []DescribeShareGroupOffsetsResponseGroupTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsResponseGroupTopic. +func (v *DescribeShareGroupOffsetsResponseGroupTopic) Default() { +} + +// NewDescribeShareGroupOffsetsResponseGroupTopic returns a default DescribeShareGroupOffsetsResponseGroupTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsResponseGroupTopic() DescribeShareGroupOffsetsResponseGroupTopic { + var v DescribeShareGroupOffsetsResponseGroupTopic + v.Default() + return v +} + +type DescribeShareGroupOffsetsResponseGroup struct { + // GroupID is the group identifier. + GroupID string + + // Topics are the results for each topic. + Topics []DescribeShareGroupOffsetsResponseGroupTopic + + // ErrorCode is the group-level error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the group-level error message, or null if there was no + // error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsResponseGroup. +func (v *DescribeShareGroupOffsetsResponseGroup) Default() { +} + +// NewDescribeShareGroupOffsetsResponseGroup returns a default DescribeShareGroupOffsetsResponseGroup +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsResponseGroup() DescribeShareGroupOffsetsResponseGroup { + var v DescribeShareGroupOffsetsResponseGroup + v.Default() + return v +} + +// DescribeShareGroupOffsetsResponse is a response for a DescribeShareGroupOffsetsRequest. +// +// Supported errors: +// - GROUP_AUTHORIZATION_FAILED (version 0+) +// - TOPIC_AUTHORIZATION_FAILED (version 0+) +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - INVALID_REQUEST (version 0+) +// - UNKNOWN_SERVER_ERROR (version 0+) +type DescribeShareGroupOffsetsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // Groups are the results for each group. + Groups []DescribeShareGroupOffsetsResponseGroup + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DescribeShareGroupOffsetsResponse) Key() int16 { return 90 } +func (*DescribeShareGroupOffsetsResponse) MaxVersion() int16 { return 1 } +func (v *DescribeShareGroupOffsetsResponse) SetVersion(version int16) { v.Version = version } +func (v *DescribeShareGroupOffsetsResponse) GetVersion() int16 { return v.Version } +func (v *DescribeShareGroupOffsetsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DescribeShareGroupOffsetsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DescribeShareGroupOffsetsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DescribeShareGroupOffsetsResponse) RequestKind() Request { + return &DescribeShareGroupOffsetsRequest{Version: v.Version} +} + +func (v *DescribeShareGroupOffsetsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.Groups + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + { + v := v.LeaderEpoch + dst = kbin.AppendInt32(dst, v) + } + if version >= 1 { + v := v.Lag + dst = kbin.AppendInt64(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DescribeShareGroupOffsetsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DescribeShareGroupOffsetsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DescribeShareGroupOffsetsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := s.Groups + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeShareGroupOffsetsResponseGroup, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeShareGroupOffsetsResponseGroupTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DescribeShareGroupOffsetsResponseGroupTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.StartOffset = v + } + { + v := b.Int32() + s.LeaderEpoch = v + } + if version >= 1 { + v := b.Int64() + s.Lag = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Groups = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDescribeShareGroupOffsetsResponse returns a pointer to a default DescribeShareGroupOffsetsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDescribeShareGroupOffsetsResponse() *DescribeShareGroupOffsetsResponse { + var v DescribeShareGroupOffsetsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DescribeShareGroupOffsetsResponse. +func (v *DescribeShareGroupOffsetsResponse) Default() { +} + +// NewDescribeShareGroupOffsetsResponse returns a default DescribeShareGroupOffsetsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDescribeShareGroupOffsetsResponse() DescribeShareGroupOffsetsResponse { + var v DescribeShareGroupOffsetsResponse + v.Default() + return v +} + +type AlterShareGroupOffsetsRequestTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // StartOffset is the share-partition start offset. + StartOffset int64 + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsRequestTopicPartition. +func (v *AlterShareGroupOffsetsRequestTopicPartition) Default() { +} + +// NewAlterShareGroupOffsetsRequestTopicPartition returns a default AlterShareGroupOffsetsRequestTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsRequestTopicPartition() AlterShareGroupOffsetsRequestTopicPartition { + var v AlterShareGroupOffsetsRequestTopicPartition + v.Default() + return v +} + +type AlterShareGroupOffsetsRequestTopic struct { + // Topic is the topic name. + Topic string + + // Partitions are each partition to alter offsets for. + Partitions []AlterShareGroupOffsetsRequestTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsRequestTopic. +func (v *AlterShareGroupOffsetsRequestTopic) Default() { +} + +// NewAlterShareGroupOffsetsRequestTopic returns a default AlterShareGroupOffsetsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsRequestTopic() AlterShareGroupOffsetsRequestTopic { + var v AlterShareGroupOffsetsRequestTopic + v.Default() + return v +} + +// AlterShareGroupOffsetsRequest is a request to alter share group offsets. +type AlterShareGroupOffsetsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the topics to alter offsets for. + Topics []AlterShareGroupOffsetsRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterShareGroupOffsetsRequest) Key() int16 { return 91 } +func (*AlterShareGroupOffsetsRequest) MaxVersion() int16 { return 0 } +func (v *AlterShareGroupOffsetsRequest) SetVersion(version int16) { v.Version = version } +func (v *AlterShareGroupOffsetsRequest) GetVersion() int16 { return v.Version } +func (v *AlterShareGroupOffsetsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterShareGroupOffsetsRequest) IsGroupCoordinatorRequest() {} +func (v *AlterShareGroupOffsetsRequest) ResponseKind() Response { + r := &AlterShareGroupOffsetsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *AlterShareGroupOffsetsRequest) RequestWith(ctx context.Context, r Requestor) (*AlterShareGroupOffsetsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*AlterShareGroupOffsetsResponse) + return resp, err +} + +func (v *AlterShareGroupOffsetsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.StartOffset + dst = kbin.AppendInt64(dst, v) + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterShareGroupOffsetsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterShareGroupOffsetsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterShareGroupOffsetsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterShareGroupOffsetsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterShareGroupOffsetsRequestTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int64() + s.StartOffset = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterShareGroupOffsetsRequest returns a pointer to a default AlterShareGroupOffsetsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterShareGroupOffsetsRequest() *AlterShareGroupOffsetsRequest { + var v AlterShareGroupOffsetsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsRequest. +func (v *AlterShareGroupOffsetsRequest) Default() { +} + +// NewAlterShareGroupOffsetsRequest returns a default AlterShareGroupOffsetsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsRequest() AlterShareGroupOffsetsRequest { + var v AlterShareGroupOffsetsRequest + v.Default() + return v +} + +type AlterShareGroupOffsetsResponseTopicPartition struct { + // Partition is the partition index. + Partition int32 + + // ErrorCode is the error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the error message, or null if there was no error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsResponseTopicPartition. +func (v *AlterShareGroupOffsetsResponseTopicPartition) Default() { +} + +// NewAlterShareGroupOffsetsResponseTopicPartition returns a default AlterShareGroupOffsetsResponseTopicPartition +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsResponseTopicPartition() AlterShareGroupOffsetsResponseTopicPartition { + var v AlterShareGroupOffsetsResponseTopicPartition + v.Default() + return v +} + +type AlterShareGroupOffsetsResponseTopic struct { + // Topic is the topic name. + Topic string + + // TopicID is the unique topic ID. + TopicID [16]byte + + // Partitions are the partition results. + Partitions []AlterShareGroupOffsetsResponseTopicPartition + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsResponseTopic. +func (v *AlterShareGroupOffsetsResponseTopic) Default() { +} + +// NewAlterShareGroupOffsetsResponseTopic returns a default AlterShareGroupOffsetsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsResponseTopic() AlterShareGroupOffsetsResponseTopic { + var v AlterShareGroupOffsetsResponseTopic + v.Default() + return v +} + +// AlterShareGroupOffsetsResponse is a response for an AlterShareGroupOffsetsRequest. +// +// Supported errors: +// - GROUP_AUTHORIZATION_FAILED (version 0+) +// - TOPIC_AUTHORIZATION_FAILED (version 0+) +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - NON_EMPTY_GROUP (version 0+) +// - KAFKA_STORAGE_ERROR (version 0+) +// - INVALID_REQUEST (version 0+) +// - UNKNOWN_SERVER_ERROR (version 0+) +type AlterShareGroupOffsetsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the top-level error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // Topics are the results for each topic. + Topics []AlterShareGroupOffsetsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*AlterShareGroupOffsetsResponse) Key() int16 { return 91 } +func (*AlterShareGroupOffsetsResponse) MaxVersion() int16 { return 0 } +func (v *AlterShareGroupOffsetsResponse) SetVersion(version int16) { v.Version = version } +func (v *AlterShareGroupOffsetsResponse) GetVersion() int16 { return v.Version } +func (v *AlterShareGroupOffsetsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *AlterShareGroupOffsetsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *AlterShareGroupOffsetsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *AlterShareGroupOffsetsResponse) RequestKind() Request { + return &AlterShareGroupOffsetsRequest{Version: v.Version} +} + +func (v *AlterShareGroupOffsetsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.Partitions + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Partition + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *AlterShareGroupOffsetsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *AlterShareGroupOffsetsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *AlterShareGroupOffsetsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterShareGroupOffsetsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Uuid() + s.TopicID = v + } + { + v := s.Partitions + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]AlterShareGroupOffsetsResponseTopicPartition, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + v := b.Int32() + s.Partition = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Partitions = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrAlterShareGroupOffsetsResponse returns a pointer to a default AlterShareGroupOffsetsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrAlterShareGroupOffsetsResponse() *AlterShareGroupOffsetsResponse { + var v AlterShareGroupOffsetsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to AlterShareGroupOffsetsResponse. +func (v *AlterShareGroupOffsetsResponse) Default() { +} + +// NewAlterShareGroupOffsetsResponse returns a default AlterShareGroupOffsetsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewAlterShareGroupOffsetsResponse() AlterShareGroupOffsetsResponse { + var v AlterShareGroupOffsetsResponse + v.Default() + return v +} + +type DeleteShareGroupOffsetsRequestTopic struct { + // Topic is the topic name. + Topic string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupOffsetsRequestTopic. +func (v *DeleteShareGroupOffsetsRequestTopic) Default() { +} + +// NewDeleteShareGroupOffsetsRequestTopic returns a default DeleteShareGroupOffsetsRequestTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupOffsetsRequestTopic() DeleteShareGroupOffsetsRequestTopic { + var v DeleteShareGroupOffsetsRequestTopic + v.Default() + return v +} + +// DeleteShareGroupOffsetsRequest is a request to delete share group offsets. +type DeleteShareGroupOffsetsRequest struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // GroupID is the group identifier. + GroupID string + + // Topics are the topics to delete offsets for. + Topics []DeleteShareGroupOffsetsRequestTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DeleteShareGroupOffsetsRequest) Key() int16 { return 92 } +func (*DeleteShareGroupOffsetsRequest) MaxVersion() int16 { return 0 } +func (v *DeleteShareGroupOffsetsRequest) SetVersion(version int16) { v.Version = version } +func (v *DeleteShareGroupOffsetsRequest) GetVersion() int16 { return v.Version } +func (v *DeleteShareGroupOffsetsRequest) IsFlexible() bool { return v.Version >= 0 } +func (v *DeleteShareGroupOffsetsRequest) IsGroupCoordinatorRequest() {} +func (v *DeleteShareGroupOffsetsRequest) ResponseKind() Response { + r := &DeleteShareGroupOffsetsResponse{Version: v.Version} + r.Default() + return r +} + +// RequestWith is requests v on r and returns the response or an error. +// For sharded requests, the response may be merged and still return an error. +// It is better to rely on client.RequestSharded than to rely on proper merging behavior. +func (v *DeleteShareGroupOffsetsRequest) RequestWith(ctx context.Context, r Requestor) (*DeleteShareGroupOffsetsResponse, error) { + kresp, err := r.Request(ctx, v) + resp, _ := kresp.(*DeleteShareGroupOffsetsResponse) + return resp, err +} + +func (v *DeleteShareGroupOffsetsRequest) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.GroupID + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteShareGroupOffsetsRequest) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteShareGroupOffsetsRequest) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteShareGroupOffsetsRequest) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.GroupID = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupOffsetsRequestTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteShareGroupOffsetsRequest returns a pointer to a default DeleteShareGroupOffsetsRequest +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteShareGroupOffsetsRequest() *DeleteShareGroupOffsetsRequest { + var v DeleteShareGroupOffsetsRequest + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupOffsetsRequest. +func (v *DeleteShareGroupOffsetsRequest) Default() { +} + +// NewDeleteShareGroupOffsetsRequest returns a default DeleteShareGroupOffsetsRequest +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupOffsetsRequest() DeleteShareGroupOffsetsRequest { + var v DeleteShareGroupOffsetsRequest + v.Default() + return v +} + +type DeleteShareGroupOffsetsResponseTopic struct { + // Topic is the topic name. + Topic string + + // TopicID is the unique topic ID. + TopicID [16]byte + + // ErrorCode is the topic-level error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the topic-level error message, or null if there was no + // error. + ErrorMessage *string + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupOffsetsResponseTopic. +func (v *DeleteShareGroupOffsetsResponseTopic) Default() { +} + +// NewDeleteShareGroupOffsetsResponseTopic returns a default DeleteShareGroupOffsetsResponseTopic +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupOffsetsResponseTopic() DeleteShareGroupOffsetsResponseTopic { + var v DeleteShareGroupOffsetsResponseTopic + v.Default() + return v +} + +// DeleteShareGroupOffsetsResponse is a response for a DeleteShareGroupOffsetsRequest. +// +// Supported errors: +// - GROUP_AUTHORIZATION_FAILED (version 0+) +// - TOPIC_AUTHORIZATION_FAILED (version 0+) +// - NOT_COORDINATOR (version 0+) +// - COORDINATOR_NOT_AVAILABLE (version 0+) +// - COORDINATOR_LOAD_IN_PROGRESS (version 0+) +// - GROUP_ID_NOT_FOUND (version 0+) +// - NON_EMPTY_GROUP (version 0+) +// - KAFKA_STORAGE_ERROR (version 0+) +// - INVALID_REQUEST (version 0+) +// - UNKNOWN_SERVER_ERROR (version 0+) +// - UNKNOWN_TOPIC_OR_PARTITION (version 0+) +type DeleteShareGroupOffsetsResponse struct { + // Version is the version of this message used with a Kafka broker. + Version int16 + + // ThrottleMillis is how long of a throttle Kafka will apply to the client + // after responding to this request. + ThrottleMillis int32 + + // ErrorCode is the top-level error code, or 0 if there was no error. + ErrorCode int16 + + // ErrorMessage is the top-level error message, or null if there was no + // error. + ErrorMessage *string + + // Topics are the results for each topic. + Topics []DeleteShareGroupOffsetsResponseTopic + + // UnknownTags are tags Kafka sent that we do not know the purpose of. + UnknownTags Tags +} + +func (*DeleteShareGroupOffsetsResponse) Key() int16 { return 92 } +func (*DeleteShareGroupOffsetsResponse) MaxVersion() int16 { return 0 } +func (v *DeleteShareGroupOffsetsResponse) SetVersion(version int16) { v.Version = version } +func (v *DeleteShareGroupOffsetsResponse) GetVersion() int16 { return v.Version } +func (v *DeleteShareGroupOffsetsResponse) IsFlexible() bool { return v.Version >= 0 } +func (v *DeleteShareGroupOffsetsResponse) Throttle() (int32, bool) { + return v.ThrottleMillis, v.Version >= 0 +} + +func (v *DeleteShareGroupOffsetsResponse) SetThrottle(throttleMillis int32) { + v.ThrottleMillis = throttleMillis +} + +func (v *DeleteShareGroupOffsetsResponse) RequestKind() Request { + return &DeleteShareGroupOffsetsRequest{Version: v.Version} +} + +func (v *DeleteShareGroupOffsetsResponse) AppendTo(dst []byte) []byte { + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + { + v := v.ThrottleMillis + dst = kbin.AppendInt32(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + { + v := v.Topics + if isFlexible { + dst = kbin.AppendCompactArrayLen(dst, len(v)) + } else { + dst = kbin.AppendArrayLen(dst, len(v)) + } + for i := range v { + v := &v[i] + { + v := v.Topic + if isFlexible { + dst = kbin.AppendCompactString(dst, v) + } else { + dst = kbin.AppendString(dst, v) + } + } + { + v := v.TopicID + dst = kbin.AppendUuid(dst, v) + } + { + v := v.ErrorCode + dst = kbin.AppendInt16(dst, v) + } + { + v := v.ErrorMessage + if isFlexible { + dst = kbin.AppendCompactNullableString(dst, v) + } else { + dst = kbin.AppendNullableString(dst, v) + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + } + } + if isFlexible { + dst = kbin.AppendUvarint(dst, 0+uint32(v.UnknownTags.Len())) + dst = v.UnknownTags.AppendEach(dst) + } + return dst +} + +func (v *DeleteShareGroupOffsetsResponse) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *DeleteShareGroupOffsetsResponse) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *DeleteShareGroupOffsetsResponse) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + version := v.Version + _ = version + isFlexible := version >= 0 + _ = isFlexible + s := v + { + v := b.Int32() + s.ThrottleMillis = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + { + v := s.Topics + a := v + var l int32 + if isFlexible { + l = b.CompactArrayLen() + } else { + l = b.ArrayLen() + } + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]DeleteShareGroupOffsetsResponseTopic, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + if isFlexible { + v = b.UnsafeCompactString() + } else { + v = b.UnsafeString() + } + } else { + if isFlexible { + v = b.CompactString() + } else { + v = b.String() + } + } + s.Topic = v + } + { + v := b.Uuid() + s.TopicID = v + } + { + v := b.Int16() + s.ErrorCode = v + } + { + var v *string + if isFlexible { + if unsafe { + v = b.UnsafeCompactNullableString() + } else { + v = b.CompactNullableString() + } + } else { + if unsafe { + v = b.UnsafeNullableString() + } else { + v = b.NullableString() + } + } + s.ErrorMessage = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + } + v = a + s.Topics = v + } + if isFlexible { + s.UnknownTags = internalReadTags(&b) + } + return b.Complete() +} + +// NewPtrDeleteShareGroupOffsetsResponse returns a pointer to a default DeleteShareGroupOffsetsResponse +// This is a shortcut for creating a new(struct) and calling Default yourself. +func NewPtrDeleteShareGroupOffsetsResponse() *DeleteShareGroupOffsetsResponse { + var v DeleteShareGroupOffsetsResponse + v.Default() + return &v +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to DeleteShareGroupOffsetsResponse. +func (v *DeleteShareGroupOffsetsResponse) Default() { +} + +// NewDeleteShareGroupOffsetsResponse returns a default DeleteShareGroupOffsetsResponse +// This is a shortcut for creating a struct and calling Default yourself. +func NewDeleteShareGroupOffsetsResponse() DeleteShareGroupOffsetsResponse { + var v DeleteShareGroupOffsetsResponse + v.Default() + return v +} + +// RequestForKey returns the request corresponding to the given request key +// or nil if the key is unknown. +func RequestForKey(key int16) Request { + switch key { + default: + return nil + case 0: + return NewPtrProduceRequest() + case 1: + return NewPtrFetchRequest() + case 2: + return NewPtrListOffsetsRequest() + case 3: + return NewPtrMetadataRequest() + case 4: + return NewPtrLeaderAndISRRequest() + case 5: + return NewPtrStopReplicaRequest() + case 6: + return NewPtrUpdateMetadataRequest() + case 7: + return NewPtrControlledShutdownRequest() + case 8: + return NewPtrOffsetCommitRequest() + case 9: + return NewPtrOffsetFetchRequest() + case 10: + return NewPtrFindCoordinatorRequest() + case 11: + return NewPtrJoinGroupRequest() + case 12: + return NewPtrHeartbeatRequest() + case 13: + return NewPtrLeaveGroupRequest() + case 14: + return NewPtrSyncGroupRequest() + case 15: + return NewPtrDescribeGroupsRequest() + case 16: + return NewPtrListGroupsRequest() + case 17: + return NewPtrSASLHandshakeRequest() + case 18: + return NewPtrApiVersionsRequest() + case 19: + return NewPtrCreateTopicsRequest() + case 20: + return NewPtrDeleteTopicsRequest() + case 21: + return NewPtrDeleteRecordsRequest() + case 22: + return NewPtrInitProducerIDRequest() + case 23: + return NewPtrOffsetForLeaderEpochRequest() + case 24: + return NewPtrAddPartitionsToTxnRequest() + case 25: + return NewPtrAddOffsetsToTxnRequest() + case 26: + return NewPtrEndTxnRequest() + case 27: + return NewPtrWriteTxnMarkersRequest() + case 28: + return NewPtrTxnOffsetCommitRequest() + case 29: + return NewPtrDescribeACLsRequest() + case 30: + return NewPtrCreateACLsRequest() + case 31: + return NewPtrDeleteACLsRequest() + case 32: + return NewPtrDescribeConfigsRequest() + case 33: + return NewPtrAlterConfigsRequest() + case 34: + return NewPtrAlterReplicaLogDirsRequest() + case 35: + return NewPtrDescribeLogDirsRequest() + case 36: + return NewPtrSASLAuthenticateRequest() + case 37: + return NewPtrCreatePartitionsRequest() + case 38: + return NewPtrCreateDelegationTokenRequest() + case 39: + return NewPtrRenewDelegationTokenRequest() + case 40: + return NewPtrExpireDelegationTokenRequest() + case 41: + return NewPtrDescribeDelegationTokenRequest() + case 42: + return NewPtrDeleteGroupsRequest() + case 43: + return NewPtrElectLeadersRequest() + case 44: + return NewPtrIncrementalAlterConfigsRequest() + case 45: + return NewPtrAlterPartitionAssignmentsRequest() + case 46: + return NewPtrListPartitionReassignmentsRequest() + case 47: + return NewPtrOffsetDeleteRequest() + case 48: + return NewPtrDescribeClientQuotasRequest() + case 49: + return NewPtrAlterClientQuotasRequest() + case 50: + return NewPtrDescribeUserSCRAMCredentialsRequest() + case 51: + return NewPtrAlterUserSCRAMCredentialsRequest() + case 52: + return NewPtrVoteRequest() + case 53: + return NewPtrBeginQuorumEpochRequest() + case 54: + return NewPtrEndQuorumEpochRequest() + case 55: + return NewPtrDescribeQuorumRequest() + case 56: + return NewPtrAlterPartitionRequest() + case 57: + return NewPtrUpdateFeaturesRequest() + case 58: + return NewPtrEnvelopeRequest() + case 59: + return NewPtrFetchSnapshotRequest() + case 60: + return NewPtrDescribeClusterRequest() + case 61: + return NewPtrDescribeProducersRequest() + case 62: + return NewPtrBrokerRegistrationRequest() + case 63: + return NewPtrBrokerHeartbeatRequest() + case 64: + return NewPtrUnregisterBrokerRequest() + case 65: + return NewPtrDescribeTransactionsRequest() + case 66: + return NewPtrListTransactionsRequest() + case 67: + return NewPtrAllocateProducerIDsRequest() + case 68: + return NewPtrConsumerGroupHeartbeatRequest() + case 69: + return NewPtrConsumerGroupDescribeRequest() + case 70: + return NewPtrControllerRegistrationRequest() + case 71: + return NewPtrGetTelemetrySubscriptionsRequest() + case 72: + return NewPtrPushTelemetryRequest() + case 73: + return NewPtrAssignReplicasToDirsRequest() + case 74: + return NewPtrListConfigResourcesRequest() + case 75: + return NewPtrDescribeTopicPartitionsRequest() + case 76: + return NewPtrShareGroupHeartbeatRequest() + case 77: + return NewPtrShareGroupDescribeRequest() + case 78: + return NewPtrShareFetchRequest() + case 79: + return NewPtrShareAcknowledgeRequest() + case 80: + return NewPtrAddRaftVoterRequest() + case 81: + return NewPtrRemoveRaftVoterRequest() + case 82: + return NewPtrUpdateRaftVoterRequest() + case 83: + return NewPtrInitializeShareGroupStateRequest() + case 84: + return NewPtrReadShareGroupStateRequest() + case 85: + return NewPtrWriteShareGroupStateRequest() + case 86: + return NewPtrDeleteShareGroupStateRequest() + case 87: + return NewPtrReadShareGroupStateSummaryRequest() + case 88: + return NewPtrStreamsGroupHeartbeatRequest() + case 89: + return NewPtrStreamsGroupDescribeRequest() + case 90: + return NewPtrDescribeShareGroupOffsetsRequest() + case 91: + return NewPtrAlterShareGroupOffsetsRequest() + case 92: + return NewPtrDeleteShareGroupOffsetsRequest() + } +} + +// ResponseForKey returns the response corresponding to the given request key +// or nil if the key is unknown. +func ResponseForKey(key int16) Response { + switch key { + default: + return nil + case 0: + return NewPtrProduceResponse() + case 1: + return NewPtrFetchResponse() + case 2: + return NewPtrListOffsetsResponse() + case 3: + return NewPtrMetadataResponse() + case 4: + return NewPtrLeaderAndISRResponse() + case 5: + return NewPtrStopReplicaResponse() + case 6: + return NewPtrUpdateMetadataResponse() + case 7: + return NewPtrControlledShutdownResponse() + case 8: + return NewPtrOffsetCommitResponse() + case 9: + return NewPtrOffsetFetchResponse() + case 10: + return NewPtrFindCoordinatorResponse() + case 11: + return NewPtrJoinGroupResponse() + case 12: + return NewPtrHeartbeatResponse() + case 13: + return NewPtrLeaveGroupResponse() + case 14: + return NewPtrSyncGroupResponse() + case 15: + return NewPtrDescribeGroupsResponse() + case 16: + return NewPtrListGroupsResponse() + case 17: + return NewPtrSASLHandshakeResponse() + case 18: + return NewPtrApiVersionsResponse() + case 19: + return NewPtrCreateTopicsResponse() + case 20: + return NewPtrDeleteTopicsResponse() + case 21: + return NewPtrDeleteRecordsResponse() + case 22: + return NewPtrInitProducerIDResponse() + case 23: + return NewPtrOffsetForLeaderEpochResponse() + case 24: + return NewPtrAddPartitionsToTxnResponse() + case 25: + return NewPtrAddOffsetsToTxnResponse() + case 26: + return NewPtrEndTxnResponse() + case 27: + return NewPtrWriteTxnMarkersResponse() + case 28: + return NewPtrTxnOffsetCommitResponse() + case 29: + return NewPtrDescribeACLsResponse() + case 30: + return NewPtrCreateACLsResponse() + case 31: + return NewPtrDeleteACLsResponse() + case 32: + return NewPtrDescribeConfigsResponse() + case 33: + return NewPtrAlterConfigsResponse() + case 34: + return NewPtrAlterReplicaLogDirsResponse() + case 35: + return NewPtrDescribeLogDirsResponse() + case 36: + return NewPtrSASLAuthenticateResponse() + case 37: + return NewPtrCreatePartitionsResponse() + case 38: + return NewPtrCreateDelegationTokenResponse() + case 39: + return NewPtrRenewDelegationTokenResponse() + case 40: + return NewPtrExpireDelegationTokenResponse() + case 41: + return NewPtrDescribeDelegationTokenResponse() + case 42: + return NewPtrDeleteGroupsResponse() + case 43: + return NewPtrElectLeadersResponse() + case 44: + return NewPtrIncrementalAlterConfigsResponse() + case 45: + return NewPtrAlterPartitionAssignmentsResponse() + case 46: + return NewPtrListPartitionReassignmentsResponse() + case 47: + return NewPtrOffsetDeleteResponse() + case 48: + return NewPtrDescribeClientQuotasResponse() + case 49: + return NewPtrAlterClientQuotasResponse() + case 50: + return NewPtrDescribeUserSCRAMCredentialsResponse() + case 51: + return NewPtrAlterUserSCRAMCredentialsResponse() + case 52: + return NewPtrVoteResponse() + case 53: + return NewPtrBeginQuorumEpochResponse() + case 54: + return NewPtrEndQuorumEpochResponse() + case 55: + return NewPtrDescribeQuorumResponse() + case 56: + return NewPtrAlterPartitionResponse() + case 57: + return NewPtrUpdateFeaturesResponse() + case 58: + return NewPtrEnvelopeResponse() + case 59: + return NewPtrFetchSnapshotResponse() + case 60: + return NewPtrDescribeClusterResponse() + case 61: + return NewPtrDescribeProducersResponse() + case 62: + return NewPtrBrokerRegistrationResponse() + case 63: + return NewPtrBrokerHeartbeatResponse() + case 64: + return NewPtrUnregisterBrokerResponse() + case 65: + return NewPtrDescribeTransactionsResponse() + case 66: + return NewPtrListTransactionsResponse() + case 67: + return NewPtrAllocateProducerIDsResponse() + case 68: + return NewPtrConsumerGroupHeartbeatResponse() + case 69: + return NewPtrConsumerGroupDescribeResponse() + case 70: + return NewPtrControllerRegistrationResponse() + case 71: + return NewPtrGetTelemetrySubscriptionsResponse() + case 72: + return NewPtrPushTelemetryResponse() + case 73: + return NewPtrAssignReplicasToDirsResponse() + case 74: + return NewPtrListConfigResourcesResponse() + case 75: + return NewPtrDescribeTopicPartitionsResponse() + case 76: + return NewPtrShareGroupHeartbeatResponse() + case 77: + return NewPtrShareGroupDescribeResponse() + case 78: + return NewPtrShareFetchResponse() + case 79: + return NewPtrShareAcknowledgeResponse() + case 80: + return NewPtrAddRaftVoterResponse() + case 81: + return NewPtrRemoveRaftVoterResponse() + case 82: + return NewPtrUpdateRaftVoterResponse() + case 83: + return NewPtrInitializeShareGroupStateResponse() + case 84: + return NewPtrReadShareGroupStateResponse() + case 85: + return NewPtrWriteShareGroupStateResponse() + case 86: + return NewPtrDeleteShareGroupStateResponse() + case 87: + return NewPtrReadShareGroupStateSummaryResponse() + case 88: + return NewPtrStreamsGroupHeartbeatResponse() + case 89: + return NewPtrStreamsGroupDescribeResponse() + case 90: + return NewPtrDescribeShareGroupOffsetsResponse() + case 91: + return NewPtrAlterShareGroupOffsetsResponse() + case 92: + return NewPtrDeleteShareGroupOffsetsResponse() + } +} + +// NameForKey returns the name (e.g., "Fetch") corresponding to a given request key +// or "" if the key is unknown. +func NameForKey(key int16) string { + switch key { + default: + return "Unknown" + case 0: + return "Produce" + case 1: + return "Fetch" + case 2: + return "ListOffsets" + case 3: + return "Metadata" + case 4: + return "LeaderAndISR" + case 5: + return "StopReplica" + case 6: + return "UpdateMetadata" + case 7: + return "ControlledShutdown" + case 8: + return "OffsetCommit" + case 9: + return "OffsetFetch" + case 10: + return "FindCoordinator" + case 11: + return "JoinGroup" + case 12: + return "Heartbeat" + case 13: + return "LeaveGroup" + case 14: + return "SyncGroup" + case 15: + return "DescribeGroups" + case 16: + return "ListGroups" + case 17: + return "SASLHandshake" + case 18: + return "ApiVersions" + case 19: + return "CreateTopics" + case 20: + return "DeleteTopics" + case 21: + return "DeleteRecords" + case 22: + return "InitProducerID" + case 23: + return "OffsetForLeaderEpoch" + case 24: + return "AddPartitionsToTxn" + case 25: + return "AddOffsetsToTxn" + case 26: + return "EndTxn" + case 27: + return "WriteTxnMarkers" + case 28: + return "TxnOffsetCommit" + case 29: + return "DescribeACLs" + case 30: + return "CreateACLs" + case 31: + return "DeleteACLs" + case 32: + return "DescribeConfigs" + case 33: + return "AlterConfigs" + case 34: + return "AlterReplicaLogDirs" + case 35: + return "DescribeLogDirs" + case 36: + return "SASLAuthenticate" + case 37: + return "CreatePartitions" + case 38: + return "CreateDelegationToken" + case 39: + return "RenewDelegationToken" + case 40: + return "ExpireDelegationToken" + case 41: + return "DescribeDelegationToken" + case 42: + return "DeleteGroups" + case 43: + return "ElectLeaders" + case 44: + return "IncrementalAlterConfigs" + case 45: + return "AlterPartitionAssignments" + case 46: + return "ListPartitionReassignments" + case 47: + return "OffsetDelete" + case 48: + return "DescribeClientQuotas" + case 49: + return "AlterClientQuotas" + case 50: + return "DescribeUserSCRAMCredentials" + case 51: + return "AlterUserSCRAMCredentials" + case 52: + return "Vote" + case 53: + return "BeginQuorumEpoch" + case 54: + return "EndQuorumEpoch" + case 55: + return "DescribeQuorum" + case 56: + return "AlterPartition" + case 57: + return "UpdateFeatures" + case 58: + return "Envelope" + case 59: + return "FetchSnapshot" + case 60: + return "DescribeCluster" + case 61: + return "DescribeProducers" + case 62: + return "BrokerRegistration" + case 63: + return "BrokerHeartbeat" + case 64: + return "UnregisterBroker" + case 65: + return "DescribeTransactions" + case 66: + return "ListTransactions" + case 67: + return "AllocateProducerIDs" + case 68: + return "ConsumerGroupHeartbeat" + case 69: + return "ConsumerGroupDescribe" + case 70: + return "ControllerRegistration" + case 71: + return "GetTelemetrySubscriptions" + case 72: + return "PushTelemetry" + case 73: + return "AssignReplicasToDirs" + case 74: + return "ListConfigResources" + case 75: + return "DescribeTopicPartitions" + case 76: + return "ShareGroupHeartbeat" + case 77: + return "ShareGroupDescribe" + case 78: + return "ShareFetch" + case 79: + return "ShareAcknowledge" + case 80: + return "AddRaftVoter" + case 81: + return "RemoveRaftVoter" + case 82: + return "UpdateRaftVoter" + case 83: + return "InitializeShareGroupState" + case 84: + return "ReadShareGroupState" + case 85: + return "WriteShareGroupState" + case 86: + return "DeleteShareGroupState" + case 87: + return "ReadShareGroupStateSummary" + case 88: + return "StreamsGroupHeartbeat" + case 89: + return "StreamsGroupDescribe" + case 90: + return "DescribeShareGroupOffsets" + case 91: + return "AlterShareGroupOffsets" + case 92: + return "DeleteShareGroupOffsets" + } +} + +// Key is a typed representation of a request key, with helper functions. +type Key int16 + +const ( + Produce Key = 0 + Fetch Key = 1 + ListOffsets Key = 2 + Metadata Key = 3 + LeaderAndISR Key = 4 + StopReplica Key = 5 + UpdateMetadata Key = 6 + ControlledShutdown Key = 7 + OffsetCommit Key = 8 + OffsetFetch Key = 9 + FindCoordinator Key = 10 + JoinGroup Key = 11 + Heartbeat Key = 12 + LeaveGroup Key = 13 + SyncGroup Key = 14 + DescribeGroups Key = 15 + ListGroups Key = 16 + SASLHandshake Key = 17 + ApiVersions Key = 18 + CreateTopics Key = 19 + DeleteTopics Key = 20 + DeleteRecords Key = 21 + InitProducerID Key = 22 + OffsetForLeaderEpoch Key = 23 + AddPartitionsToTxn Key = 24 + AddOffsetsToTxn Key = 25 + EndTxn Key = 26 + WriteTxnMarkers Key = 27 + TxnOffsetCommit Key = 28 + DescribeACLs Key = 29 + CreateACLs Key = 30 + DeleteACLs Key = 31 + DescribeConfigs Key = 32 + AlterConfigs Key = 33 + AlterReplicaLogDirs Key = 34 + DescribeLogDirs Key = 35 + SASLAuthenticate Key = 36 + CreatePartitions Key = 37 + CreateDelegationToken Key = 38 + RenewDelegationToken Key = 39 + ExpireDelegationToken Key = 40 + DescribeDelegationToken Key = 41 + DeleteGroups Key = 42 + ElectLeaders Key = 43 + IncrementalAlterConfigs Key = 44 + AlterPartitionAssignments Key = 45 + ListPartitionReassignments Key = 46 + OffsetDelete Key = 47 + DescribeClientQuotas Key = 48 + AlterClientQuotas Key = 49 + DescribeUserSCRAMCredentials Key = 50 + AlterUserSCRAMCredentials Key = 51 + Vote Key = 52 + BeginQuorumEpoch Key = 53 + EndQuorumEpoch Key = 54 + DescribeQuorum Key = 55 + AlterPartition Key = 56 + UpdateFeatures Key = 57 + Envelope Key = 58 + FetchSnapshot Key = 59 + DescribeCluster Key = 60 + DescribeProducers Key = 61 + BrokerRegistration Key = 62 + BrokerHeartbeat Key = 63 + UnregisterBroker Key = 64 + DescribeTransactions Key = 65 + ListTransactions Key = 66 + AllocateProducerIDs Key = 67 + ConsumerGroupHeartbeat Key = 68 + ConsumerGroupDescribe Key = 69 + ControllerRegistration Key = 70 + GetTelemetrySubscriptions Key = 71 + PushTelemetry Key = 72 + AssignReplicasToDirs Key = 73 + ListConfigResources Key = 74 + DescribeTopicPartitions Key = 75 + ShareGroupHeartbeat Key = 76 + ShareGroupDescribe Key = 77 + ShareFetch Key = 78 + ShareAcknowledge Key = 79 + AddRaftVoter Key = 80 + RemoveRaftVoter Key = 81 + UpdateRaftVoter Key = 82 + InitializeShareGroupState Key = 83 + ReadShareGroupState Key = 84 + WriteShareGroupState Key = 85 + DeleteShareGroupState Key = 86 + ReadShareGroupStateSummary Key = 87 + StreamsGroupHeartbeat Key = 88 + StreamsGroupDescribe Key = 89 + DescribeShareGroupOffsets Key = 90 + AlterShareGroupOffsets Key = 91 + DeleteShareGroupOffsets Key = 92 +) + +// Name returns the name for this key. +func (k Key) Name() string { return NameForKey(int16(k)) } + +// Request returns a new request for this key if the key is known. +func (k Key) Request() Request { return RequestForKey(int16(k)) } + +// Response returns a new response for this key if the key is known. +func (k Key) Response() Response { return ResponseForKey(int16(k)) } + +// Int16 is an alias for int16(k). +func (k Key) Int16() int16 { return int16(k) } + +// A type of config. +// +// Possible values and their meanings: +// +// * 2 (TOPIC) +// +// * 4 (BROKER) +// +// * 8 (BROKER_LOGGER) +// +// * 16 (CLIENT_METRICS) +// +// * 32 (GROUP_CONFIG) +type ConfigResourceType int8 + +func (v ConfigResourceType) String() string { + switch v { + default: + return "UNKNOWN" + case 2: + return "TOPIC" + case 4: + return "BROKER" + case 8: + return "BROKER_LOGGER" + case 16: + return "CLIENT_METRICS" + case 32: + return "GROUP_CONFIG" + } +} + +func ConfigResourceTypeStrings() []string { + return []string{ + "TOPIC", + "BROKER", + "BROKER_LOGGER", + "CLIENT_METRICS", + "GROUP_CONFIG", + } +} + +// ParseConfigResourceType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseConfigResourceType(s string) (ConfigResourceType, error) { + switch strnorm(s) { + case "topic": + return 2, nil + case "broker": + return 4, nil + case "brokerlogger": + return 8, nil + case "clientmetrics": + return 16, nil + case "groupconfig": + return 32, nil + default: + return 0, fmt.Errorf("ConfigResourceType: unable to parse %q", s) + } +} + +const ( + ConfigResourceTypeUnknown ConfigResourceType = 0 + ConfigResourceTypeTopic ConfigResourceType = 2 + ConfigResourceTypeBroker ConfigResourceType = 4 + ConfigResourceTypeBrokerLogger ConfigResourceType = 8 + ConfigResourceTypeClientMetrics ConfigResourceType = 16 + ConfigResourceTypeGroupConfig ConfigResourceType = 32 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ConfigResourceType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ConfigResourceType) UnmarshalText(text []byte) error { + v, err := ParseConfigResourceType(string(text)) + *e = v + return err +} + +// Where a config entry is from. If there are no config synonyms, +// the source is DEFAULT_CONFIG. +// +// Possible values and their meanings: +// +// * 1 (DYNAMIC_TOPIC_CONFIG) +// Dynamic topic config for a specific topic. +// +// * 2 (DYNAMIC_BROKER_CONFIG) +// Dynamic broker config for a specific broker. +// +// * 3 (DYNAMIC_DEFAULT_BROKER_CONFIG) +// Dynamic broker config used as the default for all brokers in a cluster. +// +// * 4 (STATIC_BROKER_CONFIG) +// Static broker config provided at start up. +// +// * 5 (DEFAULT_CONFIG) +// Built-in default configuration for those that have defaults. +// +// * 6 (DYNAMIC_BROKER_LOGGER_CONFIG) +// Broker logger; see KIP-412. +// +// * 7 (CLIENT_METRICS_CONFIG) +// Client metrics; see KIP-714 / Kafka commit a53147e7d90. +// +// * 8 (GROUP_CONFIG) +// Group configs; see KAFKA-14511 / Kafka commit bbdf79e1b4f> +type ConfigSource int8 + +func (v ConfigSource) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "DYNAMIC_TOPIC_CONFIG" + case 2: + return "DYNAMIC_BROKER_CONFIG" + case 3: + return "DYNAMIC_DEFAULT_BROKER_CONFIG" + case 4: + return "STATIC_BROKER_CONFIG" + case 5: + return "DEFAULT_CONFIG" + case 6: + return "DYNAMIC_BROKER_LOGGER_CONFIG" + case 7: + return "CLIENT_METRICS_CONFIG" + case 8: + return "GROUP_CONFIG" + } +} + +func ConfigSourceStrings() []string { + return []string{ + "DYNAMIC_TOPIC_CONFIG", + "DYNAMIC_BROKER_CONFIG", + "DYNAMIC_DEFAULT_BROKER_CONFIG", + "STATIC_BROKER_CONFIG", + "DEFAULT_CONFIG", + "DYNAMIC_BROKER_LOGGER_CONFIG", + "CLIENT_METRICS_CONFIG", + "GROUP_CONFIG", + } +} + +// ParseConfigSource normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseConfigSource(s string) (ConfigSource, error) { + switch strnorm(s) { + case "dynamictopicconfig": + return 1, nil + case "dynamicbrokerconfig": + return 2, nil + case "dynamicdefaultbrokerconfig": + return 3, nil + case "staticbrokerconfig": + return 4, nil + case "defaultconfig": + return 5, nil + case "dynamicbrokerloggerconfig": + return 6, nil + case "clientmetricsconfig": + return 7, nil + case "groupconfig": + return 8, nil + default: + return 0, fmt.Errorf("ConfigSource: unable to parse %q", s) + } +} + +const ( + ConfigSourceUnknown ConfigSource = 0 + ConfigSourceDynamicTopicConfig ConfigSource = 1 + ConfigSourceDynamicBrokerConfig ConfigSource = 2 + ConfigSourceDynamicDefaultBrokerConfig ConfigSource = 3 + ConfigSourceStaticBrokerConfig ConfigSource = 4 + ConfigSourceDefaultConfig ConfigSource = 5 + ConfigSourceDynamicBrokerLoggerConfig ConfigSource = 6 + ConfigSourceClientMetricsConfig ConfigSource = 7 + ConfigSourceGroupConfig ConfigSource = 8 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ConfigSource) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ConfigSource) UnmarshalText(text []byte) error { + v, err := ParseConfigSource(string(text)) + *e = v + return err +} + +// A configuration data type. +// +// Possible values and their meanings: +// +// * 1 (BOOLEAN) +// +// * 2 (STRING) +// +// * 3 (INT) +// +// * 4 (SHORT) +// +// * 5 (LONG) +// +// * 6 (DOUBLE) +// +// * 7 (LIST) +// +// * 8 (CLASS) +// +// * 9 (PASSWORD) +type ConfigType int8 + +func (v ConfigType) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "BOOLEAN" + case 2: + return "STRING" + case 3: + return "INT" + case 4: + return "SHORT" + case 5: + return "LONG" + case 6: + return "DOUBLE" + case 7: + return "LIST" + case 8: + return "CLASS" + case 9: + return "PASSWORD" + } +} + +func ConfigTypeStrings() []string { + return []string{ + "BOOLEAN", + "STRING", + "INT", + "SHORT", + "LONG", + "DOUBLE", + "LIST", + "CLASS", + "PASSWORD", + } +} + +// ParseConfigType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseConfigType(s string) (ConfigType, error) { + switch strnorm(s) { + case "boolean": + return 1, nil + case "string": + return 2, nil + case "int": + return 3, nil + case "short": + return 4, nil + case "long": + return 5, nil + case "double": + return 6, nil + case "list": + return 7, nil + case "class": + return 8, nil + case "password": + return 9, nil + default: + return 0, fmt.Errorf("ConfigType: unable to parse %q", s) + } +} + +const ( + ConfigTypeUnknown ConfigType = 0 + ConfigTypeBoolean ConfigType = 1 + ConfigTypeString ConfigType = 2 + ConfigTypeInt ConfigType = 3 + ConfigTypeShort ConfigType = 4 + ConfigTypeLong ConfigType = 5 + ConfigTypeDouble ConfigType = 6 + ConfigTypeList ConfigType = 7 + ConfigTypeClass ConfigType = 8 + ConfigTypePassword ConfigType = 9 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ConfigType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ConfigType) UnmarshalText(text []byte) error { + v, err := ParseConfigType(string(text)) + *e = v + return err +} + +// An incremental configuration operation. +// +// Possible values and their meanings: +// +// * 0 (SET) +// +// * 1 (DELETE) +// +// * 2 (APPEND) +// +// * 3 (SUBTRACT) +type IncrementalAlterConfigOp int8 + +func (v IncrementalAlterConfigOp) String() string { + switch v { + default: + return "UNKNOWN" + case 0: + return "SET" + case 1: + return "DELETE" + case 2: + return "APPEND" + case 3: + return "SUBTRACT" + } +} + +func IncrementalAlterConfigOpStrings() []string { + return []string{ + "SET", + "DELETE", + "APPEND", + "SUBTRACT", + } +} + +// ParseIncrementalAlterConfigOp normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseIncrementalAlterConfigOp(s string) (IncrementalAlterConfigOp, error) { + switch strnorm(s) { + case "set": + return 0, nil + case "delete": + return 1, nil + case "append": + return 2, nil + case "subtract": + return 3, nil + default: + return 0, fmt.Errorf("IncrementalAlterConfigOp: unable to parse %q", s) + } +} + +const ( + IncrementalAlterConfigOpSet IncrementalAlterConfigOp = 0 + IncrementalAlterConfigOpDelete IncrementalAlterConfigOp = 1 + IncrementalAlterConfigOpAppend IncrementalAlterConfigOp = 2 + IncrementalAlterConfigOpSubtract IncrementalAlterConfigOp = 3 +) + +// MarshalText implements encoding.TextMarshaler. +func (e IncrementalAlterConfigOp) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *IncrementalAlterConfigOp) UnmarshalText(text []byte) error { + v, err := ParseIncrementalAlterConfigOp(string(text)) + *e = v + return err +} + +// ACLResourceType is a type of resource to use for ACLs. +// +// Possible values and their meanings: +// +// * 1 (ANY) +// +// * 2 (TOPIC) +// +// * 3 (GROUP) +// +// * 4 (CLUSTER) +// +// * 5 (TRANSACTIONAL_ID) +// +// * 6 (DELEGATION_TOKEN) +// +// * 7 (USER) +type ACLResourceType int8 + +func (v ACLResourceType) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "ANY" + case 2: + return "TOPIC" + case 3: + return "GROUP" + case 4: + return "CLUSTER" + case 5: + return "TRANSACTIONAL_ID" + case 6: + return "DELEGATION_TOKEN" + case 7: + return "USER" + } +} + +func ACLResourceTypeStrings() []string { + return []string{ + "ANY", + "TOPIC", + "GROUP", + "CLUSTER", + "TRANSACTIONAL_ID", + "DELEGATION_TOKEN", + "USER", + } +} + +// ParseACLResourceType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseACLResourceType(s string) (ACLResourceType, error) { + switch strnorm(s) { + case "any": + return 1, nil + case "topic": + return 2, nil + case "group": + return 3, nil + case "cluster": + return 4, nil + case "transactionalid": + return 5, nil + case "delegationtoken": + return 6, nil + case "user": + return 7, nil + default: + return 0, fmt.Errorf("ACLResourceType: unable to parse %q", s) + } +} + +const ( + ACLResourceTypeUnknown ACLResourceType = 0 + ACLResourceTypeAny ACLResourceType = 1 + ACLResourceTypeTopic ACLResourceType = 2 + ACLResourceTypeGroup ACLResourceType = 3 + ACLResourceTypeCluster ACLResourceType = 4 + ACLResourceTypeTransactionalId ACLResourceType = 5 + ACLResourceTypeDelegationToken ACLResourceType = 6 + ACLResourceTypeUser ACLResourceType = 7 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ACLResourceType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ACLResourceType) UnmarshalText(text []byte) error { + v, err := ParseACLResourceType(string(text)) + *e = v + return err +} + +// ACLResourcePatternType is how an acl's ResourceName is understood. +// +// This field was added with Kafka 2.0.0 for KIP-290. +// +// Possible values and their meanings: +// +// * 1 (ANY) +// Matches anything. +// +// * 2 (MATCH) +// Performs pattern matching; i.e., a literal match, or a prefix match, or wildcard. +// +// * 3 (LITERAL) +// The name must be an exact match. +// +// * 4 (PREFIXED) +// The name must have our requested name as a prefix (that is, "foo" will match on "foobar"). +type ACLResourcePatternType int8 + +func (v ACLResourcePatternType) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "ANY" + case 2: + return "MATCH" + case 3: + return "LITERAL" + case 4: + return "PREFIXED" + } +} + +func ACLResourcePatternTypeStrings() []string { + return []string{ + "ANY", + "MATCH", + "LITERAL", + "PREFIXED", + } +} + +// ParseACLResourcePatternType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseACLResourcePatternType(s string) (ACLResourcePatternType, error) { + switch strnorm(s) { + case "any": + return 1, nil + case "match": + return 2, nil + case "literal": + return 3, nil + case "prefixed": + return 4, nil + default: + return 0, fmt.Errorf("ACLResourcePatternType: unable to parse %q", s) + } +} + +const ( + ACLResourcePatternTypeUnknown ACLResourcePatternType = 0 + ACLResourcePatternTypeAny ACLResourcePatternType = 1 + ACLResourcePatternTypeMatch ACLResourcePatternType = 2 + ACLResourcePatternTypeLiteral ACLResourcePatternType = 3 + ACLResourcePatternTypePrefixed ACLResourcePatternType = 4 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ACLResourcePatternType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ACLResourcePatternType) UnmarshalText(text []byte) error { + v, err := ParseACLResourcePatternType(string(text)) + *e = v + return err +} + +// An ACL permission type. +// +// Possible values and their meanings: +// +// * 1 (ANY) +// Any permission. +// +// * 2 (DENY) +// Any deny permission. +// +// * 3 (ALLOW) +// Any allow permission. +type ACLPermissionType int8 + +func (v ACLPermissionType) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "ANY" + case 2: + return "DENY" + case 3: + return "ALLOW" + } +} + +func ACLPermissionTypeStrings() []string { + return []string{ + "ANY", + "DENY", + "ALLOW", + } +} + +// ParseACLPermissionType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseACLPermissionType(s string) (ACLPermissionType, error) { + switch strnorm(s) { + case "any": + return 1, nil + case "deny": + return 2, nil + case "allow": + return 3, nil + default: + return 0, fmt.Errorf("ACLPermissionType: unable to parse %q", s) + } +} + +const ( + ACLPermissionTypeUnknown ACLPermissionType = 0 + ACLPermissionTypeAny ACLPermissionType = 1 + ACLPermissionTypeDeny ACLPermissionType = 2 + ACLPermissionTypeAllow ACLPermissionType = 3 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ACLPermissionType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ACLPermissionType) UnmarshalText(text []byte) error { + v, err := ParseACLPermissionType(string(text)) + *e = v + return err +} + +// An ACL operation. +// +// Possible values and their meanings: +// +// * 1 (ANY) +// Matches anything. +// +// * 2 (ALL) +// Matches anything granted all permissions. +// +// * 3 (READ) +// +// * 4 (WRITE) +// +// * 5 (CREATE) +// +// * 6 (DELETE) +// +// * 7 (ALTER) +// +// * 8 (DESCRIBE) +// +// * 9 (CLUSTER_ACTION) +// +// * 10 (DESCRIBE_CONFIGS) +// +// * 11 (ALTER_CONFIGS) +// +// * 12 (IDEMPOTENT_WRITE) +// +// * 13 (CREATE_TOKENS) +// +// * 14 (DESCRIBE_TOKENS) +type ACLOperation int8 + +func (v ACLOperation) String() string { + switch v { + default: + return "UNKNOWN" + case 1: + return "ANY" + case 2: + return "ALL" + case 3: + return "READ" + case 4: + return "WRITE" + case 5: + return "CREATE" + case 6: + return "DELETE" + case 7: + return "ALTER" + case 8: + return "DESCRIBE" + case 9: + return "CLUSTER_ACTION" + case 10: + return "DESCRIBE_CONFIGS" + case 11: + return "ALTER_CONFIGS" + case 12: + return "IDEMPOTENT_WRITE" + case 13: + return "CREATE_TOKENS" + case 14: + return "DESCRIBE_TOKENS" + } +} + +func ACLOperationStrings() []string { + return []string{ + "ANY", + "ALL", + "READ", + "WRITE", + "CREATE", + "DELETE", + "ALTER", + "DESCRIBE", + "CLUSTER_ACTION", + "DESCRIBE_CONFIGS", + "ALTER_CONFIGS", + "IDEMPOTENT_WRITE", + "CREATE_TOKENS", + "DESCRIBE_TOKENS", + } +} + +// ParseACLOperation normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseACLOperation(s string) (ACLOperation, error) { + switch strnorm(s) { + case "any": + return 1, nil + case "all": + return 2, nil + case "read": + return 3, nil + case "write": + return 4, nil + case "create": + return 5, nil + case "delete": + return 6, nil + case "alter": + return 7, nil + case "describe": + return 8, nil + case "clusteraction": + return 9, nil + case "describeconfigs": + return 10, nil + case "alterconfigs": + return 11, nil + case "idempotentwrite": + return 12, nil + case "createtokens": + return 13, nil + case "describetokens": + return 14, nil + default: + return 0, fmt.Errorf("ACLOperation: unable to parse %q", s) + } +} + +const ( + ACLOperationUnknown ACLOperation = 0 + ACLOperationAny ACLOperation = 1 + ACLOperationAll ACLOperation = 2 + ACLOperationRead ACLOperation = 3 + ACLOperationWrite ACLOperation = 4 + ACLOperationCreate ACLOperation = 5 + ACLOperationDelete ACLOperation = 6 + ACLOperationAlter ACLOperation = 7 + ACLOperationDescribe ACLOperation = 8 + ACLOperationClusterAction ACLOperation = 9 + ACLOperationDescribeConfigs ACLOperation = 10 + ACLOperationAlterConfigs ACLOperation = 11 + ACLOperationIdempotentWrite ACLOperation = 12 + ACLOperationCreateTokens ACLOperation = 13 + ACLOperationDescribeTokens ACLOperation = 14 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ACLOperation) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ACLOperation) UnmarshalText(text []byte) error { + v, err := ParseACLOperation(string(text)) + *e = v + return err +} + +// TransactionState is the state of a transaction. +// +// Possible values and their meanings: +// +// * 0 (Empty) +// +// * 1 (Ongoing) +// +// * 2 (PrepareCommit) +// +// * 3 (PrepareAbort) +// +// * 4 (CompleteCommit) +// +// * 5 (CompleteAbort) +// +// * 6 (Dead) +// +// * 7 (PrepareEpochFence) +type TransactionState int8 + +func (v TransactionState) String() string { + switch v { + default: + return "Unknown" + case 0: + return "Empty" + case 1: + return "Ongoing" + case 2: + return "PrepareCommit" + case 3: + return "PrepareAbort" + case 4: + return "CompleteCommit" + case 5: + return "CompleteAbort" + case 6: + return "Dead" + case 7: + return "PrepareEpochFence" + } +} + +func TransactionStateStrings() []string { + return []string{ + "Empty", + "Ongoing", + "PrepareCommit", + "PrepareAbort", + "CompleteCommit", + "CompleteAbort", + "Dead", + "PrepareEpochFence", + } +} + +// ParseTransactionState normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseTransactionState(s string) (TransactionState, error) { + switch strnorm(s) { + case "empty": + return 0, nil + case "ongoing": + return 1, nil + case "preparecommit": + return 2, nil + case "prepareabort": + return 3, nil + case "completecommit": + return 4, nil + case "completeabort": + return 5, nil + case "dead": + return 6, nil + case "prepareepochfence": + return 7, nil + default: + return 0, fmt.Errorf("TransactionState: unable to parse %q", s) + } +} + +const ( + TransactionStateEmpty TransactionState = 0 + TransactionStateOngoing TransactionState = 1 + TransactionStatePrepareCommit TransactionState = 2 + TransactionStatePrepareAbort TransactionState = 3 + TransactionStateCompleteCommit TransactionState = 4 + TransactionStateCompleteAbort TransactionState = 5 + TransactionStateDead TransactionState = 6 + TransactionStatePrepareEpochFence TransactionState = 7 +) + +// MarshalText implements encoding.TextMarshaler. +func (e TransactionState) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *TransactionState) UnmarshalText(text []byte) error { + v, err := ParseTransactionState(string(text)) + *e = v + return err +} + +// QuotasMatchType specifies how to match a Quota entity as part of the DescribeClientQuotasRequestComponent. +// +// Possible values and their meanings: +// +// * 0 (EXACT) +// Matches all quotas for the given EntityType with names equal to the Match field. +// +// * 1 (DEFAULT) +// Matches the default for the given EntityType. +// +// * 2 (ANY) +// Matches all named quotas and default quotas for the given EntityType. +type QuotasMatchType int8 + +func (v QuotasMatchType) String() string { + switch v { + default: + return "UNKNOWN" + case 0: + return "EXACT" + case 1: + return "DEFAULT" + case 2: + return "ANY" + } +} + +func QuotasMatchTypeStrings() []string { + return []string{ + "EXACT", + "DEFAULT", + "ANY", + } +} + +// ParseQuotasMatchType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseQuotasMatchType(s string) (QuotasMatchType, error) { + switch strnorm(s) { + case "exact": + return 0, nil + case "default": + return 1, nil + case "any": + return 2, nil + default: + return 0, fmt.Errorf("QuotasMatchType: unable to parse %q", s) + } +} + +const ( + QuotasMatchTypeExact QuotasMatchType = 0 + QuotasMatchTypeDefault QuotasMatchType = 1 + QuotasMatchTypeAny QuotasMatchType = 2 +) + +// MarshalText implements encoding.TextMarshaler. +func (e QuotasMatchType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *QuotasMatchType) UnmarshalText(text []byte) error { + v, err := ParseQuotasMatchType(string(text)) + *e = v + return err +} + +// Possible values and their meanings: +// +// * 0 (ABORT) +// +// * 1 (COMMIT) +// +// * 2 (QUORUM_REASSIGNMENT) +// QUORUM_REASSIGNMENT was renamed to LEADER_CHANGE in Kafka 3.0 +// (KAFKA-12952 commit d3ec9f940c), but the meaning is the same. +// +// * 3 (SNAPSHOT_HEADER) +// +// * 4 (SNAPSHOT_FOOTER) +// +// * 5 (KRAFT_VERSION) +// +// * 6 (KRAFT_VOTERS) +type ControlRecordKeyType int16 + +func (v ControlRecordKeyType) String() string { + switch v { + default: + return "UNKNOWN" + case 0: + return "ABORT" + case 1: + return "COMMIT" + case 2: + return "QUORUM_REASSIGNMENT" + case 3: + return "SNAPSHOT_HEADER" + case 4: + return "SNAPSHOT_FOOTER" + case 5: + return "KRAFT_VERSION" + case 6: + return "KRAFT_VOTERS" + } +} + +func ControlRecordKeyTypeStrings() []string { + return []string{ + "ABORT", + "COMMIT", + "QUORUM_REASSIGNMENT", + "SNAPSHOT_HEADER", + "SNAPSHOT_FOOTER", + "KRAFT_VERSION", + "KRAFT_VOTERS", + } +} + +// ParseControlRecordKeyType normalizes the input s and returns +// the value represented by the string. +// +// Normalizing works by stripping all dots, underscores, and dashes, +// trimming spaces, and lowercasing. +func ParseControlRecordKeyType(s string) (ControlRecordKeyType, error) { + switch strnorm(s) { + case "abort": + return 0, nil + case "commit": + return 1, nil + case "quorumreassignment": + return 2, nil + case "snapshotheader": + return 3, nil + case "snapshotfooter": + return 4, nil + case "kraftversion": + return 5, nil + case "kraftvoters": + return 6, nil + default: + return 0, fmt.Errorf("ControlRecordKeyType: unable to parse %q", s) + } +} + +const ( + ControlRecordKeyTypeAbort ControlRecordKeyType = 0 + ControlRecordKeyTypeCommit ControlRecordKeyType = 1 + ControlRecordKeyTypeQuorumReassignment ControlRecordKeyType = 2 + ControlRecordKeyTypeSnapshotHeader ControlRecordKeyType = 3 + ControlRecordKeyTypeSnapshotFooter ControlRecordKeyType = 4 + ControlRecordKeyTypeKraftVersion ControlRecordKeyType = 5 + ControlRecordKeyTypeKraftVoters ControlRecordKeyType = 6 +) + +// MarshalText implements encoding.TextMarshaler. +func (e ControlRecordKeyType) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (e *ControlRecordKeyType) UnmarshalText(text []byte) error { + v, err := ParseControlRecordKeyType(string(text)) + *e = v + return err +} + +func strnorm(s string) string { + s = strings.ReplaceAll(s, ".", "") + s = strings.ReplaceAll(s, "_", "") + s = strings.ReplaceAll(s, "-", "") + s = strings.TrimSpace(s) + s = strings.ToLower(s) + return s +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kmsg/internal/kbin/primitives.go b/vendor/github.com/twmb/franz-go/pkg/kmsg/internal/kbin/primitives.go new file mode 100644 index 00000000000..487e7f6c2a3 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kmsg/internal/kbin/primitives.go @@ -0,0 +1,856 @@ +// Package kbin contains Kafka primitive reading and writing functions. +package kbin + +import ( + "encoding/binary" + "errors" + "math" + "math/bits" + "reflect" + "unsafe" +) + +// This file contains primitive type encoding and decoding. +// +// The Reader helper can be used even when content runs out +// or an error is hit; all other number requests will return +// zero so a decode will basically no-op. + +// ErrNotEnoughData is returned when a type could not fully decode +// from a slice because the slice did not have enough data. +var ErrNotEnoughData = errors.New("response did not contain enough data to be valid") + +// AppendBool appends 1 for true or 0 for false to dst. +func AppendBool(dst []byte, v bool) []byte { + if v { + return append(dst, 1) + } + return append(dst, 0) +} + +// AppendInt8 appends an int8 to dst. +func AppendInt8(dst []byte, i int8) []byte { + return append(dst, byte(i)) +} + +// AppendInt16 appends a big endian int16 to dst. +func AppendInt16(dst []byte, i int16) []byte { + return AppendUint16(dst, uint16(i)) +} + +// AppendUint16 appends a big endian uint16 to dst. +func AppendUint16(dst []byte, u uint16) []byte { + return append(dst, byte(u>>8), byte(u)) +} + +// AppendInt32 appends a big endian int32 to dst. +func AppendInt32(dst []byte, i int32) []byte { + return AppendUint32(dst, uint32(i)) +} + +// AppendInt64 appends a big endian int64 to dst. +func AppendInt64(dst []byte, i int64) []byte { + return appendUint64(dst, uint64(i)) +} + +// AppendFloat64 appends a big endian float64 to dst. +func AppendFloat64(dst []byte, f float64) []byte { + return appendUint64(dst, math.Float64bits(f)) +} + +// AppendUuid appends the 16 uuid bytes to dst. +func AppendUuid(dst []byte, uuid [16]byte) []byte { + return append(dst, uuid[:]...) +} + +func appendUint64(dst []byte, u uint64) []byte { + return append(dst, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), + byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +// AppendUint32 appends a big endian uint32 to dst. +func AppendUint32(dst []byte, u uint32) []byte { + return append(dst, byte(u>>24), byte(u>>16), byte(u>>8), byte(u)) +} + +// uvarintLens could only be length 65, but using 256 allows bounds check +// elimination on lookup. +const uvarintLens = "\x01\x01\x01\x01\x01\x01\x01\x01\x02\x02\x02\x02\x02\x02\x02\x03\x03\x03\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04\x04\x05\x05\x05\x05\x05\x05\x05\x06\x06\x06\x06\x06\x06\x06\x07\x07\x07\x07\x07\x07\x07\x08\x08\x08\x08\x08\x08\x08\x09\x09\x09\x09\x09\x09\x09\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +// VarintLen returns how long i would be if it were varint encoded. +func VarintLen(i int32) int { + u := uint32(i)<<1 ^ uint32(i>>31) + return UvarintLen(u) +} + +// UvarintLen returns how long u would be if it were uvarint encoded. +func UvarintLen(u uint32) int { + return int(uvarintLens[byte(bits.Len32(u))]) +} + +// VarlongLen returns how long i would be if it were varlong encoded. +func VarlongLen(i int64) int { + u := uint64(i)<<1 ^ uint64(i>>63) + return uvarlongLen(u) +} + +func uvarlongLen(u uint64) int { + return int(uvarintLens[byte(bits.Len64(u))]) +} + +// Varint is a loop unrolled 32 bit varint decoder. The return semantics +// are the same as binary.Varint, with the added benefit that overflows +// in 5 byte encodings are handled rather than left to the user. +func Varint(in []byte) (int32, int) { + x, n := Uvarint(in) + return int32((x >> 1) ^ -(x & 1)), n +} + +// Uvarint is a loop unrolled 32 bit uvarint decoder. The return semantics +// are the same as binary.Uvarint, with the added benefit that overflows +// in 5 byte encodings are handled rather than left to the user. +func Uvarint(in []byte) (uint32, int) { + var x uint32 + var overflow int + + if len(in) < 1 { + goto fail + } + + x = uint32(in[0] & 0x7f) + if in[0]&0x80 == 0 { + return x, 1 + } else if len(in) < 2 { + goto fail + } + + x |= uint32(in[1]&0x7f) << 7 + if in[1]&0x80 == 0 { + return x, 2 + } else if len(in) < 3 { + goto fail + } + + x |= uint32(in[2]&0x7f) << 14 + if in[2]&0x80 == 0 { + return x, 3 + } else if len(in) < 4 { + goto fail + } + + x |= uint32(in[3]&0x7f) << 21 + if in[3]&0x80 == 0 { + return x, 4 + } else if len(in) < 5 { + goto fail + } + + x |= uint32(in[4]) << 28 + if in[4] <= 0x0f { + return x, 5 + } + + overflow = -5 + +fail: + return 0, overflow +} + +// Varlong is a loop unrolled 64 bit varint decoder. The return semantics +// are the same as binary.Varint, with the added benefit that overflows +// in 10 byte encodings are handled rather than left to the user. +func Varlong(in []byte) (int64, int) { + x, n := uvarlong(in) + return int64((x >> 1) ^ -(x & 1)), n +} + +func uvarlong(in []byte) (uint64, int) { + var x uint64 + var overflow int + + if len(in) < 1 { + goto fail + } + + x = uint64(in[0] & 0x7f) + if in[0]&0x80 == 0 { + return x, 1 + } else if len(in) < 2 { + goto fail + } + + x |= uint64(in[1]&0x7f) << 7 + if in[1]&0x80 == 0 { + return x, 2 + } else if len(in) < 3 { + goto fail + } + + x |= uint64(in[2]&0x7f) << 14 + if in[2]&0x80 == 0 { + return x, 3 + } else if len(in) < 4 { + goto fail + } + + x |= uint64(in[3]&0x7f) << 21 + if in[3]&0x80 == 0 { + return x, 4 + } else if len(in) < 5 { + goto fail + } + + x |= uint64(in[4]&0x7f) << 28 + if in[4]&0x80 == 0 { + return x, 5 + } else if len(in) < 6 { + goto fail + } + + x |= uint64(in[5]&0x7f) << 35 + if in[5]&0x80 == 0 { + return x, 6 + } else if len(in) < 7 { + goto fail + } + + x |= uint64(in[6]&0x7f) << 42 + if in[6]&0x80 == 0 { + return x, 7 + } else if len(in) < 8 { + goto fail + } + + x |= uint64(in[7]&0x7f) << 49 + if in[7]&0x80 == 0 { + return x, 8 + } else if len(in) < 9 { + goto fail + } + + x |= uint64(in[8]&0x7f) << 56 + if in[8]&0x80 == 0 { + return x, 9 + } else if len(in) < 10 { + goto fail + } + + x |= uint64(in[9]) << 63 + if in[9] <= 0x01 { + return x, 10 + } + + overflow = -10 + +fail: + return 0, overflow +} + +// AppendVarint appends a varint encoded i to dst. +func AppendVarint(dst []byte, i int32) []byte { + return AppendUvarint(dst, uint32(i)<<1^uint32(i>>31)) +} + +// AppendUvarint appends a uvarint encoded u to dst. +func AppendUvarint(dst []byte, u uint32) []byte { + switch UvarintLen(u) { + case 5: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte(u>>28)) + case 4: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte(u>>21)) + case 3: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte(u>>14)) + case 2: + return append(dst, + byte(u&0x7f|0x80), + byte(u>>7)) + case 1: + return append(dst, byte(u)) + } + return dst +} + +// AppendVarlong appends a varint encoded i to dst. +func AppendVarlong(dst []byte, i int64) []byte { + return appendUvarlong(dst, uint64(i)<<1^uint64(i>>63)) +} + +func appendUvarlong(dst []byte, u uint64) []byte { + switch uvarlongLen(u) { + case 10: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte((u>>49)&0x7f|0x80), + byte((u>>56)&0x7f|0x80), + byte(u>>63)) + case 9: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte((u>>49)&0x7f|0x80), + byte(u>>56)) + case 8: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte((u>>42)&0x7f|0x80), + byte(u>>49)) + case 7: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte((u>>35)&0x7f|0x80), + byte(u>>42)) + case 6: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte((u>>28)&0x7f|0x80), + byte(u>>35)) + case 5: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte((u>>21)&0x7f|0x80), + byte(u>>28)) + case 4: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte((u>>14)&0x7f|0x80), + byte(u>>21)) + case 3: + return append(dst, + byte(u&0x7f|0x80), + byte((u>>7)&0x7f|0x80), + byte(u>>14)) + case 2: + return append(dst, + byte(u&0x7f|0x80), + byte(u>>7)) + case 1: + return append(dst, byte(u)) + } + return dst +} + +// AppendString appends a string to dst prefixed with its int16 length. +func AppendString(dst []byte, s string) []byte { + dst = AppendInt16(dst, int16(len(s))) + return append(dst, s...) +} + +// AppendCompactString appends a string to dst prefixed with its uvarint length +// starting at 1; 0 is reserved for null, which compact strings are not +// (nullable compact ones are!). Thus, the length is the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactString(dst []byte, s string) []byte { + dst = AppendUvarint(dst, 1+uint32(len(s))) + return append(dst, s...) +} + +// AppendNullableString appends potentially nil string to dst prefixed with its +// int16 length or int16(-1) if nil. +func AppendNullableString(dst []byte, s *string) []byte { + if s == nil { + return AppendInt16(dst, -1) + } + return AppendString(dst, *s) +} + +// AppendCompactNullableString appends a potentially nil string to dst with its +// uvarint length starting at 1, with 0 indicating null. Thus, the length is +// the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactNullableString(dst []byte, s *string) []byte { + if s == nil { + return AppendUvarint(dst, 0) + } + return AppendCompactString(dst, *s) +} + +// AppendBytes appends bytes to dst prefixed with its int32 length. +func AppendBytes(dst, b []byte) []byte { + dst = AppendInt32(dst, int32(len(b))) + return append(dst, b...) +} + +// AppendCompactBytes appends bytes to dst prefixed with a its uvarint length +// starting at 1; 0 is reserved for null, which compact bytes are not (nullable +// compact ones are!). Thus, the length is the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactBytes(dst, b []byte) []byte { + dst = AppendUvarint(dst, 1+uint32(len(b))) + return append(dst, b...) +} + +// AppendNullableBytes appends a potentially nil slice to dst prefixed with its +// int32 length or int32(-1) if nil. +func AppendNullableBytes(dst, b []byte) []byte { + if b == nil { + return AppendInt32(dst, -1) + } + return AppendBytes(dst, b) +} + +// AppendCompactNullableBytes appends a potentially nil slice to dst with its +// uvarint length starting at 1, with 0 indicating null. Thus, the length is +// the decoded uvarint - 1. +// +// For KIP-482. +func AppendCompactNullableBytes(dst, b []byte) []byte { + if b == nil { + return AppendUvarint(dst, 0) + } + return AppendCompactBytes(dst, b) +} + +// AppendVarintString appends a string to dst prefixed with its length encoded +// as a varint. +func AppendVarintString(dst []byte, s string) []byte { + dst = AppendVarint(dst, int32(len(s))) + return append(dst, s...) +} + +// AppendVarintBytes appends a slice to dst prefixed with its length encoded as +// a varint. +func AppendVarintBytes(dst, b []byte) []byte { + if b == nil { + return AppendVarint(dst, -1) + } + dst = AppendVarint(dst, int32(len(b))) + return append(dst, b...) +} + +// AppendArrayLen appends the length of an array as an int32 to dst. +func AppendArrayLen(dst []byte, l int) []byte { + return AppendInt32(dst, int32(l)) +} + +// AppendCompactArrayLen appends the length of an array as a uvarint to dst +// as the length + 1. +// +// For KIP-482. +func AppendCompactArrayLen(dst []byte, l int) []byte { + return AppendUvarint(dst, 1+uint32(l)) +} + +// AppendNullableArrayLen appends the length of an array as an int32 to dst, +// or -1 if isNil is true. +func AppendNullableArrayLen(dst []byte, l int, isNil bool) []byte { + if isNil { + return AppendInt32(dst, -1) + } + return AppendInt32(dst, int32(l)) +} + +// AppendCompactNullableArrayLen appends the length of an array as a uvarint to +// dst as the length + 1; if isNil is true, this appends 0 as a uvarint. +// +// For KIP-482. +func AppendCompactNullableArrayLen(dst []byte, l int, isNil bool) []byte { + if isNil { + return AppendUvarint(dst, 0) + } + return AppendUvarint(dst, 1+uint32(l)) +} + +// Reader is used to decode Kafka messages. +// +// For all functions on Reader, if the reader has been invalidated, functions +// return defaults (false, 0, nil, ""). Use Complete to detect if the reader +// was invalidated or if the reader has remaining data. +type Reader struct { + Src []byte + bad bool +} + +// Bool returns a bool from the reader. +func (b *Reader) Bool() bool { + if len(b.Src) < 1 { + b.bad = true + b.Src = nil + return false + } + t := b.Src[0] != 0 // if '0', false + b.Src = b.Src[1:] + return t +} + +// Int8 returns an int8 from the reader. +func (b *Reader) Int8() int8 { + if len(b.Src) < 1 { + b.bad = true + b.Src = nil + return 0 + } + r := b.Src[0] + b.Src = b.Src[1:] + return int8(r) +} + +// Int16 returns an int16 from the reader. +func (b *Reader) Int16() int16 { + if len(b.Src) < 2 { + b.bad = true + b.Src = nil + return 0 + } + r := int16(binary.BigEndian.Uint16(b.Src)) + b.Src = b.Src[2:] + return r +} + +// Uint16 returns an uint16 from the reader. +func (b *Reader) Uint16() uint16 { + if len(b.Src) < 2 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint16(b.Src) + b.Src = b.Src[2:] + return r +} + +// Int32 returns an int32 from the reader. +func (b *Reader) Int32() int32 { + if len(b.Src) < 4 { + b.bad = true + b.Src = nil + return 0 + } + r := int32(binary.BigEndian.Uint32(b.Src)) + b.Src = b.Src[4:] + return r +} + +// Int64 returns an int64 from the reader. +func (b *Reader) Int64() int64 { + return int64(b.readUint64()) +} + +// Uuid returns a uuid from the reader. +func (b *Reader) Uuid() [16]byte { + var r [16]byte + copy(r[:], b.Span(16)) + return r +} + +// Float64 returns a float64 from the reader. +func (b *Reader) Float64() float64 { + return math.Float64frombits(b.readUint64()) +} + +func (b *Reader) readUint64() uint64 { + if len(b.Src) < 8 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint64(b.Src) + b.Src = b.Src[8:] + return r +} + +// Uint32 returns a uint32 from the reader. +func (b *Reader) Uint32() uint32 { + if len(b.Src) < 4 { + b.bad = true + b.Src = nil + return 0 + } + r := binary.BigEndian.Uint32(b.Src) + b.Src = b.Src[4:] + return r +} + +// Varint returns a varint int32 from the reader. +func (b *Reader) Varint() int32 { + val, n := Varint(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Varlong returns a varlong int64 from the reader. +func (b *Reader) Varlong() int64 { + val, n := Varlong(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Uvarint returns a uvarint encoded uint32 from the reader. +func (b *Reader) Uvarint() uint32 { + val, n := Uvarint(b.Src) + if n <= 0 { + b.bad = true + b.Src = nil + return 0 + } + b.Src = b.Src[n:] + return val +} + +// Span returns l bytes from the reader. +func (b *Reader) Span(l int) []byte { + if len(b.Src) < l || l < 0 { + b.bad = true + b.Src = nil + return nil + } + r := b.Src[:l:l] + b.Src = b.Src[l:] + return r +} + +// UnsafeString returns a Kafka string from the reader without allocating using +// the unsafe package. This must be used with care; note the string holds a +// reference to the original slice. +func (b *Reader) UnsafeString() string { + l := b.Int16() + return UnsafeString(b.Span(int(l))) +} + +// String returns a Kafka string from the reader. +func (b *Reader) String() string { + l := b.Int16() + return string(b.Span(int(l))) +} + +// UnsafeCompactString returns a Kafka compact string from the reader without +// allocating using the unsafe package. This must be used with care; note the +// string holds a reference to the original slice. +func (b *Reader) UnsafeCompactString() string { + l := int(b.Uvarint()) - 1 + return UnsafeString(b.Span(l)) +} + +// CompactString returns a Kafka compact string from the reader. +func (b *Reader) CompactString() string { + l := int(b.Uvarint()) - 1 + return string(b.Span(l)) +} + +// UnsafeNullableString returns a Kafka nullable string from the reader without +// allocating using the unsafe package. This must be used with care; note the +// string holds a reference to the original slice. +func (b *Reader) UnsafeNullableString() *string { + l := b.Int16() + if l < 0 { + return nil + } + s := UnsafeString(b.Span(int(l))) + return &s +} + +// NullableString returns a Kafka nullable string from the reader. +func (b *Reader) NullableString() *string { + l := b.Int16() + if l < 0 { + return nil + } + s := string(b.Span(int(l))) + return &s +} + +// UnsafeCompactNullableString returns a Kafka compact nullable string from the +// reader without allocating using the unsafe package. This must be used with +// care; note the string holds a reference to the original slice. +func (b *Reader) UnsafeCompactNullableString() *string { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + s := UnsafeString(b.Span(l)) + return &s +} + +// CompactNullableString returns a Kafka compact nullable string from the +// reader. +func (b *Reader) CompactNullableString() *string { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + s := string(b.Span(l)) + return &s +} + +// Bytes returns a Kafka byte array from the reader. +// +// This never returns nil. +func (b *Reader) Bytes() []byte { + l := b.Int32() + // This is not to spec, but it is not clearly documented and Microsoft + // EventHubs fails here. -1 means null, which should throw an + // exception. EventHubs uses -1 to mean "does not exist" on some + // non-nullable fields. + // + // Until EventHubs is fixed, we return an empty byte slice for null. + if l == -1 { + return []byte{} + } + return b.Span(int(l)) +} + +// CompactBytes returns a Kafka compact byte array from the reader. +// +// This never returns nil. +func (b *Reader) CompactBytes() []byte { + l := int(b.Uvarint()) - 1 + if l == -1 { // same as above: -1 should not be allowed here + return []byte{} + } + return b.Span(l) +} + +// NullableBytes returns a Kafka nullable byte array from the reader, returning +// nil as appropriate. +func (b *Reader) NullableBytes() []byte { + l := b.Int32() + if l < 0 { + return nil + } + r := b.Span(int(l)) + return r +} + +// CompactNullableBytes returns a Kafka compact nullable byte array from the +// reader, returning nil as appropriate. +func (b *Reader) CompactNullableBytes() []byte { + l := int(b.Uvarint()) - 1 + if l < 0 { + return nil + } + r := b.Span(l) + return r +} + +// ArrayLen returns a Kafka array length from the reader. +func (b *Reader) ArrayLen() int32 { + r := b.Int32() + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// VarintArrayLen returns a Kafka array length from the reader. +func (b *Reader) VarintArrayLen() int32 { + r := b.Varint() + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// CompactArrayLen returns a Kafka compact array length from the reader. +func (b *Reader) CompactArrayLen() int32 { + r := int32(b.Uvarint()) - 1 + // The min size of a Kafka type is a byte, so if we do not have + // at least the array length of bytes left, it is bad. + if len(b.Src) < int(r) { + b.bad = true + b.Src = nil + return 0 + } + return r +} + +// VarintBytes returns a Kafka encoded varint array from the reader, returning +// nil as appropriate. +func (b *Reader) VarintBytes() []byte { + l := b.Varint() + if l < 0 { + return nil + } + return b.Span(int(l)) +} + +// UnsafeVarintString returns a Kafka encoded varint string from the reader +// without allocating using the unsafe package. This must be used with care; +// note the string holds a reference to the original slice. +func (b *Reader) UnsafeVarintString() string { + return UnsafeString(b.VarintBytes()) +} + +// VarintString returns a Kafka encoded varint string from the reader. +func (b *Reader) VarintString() string { + return string(b.VarintBytes()) +} + +// Complete returns ErrNotEnoughData if the source ran out while decoding. +func (b *Reader) Complete() error { + if b.bad { + return ErrNotEnoughData + } + return nil +} + +// Ok returns true if the reader is still ok. +func (b *Reader) Ok() bool { + return !b.bad +} + +// UnsafeString returns the slice as a string using unsafe rule (6). +func UnsafeString(slice []byte) string { + var str string + strhdr := (*reflect.StringHeader)(unsafe.Pointer(&str)) //nolint:gosec // known way to convert slice to string + strhdr.Data = ((*reflect.SliceHeader)(unsafe.Pointer(&slice))).Data //nolint:gosec // known way to convert slice to string + strhdr.Len = len(slice) + return str +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kmsg/record.go b/vendor/github.com/twmb/franz-go/pkg/kmsg/record.go new file mode 100644 index 00000000000..1108f24f7ad --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kmsg/record.go @@ -0,0 +1,173 @@ +package kmsg + +import "github.com/twmb/franz-go/pkg/kmsg/internal/kbin" + +// A Record is a Kafka v0.11.0.0 record. It corresponds to an individual +// message as it is written on the wire. +type Record struct { + // Length is the length of this record on the wire of everything that + // follows this field. It is an int32 encoded as a varint. + Length int32 + + // Attributes are record level attributes. This field currently is unused. + Attributes int8 + + // TimestampDelta is the millisecond delta of this record's timestamp + // from the record's RecordBatch's FirstTimestamp. + // + // NOTE: this is actually an int64 but we cannot change the type for + // backwards compatibility. Use TimestampDelta64. + TimestampDelta int32 + TimestampDelta64 int64 + + // OffsetDelta is the delta of this record's offset from the record's + // RecordBatch's FirstOffset. + // + // For producing, this is usually equal to the index of the record in + // the record batch. + OffsetDelta int32 + + // Key is an blob of data for a record. + // + // Key's are usually used for hashing the record to specific Kafka partitions. + Key []byte + + // Value is a blob of data. This field is the main "message" portion of a + // record. + Value []byte + + // Headers are optional user provided metadata for records. Unlike normal + // arrays, the number of headers is encoded as a varint. + Headers []Header +} + +func (v *Record) AppendTo(dst []byte) []byte { + { + v := v.Length + dst = kbin.AppendVarint(dst, v) + } + { + v := v.Attributes + dst = kbin.AppendInt8(dst, v) + } + { + d := v.TimestampDelta64 + if d == 0 { + d = int64(v.TimestampDelta) + } + dst = kbin.AppendVarlong(dst, d) + } + { + v := v.OffsetDelta + dst = kbin.AppendVarint(dst, v) + } + { + v := v.Key + dst = kbin.AppendVarintBytes(dst, v) + } + { + v := v.Value + dst = kbin.AppendVarintBytes(dst, v) + } + { + v := v.Headers + dst = kbin.AppendVarint(dst, int32(len(v))) + for i := range v { + v := &v[i] + { + v := v.Key + dst = kbin.AppendVarintString(dst, v) + } + { + v := v.Value + dst = kbin.AppendVarintBytes(dst, v) + } + } + } + return dst +} + +func (v *Record) ReadFrom(src []byte) error { + return v.readFrom(src, false) +} + +func (v *Record) UnsafeReadFrom(src []byte) error { + return v.readFrom(src, true) +} + +func (v *Record) readFrom(src []byte, unsafe bool) error { + v.Default() + b := kbin.Reader{Src: src} + s := v + { + v := b.Varint() + s.Length = v + } + { + v := b.Int8() + s.Attributes = v + } + { + v := b.Varlong() + s.TimestampDelta64 = v + s.TimestampDelta = int32(v) + } + { + v := b.Varint() + s.OffsetDelta = v + } + { + v := b.VarintBytes() + s.Key = v + } + { + v := b.VarintBytes() + s.Value = v + } + { + v := s.Headers + a := v + l := b.VarintArrayLen() + if !b.Ok() { + return b.Complete() + } + a = a[:0] + if l > 0 { + a = append(a, make([]Header, l)...) + } + for i := int32(0); i < l; i++ { + v := &a[i] + v.Default() + s := v + { + var v string + if unsafe { + v = b.UnsafeVarintString() + } else { + v = b.VarintString() + } + s.Key = v + } + { + v := b.VarintBytes() + s.Value = v + } + } + v = a + s.Headers = v + } + return b.Complete() +} + +// Default sets any default fields. Calling this allows for future compatibility +// if new fields are added to Record. +func (*Record) Default() { +} + +// NewRecord returns a default Record +// This is a shortcut for creating a struct and calling Default yourself. +func NewRecord() Record { + var v Record + v.Default() + return v +} diff --git a/vendor/github.com/twmb/franz-go/pkg/kversion/kversion.go b/vendor/github.com/twmb/franz-go/pkg/kversion/kversion.go new file mode 100644 index 00000000000..aa66a2572fc --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kversion/kversion.go @@ -0,0 +1,352 @@ +// Package kversion specifies versions for Kafka request keys. +// +// Kafka technically has internal broker versions that bump multiple times per +// release. This package only defines releases and tip. +package kversion + +import ( + "bytes" + "fmt" + "maps" + "regexp" + "slices" + "sync" + "text/tabwriter" + + "github.com/twmb/franz-go/pkg/kmsg" +) + +// Versions is a list of versions, with each item corresponding to a Kafka key +// and each item's value corresponding to the max version supported. +type Versions struct { + reqs map[int16]req +} + +func (vs *Versions) lazyInit() { + if vs.reqs == nil { + vs.reqs = make(map[int16]req) + } +} + +var ( + reFromString *regexp.Regexp + reFromStringOnce sync.Once +) + +// VersionStrings returns all recognized versions, minus any patch, that can be +// used as input to FromString. +func VersionStrings() []string { + var vs []string + b := btip() + for b != nil && b.major >= 4 { + vs = append(vs, b.name()) + b = b.prior + } + zk := ztip() + for zk != nil { + vs = append(vs, zk.name()) + zk = zk.prior + } + return vs +} + +// FromString returns a Versions from v. +// The expected input is: +// - for v0, v0.#.# or v0.#.#.# +// - for v1, v1.# or v1.#.# +// +// The "v" is optional. +func FromString(v string) *Versions { + reFromStringOnce.Do(func() { + // 0: entire string + // 1: v1+ match, minus patch + // 2: v0 match, minus subpatch + reFromString = regexp.MustCompile(`^(?:(v?[1-9]+\.\d+)(?:\.\d+)?|(v?0\.\d+\.\d+)(?:\.\d+)?)$`) + }) + m := reFromString.FindStringSubmatch(v) + if m == nil { + return nil + } + v = m[1] + if m[2] != "" { + v = m[2] + } + + withv := "v" + v + b := btip() + for b != nil && b.major >= 4 { + if n := b.name(); n == v || n == withv { + return &Versions{reqs: b.reqs} + } + b = b.prior + } + zk := ztip() + for zk != nil { + if n := zk.name(); n == v || n == withv { + return &Versions{reqs: zk.reqs} + } + zk = zk.prior + } + return nil +} + +// FromApiVersionsResponse returns a Versions from a kmsg.ApiVersionsResponse. +func FromApiVersionsResponse(r *kmsg.ApiVersionsResponse) *Versions { + return &Versions{reqs: reqsFromApiVersions(r)} +} + +// HasKey returns true if the versions contains the given key. +func (vs *Versions) HasKey(k int16) bool { + _, has := vs.LookupMaxKeyVersion(k) + return has +} + +// LookupMaxKeyVersion returns the version for the given key and whether the +// key exists. If the key does not exist, this returns (-1, false). +func (vs *Versions) LookupMaxKeyVersion(k int16) (int16, bool) { + vs.lazyInit() + req, ok := vs.reqs[k] + if !ok { + return -1, false + } + return req.vmax, true +} + +// SetMaxKeyVersion sets the max version for the given key. Setting a version +// to -1 removes the key entirely. +func (vs *Versions) SetMaxKeyVersion(k, v int16) { + vs.lazyInit() + if k < 0 || v < 0 { + delete(vs.reqs, k) + return + } + req := vs.reqs[k] + req.vmax = v + req.key = k // in case the key did not exist + vs.reqs[k] = req +} + +// Equal returns whether two versions are equal. +func (vs *Versions) Equal(other *Versions) bool { + vs.lazyInit() + mereqs := maps.Clone(vs.reqs) + for k, oreq := range other.reqs { + mreq, ok := mereqs[k] + if !ok || mreq != oreq { + return false + } + delete(mereqs, k) + } + return len(mereqs) == 0 +} + +// EachMaxKeyVersion calls fn for each key and max version +func (vs *Versions) EachMaxKeyVersion(fn func(k, v int16)) { + vs.lazyInit() + keys := make([]int16, 0, len(vs.reqs)) + for k := range vs.reqs { + keys = append(keys, k) + } + slices.Sort(keys) + for _, k := range keys { + fn(k, vs.reqs[k].vmax) + } +} + +// VersionGuessOpt is an option to change how version guessing is done. +type VersionGuessOpt interface { + apply(*guessCfg) +} + +type guessOpt struct{ fn func(*guessCfg) } + +func (opt guessOpt) apply(cfg *guessCfg) { opt.fn(cfg) } + +// SkipKeys skips the given keys while guessing versions. +func SkipKeys(keys ...int16) VersionGuessOpt { + return guessOpt{func(cfg *guessCfg) { cfg.skipKeys = keys }} +} + +// TryRaftBroker previously attempted to version guess selecting for a raft +// broker. +// +// Deprecated: Zookeeper, KRaft broker, and KRaft controller are all checked +// and the best pick is chosen. +func TryRaftBroker() VersionGuessOpt { + return guessOpt{func(*guessCfg) {}} +} + +// TryRaftController previously attempted to version guest selecting for a +// raft controller. +// +// Deprecated: Zookeeper, KRaft broker, and KRaft controller are all checked +// and the best pick is chosen. +func TryRaftController() VersionGuessOpt { + return guessOpt{func(*guessCfg) {}} +} + +type guessCfg struct { + skipKeys []int16 +} + +// VersionGuess attempts to guess which version of Kafka these versions belong +// to. If an exact match can be determined, this returns a string in the format +// v0.#.# or v#.# (depending on whether Kafka is pre-1.0 or post). For +// example, v0.8.0 or v2.7. +// +// Patch numbers are not included in the guess as it is not possible to +// determine the Kafka patch version being used as a client. +// +// If the version is determined to be higher than kversion knows of or is tip, +// this package returns "at least v#.#". +// +// Custom versions, or in-between versions, are detected and return slightly +// more verbose strings. +// +// Options can be specified to change how version guessing is performed, for +// example, certain keys can be skipped, or the guessing can try evaluating the +// versions as Raft broker based versions. +// +// Internally, this function tries guessing the version against both KRaft and +// Kafka APIs. The more exact match is returned. +func (vs *Versions) VersionGuess(opts ...VersionGuessOpt) string { + zk := vs.versionGuess2(ztip(), opts...) + broker := vs.versionGuess2(btip(), opts...) + controller := vs.versionGuess2(ctip(), opts...) + + ord := []guess{broker, zk, controller} + + for _, g := range ord { + if g.how == guessExact { + return g.String() + } + } + for _, g := range ord { + if g.how == guessAtLeast { + return g.String() + } + } + + // This is a custom version. We could do some advanced logic to try to + // return highest of all three guesses, but that may be inaccurate: + // KRaft may detect a higher guess because not all requests exist in + // KRaft. Instead, we just return our standard guess. + return zk.String() +} + +type guess struct { + v1 string + v2 string // for between + how int8 +} + +const ( + guessExact = iota + guessAtLeast + guessCustomUnknown + guessCustomAtLeast + guessBetween + guessNotEven +) + +func (g guess) String() string { + switch g.how { + case guessExact: + return g.v1 + case guessAtLeast: + return "at least " + g.v1 + case guessCustomUnknown: + return "unknown custom version" + case guessCustomAtLeast: + return "unknown custom version at least " + g.v1 + case guessBetween: + return "between " + g.v1 + " and " + g.v2 + case guessNotEven: + return "not even " + g.v1 + } + return g.v1 +} + +// String returns a string representation of the versions; the format may +// change. +func (vs *Versions) String() string { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) + keys := make([]int16, 0, len(vs.reqs)) + for k := range vs.reqs { + keys = append(keys, k) + } + for _, k := range keys { + name := kmsg.NameForKey(k) + if name == "" { + name = "Unknown" + } + fmt.Fprintf(w, "%s\t%d\n", name, vs.reqs[k].vmax) + } + w.Flush() + return buf.String() +} + +func relversion(fns ...func() *release) *Versions { + // For 4.0+, we merge the Raft broker, Raft controller, and Zk broker + // requests. We merge *in order*: any key that exists is kept, any key + // that does not exist is merged. + var reqs map[int16]req + for _, fn := range fns { + if reqs == nil { + reqs = fn().reqs + continue + } + merge := fn().reqs + for k, req := range merge { + if _, ok := reqs[k]; ok { + continue + } + reqs[k] = req + } + } + return &Versions{reqs: reqs} +} + +// Stable is a shortcut for the latest _released_ Kafka versions. +// +// This is the default version used in kgo to avoid breaking tip changes. +// The stable version is only bumped once kgo internally supports all +// features in the release. +func Stable() *Versions { return relversion(b42, c42, z39) } + +// Tip is the latest defined Kafka key versions; this may be slightly out of date. +func Tip() *Versions { return relversion(ztip) } + +func V0_8_0() *Versions { return relversion(z080) } +func V0_8_1() *Versions { return relversion(z081) } +func V0_8_2() *Versions { return relversion(z082) } +func V0_9_0() *Versions { return relversion(z090) } +func V0_10_0() *Versions { return relversion(z0100) } +func V0_10_1() *Versions { return relversion(z0101) } +func V0_10_2() *Versions { return relversion(z0102) } +func V0_11_0() *Versions { return relversion(z0110) } +func V1_0_0() *Versions { return relversion(z10) } +func V1_1_0() *Versions { return relversion(z11) } +func V2_0_0() *Versions { return relversion(z20) } +func V2_1_0() *Versions { return relversion(z21) } +func V2_2_0() *Versions { return relversion(z22) } +func V2_3_0() *Versions { return relversion(z23) } +func V2_4_0() *Versions { return relversion(z24) } +func V2_5_0() *Versions { return relversion(z25) } +func V2_6_0() *Versions { return relversion(z26) } +func V2_7_0() *Versions { return relversion(z27) } +func V2_8_0() *Versions { return relversion(z28) } +func V3_0_0() *Versions { return relversion(z30) } +func V3_1_0() *Versions { return relversion(z31) } +func V3_2_0() *Versions { return relversion(z32) } +func V3_3_0() *Versions { return relversion(z33) } +func V3_4_0() *Versions { return relversion(z34) } +func V3_5_0() *Versions { return relversion(z35) } +func V3_6_0() *Versions { return relversion(z36) } +func V3_7_0() *Versions { return relversion(z37) } +func V3_8_0() *Versions { return relversion(z38) } +func V3_9_0() *Versions { return relversion(z39) } +func V4_0_0() *Versions { return relversion(b40) } +func V4_1_0() *Versions { return relversion(b41) } +func V4_2_0() *Versions { return relversion(b42) } diff --git a/vendor/github.com/twmb/franz-go/pkg/kversion/requests.go b/vendor/github.com/twmb/franz-go/pkg/kversion/requests.go new file mode 100644 index 00000000000..15337537c60 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/kversion/requests.go @@ -0,0 +1,1309 @@ +package kversion + +import ( + "fmt" + "maps" + + "github.com/twmb/franz-go/pkg/kmsg" +) + +const ( + maj0_8 uint8 = 253 + maj0_9 uint8 = 254 + maj0_10 uint8 = 255 + maj0_11 uint8 = 0 +) + +type releaseKind uint8 + +const ( + kindBroker releaseKind = 0 + kindController releaseKind = 1 + kindZk releaseKind = 2 +) + +type release struct { + major uint8 + minor uint8 + kind releaseKind + reqs map[int16]req + prior *release +} + +func (r *release) clone(nextMajor, nextMinor uint8) *release { + return &release{ + major: nextMajor, + minor: nextMinor, + kind: r.kind, + reqs: maps.Clone(r.reqs), + prior: r, + } +} + +func (r *release) name() string { + switch r.major { + case maj0_8, maj0_9, maj0_10, maj0_11: + return fmt.Sprintf("v0.%d.%d", r.major+11, r.minor) + default: + return fmt.Sprintf("v%d.%d", r.major, r.minor) + } +} + +type req struct { + key int16 + vmin int16 + vmax int16 +} + +func (r *release) incmax(key, vmax int16) { + req, ok := r.reqs[key] + if !ok { + panic(fmt.Sprintf("key %d does not yet exist to incmax", key)) + } + if req.vmax+1 != vmax { + panic(fmt.Sprintf("key %d next max %d != exp %d", key, req.vmax+1, vmax)) + } + req.vmax++ + r.reqs[key] = req +} + +func (r *release) addkey(key int16) { + r.addkeyver(key, 0) +} + +func (r *release) addkeyver(key, vmax int16) { + if _, ok := r.reqs[key]; ok { + panic(fmt.Sprintf("key %d already exists", key)) + } + r.reqs[key] = req{key: key, vmax: vmax} +} + +func (r *release) setmin(key, vmin int16) { + req, ok := r.reqs[key] + if !ok { + panic(fmt.Sprintf("setmin on non-existent key %d", key)) + } + req.vmin = vmin + r.reqs[key] = req +} + +func reqsFromApiVersions(r *kmsg.ApiVersionsResponse) map[int16]req { + m := make(map[int16]req, len(r.ApiKeys)) + for _, k := range r.ApiKeys { + m[k.ApiKey] = req{ + key: k.ApiKey, + vmin: k.MinVersion, + vmax: k.MaxVersion, + } + } + return m +} + +func (vs *Versions) versionGuess2(cmp *release, opts ...VersionGuessOpt) guess { + // KRaft at 2.8 had two requests that were not in 3.0, which makes + // version detection of 2.8 more difficult. We don't handle it + // properly; KRaft 2.8 was feature preview only. + cfg := guessCfg{ + // We skip: + // * (4) LeaderAndISR + // * (5) StopReplica + // * (6) UpdateMetadata + // * (7) ControlledShutdown + // * (27) WriteTxnMarkers + // * (56) AlterISR + // * (57) UpdateFeatures + // * (58) Envelope + // * (67) AllocateProducerIDs + // * (71) GetTelemetrySubscriptions + // * (72) PushTelemetry + // + // Most of these keys are broker-to-broker only requests, and + // most non-Kafka implementations do not implement them. + // + // We skip 71 and 72 because telemetry requests are only + // advertised if the broker is configured to support it. + skipKeys: []int16{4, 5, 6, 7, 27, 56, 57, 58, 67, 71, 72}, + } + for _, opt := range opts { + opt.apply(&cfg) + } + + // For comparison checking, we only check the max key version. + var higher *release + for { + for _, k := range cfg.skipKeys { + delete(cmp.reqs, k) + delete(vs.reqs, k) + } + + var under, equal, over bool + + for k, req := range vs.reqs { + cmpreq, ok := cmp.reqs[k] + if ok { + if req.vmax < cmpreq.vmax { + under = true + } else if req.vmax > cmpreq.vmax { + over = true + } else { + equal = true + } + delete(cmp.reqs, k) + } else { + over = true // key we do not recognize: by definition the broker is higher than this cmp version + } + } + + // If our versions did not clear out what we are comparing against, we + // do not have all keys that we need for this version. + if len(cmp.reqs) > 0 { + under = true + } + + switch { + case under && over: + if cmp.prior == nil { + return guess{v1: cmp.name(), how: guessCustomUnknown} + } + + case under: + if cmp.prior == nil { + return guess{v1: cmp.name(), how: guessNotEven} + } + + case over: + if higher != nil { + return guess{v1: cmp.name(), v2: higher.name(), how: guessBetween} + } + return guess{v1: cmp.name(), how: guessAtLeast} + + case equal: + return guess{v1: cmp.name(), how: guessExact} + } + higher = cmp + cmp = cmp.prior + } +} + +//////////////////////// +// ZOOKEEPER VERSIONS // +//////////////////////// + +func z080() *release { + return &release{ + major: maj0_8, + minor: 0, + kind: kindZk, + reqs: map[int16]req{ + 0: {key: 0}, // 0 produce + 1: {key: 1}, // 1 fetch + 2: {key: 2}, // 2 list offset + 3: {key: 3}, // 3 metadata + 4: {key: 4}, // 4 leader and isr + 5: {key: 5}, // 5 stop replica + 6: {key: 6}, // 6 update metadata (actually not supported for a bit) + 7: {key: 7}, // 7 controlled shutdown, actually not supported for a bit + }, + } +} + +func z081() *release { + prior := z080() + now := prior.clone(maj0_8, 1) + + now.addkey(8) // 8 offset commit KAFKA-965 db37ed0054 + now.addkey(9) // 9 offset fetch (same) + return now +} + +func z082() *release { + now := z081().clone(maj0_8, 2) + + now.incmax(8, 1) // 1 offset commit KAFKA-1462 + now.incmax(9, 1) // 1 offset fetch KAFKA-1841 161b1aa16e I think? + + now.addkey(10) // 10 find coordinator KAFKA-1012 a670537aa3 + now.addkey(11) // 11 join group (same) + now.addkey(12) // 12 heartbeat (same) + return now +} + +func z090() *release { + now := z082().clone(maj0_9, 0) + + now.incmax(0, 1) // 1 produce KAFKA-2136 436b7ddc38; KAFKA-2083 ?? KIP-13 + now.incmax(1, 1) // 1 fetch (same) + now.incmax(6, 1) // 1 update metadata KAFKA-2411 d02ca36ca1 + now.incmax(7, 1) // 1 controlled shutdown (same) + now.incmax(8, 2) // 2 offset commit KAFKA-1634 + + now.addkey(13) // 13 leave group KAFKA-2397 636e14a991 + now.addkey(14) // 14 sync group KAFKA-2464 86eb74d923 + now.addkey(15) // 15 describe groups KAFKA-2687 596c203af1 + now.addkey(16) // 16 list groups KAFKA-2687 596c203af1 + return now +} + +func z0100() *release { + now := z090().clone(maj0_10, 0) + + now.incmax(0, 2) // 2 produce KAFKA-3025 45c8195fa1 KIP-31 KIP-32 + now.incmax(1, 2) // 2 fetch (same) + now.incmax(3, 1) // 1 metadata KAFKA-3306 33d745e2dc + now.incmax(6, 2) // 2 update metadata KAFKA-1215 951e30adc6 + + now.addkey(17) // 17 sasl handshake KAFKA-3149 5b375d7bf9 + now.addkey(18) // 18 api versions KAFKA-3307 8407dac6ee + return now +} + +func z0101() *release { + now := z0100().clone(maj0_10, 1) + + now.incmax(1, 3) // 3 fetch KAFKA-2063 d04b0998c0 KIP-74 + now.incmax(2, 1) // 1 list offset KAFKA-4148 eaaa433fc9 KIP-79 + now.incmax(3, 2) // 2 metadata KAFKA-4093 ecc1fb10fa KIP-78 + now.incmax(11, 1) // 1 join group KAFKA-3888 40b1dd3f49 KIP-62 + + now.addkey(19) // 19 create topics KAFKA-2945 fc47b9fa6b + now.addkey(20) // 20 delete topics KAFKA-2946 539633ba0e + return now +} + +func z0102() *release { + now := z0101().clone(maj0_10, 2) + + now.incmax(6, 3) // 3 update metadata KAFKA-4565 d25671884b KIP-103 + now.incmax(19, 1) // 1 create topics KAFKA-4591 da57bc27e7 KIP-108 + return now +} + +func z0110() *release { + now := z0102().clone(maj0_11, 0) + + now.incmax(0, 3) // 3 produce KAFKA-4816 5bd06f1d54 KIP-98 + now.incmax(1, 4) // 4 fetch (same) + now.incmax(1, 5) // 5 fetch KAFKA-4586 8b05ad406d KIP-107 + now.incmax(9, 2) // 2 offset fetch KAFKA-3853 c2d9b95f36 KIP-98 + now.incmax(10, 1) // 1 find coordinator KAFKA-5043 d0e7c6b930 KIP-98 + + now.addkey(21) // 21 delete records KAFKA-4586 see above + now.addkey(22) // 22 init producer id KAFKA-4817 bdf4cba047 KIP-98 (raft added in KAFKA-12620 e97cff2702b6ba836c7925caa36ab18066a7c95d KIP-730) + now.addkey(23) // 23 offset for leader epoch KAFKA-1211 0baea2ac13 KIP-101 + + now.addkey(24) // 24 add partitions to txn KAFKA-4990 865d82af2c KIP-98 (raft 3.0 6e857c531f14d07d5b05f174e6063a124c917324) + now.addkey(25) // 25 add offsets to txn (same, same raft) + now.addkey(26) // 26 end txn (same, same raft) + now.addkey(27) // 27 write txn markers (same) + now.addkey(28) // 28 txn offset commit (same, same raft) + + // raft broker / controller added in 5b0c58ed53c420e93957369516f34346580dac95 + now.addkey(29) // 29 describe acls KAFKA-3266 9815e18fef KIP-140 + now.addkey(30) // 30 create acls (same) + now.addkey(31) // 31 delete acls (same) + + now.addkey(32) // 32 describe configs KAFKA-3267 972b754536 KIP-133 + now.addkey(33) // 33 alter configs (same) (raft broker 3.0 6e857c531f14d07d5b05f174e6063a124c917324, controller 273d66479dbee2398b09e478ffaf996498d1ab34) + + // KAFKA-4954 0104b657a1 KIP-124 + now.incmax(2, 2) // 2 list offset (reused in e71dce89c0 KIP-98) + now.incmax(3, 3) // 3 metadata + now.incmax(8, 3) // 3 offset commit + now.incmax(9, 3) // 3 offset fetch + now.incmax(11, 2) // 2 join group + now.incmax(12, 1) // 1 heartbeat + now.incmax(13, 1) // 1 leave group + now.incmax(14, 1) // 1 sync group + now.incmax(15, 1) // 1 describe groups + now.incmax(16, 1) // 1 list group + now.incmax(18, 1) // 1 api versions + now.incmax(19, 2) // 2 create topics + now.incmax(20, 1) // 1 delete topics + + now.incmax(3, 4) // 4 metadata KAFKA-5291 7311dcbc53 + + return now +} + +func z10() *release { + now := z0110().clone(1, 0) + + now.incmax(0, 4) // 4 produce KAFKA-4763 fc93fb4b61 KIP-112 + now.incmax(1, 6) // 6 fetch (same) + now.incmax(3, 5) // 5 metadata (same) + now.incmax(4, 1) // 1 leader and isr (same) + now.incmax(6, 4) // 4 update metadata (same) + + now.incmax(0, 5) // 5 produce KAFKA-5793 94692288be + now.incmax(17, 1) // 1 sasl handshake KAFKA-4764 8fca432223 KIP-152 + + now.addkey(34) // 34 alter replica log dirs KAFKA-5694 adefc8ea07 KIP-113 + now.addkey(35) // 35 describe log dirs (same) + now.addkey(36) // 36 sasl authenticate KAFKA-4764 (see above) + now.addkey(37) // 37 create partitions KAFKA-5856 5f6393f9b1 KIP-195 (raft 3.0 6e857c531f14d07d5b05f174e6063a124c917324) + return now +} + +func z11() *release { + now := z10().clone(1, 1) + + now.addkey(38) // 38 create delegation token KAFKA-4541 27a8d0f9e7 under KAFKA-1696 KIP-48 + now.addkey(39) // 39 renew delegation token (same) + now.addkey(40) // 40 expire delegation token (same) + now.addkey(41) // 41 describe delegation token (same) + now.addkey(42) // 42 delete groups KAFKA-6275 1ed6da7cc8 KIP-229 + + now.incmax(1, 7) // 7 fetch KAFKA-6254 7fe1c2b3d3 KIP-227 + now.incmax(32, 1) // 1 describe configs KAFKA-6241 b814a16b96 KIP-226 + + return now +} + +func z20() *release { + now := z11().clone(2, 0) + + now.incmax(0, 6) // 6 produce KAFKA-6028 1facab387f KIP-219 + now.incmax(1, 8) // 8 fetch (same) + now.incmax(2, 3) // 3 list offset (same) + now.incmax(3, 6) // 6 metadata (same) + now.incmax(8, 4) // 4 offset commit (same) + now.incmax(9, 4) // 4 offset fetch (same) + now.incmax(10, 2) // 2 find coordinator (same) + now.incmax(11, 3) // 3 join group (same) + now.incmax(12, 2) // 2 heartbeat (same) + now.incmax(13, 2) // 2 leave group (same) + now.incmax(14, 2) // 2 sync group (same) + now.incmax(15, 2) // 2 describe groups (same) + now.incmax(16, 2) // 2 list group (same) + now.incmax(18, 2) // 2 api versions (same) + now.incmax(19, 3) // 3 create topics (same) + now.incmax(20, 2) // 2 delete topics (same) + now.incmax(21, 1) // 1 delete records (same) + now.incmax(22, 1) // 1 init producer id (same) + now.incmax(24, 1) // 1 add partitions to txn (same) + now.incmax(25, 1) // 1 add offsets to txn (same) + now.incmax(26, 1) // 1 end txn (same) + now.incmax(28, 1) // 1 txn offset commit (same) + // 29, 30, 31 bumped below, but also had throttle changes + now.incmax(32, 2) // 2 describe configs (same) + now.incmax(33, 1) // 1 alter configs (same) + now.incmax(34, 1) // 1 alter replica log dirs (same) + now.incmax(35, 1) // 1 describe log dirs (same) + now.incmax(37, 1) // 1 create partitions (same) + now.incmax(38, 1) // 1 create delegation token (same) + now.incmax(39, 1) // 1 renew delegation token (same) + now.incmax(40, 1) // 1 expire delegation token (same) + now.incmax(41, 1) // 1 describe delegation token (same) + now.incmax(42, 1) // 1 delete groups (same) + + now.incmax(29, 1) // 1 describe acls KAFKA-6841 b3aa655a70 KIP-290 + now.incmax(30, 1) // 1 create acls (same) + now.incmax(31, 1) // 1 delete acls (same) + + now.incmax(23, 1) // 1 offset for leader epoch KAFKA-6361 9679c44d2b KIP-279 + return now +} + +func z21() *release { + now := z20().clone(2, 1) + + now.incmax(8, 5) // 5 offset commit KAFKA-4682 418a91b5d4 KIP-211 + + now.incmax(20, 3) // 3 delete topics KAFKA-5975 04770916a7 KIP-322 + + now.incmax(1, 9) // 9 fetch KAFKA-7333 05ba5aa008 KIP-320 + now.incmax(2, 4) // 4 list offset (same) + now.incmax(3, 7) // 7 metadata (same) + now.incmax(8, 6) // 6 offset commit (same) + now.incmax(9, 5) // 5 offset fetch (same) + now.incmax(23, 2) // 2 offset for leader epoch (same, also in Kafka PR #5635 79ad9026a6) + now.incmax(28, 2) // 2 txn offset commit (same) + + now.incmax(0, 7) // 7 produce KAFKA-4514 741cb761c5 KIP-110 + now.incmax(1, 10) // 10 fetch (same) + + return now +} + +func z22() *release { + now := z21().clone(2, 2) + + now.incmax(2, 5) // 5 list offset KAFKA-2334 152292994e KIP-207 + now.incmax(11, 4) // 4 join group KAFKA-7824 9a9310d074 KIP-394 + now.incmax(36, 1) // 1 sasl authenticate KAFKA-7352 e8a3bc7425 KIP-368 + + now.incmax(4, 2) // 2 leader and isr KAFKA-7235 2155c6d54b KIP-380 + now.incmax(5, 1) // 1 stop replica (same) + now.incmax(6, 5) // 5 update metadata (same) + now.incmax(7, 2) // 2 controlled shutdown (same) + + now.addkey(43) // 43 elect preferred leaders KAFKA-5692 269b65279c KIP-183 (raft 3.0 6e857c531f14d07d5b05f174e6063a124c917324) + return now +} + +func z23() *release { + now := z22().clone(2, 3) + + now.incmax(3, 8) // 8 metadata KAFKA-7922 a42f16f980 KIP-430 + now.incmax(15, 3) // 3 describe groups KAFKA-7922 f11fa5ef40 KIP-430 + + now.incmax(1, 11) // 11 fetch KAFKA-8365 e2847e8603 KIP-392 + now.incmax(23, 3) // 3 offset for leader epoch (same) + + now.incmax(11, 5) // 5 join group KAFKA-7862 0f995ba6be KIP-345 + now.incmax(8, 7) // 7 offset commit KAFKA-8225 9fa331b811 KIP-345 + now.incmax(12, 3) // 3 heartbeat (same) + now.incmax(14, 3) // 3 sync group (same) + + now.addkey(44) // 44 incremental alter configs KAFKA-7466 3b1524c5df KIP-339 + return now +} + +func z24() *release { + now := z23().clone(2, 4) + + now.incmax(4, 3) // 3 leader and isr KAFKA-8345 81900d0ba0 KIP-455 + now.incmax(15, 4) // 4 describe groups KAFKA-8538 f8db022b08 KIP-345 + now.incmax(19, 4) // 4 create topics KAFKA-8305 8e161580b8 KIP-464 + now.incmax(43, 1) // 1 elect preferred leaders KAFKA-8286 121308cc7a KIP-460 + + // raft added in e07de97a4ce730a2755db7eeacb9b3e1f69a12c8 for the following two + now.addkey(45) // 45 alter partition reassignments KAFKA-8345 81900d0ba0 KIP-455 + now.addkey(46) // 46 list partition reassignments (same) + now.addkey(47) // 47 offset delete KAFKA-8730 e24d0e22ab KIP-496 + + now.incmax(13, 3) // 3 leave group KAFKA-8221 74c90f46c3 KIP-345 + + // introducing flexible versions; 24 were bumped + now.incmax(3, 9) // 9 metadata KAFKA-8885 apache/kafka#7325 KIP-482 + now.incmax(4, 4) // 4 leader and isr (same) + now.incmax(5, 2) // 2 stop replica (same) + now.incmax(6, 6) // 6 update metadata (same) + now.incmax(7, 3) // 3 controlled shutdown (same) + now.incmax(8, 8) // 8 offset commit (same) + now.incmax(9, 6) // 6 offset fetch (same) + now.incmax(10, 3) // 3 find coordinator (same) + now.incmax(11, 6) // 6 join group (same) + now.incmax(12, 4) // 4 heartbeat (same) + now.incmax(13, 4) // 4 leave group (same) + now.incmax(14, 4) // 4 sync group (same) + now.incmax(15, 5) // 5 describe groups (same) + now.incmax(16, 3) // 3 list group (same) + now.incmax(18, 3) // 3 api versions (same, also KIP-511 [non-flexible fields added]) + now.incmax(19, 5) // 5 create topics (same) + now.incmax(20, 4) // 4 delete topics (same) + now.incmax(22, 2) // 2 init producer id (same) + now.incmax(38, 2) // 2 create delegation token (same) + now.incmax(42, 2) // 2 delete groups (same) + now.incmax(43, 2) // 2 elect preferred leaders (same) + now.incmax(44, 1) // 1 incremental alter configs (same) + // also 45, 46; not bumped since in same release + + // Create topics (19) was bumped up to 5 in KAFKA-8907 5d0052fe00 + // KIP-525, then 6 in the above bump, then back down to 5 once the + // tagged PR was merged (KAFKA-8932 1f1179ea64 for the bump down). + + now.incmax(0, 8) // 8 produce KAFKA-8729 f6f24c4700 KIP-467 + + return now +} + +func z25() *release { + now := z24().clone(2, 5) + + now.incmax(22, 3) // 3 init producer id KAFKA-8710 fecb977b25 KIP-360 + now.incmax(9, 7) // 7 offset fetch KAFKA-9346 6da70f9b95 KIP-447 + + // more flexible versions, KAFKA-9420 0a2569e2b99 KIP-482 + // 6 bumped, then sasl handshake reverted later in 1a8dcffe4 + now.incmax(36, 2) // 2 sasl authenticate + now.incmax(37, 2) // 2 create partitions + now.incmax(39, 2) // 2 renew delegation token + now.incmax(40, 2) // 2 expire delegation token + now.incmax(41, 2) // 2 describe delegation token + + now.incmax(28, 3) // 3 txn offset commit KAFKA-9365 ed7c071e07f KIP-447 + + now.incmax(29, 2) // 2 describe acls KAFKA-9026 40b35178e5 KIP-482 (for flexible versions) + now.incmax(30, 2) // 2 create acls KAFKA-9027 738e14edb KIP-482 (flexible) + now.incmax(31, 2) // 2 delete acls KAFKA-9028 738e14edb KIP-482 (flexible) + + now.incmax(11, 7) // 7 join group KAFKA-9437 96c4ce480 KIP-559 + now.incmax(14, 5) // 5 sync group (same) + + return now +} + +func z26() *release { + now := z25().clone(2, 6) + + now.incmax(21, 2) // 2 delete records KAFKA-8768 f869e33ab KIP-482 (opportunistic bump for flexible versions) + now.incmax(35, 2) // 2 describe log dirs KAFKA-9435 4f1e8331ff9 KIP-482 (same) + + now.addkey(48) // 48 describe client quotas KAFKA-7740 227a7322b KIP-546 (raft in 5964401bf9aab611bd4a072941bd1c927e044258) + now.addkey(49) // 49 alter client quotas (same) + + now.incmax(5, 3) // 3 stop replica KAFKA-9539 7c7d55dbd KIP-570 + + now.incmax(16, 4) // 4 list group KAFKA-9130 fe948d39e KIP-518 + now.incmax(32, 3) // 3 describe configs KAFKA-9494 af3b8b50f2 KIP-569 + + return now +} + +func z27() *release { + now := z26().clone(2, 7) + + // KAFKA-10163 a5ffd1ca44c KIP-599 + now.incmax(37, 3) // 3 create partitions + now.incmax(19, 6) // 6 create topics (same) + now.incmax(20, 5) // 5 delete topics (same) + + // KAFKA-9911 b937ec7567 KIP-588 + now.incmax(22, 4) // 4 init producer id + now.incmax(24, 2) // 2 add partitions to txn + now.incmax(25, 2) // 2 add offsets to txn + now.incmax(26, 2) // 2 end txn + + now.addkey(50) // 50 describe user scram creds, KAFKA-10259 e8524ccd8fca0caac79b844d87e98e9c055f76fb KIP-554; 38c409cf33c kraft + now.addkey(51) // 51 alter user scram creds, same + + // KAFKA-10435 634c9175054cc69d10b6da22ea1e95edff6a4747 KIP-595 + // This opted in fetch request to flexible versions. + // + // KAFKA-10487: further change in aa5263fba903c85812c0c31443f7d49ee371e9db + now.incmax(1, 12) // 12 fetch + + // KAFKA-8836 57de67db22eb373f92ec5dd449d317ed2bc8b8d1 KIP-497 + now.addkey(56) // 56 alter isr + + // KAFKA-10028 fb4f297207ef62f71e4a6d2d0dac75752933043d KIP-584 + now.addkey(57) // 57 update features (rbroker 3.0 6e857c531f14d07d5b05f174e6063a124c917324; rcontroller 3.2 55ff5d360381af370fe5b3a215831beac49571a4 KIP-778 KAFKA-13823) + return now +} + +func z28() *release { + now := z27().clone(2, 8) + + // KAFKA-10729 85f94d50271c952c3e9ee49c4fc814c0da411618 KIP-482 + // (flexible bumps) + now.incmax(0, 9) // 9 produce + now.incmax(2, 6) // 6 list offsets + now.incmax(23, 4) // 4 offset for leader epoch + now.incmax(24, 3) // 3 add partitions to txn + now.incmax(25, 3) // 3 add offsets to txn + now.incmax(26, 3) // 3 end txn + now.incmax(27, 1) // 1 write txn markers + now.incmax(32, 4) // 4 describe configs + now.incmax(33, 2) // 2 alter configs + now.incmax(34, 2) // 2 alter replica log dirs + now.incmax(48, 1) // 1 describe client quotas + now.incmax(49, 1) // 1 alter client quotas + + // KAFKA-10547 5c921afa4a593478f7d1c49e5db9d787558d0d5e KIP-516 + now.incmax(3, 10) // 10 metadata + now.incmax(6, 7) // 7 update metadata + + // KAFKA-10545 1dd1e7f945d7a8c1dc177223cd88800680f1ff46 KIP-516 + now.incmax(4, 5) // 5 leader and isr + + // KAFKA-12204 / KAFKA-10851 302eee63c479fd4b955c44f1058a5e5d111acb57 KIP-700 + now.addkey(60) // 60 describe cluster; rController in KAFKA-15396 41b695b6e30baa4243d9ca4f359b833e17ed0e77 KIP-919 + + // KAFKA-12212 7a1d1d9a69a241efd68e572badee999229b3942f KIP-700 + now.incmax(3, 11) // 11 metadata + + // KAFKA-10764 4f588f7ca2a1c5e8dd845863da81425ac69bac92 KIP-516 + now.incmax(19, 7) // 7 create topics + now.incmax(20, 6) // 6 delete topics + + // KAFKA-12238 e9edf104866822d9e6c3b637ffbf338767b5bf27 KIP-664 + now.addkey(61) // 61 describe producers + return now +} + +func z30() *release { + now := z28().clone(3, 0) + + // KAFKA-12267 3f09fb97b6943c0612488dfa8e5eab8078fd7ca0 KIP-664 + now.addkey(65) // 65 describe transactions + + // KAFKA-12369 3708a7c6c1ecf1304f091dda1e79ae53ba2df489 KIP-664 + now.addkey(66) // 66 list transactions + + // KAFKA-12620 72d108274c98dca44514007254552481c731c958 KIP-730 + // raft broker added in e97cff2702b6ba836c7925caa36ab18066a7c95d + now.addkey(67) // 67 allocate producer ids + + // KAFKA-12541 bd72ef1bf1e40feb3bc17349a385b479fa5fa530 KIP-734 + now.incmax(2, 7) // 7 list offsets + + // KAFKA-12663 f5d5f654db359af077088685e29fbe5ea69616cf KIP-699 + now.incmax(10, 4) // 4 find coordinator + + // KAFKA-12234 e00c0f3719ad0803620752159ef8315d668735d6 KIP-709 + now.incmax(9, 8) // 8 offset fetch + + return now +} + +func z31() *release { + now := z30().clone(3, 1) + + // KAFKA-10580 2b8aff58b575c199ee8372e5689420c9d77357a5 KIP-516 + now.incmax(1, 13) // 13 fetch + + // KAFKA-10744 1d22b0d70686aef5689b775ea2ea7610a37f3e8c KIP-516 + now.incmax(3, 12) // 12 metadata + + return now +} + +func z32() *release { + now := z31().clone(3, 2) + + // KAFKA-13495 69645f1fe5103adb00de6fa43152e7df989f3aea KIP-800 + now.incmax(11, 8) // 8 join group + + // KAFKA-13496 bf609694f83931990ce63e0123f811e6475820c5 KIP-800 + now.incmax(13, 5) // 5 leave group + + // KAFKA-13527 31fca1611a6780e8a8aa3ac21618135201718e32 KIP-784 + now.incmax(35, 3) // 3 describe log dirs + + // KAFKA-13435 c8fbe26f3bd3a7c018e7619deba002ee454208b9 KIP-814 + now.incmax(11, 9) // 9 join group + + // KAFKA-13587 52621613fd386203773ba93903abd50b46fa093a KIP-704 + now.incmax(4, 6) // 6 leader and isr + now.incmax(56, 1) // 1 alter isr => alter partition + + return now +} + +func z33() *release { + now := z32().clone(3, 3) + + // KAFKA-13823 55ff5d360381af370fe5b3a215831beac49571a4 KIP-778 + now.incmax(57, 1) // 1 update features + + // KAFKA-13958 4fcfd9ddc4a8da3d4cfbb69268c06763352e29a9 KIP-827 + now.incmax(35, 4) // 4 describe log dirs + + // KAFKA-841 f83d95d9a28 KIP-841 + now.incmax(56, 2) // 2 alter partition + + // KAFKA-6945 d65d8867983 KIP-373 + now.incmax(29, 3) // 3 describe acls + now.incmax(30, 3) // 3 create acls + now.incmax(31, 3) // 3 delete acls + now.incmax(38, 3) // 3 create delegation token + now.incmax(41, 3) // 3 describe delegation token + + return now +} + +func z34() *release { + now := z33().clone(3, 4) + + // KAFKA-14304 7b7e40a536a79cebf35cc278b9375c8352d342b9 KIP-866 + // KAFKA-14448 67c72596afe58363eceeb32084c5c04637a33831 added BrokerRegistration + // KAFKA-14493 db490707606855c265bc938e1b236070e0e2eba5 changed BrokerRegistration + // KAFKA-14304 0bb05d8679b684ad8fbb2eb40dfc00066186a75a changed BrokerRegistration back to a bool... + // 5b521031edea8ea7cbcca7dc24a58429423740ff added tag to ApiVersions + now.incmax(4, 7) // 7 leader and isr + now.incmax(5, 4) // 4 stop replica + now.incmax(6, 8) // 8 update metadata + + // KAFKA-14446 8b045dcbf6b89e1a9594ff95642d4882765e4b0d KIP-866 Kafka 3.4 + now.addkey(58) // 58 envelope + + return now +} + +func z35() *release { + now := z34().clone(3, 5) + + // KAFKA-13369 7146ac57ba9ddd035dac992b9f188a8e7677c08d KIP-405 + now.incmax(1, 14) // 14 fetch + now.incmax(2, 8) // 8 list offsets + + now.incmax(1, 15) // 15 fetch // KAFKA-14617 79b5f7f1ce2 KIP-903 + now.incmax(56, 3) // 3 alter partition // KAFKA-14617 8c88cdb7186b1d594f991eb324356dcfcabdf18a KIP-903 + return now +} + +func z36() *release { + now := z35().clone(3, 6) + + // KAFKA-14402 29a1a16668d76a1cc04ec9e39ea13026f2dce1de KIP-890 + // Later commit swapped to stable + now.incmax(24, 4) // 4 add partitions to txn + return now +} + +func z37() *release { + now := z36().clone(3, 7) + + // KAFKA-15661 c8f687ac1505456cb568de2b60df235eb1ceb5f0 KIP-951 + now.incmax(0, 10) // 10 produce + now.incmax(1, 16) // 16 fetch + + // 7826d5fc8ab695a5ad927338469ddc01b435a298 KIP-848 + // (change introduced in 3.6 but was marked unstable and not visible) + now.incmax(8, 9) // 9 offset commit + // KAFKA-14499 7054625c45dc6edb3c07271fe4a6c24b4638424f KIP-848 (and prior) + now.incmax(9, 9) // 9 offset fetch + + // KAFKA-15368 41b695b6e30baa4243d9ca4f359b833e17ed0e77 KIP-919 + // (added rController as well, see above) + now.incmax(60, 1) // 1 describe cluster + + // KAFKA-14391 3be7f7d611d0786f2f98159d5c7492b0d94a2bb7 KIP-848 + // as well as some patches following + now.addkey(68) // 68 consumer group heartbeat + + return now +} + +func z38() *release { + now := z37().clone(3, 8) + + // KAFKA-16314 2e8d69b78ca52196decd851c8520798aa856c073 KIP-890 + // Then error rename in cf1ba099c0723f9cf65dda4cd334d36b7ede6327 + now.incmax(0, 11) // 11 produce + now.incmax(10, 5) // 5 find coordinator + now.incmax(22, 5) // 5 init producer id + now.incmax(24, 5) // 5 add partitions to txn + now.incmax(25, 4) // 4 add offsets to txn + now.incmax(26, 4) // 4 end txn + now.incmax(28, 4) // 4 txn offset commit + + // KAFKA-15460 68745ef21a9d8fe0f37a8c5fbc7761a598718d46 KIP-848 + now.incmax(16, 5) // 5 list groups + + // KAFKA-14509 90e646052a17e3f6ec1a013d76c1e6af2fbb756e KIP-848 added + // 7b0352f1bd9b923b79e60b18b40f570d4bfafcc0 + // b7c99e22a77392d6053fe231209e1de32b50a98b + // 68389c244e720566aaa8443cd3fc0b9d2ec4bb7a + // 5f410ceb04878ca44d2d007655155b5303a47907 stabilized + now.addkey(69) // 69 consumer group describe + + // KAFKA-16265 b4e96913cc6c827968e47a31261e0bd8fdf677b5 KIP-994 (part 1) + now.incmax(66, 1) // 1 list transactions + + return now +} + +func z39() *release { + now := z38().clone(3, 9) + + // KAFKA-16527 adee6f0cc11 KIP-853 + // This introduces a tag only, which does not require bumping the + // version bump. My skepticism on on the entire tag concept years + // ago continues to play out. + now.incmax(1, 17) // 17 fetch + + // ebaa108967f KIP-1005 + // This allows sending -5 to the broker as the timestamp. + // No end user changes otherwise. + now.incmax(2, 9) // 9 list offsets + + // KAFKA-16713 8f82f14a483 KIP-932 + now.incmax(10, 6) // 6 find coordinator + + // KAFKA-17011 ede289db93f + now.incmax(18, 4) // 4 api versions + + return now +} + +func ztip() *release { + return z39() +} + +/////////////////////////// +// KRAFT BROKER VERSIONS // +/////////////////////////// + +func b28() *release { + return &release{ + major: 2, + minor: 8, + kind: kindBroker, + reqs: map[int16]req{ + 0: {key: 0, vmax: 9}, // produce + 1: {key: 1, vmax: 12}, // fetch + 2: {key: 2, vmax: 6}, // list offsets + 3: {key: 3, vmax: 11}, // metadata + 8: {key: 8, vmax: 8}, // offset fetch + 9: {key: 9, vmax: 7}, // offset commit + 10: {key: 10, vmax: 3}, // find coordinator + 11: {key: 11, vmax: 7}, // join group + 12: {key: 12, vmax: 4}, // heartbeat + 13: {key: 13, vmax: 4}, // leave group + 14: {key: 14, vmax: 5}, // sync group + 15: {key: 15, vmax: 5}, // describe groups + 16: {key: 16, vmax: 4}, // list groups + 17: {key: 17, vmax: 1}, // sasl handshake + 18: {key: 18, vmax: 3}, // api versions + 19: {key: 19, vmax: 7}, // create topics + 20: {key: 20, vmax: 6}, // delete topics + 21: {key: 21, vmax: 2}, // delete records + 23: {key: 23, vmax: 4}, // offset for leader epoch + 27: {key: 27, vmax: 1}, // write txn markers + 32: {key: 32, vmax: 4}, // describe configs + 34: {key: 34, vmax: 2}, // alter replica log dirs + 35: {key: 35, vmax: 2}, // describe log dirs + 36: {key: 36, vmax: 2}, // sasl authenticate + 42: {key: 42, vmax: 2}, // delete groups + 44: {key: 44, vmax: 1}, // incremental alter configs + 47: {key: 47, vmax: 0}, // offset delete + 49: {key: 49, vmax: 1}, // alter client quotas + + // KAFKA-10492 b7c8490cf47b0c18253d6a776b2b35c76c71c65d KIP-595 + // (described below) + 55: {key: 55, vmax: 0}, // describe quorum + + 60: {key: 60, vmax: 0}, // describe cluster + 61: {key: 61, vmax: 0}, // describe producers + + // KAFKA-10181 KAFKA-10181 KIP-590 + 58: {key: 58, vmax: 0}, // envelope + }, + } +} + +func b30() *release { + now := b28().clone(3, 0) + + delete(now.reqs, 55) // describe quorum not present in 3.0 + + now.incmax(2, 7) + now.incmax(9, 8) + now.incmax(10, 4) + + now.addkeyver(22, 4) + now.addkeyver(24, 3) + now.addkeyver(25, 3) + now.addkeyver(26, 3) + now.addkeyver(28, 3) + now.addkeyver(29, 2) + now.addkeyver(30, 2) + now.addkeyver(31, 2) + now.addkeyver(33, 2) + now.addkeyver(37, 3) + now.addkeyver(43, 2) + now.addkey(45) + now.addkey(46) + now.addkeyver(48, 1) + now.addkey(57) + now.addkey(65) + now.addkey(66) + + return now +} + +func b31() *release { + now := b30().clone(3, 1) + now.incmax(1, 13) + now.incmax(3, 12) + return now +} + +func b32() *release { + now := b31().clone(3, 2) + now.incmax(11, 8) + now.incmax(11, 9) + now.incmax(13, 5) + now.incmax(35, 3) + delete(now.reqs, 58) + return now +} + +func b33() *release { + now := b32().clone(3, 3) + now.incmax(29, 3) + now.incmax(30, 3) + now.incmax(31, 3) + now.incmax(35, 4) + now.addkeyver(55, 1) + now.incmax(57, 1) + now.addkey(64) + return now +} + +func b34() *release { + now := b33().clone(3, 4) // no change for broker versions + return now +} + +func b35() *release { + now := b34().clone(3, 5) + now.incmax(1, 14) + now.incmax(1, 15) + now.incmax(2, 8) + now.addkey(50) + now.addkey(51) + return now +} + +func b36() *release { + now := b35().clone(3, 6) + now.incmax(24, 4) + now.addkeyver(38, 3) + now.addkeyver(39, 2) + now.addkeyver(40, 2) + now.addkeyver(41, 3) + return now +} + +func b37() *release { + now := b36().clone(3, 7) + now.incmax(0, 10) + now.incmax(1, 16) + now.incmax(8, 9) + now.incmax(9, 9) + now.incmax(60, 1) + + // KAFKA-15604 36abc8dcea1 KIP-714, only exposed if broker is configured + now.addkey(71) + now.addkey(72) + + now.addkey(68) + // KAFKA-15831 587f50d48f8 KIP-1000 + now.addkey(74) // 74 list client metrics + return now +} + +func b38() *release { + now := b37().clone(3, 8) + now.incmax(0, 11) + now.incmax(10, 5) + now.incmax(16, 5) + now.incmax(22, 5) + now.incmax(24, 5) + now.incmax(25, 4) + now.incmax(26, 4) + now.incmax(28, 4) + now.incmax(66, 1) + + now.addkey(69) + // KAFKA-15585 7e5ef9b509a KIP-966 + now.addkey(75) // 75 describe topic partition + return now +} + +func b39() *release { + now := b38().clone(3, 9) + // All version bumps here are commented above in zk or below in c. + now.incmax(1, 17) + now.incmax(2, 9) + now.incmax(10, 6) + now.incmax(18, 4) + now.incmax(55, 2) + now.addkey(80) + now.addkey(81) + return now +} + +func b40() *release { + now := b39().clone(4, 0) + + // KIP-896 + now.setmin(1, 4) + now.setmin(2, 1) + now.setmin(8, 2) + now.setmin(9, 1) + now.setmin(11, 2) + now.setmin(19, 2) + now.setmin(20, 1) + now.setmin(23, 2) + now.setmin(27, 1) + now.setmin(29, 1) + now.setmin(30, 1) + now.setmin(31, 1) + now.setmin(32, 1) + now.setmin(34, 1) + now.setmin(35, 1) + now.setmin(38, 1) + now.setmin(39, 1) + now.setmin(40, 1) + now.setmin(41, 1) + + now.incmax(0, 12) // 12 produce KAFKA-14563 755adf8a566 KIP-890 + now.incmax(2, 10) // 10 list offsets KAFKA-15859 560076ba9e8 KIP-1075 + now.incmax(3, 13) // 13 metadata KAFKA-17885 52d2fa5c8b3 KIP-1102 + now.incmax(15, 6) // 6 describe groups KAFKA-17550 e7d986e48c2 KIP-1043 + now.incmax(26, 5) // 5 end txn KAFKA-14562 ede0c94aaae KIP-890 + now.incmax(28, 5) // 5 txn offset commit KAFKA-14563 755adf8a566 KIP-890 + now.incmax(57, 2) // documented on controller + now.incmax(60, 2) // documented on controller + now.incmax(68, 1) // 1 consumer group heartbeat KAFKA-17592 ab0df20489a KIP-848; includes KAFKA-17116 6f040cabc7c KIP-1082 in same release + now.incmax(69, 1) // 1 consumer group describe KAFKA-17750 fe88232b07c KIP-858 + + return now +} + +func b41() *release { + now := b40().clone(4, 1) + + now.setmin(11, 0) // KAFKA-19444 487af011ca5 -- re-add JoinGroup v0 & v1 + + now.incmax(0, 13) // 13 produce KAFKA-10551 6f783f85362 KIP-516 + now.incmax(1, 18) // 18 fetch KAFKA-14145 742b327025f KIP-1166 + now.incmax(45, 1) // 1 alter partition assignments KAFKA-14121 cbd72cc216e KIP-860 + now.incmax(66, 2) // 2 list transactions KAFKA-19073 0c1fbf3aebb KIP-1152 + now.incmax(74, 1) // 1 list config resources (rename & extend) KAFKA-18904 c26b09c6092 KIP-1142 + + // 8f82f14a483 for v0, KAFKA-16713 66147d5de7c KIP-932 for v1 + now.addkeyver(76, 1) // 1 share group heartbeat + now.addkeyver(77, 1) // 1 share group describe + now.addkeyver(78, 1) // 1 share fetch + now.addkeyver(79, 1) // 1 share acknowledge + + // KAFKA-16950 fecbfb81332 KIP-932 + now.addkey(83) // 0 initialize share group state + now.addkey(84) // 0 read share group state + now.addkey(85) // 0 write share group state + now.addkey(86) // 0 delete share group state + now.addkey(87) // 0 read share group state summary + + now.addkey(90) // 0 describe share group offsets e3e4c179592, then KAFKA-16720 952113e8e0e KIP-932 + now.addkey(91) // 0 alter share group offsets KAFKA-16717 6a6b80215d8 KIP-932 + now.addkey(92) // 0 delete share group offsets KAFKA-16718 63229a768ce KIP-932 + + return now +} + +func b42() *release { + now := b41().clone(4, 2) + + now.incmax(2, 11) // 11 list offsets KAFKA-17108 8d93d1096c2 KIP-1023 + now.incmax(8, 10) // 10 offset commit KAFKA-19186 9599143bfd9 KIP-848 + now.incmax(9, 10) // 10 offset fetch KAFKA-19186 9599143bfd9 KIP-848 + now.incmax(27, 2) // 2 write txn markers KAFKA-19446 faad21fcb9a KIP-1228 + now.incmax(78, 2) // 2 share fetch KAFKA-19814 05c9322ba9c KIP-1206, KIP-1222 + now.incmax(79, 2) // 2 share acknowledge KAFKA-19814 05c9322ba9c KIP-1222 + now.incmax(80, 1) // 1 add raft voter KAFKA-19400 93447d5b883 KIP-1186 + now.incmax(85, 1) // 1 write share group state KAFKA-19797 b900f2fe5a0 KIP-1226 + now.incmax(87, 1) // 1 read share group state summary KAFKA-19799 5571821240 KIP-1226 + now.incmax(90, 1) // 1 describe share group offsets KAFKA-19800 1146f97770c KIP-1226 + + now.addkey(88) // 0 streams group heartbeat KAFKA-19869 497072a5644 KIP-1071 + now.addkey(89) // 0 streams group describe KAFKA-19869 497072a5644 KIP-1071 + + return now +} + +func btip() *release { + return b42() +} + +/////////////////////////////// +// KRAFT CONTROLLER VERSIONS // +/////////////////////////////// + +func c28() *release { + return &release{ + major: 2, + minor: 8, + kind: kindController, + reqs: map[int16]req{ + 1: {key: 1, vmax: 12}, // fetch + 3: {key: 3, vmax: 11}, // metadata + 7: {key: 7, vmax: 3}, // controlled shutdown + 17: {key: 17, vmax: 1}, // sasl handshake + 18: {key: 18, vmax: 3}, // api versions + 19: {key: 19, vmax: 7}, // create topics + 20: {key: 20, vmax: 6}, // delete topics + 36: {key: 36, vmax: 2}, // sasl authenticate + 44: {key: 44, vmax: 1}, // incremental alter configs + 49: {key: 49, vmax: 1}, // alter client quotas + + // KAFKA-10492 b7c8490cf47b0c18253d6a776b2b35c76c71c65d KIP-595 + // + // These are the first requests that are raft only. The commits + // showed up in the 2.7 branch, but KRaft as preview is only + // available at 2.8. + 52: {key: 52, vmax: 0}, // vote + 53: {key: 53, vmax: 0}, // begin quorum epoch + 54: {key: 54, vmax: 0}, // end quorum epoch + 55: {key: 55, vmax: 0}, // describe quorum + 56: {key: 56, vmax: 0}, // alter partition + + // KAFKA-10181 KAFKA-10181 KIP-590 + 58: {key: 58, vmax: 0}, // 58 envelope + 59: {key: 59, vmax: 0}, // 59 fetch snapshot + + // KAFKA-12248 a022072df3c8175950c03263d2bbf2e3ea7a7a5d KIP-500 + // (commit mentions KIP-500, these are actually described in KIP-631) + // Broker registration was later updated in d9bb2ef596343da9402bff4903b129cff1f7c22b + 62: {key: 62, vmax: 0}, // 62 broker registration + 63: {key: 63, vmax: 0}, // 63 broker heartbeat + + // KAFKA-12249 3f36f9a7ca153a9d221f6bedeb7d1503aa18eff1 KIP-500 / KIP-631 + // Renamed from Decommission to Unregister in 06dce721ec0185d49fac37775dbf191d0e80e687 + 64: {key: 64, vmax: 0}, // 64 unregister broker + }, + } +} + +func c30() *release { + now := c28().clone(3, 0) + + delete(now.reqs, 3) // metadata not present in controller 3.0 + + now.addkeyver(29, 2) // describe acls + now.addkeyver(30, 2) // create acls + now.addkeyver(31, 2) // delete acls + now.addkeyver(33, 2) // alter configs + + now.addkeyver(37, 3) // create partitions + now.addkeyver(43, 2) // elect leaders + now.addkey(45) // alter partition assignments + now.addkey(46) // list partition reassignments + + // KAFKA-12620 72d108274c98dca44514007254552481c731c958 KIP-730 + // raft broker added in e97cff2702b6ba836c7925caa36ab18066a7c95d + now.addkey(67) + + return now +} + +func c31() *release { + now := c30().clone(3, 1) + now.incmax(1, 13) + return now +} + +func c32() *release { + now := c31().clone(3, 2) + now.incmax(56, 1) + return now +} + +func c33() *release { + now := c32().clone(3, 3) + now.incmax(29, 3) + now.incmax(30, 3) + now.incmax(31, 3) + now.incmax(55, 1) + now.incmax(56, 2) + now.addkeyver(57, 1) + return now +} + +func c34() *release { + now := c33().clone(3, 4) + now.incmax(62, 1) // KAFKA-14304 7b7e40a536a (type change in 0bb05d8679b) KIP-866 + return now +} + +func c35() *release { + now := c34().clone(3, 5) + now.incmax(1, 14) + now.incmax(1, 15) + now.addkey(50) + now.addkey(51) + now.incmax(56, 3) + return now +} + +func c36() *release { + now := c35().clone(3, 6) + now.addkeyver(38, 3) + now.addkeyver(39, 2) + now.addkeyver(40, 2) + now.addkeyver(41, 3) + return now +} + +func c37() *release { + now := c36().clone(3, 7) + now.incmax(1, 16) + now.addkeyver(32, 4) + now.addkeyver(60, 1) + now.incmax(62, 2) // KAFKA-15355 a94bc8d6d52 KIP-858 + now.incmax(62, 3) // KAFKA-15582 14029e2ddd1 KIP-966 (some later commit swapped the docs on which came first between this and prior line) + now.incmax(63, 1) // KAFKA-15355 0390d5b1a24 KIP-858 (adds tag... no version bump actually needed...) + + // KAFKA-15369 41b695b6e30 KIP-919 + now.addkey(70) // 70 controller registration + // KAFKA-15355 0390d5b1a24 KIP-858 + now.addkey(73) // 73 assign replica to dirs + return now +} + +func c38() *release { + now := c37().clone(3, 8) // no changes for controller broker this version + return now +} + +func c39() *release { + now := c38().clone(3, 9) + now.incmax(1, 17) + now.incmax(18, 4) + + // KAFKA-16527 adee6f0cc11 KIP-853 adds a bunch of fields to Vote/{Begin,End}QuorumEpoch + now.incmax(52, 1) + now.incmax(53, 1) + now.incmax(54, 1) + now.incmax(59, 1) + now.incmax(55, 2) // KAFKA-16520 aecaf444756 KIP-853 + + // KAFKA-17001 ede289db93f -- no kip, this is a bugfix + now.incmax(62, 4) + + // KAFKA-16535 9ceed8f18f4 KIP-853 + now.addkey(80) // 80 add raft voter + now.addkey(81) // 81 remove raft voter + now.addkey(82) // 82 update raft voter + return now +} + +func c40() *release { + now := c39().clone(4, 0) + + // KIP-896 + now.setmin(1, 4) + now.setmin(19, 2) + now.setmin(20, 1) + now.setmin(29, 1) + now.setmin(30, 1) + now.setmin(31, 1) + now.setmin(32, 1) + now.setmin(38, 1) + now.setmin(39, 1) + now.setmin(40, 1) + now.setmin(41, 1) + now.setmin(56, 2) + delete(now.reqs, 7) + + now.incmax(52, 2) // KAFKA-17641 b73e31eb159 KIP-996 adds PreVote to Vote + now.incmax(57, 2) // KAFKA-16308 49d7ea6c6a2; no kip referenced + now.incmax(60, 2) // KAFKA-17094 747dc172e87 KIP-1073 + + return now +} + +func c41() *release { + now := c40().clone(4, 1) + + now.incmax(1, 18) + now.incmax(45, 1) + + return now +} + +func c42() *release { + now := c41().clone(4, 2) + + now.incmax(80, 1) // 1 add raft voter KAFKA-19400 93447d5b883 KIP-1186 + + return now +} + +func ctip() *release { + return c42() +} diff --git a/vendor/github.com/twmb/franz-go/pkg/sasl/sasl.go b/vendor/github.com/twmb/franz-go/pkg/sasl/sasl.go new file mode 100644 index 00000000000..dd85a02a188 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/pkg/sasl/sasl.go @@ -0,0 +1,41 @@ +// Package sasl specifies interfaces that any SASL authentication must provide +// to interop with Kafka SASL. +package sasl + +import "context" + +// Session is an authentication session. +type Session interface { + // Challenge is called with a server response. This must return + // if the authentication is done, or, if not, the next message + // to send. If the authentication is done, this can return an + // additional last message to be written (for which we will not + // read a response). + // + // Returning an error stops the authentication flow. + Challenge([]byte) (bool, []byte, error) +} + +// Mechanism authenticates with SASL. +type Mechanism interface { + // Name is the name of this SASL authentication mechanism. + Name() string + + // Authenticate initializes an authentication session to the provided + // host:port. If the mechanism is a client-first authentication + // mechanism, this also returns the first message to write. + // + // If initializing a session fails, this can return an error to stop + // the authentication flow. + // + // The provided context can be used through the duration of the session. + Authenticate(ctx context.Context, host string) (Session, []byte, error) +} + +// ClosingMechanism is an optional interface for SASL mechanisms. Implementing +// this interface signals that the mechanism should be closed if it will never +// be used again. +type ClosingMechanism interface { + // Close permanently closes a mechanism. + Close() +} diff --git a/vendor/github.com/twmb/franz-go/plugin/kslog/LICENSE b/vendor/github.com/twmb/franz-go/plugin/kslog/LICENSE new file mode 100644 index 00000000000..36e18034325 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/plugin/kslog/LICENSE @@ -0,0 +1,24 @@ +Copyright 2020, Travis Bischel. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/twmb/franz-go/plugin/kslog/README.md b/vendor/github.com/twmb/franz-go/plugin/kslog/README.md new file mode 100644 index 00000000000..b51a44bfc60 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/plugin/kslog/README.md @@ -0,0 +1,13 @@ +kslog +=== + +kslog is a plug-in package to use [slog](https://pkg.go.dev/log/slog) as a [`kgo.Logger`](https://pkg.go.dev/github.com/twmb/franz-go/pkg/kgo#Logger) + +To use, + +```go +cl, err := kgo.NewClient( + kgo.WithLogger(kslog.New(slog.Default())), + // ...other opts +) +``` diff --git a/vendor/github.com/twmb/franz-go/plugin/kslog/kslog.go b/vendor/github.com/twmb/franz-go/plugin/kslog/kslog.go new file mode 100644 index 00000000000..d33896657d6 --- /dev/null +++ b/vendor/github.com/twmb/franz-go/plugin/kslog/kslog.go @@ -0,0 +1,66 @@ +// Package kslog provides a plug-in kgo.Logger wrapping slog.Logger for usage in +// a kgo.Client. +// +// This can be used like so: +// +// cl, err := kgo.NewClient( +// kgo.WithLogger(kslog.New(slog.Default())), +// // ...other opts +// ) +package kslog + +import ( + "context" + "log/slog" + + "github.com/twmb/franz-go/pkg/kgo" +) + +// Logger provides the kgo.Logger interface for usage in kgo.WithLogger when +// initializing a client. +type Logger struct { + sl *slog.Logger +} + +// New returns a new kgo.Logger that wraps an slog.Logger. +func New(sl *slog.Logger) *Logger { + return &Logger{sl} +} + +// Level is for the kgo.Logger interface. +func (l *Logger) Level() kgo.LogLevel { + ctx := context.Background() + switch { + case l.sl.Enabled(ctx, slog.LevelDebug): + return kgo.LogLevelDebug + case l.sl.Enabled(ctx, slog.LevelInfo): + return kgo.LogLevelInfo + case l.sl.Enabled(ctx, slog.LevelWarn): + return kgo.LogLevelWarn + case l.sl.Enabled(ctx, slog.LevelError): + return kgo.LogLevelError + default: + return kgo.LogLevelNone + } +} + +// Log is for the kgo.Logger interface. +func (l *Logger) Log(level kgo.LogLevel, msg string, keyvals ...any) { + l.sl.Log(context.Background(), kgoToSlogLevel(level), msg, keyvals...) +} + +func kgoToSlogLevel(level kgo.LogLevel) slog.Level { + switch level { + case kgo.LogLevelError: + return slog.LevelError + case kgo.LogLevelWarn: + return slog.LevelWarn + case kgo.LogLevelInfo: + return slog.LevelInfo + case kgo.LogLevelDebug: + return slog.LevelDebug + default: + // Using the default level for slog + return slog.LevelInfo + } +} diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/client.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/client.go index 54c8396e096..353ed2ff53c 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/client.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/client.go @@ -19,8 +19,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/semconv/v1.39.0" - "go.opentelemetry.io/otel/semconv/v1.39.0/httpconv" + "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/semconv/v1.40.0/httpconv" ) type HTTPClient struct { diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/server.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/server.go index 7b646731ee1..332057bafcd 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/server.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/server.go @@ -20,8 +20,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/semconv/v1.39.0" - "go.opentelemetry.io/otel/semconv/v1.39.0/httpconv" + "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/semconv/v1.40.0/httpconv" ) type RequestTraceAttrsOpts struct { @@ -364,7 +364,9 @@ func (n HTTPServer) MetricAttributes(server string, req *http.Request, statusCod if statusCode > 0 { num++ } - + if route == "" && req.Pattern != "" { + route = httpRoute(req.Pattern) + } if route != "" { num++ } diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/util.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/util.go index 0a63aea5ff2..b2dc8548f41 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/util.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv/util.go @@ -15,7 +15,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - semconvNew "go.opentelemetry.io/otel/semconv/v1.39.0" + semconvNew "go.opentelemetry.io/otel/semconv/v1.40.0" ) // SplitHostPort splits a network address hostport of the form "host", diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/version.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/version.go index 30c2927d147..5473bb88415 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/version.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/version.go @@ -4,4 +4,4 @@ package otelhttptrace // import "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" // Version is the current release version of the httptrace instrumentation. -const Version = "0.66.0" +const Version = "0.68.0" diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request/resp_writer_wrapper.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request/resp_writer_wrapper.go index ca2e4c14c71..f29f9b7c962 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request/resp_writer_wrapper.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request/resp_writer_wrapper.go @@ -61,7 +61,7 @@ func (w *RespWriterWrapper) Write(p []byte) (int, error) { // WriteHeader persists initial statusCode for span attribution. // All calls to WriteHeader will be propagated to the underlying ResponseWriter -// and will persist the statusCode from the first call. +// and will persist the statusCode from the first call (except for informational response status codes). // Blocking consecutive calls to WriteHeader alters expected behavior and will // remove warning logs from net/http where developers will notice incorrect handler implementations. func (w *RespWriterWrapper) WriteHeader(statusCode int) { @@ -77,6 +77,13 @@ func (w *RespWriterWrapper) WriteHeader(statusCode int) { // parent method. func (w *RespWriterWrapper) writeHeader(statusCode int) { if !w.wroteHeader { + // Ignore informational response status codes. + // Based on https://github.com/golang/go/blob/go1.24.1/src/net/http/server.go#L1216 + if statusCode >= 100 && statusCode <= 199 && statusCode != http.StatusSwitchingProtocols { + w.ResponseWriter.WriteHeader(statusCode) + return + } + w.wroteHeader = true w.statusCode = statusCode } diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/client.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/client.go index bf14b12410d..1398d85c2e9 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/client.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/client.go @@ -19,8 +19,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/semconv/v1.39.0" - "go.opentelemetry.io/otel/semconv/v1.39.0/httpconv" + "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/semconv/v1.40.0/httpconv" ) type HTTPClient struct { diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/server.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/server.go index dbae2ad8486..83c6ae24653 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/server.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/server.go @@ -20,8 +20,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/semconv/v1.39.0" - "go.opentelemetry.io/otel/semconv/v1.39.0/httpconv" + "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/semconv/v1.40.0/httpconv" ) type RequestTraceAttrsOpts struct { @@ -364,7 +364,9 @@ func (n HTTPServer) MetricAttributes(server string, req *http.Request, statusCod if statusCode > 0 { num++ } - + if route == "" && req.Pattern != "" { + route = httpRoute(req.Pattern) + } if route != "" { num++ } diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/util.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/util.go index 40e81b5e33d..2eab2ecabde 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/util.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv/util.go @@ -15,7 +15,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - semconvNew "go.opentelemetry.io/otel/semconv/v1.39.0" + semconvNew "go.opentelemetry.io/otel/semconv/v1.40.0" ) // SplitHostPort splits a network address hostport of the form "host", diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go index 541350ff8d8..d8d204d1f8a 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go @@ -5,7 +5,6 @@ package otelhttp // import "go.opentelemetry.io/contrib/instrumentation/net/http import ( "context" - "fmt" "io" "net/http" "net/http/httptrace" @@ -16,7 +15,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/propagation" - otelsemconv "go.opentelemetry.io/otel/semconv/v1.39.0" + otelsemconv "go.opentelemetry.io/otel/semconv/v1.40.0" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request" @@ -85,8 +84,6 @@ func defaultTransportFormatter(_ string, r *http.Request) string { // RoundTrip creates a Span and propagates its context via the provided request's headers // before handing the request to the configured base RoundTripper. The created span will // end when the response body is closed or when a read from the body returns io.EOF. -// If GetBody returns an error, the error is reported via otel.Handle and the request -// continues with the original Body. func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) { requestStartTime := time.Now() for _, f := range t.filters { @@ -119,23 +116,26 @@ func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) { r = r.Clone(ctx) // According to RoundTripper spec, we shouldn't modify the origin request. - // GetBody is preferred over direct access to Body if the function is set. - // If the resulting body is nil or is NoBody, we don't want to mutate the body as it - // will affect the identity of it in an unforeseeable way because we assert - // ReadCloser fulfills a certain interface and it is indeed nil or NoBody. - body := r.Body - if r.GetBody != nil { - b, err := r.GetBody() - if err != nil { - otel.Handle(fmt.Errorf("http.Request GetBody returned an error: %w", err)) - } else { - body = b + var lastBW *request.BodyWrapper // Records the last body wrapper. Can be nil. + maybeWrapBody := func(body io.ReadCloser) io.ReadCloser { + if body == nil || body == http.NoBody { + return body } + bw := request.NewBodyWrapper(body, func(int64) {}) + lastBW = bw + return bw } - - bw := request.NewBodyWrapper(body, func(int64) {}) - if body != nil && body != http.NoBody { - r.Body = bw + r.Body = maybeWrapBody(r.Body) + if r.GetBody != nil { + originalGetBody := r.GetBody + r.GetBody = func() (io.ReadCloser, error) { + b, err := originalGetBody() + if err != nil { + lastBW = nil // The underlying transport will fail to make a retry request, hence, record no data. + return nil, err + } + return maybeWrapBody(b), nil + } } span.SetAttributes(t.semconv.RequestTraceAttrs(r)...) @@ -148,10 +148,14 @@ func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) { if err == nil { statusCode = res.StatusCode } + var requestSize int64 + if lastBW != nil { + requestSize = lastBW.BytesRead() + } t.semconv.RecordMetrics( ctx, semconv.MetricData{ - RequestSize: bw.BytesRead(), + RequestSize: requestSize, RequestDuration: time.Since(requestStartTime), }, t.semconv.MetricOptions(semconv.MetricAttributes{ diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go index 0f2518f0966..835ec5aa7e8 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go @@ -4,4 +4,4 @@ package otelhttp // import "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" // Version is the current release version of the otelhttp instrumentation. -const Version = "0.66.0" +const Version = "0.68.0" diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go index 05cb2334317..4ae569ff4b1 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go @@ -32,6 +32,13 @@ import ( const contentTypeProto = "application/x-protobuf" +// maxResponseBodySize is the maximum number of bytes to read from a response +// body. It is set to 4 MiB per the OTLP specification recommendation to +// mitigate excessive memory usage caused by a misconfigured or malicious +// server. If exceeded, the response is treated as a not-retryable error. +// This is a variable to allow tests to override it. +var maxResponseBodySize int64 = 4 * 1024 * 1024 + var gzPool = sync.Pool{ New: func() any { w := gzip.NewWriter(io.Discard) @@ -203,7 +210,11 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc // Success, do not retry. // Read the partial success message, if any. var respData bytes.Buffer - if _, err := io.Copy(&respData, resp.Body); err != nil { + if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit) + } return err } if respData.Len() == 0 { @@ -234,7 +245,11 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc // message to be returned. It will help in // debugging the actual issue. var respData bytes.Buffer - if _, err := io.Copy(&respData, resp.Body); err != nil { + if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit) + } return err } respStr := strings.TrimSpace(respData.String()) @@ -259,7 +274,7 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc func (d *client) newRequest(body []byte) (request, error) { u := url.URL{Scheme: d.getScheme(), Host: d.cfg.Endpoint, Path: d.cfg.URLPath} - r, err := http.NewRequest(http.MethodPost, u.String(), http.NoBody) + r, err := http.NewRequestWithContext(context.Background(), http.MethodPost, u.String(), http.NoBody) if err != nil { return request{Request: r}, err } diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/doc.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/doc.go index 9fea75ad19c..85645e118e5 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/doc.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/doc.go @@ -24,6 +24,11 @@ The value may additionally contain a port and a path. The value should not contain a query string or fragment. The configuration can be overridden by [WithEndpoint], [WithEndpointURL], [WithInsecure], and [WithURLPath] options. +OTEL_EXPORTER_OTLP_INSECURE, OTEL_EXPORTER_OTLP_TRACES_INSECURE (default: "false") - +setting "true" disables client transport security for the exporter's HTTP connection. +OTEL_EXPORTER_OTLP_TRACES_INSECURE takes precedence over OTEL_EXPORTER_OTLP_INSECURE. +The configuration can be overridden by [WithInsecure] and [WithTLSClientConfig] options. + OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_TRACES_HEADERS (default: none) - key-value pairs used as headers associated with HTTP requests. The value is expected to be represented in a format matching the [W3C Baggage HTTP Header Content Format], diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go index 5def1ab81ce..3f2556e7a60 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go @@ -23,8 +23,8 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/x" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.39.0" - "go.opentelemetry.io/otel/semconv/v1.39.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" ) const ( diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go index e6176c1bba9..3e43f771131 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go @@ -5,4 +5,4 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlptrace/ot // Version is the current release version of the OpenTelemetry OTLP HTTP trace // exporter in use. -const Version = "1.41.0" +const Version = "1.43.0" diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md deleted file mode 100644 index fed7013e6ff..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md +++ /dev/null @@ -1,78 +0,0 @@ - -# Migration from v1.38.0 to v1.39.0 - -The `go.opentelemetry.io/otel/semconv/v1.39.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.38.0` with the following exceptions. - -## Removed - -The following declarations have been removed. -Refer to the [OpenTelemetry Semantic Conventions documentation] for deprecation instructions. - -If the type is not listed in the documentation as deprecated, it has been removed in this version due to lack of applicability or use. -If you use any of these non-deprecated declarations in your Go application, please [open an issue] describing your use-case. - -- `LinuxMemorySlabStateKey` -- `LinuxMemorySlabStateReclaimable` -- `LinuxMemorySlabStateUnreclaimable` -- `PeerService` -- `PeerServiceKey` -- `RPCConnectRPCErrorCodeAborted` -- `RPCConnectRPCErrorCodeAlreadyExists` -- `RPCConnectRPCErrorCodeCancelled` -- `RPCConnectRPCErrorCodeDataLoss` -- `RPCConnectRPCErrorCodeDeadlineExceeded` -- `RPCConnectRPCErrorCodeFailedPrecondition` -- `RPCConnectRPCErrorCodeInternal` -- `RPCConnectRPCErrorCodeInvalidArgument` -- `RPCConnectRPCErrorCodeKey` -- `RPCConnectRPCErrorCodeNotFound` -- `RPCConnectRPCErrorCodeOutOfRange` -- `RPCConnectRPCErrorCodePermissionDenied` -- `RPCConnectRPCErrorCodeResourceExhausted` -- `RPCConnectRPCErrorCodeUnauthenticated` -- `RPCConnectRPCErrorCodeUnavailable` -- `RPCConnectRPCErrorCodeUnimplemented` -- `RPCConnectRPCErrorCodeUnknown` -- `RPCConnectRPCRequestMetadata` -- `RPCConnectRPCResponseMetadata` -- `RPCGRPCRequestMetadata` -- `RPCGRPCResponseMetadata` -- `RPCGRPCStatusCodeAborted` -- `RPCGRPCStatusCodeAlreadyExists` -- `RPCGRPCStatusCodeCancelled` -- `RPCGRPCStatusCodeDataLoss` -- `RPCGRPCStatusCodeDeadlineExceeded` -- `RPCGRPCStatusCodeFailedPrecondition` -- `RPCGRPCStatusCodeInternal` -- `RPCGRPCStatusCodeInvalidArgument` -- `RPCGRPCStatusCodeKey` -- `RPCGRPCStatusCodeNotFound` -- `RPCGRPCStatusCodeOk` -- `RPCGRPCStatusCodeOutOfRange` -- `RPCGRPCStatusCodePermissionDenied` -- `RPCGRPCStatusCodeResourceExhausted` -- `RPCGRPCStatusCodeUnauthenticated` -- `RPCGRPCStatusCodeUnavailable` -- `RPCGRPCStatusCodeUnimplemented` -- `RPCGRPCStatusCodeUnknown` -- `RPCJSONRPCErrorCode` -- `RPCJSONRPCErrorCodeKey` -- `RPCJSONRPCErrorMessage` -- `RPCJSONRPCErrorMessageKey` -- `RPCJSONRPCRequestID` -- `RPCJSONRPCRequestIDKey` -- `RPCJSONRPCVersion` -- `RPCJSONRPCVersionKey` -- `RPCService` -- `RPCServiceKey` -- `RPCSystemApacheDubbo` -- `RPCSystemConnectRPC` -- `RPCSystemDotnetWcf` -- `RPCSystemGRPC` -- `RPCSystemJSONRPC` -- `RPCSystemJavaRmi` -- `RPCSystemKey` -- `RPCSystemOncRPC` - -[OpenTelemetry Semantic Conventions documentation]: https://github.com/open-telemetry/semantic-conventions -[open an issue]: https://github.com/open-telemetry/opentelemetry-go/issues/new?template=Blank+issue diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md deleted file mode 100644 index 4b0e6f7f3eb..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Semconv v1.39.0 - -[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/semconv/v1.39.0)](https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.39.0) diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go deleted file mode 100644 index dfcee964a62..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go +++ /dev/null @@ -1,16243 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -// Code generated from semantic convention specification. DO NOT EDIT. - -package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" - -import "go.opentelemetry.io/otel/attribute" - -// Namespace: android -const ( - // AndroidAppStateKey is the attribute Key conforming to the "android.app.state" - // semantic conventions. It represents the this attribute represents the state - // of the application. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "created" - // Note: The Android lifecycle states are defined in - // [Activity lifecycle callbacks], and from which the `OS identifiers` are - // derived. - // - // [Activity lifecycle callbacks]: https://developer.android.com/guide/components/activities/activity-lifecycle#lc - AndroidAppStateKey = attribute.Key("android.app.state") - - // AndroidOSAPILevelKey is the attribute Key conforming to the - // "android.os.api_level" semantic conventions. It represents the uniquely - // identifies the framework API revision offered by a version (`os.version`) of - // the android operating system. More information can be found in the - // [Android API levels documentation]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "33", "32" - // - // [Android API levels documentation]: https://developer.android.com/guide/topics/manifest/uses-sdk-element#ApiLevels - AndroidOSAPILevelKey = attribute.Key("android.os.api_level") -) - -// AndroidOSAPILevel returns an attribute KeyValue conforming to the -// "android.os.api_level" semantic conventions. It represents the uniquely -// identifies the framework API revision offered by a version (`os.version`) of -// the android operating system. More information can be found in the -// [Android API levels documentation]. -// -// [Android API levels documentation]: https://developer.android.com/guide/topics/manifest/uses-sdk-element#ApiLevels -func AndroidOSAPILevel(val string) attribute.KeyValue { - return AndroidOSAPILevelKey.String(val) -} - -// Enum values for android.app.state -var ( - // Any time before Activity.onResume() or, if the app has no Activity, - // Context.startService() has been called in the app for the first time. - // - // Stability: development - AndroidAppStateCreated = AndroidAppStateKey.String("created") - // Any time after Activity.onPause() or, if the app has no Activity, - // Context.stopService() has been called when the app was in the foreground - // state. - // - // Stability: development - AndroidAppStateBackground = AndroidAppStateKey.String("background") - // Any time after Activity.onResume() or, if the app has no Activity, - // Context.startService() has been called when the app was in either the created - // or background states. - // - // Stability: development - AndroidAppStateForeground = AndroidAppStateKey.String("foreground") -) - -// Namespace: app -const ( - // AppBuildIDKey is the attribute Key conforming to the "app.build_id" semantic - // conventions. It represents the unique identifier for a particular build or - // compilation of the application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "6cff0a7e-cefc-4668-96f5-1273d8b334d0", - // "9f2b833506aa6973a92fde9733e6271f", "my-app-1.0.0-code-123" - AppBuildIDKey = attribute.Key("app.build_id") - - // AppInstallationIDKey is the attribute Key conforming to the - // "app.installation.id" semantic conventions. It represents a unique identifier - // representing the installation of an application on a specific device. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2ab2916d-a51f-4ac8-80ee-45ac31a28092" - // Note: Its value SHOULD persist across launches of the same application - // installation, including through application upgrades. - // It SHOULD change if the application is uninstalled or if all applications of - // the vendor are uninstalled. - // Additionally, users might be able to reset this value (e.g. by clearing - // application data). - // If an app is installed multiple times on the same device (e.g. in different - // accounts on Android), each `app.installation.id` SHOULD have a different - // value. - // If multiple OpenTelemetry SDKs are used within the same application, they - // SHOULD use the same value for `app.installation.id`. - // Hardware IDs (e.g. serial number, IMEI, MAC address) MUST NOT be used as the - // `app.installation.id`. - // - // For iOS, this value SHOULD be equal to the [vendor identifier]. - // - // For Android, examples of `app.installation.id` implementations include: - // - // - [Firebase Installation ID]. - // - A globally unique UUID which is persisted across sessions in your - // application. - // - [App set ID]. - // - [`Settings.getString(Settings.Secure.ANDROID_ID)`]. - // - // More information about Android identifier best practices can be found in the - // [Android user data IDs guide]. - // - // [vendor identifier]: https://developer.apple.com/documentation/uikit/uidevice/identifierforvendor - // [Firebase Installation ID]: https://firebase.google.com/docs/projects/manage-installations - // [App set ID]: https://developer.android.com/identity/app-set-id - // [`Settings.getString(Settings.Secure.ANDROID_ID)`]: https://developer.android.com/reference/android/provider/Settings.Secure#ANDROID_ID - // [Android user data IDs guide]: https://developer.android.com/training/articles/user-data-ids - AppInstallationIDKey = attribute.Key("app.installation.id") - - // AppJankFrameCountKey is the attribute Key conforming to the - // "app.jank.frame_count" semantic conventions. It represents a number of frame - // renders that experienced jank. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 9, 42 - // Note: Depending on platform limitations, the value provided MAY be - // approximation. - AppJankFrameCountKey = attribute.Key("app.jank.frame_count") - - // AppJankPeriodKey is the attribute Key conforming to the "app.jank.period" - // semantic conventions. It represents the time period, in seconds, for which - // this jank is being reported. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0, 5.0, 10.24 - AppJankPeriodKey = attribute.Key("app.jank.period") - - // AppJankThresholdKey is the attribute Key conforming to the - // "app.jank.threshold" semantic conventions. It represents the minimum - // rendering threshold for this jank, in seconds. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0.016, 0.7, 1.024 - AppJankThresholdKey = attribute.Key("app.jank.threshold") - - // AppScreenCoordinateXKey is the attribute Key conforming to the - // "app.screen.coordinate.x" semantic conventions. It represents the x - // (horizontal) coordinate of a screen coordinate, in screen pixels. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0, 131 - AppScreenCoordinateXKey = attribute.Key("app.screen.coordinate.x") - - // AppScreenCoordinateYKey is the attribute Key conforming to the - // "app.screen.coordinate.y" semantic conventions. It represents the y - // (vertical) component of a screen coordinate, in screen pixels. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 12, 99 - AppScreenCoordinateYKey = attribute.Key("app.screen.coordinate.y") - - // AppScreenIDKey is the attribute Key conforming to the "app.screen.id" - // semantic conventions. It represents an identifier that uniquely - // differentiates this screen from other screens in the same application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "f9bc787d-ff05-48ad-90e1-fca1d46130b3", - // "com.example.app.MainActivity", "com.example.shop.ProductDetailFragment", - // "MyApp.ProfileView", "MyApp.ProfileViewController" - // Note: A screen represents only the part of the device display drawn by the - // app. It typically contains multiple widgets or UI components and is larger in - // scope than individual widgets. Multiple screens can coexist on the same - // display simultaneously (e.g., split view on tablets). - AppScreenIDKey = attribute.Key("app.screen.id") - - // AppScreenNameKey is the attribute Key conforming to the "app.screen.name" - // semantic conventions. It represents the name of an application screen. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MainActivity", "ProductDetailFragment", "ProfileView", - // "ProfileViewController" - // Note: A screen represents only the part of the device display drawn by the - // app. It typically contains multiple widgets or UI components and is larger in - // scope than individual widgets. Multiple screens can coexist on the same - // display simultaneously (e.g., split view on tablets). - AppScreenNameKey = attribute.Key("app.screen.name") - - // AppWidgetIDKey is the attribute Key conforming to the "app.widget.id" - // semantic conventions. It represents an identifier that uniquely - // differentiates this widget from other widgets in the same application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "f9bc787d-ff05-48ad-90e1-fca1d46130b3", "submit_order_1829" - // Note: A widget is an application component, typically an on-screen visual GUI - // element. - AppWidgetIDKey = attribute.Key("app.widget.id") - - // AppWidgetNameKey is the attribute Key conforming to the "app.widget.name" - // semantic conventions. It represents the name of an application widget. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "submit", "attack", "Clear Cart" - // Note: A widget is an application component, typically an on-screen visual GUI - // element. - AppWidgetNameKey = attribute.Key("app.widget.name") -) - -// AppBuildID returns an attribute KeyValue conforming to the "app.build_id" -// semantic conventions. It represents the unique identifier for a particular -// build or compilation of the application. -func AppBuildID(val string) attribute.KeyValue { - return AppBuildIDKey.String(val) -} - -// AppInstallationID returns an attribute KeyValue conforming to the -// "app.installation.id" semantic conventions. It represents a unique identifier -// representing the installation of an application on a specific device. -func AppInstallationID(val string) attribute.KeyValue { - return AppInstallationIDKey.String(val) -} - -// AppJankFrameCount returns an attribute KeyValue conforming to the -// "app.jank.frame_count" semantic conventions. It represents a number of frame -// renders that experienced jank. -func AppJankFrameCount(val int) attribute.KeyValue { - return AppJankFrameCountKey.Int(val) -} - -// AppJankPeriod returns an attribute KeyValue conforming to the -// "app.jank.period" semantic conventions. It represents the time period, in -// seconds, for which this jank is being reported. -func AppJankPeriod(val float64) attribute.KeyValue { - return AppJankPeriodKey.Float64(val) -} - -// AppJankThreshold returns an attribute KeyValue conforming to the -// "app.jank.threshold" semantic conventions. It represents the minimum rendering -// threshold for this jank, in seconds. -func AppJankThreshold(val float64) attribute.KeyValue { - return AppJankThresholdKey.Float64(val) -} - -// AppScreenCoordinateX returns an attribute KeyValue conforming to the -// "app.screen.coordinate.x" semantic conventions. It represents the x -// (horizontal) coordinate of a screen coordinate, in screen pixels. -func AppScreenCoordinateX(val int) attribute.KeyValue { - return AppScreenCoordinateXKey.Int(val) -} - -// AppScreenCoordinateY returns an attribute KeyValue conforming to the -// "app.screen.coordinate.y" semantic conventions. It represents the y (vertical) -// component of a screen coordinate, in screen pixels. -func AppScreenCoordinateY(val int) attribute.KeyValue { - return AppScreenCoordinateYKey.Int(val) -} - -// AppScreenID returns an attribute KeyValue conforming to the "app.screen.id" -// semantic conventions. It represents an identifier that uniquely differentiates -// this screen from other screens in the same application. -func AppScreenID(val string) attribute.KeyValue { - return AppScreenIDKey.String(val) -} - -// AppScreenName returns an attribute KeyValue conforming to the -// "app.screen.name" semantic conventions. It represents the name of an -// application screen. -func AppScreenName(val string) attribute.KeyValue { - return AppScreenNameKey.String(val) -} - -// AppWidgetID returns an attribute KeyValue conforming to the "app.widget.id" -// semantic conventions. It represents an identifier that uniquely differentiates -// this widget from other widgets in the same application. -func AppWidgetID(val string) attribute.KeyValue { - return AppWidgetIDKey.String(val) -} - -// AppWidgetName returns an attribute KeyValue conforming to the -// "app.widget.name" semantic conventions. It represents the name of an -// application widget. -func AppWidgetName(val string) attribute.KeyValue { - return AppWidgetNameKey.String(val) -} - -// Namespace: artifact -const ( - // ArtifactAttestationFilenameKey is the attribute Key conforming to the - // "artifact.attestation.filename" semantic conventions. It represents the - // provenance filename of the built attestation which directly relates to the - // build artifact filename. This filename SHOULD accompany the artifact at - // publish time. See the [SLSA Relationship] specification for more information. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "golang-binary-amd64-v0.1.0.attestation", - // "docker-image-amd64-v0.1.0.intoto.json1", "release-1.tar.gz.attestation", - // "file-name-package.tar.gz.intoto.json1" - // - // [SLSA Relationship]: https://slsa.dev/spec/v1.0/distributing-provenance#relationship-between-artifacts-and-attestations - ArtifactAttestationFilenameKey = attribute.Key("artifact.attestation.filename") - - // ArtifactAttestationHashKey is the attribute Key conforming to the - // "artifact.attestation.hash" semantic conventions. It represents the full - // [hash value (see glossary)], of the built attestation. Some envelopes in the - // [software attestation space] also refer to this as the **digest**. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1b31dfcd5b7f9267bf2ff47651df1cfb9147b9e4df1f335accf65b4cda498408" - // - // [hash value (see glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf - // [software attestation space]: https://github.com/in-toto/attestation/tree/main/spec - ArtifactAttestationHashKey = attribute.Key("artifact.attestation.hash") - - // ArtifactAttestationIDKey is the attribute Key conforming to the - // "artifact.attestation.id" semantic conventions. It represents the id of the - // build [software attestation]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "123" - // - // [software attestation]: https://slsa.dev/attestation-model - ArtifactAttestationIDKey = attribute.Key("artifact.attestation.id") - - // ArtifactFilenameKey is the attribute Key conforming to the - // "artifact.filename" semantic conventions. It represents the human readable - // file name of the artifact, typically generated during build and release - // processes. Often includes the package name and version in the file name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "golang-binary-amd64-v0.1.0", "docker-image-amd64-v0.1.0", - // "release-1.tar.gz", "file-name-package.tar.gz" - // Note: This file name can also act as the [Package Name] - // in cases where the package ecosystem maps accordingly. - // Additionally, the artifact [can be published] - // for others, but that is not a guarantee. - // - // [Package Name]: https://slsa.dev/spec/v1.0/terminology#package-model - // [can be published]: https://slsa.dev/spec/v1.0/terminology#software-supply-chain - ArtifactFilenameKey = attribute.Key("artifact.filename") - - // ArtifactHashKey is the attribute Key conforming to the "artifact.hash" - // semantic conventions. It represents the full [hash value (see glossary)], - // often found in checksum.txt on a release of the artifact and used to verify - // package integrity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9ff4c52759e2c4ac70b7d517bc7fcdc1cda631ca0045271ddd1b192544f8a3e9" - // Note: The specific algorithm used to create the cryptographic hash value is - // not defined. In situations where an artifact has multiple - // cryptographic hashes, it is up to the implementer to choose which - // hash value to set here; this should be the most secure hash algorithm - // that is suitable for the situation and consistent with the - // corresponding attestation. The implementer can then provide the other - // hash values through an additional set of attribute extensions as they - // deem necessary. - // - // [hash value (see glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf - ArtifactHashKey = attribute.Key("artifact.hash") - - // ArtifactPurlKey is the attribute Key conforming to the "artifact.purl" - // semantic conventions. It represents the [Package URL] of the - // [package artifact] provides a standard way to identify and locate the - // packaged artifact. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "pkg:github/package-url/purl-spec@1209109710924", - // "pkg:npm/foo@12.12.3" - // - // [Package URL]: https://github.com/package-url/purl-spec - // [package artifact]: https://slsa.dev/spec/v1.0/terminology#package-model - ArtifactPurlKey = attribute.Key("artifact.purl") - - // ArtifactVersionKey is the attribute Key conforming to the "artifact.version" - // semantic conventions. It represents the version of the artifact. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "v0.1.0", "1.2.1", "122691-build" - ArtifactVersionKey = attribute.Key("artifact.version") -) - -// ArtifactAttestationFilename returns an attribute KeyValue conforming to the -// "artifact.attestation.filename" semantic conventions. It represents the -// provenance filename of the built attestation which directly relates to the -// build artifact filename. This filename SHOULD accompany the artifact at -// publish time. See the [SLSA Relationship] specification for more information. -// -// [SLSA Relationship]: https://slsa.dev/spec/v1.0/distributing-provenance#relationship-between-artifacts-and-attestations -func ArtifactAttestationFilename(val string) attribute.KeyValue { - return ArtifactAttestationFilenameKey.String(val) -} - -// ArtifactAttestationHash returns an attribute KeyValue conforming to the -// "artifact.attestation.hash" semantic conventions. It represents the full -// [hash value (see glossary)], of the built attestation. Some envelopes in the -// [software attestation space] also refer to this as the **digest**. -// -// [hash value (see glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf -// [software attestation space]: https://github.com/in-toto/attestation/tree/main/spec -func ArtifactAttestationHash(val string) attribute.KeyValue { - return ArtifactAttestationHashKey.String(val) -} - -// ArtifactAttestationID returns an attribute KeyValue conforming to the -// "artifact.attestation.id" semantic conventions. It represents the id of the -// build [software attestation]. -// -// [software attestation]: https://slsa.dev/attestation-model -func ArtifactAttestationID(val string) attribute.KeyValue { - return ArtifactAttestationIDKey.String(val) -} - -// ArtifactFilename returns an attribute KeyValue conforming to the -// "artifact.filename" semantic conventions. It represents the human readable -// file name of the artifact, typically generated during build and release -// processes. Often includes the package name and version in the file name. -func ArtifactFilename(val string) attribute.KeyValue { - return ArtifactFilenameKey.String(val) -} - -// ArtifactHash returns an attribute KeyValue conforming to the "artifact.hash" -// semantic conventions. It represents the full [hash value (see glossary)], -// often found in checksum.txt on a release of the artifact and used to verify -// package integrity. -// -// [hash value (see glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf -func ArtifactHash(val string) attribute.KeyValue { - return ArtifactHashKey.String(val) -} - -// ArtifactPurl returns an attribute KeyValue conforming to the "artifact.purl" -// semantic conventions. It represents the [Package URL] of the -// [package artifact] provides a standard way to identify and locate the packaged -// artifact. -// -// [Package URL]: https://github.com/package-url/purl-spec -// [package artifact]: https://slsa.dev/spec/v1.0/terminology#package-model -func ArtifactPurl(val string) attribute.KeyValue { - return ArtifactPurlKey.String(val) -} - -// ArtifactVersion returns an attribute KeyValue conforming to the -// "artifact.version" semantic conventions. It represents the version of the -// artifact. -func ArtifactVersion(val string) attribute.KeyValue { - return ArtifactVersionKey.String(val) -} - -// Namespace: aws -const ( - // AWSBedrockGuardrailIDKey is the attribute Key conforming to the - // "aws.bedrock.guardrail.id" semantic conventions. It represents the unique - // identifier of the AWS Bedrock Guardrail. A [guardrail] helps safeguard and - // prevent unwanted behavior from model responses or user messages. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "sgi5gkybzqak" - // - // [guardrail]: https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html - AWSBedrockGuardrailIDKey = attribute.Key("aws.bedrock.guardrail.id") - - // AWSBedrockKnowledgeBaseIDKey is the attribute Key conforming to the - // "aws.bedrock.knowledge_base.id" semantic conventions. It represents the - // unique identifier of the AWS Bedrock Knowledge base. A [knowledge base] is a - // bank of information that can be queried by models to generate more relevant - // responses and augment prompts. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "XFWUPB9PAW" - // - // [knowledge base]: https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html - AWSBedrockKnowledgeBaseIDKey = attribute.Key("aws.bedrock.knowledge_base.id") - - // AWSDynamoDBAttributeDefinitionsKey is the attribute Key conforming to the - // "aws.dynamodb.attribute_definitions" semantic conventions. It represents the - // JSON-serialized value of each item in the `AttributeDefinitions` request - // field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "AttributeName": "string", "AttributeType": "string" }" - AWSDynamoDBAttributeDefinitionsKey = attribute.Key("aws.dynamodb.attribute_definitions") - - // AWSDynamoDBAttributesToGetKey is the attribute Key conforming to the - // "aws.dynamodb.attributes_to_get" semantic conventions. It represents the - // value of the `AttributesToGet` request parameter. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "lives", "id" - AWSDynamoDBAttributesToGetKey = attribute.Key("aws.dynamodb.attributes_to_get") - - // AWSDynamoDBConsistentReadKey is the attribute Key conforming to the - // "aws.dynamodb.consistent_read" semantic conventions. It represents the value - // of the `ConsistentRead` request parameter. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - AWSDynamoDBConsistentReadKey = attribute.Key("aws.dynamodb.consistent_read") - - // AWSDynamoDBConsumedCapacityKey is the attribute Key conforming to the - // "aws.dynamodb.consumed_capacity" semantic conventions. It represents the - // JSON-serialized value of each item in the `ConsumedCapacity` response field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "CapacityUnits": number, "GlobalSecondaryIndexes": { "string" : - // { "CapacityUnits": number, "ReadCapacityUnits": number, "WriteCapacityUnits": - // number } }, "LocalSecondaryIndexes": { "string" : { "CapacityUnits": number, - // "ReadCapacityUnits": number, "WriteCapacityUnits": number } }, - // "ReadCapacityUnits": number, "Table": { "CapacityUnits": number, - // "ReadCapacityUnits": number, "WriteCapacityUnits": number }, "TableName": - // "string", "WriteCapacityUnits": number }" - AWSDynamoDBConsumedCapacityKey = attribute.Key("aws.dynamodb.consumed_capacity") - - // AWSDynamoDBCountKey is the attribute Key conforming to the - // "aws.dynamodb.count" semantic conventions. It represents the value of the - // `Count` response parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 10 - AWSDynamoDBCountKey = attribute.Key("aws.dynamodb.count") - - // AWSDynamoDBExclusiveStartTableKey is the attribute Key conforming to the - // "aws.dynamodb.exclusive_start_table" semantic conventions. It represents the - // value of the `ExclusiveStartTableName` request parameter. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Users", "CatsTable" - AWSDynamoDBExclusiveStartTableKey = attribute.Key("aws.dynamodb.exclusive_start_table") - - // AWSDynamoDBGlobalSecondaryIndexUpdatesKey is the attribute Key conforming to - // the "aws.dynamodb.global_secondary_index_updates" semantic conventions. It - // represents the JSON-serialized value of each item in the - // `GlobalSecondaryIndexUpdates` request field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "Create": { "IndexName": "string", "KeySchema": [ { - // "AttributeName": "string", "KeyType": "string" } ], "Projection": { - // "NonKeyAttributes": [ "string" ], "ProjectionType": "string" }, - // "ProvisionedThroughput": { "ReadCapacityUnits": number, "WriteCapacityUnits": - // number } }" - AWSDynamoDBGlobalSecondaryIndexUpdatesKey = attribute.Key("aws.dynamodb.global_secondary_index_updates") - - // AWSDynamoDBGlobalSecondaryIndexesKey is the attribute Key conforming to the - // "aws.dynamodb.global_secondary_indexes" semantic conventions. It represents - // the JSON-serialized value of each item of the `GlobalSecondaryIndexes` - // request field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "IndexName": "string", "KeySchema": [ { "AttributeName": - // "string", "KeyType": "string" } ], "Projection": { "NonKeyAttributes": [ - // "string" ], "ProjectionType": "string" }, "ProvisionedThroughput": { - // "ReadCapacityUnits": number, "WriteCapacityUnits": number } }" - AWSDynamoDBGlobalSecondaryIndexesKey = attribute.Key("aws.dynamodb.global_secondary_indexes") - - // AWSDynamoDBIndexNameKey is the attribute Key conforming to the - // "aws.dynamodb.index_name" semantic conventions. It represents the value of - // the `IndexName` request parameter. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "name_to_group" - AWSDynamoDBIndexNameKey = attribute.Key("aws.dynamodb.index_name") - - // AWSDynamoDBItemCollectionMetricsKey is the attribute Key conforming to the - // "aws.dynamodb.item_collection_metrics" semantic conventions. It represents - // the JSON-serialized value of the `ItemCollectionMetrics` response field. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "string" : [ { "ItemCollectionKey": { "string" : { "B": blob, - // "BOOL": boolean, "BS": [ blob ], "L": [ "AttributeValue" ], "M": { "string" : - // "AttributeValue" }, "N": "string", "NS": [ "string" ], "NULL": boolean, "S": - // "string", "SS": [ "string" ] } }, "SizeEstimateRangeGB": [ number ] } ] }" - AWSDynamoDBItemCollectionMetricsKey = attribute.Key("aws.dynamodb.item_collection_metrics") - - // AWSDynamoDBLimitKey is the attribute Key conforming to the - // "aws.dynamodb.limit" semantic conventions. It represents the value of the - // `Limit` request parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 10 - AWSDynamoDBLimitKey = attribute.Key("aws.dynamodb.limit") - - // AWSDynamoDBLocalSecondaryIndexesKey is the attribute Key conforming to the - // "aws.dynamodb.local_secondary_indexes" semantic conventions. It represents - // the JSON-serialized value of each item of the `LocalSecondaryIndexes` request - // field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{ "IndexArn": "string", "IndexName": "string", "IndexSizeBytes": - // number, "ItemCount": number, "KeySchema": [ { "AttributeName": "string", - // "KeyType": "string" } ], "Projection": { "NonKeyAttributes": [ "string" ], - // "ProjectionType": "string" } }" - AWSDynamoDBLocalSecondaryIndexesKey = attribute.Key("aws.dynamodb.local_secondary_indexes") - - // AWSDynamoDBProjectionKey is the attribute Key conforming to the - // "aws.dynamodb.projection" semantic conventions. It represents the value of - // the `ProjectionExpression` request parameter. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Title", "Title, Price, Color", "Title, Description, RelatedItems, - // ProductReviews" - AWSDynamoDBProjectionKey = attribute.Key("aws.dynamodb.projection") - - // AWSDynamoDBProvisionedReadCapacityKey is the attribute Key conforming to the - // "aws.dynamodb.provisioned_read_capacity" semantic conventions. It represents - // the value of the `ProvisionedThroughput.ReadCapacityUnits` request parameter. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0, 2.0 - AWSDynamoDBProvisionedReadCapacityKey = attribute.Key("aws.dynamodb.provisioned_read_capacity") - - // AWSDynamoDBProvisionedWriteCapacityKey is the attribute Key conforming to the - // "aws.dynamodb.provisioned_write_capacity" semantic conventions. It represents - // the value of the `ProvisionedThroughput.WriteCapacityUnits` request - // parameter. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0, 2.0 - AWSDynamoDBProvisionedWriteCapacityKey = attribute.Key("aws.dynamodb.provisioned_write_capacity") - - // AWSDynamoDBScanForwardKey is the attribute Key conforming to the - // "aws.dynamodb.scan_forward" semantic conventions. It represents the value of - // the `ScanIndexForward` request parameter. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - AWSDynamoDBScanForwardKey = attribute.Key("aws.dynamodb.scan_forward") - - // AWSDynamoDBScannedCountKey is the attribute Key conforming to the - // "aws.dynamodb.scanned_count" semantic conventions. It represents the value of - // the `ScannedCount` response parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 50 - AWSDynamoDBScannedCountKey = attribute.Key("aws.dynamodb.scanned_count") - - // AWSDynamoDBSegmentKey is the attribute Key conforming to the - // "aws.dynamodb.segment" semantic conventions. It represents the value of the - // `Segment` request parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 10 - AWSDynamoDBSegmentKey = attribute.Key("aws.dynamodb.segment") - - // AWSDynamoDBSelectKey is the attribute Key conforming to the - // "aws.dynamodb.select" semantic conventions. It represents the value of the - // `Select` request parameter. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ALL_ATTRIBUTES", "COUNT" - AWSDynamoDBSelectKey = attribute.Key("aws.dynamodb.select") - - // AWSDynamoDBTableCountKey is the attribute Key conforming to the - // "aws.dynamodb.table_count" semantic conventions. It represents the number of - // items in the `TableNames` response parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 20 - AWSDynamoDBTableCountKey = attribute.Key("aws.dynamodb.table_count") - - // AWSDynamoDBTableNamesKey is the attribute Key conforming to the - // "aws.dynamodb.table_names" semantic conventions. It represents the keys in - // the `RequestItems` object field. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Users", "Cats" - AWSDynamoDBTableNamesKey = attribute.Key("aws.dynamodb.table_names") - - // AWSDynamoDBTotalSegmentsKey is the attribute Key conforming to the - // "aws.dynamodb.total_segments" semantic conventions. It represents the value - // of the `TotalSegments` request parameter. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 100 - AWSDynamoDBTotalSegmentsKey = attribute.Key("aws.dynamodb.total_segments") - - // AWSECSClusterARNKey is the attribute Key conforming to the - // "aws.ecs.cluster.arn" semantic conventions. It represents the ARN of an - // [ECS cluster]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster" - // - // [ECS cluster]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html - AWSECSClusterARNKey = attribute.Key("aws.ecs.cluster.arn") - - // AWSECSContainerARNKey is the attribute Key conforming to the - // "aws.ecs.container.arn" semantic conventions. It represents the Amazon - // Resource Name (ARN) of an [ECS container instance]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "arn:aws:ecs:us-west-1:123456789123:container/32624152-9086-4f0e-acae-1a75b14fe4d9" - // - // [ECS container instance]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_instances.html - AWSECSContainerARNKey = attribute.Key("aws.ecs.container.arn") - - // AWSECSLaunchtypeKey is the attribute Key conforming to the - // "aws.ecs.launchtype" semantic conventions. It represents the [launch type] - // for an ECS task. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [launch type]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/launch_types.html - AWSECSLaunchtypeKey = attribute.Key("aws.ecs.launchtype") - - // AWSECSTaskARNKey is the attribute Key conforming to the "aws.ecs.task.arn" - // semantic conventions. It represents the ARN of a running [ECS task]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "arn:aws:ecs:us-west-1:123456789123:task/10838bed-421f-43ef-870a-f43feacbbb5b", - // "arn:aws:ecs:us-west-1:123456789123:task/my-cluster/task-id/23ebb8ac-c18f-46c6-8bbe-d55d0e37cfbd" - // - // [ECS task]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-account-settings.html#ecs-resource-ids - AWSECSTaskARNKey = attribute.Key("aws.ecs.task.arn") - - // AWSECSTaskFamilyKey is the attribute Key conforming to the - // "aws.ecs.task.family" semantic conventions. It represents the family name of - // the [ECS task definition] used to create the ECS task. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry-family" - // - // [ECS task definition]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html - AWSECSTaskFamilyKey = attribute.Key("aws.ecs.task.family") - - // AWSECSTaskIDKey is the attribute Key conforming to the "aws.ecs.task.id" - // semantic conventions. It represents the ID of a running ECS task. The ID MUST - // be extracted from `task.arn`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "10838bed-421f-43ef-870a-f43feacbbb5b", - // "23ebb8ac-c18f-46c6-8bbe-d55d0e37cfbd" - AWSECSTaskIDKey = attribute.Key("aws.ecs.task.id") - - // AWSECSTaskRevisionKey is the attribute Key conforming to the - // "aws.ecs.task.revision" semantic conventions. It represents the revision for - // the task definition used to create the ECS task. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "8", "26" - AWSECSTaskRevisionKey = attribute.Key("aws.ecs.task.revision") - - // AWSEKSClusterARNKey is the attribute Key conforming to the - // "aws.eks.cluster.arn" semantic conventions. It represents the ARN of an EKS - // cluster. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster" - AWSEKSClusterARNKey = attribute.Key("aws.eks.cluster.arn") - - // AWSExtendedRequestIDKey is the attribute Key conforming to the - // "aws.extended_request_id" semantic conventions. It represents the AWS - // extended request ID as returned in the response header `x-amz-id-2`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "wzHcyEWfmOGDIE5QOhTAqFDoDWP3y8IUvpNINCwL9N4TEHbUw0/gZJ+VZTmCNCWR7fezEN3eCiQ=" - AWSExtendedRequestIDKey = attribute.Key("aws.extended_request_id") - - // AWSKinesisStreamNameKey is the attribute Key conforming to the - // "aws.kinesis.stream_name" semantic conventions. It represents the name of the - // AWS Kinesis [stream] the request refers to. Corresponds to the - // `--stream-name` parameter of the Kinesis [describe-stream] operation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "some-stream-name" - // - // [stream]: https://docs.aws.amazon.com/streams/latest/dev/introduction.html - // [describe-stream]: https://docs.aws.amazon.com/cli/latest/reference/kinesis/describe-stream.html - AWSKinesisStreamNameKey = attribute.Key("aws.kinesis.stream_name") - - // AWSLambdaInvokedARNKey is the attribute Key conforming to the - // "aws.lambda.invoked_arn" semantic conventions. It represents the full invoked - // ARN as provided on the `Context` passed to the function ( - // `Lambda-Runtime-Invoked-Function-Arn` header on the - // `/runtime/invocation/next` applicable). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:lambda:us-east-1:123456:function:myfunction:myalias" - // Note: This may be different from `cloud.resource_id` if an alias is involved. - AWSLambdaInvokedARNKey = attribute.Key("aws.lambda.invoked_arn") - - // AWSLambdaResourceMappingIDKey is the attribute Key conforming to the - // "aws.lambda.resource_mapping.id" semantic conventions. It represents the UUID - // of the [AWS Lambda EvenSource Mapping]. An event source is mapped to a lambda - // function. It's contents are read by Lambda and used to trigger a function. - // This isn't available in the lambda execution context or the lambda runtime - // environtment. This is going to be populated by the AWS SDK for each language - // when that UUID is present. Some of these operations are - // Create/Delete/Get/List/Update EventSourceMapping. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "587ad24b-03b9-4413-8202-bbd56b36e5b7" - // - // [AWS Lambda EvenSource Mapping]: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html - AWSLambdaResourceMappingIDKey = attribute.Key("aws.lambda.resource_mapping.id") - - // AWSLogGroupARNsKey is the attribute Key conforming to the - // "aws.log.group.arns" semantic conventions. It represents the Amazon Resource - // Name(s) (ARN) of the AWS log group(s). - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:logs:us-west-1:123456789012:log-group:/aws/my/group:*" - // Note: See the [log group ARN format documentation]. - // - // [log group ARN format documentation]: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/iam-access-control-overview-cwl.html#CWL_ARN_Format - AWSLogGroupARNsKey = attribute.Key("aws.log.group.arns") - - // AWSLogGroupNamesKey is the attribute Key conforming to the - // "aws.log.group.names" semantic conventions. It represents the name(s) of the - // AWS log group(s) an application is writing to. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/aws/lambda/my-function", "opentelemetry-service" - // Note: Multiple log groups must be supported for cases like multi-container - // applications, where a single application has sidecar containers, and each - // write to their own log group. - AWSLogGroupNamesKey = attribute.Key("aws.log.group.names") - - // AWSLogStreamARNsKey is the attribute Key conforming to the - // "aws.log.stream.arns" semantic conventions. It represents the ARN(s) of the - // AWS log stream(s). - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "arn:aws:logs:us-west-1:123456789012:log-group:/aws/my/group:log-stream:logs/main/10838bed-421f-43ef-870a-f43feacbbb5b" - // Note: See the [log stream ARN format documentation]. One log group can - // contain several log streams, so these ARNs necessarily identify both a log - // group and a log stream. - // - // [log stream ARN format documentation]: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/iam-access-control-overview-cwl.html#CWL_ARN_Format - AWSLogStreamARNsKey = attribute.Key("aws.log.stream.arns") - - // AWSLogStreamNamesKey is the attribute Key conforming to the - // "aws.log.stream.names" semantic conventions. It represents the name(s) of the - // AWS log stream(s) an application is writing to. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "logs/main/10838bed-421f-43ef-870a-f43feacbbb5b" - AWSLogStreamNamesKey = attribute.Key("aws.log.stream.names") - - // AWSRequestIDKey is the attribute Key conforming to the "aws.request_id" - // semantic conventions. It represents the AWS request ID as returned in the - // response headers `x-amzn-requestid`, `x-amzn-request-id` or - // `x-amz-request-id`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "79b9da39-b7ae-508a-a6bc-864b2829c622", "C9ER4AJX75574TDJ" - AWSRequestIDKey = attribute.Key("aws.request_id") - - // AWSS3BucketKey is the attribute Key conforming to the "aws.s3.bucket" - // semantic conventions. It represents the S3 bucket name the request refers to. - // Corresponds to the `--bucket` parameter of the [S3 API] operations. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "some-bucket-name" - // Note: The `bucket` attribute is applicable to all S3 operations that - // reference a bucket, i.e. that require the bucket name as a mandatory - // parameter. - // This applies to almost all S3 operations except `list-buckets`. - // - // [S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/index.html - AWSS3BucketKey = attribute.Key("aws.s3.bucket") - - // AWSS3CopySourceKey is the attribute Key conforming to the - // "aws.s3.copy_source" semantic conventions. It represents the source object - // (in the form `bucket`/`key`) for the copy operation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "someFile.yml" - // Note: The `copy_source` attribute applies to S3 copy operations and - // corresponds to the `--copy-source` parameter - // of the [copy-object operation within the S3 API]. - // This applies in particular to the following operations: - // - // - [copy-object] - // - [upload-part-copy] - // - // - // [copy-object operation within the S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/copy-object.html - // [copy-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/copy-object.html - // [upload-part-copy]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part-copy.html - AWSS3CopySourceKey = attribute.Key("aws.s3.copy_source") - - // AWSS3DeleteKey is the attribute Key conforming to the "aws.s3.delete" - // semantic conventions. It represents the delete request container that - // specifies the objects to be deleted. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "Objects=[{Key=string,VersionId=string},{Key=string,VersionId=string}],Quiet=boolean" - // Note: The `delete` attribute is only applicable to the [delete-object] - // operation. - // The `delete` attribute corresponds to the `--delete` parameter of the - // [delete-objects operation within the S3 API]. - // - // [delete-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/delete-object.html - // [delete-objects operation within the S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/delete-objects.html - AWSS3DeleteKey = attribute.Key("aws.s3.delete") - - // AWSS3KeyKey is the attribute Key conforming to the "aws.s3.key" semantic - // conventions. It represents the S3 object key the request refers to. - // Corresponds to the `--key` parameter of the [S3 API] operations. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "someFile.yml" - // Note: The `key` attribute is applicable to all object-related S3 operations, - // i.e. that require the object key as a mandatory parameter. - // This applies in particular to the following operations: - // - // - [copy-object] - // - [delete-object] - // - [get-object] - // - [head-object] - // - [put-object] - // - [restore-object] - // - [select-object-content] - // - [abort-multipart-upload] - // - [complete-multipart-upload] - // - [create-multipart-upload] - // - [list-parts] - // - [upload-part] - // - [upload-part-copy] - // - // - // [S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/index.html - // [copy-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/copy-object.html - // [delete-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/delete-object.html - // [get-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/get-object.html - // [head-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/head-object.html - // [put-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/put-object.html - // [restore-object]: https://docs.aws.amazon.com/cli/latest/reference/s3api/restore-object.html - // [select-object-content]: https://docs.aws.amazon.com/cli/latest/reference/s3api/select-object-content.html - // [abort-multipart-upload]: https://docs.aws.amazon.com/cli/latest/reference/s3api/abort-multipart-upload.html - // [complete-multipart-upload]: https://docs.aws.amazon.com/cli/latest/reference/s3api/complete-multipart-upload.html - // [create-multipart-upload]: https://docs.aws.amazon.com/cli/latest/reference/s3api/create-multipart-upload.html - // [list-parts]: https://docs.aws.amazon.com/cli/latest/reference/s3api/list-parts.html - // [upload-part]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part.html - // [upload-part-copy]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part-copy.html - AWSS3KeyKey = attribute.Key("aws.s3.key") - - // AWSS3PartNumberKey is the attribute Key conforming to the - // "aws.s3.part_number" semantic conventions. It represents the part number of - // the part being uploaded in a multipart-upload operation. This is a positive - // integer between 1 and 10,000. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 3456 - // Note: The `part_number` attribute is only applicable to the [upload-part] - // and [upload-part-copy] operations. - // The `part_number` attribute corresponds to the `--part-number` parameter of - // the - // [upload-part operation within the S3 API]. - // - // [upload-part]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part.html - // [upload-part-copy]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part-copy.html - // [upload-part operation within the S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part.html - AWSS3PartNumberKey = attribute.Key("aws.s3.part_number") - - // AWSS3UploadIDKey is the attribute Key conforming to the "aws.s3.upload_id" - // semantic conventions. It represents the upload ID that identifies the - // multipart upload. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "dfRtDYWFbkRONycy.Yxwh66Yjlx.cph0gtNBtJ" - // Note: The `upload_id` attribute applies to S3 multipart-upload operations and - // corresponds to the `--upload-id` parameter - // of the [S3 API] multipart operations. - // This applies in particular to the following operations: - // - // - [abort-multipart-upload] - // - [complete-multipart-upload] - // - [list-parts] - // - [upload-part] - // - [upload-part-copy] - // - // - // [S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/index.html - // [abort-multipart-upload]: https://docs.aws.amazon.com/cli/latest/reference/s3api/abort-multipart-upload.html - // [complete-multipart-upload]: https://docs.aws.amazon.com/cli/latest/reference/s3api/complete-multipart-upload.html - // [list-parts]: https://docs.aws.amazon.com/cli/latest/reference/s3api/list-parts.html - // [upload-part]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part.html - // [upload-part-copy]: https://docs.aws.amazon.com/cli/latest/reference/s3api/upload-part-copy.html - AWSS3UploadIDKey = attribute.Key("aws.s3.upload_id") - - // AWSSecretsmanagerSecretARNKey is the attribute Key conforming to the - // "aws.secretsmanager.secret.arn" semantic conventions. It represents the ARN - // of the Secret stored in the Secrets Mangger. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "arn:aws:secretsmanager:us-east-1:123456789012:secret:SecretName-6RandomCharacters" - AWSSecretsmanagerSecretARNKey = attribute.Key("aws.secretsmanager.secret.arn") - - // AWSSNSTopicARNKey is the attribute Key conforming to the "aws.sns.topic.arn" - // semantic conventions. It represents the ARN of the AWS SNS Topic. An Amazon - // SNS [topic] is a logical access point that acts as a communication channel. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:sns:us-east-1:123456789012:mystack-mytopic-NZJ5JSMVGFIE" - // - // [topic]: https://docs.aws.amazon.com/sns/latest/dg/sns-create-topic.html - AWSSNSTopicARNKey = attribute.Key("aws.sns.topic.arn") - - // AWSSQSQueueURLKey is the attribute Key conforming to the "aws.sqs.queue.url" - // semantic conventions. It represents the URL of the AWS SQS Queue. It's a - // unique identifier for a queue in Amazon Simple Queue Service (SQS) and is - // used to access the queue and perform actions on it. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue" - AWSSQSQueueURLKey = attribute.Key("aws.sqs.queue.url") - - // AWSStepFunctionsActivityARNKey is the attribute Key conforming to the - // "aws.step_functions.activity.arn" semantic conventions. It represents the ARN - // of the AWS Step Functions Activity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:states:us-east-1:123456789012:activity:get-greeting" - AWSStepFunctionsActivityARNKey = attribute.Key("aws.step_functions.activity.arn") - - // AWSStepFunctionsStateMachineARNKey is the attribute Key conforming to the - // "aws.step_functions.state_machine.arn" semantic conventions. It represents - // the ARN of the AWS Step Functions State Machine. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "arn:aws:states:us-east-1:123456789012:stateMachine:myStateMachine:1" - AWSStepFunctionsStateMachineARNKey = attribute.Key("aws.step_functions.state_machine.arn") -) - -// AWSBedrockGuardrailID returns an attribute KeyValue conforming to the -// "aws.bedrock.guardrail.id" semantic conventions. It represents the unique -// identifier of the AWS Bedrock Guardrail. A [guardrail] helps safeguard and -// prevent unwanted behavior from model responses or user messages. -// -// [guardrail]: https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html -func AWSBedrockGuardrailID(val string) attribute.KeyValue { - return AWSBedrockGuardrailIDKey.String(val) -} - -// AWSBedrockKnowledgeBaseID returns an attribute KeyValue conforming to the -// "aws.bedrock.knowledge_base.id" semantic conventions. It represents the unique -// identifier of the AWS Bedrock Knowledge base. A [knowledge base] is a bank of -// information that can be queried by models to generate more relevant responses -// and augment prompts. -// -// [knowledge base]: https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html -func AWSBedrockKnowledgeBaseID(val string) attribute.KeyValue { - return AWSBedrockKnowledgeBaseIDKey.String(val) -} - -// AWSDynamoDBAttributeDefinitions returns an attribute KeyValue conforming to -// the "aws.dynamodb.attribute_definitions" semantic conventions. It represents -// the JSON-serialized value of each item in the `AttributeDefinitions` request -// field. -func AWSDynamoDBAttributeDefinitions(val ...string) attribute.KeyValue { - return AWSDynamoDBAttributeDefinitionsKey.StringSlice(val) -} - -// AWSDynamoDBAttributesToGet returns an attribute KeyValue conforming to the -// "aws.dynamodb.attributes_to_get" semantic conventions. It represents the value -// of the `AttributesToGet` request parameter. -func AWSDynamoDBAttributesToGet(val ...string) attribute.KeyValue { - return AWSDynamoDBAttributesToGetKey.StringSlice(val) -} - -// AWSDynamoDBConsistentRead returns an attribute KeyValue conforming to the -// "aws.dynamodb.consistent_read" semantic conventions. It represents the value -// of the `ConsistentRead` request parameter. -func AWSDynamoDBConsistentRead(val bool) attribute.KeyValue { - return AWSDynamoDBConsistentReadKey.Bool(val) -} - -// AWSDynamoDBConsumedCapacity returns an attribute KeyValue conforming to the -// "aws.dynamodb.consumed_capacity" semantic conventions. It represents the -// JSON-serialized value of each item in the `ConsumedCapacity` response field. -func AWSDynamoDBConsumedCapacity(val ...string) attribute.KeyValue { - return AWSDynamoDBConsumedCapacityKey.StringSlice(val) -} - -// AWSDynamoDBCount returns an attribute KeyValue conforming to the -// "aws.dynamodb.count" semantic conventions. It represents the value of the -// `Count` response parameter. -func AWSDynamoDBCount(val int) attribute.KeyValue { - return AWSDynamoDBCountKey.Int(val) -} - -// AWSDynamoDBExclusiveStartTable returns an attribute KeyValue conforming to the -// "aws.dynamodb.exclusive_start_table" semantic conventions. It represents the -// value of the `ExclusiveStartTableName` request parameter. -func AWSDynamoDBExclusiveStartTable(val string) attribute.KeyValue { - return AWSDynamoDBExclusiveStartTableKey.String(val) -} - -// AWSDynamoDBGlobalSecondaryIndexUpdates returns an attribute KeyValue -// conforming to the "aws.dynamodb.global_secondary_index_updates" semantic -// conventions. It represents the JSON-serialized value of each item in the -// `GlobalSecondaryIndexUpdates` request field. -func AWSDynamoDBGlobalSecondaryIndexUpdates(val ...string) attribute.KeyValue { - return AWSDynamoDBGlobalSecondaryIndexUpdatesKey.StringSlice(val) -} - -// AWSDynamoDBGlobalSecondaryIndexes returns an attribute KeyValue conforming to -// the "aws.dynamodb.global_secondary_indexes" semantic conventions. It -// represents the JSON-serialized value of each item of the -// `GlobalSecondaryIndexes` request field. -func AWSDynamoDBGlobalSecondaryIndexes(val ...string) attribute.KeyValue { - return AWSDynamoDBGlobalSecondaryIndexesKey.StringSlice(val) -} - -// AWSDynamoDBIndexName returns an attribute KeyValue conforming to the -// "aws.dynamodb.index_name" semantic conventions. It represents the value of the -// `IndexName` request parameter. -func AWSDynamoDBIndexName(val string) attribute.KeyValue { - return AWSDynamoDBIndexNameKey.String(val) -} - -// AWSDynamoDBItemCollectionMetrics returns an attribute KeyValue conforming to -// the "aws.dynamodb.item_collection_metrics" semantic conventions. It represents -// the JSON-serialized value of the `ItemCollectionMetrics` response field. -func AWSDynamoDBItemCollectionMetrics(val string) attribute.KeyValue { - return AWSDynamoDBItemCollectionMetricsKey.String(val) -} - -// AWSDynamoDBLimit returns an attribute KeyValue conforming to the -// "aws.dynamodb.limit" semantic conventions. It represents the value of the -// `Limit` request parameter. -func AWSDynamoDBLimit(val int) attribute.KeyValue { - return AWSDynamoDBLimitKey.Int(val) -} - -// AWSDynamoDBLocalSecondaryIndexes returns an attribute KeyValue conforming to -// the "aws.dynamodb.local_secondary_indexes" semantic conventions. It represents -// the JSON-serialized value of each item of the `LocalSecondaryIndexes` request -// field. -func AWSDynamoDBLocalSecondaryIndexes(val ...string) attribute.KeyValue { - return AWSDynamoDBLocalSecondaryIndexesKey.StringSlice(val) -} - -// AWSDynamoDBProjection returns an attribute KeyValue conforming to the -// "aws.dynamodb.projection" semantic conventions. It represents the value of the -// `ProjectionExpression` request parameter. -func AWSDynamoDBProjection(val string) attribute.KeyValue { - return AWSDynamoDBProjectionKey.String(val) -} - -// AWSDynamoDBProvisionedReadCapacity returns an attribute KeyValue conforming to -// the "aws.dynamodb.provisioned_read_capacity" semantic conventions. It -// represents the value of the `ProvisionedThroughput.ReadCapacityUnits` request -// parameter. -func AWSDynamoDBProvisionedReadCapacity(val float64) attribute.KeyValue { - return AWSDynamoDBProvisionedReadCapacityKey.Float64(val) -} - -// AWSDynamoDBProvisionedWriteCapacity returns an attribute KeyValue conforming -// to the "aws.dynamodb.provisioned_write_capacity" semantic conventions. It -// represents the value of the `ProvisionedThroughput.WriteCapacityUnits` request -// parameter. -func AWSDynamoDBProvisionedWriteCapacity(val float64) attribute.KeyValue { - return AWSDynamoDBProvisionedWriteCapacityKey.Float64(val) -} - -// AWSDynamoDBScanForward returns an attribute KeyValue conforming to the -// "aws.dynamodb.scan_forward" semantic conventions. It represents the value of -// the `ScanIndexForward` request parameter. -func AWSDynamoDBScanForward(val bool) attribute.KeyValue { - return AWSDynamoDBScanForwardKey.Bool(val) -} - -// AWSDynamoDBScannedCount returns an attribute KeyValue conforming to the -// "aws.dynamodb.scanned_count" semantic conventions. It represents the value of -// the `ScannedCount` response parameter. -func AWSDynamoDBScannedCount(val int) attribute.KeyValue { - return AWSDynamoDBScannedCountKey.Int(val) -} - -// AWSDynamoDBSegment returns an attribute KeyValue conforming to the -// "aws.dynamodb.segment" semantic conventions. It represents the value of the -// `Segment` request parameter. -func AWSDynamoDBSegment(val int) attribute.KeyValue { - return AWSDynamoDBSegmentKey.Int(val) -} - -// AWSDynamoDBSelect returns an attribute KeyValue conforming to the -// "aws.dynamodb.select" semantic conventions. It represents the value of the -// `Select` request parameter. -func AWSDynamoDBSelect(val string) attribute.KeyValue { - return AWSDynamoDBSelectKey.String(val) -} - -// AWSDynamoDBTableCount returns an attribute KeyValue conforming to the -// "aws.dynamodb.table_count" semantic conventions. It represents the number of -// items in the `TableNames` response parameter. -func AWSDynamoDBTableCount(val int) attribute.KeyValue { - return AWSDynamoDBTableCountKey.Int(val) -} - -// AWSDynamoDBTableNames returns an attribute KeyValue conforming to the -// "aws.dynamodb.table_names" semantic conventions. It represents the keys in the -// `RequestItems` object field. -func AWSDynamoDBTableNames(val ...string) attribute.KeyValue { - return AWSDynamoDBTableNamesKey.StringSlice(val) -} - -// AWSDynamoDBTotalSegments returns an attribute KeyValue conforming to the -// "aws.dynamodb.total_segments" semantic conventions. It represents the value of -// the `TotalSegments` request parameter. -func AWSDynamoDBTotalSegments(val int) attribute.KeyValue { - return AWSDynamoDBTotalSegmentsKey.Int(val) -} - -// AWSECSClusterARN returns an attribute KeyValue conforming to the -// "aws.ecs.cluster.arn" semantic conventions. It represents the ARN of an -// [ECS cluster]. -// -// [ECS cluster]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html -func AWSECSClusterARN(val string) attribute.KeyValue { - return AWSECSClusterARNKey.String(val) -} - -// AWSECSContainerARN returns an attribute KeyValue conforming to the -// "aws.ecs.container.arn" semantic conventions. It represents the Amazon -// Resource Name (ARN) of an [ECS container instance]. -// -// [ECS container instance]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_instances.html -func AWSECSContainerARN(val string) attribute.KeyValue { - return AWSECSContainerARNKey.String(val) -} - -// AWSECSTaskARN returns an attribute KeyValue conforming to the -// "aws.ecs.task.arn" semantic conventions. It represents the ARN of a running -// [ECS task]. -// -// [ECS task]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-account-settings.html#ecs-resource-ids -func AWSECSTaskARN(val string) attribute.KeyValue { - return AWSECSTaskARNKey.String(val) -} - -// AWSECSTaskFamily returns an attribute KeyValue conforming to the -// "aws.ecs.task.family" semantic conventions. It represents the family name of -// the [ECS task definition] used to create the ECS task. -// -// [ECS task definition]: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html -func AWSECSTaskFamily(val string) attribute.KeyValue { - return AWSECSTaskFamilyKey.String(val) -} - -// AWSECSTaskID returns an attribute KeyValue conforming to the "aws.ecs.task.id" -// semantic conventions. It represents the ID of a running ECS task. The ID MUST -// be extracted from `task.arn`. -func AWSECSTaskID(val string) attribute.KeyValue { - return AWSECSTaskIDKey.String(val) -} - -// AWSECSTaskRevision returns an attribute KeyValue conforming to the -// "aws.ecs.task.revision" semantic conventions. It represents the revision for -// the task definition used to create the ECS task. -func AWSECSTaskRevision(val string) attribute.KeyValue { - return AWSECSTaskRevisionKey.String(val) -} - -// AWSEKSClusterARN returns an attribute KeyValue conforming to the -// "aws.eks.cluster.arn" semantic conventions. It represents the ARN of an EKS -// cluster. -func AWSEKSClusterARN(val string) attribute.KeyValue { - return AWSEKSClusterARNKey.String(val) -} - -// AWSExtendedRequestID returns an attribute KeyValue conforming to the -// "aws.extended_request_id" semantic conventions. It represents the AWS extended -// request ID as returned in the response header `x-amz-id-2`. -func AWSExtendedRequestID(val string) attribute.KeyValue { - return AWSExtendedRequestIDKey.String(val) -} - -// AWSKinesisStreamName returns an attribute KeyValue conforming to the -// "aws.kinesis.stream_name" semantic conventions. It represents the name of the -// AWS Kinesis [stream] the request refers to. Corresponds to the `--stream-name` -// -// parameter of the Kinesis [describe-stream] operation. -// -// [stream]: https://docs.aws.amazon.com/streams/latest/dev/introduction.html -// -// [describe-stream]: https://docs.aws.amazon.com/cli/latest/reference/kinesis/describe-stream.html -func AWSKinesisStreamName(val string) attribute.KeyValue { - return AWSKinesisStreamNameKey.String(val) -} - -// AWSLambdaInvokedARN returns an attribute KeyValue conforming to the -// "aws.lambda.invoked_arn" semantic conventions. It represents the full invoked -// ARN as provided on the `Context` passed to the function ( -// `Lambda-Runtime-Invoked-Function-Arn` header on the `/runtime/invocation/next` -// -// applicable). -func AWSLambdaInvokedARN(val string) attribute.KeyValue { - return AWSLambdaInvokedARNKey.String(val) -} - -// AWSLambdaResourceMappingID returns an attribute KeyValue conforming to the -// "aws.lambda.resource_mapping.id" semantic conventions. It represents the UUID -// of the [AWS Lambda EvenSource Mapping]. An event source is mapped to a lambda -// function. It's contents are read by Lambda and used to trigger a function. -// This isn't available in the lambda execution context or the lambda runtime -// environtment. This is going to be populated by the AWS SDK for each language -// when that UUID is present. Some of these operations are -// Create/Delete/Get/List/Update EventSourceMapping. -// -// [AWS Lambda EvenSource Mapping]: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html -func AWSLambdaResourceMappingID(val string) attribute.KeyValue { - return AWSLambdaResourceMappingIDKey.String(val) -} - -// AWSLogGroupARNs returns an attribute KeyValue conforming to the -// "aws.log.group.arns" semantic conventions. It represents the Amazon Resource -// Name(s) (ARN) of the AWS log group(s). -func AWSLogGroupARNs(val ...string) attribute.KeyValue { - return AWSLogGroupARNsKey.StringSlice(val) -} - -// AWSLogGroupNames returns an attribute KeyValue conforming to the -// "aws.log.group.names" semantic conventions. It represents the name(s) of the -// AWS log group(s) an application is writing to. -func AWSLogGroupNames(val ...string) attribute.KeyValue { - return AWSLogGroupNamesKey.StringSlice(val) -} - -// AWSLogStreamARNs returns an attribute KeyValue conforming to the -// "aws.log.stream.arns" semantic conventions. It represents the ARN(s) of the -// AWS log stream(s). -func AWSLogStreamARNs(val ...string) attribute.KeyValue { - return AWSLogStreamARNsKey.StringSlice(val) -} - -// AWSLogStreamNames returns an attribute KeyValue conforming to the -// "aws.log.stream.names" semantic conventions. It represents the name(s) of the -// AWS log stream(s) an application is writing to. -func AWSLogStreamNames(val ...string) attribute.KeyValue { - return AWSLogStreamNamesKey.StringSlice(val) -} - -// AWSRequestID returns an attribute KeyValue conforming to the "aws.request_id" -// semantic conventions. It represents the AWS request ID as returned in the -// response headers `x-amzn-requestid`, `x-amzn-request-id` or `x-amz-request-id` -// . -func AWSRequestID(val string) attribute.KeyValue { - return AWSRequestIDKey.String(val) -} - -// AWSS3Bucket returns an attribute KeyValue conforming to the "aws.s3.bucket" -// semantic conventions. It represents the S3 bucket name the request refers to. -// Corresponds to the `--bucket` parameter of the [S3 API] operations. -// -// [S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/index.html -func AWSS3Bucket(val string) attribute.KeyValue { - return AWSS3BucketKey.String(val) -} - -// AWSS3CopySource returns an attribute KeyValue conforming to the -// "aws.s3.copy_source" semantic conventions. It represents the source object (in -// the form `bucket`/`key`) for the copy operation. -func AWSS3CopySource(val string) attribute.KeyValue { - return AWSS3CopySourceKey.String(val) -} - -// AWSS3Delete returns an attribute KeyValue conforming to the "aws.s3.delete" -// semantic conventions. It represents the delete request container that -// specifies the objects to be deleted. -func AWSS3Delete(val string) attribute.KeyValue { - return AWSS3DeleteKey.String(val) -} - -// AWSS3Key returns an attribute KeyValue conforming to the "aws.s3.key" semantic -// conventions. It represents the S3 object key the request refers to. -// Corresponds to the `--key` parameter of the [S3 API] operations. -// -// [S3 API]: https://docs.aws.amazon.com/cli/latest/reference/s3api/index.html -func AWSS3Key(val string) attribute.KeyValue { - return AWSS3KeyKey.String(val) -} - -// AWSS3PartNumber returns an attribute KeyValue conforming to the -// "aws.s3.part_number" semantic conventions. It represents the part number of -// the part being uploaded in a multipart-upload operation. This is a positive -// integer between 1 and 10,000. -func AWSS3PartNumber(val int) attribute.KeyValue { - return AWSS3PartNumberKey.Int(val) -} - -// AWSS3UploadID returns an attribute KeyValue conforming to the -// "aws.s3.upload_id" semantic conventions. It represents the upload ID that -// identifies the multipart upload. -func AWSS3UploadID(val string) attribute.KeyValue { - return AWSS3UploadIDKey.String(val) -} - -// AWSSecretsmanagerSecretARN returns an attribute KeyValue conforming to the -// "aws.secretsmanager.secret.arn" semantic conventions. It represents the ARN of -// the Secret stored in the Secrets Mangger. -func AWSSecretsmanagerSecretARN(val string) attribute.KeyValue { - return AWSSecretsmanagerSecretARNKey.String(val) -} - -// AWSSNSTopicARN returns an attribute KeyValue conforming to the -// "aws.sns.topic.arn" semantic conventions. It represents the ARN of the AWS SNS -// Topic. An Amazon SNS [topic] is a logical access point that acts as a -// communication channel. -// -// [topic]: https://docs.aws.amazon.com/sns/latest/dg/sns-create-topic.html -func AWSSNSTopicARN(val string) attribute.KeyValue { - return AWSSNSTopicARNKey.String(val) -} - -// AWSSQSQueueURL returns an attribute KeyValue conforming to the -// "aws.sqs.queue.url" semantic conventions. It represents the URL of the AWS SQS -// Queue. It's a unique identifier for a queue in Amazon Simple Queue Service -// (SQS) and is used to access the queue and perform actions on it. -func AWSSQSQueueURL(val string) attribute.KeyValue { - return AWSSQSQueueURLKey.String(val) -} - -// AWSStepFunctionsActivityARN returns an attribute KeyValue conforming to the -// "aws.step_functions.activity.arn" semantic conventions. It represents the ARN -// of the AWS Step Functions Activity. -func AWSStepFunctionsActivityARN(val string) attribute.KeyValue { - return AWSStepFunctionsActivityARNKey.String(val) -} - -// AWSStepFunctionsStateMachineARN returns an attribute KeyValue conforming to -// the "aws.step_functions.state_machine.arn" semantic conventions. It represents -// the ARN of the AWS Step Functions State Machine. -func AWSStepFunctionsStateMachineARN(val string) attribute.KeyValue { - return AWSStepFunctionsStateMachineARNKey.String(val) -} - -// Enum values for aws.ecs.launchtype -var ( - // Amazon EC2 - // Stability: development - AWSECSLaunchtypeEC2 = AWSECSLaunchtypeKey.String("ec2") - // Amazon Fargate - // Stability: development - AWSECSLaunchtypeFargate = AWSECSLaunchtypeKey.String("fargate") -) - -// Namespace: azure -const ( - // AzureClientIDKey is the attribute Key conforming to the "azure.client.id" - // semantic conventions. It represents the unique identifier of the client - // instance. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "3ba4827d-4422-483f-b59f-85b74211c11d", "storage-client-1" - AzureClientIDKey = attribute.Key("azure.client.id") - - // AzureCosmosDBConnectionModeKey is the attribute Key conforming to the - // "azure.cosmosdb.connection.mode" semantic conventions. It represents the - // cosmos client connection mode. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - AzureCosmosDBConnectionModeKey = attribute.Key("azure.cosmosdb.connection.mode") - - // AzureCosmosDBConsistencyLevelKey is the attribute Key conforming to the - // "azure.cosmosdb.consistency.level" semantic conventions. It represents the - // account or request [consistency level]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Eventual", "ConsistentPrefix", "BoundedStaleness", "Strong", - // "Session" - // - // [consistency level]: https://learn.microsoft.com/azure/cosmos-db/consistency-levels - AzureCosmosDBConsistencyLevelKey = attribute.Key("azure.cosmosdb.consistency.level") - - // AzureCosmosDBOperationContactedRegionsKey is the attribute Key conforming to - // the "azure.cosmosdb.operation.contacted_regions" semantic conventions. It - // represents the list of regions contacted during operation in the order that - // they were contacted. If there is more than one region listed, it indicates - // that the operation was performed on multiple regions i.e. cross-regional - // call. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "North Central US", "Australia East", "Australia Southeast" - // Note: Region name matches the format of `displayName` in [Azure Location API] - // - // [Azure Location API]: https://learn.microsoft.com/rest/api/resources/subscriptions/list-locations - AzureCosmosDBOperationContactedRegionsKey = attribute.Key("azure.cosmosdb.operation.contacted_regions") - - // AzureCosmosDBOperationRequestChargeKey is the attribute Key conforming to the - // "azure.cosmosdb.operation.request_charge" semantic conventions. It represents - // the number of request units consumed by the operation. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 46.18, 1.0 - AzureCosmosDBOperationRequestChargeKey = attribute.Key("azure.cosmosdb.operation.request_charge") - - // AzureCosmosDBRequestBodySizeKey is the attribute Key conforming to the - // "azure.cosmosdb.request.body.size" semantic conventions. It represents the - // request payload size in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - AzureCosmosDBRequestBodySizeKey = attribute.Key("azure.cosmosdb.request.body.size") - - // AzureCosmosDBResponseSubStatusCodeKey is the attribute Key conforming to the - // "azure.cosmosdb.response.sub_status_code" semantic conventions. It represents - // the cosmos DB sub status code. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1000, 1002 - AzureCosmosDBResponseSubStatusCodeKey = attribute.Key("azure.cosmosdb.response.sub_status_code") - - // AzureResourceProviderNamespaceKey is the attribute Key conforming to the - // "azure.resource_provider.namespace" semantic conventions. It represents the - // [Azure Resource Provider Namespace] as recognized by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Microsoft.Storage", "Microsoft.KeyVault", "Microsoft.ServiceBus" - // - // [Azure Resource Provider Namespace]: https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers - AzureResourceProviderNamespaceKey = attribute.Key("azure.resource_provider.namespace") - - // AzureServiceRequestIDKey is the attribute Key conforming to the - // "azure.service.request.id" semantic conventions. It represents the unique - // identifier of the service request. It's generated by the Azure service and - // returned with the response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "00000000-0000-0000-0000-000000000000" - AzureServiceRequestIDKey = attribute.Key("azure.service.request.id") -) - -// AzureClientID returns an attribute KeyValue conforming to the -// "azure.client.id" semantic conventions. It represents the unique identifier of -// the client instance. -func AzureClientID(val string) attribute.KeyValue { - return AzureClientIDKey.String(val) -} - -// AzureCosmosDBOperationContactedRegions returns an attribute KeyValue -// conforming to the "azure.cosmosdb.operation.contacted_regions" semantic -// conventions. It represents the list of regions contacted during operation in -// the order that they were contacted. If there is more than one region listed, -// it indicates that the operation was performed on multiple regions i.e. -// cross-regional call. -func AzureCosmosDBOperationContactedRegions(val ...string) attribute.KeyValue { - return AzureCosmosDBOperationContactedRegionsKey.StringSlice(val) -} - -// AzureCosmosDBOperationRequestCharge returns an attribute KeyValue conforming -// to the "azure.cosmosdb.operation.request_charge" semantic conventions. It -// represents the number of request units consumed by the operation. -func AzureCosmosDBOperationRequestCharge(val float64) attribute.KeyValue { - return AzureCosmosDBOperationRequestChargeKey.Float64(val) -} - -// AzureCosmosDBRequestBodySize returns an attribute KeyValue conforming to the -// "azure.cosmosdb.request.body.size" semantic conventions. It represents the -// request payload size in bytes. -func AzureCosmosDBRequestBodySize(val int) attribute.KeyValue { - return AzureCosmosDBRequestBodySizeKey.Int(val) -} - -// AzureCosmosDBResponseSubStatusCode returns an attribute KeyValue conforming to -// the "azure.cosmosdb.response.sub_status_code" semantic conventions. It -// represents the cosmos DB sub status code. -func AzureCosmosDBResponseSubStatusCode(val int) attribute.KeyValue { - return AzureCosmosDBResponseSubStatusCodeKey.Int(val) -} - -// AzureResourceProviderNamespace returns an attribute KeyValue conforming to the -// "azure.resource_provider.namespace" semantic conventions. It represents the -// [Azure Resource Provider Namespace] as recognized by the client. -// -// [Azure Resource Provider Namespace]: https://learn.microsoft.com/azure/azure-resource-manager/management/azure-services-resource-providers -func AzureResourceProviderNamespace(val string) attribute.KeyValue { - return AzureResourceProviderNamespaceKey.String(val) -} - -// AzureServiceRequestID returns an attribute KeyValue conforming to the -// "azure.service.request.id" semantic conventions. It represents the unique -// identifier of the service request. It's generated by the Azure service and -// returned with the response. -func AzureServiceRequestID(val string) attribute.KeyValue { - return AzureServiceRequestIDKey.String(val) -} - -// Enum values for azure.cosmosdb.connection.mode -var ( - // Gateway (HTTP) connection. - // Stability: development - AzureCosmosDBConnectionModeGateway = AzureCosmosDBConnectionModeKey.String("gateway") - // Direct connection. - // Stability: development - AzureCosmosDBConnectionModeDirect = AzureCosmosDBConnectionModeKey.String("direct") -) - -// Enum values for azure.cosmosdb.consistency.level -var ( - // Strong - // Stability: development - AzureCosmosDBConsistencyLevelStrong = AzureCosmosDBConsistencyLevelKey.String("Strong") - // Bounded Staleness - // Stability: development - AzureCosmosDBConsistencyLevelBoundedStaleness = AzureCosmosDBConsistencyLevelKey.String("BoundedStaleness") - // Session - // Stability: development - AzureCosmosDBConsistencyLevelSession = AzureCosmosDBConsistencyLevelKey.String("Session") - // Eventual - // Stability: development - AzureCosmosDBConsistencyLevelEventual = AzureCosmosDBConsistencyLevelKey.String("Eventual") - // Consistent Prefix - // Stability: development - AzureCosmosDBConsistencyLevelConsistentPrefix = AzureCosmosDBConsistencyLevelKey.String("ConsistentPrefix") -) - -// Namespace: browser -const ( - // BrowserBrandsKey is the attribute Key conforming to the "browser.brands" - // semantic conventions. It represents the array of brand name and version - // separated by a space. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: " Not A;Brand 99", "Chromium 99", "Chrome 99" - // Note: This value is intended to be taken from the [UA client hints API] ( - // `navigator.userAgentData.brands`). - // - // [UA client hints API]: https://wicg.github.io/ua-client-hints/#interface - BrowserBrandsKey = attribute.Key("browser.brands") - - // BrowserLanguageKey is the attribute Key conforming to the "browser.language" - // semantic conventions. It represents the preferred language of the user using - // the browser. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "en", "en-US", "fr", "fr-FR" - // Note: This value is intended to be taken from the Navigator API - // `navigator.language`. - BrowserLanguageKey = attribute.Key("browser.language") - - // BrowserMobileKey is the attribute Key conforming to the "browser.mobile" - // semantic conventions. It represents a boolean that is true if the browser is - // running on a mobile device. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: This value is intended to be taken from the [UA client hints API] ( - // `navigator.userAgentData.mobile`). If unavailable, this attribute SHOULD be - // left unset. - // - // [UA client hints API]: https://wicg.github.io/ua-client-hints/#interface - BrowserMobileKey = attribute.Key("browser.mobile") - - // BrowserPlatformKey is the attribute Key conforming to the "browser.platform" - // semantic conventions. It represents the platform on which the browser is - // running. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Windows", "macOS", "Android" - // Note: This value is intended to be taken from the [UA client hints API] ( - // `navigator.userAgentData.platform`). If unavailable, the legacy - // `navigator.platform` API SHOULD NOT be used instead and this attribute SHOULD - // be left unset in order for the values to be consistent. - // The list of possible values is defined in the - // [W3C User-Agent Client Hints specification]. Note that some (but not all) of - // these values can overlap with values in the - // [`os.type` and `os.name` attributes]. However, for consistency, the values in - // the `browser.platform` attribute should capture the exact value that the user - // agent provides. - // - // [UA client hints API]: https://wicg.github.io/ua-client-hints/#interface - // [W3C User-Agent Client Hints specification]: https://wicg.github.io/ua-client-hints/#sec-ch-ua-platform - // [`os.type` and `os.name` attributes]: ./os.md - BrowserPlatformKey = attribute.Key("browser.platform") -) - -// BrowserBrands returns an attribute KeyValue conforming to the "browser.brands" -// semantic conventions. It represents the array of brand name and version -// separated by a space. -func BrowserBrands(val ...string) attribute.KeyValue { - return BrowserBrandsKey.StringSlice(val) -} - -// BrowserLanguage returns an attribute KeyValue conforming to the -// "browser.language" semantic conventions. It represents the preferred language -// of the user using the browser. -func BrowserLanguage(val string) attribute.KeyValue { - return BrowserLanguageKey.String(val) -} - -// BrowserMobile returns an attribute KeyValue conforming to the "browser.mobile" -// semantic conventions. It represents a boolean that is true if the browser is -// running on a mobile device. -func BrowserMobile(val bool) attribute.KeyValue { - return BrowserMobileKey.Bool(val) -} - -// BrowserPlatform returns an attribute KeyValue conforming to the -// "browser.platform" semantic conventions. It represents the platform on which -// the browser is running. -func BrowserPlatform(val string) attribute.KeyValue { - return BrowserPlatformKey.String(val) -} - -// Namespace: cassandra -const ( - // CassandraConsistencyLevelKey is the attribute Key conforming to the - // "cassandra.consistency.level" semantic conventions. It represents the - // consistency level of the query. Based on consistency values from [CQL]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [CQL]: https://docs.datastax.com/en/cassandra-oss/3.0/cassandra/dml/dmlConfigConsistency.html - CassandraConsistencyLevelKey = attribute.Key("cassandra.consistency.level") - - // CassandraCoordinatorDCKey is the attribute Key conforming to the - // "cassandra.coordinator.dc" semantic conventions. It represents the data - // center of the coordinating node for a query. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: us-west-2 - CassandraCoordinatorDCKey = attribute.Key("cassandra.coordinator.dc") - - // CassandraCoordinatorIDKey is the attribute Key conforming to the - // "cassandra.coordinator.id" semantic conventions. It represents the ID of the - // coordinating node for a query. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: be13faa2-8574-4d71-926d-27f16cf8a7af - CassandraCoordinatorIDKey = attribute.Key("cassandra.coordinator.id") - - // CassandraPageSizeKey is the attribute Key conforming to the - // "cassandra.page.size" semantic conventions. It represents the fetch size used - // for paging, i.e. how many rows will be returned at once. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 5000 - CassandraPageSizeKey = attribute.Key("cassandra.page.size") - - // CassandraQueryIdempotentKey is the attribute Key conforming to the - // "cassandra.query.idempotent" semantic conventions. It represents the whether - // or not the query is idempotent. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - CassandraQueryIdempotentKey = attribute.Key("cassandra.query.idempotent") - - // CassandraSpeculativeExecutionCountKey is the attribute Key conforming to the - // "cassandra.speculative_execution.count" semantic conventions. It represents - // the number of times a query was speculatively executed. Not set or `0` if the - // query was not executed speculatively. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0, 2 - CassandraSpeculativeExecutionCountKey = attribute.Key("cassandra.speculative_execution.count") -) - -// CassandraCoordinatorDC returns an attribute KeyValue conforming to the -// "cassandra.coordinator.dc" semantic conventions. It represents the data center -// of the coordinating node for a query. -func CassandraCoordinatorDC(val string) attribute.KeyValue { - return CassandraCoordinatorDCKey.String(val) -} - -// CassandraCoordinatorID returns an attribute KeyValue conforming to the -// "cassandra.coordinator.id" semantic conventions. It represents the ID of the -// coordinating node for a query. -func CassandraCoordinatorID(val string) attribute.KeyValue { - return CassandraCoordinatorIDKey.String(val) -} - -// CassandraPageSize returns an attribute KeyValue conforming to the -// "cassandra.page.size" semantic conventions. It represents the fetch size used -// for paging, i.e. how many rows will be returned at once. -func CassandraPageSize(val int) attribute.KeyValue { - return CassandraPageSizeKey.Int(val) -} - -// CassandraQueryIdempotent returns an attribute KeyValue conforming to the -// "cassandra.query.idempotent" semantic conventions. It represents the whether -// or not the query is idempotent. -func CassandraQueryIdempotent(val bool) attribute.KeyValue { - return CassandraQueryIdempotentKey.Bool(val) -} - -// CassandraSpeculativeExecutionCount returns an attribute KeyValue conforming to -// the "cassandra.speculative_execution.count" semantic conventions. It -// represents the number of times a query was speculatively executed. Not set or -// `0` if the query was not executed speculatively. -func CassandraSpeculativeExecutionCount(val int) attribute.KeyValue { - return CassandraSpeculativeExecutionCountKey.Int(val) -} - -// Enum values for cassandra.consistency.level -var ( - // All - // Stability: development - CassandraConsistencyLevelAll = CassandraConsistencyLevelKey.String("all") - // Each Quorum - // Stability: development - CassandraConsistencyLevelEachQuorum = CassandraConsistencyLevelKey.String("each_quorum") - // Quorum - // Stability: development - CassandraConsistencyLevelQuorum = CassandraConsistencyLevelKey.String("quorum") - // Local Quorum - // Stability: development - CassandraConsistencyLevelLocalQuorum = CassandraConsistencyLevelKey.String("local_quorum") - // One - // Stability: development - CassandraConsistencyLevelOne = CassandraConsistencyLevelKey.String("one") - // Two - // Stability: development - CassandraConsistencyLevelTwo = CassandraConsistencyLevelKey.String("two") - // Three - // Stability: development - CassandraConsistencyLevelThree = CassandraConsistencyLevelKey.String("three") - // Local One - // Stability: development - CassandraConsistencyLevelLocalOne = CassandraConsistencyLevelKey.String("local_one") - // Any - // Stability: development - CassandraConsistencyLevelAny = CassandraConsistencyLevelKey.String("any") - // Serial - // Stability: development - CassandraConsistencyLevelSerial = CassandraConsistencyLevelKey.String("serial") - // Local Serial - // Stability: development - CassandraConsistencyLevelLocalSerial = CassandraConsistencyLevelKey.String("local_serial") -) - -// Namespace: cicd -const ( - // CICDPipelineActionNameKey is the attribute Key conforming to the - // "cicd.pipeline.action.name" semantic conventions. It represents the kind of - // action a pipeline run is performing. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "BUILD", "RUN", "SYNC" - CICDPipelineActionNameKey = attribute.Key("cicd.pipeline.action.name") - - // CICDPipelineNameKey is the attribute Key conforming to the - // "cicd.pipeline.name" semantic conventions. It represents the human readable - // name of the pipeline within a CI/CD system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Build and Test", "Lint", "Deploy Go Project", - // "deploy_to_environment" - CICDPipelineNameKey = attribute.Key("cicd.pipeline.name") - - // CICDPipelineResultKey is the attribute Key conforming to the - // "cicd.pipeline.result" semantic conventions. It represents the result of a - // pipeline run. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "success", "failure", "timeout", "skipped" - CICDPipelineResultKey = attribute.Key("cicd.pipeline.result") - - // CICDPipelineRunIDKey is the attribute Key conforming to the - // "cicd.pipeline.run.id" semantic conventions. It represents the unique - // identifier of a pipeline run within a CI/CD system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "120912" - CICDPipelineRunIDKey = attribute.Key("cicd.pipeline.run.id") - - // CICDPipelineRunStateKey is the attribute Key conforming to the - // "cicd.pipeline.run.state" semantic conventions. It represents the pipeline - // run goes through these states during its lifecycle. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "pending", "executing", "finalizing" - CICDPipelineRunStateKey = attribute.Key("cicd.pipeline.run.state") - - // CICDPipelineRunURLFullKey is the attribute Key conforming to the - // "cicd.pipeline.run.url.full" semantic conventions. It represents the [URL] of - // the pipeline run, providing the complete address in order to locate and - // identify the pipeline run. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "https://github.com/open-telemetry/semantic-conventions/actions/runs/9753949763?pr=1075" - // - // [URL]: https://wikipedia.org/wiki/URL - CICDPipelineRunURLFullKey = attribute.Key("cicd.pipeline.run.url.full") - - // CICDPipelineTaskNameKey is the attribute Key conforming to the - // "cicd.pipeline.task.name" semantic conventions. It represents the human - // readable name of a task within a pipeline. Task here most closely aligns with - // a [computing process] in a pipeline. Other terms for tasks include commands, - // steps, and procedures. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Run GoLang Linter", "Go Build", "go-test", "deploy_binary" - // - // [computing process]: https://wikipedia.org/wiki/Pipeline_(computing) - CICDPipelineTaskNameKey = attribute.Key("cicd.pipeline.task.name") - - // CICDPipelineTaskRunIDKey is the attribute Key conforming to the - // "cicd.pipeline.task.run.id" semantic conventions. It represents the unique - // identifier of a task run within a pipeline. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "12097" - CICDPipelineTaskRunIDKey = attribute.Key("cicd.pipeline.task.run.id") - - // CICDPipelineTaskRunResultKey is the attribute Key conforming to the - // "cicd.pipeline.task.run.result" semantic conventions. It represents the - // result of a task run. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "success", "failure", "timeout", "skipped" - CICDPipelineTaskRunResultKey = attribute.Key("cicd.pipeline.task.run.result") - - // CICDPipelineTaskRunURLFullKey is the attribute Key conforming to the - // "cicd.pipeline.task.run.url.full" semantic conventions. It represents the - // [URL] of the pipeline task run, providing the complete address in order to - // locate and identify the pipeline task run. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "https://github.com/open-telemetry/semantic-conventions/actions/runs/9753949763/job/26920038674?pr=1075" - // - // [URL]: https://wikipedia.org/wiki/URL - CICDPipelineTaskRunURLFullKey = attribute.Key("cicd.pipeline.task.run.url.full") - - // CICDPipelineTaskTypeKey is the attribute Key conforming to the - // "cicd.pipeline.task.type" semantic conventions. It represents the type of the - // task within a pipeline. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "build", "test", "deploy" - CICDPipelineTaskTypeKey = attribute.Key("cicd.pipeline.task.type") - - // CICDSystemComponentKey is the attribute Key conforming to the - // "cicd.system.component" semantic conventions. It represents the name of a - // component of the CICD system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "controller", "scheduler", "agent" - CICDSystemComponentKey = attribute.Key("cicd.system.component") - - // CICDWorkerIDKey is the attribute Key conforming to the "cicd.worker.id" - // semantic conventions. It represents the unique identifier of a worker within - // a CICD system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "abc123", "10.0.1.2", "controller" - CICDWorkerIDKey = attribute.Key("cicd.worker.id") - - // CICDWorkerNameKey is the attribute Key conforming to the "cicd.worker.name" - // semantic conventions. It represents the name of a worker within a CICD - // system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "agent-abc", "controller", "Ubuntu LTS" - CICDWorkerNameKey = attribute.Key("cicd.worker.name") - - // CICDWorkerStateKey is the attribute Key conforming to the "cicd.worker.state" - // semantic conventions. It represents the state of a CICD worker / agent. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "idle", "busy", "down" - CICDWorkerStateKey = attribute.Key("cicd.worker.state") - - // CICDWorkerURLFullKey is the attribute Key conforming to the - // "cicd.worker.url.full" semantic conventions. It represents the [URL] of the - // worker, providing the complete address in order to locate and identify the - // worker. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://cicd.example.org/worker/abc123" - // - // [URL]: https://wikipedia.org/wiki/URL - CICDWorkerURLFullKey = attribute.Key("cicd.worker.url.full") -) - -// CICDPipelineName returns an attribute KeyValue conforming to the -// "cicd.pipeline.name" semantic conventions. It represents the human readable -// name of the pipeline within a CI/CD system. -func CICDPipelineName(val string) attribute.KeyValue { - return CICDPipelineNameKey.String(val) -} - -// CICDPipelineRunID returns an attribute KeyValue conforming to the -// "cicd.pipeline.run.id" semantic conventions. It represents the unique -// identifier of a pipeline run within a CI/CD system. -func CICDPipelineRunID(val string) attribute.KeyValue { - return CICDPipelineRunIDKey.String(val) -} - -// CICDPipelineRunURLFull returns an attribute KeyValue conforming to the -// "cicd.pipeline.run.url.full" semantic conventions. It represents the [URL] of -// the pipeline run, providing the complete address in order to locate and -// identify the pipeline run. -// -// [URL]: https://wikipedia.org/wiki/URL -func CICDPipelineRunURLFull(val string) attribute.KeyValue { - return CICDPipelineRunURLFullKey.String(val) -} - -// CICDPipelineTaskName returns an attribute KeyValue conforming to the -// "cicd.pipeline.task.name" semantic conventions. It represents the human -// readable name of a task within a pipeline. Task here most closely aligns with -// a [computing process] in a pipeline. Other terms for tasks include commands, -// steps, and procedures. -// -// [computing process]: https://wikipedia.org/wiki/Pipeline_(computing) -func CICDPipelineTaskName(val string) attribute.KeyValue { - return CICDPipelineTaskNameKey.String(val) -} - -// CICDPipelineTaskRunID returns an attribute KeyValue conforming to the -// "cicd.pipeline.task.run.id" semantic conventions. It represents the unique -// identifier of a task run within a pipeline. -func CICDPipelineTaskRunID(val string) attribute.KeyValue { - return CICDPipelineTaskRunIDKey.String(val) -} - -// CICDPipelineTaskRunURLFull returns an attribute KeyValue conforming to the -// "cicd.pipeline.task.run.url.full" semantic conventions. It represents the -// [URL] of the pipeline task run, providing the complete address in order to -// locate and identify the pipeline task run. -// -// [URL]: https://wikipedia.org/wiki/URL -func CICDPipelineTaskRunURLFull(val string) attribute.KeyValue { - return CICDPipelineTaskRunURLFullKey.String(val) -} - -// CICDSystemComponent returns an attribute KeyValue conforming to the -// "cicd.system.component" semantic conventions. It represents the name of a -// component of the CICD system. -func CICDSystemComponent(val string) attribute.KeyValue { - return CICDSystemComponentKey.String(val) -} - -// CICDWorkerID returns an attribute KeyValue conforming to the "cicd.worker.id" -// semantic conventions. It represents the unique identifier of a worker within a -// CICD system. -func CICDWorkerID(val string) attribute.KeyValue { - return CICDWorkerIDKey.String(val) -} - -// CICDWorkerName returns an attribute KeyValue conforming to the -// "cicd.worker.name" semantic conventions. It represents the name of a worker -// within a CICD system. -func CICDWorkerName(val string) attribute.KeyValue { - return CICDWorkerNameKey.String(val) -} - -// CICDWorkerURLFull returns an attribute KeyValue conforming to the -// "cicd.worker.url.full" semantic conventions. It represents the [URL] of the -// worker, providing the complete address in order to locate and identify the -// worker. -// -// [URL]: https://wikipedia.org/wiki/URL -func CICDWorkerURLFull(val string) attribute.KeyValue { - return CICDWorkerURLFullKey.String(val) -} - -// Enum values for cicd.pipeline.action.name -var ( - // The pipeline run is executing a build. - // Stability: development - CICDPipelineActionNameBuild = CICDPipelineActionNameKey.String("BUILD") - // The pipeline run is executing. - // Stability: development - CICDPipelineActionNameRun = CICDPipelineActionNameKey.String("RUN") - // The pipeline run is executing a sync. - // Stability: development - CICDPipelineActionNameSync = CICDPipelineActionNameKey.String("SYNC") -) - -// Enum values for cicd.pipeline.result -var ( - // The pipeline run finished successfully. - // Stability: development - CICDPipelineResultSuccess = CICDPipelineResultKey.String("success") - // The pipeline run did not finish successfully, eg. due to a compile error or a - // failing test. Such failures are usually detected by non-zero exit codes of - // the tools executed in the pipeline run. - // Stability: development - CICDPipelineResultFailure = CICDPipelineResultKey.String("failure") - // The pipeline run failed due to an error in the CICD system, eg. due to the - // worker being killed. - // Stability: development - CICDPipelineResultError = CICDPipelineResultKey.String("error") - // A timeout caused the pipeline run to be interrupted. - // Stability: development - CICDPipelineResultTimeout = CICDPipelineResultKey.String("timeout") - // The pipeline run was cancelled, eg. by a user manually cancelling the - // pipeline run. - // Stability: development - CICDPipelineResultCancellation = CICDPipelineResultKey.String("cancellation") - // The pipeline run was skipped, eg. due to a precondition not being met. - // Stability: development - CICDPipelineResultSkip = CICDPipelineResultKey.String("skip") -) - -// Enum values for cicd.pipeline.run.state -var ( - // The run pending state spans from the event triggering the pipeline run until - // the execution of the run starts (eg. time spent in a queue, provisioning - // agents, creating run resources). - // - // Stability: development - CICDPipelineRunStatePending = CICDPipelineRunStateKey.String("pending") - // The executing state spans the execution of any run tasks (eg. build, test). - // Stability: development - CICDPipelineRunStateExecuting = CICDPipelineRunStateKey.String("executing") - // The finalizing state spans from when the run has finished executing (eg. - // cleanup of run resources). - // Stability: development - CICDPipelineRunStateFinalizing = CICDPipelineRunStateKey.String("finalizing") -) - -// Enum values for cicd.pipeline.task.run.result -var ( - // The task run finished successfully. - // Stability: development - CICDPipelineTaskRunResultSuccess = CICDPipelineTaskRunResultKey.String("success") - // The task run did not finish successfully, eg. due to a compile error or a - // failing test. Such failures are usually detected by non-zero exit codes of - // the tools executed in the task run. - // Stability: development - CICDPipelineTaskRunResultFailure = CICDPipelineTaskRunResultKey.String("failure") - // The task run failed due to an error in the CICD system, eg. due to the worker - // being killed. - // Stability: development - CICDPipelineTaskRunResultError = CICDPipelineTaskRunResultKey.String("error") - // A timeout caused the task run to be interrupted. - // Stability: development - CICDPipelineTaskRunResultTimeout = CICDPipelineTaskRunResultKey.String("timeout") - // The task run was cancelled, eg. by a user manually cancelling the task run. - // Stability: development - CICDPipelineTaskRunResultCancellation = CICDPipelineTaskRunResultKey.String("cancellation") - // The task run was skipped, eg. due to a precondition not being met. - // Stability: development - CICDPipelineTaskRunResultSkip = CICDPipelineTaskRunResultKey.String("skip") -) - -// Enum values for cicd.pipeline.task.type -var ( - // build - // Stability: development - CICDPipelineTaskTypeBuild = CICDPipelineTaskTypeKey.String("build") - // test - // Stability: development - CICDPipelineTaskTypeTest = CICDPipelineTaskTypeKey.String("test") - // deploy - // Stability: development - CICDPipelineTaskTypeDeploy = CICDPipelineTaskTypeKey.String("deploy") -) - -// Enum values for cicd.worker.state -var ( - // The worker is not performing work for the CICD system. It is available to the - // CICD system to perform work on (online / idle). - // Stability: development - CICDWorkerStateAvailable = CICDWorkerStateKey.String("available") - // The worker is performing work for the CICD system. - // Stability: development - CICDWorkerStateBusy = CICDWorkerStateKey.String("busy") - // The worker is not available to the CICD system (disconnected / down). - // Stability: development - CICDWorkerStateOffline = CICDWorkerStateKey.String("offline") -) - -// Namespace: client -const ( - // ClientAddressKey is the attribute Key conforming to the "client.address" - // semantic conventions. It represents the client address - domain name if - // available without reverse DNS lookup; otherwise, IP address or Unix domain - // socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "client.example.com", "10.1.2.80", "/tmp/my.sock" - // Note: When observed from the server side, and when communicating through an - // intermediary, `client.address` SHOULD represent the client address behind any - // intermediaries, for example proxies, if it's available. - ClientAddressKey = attribute.Key("client.address") - - // ClientPortKey is the attribute Key conforming to the "client.port" semantic - // conventions. It represents the client port number. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 65123 - // Note: When observed from the server side, and when communicating through an - // intermediary, `client.port` SHOULD represent the client port behind any - // intermediaries, for example proxies, if it's available. - ClientPortKey = attribute.Key("client.port") -) - -// ClientAddress returns an attribute KeyValue conforming to the "client.address" -// semantic conventions. It represents the client address - domain name if -// available without reverse DNS lookup; otherwise, IP address or Unix domain -// socket name. -func ClientAddress(val string) attribute.KeyValue { - return ClientAddressKey.String(val) -} - -// ClientPort returns an attribute KeyValue conforming to the "client.port" -// semantic conventions. It represents the client port number. -func ClientPort(val int) attribute.KeyValue { - return ClientPortKey.Int(val) -} - -// Namespace: cloud -const ( - // CloudAccountIDKey is the attribute Key conforming to the "cloud.account.id" - // semantic conventions. It represents the cloud account ID the resource is - // assigned to. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "111111111111", "opentelemetry" - CloudAccountIDKey = attribute.Key("cloud.account.id") - - // CloudAvailabilityZoneKey is the attribute Key conforming to the - // "cloud.availability_zone" semantic conventions. It represents the cloud - // regions often have multiple, isolated locations known as zones to increase - // availability. Availability zone represents the zone where the resource is - // running. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "us-east-1c" - // Note: Availability zones are called "zones" on Alibaba Cloud and Google - // Cloud. - CloudAvailabilityZoneKey = attribute.Key("cloud.availability_zone") - - // CloudPlatformKey is the attribute Key conforming to the "cloud.platform" - // semantic conventions. It represents the cloud platform in use. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The prefix of the service SHOULD match the one specified in - // `cloud.provider`. - CloudPlatformKey = attribute.Key("cloud.platform") - - // CloudProviderKey is the attribute Key conforming to the "cloud.provider" - // semantic conventions. It represents the name of the cloud provider. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - CloudProviderKey = attribute.Key("cloud.provider") - - // CloudRegionKey is the attribute Key conforming to the "cloud.region" semantic - // conventions. It represents the geographical region within a cloud provider. - // When associated with a resource, this attribute specifies the region where - // the resource operates. When calling services or APIs deployed on a cloud, - // this attribute identifies the region where the called destination is - // deployed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "us-central1", "us-east-1" - // Note: Refer to your provider's docs to see the available regions, for example - // [Alibaba Cloud regions], [AWS regions], [Azure regions], - // [Google Cloud regions], or [Tencent Cloud regions]. - // - // [Alibaba Cloud regions]: https://www.alibabacloud.com/help/doc-detail/40654.htm - // [AWS regions]: https://aws.amazon.com/about-aws/global-infrastructure/regions_az/ - // [Azure regions]: https://azure.microsoft.com/global-infrastructure/geographies/ - // [Google Cloud regions]: https://cloud.google.com/about/locations - // [Tencent Cloud regions]: https://www.tencentcloud.com/document/product/213/6091 - CloudRegionKey = attribute.Key("cloud.region") - - // CloudResourceIDKey is the attribute Key conforming to the "cloud.resource_id" - // semantic conventions. It represents the cloud provider-specific native - // identifier of the monitored cloud resource (e.g. an [ARN] on AWS, a - // [fully qualified resource ID] on Azure, a [full resource name] on GCP). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "arn:aws:lambda:REGION:ACCOUNT_ID:function:my-function", - // "//run.googleapis.com/projects/PROJECT_ID/locations/LOCATION_ID/services/SERVICE_ID", - // "/subscriptions//resourceGroups/ - // /providers/Microsoft.Web/sites//functions/" - // Note: On some cloud providers, it may not be possible to determine the full - // ID at startup, - // so it may be necessary to set `cloud.resource_id` as a span attribute - // instead. - // - // The exact value to use for `cloud.resource_id` depends on the cloud provider. - // The following well-known definitions MUST be used if you set this attribute - // and they apply: - // - // - **AWS Lambda:** The function [ARN]. - // Take care not to use the "invoked ARN" directly but replace any - // [alias suffix] - // with the resolved function version, as the same runtime instance may be - // invocable with - // multiple different aliases. - // - **GCP:** The [URI of the resource] - // - **Azure:** The [Fully Qualified Resource ID] of the invoked function, - // *not* the function app, having the form - // - // `/subscriptions//resourceGroups//providers/Microsoft.Web/sites//functions/` - // . - // This means that a span attribute MUST be used, as an Azure function app - // can host multiple functions that would usually share - // a TracerProvider. - // - // - // [ARN]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html - // [fully qualified resource ID]: https://learn.microsoft.com/rest/api/resources/resources/get-by-id - // [full resource name]: https://google.aip.dev/122#full-resource-names - // [ARN]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html - // [alias suffix]: https://docs.aws.amazon.com/lambda/latest/dg/configuration-aliases.html - // [URI of the resource]: https://cloud.google.com/iam/docs/full-resource-names - // [Fully Qualified Resource ID]: https://learn.microsoft.com/rest/api/resources/resources/get-by-id - CloudResourceIDKey = attribute.Key("cloud.resource_id") -) - -// CloudAccountID returns an attribute KeyValue conforming to the -// "cloud.account.id" semantic conventions. It represents the cloud account ID -// the resource is assigned to. -func CloudAccountID(val string) attribute.KeyValue { - return CloudAccountIDKey.String(val) -} - -// CloudAvailabilityZone returns an attribute KeyValue conforming to the -// "cloud.availability_zone" semantic conventions. It represents the cloud -// regions often have multiple, isolated locations known as zones to increase -// availability. Availability zone represents the zone where the resource is -// running. -func CloudAvailabilityZone(val string) attribute.KeyValue { - return CloudAvailabilityZoneKey.String(val) -} - -// CloudRegion returns an attribute KeyValue conforming to the "cloud.region" -// semantic conventions. It represents the geographical region within a cloud -// provider. When associated with a resource, this attribute specifies the region -// where the resource operates. When calling services or APIs deployed on a -// cloud, this attribute identifies the region where the called destination is -// deployed. -func CloudRegion(val string) attribute.KeyValue { - return CloudRegionKey.String(val) -} - -// CloudResourceID returns an attribute KeyValue conforming to the -// "cloud.resource_id" semantic conventions. It represents the cloud -// provider-specific native identifier of the monitored cloud resource (e.g. an -// [ARN] on AWS, a [fully qualified resource ID] on Azure, a [full resource name] -// -// on GCP). -// -// [ARN]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html -// [fully qualified resource ID]: https://learn.microsoft.com/rest/api/resources/resources/get-by-id -// [full resource name]: https://google.aip.dev/122#full-resource-names -func CloudResourceID(val string) attribute.KeyValue { - return CloudResourceIDKey.String(val) -} - -// Enum values for cloud.platform -var ( - // Akamai Cloud Compute - // Stability: development - CloudPlatformAkamaiCloudCompute = CloudPlatformKey.String("akamai_cloud.compute") - // Alibaba Cloud Elastic Compute Service - // Stability: development - CloudPlatformAlibabaCloudECS = CloudPlatformKey.String("alibaba_cloud_ecs") - // Alibaba Cloud Function Compute - // Stability: development - CloudPlatformAlibabaCloudFC = CloudPlatformKey.String("alibaba_cloud_fc") - // Red Hat OpenShift on Alibaba Cloud - // Stability: development - CloudPlatformAlibabaCloudOpenShift = CloudPlatformKey.String("alibaba_cloud_openshift") - // AWS Elastic Compute Cloud - // Stability: development - CloudPlatformAWSEC2 = CloudPlatformKey.String("aws_ec2") - // AWS Elastic Container Service - // Stability: development - CloudPlatformAWSECS = CloudPlatformKey.String("aws_ecs") - // AWS Elastic Kubernetes Service - // Stability: development - CloudPlatformAWSEKS = CloudPlatformKey.String("aws_eks") - // AWS Lambda - // Stability: development - CloudPlatformAWSLambda = CloudPlatformKey.String("aws_lambda") - // AWS Elastic Beanstalk - // Stability: development - CloudPlatformAWSElasticBeanstalk = CloudPlatformKey.String("aws_elastic_beanstalk") - // AWS App Runner - // Stability: development - CloudPlatformAWSAppRunner = CloudPlatformKey.String("aws_app_runner") - // Red Hat OpenShift on AWS (ROSA) - // Stability: development - CloudPlatformAWSOpenShift = CloudPlatformKey.String("aws_openshift") - // Azure Virtual Machines - // Stability: development - CloudPlatformAzureVM = CloudPlatformKey.String("azure.vm") - // Azure Container Apps - // Stability: development - CloudPlatformAzureContainerApps = CloudPlatformKey.String("azure.container_apps") - // Azure Container Instances - // Stability: development - CloudPlatformAzureContainerInstances = CloudPlatformKey.String("azure.container_instances") - // Azure Kubernetes Service - // Stability: development - CloudPlatformAzureAKS = CloudPlatformKey.String("azure.aks") - // Azure Functions - // Stability: development - CloudPlatformAzureFunctions = CloudPlatformKey.String("azure.functions") - // Azure App Service - // Stability: development - CloudPlatformAzureAppService = CloudPlatformKey.String("azure.app_service") - // Azure Red Hat OpenShift - // Stability: development - CloudPlatformAzureOpenShift = CloudPlatformKey.String("azure.openshift") - // Google Vertex AI Agent Engine - // Stability: development - CloudPlatformGCPAgentEngine = CloudPlatformKey.String("gcp.agent_engine") - // Google Bare Metal Solution (BMS) - // Stability: development - CloudPlatformGCPBareMetalSolution = CloudPlatformKey.String("gcp_bare_metal_solution") - // Google Cloud Compute Engine (GCE) - // Stability: development - CloudPlatformGCPComputeEngine = CloudPlatformKey.String("gcp_compute_engine") - // Google Cloud Run - // Stability: development - CloudPlatformGCPCloudRun = CloudPlatformKey.String("gcp_cloud_run") - // Google Cloud Kubernetes Engine (GKE) - // Stability: development - CloudPlatformGCPKubernetesEngine = CloudPlatformKey.String("gcp_kubernetes_engine") - // Google Cloud Functions (GCF) - // Stability: development - CloudPlatformGCPCloudFunctions = CloudPlatformKey.String("gcp_cloud_functions") - // Google Cloud App Engine (GAE) - // Stability: development - CloudPlatformGCPAppEngine = CloudPlatformKey.String("gcp_app_engine") - // Red Hat OpenShift on Google Cloud - // Stability: development - CloudPlatformGCPOpenShift = CloudPlatformKey.String("gcp_openshift") - // Server on Hetzner Cloud - // Stability: development - CloudPlatformHetznerCloudServer = CloudPlatformKey.String("hetzner.cloud_server") - // Red Hat OpenShift on IBM Cloud - // Stability: development - CloudPlatformIBMCloudOpenShift = CloudPlatformKey.String("ibm_cloud_openshift") - // Compute on Oracle Cloud Infrastructure (OCI) - // Stability: development - CloudPlatformOracleCloudCompute = CloudPlatformKey.String("oracle_cloud_compute") - // Kubernetes Engine (OKE) on Oracle Cloud Infrastructure (OCI) - // Stability: development - CloudPlatformOracleCloudOKE = CloudPlatformKey.String("oracle_cloud_oke") - // Tencent Cloud Cloud Virtual Machine (CVM) - // Stability: development - CloudPlatformTencentCloudCVM = CloudPlatformKey.String("tencent_cloud_cvm") - // Tencent Cloud Elastic Kubernetes Service (EKS) - // Stability: development - CloudPlatformTencentCloudEKS = CloudPlatformKey.String("tencent_cloud_eks") - // Tencent Cloud Serverless Cloud Function (SCF) - // Stability: development - CloudPlatformTencentCloudSCF = CloudPlatformKey.String("tencent_cloud_scf") - // Vultr Cloud Compute - // Stability: development - CloudPlatformVultrCloudCompute = CloudPlatformKey.String("vultr.cloud_compute") -) - -// Enum values for cloud.provider -var ( - // Akamai Cloud - // Stability: development - CloudProviderAkamaiCloud = CloudProviderKey.String("akamai_cloud") - // Alibaba Cloud - // Stability: development - CloudProviderAlibabaCloud = CloudProviderKey.String("alibaba_cloud") - // Amazon Web Services - // Stability: development - CloudProviderAWS = CloudProviderKey.String("aws") - // Microsoft Azure - // Stability: development - CloudProviderAzure = CloudProviderKey.String("azure") - // Google Cloud Platform - // Stability: development - CloudProviderGCP = CloudProviderKey.String("gcp") - // Heroku Platform as a Service - // Stability: development - CloudProviderHeroku = CloudProviderKey.String("heroku") - // Hetzner - // Stability: development - CloudProviderHetzner = CloudProviderKey.String("hetzner") - // IBM Cloud - // Stability: development - CloudProviderIBMCloud = CloudProviderKey.String("ibm_cloud") - // Oracle Cloud Infrastructure (OCI) - // Stability: development - CloudProviderOracleCloud = CloudProviderKey.String("oracle_cloud") - // Tencent Cloud - // Stability: development - CloudProviderTencentCloud = CloudProviderKey.String("tencent_cloud") - // Vultr - // Stability: development - CloudProviderVultr = CloudProviderKey.String("vultr") -) - -// Namespace: cloudevents -const ( - // CloudEventsEventIDKey is the attribute Key conforming to the - // "cloudevents.event_id" semantic conventions. It represents the [event_id] - // uniquely identifies the event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "123e4567-e89b-12d3-a456-426614174000", "0001" - // - // [event_id]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#id - CloudEventsEventIDKey = attribute.Key("cloudevents.event_id") - - // CloudEventsEventSourceKey is the attribute Key conforming to the - // "cloudevents.event_source" semantic conventions. It represents the [source] - // identifies the context in which an event happened. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://github.com/cloudevents", "/cloudevents/spec/pull/123", - // "my-service" - // - // [source]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#source-1 - CloudEventsEventSourceKey = attribute.Key("cloudevents.event_source") - - // CloudEventsEventSpecVersionKey is the attribute Key conforming to the - // "cloudevents.event_spec_version" semantic conventions. It represents the - // [version of the CloudEvents specification] which the event uses. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0 - // - // [version of the CloudEvents specification]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#specversion - CloudEventsEventSpecVersionKey = attribute.Key("cloudevents.event_spec_version") - - // CloudEventsEventSubjectKey is the attribute Key conforming to the - // "cloudevents.event_subject" semantic conventions. It represents the [subject] - // of the event in the context of the event producer (identified by source). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: mynewfile.jpg - // - // [subject]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#subject - CloudEventsEventSubjectKey = attribute.Key("cloudevents.event_subject") - - // CloudEventsEventTypeKey is the attribute Key conforming to the - // "cloudevents.event_type" semantic conventions. It represents the [event_type] - // contains a value describing the type of event related to the originating - // occurrence. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "com.github.pull_request.opened", "com.example.object.deleted.v2" - // - // [event_type]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#type - CloudEventsEventTypeKey = attribute.Key("cloudevents.event_type") -) - -// CloudEventsEventID returns an attribute KeyValue conforming to the -// "cloudevents.event_id" semantic conventions. It represents the [event_id] -// uniquely identifies the event. -// -// [event_id]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#id -func CloudEventsEventID(val string) attribute.KeyValue { - return CloudEventsEventIDKey.String(val) -} - -// CloudEventsEventSource returns an attribute KeyValue conforming to the -// "cloudevents.event_source" semantic conventions. It represents the [source] -// identifies the context in which an event happened. -// -// [source]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#source-1 -func CloudEventsEventSource(val string) attribute.KeyValue { - return CloudEventsEventSourceKey.String(val) -} - -// CloudEventsEventSpecVersion returns an attribute KeyValue conforming to the -// "cloudevents.event_spec_version" semantic conventions. It represents the -// [version of the CloudEvents specification] which the event uses. -// -// [version of the CloudEvents specification]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#specversion -func CloudEventsEventSpecVersion(val string) attribute.KeyValue { - return CloudEventsEventSpecVersionKey.String(val) -} - -// CloudEventsEventSubject returns an attribute KeyValue conforming to the -// "cloudevents.event_subject" semantic conventions. It represents the [subject] -// of the event in the context of the event producer (identified by source). -// -// [subject]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#subject -func CloudEventsEventSubject(val string) attribute.KeyValue { - return CloudEventsEventSubjectKey.String(val) -} - -// CloudEventsEventType returns an attribute KeyValue conforming to the -// "cloudevents.event_type" semantic conventions. It represents the [event_type] -// contains a value describing the type of event related to the originating -// occurrence. -// -// [event_type]: https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md#type -func CloudEventsEventType(val string) attribute.KeyValue { - return CloudEventsEventTypeKey.String(val) -} - -// Namespace: cloudfoundry -const ( - // CloudFoundryAppIDKey is the attribute Key conforming to the - // "cloudfoundry.app.id" semantic conventions. It represents the guid of the - // application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.application_id`. This is the same value as - // reported by `cf app --guid`. - CloudFoundryAppIDKey = attribute.Key("cloudfoundry.app.id") - - // CloudFoundryAppInstanceIDKey is the attribute Key conforming to the - // "cloudfoundry.app.instance.id" semantic conventions. It represents the index - // of the application instance. 0 when just one instance is active. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0", "1" - // Note: CloudFoundry defines the `instance_id` in the [Loggregator v2 envelope] - // . - // It is used for logs and metrics emitted by CloudFoundry. It is - // supposed to contain the application instance index for applications - // deployed on the runtime. - // - // Application instrumentation should use the value from environment - // variable `CF_INSTANCE_INDEX`. - // - // [Loggregator v2 envelope]: https://github.com/cloudfoundry/loggregator-api#v2-envelope - CloudFoundryAppInstanceIDKey = attribute.Key("cloudfoundry.app.instance.id") - - // CloudFoundryAppNameKey is the attribute Key conforming to the - // "cloudfoundry.app.name" semantic conventions. It represents the name of the - // application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-app-name" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.application_name`. This is the same value - // as reported by `cf apps`. - CloudFoundryAppNameKey = attribute.Key("cloudfoundry.app.name") - - // CloudFoundryOrgIDKey is the attribute Key conforming to the - // "cloudfoundry.org.id" semantic conventions. It represents the guid of the - // CloudFoundry org the application is running in. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.org_id`. This is the same value as - // reported by `cf org --guid`. - CloudFoundryOrgIDKey = attribute.Key("cloudfoundry.org.id") - - // CloudFoundryOrgNameKey is the attribute Key conforming to the - // "cloudfoundry.org.name" semantic conventions. It represents the name of the - // CloudFoundry organization the app is running in. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-org-name" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.org_name`. This is the same value as - // reported by `cf orgs`. - CloudFoundryOrgNameKey = attribute.Key("cloudfoundry.org.name") - - // CloudFoundryProcessIDKey is the attribute Key conforming to the - // "cloudfoundry.process.id" semantic conventions. It represents the UID - // identifying the process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.process_id`. It is supposed to be equal to - // `VCAP_APPLICATION.app_id` for applications deployed to the runtime. - // For system components, this could be the actual PID. - CloudFoundryProcessIDKey = attribute.Key("cloudfoundry.process.id") - - // CloudFoundryProcessTypeKey is the attribute Key conforming to the - // "cloudfoundry.process.type" semantic conventions. It represents the type of - // process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "web" - // Note: CloudFoundry applications can consist of multiple jobs. Usually the - // main process will be of type `web`. There can be additional background - // tasks or side-cars with different process types. - CloudFoundryProcessTypeKey = attribute.Key("cloudfoundry.process.type") - - // CloudFoundrySpaceIDKey is the attribute Key conforming to the - // "cloudfoundry.space.id" semantic conventions. It represents the guid of the - // CloudFoundry space the application is running in. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.space_id`. This is the same value as - // reported by `cf space --guid`. - CloudFoundrySpaceIDKey = attribute.Key("cloudfoundry.space.id") - - // CloudFoundrySpaceNameKey is the attribute Key conforming to the - // "cloudfoundry.space.name" semantic conventions. It represents the name of the - // CloudFoundry space the application is running in. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-space-name" - // Note: Application instrumentation should use the value from environment - // variable `VCAP_APPLICATION.space_name`. This is the same value as - // reported by `cf spaces`. - CloudFoundrySpaceNameKey = attribute.Key("cloudfoundry.space.name") - - // CloudFoundrySystemIDKey is the attribute Key conforming to the - // "cloudfoundry.system.id" semantic conventions. It represents a guid or - // another name describing the event source. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cf/gorouter" - // Note: CloudFoundry defines the `source_id` in the [Loggregator v2 envelope]. - // It is used for logs and metrics emitted by CloudFoundry. It is - // supposed to contain the component name, e.g. "gorouter", for - // CloudFoundry components. - // - // When system components are instrumented, values from the - // [Bosh spec] - // should be used. The `system.id` should be set to - // `spec.deployment/spec.name`. - // - // [Loggregator v2 envelope]: https://github.com/cloudfoundry/loggregator-api#v2-envelope - // [Bosh spec]: https://bosh.io/docs/jobs/#properties-spec - CloudFoundrySystemIDKey = attribute.Key("cloudfoundry.system.id") - - // CloudFoundrySystemInstanceIDKey is the attribute Key conforming to the - // "cloudfoundry.system.instance.id" semantic conventions. It represents a guid - // describing the concrete instance of the event source. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: CloudFoundry defines the `instance_id` in the [Loggregator v2 envelope] - // . - // It is used for logs and metrics emitted by CloudFoundry. It is - // supposed to contain the vm id for CloudFoundry components. - // - // When system components are instrumented, values from the - // [Bosh spec] - // should be used. The `system.instance.id` should be set to `spec.id`. - // - // [Loggregator v2 envelope]: https://github.com/cloudfoundry/loggregator-api#v2-envelope - // [Bosh spec]: https://bosh.io/docs/jobs/#properties-spec - CloudFoundrySystemInstanceIDKey = attribute.Key("cloudfoundry.system.instance.id") -) - -// CloudFoundryAppID returns an attribute KeyValue conforming to the -// "cloudfoundry.app.id" semantic conventions. It represents the guid of the -// application. -func CloudFoundryAppID(val string) attribute.KeyValue { - return CloudFoundryAppIDKey.String(val) -} - -// CloudFoundryAppInstanceID returns an attribute KeyValue conforming to the -// "cloudfoundry.app.instance.id" semantic conventions. It represents the index -// of the application instance. 0 when just one instance is active. -func CloudFoundryAppInstanceID(val string) attribute.KeyValue { - return CloudFoundryAppInstanceIDKey.String(val) -} - -// CloudFoundryAppName returns an attribute KeyValue conforming to the -// "cloudfoundry.app.name" semantic conventions. It represents the name of the -// application. -func CloudFoundryAppName(val string) attribute.KeyValue { - return CloudFoundryAppNameKey.String(val) -} - -// CloudFoundryOrgID returns an attribute KeyValue conforming to the -// "cloudfoundry.org.id" semantic conventions. It represents the guid of the -// CloudFoundry org the application is running in. -func CloudFoundryOrgID(val string) attribute.KeyValue { - return CloudFoundryOrgIDKey.String(val) -} - -// CloudFoundryOrgName returns an attribute KeyValue conforming to the -// "cloudfoundry.org.name" semantic conventions. It represents the name of the -// CloudFoundry organization the app is running in. -func CloudFoundryOrgName(val string) attribute.KeyValue { - return CloudFoundryOrgNameKey.String(val) -} - -// CloudFoundryProcessID returns an attribute KeyValue conforming to the -// "cloudfoundry.process.id" semantic conventions. It represents the UID -// identifying the process. -func CloudFoundryProcessID(val string) attribute.KeyValue { - return CloudFoundryProcessIDKey.String(val) -} - -// CloudFoundryProcessType returns an attribute KeyValue conforming to the -// "cloudfoundry.process.type" semantic conventions. It represents the type of -// process. -func CloudFoundryProcessType(val string) attribute.KeyValue { - return CloudFoundryProcessTypeKey.String(val) -} - -// CloudFoundrySpaceID returns an attribute KeyValue conforming to the -// "cloudfoundry.space.id" semantic conventions. It represents the guid of the -// CloudFoundry space the application is running in. -func CloudFoundrySpaceID(val string) attribute.KeyValue { - return CloudFoundrySpaceIDKey.String(val) -} - -// CloudFoundrySpaceName returns an attribute KeyValue conforming to the -// "cloudfoundry.space.name" semantic conventions. It represents the name of the -// CloudFoundry space the application is running in. -func CloudFoundrySpaceName(val string) attribute.KeyValue { - return CloudFoundrySpaceNameKey.String(val) -} - -// CloudFoundrySystemID returns an attribute KeyValue conforming to the -// "cloudfoundry.system.id" semantic conventions. It represents a guid or another -// name describing the event source. -func CloudFoundrySystemID(val string) attribute.KeyValue { - return CloudFoundrySystemIDKey.String(val) -} - -// CloudFoundrySystemInstanceID returns an attribute KeyValue conforming to the -// "cloudfoundry.system.instance.id" semantic conventions. It represents a guid -// describing the concrete instance of the event source. -func CloudFoundrySystemInstanceID(val string) attribute.KeyValue { - return CloudFoundrySystemInstanceIDKey.String(val) -} - -// Namespace: code -const ( - // CodeColumnNumberKey is the attribute Key conforming to the - // "code.column.number" semantic conventions. It represents the column number in - // `code.file.path` best representing the operation. It SHOULD point within the - // code unit named in `code.function.name`. This attribute MUST NOT be used on - // the Profile signal since the data is already captured in 'message Line'. This - // constraint is imposed to prevent redundancy and maintain data integrity. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - CodeColumnNumberKey = attribute.Key("code.column.number") - - // CodeFilePathKey is the attribute Key conforming to the "code.file.path" - // semantic conventions. It represents the source code file name that identifies - // the code unit as uniquely as possible (preferably an absolute file path). - // This attribute MUST NOT be used on the Profile signal since the data is - // already captured in 'message Function'. This constraint is imposed to prevent - // redundancy and maintain data integrity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: /usr/local/MyApplication/content_root/app/index.php - CodeFilePathKey = attribute.Key("code.file.path") - - // CodeFunctionNameKey is the attribute Key conforming to the - // "code.function.name" semantic conventions. It represents the method or - // function fully-qualified name without arguments. The value should fit the - // natural representation of the language runtime, which is also likely the same - // used within `code.stacktrace` attribute value. This attribute MUST NOT be - // used on the Profile signal since the data is already captured in 'message - // Function'. This constraint is imposed to prevent redundancy and maintain data - // integrity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "com.example.MyHttpService.serveRequest", - // "GuzzleHttp\Client::transfer", "fopen" - // Note: Values and format depends on each language runtime, thus it is - // impossible to provide an exhaustive list of examples. - // The values are usually the same (or prefixes of) the ones found in native - // stack trace representation stored in - // `code.stacktrace` without information on arguments. - // - // Examples: - // - // - Java method: `com.example.MyHttpService.serveRequest` - // - Java anonymous class method: `com.mycompany.Main$1.myMethod` - // - Java lambda method: - // `com.mycompany.Main$$Lambda/0x0000748ae4149c00.myMethod` - // - PHP function: `GuzzleHttp\Client::transfer` - // - Go function: `github.com/my/repo/pkg.foo.func5` - // - Elixir: `OpenTelemetry.Ctx.new` - // - Erlang: `opentelemetry_ctx:new` - // - Rust: `playground::my_module::my_cool_func` - // - C function: `fopen` - CodeFunctionNameKey = attribute.Key("code.function.name") - - // CodeLineNumberKey is the attribute Key conforming to the "code.line.number" - // semantic conventions. It represents the line number in `code.file.path` best - // representing the operation. It SHOULD point within the code unit named in - // `code.function.name`. This attribute MUST NOT be used on the Profile signal - // since the data is already captured in 'message Line'. This constraint is - // imposed to prevent redundancy and maintain data integrity. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - CodeLineNumberKey = attribute.Key("code.line.number") - - // CodeStacktraceKey is the attribute Key conforming to the "code.stacktrace" - // semantic conventions. It represents a stacktrace as a string in the natural - // representation for the language runtime. The representation is identical to - // [`exception.stacktrace`]. This attribute MUST NOT be used on the Profile - // signal since the data is already captured in 'message Location'. This - // constraint is imposed to prevent redundancy and maintain data integrity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at - // com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at - // com.example.GenerateTrace.main(GenerateTrace.java:5) - // - // [`exception.stacktrace`]: /docs/exceptions/exceptions-spans.md#stacktrace-representation - CodeStacktraceKey = attribute.Key("code.stacktrace") -) - -// CodeColumnNumber returns an attribute KeyValue conforming to the -// "code.column.number" semantic conventions. It represents the column number in -// `code.file.path` best representing the operation. It SHOULD point within the -// code unit named in `code.function.name`. This attribute MUST NOT be used on -// the Profile signal since the data is already captured in 'message Line'. This -// constraint is imposed to prevent redundancy and maintain data integrity. -func CodeColumnNumber(val int) attribute.KeyValue { - return CodeColumnNumberKey.Int(val) -} - -// CodeFilePath returns an attribute KeyValue conforming to the "code.file.path" -// semantic conventions. It represents the source code file name that identifies -// the code unit as uniquely as possible (preferably an absolute file path). This -// attribute MUST NOT be used on the Profile signal since the data is already -// captured in 'message Function'. This constraint is imposed to prevent -// redundancy and maintain data integrity. -func CodeFilePath(val string) attribute.KeyValue { - return CodeFilePathKey.String(val) -} - -// CodeFunctionName returns an attribute KeyValue conforming to the -// "code.function.name" semantic conventions. It represents the method or -// function fully-qualified name without arguments. The value should fit the -// natural representation of the language runtime, which is also likely the same -// used within `code.stacktrace` attribute value. This attribute MUST NOT be used -// on the Profile signal since the data is already captured in 'message -// Function'. This constraint is imposed to prevent redundancy and maintain data -// integrity. -func CodeFunctionName(val string) attribute.KeyValue { - return CodeFunctionNameKey.String(val) -} - -// CodeLineNumber returns an attribute KeyValue conforming to the -// "code.line.number" semantic conventions. It represents the line number in -// `code.file.path` best representing the operation. It SHOULD point within the -// code unit named in `code.function.name`. This attribute MUST NOT be used on -// the Profile signal since the data is already captured in 'message Line'. This -// constraint is imposed to prevent redundancy and maintain data integrity. -func CodeLineNumber(val int) attribute.KeyValue { - return CodeLineNumberKey.Int(val) -} - -// CodeStacktrace returns an attribute KeyValue conforming to the -// "code.stacktrace" semantic conventions. It represents a stacktrace as a string -// in the natural representation for the language runtime. The representation is -// identical to [`exception.stacktrace`]. This attribute MUST NOT be used on the -// Profile signal since the data is already captured in 'message Location'. This -// constraint is imposed to prevent redundancy and maintain data integrity. -// -// [`exception.stacktrace`]: /docs/exceptions/exceptions-spans.md#stacktrace-representation -func CodeStacktrace(val string) attribute.KeyValue { - return CodeStacktraceKey.String(val) -} - -// Namespace: container -const ( - // ContainerCommandKey is the attribute Key conforming to the - // "container.command" semantic conventions. It represents the command used to - // run the container (i.e. the command name). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "otelcontribcol" - // Note: If using embedded credentials or sensitive data, it is recommended to - // remove them to prevent potential leakage. - ContainerCommandKey = attribute.Key("container.command") - - // ContainerCommandArgsKey is the attribute Key conforming to the - // "container.command_args" semantic conventions. It represents the all the - // command arguments (including the command/executable itself) run by the - // container. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "otelcontribcol", "--config", "config.yaml" - ContainerCommandArgsKey = attribute.Key("container.command_args") - - // ContainerCommandLineKey is the attribute Key conforming to the - // "container.command_line" semantic conventions. It represents the full command - // run by the container as a single string representing the full command. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "otelcontribcol --config config.yaml" - ContainerCommandLineKey = attribute.Key("container.command_line") - - // ContainerCSIPluginNameKey is the attribute Key conforming to the - // "container.csi.plugin.name" semantic conventions. It represents the name of - // the CSI ([Container Storage Interface]) plugin used by the volume. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "pd.csi.storage.gke.io" - // Note: This can sometimes be referred to as a "driver" in CSI implementations. - // This should represent the `name` field of the GetPluginInfo RPC. - // - // [Container Storage Interface]: https://github.com/container-storage-interface/spec - ContainerCSIPluginNameKey = attribute.Key("container.csi.plugin.name") - - // ContainerCSIVolumeIDKey is the attribute Key conforming to the - // "container.csi.volume.id" semantic conventions. It represents the unique - // volume ID returned by the CSI ([Container Storage Interface]) plugin. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "projects/my-gcp-project/zones/my-gcp-zone/disks/my-gcp-disk" - // Note: This can sometimes be referred to as a "volume handle" in CSI - // implementations. This should represent the `Volume.volume_id` field in CSI - // spec. - // - // [Container Storage Interface]: https://github.com/container-storage-interface/spec - ContainerCSIVolumeIDKey = attribute.Key("container.csi.volume.id") - - // ContainerIDKey is the attribute Key conforming to the "container.id" semantic - // conventions. It represents the container ID. Usually a UUID, as for example - // used to [identify Docker containers]. The UUID might be abbreviated. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "a3bf90e006b2" - // - // [identify Docker containers]: https://docs.docker.com/engine/containers/run/#container-identification - ContainerIDKey = attribute.Key("container.id") - - // ContainerImageIDKey is the attribute Key conforming to the - // "container.image.id" semantic conventions. It represents the runtime specific - // image identifier. Usually a hash algorithm followed by a UUID. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "sha256:19c92d0a00d1b66d897bceaa7319bee0dd38a10a851c60bcec9474aa3f01e50f" - // Note: Docker defines a sha256 of the image id; `container.image.id` - // corresponds to the `Image` field from the Docker container inspect [API] - // endpoint. - // K8s defines a link to the container registry repository with digest - // `"imageID": "registry.azurecr.io /namespace/service/dockerfile@sha256:bdeabd40c3a8a492eaf9e8e44d0ebbb84bac7ee25ac0cf8a7159d25f62555625"` - // . - // The ID is assigned by the container runtime and can vary in different - // environments. Consider using `oci.manifest.digest` if it is important to - // identify the same image in different environments/runtimes. - // - // [API]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Container/operation/ContainerInspect - ContainerImageIDKey = attribute.Key("container.image.id") - - // ContainerImageNameKey is the attribute Key conforming to the - // "container.image.name" semantic conventions. It represents the name of the - // image the container was built on. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "gcr.io/opentelemetry/operator" - ContainerImageNameKey = attribute.Key("container.image.name") - - // ContainerImageRepoDigestsKey is the attribute Key conforming to the - // "container.image.repo_digests" semantic conventions. It represents the repo - // digests of the container image as provided by the container runtime. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: - // "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", - // "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578" - // Note: [Docker] and [CRI] report those under the `RepoDigests` field. - // - // [Docker]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect - // [CRI]: https://github.com/kubernetes/cri-api/blob/c75ef5b473bbe2d0a4fc92f82235efd665ea8e9f/pkg/apis/runtime/v1/api.proto#L1237-L1238 - ContainerImageRepoDigestsKey = attribute.Key("container.image.repo_digests") - - // ContainerImageTagsKey is the attribute Key conforming to the - // "container.image.tags" semantic conventions. It represents the container - // image tags. An example can be found in [Docker Image Inspect]. Should be only - // the `` section of the full name for example from - // `registry.example.com/my-org/my-image:`. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "v1.27.1", "3.5.7-0" - // - // [Docker Image Inspect]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect - ContainerImageTagsKey = attribute.Key("container.image.tags") - - // ContainerNameKey is the attribute Key conforming to the "container.name" - // semantic conventions. It represents the container name used by container - // runtime. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry-autoconf" - ContainerNameKey = attribute.Key("container.name") - - // ContainerRuntimeDescriptionKey is the attribute Key conforming to the - // "container.runtime.description" semantic conventions. It represents a - // description about the runtime which could include, for example details about - // the CRI/API version being used or other customisations. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "docker://19.3.1 - CRI: 1.22.0" - ContainerRuntimeDescriptionKey = attribute.Key("container.runtime.description") - - // ContainerRuntimeNameKey is the attribute Key conforming to the - // "container.runtime.name" semantic conventions. It represents the container - // runtime managing this container. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "docker", "containerd", "rkt" - ContainerRuntimeNameKey = attribute.Key("container.runtime.name") - - // ContainerRuntimeVersionKey is the attribute Key conforming to the - // "container.runtime.version" semantic conventions. It represents the version - // of the runtime of this process, as returned by the runtime without - // modification. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0.0 - ContainerRuntimeVersionKey = attribute.Key("container.runtime.version") -) - -// ContainerCommand returns an attribute KeyValue conforming to the -// "container.command" semantic conventions. It represents the command used to -// run the container (i.e. the command name). -func ContainerCommand(val string) attribute.KeyValue { - return ContainerCommandKey.String(val) -} - -// ContainerCommandArgs returns an attribute KeyValue conforming to the -// "container.command_args" semantic conventions. It represents the all the -// command arguments (including the command/executable itself) run by the -// container. -func ContainerCommandArgs(val ...string) attribute.KeyValue { - return ContainerCommandArgsKey.StringSlice(val) -} - -// ContainerCommandLine returns an attribute KeyValue conforming to the -// "container.command_line" semantic conventions. It represents the full command -// run by the container as a single string representing the full command. -func ContainerCommandLine(val string) attribute.KeyValue { - return ContainerCommandLineKey.String(val) -} - -// ContainerCSIPluginName returns an attribute KeyValue conforming to the -// "container.csi.plugin.name" semantic conventions. It represents the name of -// the CSI ([Container Storage Interface]) plugin used by the volume. -// -// [Container Storage Interface]: https://github.com/container-storage-interface/spec -func ContainerCSIPluginName(val string) attribute.KeyValue { - return ContainerCSIPluginNameKey.String(val) -} - -// ContainerCSIVolumeID returns an attribute KeyValue conforming to the -// "container.csi.volume.id" semantic conventions. It represents the unique -// volume ID returned by the CSI ([Container Storage Interface]) plugin. -// -// [Container Storage Interface]: https://github.com/container-storage-interface/spec -func ContainerCSIVolumeID(val string) attribute.KeyValue { - return ContainerCSIVolumeIDKey.String(val) -} - -// ContainerID returns an attribute KeyValue conforming to the "container.id" -// semantic conventions. It represents the container ID. Usually a UUID, as for -// example used to [identify Docker containers]. The UUID might be abbreviated. -// -// [identify Docker containers]: https://docs.docker.com/engine/containers/run/#container-identification -func ContainerID(val string) attribute.KeyValue { - return ContainerIDKey.String(val) -} - -// ContainerImageID returns an attribute KeyValue conforming to the -// "container.image.id" semantic conventions. It represents the runtime specific -// image identifier. Usually a hash algorithm followed by a UUID. -func ContainerImageID(val string) attribute.KeyValue { - return ContainerImageIDKey.String(val) -} - -// ContainerImageName returns an attribute KeyValue conforming to the -// "container.image.name" semantic conventions. It represents the name of the -// image the container was built on. -func ContainerImageName(val string) attribute.KeyValue { - return ContainerImageNameKey.String(val) -} - -// ContainerImageRepoDigests returns an attribute KeyValue conforming to the -// "container.image.repo_digests" semantic conventions. It represents the repo -// digests of the container image as provided by the container runtime. -func ContainerImageRepoDigests(val ...string) attribute.KeyValue { - return ContainerImageRepoDigestsKey.StringSlice(val) -} - -// ContainerImageTags returns an attribute KeyValue conforming to the -// "container.image.tags" semantic conventions. It represents the container image -// tags. An example can be found in [Docker Image Inspect]. Should be only the -// `` section of the full name for example from -// `registry.example.com/my-org/my-image:`. -// -// [Docker Image Inspect]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect -func ContainerImageTags(val ...string) attribute.KeyValue { - return ContainerImageTagsKey.StringSlice(val) -} - -// ContainerLabel returns an attribute KeyValue conforming to the -// "container.label" semantic conventions. It represents the container labels, -// `` being the label name, the value being the label value. -func ContainerLabel(key string, val string) attribute.KeyValue { - return attribute.String("container.label."+key, val) -} - -// ContainerName returns an attribute KeyValue conforming to the "container.name" -// semantic conventions. It represents the container name used by container -// runtime. -func ContainerName(val string) attribute.KeyValue { - return ContainerNameKey.String(val) -} - -// ContainerRuntimeDescription returns an attribute KeyValue conforming to the -// "container.runtime.description" semantic conventions. It represents a -// description about the runtime which could include, for example details about -// the CRI/API version being used or other customisations. -func ContainerRuntimeDescription(val string) attribute.KeyValue { - return ContainerRuntimeDescriptionKey.String(val) -} - -// ContainerRuntimeName returns an attribute KeyValue conforming to the -// "container.runtime.name" semantic conventions. It represents the container -// runtime managing this container. -func ContainerRuntimeName(val string) attribute.KeyValue { - return ContainerRuntimeNameKey.String(val) -} - -// ContainerRuntimeVersion returns an attribute KeyValue conforming to the -// "container.runtime.version" semantic conventions. It represents the version of -// the runtime of this process, as returned by the runtime without modification. -func ContainerRuntimeVersion(val string) attribute.KeyValue { - return ContainerRuntimeVersionKey.String(val) -} - -// Namespace: cpu -const ( - // CPULogicalNumberKey is the attribute Key conforming to the - // "cpu.logical_number" semantic conventions. It represents the logical CPU - // number [0..n-1]. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1 - CPULogicalNumberKey = attribute.Key("cpu.logical_number") - - // CPUModeKey is the attribute Key conforming to the "cpu.mode" semantic - // conventions. It represents the mode of the CPU. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "user", "system" - CPUModeKey = attribute.Key("cpu.mode") -) - -// CPULogicalNumber returns an attribute KeyValue conforming to the -// "cpu.logical_number" semantic conventions. It represents the logical CPU -// number [0..n-1]. -func CPULogicalNumber(val int) attribute.KeyValue { - return CPULogicalNumberKey.Int(val) -} - -// Enum values for cpu.mode -var ( - // User - // Stability: development - CPUModeUser = CPUModeKey.String("user") - // System - // Stability: development - CPUModeSystem = CPUModeKey.String("system") - // Nice - // Stability: development - CPUModeNice = CPUModeKey.String("nice") - // Idle - // Stability: development - CPUModeIdle = CPUModeKey.String("idle") - // IO Wait - // Stability: development - CPUModeIOWait = CPUModeKey.String("iowait") - // Interrupt - // Stability: development - CPUModeInterrupt = CPUModeKey.String("interrupt") - // Steal - // Stability: development - CPUModeSteal = CPUModeKey.String("steal") - // Kernel - // Stability: development - CPUModeKernel = CPUModeKey.String("kernel") -) - -// Namespace: db -const ( - // DBClientConnectionPoolNameKey is the attribute Key conforming to the - // "db.client.connection.pool.name" semantic conventions. It represents the name - // of the connection pool; unique within the instrumented application. In case - // the connection pool implementation doesn't provide a name, instrumentation - // SHOULD use a combination of parameters that would make the name unique, for - // example, combining attributes `server.address`, `server.port`, and - // `db.namespace`, formatted as `server.address:server.port/db.namespace`. - // Instrumentations that generate connection pool name following different - // patterns SHOULD document it. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "myDataSource" - DBClientConnectionPoolNameKey = attribute.Key("db.client.connection.pool.name") - - // DBClientConnectionStateKey is the attribute Key conforming to the - // "db.client.connection.state" semantic conventions. It represents the state of - // a connection in the pool. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "idle" - DBClientConnectionStateKey = attribute.Key("db.client.connection.state") - - // DBCollectionNameKey is the attribute Key conforming to the - // "db.collection.name" semantic conventions. It represents the name of a - // collection (table, container) within the database. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "public.users", "customers" - // Note: It is RECOMMENDED to capture the value as provided by the application - // without attempting to do any case normalization. - // - // The collection name SHOULD NOT be extracted from `db.query.text`, - // when the database system supports query text with multiple collections - // in non-batch operations. - // - // For batch operations, if the individual operations are known to have the same - // collection name then that collection name SHOULD be used. - DBCollectionNameKey = attribute.Key("db.collection.name") - - // DBNamespaceKey is the attribute Key conforming to the "db.namespace" semantic - // conventions. It represents the name of the database, fully qualified within - // the server address and port. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "customers", "test.users" - // Note: If a database system has multiple namespace components, they SHOULD be - // concatenated from the most general to the most specific namespace component, - // using `|` as a separator between the components. Any missing components (and - // their associated separators) SHOULD be omitted. - // Semantic conventions for individual database systems SHOULD document what - // `db.namespace` means in the context of that system. - // It is RECOMMENDED to capture the value as provided by the application without - // attempting to do any case normalization. - DBNamespaceKey = attribute.Key("db.namespace") - - // DBOperationBatchSizeKey is the attribute Key conforming to the - // "db.operation.batch.size" semantic conventions. It represents the number of - // queries included in a batch operation. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 2, 3, 4 - // Note: Operations are only considered batches when they contain two or more - // operations, and so `db.operation.batch.size` SHOULD never be `1`. - DBOperationBatchSizeKey = attribute.Key("db.operation.batch.size") - - // DBOperationNameKey is the attribute Key conforming to the "db.operation.name" - // semantic conventions. It represents the name of the operation or command - // being executed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "findAndModify", "HMSET", "SELECT" - // Note: It is RECOMMENDED to capture the value as provided by the application - // without attempting to do any case normalization. - // - // The operation name SHOULD NOT be extracted from `db.query.text`, - // when the database system supports query text with multiple operations - // in non-batch operations. - // - // If spaces can occur in the operation name, multiple consecutive spaces - // SHOULD be normalized to a single space. - // - // For batch operations, if the individual operations are known to have the same - // operation name - // then that operation name SHOULD be used prepended by `BATCH `, - // otherwise `db.operation.name` SHOULD be `BATCH` or some other database - // system specific term if more applicable. - DBOperationNameKey = attribute.Key("db.operation.name") - - // DBQuerySummaryKey is the attribute Key conforming to the "db.query.summary" - // semantic conventions. It represents the low cardinality summary of a database - // query. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "SELECT wuser_table", "INSERT shipping_details SELECT orders", "get - // user by id" - // Note: The query summary describes a class of database queries and is useful - // as a grouping key, especially when analyzing telemetry for database - // calls involving complex queries. - // - // Summary may be available to the instrumentation through - // instrumentation hooks or other means. If it is not available, - // instrumentations - // that support query parsing SHOULD generate a summary following - // [Generating query summary] - // section. - // - // [Generating query summary]: /docs/db/database-spans.md#generating-a-summary-of-the-query - DBQuerySummaryKey = attribute.Key("db.query.summary") - - // DBQueryTextKey is the attribute Key conforming to the "db.query.text" - // semantic conventions. It represents the database query being executed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "SELECT * FROM wuser_table where username = ?", "SET mykey ?" - // Note: For sanitization see [Sanitization of `db.query.text`]. - // For batch operations, if the individual operations are known to have the same - // query text then that query text SHOULD be used, otherwise all of the - // individual query texts SHOULD be concatenated with separator `; ` or some - // other database system specific separator if more applicable. - // Parameterized query text SHOULD NOT be sanitized. Even though parameterized - // query text can potentially have sensitive data, by using a parameterized - // query the user is giving a strong signal that any sensitive data will be - // passed as parameter values, and the benefit to observability of capturing the - // static part of the query text by default outweighs the risk. - // - // [Sanitization of `db.query.text`]: /docs/db/database-spans.md#sanitization-of-dbquerytext - DBQueryTextKey = attribute.Key("db.query.text") - - // DBResponseReturnedRowsKey is the attribute Key conforming to the - // "db.response.returned_rows" semantic conventions. It represents the number of - // rows returned by the operation. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 10, 30, 1000 - DBResponseReturnedRowsKey = attribute.Key("db.response.returned_rows") - - // DBResponseStatusCodeKey is the attribute Key conforming to the - // "db.response.status_code" semantic conventions. It represents the database - // response status code. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "102", "ORA-17002", "08P01", "404" - // Note: The status code returned by the database. Usually it represents an - // error code, but may also represent partial success, warning, or differentiate - // between various types of successful outcomes. - // Semantic conventions for individual database systems SHOULD document what - // `db.response.status_code` means in the context of that system. - DBResponseStatusCodeKey = attribute.Key("db.response.status_code") - - // DBStoredProcedureNameKey is the attribute Key conforming to the - // "db.stored_procedure.name" semantic conventions. It represents the name of a - // stored procedure within the database. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "GetCustomer" - // Note: It is RECOMMENDED to capture the value as provided by the application - // without attempting to do any case normalization. - // - // For batch operations, if the individual operations are known to have the same - // stored procedure name then that stored procedure name SHOULD be used. - DBStoredProcedureNameKey = attribute.Key("db.stored_procedure.name") - - // DBSystemNameKey is the attribute Key conforming to the "db.system.name" - // semantic conventions. It represents the database management system (DBMS) - // product as identified by the client instrumentation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: - // Note: The actual DBMS may differ from the one identified by the client. For - // example, when using PostgreSQL client libraries to connect to a CockroachDB, - // the `db.system.name` is set to `postgresql` based on the instrumentation's - // best knowledge. - DBSystemNameKey = attribute.Key("db.system.name") -) - -// DBClientConnectionPoolName returns an attribute KeyValue conforming to the -// "db.client.connection.pool.name" semantic conventions. It represents the name -// of the connection pool; unique within the instrumented application. In case -// the connection pool implementation doesn't provide a name, instrumentation -// SHOULD use a combination of parameters that would make the name unique, for -// example, combining attributes `server.address`, `server.port`, and -// `db.namespace`, formatted as `server.address:server.port/db.namespace`. -// Instrumentations that generate connection pool name following different -// patterns SHOULD document it. -func DBClientConnectionPoolName(val string) attribute.KeyValue { - return DBClientConnectionPoolNameKey.String(val) -} - -// DBCollectionName returns an attribute KeyValue conforming to the -// "db.collection.name" semantic conventions. It represents the name of a -// collection (table, container) within the database. -func DBCollectionName(val string) attribute.KeyValue { - return DBCollectionNameKey.String(val) -} - -// DBNamespace returns an attribute KeyValue conforming to the "db.namespace" -// semantic conventions. It represents the name of the database, fully qualified -// within the server address and port. -func DBNamespace(val string) attribute.KeyValue { - return DBNamespaceKey.String(val) -} - -// DBOperationBatchSize returns an attribute KeyValue conforming to the -// "db.operation.batch.size" semantic conventions. It represents the number of -// queries included in a batch operation. -func DBOperationBatchSize(val int) attribute.KeyValue { - return DBOperationBatchSizeKey.Int(val) -} - -// DBOperationName returns an attribute KeyValue conforming to the -// "db.operation.name" semantic conventions. It represents the name of the -// operation or command being executed. -func DBOperationName(val string) attribute.KeyValue { - return DBOperationNameKey.String(val) -} - -// DBOperationParameter returns an attribute KeyValue conforming to the -// "db.operation.parameter" semantic conventions. It represents a database -// operation parameter, with `` being the parameter name, and the attribute -// value being a string representation of the parameter value. -func DBOperationParameter(key string, val string) attribute.KeyValue { - return attribute.String("db.operation.parameter."+key, val) -} - -// DBQueryParameter returns an attribute KeyValue conforming to the -// "db.query.parameter" semantic conventions. It represents a database query -// parameter, with `` being the parameter name, and the attribute value -// being a string representation of the parameter value. -func DBQueryParameter(key string, val string) attribute.KeyValue { - return attribute.String("db.query.parameter."+key, val) -} - -// DBQuerySummary returns an attribute KeyValue conforming to the -// "db.query.summary" semantic conventions. It represents the low cardinality -// summary of a database query. -func DBQuerySummary(val string) attribute.KeyValue { - return DBQuerySummaryKey.String(val) -} - -// DBQueryText returns an attribute KeyValue conforming to the "db.query.text" -// semantic conventions. It represents the database query being executed. -func DBQueryText(val string) attribute.KeyValue { - return DBQueryTextKey.String(val) -} - -// DBResponseReturnedRows returns an attribute KeyValue conforming to the -// "db.response.returned_rows" semantic conventions. It represents the number of -// rows returned by the operation. -func DBResponseReturnedRows(val int) attribute.KeyValue { - return DBResponseReturnedRowsKey.Int(val) -} - -// DBResponseStatusCode returns an attribute KeyValue conforming to the -// "db.response.status_code" semantic conventions. It represents the database -// response status code. -func DBResponseStatusCode(val string) attribute.KeyValue { - return DBResponseStatusCodeKey.String(val) -} - -// DBStoredProcedureName returns an attribute KeyValue conforming to the -// "db.stored_procedure.name" semantic conventions. It represents the name of a -// stored procedure within the database. -func DBStoredProcedureName(val string) attribute.KeyValue { - return DBStoredProcedureNameKey.String(val) -} - -// Enum values for db.client.connection.state -var ( - // idle - // Stability: development - DBClientConnectionStateIdle = DBClientConnectionStateKey.String("idle") - // used - // Stability: development - DBClientConnectionStateUsed = DBClientConnectionStateKey.String("used") -) - -// Enum values for db.system.name -var ( - // Some other SQL database. Fallback only. - // Stability: development - DBSystemNameOtherSQL = DBSystemNameKey.String("other_sql") - // [Adabas (Adaptable Database System)] - // Stability: development - // - // [Adabas (Adaptable Database System)]: https://documentation.softwareag.com/?pf=adabas - DBSystemNameSoftwareagAdabas = DBSystemNameKey.String("softwareag.adabas") - // [Actian Ingres] - // Stability: development - // - // [Actian Ingres]: https://www.actian.com/databases/ingres/ - DBSystemNameActianIngres = DBSystemNameKey.String("actian.ingres") - // [Amazon DynamoDB] - // Stability: development - // - // [Amazon DynamoDB]: https://aws.amazon.com/pm/dynamodb/ - DBSystemNameAWSDynamoDB = DBSystemNameKey.String("aws.dynamodb") - // [Amazon Redshift] - // Stability: development - // - // [Amazon Redshift]: https://aws.amazon.com/redshift/ - DBSystemNameAWSRedshift = DBSystemNameKey.String("aws.redshift") - // [Azure Cosmos DB] - // Stability: development - // - // [Azure Cosmos DB]: https://learn.microsoft.com/azure/cosmos-db - DBSystemNameAzureCosmosDB = DBSystemNameKey.String("azure.cosmosdb") - // [InterSystems Caché] - // Stability: development - // - // [InterSystems Caché]: https://www.intersystems.com/products/cache/ - DBSystemNameIntersystemsCache = DBSystemNameKey.String("intersystems.cache") - // [Apache Cassandra] - // Stability: development - // - // [Apache Cassandra]: https://cassandra.apache.org/ - DBSystemNameCassandra = DBSystemNameKey.String("cassandra") - // [ClickHouse] - // Stability: development - // - // [ClickHouse]: https://clickhouse.com/ - DBSystemNameClickHouse = DBSystemNameKey.String("clickhouse") - // [CockroachDB] - // Stability: development - // - // [CockroachDB]: https://www.cockroachlabs.com/ - DBSystemNameCockroachDB = DBSystemNameKey.String("cockroachdb") - // [Couchbase] - // Stability: development - // - // [Couchbase]: https://www.couchbase.com/ - DBSystemNameCouchbase = DBSystemNameKey.String("couchbase") - // [Apache CouchDB] - // Stability: development - // - // [Apache CouchDB]: https://couchdb.apache.org/ - DBSystemNameCouchDB = DBSystemNameKey.String("couchdb") - // [Apache Derby] - // Stability: development - // - // [Apache Derby]: https://db.apache.org/derby/ - DBSystemNameDerby = DBSystemNameKey.String("derby") - // [Elasticsearch] - // Stability: development - // - // [Elasticsearch]: https://www.elastic.co/elasticsearch - DBSystemNameElasticsearch = DBSystemNameKey.String("elasticsearch") - // [Firebird] - // Stability: development - // - // [Firebird]: https://www.firebirdsql.org/ - DBSystemNameFirebirdSQL = DBSystemNameKey.String("firebirdsql") - // [Google Cloud Spanner] - // Stability: development - // - // [Google Cloud Spanner]: https://cloud.google.com/spanner - DBSystemNameGCPSpanner = DBSystemNameKey.String("gcp.spanner") - // [Apache Geode] - // Stability: development - // - // [Apache Geode]: https://geode.apache.org/ - DBSystemNameGeode = DBSystemNameKey.String("geode") - // [H2 Database] - // Stability: development - // - // [H2 Database]: https://h2database.com/ - DBSystemNameH2database = DBSystemNameKey.String("h2database") - // [Apache HBase] - // Stability: development - // - // [Apache HBase]: https://hbase.apache.org/ - DBSystemNameHBase = DBSystemNameKey.String("hbase") - // [Apache Hive] - // Stability: development - // - // [Apache Hive]: https://hive.apache.org/ - DBSystemNameHive = DBSystemNameKey.String("hive") - // [HyperSQL Database] - // Stability: development - // - // [HyperSQL Database]: https://hsqldb.org/ - DBSystemNameHSQLDB = DBSystemNameKey.String("hsqldb") - // [IBM Db2] - // Stability: development - // - // [IBM Db2]: https://www.ibm.com/db2 - DBSystemNameIBMDB2 = DBSystemNameKey.String("ibm.db2") - // [IBM Informix] - // Stability: development - // - // [IBM Informix]: https://www.ibm.com/products/informix - DBSystemNameIBMInformix = DBSystemNameKey.String("ibm.informix") - // [IBM Netezza] - // Stability: development - // - // [IBM Netezza]: https://www.ibm.com/products/netezza - DBSystemNameIBMNetezza = DBSystemNameKey.String("ibm.netezza") - // [InfluxDB] - // Stability: development - // - // [InfluxDB]: https://www.influxdata.com/ - DBSystemNameInfluxDB = DBSystemNameKey.String("influxdb") - // [Instant] - // Stability: development - // - // [Instant]: https://www.instantdb.com/ - DBSystemNameInstantDB = DBSystemNameKey.String("instantdb") - // [MariaDB] - // Stability: stable - // - // [MariaDB]: https://mariadb.org/ - DBSystemNameMariaDB = DBSystemNameKey.String("mariadb") - // [Memcached] - // Stability: development - // - // [Memcached]: https://memcached.org/ - DBSystemNameMemcached = DBSystemNameKey.String("memcached") - // [MongoDB] - // Stability: development - // - // [MongoDB]: https://www.mongodb.com/ - DBSystemNameMongoDB = DBSystemNameKey.String("mongodb") - // [Microsoft SQL Server] - // Stability: stable - // - // [Microsoft SQL Server]: https://www.microsoft.com/sql-server - DBSystemNameMicrosoftSQLServer = DBSystemNameKey.String("microsoft.sql_server") - // [MySQL] - // Stability: stable - // - // [MySQL]: https://www.mysql.com/ - DBSystemNameMySQL = DBSystemNameKey.String("mysql") - // [Neo4j] - // Stability: development - // - // [Neo4j]: https://neo4j.com/ - DBSystemNameNeo4j = DBSystemNameKey.String("neo4j") - // [OpenSearch] - // Stability: development - // - // [OpenSearch]: https://opensearch.org/ - DBSystemNameOpenSearch = DBSystemNameKey.String("opensearch") - // [Oracle Database] - // Stability: development - // - // [Oracle Database]: https://www.oracle.com/database/ - DBSystemNameOracleDB = DBSystemNameKey.String("oracle.db") - // [PostgreSQL] - // Stability: stable - // - // [PostgreSQL]: https://www.postgresql.org/ - DBSystemNamePostgreSQL = DBSystemNameKey.String("postgresql") - // [Redis] - // Stability: development - // - // [Redis]: https://redis.io/ - DBSystemNameRedis = DBSystemNameKey.String("redis") - // [SAP HANA] - // Stability: development - // - // [SAP HANA]: https://www.sap.com/products/technology-platform/hana/what-is-sap-hana.html - DBSystemNameSAPHANA = DBSystemNameKey.String("sap.hana") - // [SAP MaxDB] - // Stability: development - // - // [SAP MaxDB]: https://maxdb.sap.com/ - DBSystemNameSAPMaxDB = DBSystemNameKey.String("sap.maxdb") - // [SQLite] - // Stability: development - // - // [SQLite]: https://www.sqlite.org/ - DBSystemNameSQLite = DBSystemNameKey.String("sqlite") - // [Teradata] - // Stability: development - // - // [Teradata]: https://www.teradata.com/ - DBSystemNameTeradata = DBSystemNameKey.String("teradata") - // [Trino] - // Stability: development - // - // [Trino]: https://trino.io/ - DBSystemNameTrino = DBSystemNameKey.String("trino") -) - -// Namespace: deployment -const ( - // DeploymentEnvironmentNameKey is the attribute Key conforming to the - // "deployment.environment.name" semantic conventions. It represents the name of - // the [deployment environment] (aka deployment tier). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "staging", "production" - // Note: `deployment.environment.name` does not affect the uniqueness - // constraints defined through - // the `service.namespace`, `service.name` and `service.instance.id` resource - // attributes. - // This implies that resources carrying the following attribute combinations - // MUST be - // considered to be identifying the same service: - // - // - `service.name=frontend`, `deployment.environment.name=production` - // - `service.name=frontend`, `deployment.environment.name=staging`. - // - // - // [deployment environment]: https://wikipedia.org/wiki/Deployment_environment - DeploymentEnvironmentNameKey = attribute.Key("deployment.environment.name") - - // DeploymentIDKey is the attribute Key conforming to the "deployment.id" - // semantic conventions. It represents the id of the deployment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1208" - DeploymentIDKey = attribute.Key("deployment.id") - - // DeploymentNameKey is the attribute Key conforming to the "deployment.name" - // semantic conventions. It represents the name of the deployment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "deploy my app", "deploy-frontend" - DeploymentNameKey = attribute.Key("deployment.name") - - // DeploymentStatusKey is the attribute Key conforming to the - // "deployment.status" semantic conventions. It represents the status of the - // deployment. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - DeploymentStatusKey = attribute.Key("deployment.status") -) - -// DeploymentEnvironmentName returns an attribute KeyValue conforming to the -// "deployment.environment.name" semantic conventions. It represents the name of -// the [deployment environment] (aka deployment tier). -// -// [deployment environment]: https://wikipedia.org/wiki/Deployment_environment -func DeploymentEnvironmentName(val string) attribute.KeyValue { - return DeploymentEnvironmentNameKey.String(val) -} - -// DeploymentID returns an attribute KeyValue conforming to the "deployment.id" -// semantic conventions. It represents the id of the deployment. -func DeploymentID(val string) attribute.KeyValue { - return DeploymentIDKey.String(val) -} - -// DeploymentName returns an attribute KeyValue conforming to the -// "deployment.name" semantic conventions. It represents the name of the -// deployment. -func DeploymentName(val string) attribute.KeyValue { - return DeploymentNameKey.String(val) -} - -// Enum values for deployment.status -var ( - // failed - // Stability: development - DeploymentStatusFailed = DeploymentStatusKey.String("failed") - // succeeded - // Stability: development - DeploymentStatusSucceeded = DeploymentStatusKey.String("succeeded") -) - -// Namespace: destination -const ( - // DestinationAddressKey is the attribute Key conforming to the - // "destination.address" semantic conventions. It represents the destination - // address - domain name if available without reverse DNS lookup; otherwise, IP - // address or Unix domain socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "destination.example.com", "10.1.2.80", "/tmp/my.sock" - // Note: When observed from the source side, and when communicating through an - // intermediary, `destination.address` SHOULD represent the destination address - // behind any intermediaries, for example proxies, if it's available. - DestinationAddressKey = attribute.Key("destination.address") - - // DestinationPortKey is the attribute Key conforming to the "destination.port" - // semantic conventions. It represents the destination port number. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 3389, 2888 - DestinationPortKey = attribute.Key("destination.port") -) - -// DestinationAddress returns an attribute KeyValue conforming to the -// "destination.address" semantic conventions. It represents the destination -// address - domain name if available without reverse DNS lookup; otherwise, IP -// address or Unix domain socket name. -func DestinationAddress(val string) attribute.KeyValue { - return DestinationAddressKey.String(val) -} - -// DestinationPort returns an attribute KeyValue conforming to the -// "destination.port" semantic conventions. It represents the destination port -// number. -func DestinationPort(val int) attribute.KeyValue { - return DestinationPortKey.Int(val) -} - -// Namespace: device -const ( - // DeviceIDKey is the attribute Key conforming to the "device.id" semantic - // conventions. It represents a unique identifier representing the device. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "123456789012345", "01:23:45:67:89:AB" - // Note: Its value SHOULD be identical for all apps on a device and it SHOULD - // NOT change if an app is uninstalled and re-installed. - // However, it might be resettable by the user for all apps on a device. - // Hardware IDs (e.g. vendor-specific serial number, IMEI or MAC address) MAY be - // used as values. - // - // More information about Android identifier best practices can be found in the - // [Android user data IDs guide]. - // - // > [!WARNING]> This attribute may contain sensitive (PII) information. Caution - // > should be taken when storing personal data or anything which can identify a - // > user. GDPR and data protection laws may apply, - // > ensure you do your own due diligence.> Due to these reasons, this - // > identifier is not recommended for consumer applications and will likely - // > result in rejection from both Google Play and App Store. - // > However, it may be appropriate for specific enterprise scenarios, such as - // > kiosk devices or enterprise-managed devices, with appropriate compliance - // > clearance. - // > Any instrumentation providing this identifier MUST implement it as an - // > opt-in feature.> See [`app.installation.id`]> for a more - // > privacy-preserving alternative. - // - // [Android user data IDs guide]: https://developer.android.com/training/articles/user-data-ids - // [`app.installation.id`]: /docs/registry/attributes/app.md#app-installation-id - DeviceIDKey = attribute.Key("device.id") - - // DeviceManufacturerKey is the attribute Key conforming to the - // "device.manufacturer" semantic conventions. It represents the name of the - // device manufacturer. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Apple", "Samsung" - // Note: The Android OS provides this field via [Build]. iOS apps SHOULD - // hardcode the value `Apple`. - // - // [Build]: https://developer.android.com/reference/android/os/Build#MANUFACTURER - DeviceManufacturerKey = attribute.Key("device.manufacturer") - - // DeviceModelIdentifierKey is the attribute Key conforming to the - // "device.model.identifier" semantic conventions. It represents the model - // identifier for the device. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "iPhone3,4", "SM-G920F" - // Note: It's recommended this value represents a machine-readable version of - // the model identifier rather than the market or consumer-friendly name of the - // device. - DeviceModelIdentifierKey = attribute.Key("device.model.identifier") - - // DeviceModelNameKey is the attribute Key conforming to the "device.model.name" - // semantic conventions. It represents the marketing name for the device model. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "iPhone 6s Plus", "Samsung Galaxy S6" - // Note: It's recommended this value represents a human-readable version of the - // device model rather than a machine-readable alternative. - DeviceModelNameKey = attribute.Key("device.model.name") -) - -// DeviceID returns an attribute KeyValue conforming to the "device.id" semantic -// conventions. It represents a unique identifier representing the device. -func DeviceID(val string) attribute.KeyValue { - return DeviceIDKey.String(val) -} - -// DeviceManufacturer returns an attribute KeyValue conforming to the -// "device.manufacturer" semantic conventions. It represents the name of the -// device manufacturer. -func DeviceManufacturer(val string) attribute.KeyValue { - return DeviceManufacturerKey.String(val) -} - -// DeviceModelIdentifier returns an attribute KeyValue conforming to the -// "device.model.identifier" semantic conventions. It represents the model -// identifier for the device. -func DeviceModelIdentifier(val string) attribute.KeyValue { - return DeviceModelIdentifierKey.String(val) -} - -// DeviceModelName returns an attribute KeyValue conforming to the -// "device.model.name" semantic conventions. It represents the marketing name for -// the device model. -func DeviceModelName(val string) attribute.KeyValue { - return DeviceModelNameKey.String(val) -} - -// Namespace: disk -const ( - // DiskIODirectionKey is the attribute Key conforming to the "disk.io.direction" - // semantic conventions. It represents the disk IO operation direction. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "read" - DiskIODirectionKey = attribute.Key("disk.io.direction") -) - -// Enum values for disk.io.direction -var ( - // read - // Stability: development - DiskIODirectionRead = DiskIODirectionKey.String("read") - // write - // Stability: development - DiskIODirectionWrite = DiskIODirectionKey.String("write") -) - -// Namespace: dns -const ( - // DNSAnswersKey is the attribute Key conforming to the "dns.answers" semantic - // conventions. It represents the list of IPv4 or IPv6 addresses resolved during - // DNS lookup. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "10.0.0.1", "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - DNSAnswersKey = attribute.Key("dns.answers") - - // DNSQuestionNameKey is the attribute Key conforming to the "dns.question.name" - // semantic conventions. It represents the name being queried. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "www.example.com", "opentelemetry.io" - // Note: The name represents the queried domain name as it appears in the DNS - // query without any additional normalization. - DNSQuestionNameKey = attribute.Key("dns.question.name") -) - -// DNSAnswers returns an attribute KeyValue conforming to the "dns.answers" -// semantic conventions. It represents the list of IPv4 or IPv6 addresses -// resolved during DNS lookup. -func DNSAnswers(val ...string) attribute.KeyValue { - return DNSAnswersKey.StringSlice(val) -} - -// DNSQuestionName returns an attribute KeyValue conforming to the -// "dns.question.name" semantic conventions. It represents the name being -// queried. -func DNSQuestionName(val string) attribute.KeyValue { - return DNSQuestionNameKey.String(val) -} - -// Namespace: elasticsearch -const ( - // ElasticsearchNodeNameKey is the attribute Key conforming to the - // "elasticsearch.node.name" semantic conventions. It represents the represents - // the human-readable identifier of the node/instance to which a request was - // routed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "instance-0000000001" - ElasticsearchNodeNameKey = attribute.Key("elasticsearch.node.name") -) - -// ElasticsearchNodeName returns an attribute KeyValue conforming to the -// "elasticsearch.node.name" semantic conventions. It represents the represents -// the human-readable identifier of the node/instance to which a request was -// routed. -func ElasticsearchNodeName(val string) attribute.KeyValue { - return ElasticsearchNodeNameKey.String(val) -} - -// Namespace: enduser -const ( - // EnduserIDKey is the attribute Key conforming to the "enduser.id" semantic - // conventions. It represents the unique identifier of an end user in the - // system. It maybe a username, email address, or other identifier. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "username" - // Note: Unique identifier of an end user in the system. - // - // > [!Warning] - // > This field contains sensitive (PII) information. - EnduserIDKey = attribute.Key("enduser.id") - - // EnduserPseudoIDKey is the attribute Key conforming to the "enduser.pseudo.id" - // semantic conventions. It represents the pseudonymous identifier of an end - // user. This identifier should be a random value that is not directly linked or - // associated with the end user's actual identity. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "QdH5CAWJgqVT4rOr0qtumf" - // Note: Pseudonymous identifier of an end user. - // - // > [!Warning] - // > This field contains sensitive (linkable PII) information. - EnduserPseudoIDKey = attribute.Key("enduser.pseudo.id") -) - -// EnduserID returns an attribute KeyValue conforming to the "enduser.id" -// semantic conventions. It represents the unique identifier of an end user in -// the system. It maybe a username, email address, or other identifier. -func EnduserID(val string) attribute.KeyValue { - return EnduserIDKey.String(val) -} - -// EnduserPseudoID returns an attribute KeyValue conforming to the -// "enduser.pseudo.id" semantic conventions. It represents the pseudonymous -// identifier of an end user. This identifier should be a random value that is -// not directly linked or associated with the end user's actual identity. -func EnduserPseudoID(val string) attribute.KeyValue { - return EnduserPseudoIDKey.String(val) -} - -// Namespace: error -const ( - // ErrorMessageKey is the attribute Key conforming to the "error.message" - // semantic conventions. It represents a message providing more detail about an - // error in human-readable form. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Unexpected input type: string", "The user has exceeded their - // storage quota" - // Note: `error.message` should provide additional context and detail about an - // error. - // It is NOT RECOMMENDED to duplicate the value of `error.type` in - // `error.message`. - // It is also NOT RECOMMENDED to duplicate the value of `exception.message` in - // `error.message`. - // - // `error.message` is NOT RECOMMENDED for metrics or spans due to its unbounded - // cardinality and overlap with span status. - ErrorMessageKey = attribute.Key("error.message") - - // ErrorTypeKey is the attribute Key conforming to the "error.type" semantic - // conventions. It represents the describes a class of error the operation ended - // with. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "timeout", "java.net.UnknownHostException", - // "server_certificate_invalid", "500" - // Note: The `error.type` SHOULD be predictable, and SHOULD have low - // cardinality. - // - // When `error.type` is set to a type (e.g., an exception type), its - // canonical class name identifying the type within the artifact SHOULD be used. - // - // Instrumentations SHOULD document the list of errors they report. - // - // The cardinality of `error.type` within one instrumentation library SHOULD be - // low. - // Telemetry consumers that aggregate data from multiple instrumentation - // libraries and applications - // should be prepared for `error.type` to have high cardinality at query time - // when no - // additional filters are applied. - // - // If the operation has completed successfully, instrumentations SHOULD NOT set - // `error.type`. - // - // If a specific domain defines its own set of error identifiers (such as HTTP - // or RPC status codes), - // it's RECOMMENDED to: - // - // - Use a domain-specific attribute - // - Set `error.type` to capture all errors, regardless of whether they are - // defined within the domain-specific set or not. - ErrorTypeKey = attribute.Key("error.type") -) - -// ErrorMessage returns an attribute KeyValue conforming to the "error.message" -// semantic conventions. It represents a message providing more detail about an -// error in human-readable form. -func ErrorMessage(val string) attribute.KeyValue { - return ErrorMessageKey.String(val) -} - -// Enum values for error.type -var ( - // A fallback error value to be used when the instrumentation doesn't define a - // custom value. - // - // Stability: stable - ErrorTypeOther = ErrorTypeKey.String("_OTHER") -) - -// Namespace: exception -const ( - // ExceptionMessageKey is the attribute Key conforming to the - // "exception.message" semantic conventions. It represents the exception - // message. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "Division by zero", "Can't convert 'int' object to str implicitly" - ExceptionMessageKey = attribute.Key("exception.message") - - // ExceptionStacktraceKey is the attribute Key conforming to the - // "exception.stacktrace" semantic conventions. It represents a stacktrace as a - // string in the natural representation for the language runtime. The - // representation is to be determined and documented by each language SIG. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: Exception in thread "main" java.lang.RuntimeException: Test - // exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at - // com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at - // com.example.GenerateTrace.main(GenerateTrace.java:5) - ExceptionStacktraceKey = attribute.Key("exception.stacktrace") - - // ExceptionTypeKey is the attribute Key conforming to the "exception.type" - // semantic conventions. It represents the type of the exception (its - // fully-qualified class name, if applicable). The dynamic type of the exception - // should be preferred over the static type in languages that support it. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "java.net.ConnectException", "OSError" - ExceptionTypeKey = attribute.Key("exception.type") -) - -// ExceptionMessage returns an attribute KeyValue conforming to the -// "exception.message" semantic conventions. It represents the exception message. -func ExceptionMessage(val string) attribute.KeyValue { - return ExceptionMessageKey.String(val) -} - -// ExceptionStacktrace returns an attribute KeyValue conforming to the -// "exception.stacktrace" semantic conventions. It represents a stacktrace as a -// string in the natural representation for the language runtime. The -// representation is to be determined and documented by each language SIG. -func ExceptionStacktrace(val string) attribute.KeyValue { - return ExceptionStacktraceKey.String(val) -} - -// ExceptionType returns an attribute KeyValue conforming to the "exception.type" -// semantic conventions. It represents the type of the exception (its -// fully-qualified class name, if applicable). The dynamic type of the exception -// should be preferred over the static type in languages that support it. -func ExceptionType(val string) attribute.KeyValue { - return ExceptionTypeKey.String(val) -} - -// Namespace: faas -const ( - // FaaSColdstartKey is the attribute Key conforming to the "faas.coldstart" - // semantic conventions. It represents a boolean that is true if the serverless - // function is executed for the first time (aka cold-start). - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - FaaSColdstartKey = attribute.Key("faas.coldstart") - - // FaaSCronKey is the attribute Key conforming to the "faas.cron" semantic - // conventions. It represents a string containing the schedule period as - // [Cron Expression]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0/5 * * * ? * - // - // [Cron Expression]: https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm - FaaSCronKey = attribute.Key("faas.cron") - - // FaaSDocumentCollectionKey is the attribute Key conforming to the - // "faas.document.collection" semantic conventions. It represents the name of - // the source on which the triggering operation was performed. For example, in - // Cloud Storage or S3 corresponds to the bucket name, and in Cosmos DB to the - // database name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "myBucketName", "myDbName" - FaaSDocumentCollectionKey = attribute.Key("faas.document.collection") - - // FaaSDocumentNameKey is the attribute Key conforming to the - // "faas.document.name" semantic conventions. It represents the document - // name/table subjected to the operation. For example, in Cloud Storage or S3 is - // the name of the file, and in Cosmos DB the table name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "myFile.txt", "myTableName" - FaaSDocumentNameKey = attribute.Key("faas.document.name") - - // FaaSDocumentOperationKey is the attribute Key conforming to the - // "faas.document.operation" semantic conventions. It represents the describes - // the type of the operation that was performed on the data. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - FaaSDocumentOperationKey = attribute.Key("faas.document.operation") - - // FaaSDocumentTimeKey is the attribute Key conforming to the - // "faas.document.time" semantic conventions. It represents a string containing - // the time when the data was accessed in the [ISO 8601] format expressed in - // [UTC]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 2020-01-23T13:47:06Z - // - // [ISO 8601]: https://www.iso.org/iso-8601-date-and-time-format.html - // [UTC]: https://www.w3.org/TR/NOTE-datetime - FaaSDocumentTimeKey = attribute.Key("faas.document.time") - - // FaaSInstanceKey is the attribute Key conforming to the "faas.instance" - // semantic conventions. It represents the execution environment ID as a string, - // that will be potentially reused for other invocations to the same - // function/function version. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021/06/28/[$LATEST]2f399eb14537447da05ab2a2e39309de" - // Note: - **AWS Lambda:** Use the (full) log stream name. - FaaSInstanceKey = attribute.Key("faas.instance") - - // FaaSInvocationIDKey is the attribute Key conforming to the - // "faas.invocation_id" semantic conventions. It represents the invocation ID of - // the current function invocation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: af9d5aa4-a685-4c5f-a22b-444f80b3cc28 - FaaSInvocationIDKey = attribute.Key("faas.invocation_id") - - // FaaSInvokedNameKey is the attribute Key conforming to the "faas.invoked_name" - // semantic conventions. It represents the name of the invoked function. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: my-function - // Note: SHOULD be equal to the `faas.name` resource attribute of the invoked - // function. - FaaSInvokedNameKey = attribute.Key("faas.invoked_name") - - // FaaSInvokedProviderKey is the attribute Key conforming to the - // "faas.invoked_provider" semantic conventions. It represents the cloud - // provider of the invoked function. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: SHOULD be equal to the `cloud.provider` resource attribute of the - // invoked function. - FaaSInvokedProviderKey = attribute.Key("faas.invoked_provider") - - // FaaSInvokedRegionKey is the attribute Key conforming to the - // "faas.invoked_region" semantic conventions. It represents the cloud region of - // the invoked function. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: eu-central-1 - // Note: SHOULD be equal to the `cloud.region` resource attribute of the invoked - // function. - FaaSInvokedRegionKey = attribute.Key("faas.invoked_region") - - // FaaSMaxMemoryKey is the attribute Key conforming to the "faas.max_memory" - // semantic conventions. It represents the amount of memory available to the - // serverless function converted to Bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Note: It's recommended to set this attribute since e.g. too little memory can - // easily stop a Java AWS Lambda function from working correctly. On AWS Lambda, - // the environment variable `AWS_LAMBDA_FUNCTION_MEMORY_SIZE` provides this - // information (which must be multiplied by 1,048,576). - FaaSMaxMemoryKey = attribute.Key("faas.max_memory") - - // FaaSNameKey is the attribute Key conforming to the "faas.name" semantic - // conventions. It represents the name of the single function that this runtime - // instance executes. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-function", "myazurefunctionapp/some-function-name" - // Note: This is the name of the function as configured/deployed on the FaaS - // platform and is usually different from the name of the callback - // function (which may be stored in the - // [`code.namespace`/`code.function.name`] - // span attributes). - // - // For some cloud providers, the above definition is ambiguous. The following - // definition of function name MUST be used for this attribute - // (and consequently the span name) for the listed cloud providers/products: - // - // - **Azure:** The full name `/`, i.e., function app name - // followed by a forward slash followed by the function name (this form - // can also be seen in the resource JSON for the function). - // This means that a span attribute MUST be used, as an Azure function - // app can host multiple functions that would usually share - // a TracerProvider (see also the `cloud.resource_id` attribute). - // - // - // [`code.namespace`/`code.function.name`]: /docs/general/attributes.md#source-code-attributes - FaaSNameKey = attribute.Key("faas.name") - - // FaaSTimeKey is the attribute Key conforming to the "faas.time" semantic - // conventions. It represents a string containing the function invocation time - // in the [ISO 8601] format expressed in [UTC]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 2020-01-23T13:47:06Z - // - // [ISO 8601]: https://www.iso.org/iso-8601-date-and-time-format.html - // [UTC]: https://www.w3.org/TR/NOTE-datetime - FaaSTimeKey = attribute.Key("faas.time") - - // FaaSTriggerKey is the attribute Key conforming to the "faas.trigger" semantic - // conventions. It represents the type of the trigger which caused this function - // invocation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - FaaSTriggerKey = attribute.Key("faas.trigger") - - // FaaSVersionKey is the attribute Key conforming to the "faas.version" semantic - // conventions. It represents the immutable version of the function being - // executed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "26", "pinkfroid-00002" - // Note: Depending on the cloud provider and platform, use: - // - // - **AWS Lambda:** The [function version] - // (an integer represented as a decimal string). - // - **Google Cloud Run (Services):** The [revision] - // (i.e., the function name plus the revision suffix). - // - **Google Cloud Functions:** The value of the - // [`K_REVISION` environment variable]. - // - **Azure Functions:** Not applicable. Do not set this attribute. - // - // - // [function version]: https://docs.aws.amazon.com/lambda/latest/dg/configuration-versions.html - // [revision]: https://cloud.google.com/run/docs/managing/revisions - // [`K_REVISION` environment variable]: https://cloud.google.com/run/docs/container-contract#services-env-vars - FaaSVersionKey = attribute.Key("faas.version") -) - -// FaaSColdstart returns an attribute KeyValue conforming to the "faas.coldstart" -// semantic conventions. It represents a boolean that is true if the serverless -// function is executed for the first time (aka cold-start). -func FaaSColdstart(val bool) attribute.KeyValue { - return FaaSColdstartKey.Bool(val) -} - -// FaaSCron returns an attribute KeyValue conforming to the "faas.cron" semantic -// conventions. It represents a string containing the schedule period as -// [Cron Expression]. -// -// [Cron Expression]: https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm -func FaaSCron(val string) attribute.KeyValue { - return FaaSCronKey.String(val) -} - -// FaaSDocumentCollection returns an attribute KeyValue conforming to the -// "faas.document.collection" semantic conventions. It represents the name of the -// source on which the triggering operation was performed. For example, in Cloud -// Storage or S3 corresponds to the bucket name, and in Cosmos DB to the database -// name. -func FaaSDocumentCollection(val string) attribute.KeyValue { - return FaaSDocumentCollectionKey.String(val) -} - -// FaaSDocumentName returns an attribute KeyValue conforming to the -// "faas.document.name" semantic conventions. It represents the document -// name/table subjected to the operation. For example, in Cloud Storage or S3 is -// the name of the file, and in Cosmos DB the table name. -func FaaSDocumentName(val string) attribute.KeyValue { - return FaaSDocumentNameKey.String(val) -} - -// FaaSDocumentTime returns an attribute KeyValue conforming to the -// "faas.document.time" semantic conventions. It represents a string containing -// the time when the data was accessed in the [ISO 8601] format expressed in -// [UTC]. -// -// [ISO 8601]: https://www.iso.org/iso-8601-date-and-time-format.html -// [UTC]: https://www.w3.org/TR/NOTE-datetime -func FaaSDocumentTime(val string) attribute.KeyValue { - return FaaSDocumentTimeKey.String(val) -} - -// FaaSInstance returns an attribute KeyValue conforming to the "faas.instance" -// semantic conventions. It represents the execution environment ID as a string, -// that will be potentially reused for other invocations to the same -// function/function version. -func FaaSInstance(val string) attribute.KeyValue { - return FaaSInstanceKey.String(val) -} - -// FaaSInvocationID returns an attribute KeyValue conforming to the -// "faas.invocation_id" semantic conventions. It represents the invocation ID of -// the current function invocation. -func FaaSInvocationID(val string) attribute.KeyValue { - return FaaSInvocationIDKey.String(val) -} - -// FaaSInvokedName returns an attribute KeyValue conforming to the -// "faas.invoked_name" semantic conventions. It represents the name of the -// invoked function. -func FaaSInvokedName(val string) attribute.KeyValue { - return FaaSInvokedNameKey.String(val) -} - -// FaaSInvokedRegion returns an attribute KeyValue conforming to the -// "faas.invoked_region" semantic conventions. It represents the cloud region of -// the invoked function. -func FaaSInvokedRegion(val string) attribute.KeyValue { - return FaaSInvokedRegionKey.String(val) -} - -// FaaSMaxMemory returns an attribute KeyValue conforming to the -// "faas.max_memory" semantic conventions. It represents the amount of memory -// available to the serverless function converted to Bytes. -func FaaSMaxMemory(val int) attribute.KeyValue { - return FaaSMaxMemoryKey.Int(val) -} - -// FaaSName returns an attribute KeyValue conforming to the "faas.name" semantic -// conventions. It represents the name of the single function that this runtime -// instance executes. -func FaaSName(val string) attribute.KeyValue { - return FaaSNameKey.String(val) -} - -// FaaSTime returns an attribute KeyValue conforming to the "faas.time" semantic -// conventions. It represents a string containing the function invocation time in -// the [ISO 8601] format expressed in [UTC]. -// -// [ISO 8601]: https://www.iso.org/iso-8601-date-and-time-format.html -// [UTC]: https://www.w3.org/TR/NOTE-datetime -func FaaSTime(val string) attribute.KeyValue { - return FaaSTimeKey.String(val) -} - -// FaaSVersion returns an attribute KeyValue conforming to the "faas.version" -// semantic conventions. It represents the immutable version of the function -// being executed. -func FaaSVersion(val string) attribute.KeyValue { - return FaaSVersionKey.String(val) -} - -// Enum values for faas.document.operation -var ( - // When a new object is created. - // Stability: development - FaaSDocumentOperationInsert = FaaSDocumentOperationKey.String("insert") - // When an object is modified. - // Stability: development - FaaSDocumentOperationEdit = FaaSDocumentOperationKey.String("edit") - // When an object is deleted. - // Stability: development - FaaSDocumentOperationDelete = FaaSDocumentOperationKey.String("delete") -) - -// Enum values for faas.invoked_provider -var ( - // Alibaba Cloud - // Stability: development - FaaSInvokedProviderAlibabaCloud = FaaSInvokedProviderKey.String("alibaba_cloud") - // Amazon Web Services - // Stability: development - FaaSInvokedProviderAWS = FaaSInvokedProviderKey.String("aws") - // Microsoft Azure - // Stability: development - FaaSInvokedProviderAzure = FaaSInvokedProviderKey.String("azure") - // Google Cloud Platform - // Stability: development - FaaSInvokedProviderGCP = FaaSInvokedProviderKey.String("gcp") - // Tencent Cloud - // Stability: development - FaaSInvokedProviderTencentCloud = FaaSInvokedProviderKey.String("tencent_cloud") -) - -// Enum values for faas.trigger -var ( - // A response to some data source operation such as a database or filesystem - // read/write - // Stability: development - FaaSTriggerDatasource = FaaSTriggerKey.String("datasource") - // To provide an answer to an inbound HTTP request - // Stability: development - FaaSTriggerHTTP = FaaSTriggerKey.String("http") - // A function is set to be executed when messages are sent to a messaging system - // Stability: development - FaaSTriggerPubSub = FaaSTriggerKey.String("pubsub") - // A function is scheduled to be executed regularly - // Stability: development - FaaSTriggerTimer = FaaSTriggerKey.String("timer") - // If none of the others apply - // Stability: development - FaaSTriggerOther = FaaSTriggerKey.String("other") -) - -// Namespace: feature_flag -const ( - // FeatureFlagContextIDKey is the attribute Key conforming to the - // "feature_flag.context.id" semantic conventions. It represents the unique - // identifier for the flag evaluation context. For example, the targeting key. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "5157782b-2203-4c80-a857-dbbd5e7761db" - FeatureFlagContextIDKey = attribute.Key("feature_flag.context.id") - - // FeatureFlagKeyKey is the attribute Key conforming to the "feature_flag.key" - // semantic conventions. It represents the lookup key of the feature flag. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "logo-color" - FeatureFlagKeyKey = attribute.Key("feature_flag.key") - - // FeatureFlagProviderNameKey is the attribute Key conforming to the - // "feature_flag.provider.name" semantic conventions. It represents the - // identifies the feature flag provider. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "Flag Manager" - FeatureFlagProviderNameKey = attribute.Key("feature_flag.provider.name") - - // FeatureFlagResultReasonKey is the attribute Key conforming to the - // "feature_flag.result.reason" semantic conventions. It represents the reason - // code which shows how a feature flag value was determined. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "static", "targeting_match", "error", "default" - FeatureFlagResultReasonKey = attribute.Key("feature_flag.result.reason") - - // FeatureFlagResultValueKey is the attribute Key conforming to the - // "feature_flag.result.value" semantic conventions. It represents the evaluated - // value of the feature flag. - // - // Type: any - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "#ff0000", true, 3 - // Note: With some feature flag providers, feature flag results can be quite - // large or contain private or sensitive details. - // Because of this, `feature_flag.result.variant` is often the preferred - // attribute if it is available. - // - // It may be desirable to redact or otherwise limit the size and scope of - // `feature_flag.result.value` if possible. - // Because the evaluated flag value is unstructured and may be any type, it is - // left to the instrumentation author to determine how best to achieve this. - FeatureFlagResultValueKey = attribute.Key("feature_flag.result.value") - - // FeatureFlagResultVariantKey is the attribute Key conforming to the - // "feature_flag.result.variant" semantic conventions. It represents a semantic - // identifier for an evaluated flag value. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "red", "true", "on" - // Note: A semantic identifier, commonly referred to as a variant, provides a - // means - // for referring to a value without including the value itself. This can - // provide additional context for understanding the meaning behind a value. - // For example, the variant `red` maybe be used for the value `#c05543`. - FeatureFlagResultVariantKey = attribute.Key("feature_flag.result.variant") - - // FeatureFlagSetIDKey is the attribute Key conforming to the - // "feature_flag.set.id" semantic conventions. It represents the identifier of - // the [flag set] to which the feature flag belongs. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "proj-1", "ab98sgs", "service1/dev" - // - // [flag set]: https://openfeature.dev/specification/glossary/#flag-set - FeatureFlagSetIDKey = attribute.Key("feature_flag.set.id") - - // FeatureFlagVersionKey is the attribute Key conforming to the - // "feature_flag.version" semantic conventions. It represents the version of the - // ruleset used during the evaluation. This may be any stable value which - // uniquely identifies the ruleset. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Release_Candidate - // - // Examples: "1", "01ABCDEF" - FeatureFlagVersionKey = attribute.Key("feature_flag.version") -) - -// FeatureFlagContextID returns an attribute KeyValue conforming to the -// "feature_flag.context.id" semantic conventions. It represents the unique -// identifier for the flag evaluation context. For example, the targeting key. -func FeatureFlagContextID(val string) attribute.KeyValue { - return FeatureFlagContextIDKey.String(val) -} - -// FeatureFlagKey returns an attribute KeyValue conforming to the -// "feature_flag.key" semantic conventions. It represents the lookup key of the -// feature flag. -func FeatureFlagKey(val string) attribute.KeyValue { - return FeatureFlagKeyKey.String(val) -} - -// FeatureFlagProviderName returns an attribute KeyValue conforming to the -// "feature_flag.provider.name" semantic conventions. It represents the -// identifies the feature flag provider. -func FeatureFlagProviderName(val string) attribute.KeyValue { - return FeatureFlagProviderNameKey.String(val) -} - -// FeatureFlagResultVariant returns an attribute KeyValue conforming to the -// "feature_flag.result.variant" semantic conventions. It represents a semantic -// identifier for an evaluated flag value. -func FeatureFlagResultVariant(val string) attribute.KeyValue { - return FeatureFlagResultVariantKey.String(val) -} - -// FeatureFlagSetID returns an attribute KeyValue conforming to the -// "feature_flag.set.id" semantic conventions. It represents the identifier of -// the [flag set] to which the feature flag belongs. -// -// [flag set]: https://openfeature.dev/specification/glossary/#flag-set -func FeatureFlagSetID(val string) attribute.KeyValue { - return FeatureFlagSetIDKey.String(val) -} - -// FeatureFlagVersion returns an attribute KeyValue conforming to the -// "feature_flag.version" semantic conventions. It represents the version of the -// ruleset used during the evaluation. This may be any stable value which -// uniquely identifies the ruleset. -func FeatureFlagVersion(val string) attribute.KeyValue { - return FeatureFlagVersionKey.String(val) -} - -// Enum values for feature_flag.result.reason -var ( - // The resolved value is static (no dynamic evaluation). - // Stability: release_candidate - FeatureFlagResultReasonStatic = FeatureFlagResultReasonKey.String("static") - // The resolved value fell back to a pre-configured value (no dynamic evaluation - // occurred or dynamic evaluation yielded no result). - // Stability: release_candidate - FeatureFlagResultReasonDefault = FeatureFlagResultReasonKey.String("default") - // The resolved value was the result of a dynamic evaluation, such as a rule or - // specific user-targeting. - // Stability: release_candidate - FeatureFlagResultReasonTargetingMatch = FeatureFlagResultReasonKey.String("targeting_match") - // The resolved value was the result of pseudorandom assignment. - // Stability: release_candidate - FeatureFlagResultReasonSplit = FeatureFlagResultReasonKey.String("split") - // The resolved value was retrieved from cache. - // Stability: release_candidate - FeatureFlagResultReasonCached = FeatureFlagResultReasonKey.String("cached") - // The resolved value was the result of the flag being disabled in the - // management system. - // Stability: release_candidate - FeatureFlagResultReasonDisabled = FeatureFlagResultReasonKey.String("disabled") - // The reason for the resolved value could not be determined. - // Stability: release_candidate - FeatureFlagResultReasonUnknown = FeatureFlagResultReasonKey.String("unknown") - // The resolved value is non-authoritative or possibly out of date - // Stability: release_candidate - FeatureFlagResultReasonStale = FeatureFlagResultReasonKey.String("stale") - // The resolved value was the result of an error. - // Stability: release_candidate - FeatureFlagResultReasonError = FeatureFlagResultReasonKey.String("error") -) - -// Namespace: file -const ( - // FileAccessedKey is the attribute Key conforming to the "file.accessed" - // semantic conventions. It represents the time when the file was last accessed, - // in ISO 8601 format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T12:00:00Z" - // Note: This attribute might not be supported by some file systems — NFS, - // FAT32, in embedded OS, etc. - FileAccessedKey = attribute.Key("file.accessed") - - // FileAttributesKey is the attribute Key conforming to the "file.attributes" - // semantic conventions. It represents the array of file attributes. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "readonly", "hidden" - // Note: Attributes names depend on the OS or file system. Here’s a - // non-exhaustive list of values expected for this attribute: `archive`, - // `compressed`, `directory`, `encrypted`, `execute`, `hidden`, `immutable`, - // `journaled`, `read`, `readonly`, `symbolic link`, `system`, `temporary`, - // `write`. - FileAttributesKey = attribute.Key("file.attributes") - - // FileChangedKey is the attribute Key conforming to the "file.changed" semantic - // conventions. It represents the time when the file attributes or metadata was - // last changed, in ISO 8601 format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T12:00:00Z" - // Note: `file.changed` captures the time when any of the file's properties or - // attributes (including the content) are changed, while `file.modified` - // captures the timestamp when the file content is modified. - FileChangedKey = attribute.Key("file.changed") - - // FileCreatedKey is the attribute Key conforming to the "file.created" semantic - // conventions. It represents the time when the file was created, in ISO 8601 - // format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T12:00:00Z" - // Note: This attribute might not be supported by some file systems — NFS, - // FAT32, in embedded OS, etc. - FileCreatedKey = attribute.Key("file.created") - - // FileDirectoryKey is the attribute Key conforming to the "file.directory" - // semantic conventions. It represents the directory where the file is located. - // It should include the drive letter, when appropriate. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/home/user", "C:\Program Files\MyApp" - FileDirectoryKey = attribute.Key("file.directory") - - // FileExtensionKey is the attribute Key conforming to the "file.extension" - // semantic conventions. It represents the file extension, excluding the leading - // dot. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "png", "gz" - // Note: When the file name has multiple extensions (example.tar.gz), only the - // last one should be captured ("gz", not "tar.gz"). - FileExtensionKey = attribute.Key("file.extension") - - // FileForkNameKey is the attribute Key conforming to the "file.fork_name" - // semantic conventions. It represents the name of the fork. A fork is - // additional data associated with a filesystem object. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Zone.Identifier" - // Note: On Linux, a resource fork is used to store additional data with a - // filesystem object. A file always has at least one fork for the data portion, - // and additional forks may exist. - // On NTFS, this is analogous to an Alternate Data Stream (ADS), and the default - // data stream for a file is just called $DATA. Zone.Identifier is commonly used - // by Windows to track contents downloaded from the Internet. An ADS is - // typically of the form: C:\path\to\filename.extension:some_fork_name, and - // some_fork_name is the value that should populate `fork_name`. - // `filename.extension` should populate `file.name`, and `extension` should - // populate `file.extension`. The full path, `file.path`, will include the fork - // name. - FileForkNameKey = attribute.Key("file.fork_name") - - // FileGroupIDKey is the attribute Key conforming to the "file.group.id" - // semantic conventions. It represents the primary Group ID (GID) of the file. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1000" - FileGroupIDKey = attribute.Key("file.group.id") - - // FileGroupNameKey is the attribute Key conforming to the "file.group.name" - // semantic conventions. It represents the primary group name of the file. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "users" - FileGroupNameKey = attribute.Key("file.group.name") - - // FileInodeKey is the attribute Key conforming to the "file.inode" semantic - // conventions. It represents the inode representing the file in the filesystem. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "256383" - FileInodeKey = attribute.Key("file.inode") - - // FileModeKey is the attribute Key conforming to the "file.mode" semantic - // conventions. It represents the mode of the file in octal representation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0640" - FileModeKey = attribute.Key("file.mode") - - // FileModifiedKey is the attribute Key conforming to the "file.modified" - // semantic conventions. It represents the time when the file content was last - // modified, in ISO 8601 format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T12:00:00Z" - FileModifiedKey = attribute.Key("file.modified") - - // FileNameKey is the attribute Key conforming to the "file.name" semantic - // conventions. It represents the name of the file including the extension, - // without the directory. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "example.png" - FileNameKey = attribute.Key("file.name") - - // FileOwnerIDKey is the attribute Key conforming to the "file.owner.id" - // semantic conventions. It represents the user ID (UID) or security identifier - // (SID) of the file owner. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1000" - FileOwnerIDKey = attribute.Key("file.owner.id") - - // FileOwnerNameKey is the attribute Key conforming to the "file.owner.name" - // semantic conventions. It represents the username of the file owner. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "root" - FileOwnerNameKey = attribute.Key("file.owner.name") - - // FilePathKey is the attribute Key conforming to the "file.path" semantic - // conventions. It represents the full path to the file, including the file - // name. It should include the drive letter, when appropriate. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/home/alice/example.png", "C:\Program Files\MyApp\myapp.exe" - FilePathKey = attribute.Key("file.path") - - // FileSizeKey is the attribute Key conforming to the "file.size" semantic - // conventions. It represents the file size in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - FileSizeKey = attribute.Key("file.size") - - // FileSymbolicLinkTargetPathKey is the attribute Key conforming to the - // "file.symbolic_link.target_path" semantic conventions. It represents the path - // to the target of a symbolic link. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/usr/bin/python3" - // Note: This attribute is only applicable to symbolic links. - FileSymbolicLinkTargetPathKey = attribute.Key("file.symbolic_link.target_path") -) - -// FileAccessed returns an attribute KeyValue conforming to the "file.accessed" -// semantic conventions. It represents the time when the file was last accessed, -// in ISO 8601 format. -func FileAccessed(val string) attribute.KeyValue { - return FileAccessedKey.String(val) -} - -// FileAttributes returns an attribute KeyValue conforming to the -// "file.attributes" semantic conventions. It represents the array of file -// attributes. -func FileAttributes(val ...string) attribute.KeyValue { - return FileAttributesKey.StringSlice(val) -} - -// FileChanged returns an attribute KeyValue conforming to the "file.changed" -// semantic conventions. It represents the time when the file attributes or -// metadata was last changed, in ISO 8601 format. -func FileChanged(val string) attribute.KeyValue { - return FileChangedKey.String(val) -} - -// FileCreated returns an attribute KeyValue conforming to the "file.created" -// semantic conventions. It represents the time when the file was created, in ISO -// 8601 format. -func FileCreated(val string) attribute.KeyValue { - return FileCreatedKey.String(val) -} - -// FileDirectory returns an attribute KeyValue conforming to the "file.directory" -// semantic conventions. It represents the directory where the file is located. -// It should include the drive letter, when appropriate. -func FileDirectory(val string) attribute.KeyValue { - return FileDirectoryKey.String(val) -} - -// FileExtension returns an attribute KeyValue conforming to the "file.extension" -// semantic conventions. It represents the file extension, excluding the leading -// dot. -func FileExtension(val string) attribute.KeyValue { - return FileExtensionKey.String(val) -} - -// FileForkName returns an attribute KeyValue conforming to the "file.fork_name" -// semantic conventions. It represents the name of the fork. A fork is additional -// data associated with a filesystem object. -func FileForkName(val string) attribute.KeyValue { - return FileForkNameKey.String(val) -} - -// FileGroupID returns an attribute KeyValue conforming to the "file.group.id" -// semantic conventions. It represents the primary Group ID (GID) of the file. -func FileGroupID(val string) attribute.KeyValue { - return FileGroupIDKey.String(val) -} - -// FileGroupName returns an attribute KeyValue conforming to the -// "file.group.name" semantic conventions. It represents the primary group name -// of the file. -func FileGroupName(val string) attribute.KeyValue { - return FileGroupNameKey.String(val) -} - -// FileInode returns an attribute KeyValue conforming to the "file.inode" -// semantic conventions. It represents the inode representing the file in the -// filesystem. -func FileInode(val string) attribute.KeyValue { - return FileInodeKey.String(val) -} - -// FileMode returns an attribute KeyValue conforming to the "file.mode" semantic -// conventions. It represents the mode of the file in octal representation. -func FileMode(val string) attribute.KeyValue { - return FileModeKey.String(val) -} - -// FileModified returns an attribute KeyValue conforming to the "file.modified" -// semantic conventions. It represents the time when the file content was last -// modified, in ISO 8601 format. -func FileModified(val string) attribute.KeyValue { - return FileModifiedKey.String(val) -} - -// FileName returns an attribute KeyValue conforming to the "file.name" semantic -// conventions. It represents the name of the file including the extension, -// without the directory. -func FileName(val string) attribute.KeyValue { - return FileNameKey.String(val) -} - -// FileOwnerID returns an attribute KeyValue conforming to the "file.owner.id" -// semantic conventions. It represents the user ID (UID) or security identifier -// (SID) of the file owner. -func FileOwnerID(val string) attribute.KeyValue { - return FileOwnerIDKey.String(val) -} - -// FileOwnerName returns an attribute KeyValue conforming to the -// "file.owner.name" semantic conventions. It represents the username of the file -// owner. -func FileOwnerName(val string) attribute.KeyValue { - return FileOwnerNameKey.String(val) -} - -// FilePath returns an attribute KeyValue conforming to the "file.path" semantic -// conventions. It represents the full path to the file, including the file name. -// It should include the drive letter, when appropriate. -func FilePath(val string) attribute.KeyValue { - return FilePathKey.String(val) -} - -// FileSize returns an attribute KeyValue conforming to the "file.size" semantic -// conventions. It represents the file size in bytes. -func FileSize(val int) attribute.KeyValue { - return FileSizeKey.Int(val) -} - -// FileSymbolicLinkTargetPath returns an attribute KeyValue conforming to the -// "file.symbolic_link.target_path" semantic conventions. It represents the path -// to the target of a symbolic link. -func FileSymbolicLinkTargetPath(val string) attribute.KeyValue { - return FileSymbolicLinkTargetPathKey.String(val) -} - -// Namespace: gcp -const ( - // GCPAppHubApplicationContainerKey is the attribute Key conforming to the - // "gcp.apphub.application.container" semantic conventions. It represents the - // container within GCP where the AppHub application is defined. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "projects/my-container-project" - GCPAppHubApplicationContainerKey = attribute.Key("gcp.apphub.application.container") - - // GCPAppHubApplicationIDKey is the attribute Key conforming to the - // "gcp.apphub.application.id" semantic conventions. It represents the name of - // the application as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-application" - GCPAppHubApplicationIDKey = attribute.Key("gcp.apphub.application.id") - - // GCPAppHubApplicationLocationKey is the attribute Key conforming to the - // "gcp.apphub.application.location" semantic conventions. It represents the GCP - // zone or region where the application is defined. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "us-central1" - GCPAppHubApplicationLocationKey = attribute.Key("gcp.apphub.application.location") - - // GCPAppHubServiceCriticalityTypeKey is the attribute Key conforming to the - // "gcp.apphub.service.criticality_type" semantic conventions. It represents the - // criticality of a service indicates its importance to the business. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: [See AppHub type enum] - // - // [See AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type - GCPAppHubServiceCriticalityTypeKey = attribute.Key("gcp.apphub.service.criticality_type") - - // GCPAppHubServiceEnvironmentTypeKey is the attribute Key conforming to the - // "gcp.apphub.service.environment_type" semantic conventions. It represents the - // environment of a service is the stage of a software lifecycle. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: [See AppHub environment type] - // - // [See AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 - GCPAppHubServiceEnvironmentTypeKey = attribute.Key("gcp.apphub.service.environment_type") - - // GCPAppHubServiceIDKey is the attribute Key conforming to the - // "gcp.apphub.service.id" semantic conventions. It represents the name of the - // service as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-service" - GCPAppHubServiceIDKey = attribute.Key("gcp.apphub.service.id") - - // GCPAppHubWorkloadCriticalityTypeKey is the attribute Key conforming to the - // "gcp.apphub.workload.criticality_type" semantic conventions. It represents - // the criticality of a workload indicates its importance to the business. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: [See AppHub type enum] - // - // [See AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type - GCPAppHubWorkloadCriticalityTypeKey = attribute.Key("gcp.apphub.workload.criticality_type") - - // GCPAppHubWorkloadEnvironmentTypeKey is the attribute Key conforming to the - // "gcp.apphub.workload.environment_type" semantic conventions. It represents - // the environment of a workload is the stage of a software lifecycle. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: [See AppHub environment type] - // - // [See AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 - GCPAppHubWorkloadEnvironmentTypeKey = attribute.Key("gcp.apphub.workload.environment_type") - - // GCPAppHubWorkloadIDKey is the attribute Key conforming to the - // "gcp.apphub.workload.id" semantic conventions. It represents the name of the - // workload as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-workload" - GCPAppHubWorkloadIDKey = attribute.Key("gcp.apphub.workload.id") - - // GCPAppHubDestinationApplicationContainerKey is the attribute Key conforming - // to the "gcp.apphub_destination.application.container" semantic conventions. - // It represents the container within GCP where the AppHub destination - // application is defined. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "projects/my-container-project" - GCPAppHubDestinationApplicationContainerKey = attribute.Key("gcp.apphub_destination.application.container") - - // GCPAppHubDestinationApplicationIDKey is the attribute Key conforming to the - // "gcp.apphub_destination.application.id" semantic conventions. It represents - // the name of the destination application as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-application" - GCPAppHubDestinationApplicationIDKey = attribute.Key("gcp.apphub_destination.application.id") - - // GCPAppHubDestinationApplicationLocationKey is the attribute Key conforming to - // the "gcp.apphub_destination.application.location" semantic conventions. It - // represents the GCP zone or region where the destination application is - // defined. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "us-central1" - GCPAppHubDestinationApplicationLocationKey = attribute.Key("gcp.apphub_destination.application.location") - - // GCPAppHubDestinationServiceCriticalityTypeKey is the attribute Key conforming - // to the "gcp.apphub_destination.service.criticality_type" semantic - // conventions. It represents the criticality of a destination workload - // indicates its importance to the business as specified in [AppHub type enum]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type - GCPAppHubDestinationServiceCriticalityTypeKey = attribute.Key("gcp.apphub_destination.service.criticality_type") - - // GCPAppHubDestinationServiceEnvironmentTypeKey is the attribute Key conforming - // to the "gcp.apphub_destination.service.environment_type" semantic - // conventions. It represents the software lifecycle stage of a destination - // service as defined [AppHub environment type]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 - GCPAppHubDestinationServiceEnvironmentTypeKey = attribute.Key("gcp.apphub_destination.service.environment_type") - - // GCPAppHubDestinationServiceIDKey is the attribute Key conforming to the - // "gcp.apphub_destination.service.id" semantic conventions. It represents the - // name of the destination service as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-service" - GCPAppHubDestinationServiceIDKey = attribute.Key("gcp.apphub_destination.service.id") - - // GCPAppHubDestinationWorkloadCriticalityTypeKey is the attribute Key - // conforming to the "gcp.apphub_destination.workload.criticality_type" semantic - // conventions. It represents the criticality of a destination workload - // indicates its importance to the business as specified in [AppHub type enum]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type - GCPAppHubDestinationWorkloadCriticalityTypeKey = attribute.Key("gcp.apphub_destination.workload.criticality_type") - - // GCPAppHubDestinationWorkloadEnvironmentTypeKey is the attribute Key - // conforming to the "gcp.apphub_destination.workload.environment_type" semantic - // conventions. It represents the environment of a destination workload is the - // stage of a software lifecycle as provided in the [AppHub environment type]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 - GCPAppHubDestinationWorkloadEnvironmentTypeKey = attribute.Key("gcp.apphub_destination.workload.environment_type") - - // GCPAppHubDestinationWorkloadIDKey is the attribute Key conforming to the - // "gcp.apphub_destination.workload.id" semantic conventions. It represents the - // name of the destination workload as configured in AppHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-workload" - GCPAppHubDestinationWorkloadIDKey = attribute.Key("gcp.apphub_destination.workload.id") - - // GCPClientServiceKey is the attribute Key conforming to the - // "gcp.client.service" semantic conventions. It represents the identifies the - // Google Cloud service for which the official client library is intended. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "appengine", "run", "firestore", "alloydb", "spanner" - // Note: Intended to be a stable identifier for Google Cloud client libraries - // that is uniform across implementation languages. The value should be derived - // from the canonical service domain for the service; for example, - // 'foo.googleapis.com' should result in a value of 'foo'. - GCPClientServiceKey = attribute.Key("gcp.client.service") - - // GCPCloudRunJobExecutionKey is the attribute Key conforming to the - // "gcp.cloud_run.job.execution" semantic conventions. It represents the name of - // the Cloud Run [execution] being run for the Job, as set by the - // [`CLOUD_RUN_EXECUTION`] environment variable. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "job-name-xxxx", "sample-job-mdw84" - // - // [execution]: https://cloud.google.com/run/docs/managing/job-executions - // [`CLOUD_RUN_EXECUTION`]: https://cloud.google.com/run/docs/container-contract#jobs-env-vars - GCPCloudRunJobExecutionKey = attribute.Key("gcp.cloud_run.job.execution") - - // GCPCloudRunJobTaskIndexKey is the attribute Key conforming to the - // "gcp.cloud_run.job.task_index" semantic conventions. It represents the index - // for a task within an execution as provided by the [`CLOUD_RUN_TASK_INDEX`] - // environment variable. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0, 1 - // - // [`CLOUD_RUN_TASK_INDEX`]: https://cloud.google.com/run/docs/container-contract#jobs-env-vars - GCPCloudRunJobTaskIndexKey = attribute.Key("gcp.cloud_run.job.task_index") - - // GCPGCEInstanceHostnameKey is the attribute Key conforming to the - // "gcp.gce.instance.hostname" semantic conventions. It represents the hostname - // of a GCE instance. This is the full value of the default or [custom hostname] - // . - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-host1234.example.com", - // "sample-vm.us-west1-b.c.my-project.internal" - // - // [custom hostname]: https://cloud.google.com/compute/docs/instances/custom-hostname-vm - GCPGCEInstanceHostnameKey = attribute.Key("gcp.gce.instance.hostname") - - // GCPGCEInstanceNameKey is the attribute Key conforming to the - // "gcp.gce.instance.name" semantic conventions. It represents the instance name - // of a GCE instance. This is the value provided by `host.name`, the visible - // name of the instance in the Cloud Console UI, and the prefix for the default - // hostname of the instance as defined by the [default internal DNS name]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "instance-1", "my-vm-name" - // - // [default internal DNS name]: https://cloud.google.com/compute/docs/internal-dns#instance-fully-qualified-domain-names - GCPGCEInstanceNameKey = attribute.Key("gcp.gce.instance.name") -) - -// GCPAppHubApplicationContainer returns an attribute KeyValue conforming to the -// "gcp.apphub.application.container" semantic conventions. It represents the -// container within GCP where the AppHub application is defined. -func GCPAppHubApplicationContainer(val string) attribute.KeyValue { - return GCPAppHubApplicationContainerKey.String(val) -} - -// GCPAppHubApplicationID returns an attribute KeyValue conforming to the -// "gcp.apphub.application.id" semantic conventions. It represents the name of -// the application as configured in AppHub. -func GCPAppHubApplicationID(val string) attribute.KeyValue { - return GCPAppHubApplicationIDKey.String(val) -} - -// GCPAppHubApplicationLocation returns an attribute KeyValue conforming to the -// "gcp.apphub.application.location" semantic conventions. It represents the GCP -// zone or region where the application is defined. -func GCPAppHubApplicationLocation(val string) attribute.KeyValue { - return GCPAppHubApplicationLocationKey.String(val) -} - -// GCPAppHubServiceID returns an attribute KeyValue conforming to the -// "gcp.apphub.service.id" semantic conventions. It represents the name of the -// service as configured in AppHub. -func GCPAppHubServiceID(val string) attribute.KeyValue { - return GCPAppHubServiceIDKey.String(val) -} - -// GCPAppHubWorkloadID returns an attribute KeyValue conforming to the -// "gcp.apphub.workload.id" semantic conventions. It represents the name of the -// workload as configured in AppHub. -func GCPAppHubWorkloadID(val string) attribute.KeyValue { - return GCPAppHubWorkloadIDKey.String(val) -} - -// GCPAppHubDestinationApplicationContainer returns an attribute KeyValue -// conforming to the "gcp.apphub_destination.application.container" semantic -// conventions. It represents the container within GCP where the AppHub -// destination application is defined. -func GCPAppHubDestinationApplicationContainer(val string) attribute.KeyValue { - return GCPAppHubDestinationApplicationContainerKey.String(val) -} - -// GCPAppHubDestinationApplicationID returns an attribute KeyValue conforming to -// the "gcp.apphub_destination.application.id" semantic conventions. It -// represents the name of the destination application as configured in AppHub. -func GCPAppHubDestinationApplicationID(val string) attribute.KeyValue { - return GCPAppHubDestinationApplicationIDKey.String(val) -} - -// GCPAppHubDestinationApplicationLocation returns an attribute KeyValue -// conforming to the "gcp.apphub_destination.application.location" semantic -// conventions. It represents the GCP zone or region where the destination -// application is defined. -func GCPAppHubDestinationApplicationLocation(val string) attribute.KeyValue { - return GCPAppHubDestinationApplicationLocationKey.String(val) -} - -// GCPAppHubDestinationServiceID returns an attribute KeyValue conforming to the -// "gcp.apphub_destination.service.id" semantic conventions. It represents the -// name of the destination service as configured in AppHub. -func GCPAppHubDestinationServiceID(val string) attribute.KeyValue { - return GCPAppHubDestinationServiceIDKey.String(val) -} - -// GCPAppHubDestinationWorkloadID returns an attribute KeyValue conforming to the -// "gcp.apphub_destination.workload.id" semantic conventions. It represents the -// name of the destination workload as configured in AppHub. -func GCPAppHubDestinationWorkloadID(val string) attribute.KeyValue { - return GCPAppHubDestinationWorkloadIDKey.String(val) -} - -// GCPClientService returns an attribute KeyValue conforming to the -// "gcp.client.service" semantic conventions. It represents the identifies the -// Google Cloud service for which the official client library is intended. -func GCPClientService(val string) attribute.KeyValue { - return GCPClientServiceKey.String(val) -} - -// GCPCloudRunJobExecution returns an attribute KeyValue conforming to the -// "gcp.cloud_run.job.execution" semantic conventions. It represents the name of -// the Cloud Run [execution] being run for the Job, as set by the -// [`CLOUD_RUN_EXECUTION`] environment variable. -// -// [execution]: https://cloud.google.com/run/docs/managing/job-executions -// [`CLOUD_RUN_EXECUTION`]: https://cloud.google.com/run/docs/container-contract#jobs-env-vars -func GCPCloudRunJobExecution(val string) attribute.KeyValue { - return GCPCloudRunJobExecutionKey.String(val) -} - -// GCPCloudRunJobTaskIndex returns an attribute KeyValue conforming to the -// "gcp.cloud_run.job.task_index" semantic conventions. It represents the index -// for a task within an execution as provided by the [`CLOUD_RUN_TASK_INDEX`] -// environment variable. -// -// [`CLOUD_RUN_TASK_INDEX`]: https://cloud.google.com/run/docs/container-contract#jobs-env-vars -func GCPCloudRunJobTaskIndex(val int) attribute.KeyValue { - return GCPCloudRunJobTaskIndexKey.Int(val) -} - -// GCPGCEInstanceHostname returns an attribute KeyValue conforming to the -// "gcp.gce.instance.hostname" semantic conventions. It represents the hostname -// of a GCE instance. This is the full value of the default or [custom hostname] -// . -// -// [custom hostname]: https://cloud.google.com/compute/docs/instances/custom-hostname-vm -func GCPGCEInstanceHostname(val string) attribute.KeyValue { - return GCPGCEInstanceHostnameKey.String(val) -} - -// GCPGCEInstanceName returns an attribute KeyValue conforming to the -// "gcp.gce.instance.name" semantic conventions. It represents the instance name -// of a GCE instance. This is the value provided by `host.name`, the visible name -// of the instance in the Cloud Console UI, and the prefix for the default -// hostname of the instance as defined by the [default internal DNS name]. -// -// [default internal DNS name]: https://cloud.google.com/compute/docs/internal-dns#instance-fully-qualified-domain-names -func GCPGCEInstanceName(val string) attribute.KeyValue { - return GCPGCEInstanceNameKey.String(val) -} - -// Enum values for gcp.apphub.service.criticality_type -var ( - // Mission critical service. - // Stability: development - GCPAppHubServiceCriticalityTypeMissionCritical = GCPAppHubServiceCriticalityTypeKey.String("MISSION_CRITICAL") - // High impact. - // Stability: development - GCPAppHubServiceCriticalityTypeHigh = GCPAppHubServiceCriticalityTypeKey.String("HIGH") - // Medium impact. - // Stability: development - GCPAppHubServiceCriticalityTypeMedium = GCPAppHubServiceCriticalityTypeKey.String("MEDIUM") - // Low impact. - // Stability: development - GCPAppHubServiceCriticalityTypeLow = GCPAppHubServiceCriticalityTypeKey.String("LOW") -) - -// Enum values for gcp.apphub.service.environment_type -var ( - // Production environment. - // Stability: development - GCPAppHubServiceEnvironmentTypeProduction = GCPAppHubServiceEnvironmentTypeKey.String("PRODUCTION") - // Staging environment. - // Stability: development - GCPAppHubServiceEnvironmentTypeStaging = GCPAppHubServiceEnvironmentTypeKey.String("STAGING") - // Test environment. - // Stability: development - GCPAppHubServiceEnvironmentTypeTest = GCPAppHubServiceEnvironmentTypeKey.String("TEST") - // Development environment. - // Stability: development - GCPAppHubServiceEnvironmentTypeDevelopment = GCPAppHubServiceEnvironmentTypeKey.String("DEVELOPMENT") -) - -// Enum values for gcp.apphub.workload.criticality_type -var ( - // Mission critical service. - // Stability: development - GCPAppHubWorkloadCriticalityTypeMissionCritical = GCPAppHubWorkloadCriticalityTypeKey.String("MISSION_CRITICAL") - // High impact. - // Stability: development - GCPAppHubWorkloadCriticalityTypeHigh = GCPAppHubWorkloadCriticalityTypeKey.String("HIGH") - // Medium impact. - // Stability: development - GCPAppHubWorkloadCriticalityTypeMedium = GCPAppHubWorkloadCriticalityTypeKey.String("MEDIUM") - // Low impact. - // Stability: development - GCPAppHubWorkloadCriticalityTypeLow = GCPAppHubWorkloadCriticalityTypeKey.String("LOW") -) - -// Enum values for gcp.apphub.workload.environment_type -var ( - // Production environment. - // Stability: development - GCPAppHubWorkloadEnvironmentTypeProduction = GCPAppHubWorkloadEnvironmentTypeKey.String("PRODUCTION") - // Staging environment. - // Stability: development - GCPAppHubWorkloadEnvironmentTypeStaging = GCPAppHubWorkloadEnvironmentTypeKey.String("STAGING") - // Test environment. - // Stability: development - GCPAppHubWorkloadEnvironmentTypeTest = GCPAppHubWorkloadEnvironmentTypeKey.String("TEST") - // Development environment. - // Stability: development - GCPAppHubWorkloadEnvironmentTypeDevelopment = GCPAppHubWorkloadEnvironmentTypeKey.String("DEVELOPMENT") -) - -// Enum values for gcp.apphub_destination.service.criticality_type -var ( - // Mission critical service. - // Stability: development - GCPAppHubDestinationServiceCriticalityTypeMissionCritical = GCPAppHubDestinationServiceCriticalityTypeKey.String("MISSION_CRITICAL") - // High impact. - // Stability: development - GCPAppHubDestinationServiceCriticalityTypeHigh = GCPAppHubDestinationServiceCriticalityTypeKey.String("HIGH") - // Medium impact. - // Stability: development - GCPAppHubDestinationServiceCriticalityTypeMedium = GCPAppHubDestinationServiceCriticalityTypeKey.String("MEDIUM") - // Low impact. - // Stability: development - GCPAppHubDestinationServiceCriticalityTypeLow = GCPAppHubDestinationServiceCriticalityTypeKey.String("LOW") -) - -// Enum values for gcp.apphub_destination.service.environment_type -var ( - // Production environment. - // Stability: development - GCPAppHubDestinationServiceEnvironmentTypeProduction = GCPAppHubDestinationServiceEnvironmentTypeKey.String("PRODUCTION") - // Staging environment. - // Stability: development - GCPAppHubDestinationServiceEnvironmentTypeStaging = GCPAppHubDestinationServiceEnvironmentTypeKey.String("STAGING") - // Test environment. - // Stability: development - GCPAppHubDestinationServiceEnvironmentTypeTest = GCPAppHubDestinationServiceEnvironmentTypeKey.String("TEST") - // Development environment. - // Stability: development - GCPAppHubDestinationServiceEnvironmentTypeDevelopment = GCPAppHubDestinationServiceEnvironmentTypeKey.String("DEVELOPMENT") -) - -// Enum values for gcp.apphub_destination.workload.criticality_type -var ( - // Mission critical service. - // Stability: development - GCPAppHubDestinationWorkloadCriticalityTypeMissionCritical = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("MISSION_CRITICAL") - // High impact. - // Stability: development - GCPAppHubDestinationWorkloadCriticalityTypeHigh = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("HIGH") - // Medium impact. - // Stability: development - GCPAppHubDestinationWorkloadCriticalityTypeMedium = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("MEDIUM") - // Low impact. - // Stability: development - GCPAppHubDestinationWorkloadCriticalityTypeLow = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("LOW") -) - -// Enum values for gcp.apphub_destination.workload.environment_type -var ( - // Production environment. - // Stability: development - GCPAppHubDestinationWorkloadEnvironmentTypeProduction = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("PRODUCTION") - // Staging environment. - // Stability: development - GCPAppHubDestinationWorkloadEnvironmentTypeStaging = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("STAGING") - // Test environment. - // Stability: development - GCPAppHubDestinationWorkloadEnvironmentTypeTest = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("TEST") - // Development environment. - // Stability: development - GCPAppHubDestinationWorkloadEnvironmentTypeDevelopment = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("DEVELOPMENT") -) - -// Namespace: gen_ai -const ( - // GenAIAgentDescriptionKey is the attribute Key conforming to the - // "gen_ai.agent.description" semantic conventions. It represents the free-form - // description of the GenAI agent provided by the application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Helps with math problems", "Generates fiction stories" - GenAIAgentDescriptionKey = attribute.Key("gen_ai.agent.description") - - // GenAIAgentIDKey is the attribute Key conforming to the "gen_ai.agent.id" - // semantic conventions. It represents the unique identifier of the GenAI agent. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "asst_5j66UpCpwteGg4YSxUnt7lPY" - GenAIAgentIDKey = attribute.Key("gen_ai.agent.id") - - // GenAIAgentNameKey is the attribute Key conforming to the "gen_ai.agent.name" - // semantic conventions. It represents the human-readable name of the GenAI - // agent provided by the application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Math Tutor", "Fiction Writer" - GenAIAgentNameKey = attribute.Key("gen_ai.agent.name") - - // GenAIConversationIDKey is the attribute Key conforming to the - // "gen_ai.conversation.id" semantic conventions. It represents the unique - // identifier for a conversation (session, thread), used to store and correlate - // messages within this conversation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "conv_5j66UpCpwteGg4YSxUnt7lPY" - GenAIConversationIDKey = attribute.Key("gen_ai.conversation.id") - - // GenAIDataSourceIDKey is the attribute Key conforming to the - // "gen_ai.data_source.id" semantic conventions. It represents the data source - // identifier. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "H7STPQYOND" - // Note: Data sources are used by AI agents and RAG applications to store - // grounding data. A data source may be an external database, object store, - // document collection, website, or any other storage system used by the GenAI - // agent or application. The `gen_ai.data_source.id` SHOULD match the identifier - // used by the GenAI system rather than a name specific to the external storage, - // such as a database or object store. Semantic conventions referencing - // `gen_ai.data_source.id` MAY also leverage additional attributes, such as - // `db.*`, to further identify and describe the data source. - GenAIDataSourceIDKey = attribute.Key("gen_ai.data_source.id") - - // GenAIEmbeddingsDimensionCountKey is the attribute Key conforming to the - // "gen_ai.embeddings.dimension.count" semantic conventions. It represents the - // number of dimensions the resulting output embeddings should have. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 512, 1024 - GenAIEmbeddingsDimensionCountKey = attribute.Key("gen_ai.embeddings.dimension.count") - - // GenAIEvaluationExplanationKey is the attribute Key conforming to the - // "gen_ai.evaluation.explanation" semantic conventions. It represents a - // free-form explanation for the assigned score provided by the evaluator. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "The response is factually accurate but lacks sufficient detail to - // fully address the question." - GenAIEvaluationExplanationKey = attribute.Key("gen_ai.evaluation.explanation") - - // GenAIEvaluationNameKey is the attribute Key conforming to the - // "gen_ai.evaluation.name" semantic conventions. It represents the name of the - // evaluation metric used for the GenAI response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Relevance", "IntentResolution" - GenAIEvaluationNameKey = attribute.Key("gen_ai.evaluation.name") - - // GenAIEvaluationScoreLabelKey is the attribute Key conforming to the - // "gen_ai.evaluation.score.label" semantic conventions. It represents the human - // readable label for evaluation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "relevant", "not_relevant", "correct", "incorrect", "pass", "fail" - // Note: This attribute provides a human-readable interpretation of the - // evaluation score produced by an evaluator. For example, a score value of 1 - // could mean "relevant" in one evaluation system and "not relevant" in another, - // depending on the scoring range and evaluator. The label SHOULD have low - // cardinality. Possible values depend on the evaluation metric and evaluator - // used; implementations SHOULD document the possible values. - GenAIEvaluationScoreLabelKey = attribute.Key("gen_ai.evaluation.score.label") - - // GenAIEvaluationScoreValueKey is the attribute Key conforming to the - // "gen_ai.evaluation.score.value" semantic conventions. It represents the - // evaluation score returned by the evaluator. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 4.0 - GenAIEvaluationScoreValueKey = attribute.Key("gen_ai.evaluation.score.value") - - // GenAIInputMessagesKey is the attribute Key conforming to the - // "gen_ai.input.messages" semantic conventions. It represents the chat history - // provided to the model as an input. - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "[\n {\n "role": "user",\n "parts": [\n {\n "type": "text",\n - // "content": "Weather in Paris?"\n }\n ]\n },\n {\n "role": "assistant",\n - // "parts": [\n {\n "type": "tool_call",\n "id": - // "call_VSPygqKTWdrhaFErNvMV18Yl",\n "name": "get_weather",\n "arguments": {\n - // "location": "Paris"\n }\n }\n ]\n },\n {\n "role": "tool",\n "parts": [\n {\n - // "type": "tool_call_response",\n "id": " call_VSPygqKTWdrhaFErNvMV18Yl",\n - // "result": "rainy, 57°F"\n }\n ]\n }\n]\n" - // Note: Instrumentations MUST follow [Input messages JSON schema]. - // When the attribute is recorded on events, it MUST be recorded in structured - // form. When recorded on spans, it MAY be recorded as a JSON string if - // structured - // format is not supported and SHOULD be recorded in structured form otherwise. - // - // Messages MUST be provided in the order they were sent to the model. - // Instrumentations MAY provide a way for users to filter or truncate - // input messages. - // - // > [!Warning] - // > This attribute is likely to contain sensitive information including - // > user/PII data. - // - // See [Recording content on attributes] - // section for more details. - // - // [Input messages JSON schema]: /docs/gen-ai/gen-ai-input-messages.json - // [Recording content on attributes]: /docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes - GenAIInputMessagesKey = attribute.Key("gen_ai.input.messages") - - // GenAIOperationNameKey is the attribute Key conforming to the - // "gen_ai.operation.name" semantic conventions. It represents the name of the - // operation being performed. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: If one of the predefined values applies, but specific system uses a - // different name it's RECOMMENDED to document it in the semantic conventions - // for specific GenAI system and use system-specific name in the - // instrumentation. If a different name is not documented, instrumentation - // libraries SHOULD use applicable predefined value. - GenAIOperationNameKey = attribute.Key("gen_ai.operation.name") - - // GenAIOutputMessagesKey is the attribute Key conforming to the - // "gen_ai.output.messages" semantic conventions. It represents the messages - // returned by the model where each message represents a specific model response - // (choice, candidate). - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "[\n {\n "role": "assistant",\n "parts": [\n {\n "type": "text",\n - // "content": "The weather in Paris is currently rainy with a temperature of - // 57°F."\n }\n ],\n "finish_reason": "stop"\n }\n]\n" - // Note: Instrumentations MUST follow [Output messages JSON schema] - // - // Each message represents a single output choice/candidate generated by - // the model. Each message corresponds to exactly one generation - // (choice/candidate) and vice versa - one choice cannot be split across - // multiple messages or one message cannot contain parts from multiple choices. - // - // When the attribute is recorded on events, it MUST be recorded in structured - // form. When recorded on spans, it MAY be recorded as a JSON string if - // structured - // format is not supported and SHOULD be recorded in structured form otherwise. - // - // Instrumentations MAY provide a way for users to filter or truncate - // output messages. - // - // > [!Warning] - // > This attribute is likely to contain sensitive information including - // > user/PII data. - // - // See [Recording content on attributes] - // section for more details. - // - // [Output messages JSON schema]: /docs/gen-ai/gen-ai-output-messages.json - // [Recording content on attributes]: /docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes - GenAIOutputMessagesKey = attribute.Key("gen_ai.output.messages") - - // GenAIOutputTypeKey is the attribute Key conforming to the - // "gen_ai.output.type" semantic conventions. It represents the represents the - // content type requested by the client. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: This attribute SHOULD be used when the client requests output of a - // specific type. The model may return zero or more outputs of this type. - // This attribute specifies the output modality and not the actual output - // format. For example, if an image is requested, the actual output could be a - // URL pointing to an image file. - // Additional output format details may be recorded in the future in the - // `gen_ai.output.{type}.*` attributes. - GenAIOutputTypeKey = attribute.Key("gen_ai.output.type") - - // GenAIPromptNameKey is the attribute Key conforming to the - // "gen_ai.prompt.name" semantic conventions. It represents the name of the - // prompt that uniquely identifies it. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "analyze-code" - GenAIPromptNameKey = attribute.Key("gen_ai.prompt.name") - - // GenAIProviderNameKey is the attribute Key conforming to the - // "gen_ai.provider.name" semantic conventions. It represents the Generative AI - // provider as identified by the client or server instrumentation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The attribute SHOULD be set based on the instrumentation's best - // knowledge and may differ from the actual model provider. - // - // Multiple providers, including Azure OpenAI, Gemini, and AI hosting platforms - // are accessible using the OpenAI REST API and corresponding client libraries, - // but may proxy or host models from different providers. - // - // The `gen_ai.request.model`, `gen_ai.response.model`, and `server.address` - // attributes may help identify the actual system in use. - // - // The `gen_ai.provider.name` attribute acts as a discriminator that - // identifies the GenAI telemetry format flavor specific to that provider - // within GenAI semantic conventions. - // It SHOULD be set consistently with provider-specific attributes and signals. - // For example, GenAI spans, metrics, and events related to AWS Bedrock - // should have the `gen_ai.provider.name` set to `aws.bedrock` and include - // applicable `aws.bedrock.*` attributes and are not expected to include - // `openai.*` attributes. - GenAIProviderNameKey = attribute.Key("gen_ai.provider.name") - - // GenAIRequestChoiceCountKey is the attribute Key conforming to the - // "gen_ai.request.choice.count" semantic conventions. It represents the target - // number of candidate completions to return. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 3 - GenAIRequestChoiceCountKey = attribute.Key("gen_ai.request.choice.count") - - // GenAIRequestEncodingFormatsKey is the attribute Key conforming to the - // "gen_ai.request.encoding_formats" semantic conventions. It represents the - // encoding formats requested in an embeddings operation, if specified. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "base64"], ["float", "binary" - // Note: In some GenAI systems the encoding formats are called embedding types. - // Also, some GenAI systems only accept a single format per request. - GenAIRequestEncodingFormatsKey = attribute.Key("gen_ai.request.encoding_formats") - - // GenAIRequestFrequencyPenaltyKey is the attribute Key conforming to the - // "gen_ai.request.frequency_penalty" semantic conventions. It represents the - // frequency penalty setting for the GenAI request. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0.1 - GenAIRequestFrequencyPenaltyKey = attribute.Key("gen_ai.request.frequency_penalty") - - // GenAIRequestMaxTokensKey is the attribute Key conforming to the - // "gen_ai.request.max_tokens" semantic conventions. It represents the maximum - // number of tokens the model generates for a request. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 100 - GenAIRequestMaxTokensKey = attribute.Key("gen_ai.request.max_tokens") - - // GenAIRequestModelKey is the attribute Key conforming to the - // "gen_ai.request.model" semantic conventions. It represents the name of the - // GenAI model a request is being made to. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: gpt-4 - GenAIRequestModelKey = attribute.Key("gen_ai.request.model") - - // GenAIRequestPresencePenaltyKey is the attribute Key conforming to the - // "gen_ai.request.presence_penalty" semantic conventions. It represents the - // presence penalty setting for the GenAI request. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0.1 - GenAIRequestPresencePenaltyKey = attribute.Key("gen_ai.request.presence_penalty") - - // GenAIRequestSeedKey is the attribute Key conforming to the - // "gen_ai.request.seed" semantic conventions. It represents the requests with - // same seed value more likely to return same result. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 100 - GenAIRequestSeedKey = attribute.Key("gen_ai.request.seed") - - // GenAIRequestStopSequencesKey is the attribute Key conforming to the - // "gen_ai.request.stop_sequences" semantic conventions. It represents the list - // of sequences that the model will use to stop generating further tokens. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "forest", "lived" - GenAIRequestStopSequencesKey = attribute.Key("gen_ai.request.stop_sequences") - - // GenAIRequestTemperatureKey is the attribute Key conforming to the - // "gen_ai.request.temperature" semantic conventions. It represents the - // temperature setting for the GenAI request. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0.0 - GenAIRequestTemperatureKey = attribute.Key("gen_ai.request.temperature") - - // GenAIRequestTopKKey is the attribute Key conforming to the - // "gen_ai.request.top_k" semantic conventions. It represents the top_k sampling - // setting for the GenAI request. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0 - GenAIRequestTopKKey = attribute.Key("gen_ai.request.top_k") - - // GenAIRequestTopPKey is the attribute Key conforming to the - // "gen_ai.request.top_p" semantic conventions. It represents the top_p sampling - // setting for the GenAI request. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1.0 - GenAIRequestTopPKey = attribute.Key("gen_ai.request.top_p") - - // GenAIResponseFinishReasonsKey is the attribute Key conforming to the - // "gen_ai.response.finish_reasons" semantic conventions. It represents the - // array of reasons the model stopped generating tokens, corresponding to each - // generation received. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "stop"], ["stop", "length" - GenAIResponseFinishReasonsKey = attribute.Key("gen_ai.response.finish_reasons") - - // GenAIResponseIDKey is the attribute Key conforming to the - // "gen_ai.response.id" semantic conventions. It represents the unique - // identifier for the completion. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "chatcmpl-123" - GenAIResponseIDKey = attribute.Key("gen_ai.response.id") - - // GenAIResponseModelKey is the attribute Key conforming to the - // "gen_ai.response.model" semantic conventions. It represents the name of the - // model that generated the response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "gpt-4-0613" - GenAIResponseModelKey = attribute.Key("gen_ai.response.model") - - // GenAISystemInstructionsKey is the attribute Key conforming to the - // "gen_ai.system_instructions" semantic conventions. It represents the system - // message or instructions provided to the GenAI model separately from the chat - // history. - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "[\n {\n "type": "text",\n "content": "You are an Agent that greet - // users, always use greetings tool to respond"\n }\n]\n", "[\n {\n "type": - // "text",\n "content": "You are a language translator."\n },\n {\n "type": - // "text",\n "content": "Your mission is to translate text in English to - // French."\n }\n]\n" - // Note: This attribute SHOULD be used when the corresponding provider or API - // allows to provide system instructions or messages separately from the - // chat history. - // - // Instructions that are part of the chat history SHOULD be recorded in - // `gen_ai.input.messages` attribute instead. - // - // Instrumentations MUST follow [System instructions JSON schema]. - // - // When recorded on spans, it MAY be recorded as a JSON string if structured - // format is not supported and SHOULD be recorded in structured form otherwise. - // - // Instrumentations MAY provide a way for users to filter or truncate - // system instructions. - // - // > [!Warning] - // > This attribute may contain sensitive information. - // - // See [Recording content on attributes] - // section for more details. - // - // [System instructions JSON schema]: /docs/gen-ai/gen-ai-system-instructions.json - // [Recording content on attributes]: /docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes - GenAISystemInstructionsKey = attribute.Key("gen_ai.system_instructions") - - // GenAITokenTypeKey is the attribute Key conforming to the "gen_ai.token.type" - // semantic conventions. It represents the type of token being counted. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "input", "output" - GenAITokenTypeKey = attribute.Key("gen_ai.token.type") - - // GenAIToolCallArgumentsKey is the attribute Key conforming to the - // "gen_ai.tool.call.arguments" semantic conventions. It represents the - // parameters passed to the tool call. - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{\n "location": "San Francisco?",\n "date": "2025-10-01"\n}\n" - // Note: > [!WARNING] - // - // > This attribute may contain sensitive information. - // - // It's expected to be an object - in case a serialized string is available - // to the instrumentation, the instrumentation SHOULD do the best effort to - // deserialize it to an object. When recorded on spans, it MAY be recorded as a - // JSON string if structured format is not supported and SHOULD be recorded in - // structured form otherwise. - GenAIToolCallArgumentsKey = attribute.Key("gen_ai.tool.call.arguments") - - // GenAIToolCallIDKey is the attribute Key conforming to the - // "gen_ai.tool.call.id" semantic conventions. It represents the tool call - // identifier. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "call_mszuSIzqtI65i1wAUOE8w5H4" - GenAIToolCallIDKey = attribute.Key("gen_ai.tool.call.id") - - // GenAIToolCallResultKey is the attribute Key conforming to the - // "gen_ai.tool.call.result" semantic conventions. It represents the result - // returned by the tool call (if any and if execution was successful). - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "{\n "temperature_range": {\n "high": 75,\n "low": 60\n },\n - // "conditions": "sunny"\n}\n" - // Note: > [!WARNING] - // - // > This attribute may contain sensitive information. - // - // It's expected to be an object - in case a serialized string is available - // to the instrumentation, the instrumentation SHOULD do the best effort to - // deserialize it to an object. When recorded on spans, it MAY be recorded as a - // JSON string if structured format is not supported and SHOULD be recorded in - // structured form otherwise. - GenAIToolCallResultKey = attribute.Key("gen_ai.tool.call.result") - - // GenAIToolDefinitionsKey is the attribute Key conforming to the - // "gen_ai.tool.definitions" semantic conventions. It represents the list of - // source system tool definitions available to the GenAI agent or model. - // - // Type: any - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "[\n {\n "type": "function",\n "name": "get_current_weather",\n - // "description": "Get the current weather in a given location",\n "parameters": - // {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n - // "description": "The city and state, e.g. San Francisco, CA"\n },\n "unit": - // {\n "type": "string",\n "enum": [\n "celsius",\n "fahrenheit"\n ]\n }\n },\n - // "required": [\n "location",\n "unit"\n ]\n }\n }\n]\n" - // Note: The value of this attribute matches source system tool definition - // format. - // - // It's expected to be an array of objects where each object represents a tool - // definition. In case a serialized string is available - // to the instrumentation, the instrumentation SHOULD do the best effort to - // deserialize it to an array. When recorded on spans, it MAY be recorded as a - // JSON string if structured format is not supported and SHOULD be recorded in - // structured form otherwise. - // - // Since this attribute could be large, it's NOT RECOMMENDED to populate - // it by default. Instrumentations MAY provide a way to enable - // populating this attribute. - GenAIToolDefinitionsKey = attribute.Key("gen_ai.tool.definitions") - - // GenAIToolDescriptionKey is the attribute Key conforming to the - // "gen_ai.tool.description" semantic conventions. It represents the tool - // description. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Multiply two numbers" - GenAIToolDescriptionKey = attribute.Key("gen_ai.tool.description") - - // GenAIToolNameKey is the attribute Key conforming to the "gen_ai.tool.name" - // semantic conventions. It represents the name of the tool utilized by the - // agent. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Flights" - GenAIToolNameKey = attribute.Key("gen_ai.tool.name") - - // GenAIToolTypeKey is the attribute Key conforming to the "gen_ai.tool.type" - // semantic conventions. It represents the type of the tool utilized by the - // agent. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "function", "extension", "datastore" - // Note: Extension: A tool executed on the agent-side to directly call external - // APIs, bridging the gap between the agent and real-world systems. - // Agent-side operations involve actions that are performed by the agent on the - // server or within the agent's controlled environment. - // Function: A tool executed on the client-side, where the agent generates - // parameters for a predefined function, and the client executes the logic. - // Client-side operations are actions taken on the user's end or within the - // client application. - // Datastore: A tool used by the agent to access and query structured or - // unstructured external data for retrieval-augmented tasks or knowledge - // updates. - GenAIToolTypeKey = attribute.Key("gen_ai.tool.type") - - // GenAIUsageInputTokensKey is the attribute Key conforming to the - // "gen_ai.usage.input_tokens" semantic conventions. It represents the number of - // tokens used in the GenAI input (prompt). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 100 - GenAIUsageInputTokensKey = attribute.Key("gen_ai.usage.input_tokens") - - // GenAIUsageOutputTokensKey is the attribute Key conforming to the - // "gen_ai.usage.output_tokens" semantic conventions. It represents the number - // of tokens used in the GenAI response (completion). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 180 - GenAIUsageOutputTokensKey = attribute.Key("gen_ai.usage.output_tokens") -) - -// GenAIAgentDescription returns an attribute KeyValue conforming to the -// "gen_ai.agent.description" semantic conventions. It represents the free-form -// description of the GenAI agent provided by the application. -func GenAIAgentDescription(val string) attribute.KeyValue { - return GenAIAgentDescriptionKey.String(val) -} - -// GenAIAgentID returns an attribute KeyValue conforming to the "gen_ai.agent.id" -// semantic conventions. It represents the unique identifier of the GenAI agent. -func GenAIAgentID(val string) attribute.KeyValue { - return GenAIAgentIDKey.String(val) -} - -// GenAIAgentName returns an attribute KeyValue conforming to the -// "gen_ai.agent.name" semantic conventions. It represents the human-readable -// name of the GenAI agent provided by the application. -func GenAIAgentName(val string) attribute.KeyValue { - return GenAIAgentNameKey.String(val) -} - -// GenAIConversationID returns an attribute KeyValue conforming to the -// "gen_ai.conversation.id" semantic conventions. It represents the unique -// identifier for a conversation (session, thread), used to store and correlate -// messages within this conversation. -func GenAIConversationID(val string) attribute.KeyValue { - return GenAIConversationIDKey.String(val) -} - -// GenAIDataSourceID returns an attribute KeyValue conforming to the -// "gen_ai.data_source.id" semantic conventions. It represents the data source -// identifier. -func GenAIDataSourceID(val string) attribute.KeyValue { - return GenAIDataSourceIDKey.String(val) -} - -// GenAIEmbeddingsDimensionCount returns an attribute KeyValue conforming to the -// "gen_ai.embeddings.dimension.count" semantic conventions. It represents the -// number of dimensions the resulting output embeddings should have. -func GenAIEmbeddingsDimensionCount(val int) attribute.KeyValue { - return GenAIEmbeddingsDimensionCountKey.Int(val) -} - -// GenAIEvaluationExplanation returns an attribute KeyValue conforming to the -// "gen_ai.evaluation.explanation" semantic conventions. It represents a -// free-form explanation for the assigned score provided by the evaluator. -func GenAIEvaluationExplanation(val string) attribute.KeyValue { - return GenAIEvaluationExplanationKey.String(val) -} - -// GenAIEvaluationName returns an attribute KeyValue conforming to the -// "gen_ai.evaluation.name" semantic conventions. It represents the name of the -// evaluation metric used for the GenAI response. -func GenAIEvaluationName(val string) attribute.KeyValue { - return GenAIEvaluationNameKey.String(val) -} - -// GenAIEvaluationScoreLabel returns an attribute KeyValue conforming to the -// "gen_ai.evaluation.score.label" semantic conventions. It represents the human -// readable label for evaluation. -func GenAIEvaluationScoreLabel(val string) attribute.KeyValue { - return GenAIEvaluationScoreLabelKey.String(val) -} - -// GenAIEvaluationScoreValue returns an attribute KeyValue conforming to the -// "gen_ai.evaluation.score.value" semantic conventions. It represents the -// evaluation score returned by the evaluator. -func GenAIEvaluationScoreValue(val float64) attribute.KeyValue { - return GenAIEvaluationScoreValueKey.Float64(val) -} - -// GenAIPromptName returns an attribute KeyValue conforming to the -// "gen_ai.prompt.name" semantic conventions. It represents the name of the -// prompt that uniquely identifies it. -func GenAIPromptName(val string) attribute.KeyValue { - return GenAIPromptNameKey.String(val) -} - -// GenAIRequestChoiceCount returns an attribute KeyValue conforming to the -// "gen_ai.request.choice.count" semantic conventions. It represents the target -// number of candidate completions to return. -func GenAIRequestChoiceCount(val int) attribute.KeyValue { - return GenAIRequestChoiceCountKey.Int(val) -} - -// GenAIRequestEncodingFormats returns an attribute KeyValue conforming to the -// "gen_ai.request.encoding_formats" semantic conventions. It represents the -// encoding formats requested in an embeddings operation, if specified. -func GenAIRequestEncodingFormats(val ...string) attribute.KeyValue { - return GenAIRequestEncodingFormatsKey.StringSlice(val) -} - -// GenAIRequestFrequencyPenalty returns an attribute KeyValue conforming to the -// "gen_ai.request.frequency_penalty" semantic conventions. It represents the -// frequency penalty setting for the GenAI request. -func GenAIRequestFrequencyPenalty(val float64) attribute.KeyValue { - return GenAIRequestFrequencyPenaltyKey.Float64(val) -} - -// GenAIRequestMaxTokens returns an attribute KeyValue conforming to the -// "gen_ai.request.max_tokens" semantic conventions. It represents the maximum -// number of tokens the model generates for a request. -func GenAIRequestMaxTokens(val int) attribute.KeyValue { - return GenAIRequestMaxTokensKey.Int(val) -} - -// GenAIRequestModel returns an attribute KeyValue conforming to the -// "gen_ai.request.model" semantic conventions. It represents the name of the -// GenAI model a request is being made to. -func GenAIRequestModel(val string) attribute.KeyValue { - return GenAIRequestModelKey.String(val) -} - -// GenAIRequestPresencePenalty returns an attribute KeyValue conforming to the -// "gen_ai.request.presence_penalty" semantic conventions. It represents the -// presence penalty setting for the GenAI request. -func GenAIRequestPresencePenalty(val float64) attribute.KeyValue { - return GenAIRequestPresencePenaltyKey.Float64(val) -} - -// GenAIRequestSeed returns an attribute KeyValue conforming to the -// "gen_ai.request.seed" semantic conventions. It represents the requests with -// same seed value more likely to return same result. -func GenAIRequestSeed(val int) attribute.KeyValue { - return GenAIRequestSeedKey.Int(val) -} - -// GenAIRequestStopSequences returns an attribute KeyValue conforming to the -// "gen_ai.request.stop_sequences" semantic conventions. It represents the list -// of sequences that the model will use to stop generating further tokens. -func GenAIRequestStopSequences(val ...string) attribute.KeyValue { - return GenAIRequestStopSequencesKey.StringSlice(val) -} - -// GenAIRequestTemperature returns an attribute KeyValue conforming to the -// "gen_ai.request.temperature" semantic conventions. It represents the -// temperature setting for the GenAI request. -func GenAIRequestTemperature(val float64) attribute.KeyValue { - return GenAIRequestTemperatureKey.Float64(val) -} - -// GenAIRequestTopK returns an attribute KeyValue conforming to the -// "gen_ai.request.top_k" semantic conventions. It represents the top_k sampling -// setting for the GenAI request. -func GenAIRequestTopK(val float64) attribute.KeyValue { - return GenAIRequestTopKKey.Float64(val) -} - -// GenAIRequestTopP returns an attribute KeyValue conforming to the -// "gen_ai.request.top_p" semantic conventions. It represents the top_p sampling -// setting for the GenAI request. -func GenAIRequestTopP(val float64) attribute.KeyValue { - return GenAIRequestTopPKey.Float64(val) -} - -// GenAIResponseFinishReasons returns an attribute KeyValue conforming to the -// "gen_ai.response.finish_reasons" semantic conventions. It represents the array -// of reasons the model stopped generating tokens, corresponding to each -// generation received. -func GenAIResponseFinishReasons(val ...string) attribute.KeyValue { - return GenAIResponseFinishReasonsKey.StringSlice(val) -} - -// GenAIResponseID returns an attribute KeyValue conforming to the -// "gen_ai.response.id" semantic conventions. It represents the unique identifier -// for the completion. -func GenAIResponseID(val string) attribute.KeyValue { - return GenAIResponseIDKey.String(val) -} - -// GenAIResponseModel returns an attribute KeyValue conforming to the -// "gen_ai.response.model" semantic conventions. It represents the name of the -// model that generated the response. -func GenAIResponseModel(val string) attribute.KeyValue { - return GenAIResponseModelKey.String(val) -} - -// GenAIToolCallID returns an attribute KeyValue conforming to the -// "gen_ai.tool.call.id" semantic conventions. It represents the tool call -// identifier. -func GenAIToolCallID(val string) attribute.KeyValue { - return GenAIToolCallIDKey.String(val) -} - -// GenAIToolDescription returns an attribute KeyValue conforming to the -// "gen_ai.tool.description" semantic conventions. It represents the tool -// description. -func GenAIToolDescription(val string) attribute.KeyValue { - return GenAIToolDescriptionKey.String(val) -} - -// GenAIToolName returns an attribute KeyValue conforming to the -// "gen_ai.tool.name" semantic conventions. It represents the name of the tool -// utilized by the agent. -func GenAIToolName(val string) attribute.KeyValue { - return GenAIToolNameKey.String(val) -} - -// GenAIToolType returns an attribute KeyValue conforming to the -// "gen_ai.tool.type" semantic conventions. It represents the type of the tool -// utilized by the agent. -func GenAIToolType(val string) attribute.KeyValue { - return GenAIToolTypeKey.String(val) -} - -// GenAIUsageInputTokens returns an attribute KeyValue conforming to the -// "gen_ai.usage.input_tokens" semantic conventions. It represents the number of -// tokens used in the GenAI input (prompt). -func GenAIUsageInputTokens(val int) attribute.KeyValue { - return GenAIUsageInputTokensKey.Int(val) -} - -// GenAIUsageOutputTokens returns an attribute KeyValue conforming to the -// "gen_ai.usage.output_tokens" semantic conventions. It represents the number of -// tokens used in the GenAI response (completion). -func GenAIUsageOutputTokens(val int) attribute.KeyValue { - return GenAIUsageOutputTokensKey.Int(val) -} - -// Enum values for gen_ai.operation.name -var ( - // Chat completion operation such as [OpenAI Chat API] - // Stability: development - // - // [OpenAI Chat API]: https://platform.openai.com/docs/api-reference/chat - GenAIOperationNameChat = GenAIOperationNameKey.String("chat") - // Multimodal content generation operation such as [Gemini Generate Content] - // Stability: development - // - // [Gemini Generate Content]: https://ai.google.dev/api/generate-content - GenAIOperationNameGenerateContent = GenAIOperationNameKey.String("generate_content") - // Text completions operation such as [OpenAI Completions API (Legacy)] - // Stability: development - // - // [OpenAI Completions API (Legacy)]: https://platform.openai.com/docs/api-reference/completions - GenAIOperationNameTextCompletion = GenAIOperationNameKey.String("text_completion") - // Embeddings operation such as [OpenAI Create embeddings API] - // Stability: development - // - // [OpenAI Create embeddings API]: https://platform.openai.com/docs/api-reference/embeddings/create - GenAIOperationNameEmbeddings = GenAIOperationNameKey.String("embeddings") - // Create GenAI agent - // Stability: development - GenAIOperationNameCreateAgent = GenAIOperationNameKey.String("create_agent") - // Invoke GenAI agent - // Stability: development - GenAIOperationNameInvokeAgent = GenAIOperationNameKey.String("invoke_agent") - // Execute a tool - // Stability: development - GenAIOperationNameExecuteTool = GenAIOperationNameKey.String("execute_tool") -) - -// Enum values for gen_ai.output.type -var ( - // Plain text - // Stability: development - GenAIOutputTypeText = GenAIOutputTypeKey.String("text") - // JSON object with known or unknown schema - // Stability: development - GenAIOutputTypeJSON = GenAIOutputTypeKey.String("json") - // Image - // Stability: development - GenAIOutputTypeImage = GenAIOutputTypeKey.String("image") - // Speech - // Stability: development - GenAIOutputTypeSpeech = GenAIOutputTypeKey.String("speech") -) - -// Enum values for gen_ai.provider.name -var ( - // [OpenAI] - // Stability: development - // - // [OpenAI]: https://openai.com/ - GenAIProviderNameOpenAI = GenAIProviderNameKey.String("openai") - // Any Google generative AI endpoint - // Stability: development - GenAIProviderNameGCPGenAI = GenAIProviderNameKey.String("gcp.gen_ai") - // [Vertex AI] - // Stability: development - // - // [Vertex AI]: https://cloud.google.com/vertex-ai - GenAIProviderNameGCPVertexAI = GenAIProviderNameKey.String("gcp.vertex_ai") - // [Gemini] - // Stability: development - // - // [Gemini]: https://cloud.google.com/products/gemini - GenAIProviderNameGCPGemini = GenAIProviderNameKey.String("gcp.gemini") - // [Anthropic] - // Stability: development - // - // [Anthropic]: https://www.anthropic.com/ - GenAIProviderNameAnthropic = GenAIProviderNameKey.String("anthropic") - // [Cohere] - // Stability: development - // - // [Cohere]: https://cohere.com/ - GenAIProviderNameCohere = GenAIProviderNameKey.String("cohere") - // Azure AI Inference - // Stability: development - GenAIProviderNameAzureAIInference = GenAIProviderNameKey.String("azure.ai.inference") - // [Azure OpenAI] - // Stability: development - // - // [Azure OpenAI]: https://azure.microsoft.com/products/ai-services/openai-service/ - GenAIProviderNameAzureAIOpenAI = GenAIProviderNameKey.String("azure.ai.openai") - // [IBM Watsonx AI] - // Stability: development - // - // [IBM Watsonx AI]: https://www.ibm.com/products/watsonx-ai - GenAIProviderNameIBMWatsonxAI = GenAIProviderNameKey.String("ibm.watsonx.ai") - // [AWS Bedrock] - // Stability: development - // - // [AWS Bedrock]: https://aws.amazon.com/bedrock - GenAIProviderNameAWSBedrock = GenAIProviderNameKey.String("aws.bedrock") - // [Perplexity] - // Stability: development - // - // [Perplexity]: https://www.perplexity.ai/ - GenAIProviderNamePerplexity = GenAIProviderNameKey.String("perplexity") - // [xAI] - // Stability: development - // - // [xAI]: https://x.ai/ - GenAIProviderNameXAI = GenAIProviderNameKey.String("x_ai") - // [DeepSeek] - // Stability: development - // - // [DeepSeek]: https://www.deepseek.com/ - GenAIProviderNameDeepseek = GenAIProviderNameKey.String("deepseek") - // [Groq] - // Stability: development - // - // [Groq]: https://groq.com/ - GenAIProviderNameGroq = GenAIProviderNameKey.String("groq") - // [Mistral AI] - // Stability: development - // - // [Mistral AI]: https://mistral.ai/ - GenAIProviderNameMistralAI = GenAIProviderNameKey.String("mistral_ai") -) - -// Enum values for gen_ai.token.type -var ( - // Input tokens (prompt, input, etc.) - // Stability: development - GenAITokenTypeInput = GenAITokenTypeKey.String("input") - // Output tokens (completion, response, etc.) - // Stability: development - GenAITokenTypeOutput = GenAITokenTypeKey.String("output") -) - -// Namespace: geo -const ( - // GeoContinentCodeKey is the attribute Key conforming to the - // "geo.continent.code" semantic conventions. It represents the two-letter code - // representing continent’s name. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - GeoContinentCodeKey = attribute.Key("geo.continent.code") - - // GeoCountryISOCodeKey is the attribute Key conforming to the - // "geo.country.iso_code" semantic conventions. It represents the two-letter ISO - // Country Code ([ISO 3166-1 alpha2]). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CA" - // - // [ISO 3166-1 alpha2]: https://wikipedia.org/wiki/ISO_3166-1#Codes - GeoCountryISOCodeKey = attribute.Key("geo.country.iso_code") - - // GeoLocalityNameKey is the attribute Key conforming to the "geo.locality.name" - // semantic conventions. It represents the locality name. Represents the name of - // a city, town, village, or similar populated place. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Montreal", "Berlin" - GeoLocalityNameKey = attribute.Key("geo.locality.name") - - // GeoLocationLatKey is the attribute Key conforming to the "geo.location.lat" - // semantic conventions. It represents the latitude of the geo location in - // [WGS84]. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 45.505918 - // - // [WGS84]: https://wikipedia.org/wiki/World_Geodetic_System#WGS84 - GeoLocationLatKey = attribute.Key("geo.location.lat") - - // GeoLocationLonKey is the attribute Key conforming to the "geo.location.lon" - // semantic conventions. It represents the longitude of the geo location in - // [WGS84]. - // - // Type: double - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: -73.61483 - // - // [WGS84]: https://wikipedia.org/wiki/World_Geodetic_System#WGS84 - GeoLocationLonKey = attribute.Key("geo.location.lon") - - // GeoPostalCodeKey is the attribute Key conforming to the "geo.postal_code" - // semantic conventions. It represents the postal code associated with the - // location. Values appropriate for this field may also be known as a postcode - // or ZIP code and will vary widely from country to country. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "94040" - GeoPostalCodeKey = attribute.Key("geo.postal_code") - - // GeoRegionISOCodeKey is the attribute Key conforming to the - // "geo.region.iso_code" semantic conventions. It represents the region ISO code - // ([ISO 3166-2]). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CA-QC" - // - // [ISO 3166-2]: https://wikipedia.org/wiki/ISO_3166-2 - GeoRegionISOCodeKey = attribute.Key("geo.region.iso_code") -) - -// GeoCountryISOCode returns an attribute KeyValue conforming to the -// "geo.country.iso_code" semantic conventions. It represents the two-letter ISO -// Country Code ([ISO 3166-1 alpha2]). -// -// [ISO 3166-1 alpha2]: https://wikipedia.org/wiki/ISO_3166-1#Codes -func GeoCountryISOCode(val string) attribute.KeyValue { - return GeoCountryISOCodeKey.String(val) -} - -// GeoLocalityName returns an attribute KeyValue conforming to the -// "geo.locality.name" semantic conventions. It represents the locality name. -// Represents the name of a city, town, village, or similar populated place. -func GeoLocalityName(val string) attribute.KeyValue { - return GeoLocalityNameKey.String(val) -} - -// GeoLocationLat returns an attribute KeyValue conforming to the -// "geo.location.lat" semantic conventions. It represents the latitude of the geo -// location in [WGS84]. -// -// [WGS84]: https://wikipedia.org/wiki/World_Geodetic_System#WGS84 -func GeoLocationLat(val float64) attribute.KeyValue { - return GeoLocationLatKey.Float64(val) -} - -// GeoLocationLon returns an attribute KeyValue conforming to the -// "geo.location.lon" semantic conventions. It represents the longitude of the -// geo location in [WGS84]. -// -// [WGS84]: https://wikipedia.org/wiki/World_Geodetic_System#WGS84 -func GeoLocationLon(val float64) attribute.KeyValue { - return GeoLocationLonKey.Float64(val) -} - -// GeoPostalCode returns an attribute KeyValue conforming to the -// "geo.postal_code" semantic conventions. It represents the postal code -// associated with the location. Values appropriate for this field may also be -// known as a postcode or ZIP code and will vary widely from country to country. -func GeoPostalCode(val string) attribute.KeyValue { - return GeoPostalCodeKey.String(val) -} - -// GeoRegionISOCode returns an attribute KeyValue conforming to the -// "geo.region.iso_code" semantic conventions. It represents the region ISO code -// ([ISO 3166-2]). -// -// [ISO 3166-2]: https://wikipedia.org/wiki/ISO_3166-2 -func GeoRegionISOCode(val string) attribute.KeyValue { - return GeoRegionISOCodeKey.String(val) -} - -// Enum values for geo.continent.code -var ( - // Africa - // Stability: development - GeoContinentCodeAf = GeoContinentCodeKey.String("AF") - // Antarctica - // Stability: development - GeoContinentCodeAn = GeoContinentCodeKey.String("AN") - // Asia - // Stability: development - GeoContinentCodeAs = GeoContinentCodeKey.String("AS") - // Europe - // Stability: development - GeoContinentCodeEu = GeoContinentCodeKey.String("EU") - // North America - // Stability: development - GeoContinentCodeNa = GeoContinentCodeKey.String("NA") - // Oceania - // Stability: development - GeoContinentCodeOc = GeoContinentCodeKey.String("OC") - // South America - // Stability: development - GeoContinentCodeSa = GeoContinentCodeKey.String("SA") -) - -// Namespace: go -const ( - // GoMemoryTypeKey is the attribute Key conforming to the "go.memory.type" - // semantic conventions. It represents the type of memory. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "other", "stack" - GoMemoryTypeKey = attribute.Key("go.memory.type") -) - -// Enum values for go.memory.type -var ( - // Memory allocated from the heap that is reserved for stack space, whether or - // not it is currently in-use. - // Stability: development - GoMemoryTypeStack = GoMemoryTypeKey.String("stack") - // Memory used by the Go runtime, excluding other categories of memory usage - // described in this enumeration. - // Stability: development - GoMemoryTypeOther = GoMemoryTypeKey.String("other") -) - -// Namespace: graphql -const ( - // GraphQLDocumentKey is the attribute Key conforming to the "graphql.document" - // semantic conventions. It represents the GraphQL document being executed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: query findBookById { bookById(id: ?) { name } } - // Note: The value may be sanitized to exclude sensitive information. - GraphQLDocumentKey = attribute.Key("graphql.document") - - // GraphQLOperationNameKey is the attribute Key conforming to the - // "graphql.operation.name" semantic conventions. It represents the name of the - // operation being executed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: findBookById - GraphQLOperationNameKey = attribute.Key("graphql.operation.name") - - // GraphQLOperationTypeKey is the attribute Key conforming to the - // "graphql.operation.type" semantic conventions. It represents the type of the - // operation being executed. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "query", "mutation", "subscription" - GraphQLOperationTypeKey = attribute.Key("graphql.operation.type") -) - -// GraphQLDocument returns an attribute KeyValue conforming to the -// "graphql.document" semantic conventions. It represents the GraphQL document -// being executed. -func GraphQLDocument(val string) attribute.KeyValue { - return GraphQLDocumentKey.String(val) -} - -// GraphQLOperationName returns an attribute KeyValue conforming to the -// "graphql.operation.name" semantic conventions. It represents the name of the -// operation being executed. -func GraphQLOperationName(val string) attribute.KeyValue { - return GraphQLOperationNameKey.String(val) -} - -// Enum values for graphql.operation.type -var ( - // GraphQL query - // Stability: development - GraphQLOperationTypeQuery = GraphQLOperationTypeKey.String("query") - // GraphQL mutation - // Stability: development - GraphQLOperationTypeMutation = GraphQLOperationTypeKey.String("mutation") - // GraphQL subscription - // Stability: development - GraphQLOperationTypeSubscription = GraphQLOperationTypeKey.String("subscription") -) - -// Namespace: heroku -const ( - // HerokuAppIDKey is the attribute Key conforming to the "heroku.app.id" - // semantic conventions. It represents the unique identifier for the - // application. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2daa2797-e42b-4624-9322-ec3f968df4da" - HerokuAppIDKey = attribute.Key("heroku.app.id") - - // HerokuReleaseCommitKey is the attribute Key conforming to the - // "heroku.release.commit" semantic conventions. It represents the commit hash - // for the current release. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "e6134959463efd8966b20e75b913cafe3f5ec" - HerokuReleaseCommitKey = attribute.Key("heroku.release.commit") - - // HerokuReleaseCreationTimestampKey is the attribute Key conforming to the - // "heroku.release.creation_timestamp" semantic conventions. It represents the - // time and date the release was created. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2022-10-23T18:00:42Z" - HerokuReleaseCreationTimestampKey = attribute.Key("heroku.release.creation_timestamp") -) - -// HerokuAppID returns an attribute KeyValue conforming to the "heroku.app.id" -// semantic conventions. It represents the unique identifier for the application. -func HerokuAppID(val string) attribute.KeyValue { - return HerokuAppIDKey.String(val) -} - -// HerokuReleaseCommit returns an attribute KeyValue conforming to the -// "heroku.release.commit" semantic conventions. It represents the commit hash -// for the current release. -func HerokuReleaseCommit(val string) attribute.KeyValue { - return HerokuReleaseCommitKey.String(val) -} - -// HerokuReleaseCreationTimestamp returns an attribute KeyValue conforming to the -// "heroku.release.creation_timestamp" semantic conventions. It represents the -// time and date the release was created. -func HerokuReleaseCreationTimestamp(val string) attribute.KeyValue { - return HerokuReleaseCreationTimestampKey.String(val) -} - -// Namespace: host -const ( - // HostArchKey is the attribute Key conforming to the "host.arch" semantic - // conventions. It represents the CPU architecture the host system is running - // on. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HostArchKey = attribute.Key("host.arch") - - // HostCPUCacheL2SizeKey is the attribute Key conforming to the - // "host.cpu.cache.l2.size" semantic conventions. It represents the amount of - // level 2 memory cache available to the processor (in Bytes). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 12288000 - HostCPUCacheL2SizeKey = attribute.Key("host.cpu.cache.l2.size") - - // HostCPUFamilyKey is the attribute Key conforming to the "host.cpu.family" - // semantic conventions. It represents the family or generation of the CPU. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "6", "PA-RISC 1.1e" - HostCPUFamilyKey = attribute.Key("host.cpu.family") - - // HostCPUModelIDKey is the attribute Key conforming to the "host.cpu.model.id" - // semantic conventions. It represents the model identifier. It provides more - // granular information about the CPU, distinguishing it from other CPUs within - // the same family. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "6", "9000/778/B180L" - HostCPUModelIDKey = attribute.Key("host.cpu.model.id") - - // HostCPUModelNameKey is the attribute Key conforming to the - // "host.cpu.model.name" semantic conventions. It represents the model - // designation of the processor. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz" - HostCPUModelNameKey = attribute.Key("host.cpu.model.name") - - // HostCPUSteppingKey is the attribute Key conforming to the "host.cpu.stepping" - // semantic conventions. It represents the stepping or core revisions. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1", "r1p1" - HostCPUSteppingKey = attribute.Key("host.cpu.stepping") - - // HostCPUVendorIDKey is the attribute Key conforming to the - // "host.cpu.vendor.id" semantic conventions. It represents the processor - // manufacturer identifier. A maximum 12-character string. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "GenuineIntel" - // Note: [CPUID] command returns the vendor ID string in EBX, EDX and ECX - // registers. Writing these to memory in this order results in a 12-character - // string. - // - // [CPUID]: https://wiki.osdev.org/CPUID - HostCPUVendorIDKey = attribute.Key("host.cpu.vendor.id") - - // HostIDKey is the attribute Key conforming to the "host.id" semantic - // conventions. It represents the unique host ID. For Cloud, this must be the - // instance_id assigned by the cloud provider. For non-containerized systems, - // this should be the `machine-id`. See the table below for the sources to use - // to determine the `machine-id` based on operating system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "fdbf79e8af94cb7f9e8df36789187052" - HostIDKey = attribute.Key("host.id") - - // HostImageIDKey is the attribute Key conforming to the "host.image.id" - // semantic conventions. It represents the VM image ID or host OS image ID. For - // Cloud, this value is from the provider. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ami-07b06b442921831e5" - HostImageIDKey = attribute.Key("host.image.id") - - // HostImageNameKey is the attribute Key conforming to the "host.image.name" - // semantic conventions. It represents the name of the VM image or OS install - // the host was instantiated from. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "infra-ami-eks-worker-node-7d4ec78312", "CentOS-8-x86_64-1905" - HostImageNameKey = attribute.Key("host.image.name") - - // HostImageVersionKey is the attribute Key conforming to the - // "host.image.version" semantic conventions. It represents the version string - // of the VM image or host OS as defined in [Version Attributes]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0.1" - // - // [Version Attributes]: /docs/resource/README.md#version-attributes - HostImageVersionKey = attribute.Key("host.image.version") - - // HostIPKey is the attribute Key conforming to the "host.ip" semantic - // conventions. It represents the available IP addresses of the host, excluding - // loopback interfaces. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "192.168.1.140", "fe80::abc2:4a28:737a:609e" - // Note: IPv4 Addresses MUST be specified in dotted-quad notation. IPv6 - // addresses MUST be specified in the [RFC 5952] format. - // - // [RFC 5952]: https://www.rfc-editor.org/rfc/rfc5952.html - HostIPKey = attribute.Key("host.ip") - - // HostMacKey is the attribute Key conforming to the "host.mac" semantic - // conventions. It represents the available MAC addresses of the host, excluding - // loopback interfaces. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "AC-DE-48-23-45-67", "AC-DE-48-23-45-67-01-9F" - // Note: MAC Addresses MUST be represented in [IEEE RA hexadecimal form]: as - // hyphen-separated octets in uppercase hexadecimal form from most to least - // significant. - // - // [IEEE RA hexadecimal form]: https://standards.ieee.org/wp-content/uploads/import/documents/tutorials/eui.pdf - HostMacKey = attribute.Key("host.mac") - - // HostNameKey is the attribute Key conforming to the "host.name" semantic - // conventions. It represents the name of the host. On Unix systems, it may - // contain what the hostname command returns, or the fully qualified hostname, - // or another name specified by the user. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry-test" - HostNameKey = attribute.Key("host.name") - - // HostTypeKey is the attribute Key conforming to the "host.type" semantic - // conventions. It represents the type of host. For Cloud, this must be the - // machine type. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "n1-standard-1" - HostTypeKey = attribute.Key("host.type") -) - -// HostCPUCacheL2Size returns an attribute KeyValue conforming to the -// "host.cpu.cache.l2.size" semantic conventions. It represents the amount of -// level 2 memory cache available to the processor (in Bytes). -func HostCPUCacheL2Size(val int) attribute.KeyValue { - return HostCPUCacheL2SizeKey.Int(val) -} - -// HostCPUFamily returns an attribute KeyValue conforming to the -// "host.cpu.family" semantic conventions. It represents the family or generation -// of the CPU. -func HostCPUFamily(val string) attribute.KeyValue { - return HostCPUFamilyKey.String(val) -} - -// HostCPUModelID returns an attribute KeyValue conforming to the -// "host.cpu.model.id" semantic conventions. It represents the model identifier. -// It provides more granular information about the CPU, distinguishing it from -// other CPUs within the same family. -func HostCPUModelID(val string) attribute.KeyValue { - return HostCPUModelIDKey.String(val) -} - -// HostCPUModelName returns an attribute KeyValue conforming to the -// "host.cpu.model.name" semantic conventions. It represents the model -// designation of the processor. -func HostCPUModelName(val string) attribute.KeyValue { - return HostCPUModelNameKey.String(val) -} - -// HostCPUStepping returns an attribute KeyValue conforming to the -// "host.cpu.stepping" semantic conventions. It represents the stepping or core -// revisions. -func HostCPUStepping(val string) attribute.KeyValue { - return HostCPUSteppingKey.String(val) -} - -// HostCPUVendorID returns an attribute KeyValue conforming to the -// "host.cpu.vendor.id" semantic conventions. It represents the processor -// manufacturer identifier. A maximum 12-character string. -func HostCPUVendorID(val string) attribute.KeyValue { - return HostCPUVendorIDKey.String(val) -} - -// HostID returns an attribute KeyValue conforming to the "host.id" semantic -// conventions. It represents the unique host ID. For Cloud, this must be the -// instance_id assigned by the cloud provider. For non-containerized systems, -// this should be the `machine-id`. See the table below for the sources to use to -// determine the `machine-id` based on operating system. -func HostID(val string) attribute.KeyValue { - return HostIDKey.String(val) -} - -// HostImageID returns an attribute KeyValue conforming to the "host.image.id" -// semantic conventions. It represents the VM image ID or host OS image ID. For -// Cloud, this value is from the provider. -func HostImageID(val string) attribute.KeyValue { - return HostImageIDKey.String(val) -} - -// HostImageName returns an attribute KeyValue conforming to the -// "host.image.name" semantic conventions. It represents the name of the VM image -// or OS install the host was instantiated from. -func HostImageName(val string) attribute.KeyValue { - return HostImageNameKey.String(val) -} - -// HostImageVersion returns an attribute KeyValue conforming to the -// "host.image.version" semantic conventions. It represents the version string of -// the VM image or host OS as defined in [Version Attributes]. -// -// [Version Attributes]: /docs/resource/README.md#version-attributes -func HostImageVersion(val string) attribute.KeyValue { - return HostImageVersionKey.String(val) -} - -// HostIP returns an attribute KeyValue conforming to the "host.ip" semantic -// conventions. It represents the available IP addresses of the host, excluding -// loopback interfaces. -func HostIP(val ...string) attribute.KeyValue { - return HostIPKey.StringSlice(val) -} - -// HostMac returns an attribute KeyValue conforming to the "host.mac" semantic -// conventions. It represents the available MAC addresses of the host, excluding -// loopback interfaces. -func HostMac(val ...string) attribute.KeyValue { - return HostMacKey.StringSlice(val) -} - -// HostName returns an attribute KeyValue conforming to the "host.name" semantic -// conventions. It represents the name of the host. On Unix systems, it may -// contain what the hostname command returns, or the fully qualified hostname, or -// another name specified by the user. -func HostName(val string) attribute.KeyValue { - return HostNameKey.String(val) -} - -// HostType returns an attribute KeyValue conforming to the "host.type" semantic -// conventions. It represents the type of host. For Cloud, this must be the -// machine type. -func HostType(val string) attribute.KeyValue { - return HostTypeKey.String(val) -} - -// Enum values for host.arch -var ( - // AMD64 - // Stability: development - HostArchAMD64 = HostArchKey.String("amd64") - // ARM32 - // Stability: development - HostArchARM32 = HostArchKey.String("arm32") - // ARM64 - // Stability: development - HostArchARM64 = HostArchKey.String("arm64") - // Itanium - // Stability: development - HostArchIA64 = HostArchKey.String("ia64") - // 32-bit PowerPC - // Stability: development - HostArchPPC32 = HostArchKey.String("ppc32") - // 64-bit PowerPC - // Stability: development - HostArchPPC64 = HostArchKey.String("ppc64") - // IBM z/Architecture - // Stability: development - HostArchS390x = HostArchKey.String("s390x") - // 32-bit x86 - // Stability: development - HostArchX86 = HostArchKey.String("x86") -) - -// Namespace: http -const ( - // HTTPConnectionStateKey is the attribute Key conforming to the - // "http.connection.state" semantic conventions. It represents the state of the - // HTTP connection in the HTTP connection pool. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "active", "idle" - HTTPConnectionStateKey = attribute.Key("http.connection.state") - - // HTTPRequestBodySizeKey is the attribute Key conforming to the - // "http.request.body.size" semantic conventions. It represents the size of the - // request payload body in bytes. This is the number of bytes transferred - // excluding headers and is often, but not always, present as the - // [Content-Length] header. For requests using transport encoding, this should - // be the compressed size. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length - HTTPRequestBodySizeKey = attribute.Key("http.request.body.size") - - // HTTPRequestMethodKey is the attribute Key conforming to the - // "http.request.method" semantic conventions. It represents the HTTP request - // method. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "GET", "POST", "HEAD" - // Note: HTTP request method value SHOULD be "known" to the instrumentation. - // By default, this convention defines "known" methods as the ones listed in - // [RFC9110], - // the PATCH method defined in [RFC5789] - // and the QUERY method defined in [httpbis-safe-method-w-body]. - // - // If the HTTP request method is not known to instrumentation, it MUST set the - // `http.request.method` attribute to `_OTHER`. - // - // If the HTTP instrumentation could end up converting valid HTTP request - // methods to `_OTHER`, then it MUST provide a way to override - // the list of known HTTP methods. If this override is done via environment - // variable, then the environment variable MUST be named - // OTEL_INSTRUMENTATION_HTTP_KNOWN_METHODS and support a comma-separated list of - // case-sensitive known HTTP methods - // (this list MUST be a full override of the default known method, it is not a - // list of known methods in addition to the defaults). - // - // HTTP method names are case-sensitive and `http.request.method` attribute - // value MUST match a known HTTP method name exactly. - // Instrumentations for specific web frameworks that consider HTTP methods to be - // case insensitive, SHOULD populate a canonical equivalent. - // Tracing instrumentations that do so, MUST also set - // `http.request.method_original` to the original value. - // - // [RFC9110]: https://www.rfc-editor.org/rfc/rfc9110.html#name-methods - // [RFC5789]: https://www.rfc-editor.org/rfc/rfc5789.html - // [httpbis-safe-method-w-body]: https://datatracker.ietf.org/doc/draft-ietf-httpbis-safe-method-w-body/?include_text=1 - HTTPRequestMethodKey = attribute.Key("http.request.method") - - // HTTPRequestMethodOriginalKey is the attribute Key conforming to the - // "http.request.method_original" semantic conventions. It represents the - // original HTTP method sent by the client in the request line. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "GeT", "ACL", "foo" - HTTPRequestMethodOriginalKey = attribute.Key("http.request.method_original") - - // HTTPRequestResendCountKey is the attribute Key conforming to the - // "http.request.resend_count" semantic conventions. It represents the ordinal - // number of request resending attempt (for any reason, including redirects). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Note: The resend count SHOULD be updated each time an HTTP request gets - // resent by the client, regardless of what was the cause of the resending (e.g. - // redirection, authorization failure, 503 Server Unavailable, network issues, - // or any other). - HTTPRequestResendCountKey = attribute.Key("http.request.resend_count") - - // HTTPRequestSizeKey is the attribute Key conforming to the "http.request.size" - // semantic conventions. It represents the total size of the request in bytes. - // This should be the total number of bytes sent over the wire, including the - // request line (HTTP/1.1), framing (HTTP/2 and HTTP/3), headers, and request - // body if any. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - HTTPRequestSizeKey = attribute.Key("http.request.size") - - // HTTPResponseBodySizeKey is the attribute Key conforming to the - // "http.response.body.size" semantic conventions. It represents the size of the - // response payload body in bytes. This is the number of bytes transferred - // excluding headers and is often, but not always, present as the - // [Content-Length] header. For requests using transport encoding, this should - // be the compressed size. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length - HTTPResponseBodySizeKey = attribute.Key("http.response.body.size") - - // HTTPResponseSizeKey is the attribute Key conforming to the - // "http.response.size" semantic conventions. It represents the total size of - // the response in bytes. This should be the total number of bytes sent over the - // wire, including the status line (HTTP/1.1), framing (HTTP/2 and HTTP/3), - // headers, and response body and trailers if any. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - HTTPResponseSizeKey = attribute.Key("http.response.size") - - // HTTPResponseStatusCodeKey is the attribute Key conforming to the - // "http.response.status_code" semantic conventions. It represents the - // [HTTP response status code]. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 200 - // - // [HTTP response status code]: https://tools.ietf.org/html/rfc7231#section-6 - HTTPResponseStatusCodeKey = attribute.Key("http.response.status_code") - - // HTTPRouteKey is the attribute Key conforming to the "http.route" semantic - // conventions. It represents the matched route template for the request. This - // MUST be low-cardinality and include all static path segments, with dynamic - // path segments represented with placeholders. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "/users/:userID?", "my-controller/my-action/{id?}" - // Note: MUST NOT be populated when this is not supported by the HTTP server - // framework as the route attribute should have low-cardinality and the URI path - // can NOT substitute it. - // SHOULD include the [application root] if there is one. - // - // A static path segment is a part of the route template with a fixed, - // low-cardinality value. This includes literal strings like `/users/` and - // placeholders that - // are constrained to a finite, predefined set of values, e.g. `{controller}` or - // `{action}`. - // - // A dynamic path segment is a placeholder for a value that can have high - // cardinality and is not constrained to a predefined list like static path - // segments. - // - // Instrumentations SHOULD use routing information provided by the corresponding - // web framework. They SHOULD pick the most precise source of routing - // information and MAY - // support custom route formatting. Instrumentations SHOULD document the format - // and the API used to obtain the route string. - // - // [application root]: /docs/http/http-spans.md#http-server-definitions - HTTPRouteKey = attribute.Key("http.route") -) - -// HTTPRequestBodySize returns an attribute KeyValue conforming to the -// "http.request.body.size" semantic conventions. It represents the size of the -// request payload body in bytes. This is the number of bytes transferred -// excluding headers and is often, but not always, present as the -// [Content-Length] header. For requests using transport encoding, this should be -// the compressed size. -// -// [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length -func HTTPRequestBodySize(val int) attribute.KeyValue { - return HTTPRequestBodySizeKey.Int(val) -} - -// HTTPRequestHeader returns an attribute KeyValue conforming to the -// "http.request.header" semantic conventions. It represents the HTTP request -// headers, `` being the normalized HTTP Header name (lowercase), the value -// being the header values. -func HTTPRequestHeader(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("http.request.header."+key, val) -} - -// HTTPRequestMethodOriginal returns an attribute KeyValue conforming to the -// "http.request.method_original" semantic conventions. It represents the -// original HTTP method sent by the client in the request line. -func HTTPRequestMethodOriginal(val string) attribute.KeyValue { - return HTTPRequestMethodOriginalKey.String(val) -} - -// HTTPRequestResendCount returns an attribute KeyValue conforming to the -// "http.request.resend_count" semantic conventions. It represents the ordinal -// number of request resending attempt (for any reason, including redirects). -func HTTPRequestResendCount(val int) attribute.KeyValue { - return HTTPRequestResendCountKey.Int(val) -} - -// HTTPRequestSize returns an attribute KeyValue conforming to the -// "http.request.size" semantic conventions. It represents the total size of the -// request in bytes. This should be the total number of bytes sent over the wire, -// including the request line (HTTP/1.1), framing (HTTP/2 and HTTP/3), headers, -// and request body if any. -func HTTPRequestSize(val int) attribute.KeyValue { - return HTTPRequestSizeKey.Int(val) -} - -// HTTPResponseBodySize returns an attribute KeyValue conforming to the -// "http.response.body.size" semantic conventions. It represents the size of the -// response payload body in bytes. This is the number of bytes transferred -// excluding headers and is often, but not always, present as the -// [Content-Length] header. For requests using transport encoding, this should be -// the compressed size. -// -// [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length -func HTTPResponseBodySize(val int) attribute.KeyValue { - return HTTPResponseBodySizeKey.Int(val) -} - -// HTTPResponseHeader returns an attribute KeyValue conforming to the -// "http.response.header" semantic conventions. It represents the HTTP response -// headers, `` being the normalized HTTP Header name (lowercase), the value -// being the header values. -func HTTPResponseHeader(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("http.response.header."+key, val) -} - -// HTTPResponseSize returns an attribute KeyValue conforming to the -// "http.response.size" semantic conventions. It represents the total size of the -// response in bytes. This should be the total number of bytes sent over the -// wire, including the status line (HTTP/1.1), framing (HTTP/2 and HTTP/3), -// headers, and response body and trailers if any. -func HTTPResponseSize(val int) attribute.KeyValue { - return HTTPResponseSizeKey.Int(val) -} - -// HTTPResponseStatusCode returns an attribute KeyValue conforming to the -// "http.response.status_code" semantic conventions. It represents the -// [HTTP response status code]. -// -// [HTTP response status code]: https://tools.ietf.org/html/rfc7231#section-6 -func HTTPResponseStatusCode(val int) attribute.KeyValue { - return HTTPResponseStatusCodeKey.Int(val) -} - -// HTTPRoute returns an attribute KeyValue conforming to the "http.route" -// semantic conventions. It represents the matched route template for the -// request. This MUST be low-cardinality and include all static path segments, -// with dynamic path segments represented with placeholders. -func HTTPRoute(val string) attribute.KeyValue { - return HTTPRouteKey.String(val) -} - -// Enum values for http.connection.state -var ( - // active state. - // Stability: development - HTTPConnectionStateActive = HTTPConnectionStateKey.String("active") - // idle state. - // Stability: development - HTTPConnectionStateIdle = HTTPConnectionStateKey.String("idle") -) - -// Enum values for http.request.method -var ( - // CONNECT method. - // Stability: stable - HTTPRequestMethodConnect = HTTPRequestMethodKey.String("CONNECT") - // DELETE method. - // Stability: stable - HTTPRequestMethodDelete = HTTPRequestMethodKey.String("DELETE") - // GET method. - // Stability: stable - HTTPRequestMethodGet = HTTPRequestMethodKey.String("GET") - // HEAD method. - // Stability: stable - HTTPRequestMethodHead = HTTPRequestMethodKey.String("HEAD") - // OPTIONS method. - // Stability: stable - HTTPRequestMethodOptions = HTTPRequestMethodKey.String("OPTIONS") - // PATCH method. - // Stability: stable - HTTPRequestMethodPatch = HTTPRequestMethodKey.String("PATCH") - // POST method. - // Stability: stable - HTTPRequestMethodPost = HTTPRequestMethodKey.String("POST") - // PUT method. - // Stability: stable - HTTPRequestMethodPut = HTTPRequestMethodKey.String("PUT") - // TRACE method. - // Stability: stable - HTTPRequestMethodTrace = HTTPRequestMethodKey.String("TRACE") - // QUERY method. - // Stability: development - HTTPRequestMethodQuery = HTTPRequestMethodKey.String("QUERY") - // Any HTTP method that the instrumentation has no prior knowledge of. - // Stability: stable - HTTPRequestMethodOther = HTTPRequestMethodKey.String("_OTHER") -) - -// Namespace: hw -const ( - // HwBatteryCapacityKey is the attribute Key conforming to the - // "hw.battery.capacity" semantic conventions. It represents the design capacity - // in Watts-hours or Amper-hours. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9.3Ah", "50Wh" - HwBatteryCapacityKey = attribute.Key("hw.battery.capacity") - - // HwBatteryChemistryKey is the attribute Key conforming to the - // "hw.battery.chemistry" semantic conventions. It represents the battery - // [chemistry], e.g. Lithium-Ion, Nickel-Cadmium, etc. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Li-ion", "NiMH" - // - // [chemistry]: https://schemas.dmtf.org/wbem/cim-html/2.31.0/CIM_Battery.html - HwBatteryChemistryKey = attribute.Key("hw.battery.chemistry") - - // HwBatteryStateKey is the attribute Key conforming to the "hw.battery.state" - // semantic conventions. It represents the current state of the battery. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwBatteryStateKey = attribute.Key("hw.battery.state") - - // HwBiosVersionKey is the attribute Key conforming to the "hw.bios_version" - // semantic conventions. It represents the BIOS version of the hardware - // component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1.2.3" - HwBiosVersionKey = attribute.Key("hw.bios_version") - - // HwDriverVersionKey is the attribute Key conforming to the "hw.driver_version" - // semantic conventions. It represents the driver version for the hardware - // component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "10.2.1-3" - HwDriverVersionKey = attribute.Key("hw.driver_version") - - // HwEnclosureTypeKey is the attribute Key conforming to the "hw.enclosure.type" - // semantic conventions. It represents the type of the enclosure (useful for - // modular systems). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Computer", "Storage", "Switch" - HwEnclosureTypeKey = attribute.Key("hw.enclosure.type") - - // HwFirmwareVersionKey is the attribute Key conforming to the - // "hw.firmware_version" semantic conventions. It represents the firmware - // version of the hardware component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2.0.1" - HwFirmwareVersionKey = attribute.Key("hw.firmware_version") - - // HwGpuTaskKey is the attribute Key conforming to the "hw.gpu.task" semantic - // conventions. It represents the type of task the GPU is performing. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwGpuTaskKey = attribute.Key("hw.gpu.task") - - // HwIDKey is the attribute Key conforming to the "hw.id" semantic conventions. - // It represents an identifier for the hardware component, unique within the - // monitored host. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "win32battery_battery_testsysa33_1" - HwIDKey = attribute.Key("hw.id") - - // HwLimitTypeKey is the attribute Key conforming to the "hw.limit_type" - // semantic conventions. It represents the type of limit for hardware - // components. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwLimitTypeKey = attribute.Key("hw.limit_type") - - // HwLogicalDiskRaidLevelKey is the attribute Key conforming to the - // "hw.logical_disk.raid_level" semantic conventions. It represents the RAID - // Level of the logical disk. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "RAID0+1", "RAID5", "RAID10" - HwLogicalDiskRaidLevelKey = attribute.Key("hw.logical_disk.raid_level") - - // HwLogicalDiskStateKey is the attribute Key conforming to the - // "hw.logical_disk.state" semantic conventions. It represents the state of the - // logical disk space usage. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwLogicalDiskStateKey = attribute.Key("hw.logical_disk.state") - - // HwMemoryTypeKey is the attribute Key conforming to the "hw.memory.type" - // semantic conventions. It represents the type of the memory module. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "DDR4", "DDR5", "LPDDR5" - HwMemoryTypeKey = attribute.Key("hw.memory.type") - - // HwModelKey is the attribute Key conforming to the "hw.model" semantic - // conventions. It represents the descriptive model name of the hardware - // component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "PERC H740P", "Intel(R) Core(TM) i7-10700K", "Dell XPS 15 Battery" - HwModelKey = attribute.Key("hw.model") - - // HwNameKey is the attribute Key conforming to the "hw.name" semantic - // conventions. It represents an easily-recognizable name for the hardware - // component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "eth0" - HwNameKey = attribute.Key("hw.name") - - // HwNetworkLogicalAddressesKey is the attribute Key conforming to the - // "hw.network.logical_addresses" semantic conventions. It represents the - // logical addresses of the adapter (e.g. IP address, or WWPN). - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "172.16.8.21", "57.11.193.42" - HwNetworkLogicalAddressesKey = attribute.Key("hw.network.logical_addresses") - - // HwNetworkPhysicalAddressKey is the attribute Key conforming to the - // "hw.network.physical_address" semantic conventions. It represents the - // physical address of the adapter (e.g. MAC address, or WWNN). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "00-90-F5-E9-7B-36" - HwNetworkPhysicalAddressKey = attribute.Key("hw.network.physical_address") - - // HwParentKey is the attribute Key conforming to the "hw.parent" semantic - // conventions. It represents the unique identifier of the parent component - // (typically the `hw.id` attribute of the enclosure, or disk controller). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "dellStorage_perc_0" - HwParentKey = attribute.Key("hw.parent") - - // HwPhysicalDiskSmartAttributeKey is the attribute Key conforming to the - // "hw.physical_disk.smart_attribute" semantic conventions. It represents the - // [S.M.A.R.T.] (Self-Monitoring, Analysis, and Reporting Technology) attribute - // of the physical disk. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Spin Retry Count", "Seek Error Rate", "Raw Read Error Rate" - // - // [S.M.A.R.T.]: https://wikipedia.org/wiki/S.M.A.R.T. - HwPhysicalDiskSmartAttributeKey = attribute.Key("hw.physical_disk.smart_attribute") - - // HwPhysicalDiskStateKey is the attribute Key conforming to the - // "hw.physical_disk.state" semantic conventions. It represents the state of the - // physical disk endurance utilization. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwPhysicalDiskStateKey = attribute.Key("hw.physical_disk.state") - - // HwPhysicalDiskTypeKey is the attribute Key conforming to the - // "hw.physical_disk.type" semantic conventions. It represents the type of the - // physical disk. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "HDD", "SSD", "10K" - HwPhysicalDiskTypeKey = attribute.Key("hw.physical_disk.type") - - // HwSensorLocationKey is the attribute Key conforming to the - // "hw.sensor_location" semantic conventions. It represents the location of the - // sensor. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cpu0", "ps1", "INLET", "CPU0_DIE", "AMBIENT", "MOTHERBOARD", "PS0 - // V3_3", "MAIN_12V", "CPU_VCORE" - HwSensorLocationKey = attribute.Key("hw.sensor_location") - - // HwSerialNumberKey is the attribute Key conforming to the "hw.serial_number" - // semantic conventions. It represents the serial number of the hardware - // component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CNFCP0123456789" - HwSerialNumberKey = attribute.Key("hw.serial_number") - - // HwStateKey is the attribute Key conforming to the "hw.state" semantic - // conventions. It represents the current state of the component. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwStateKey = attribute.Key("hw.state") - - // HwTapeDriveOperationTypeKey is the attribute Key conforming to the - // "hw.tape_drive.operation_type" semantic conventions. It represents the type - // of tape drive operation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - HwTapeDriveOperationTypeKey = attribute.Key("hw.tape_drive.operation_type") - - // HwTypeKey is the attribute Key conforming to the "hw.type" semantic - // conventions. It represents the type of the component. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: Describes the category of the hardware component for which `hw.state` - // is being reported. For example, `hw.type=temperature` along with - // `hw.state=degraded` would indicate that the temperature of the hardware - // component has been reported as `degraded`. - HwTypeKey = attribute.Key("hw.type") - - // HwVendorKey is the attribute Key conforming to the "hw.vendor" semantic - // conventions. It represents the vendor name of the hardware component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Dell", "HP", "Intel", "AMD", "LSI", "Lenovo" - HwVendorKey = attribute.Key("hw.vendor") -) - -// HwBatteryCapacity returns an attribute KeyValue conforming to the -// "hw.battery.capacity" semantic conventions. It represents the design capacity -// in Watts-hours or Amper-hours. -func HwBatteryCapacity(val string) attribute.KeyValue { - return HwBatteryCapacityKey.String(val) -} - -// HwBatteryChemistry returns an attribute KeyValue conforming to the -// "hw.battery.chemistry" semantic conventions. It represents the battery -// [chemistry], e.g. Lithium-Ion, Nickel-Cadmium, etc. -// -// [chemistry]: https://schemas.dmtf.org/wbem/cim-html/2.31.0/CIM_Battery.html -func HwBatteryChemistry(val string) attribute.KeyValue { - return HwBatteryChemistryKey.String(val) -} - -// HwBiosVersion returns an attribute KeyValue conforming to the -// "hw.bios_version" semantic conventions. It represents the BIOS version of the -// hardware component. -func HwBiosVersion(val string) attribute.KeyValue { - return HwBiosVersionKey.String(val) -} - -// HwDriverVersion returns an attribute KeyValue conforming to the -// "hw.driver_version" semantic conventions. It represents the driver version for -// the hardware component. -func HwDriverVersion(val string) attribute.KeyValue { - return HwDriverVersionKey.String(val) -} - -// HwEnclosureType returns an attribute KeyValue conforming to the -// "hw.enclosure.type" semantic conventions. It represents the type of the -// enclosure (useful for modular systems). -func HwEnclosureType(val string) attribute.KeyValue { - return HwEnclosureTypeKey.String(val) -} - -// HwFirmwareVersion returns an attribute KeyValue conforming to the -// "hw.firmware_version" semantic conventions. It represents the firmware version -// of the hardware component. -func HwFirmwareVersion(val string) attribute.KeyValue { - return HwFirmwareVersionKey.String(val) -} - -// HwID returns an attribute KeyValue conforming to the "hw.id" semantic -// conventions. It represents an identifier for the hardware component, unique -// within the monitored host. -func HwID(val string) attribute.KeyValue { - return HwIDKey.String(val) -} - -// HwLogicalDiskRaidLevel returns an attribute KeyValue conforming to the -// "hw.logical_disk.raid_level" semantic conventions. It represents the RAID -// Level of the logical disk. -func HwLogicalDiskRaidLevel(val string) attribute.KeyValue { - return HwLogicalDiskRaidLevelKey.String(val) -} - -// HwMemoryType returns an attribute KeyValue conforming to the "hw.memory.type" -// semantic conventions. It represents the type of the memory module. -func HwMemoryType(val string) attribute.KeyValue { - return HwMemoryTypeKey.String(val) -} - -// HwModel returns an attribute KeyValue conforming to the "hw.model" semantic -// conventions. It represents the descriptive model name of the hardware -// component. -func HwModel(val string) attribute.KeyValue { - return HwModelKey.String(val) -} - -// HwName returns an attribute KeyValue conforming to the "hw.name" semantic -// conventions. It represents an easily-recognizable name for the hardware -// component. -func HwName(val string) attribute.KeyValue { - return HwNameKey.String(val) -} - -// HwNetworkLogicalAddresses returns an attribute KeyValue conforming to the -// "hw.network.logical_addresses" semantic conventions. It represents the logical -// addresses of the adapter (e.g. IP address, or WWPN). -func HwNetworkLogicalAddresses(val ...string) attribute.KeyValue { - return HwNetworkLogicalAddressesKey.StringSlice(val) -} - -// HwNetworkPhysicalAddress returns an attribute KeyValue conforming to the -// "hw.network.physical_address" semantic conventions. It represents the physical -// address of the adapter (e.g. MAC address, or WWNN). -func HwNetworkPhysicalAddress(val string) attribute.KeyValue { - return HwNetworkPhysicalAddressKey.String(val) -} - -// HwParent returns an attribute KeyValue conforming to the "hw.parent" semantic -// conventions. It represents the unique identifier of the parent component -// (typically the `hw.id` attribute of the enclosure, or disk controller). -func HwParent(val string) attribute.KeyValue { - return HwParentKey.String(val) -} - -// HwPhysicalDiskSmartAttribute returns an attribute KeyValue conforming to the -// "hw.physical_disk.smart_attribute" semantic conventions. It represents the -// [S.M.A.R.T.] (Self-Monitoring, Analysis, and Reporting Technology) attribute -// of the physical disk. -// -// [S.M.A.R.T.]: https://wikipedia.org/wiki/S.M.A.R.T. -func HwPhysicalDiskSmartAttribute(val string) attribute.KeyValue { - return HwPhysicalDiskSmartAttributeKey.String(val) -} - -// HwPhysicalDiskType returns an attribute KeyValue conforming to the -// "hw.physical_disk.type" semantic conventions. It represents the type of the -// physical disk. -func HwPhysicalDiskType(val string) attribute.KeyValue { - return HwPhysicalDiskTypeKey.String(val) -} - -// HwSensorLocation returns an attribute KeyValue conforming to the -// "hw.sensor_location" semantic conventions. It represents the location of the -// sensor. -func HwSensorLocation(val string) attribute.KeyValue { - return HwSensorLocationKey.String(val) -} - -// HwSerialNumber returns an attribute KeyValue conforming to the -// "hw.serial_number" semantic conventions. It represents the serial number of -// the hardware component. -func HwSerialNumber(val string) attribute.KeyValue { - return HwSerialNumberKey.String(val) -} - -// HwVendor returns an attribute KeyValue conforming to the "hw.vendor" semantic -// conventions. It represents the vendor name of the hardware component. -func HwVendor(val string) attribute.KeyValue { - return HwVendorKey.String(val) -} - -// Enum values for hw.battery.state -var ( - // Charging - // Stability: development - HwBatteryStateCharging = HwBatteryStateKey.String("charging") - // Discharging - // Stability: development - HwBatteryStateDischarging = HwBatteryStateKey.String("discharging") -) - -// Enum values for hw.gpu.task -var ( - // Decoder - // Stability: development - HwGpuTaskDecoder = HwGpuTaskKey.String("decoder") - // Encoder - // Stability: development - HwGpuTaskEncoder = HwGpuTaskKey.String("encoder") - // General - // Stability: development - HwGpuTaskGeneral = HwGpuTaskKey.String("general") -) - -// Enum values for hw.limit_type -var ( - // Critical - // Stability: development - HwLimitTypeCritical = HwLimitTypeKey.String("critical") - // Degraded - // Stability: development - HwLimitTypeDegraded = HwLimitTypeKey.String("degraded") - // High Critical - // Stability: development - HwLimitTypeHighCritical = HwLimitTypeKey.String("high.critical") - // High Degraded - // Stability: development - HwLimitTypeHighDegraded = HwLimitTypeKey.String("high.degraded") - // Low Critical - // Stability: development - HwLimitTypeLowCritical = HwLimitTypeKey.String("low.critical") - // Low Degraded - // Stability: development - HwLimitTypeLowDegraded = HwLimitTypeKey.String("low.degraded") - // Maximum - // Stability: development - HwLimitTypeMax = HwLimitTypeKey.String("max") - // Throttled - // Stability: development - HwLimitTypeThrottled = HwLimitTypeKey.String("throttled") - // Turbo - // Stability: development - HwLimitTypeTurbo = HwLimitTypeKey.String("turbo") -) - -// Enum values for hw.logical_disk.state -var ( - // Used - // Stability: development - HwLogicalDiskStateUsed = HwLogicalDiskStateKey.String("used") - // Free - // Stability: development - HwLogicalDiskStateFree = HwLogicalDiskStateKey.String("free") -) - -// Enum values for hw.physical_disk.state -var ( - // Remaining - // Stability: development - HwPhysicalDiskStateRemaining = HwPhysicalDiskStateKey.String("remaining") -) - -// Enum values for hw.state -var ( - // Degraded - // Stability: development - HwStateDegraded = HwStateKey.String("degraded") - // Failed - // Stability: development - HwStateFailed = HwStateKey.String("failed") - // Needs Cleaning - // Stability: development - HwStateNeedsCleaning = HwStateKey.String("needs_cleaning") - // OK - // Stability: development - HwStateOk = HwStateKey.String("ok") - // Predicted Failure - // Stability: development - HwStatePredictedFailure = HwStateKey.String("predicted_failure") -) - -// Enum values for hw.tape_drive.operation_type -var ( - // Mount - // Stability: development - HwTapeDriveOperationTypeMount = HwTapeDriveOperationTypeKey.String("mount") - // Unmount - // Stability: development - HwTapeDriveOperationTypeUnmount = HwTapeDriveOperationTypeKey.String("unmount") - // Clean - // Stability: development - HwTapeDriveOperationTypeClean = HwTapeDriveOperationTypeKey.String("clean") -) - -// Enum values for hw.type -var ( - // Battery - // Stability: development - HwTypeBattery = HwTypeKey.String("battery") - // CPU - // Stability: development - HwTypeCPU = HwTypeKey.String("cpu") - // Disk controller - // Stability: development - HwTypeDiskController = HwTypeKey.String("disk_controller") - // Enclosure - // Stability: development - HwTypeEnclosure = HwTypeKey.String("enclosure") - // Fan - // Stability: development - HwTypeFan = HwTypeKey.String("fan") - // GPU - // Stability: development - HwTypeGpu = HwTypeKey.String("gpu") - // Logical disk - // Stability: development - HwTypeLogicalDisk = HwTypeKey.String("logical_disk") - // Memory - // Stability: development - HwTypeMemory = HwTypeKey.String("memory") - // Network - // Stability: development - HwTypeNetwork = HwTypeKey.String("network") - // Physical disk - // Stability: development - HwTypePhysicalDisk = HwTypeKey.String("physical_disk") - // Power supply - // Stability: development - HwTypePowerSupply = HwTypeKey.String("power_supply") - // Tape drive - // Stability: development - HwTypeTapeDrive = HwTypeKey.String("tape_drive") - // Temperature - // Stability: development - HwTypeTemperature = HwTypeKey.String("temperature") - // Voltage - // Stability: development - HwTypeVoltage = HwTypeKey.String("voltage") -) - -// Namespace: ios -const ( - // IOSAppStateKey is the attribute Key conforming to the "ios.app.state" - // semantic conventions. It represents the this attribute represents the state - // of the application. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The iOS lifecycle states are defined in the - // [UIApplicationDelegate documentation], and from which the `OS terminology` - // column values are derived. - // - // [UIApplicationDelegate documentation]: https://developer.apple.com/documentation/uikit/uiapplicationdelegate - IOSAppStateKey = attribute.Key("ios.app.state") -) - -// Enum values for ios.app.state -var ( - // The app has become `active`. Associated with UIKit notification - // `applicationDidBecomeActive`. - // - // Stability: development - IOSAppStateActive = IOSAppStateKey.String("active") - // The app is now `inactive`. Associated with UIKit notification - // `applicationWillResignActive`. - // - // Stability: development - IOSAppStateInactive = IOSAppStateKey.String("inactive") - // The app is now in the background. This value is associated with UIKit - // notification `applicationDidEnterBackground`. - // - // Stability: development - IOSAppStateBackground = IOSAppStateKey.String("background") - // The app is now in the foreground. This value is associated with UIKit - // notification `applicationWillEnterForeground`. - // - // Stability: development - IOSAppStateForeground = IOSAppStateKey.String("foreground") - // The app is about to terminate. Associated with UIKit notification - // `applicationWillTerminate`. - // - // Stability: development - IOSAppStateTerminate = IOSAppStateKey.String("terminate") -) - -// Namespace: jsonrpc -const ( - // JSONRPCProtocolVersionKey is the attribute Key conforming to the - // "jsonrpc.protocol.version" semantic conventions. It represents the protocol - // version, as specified in the `jsonrpc` property of the request and its - // corresponding response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2.0", "1.0" - JSONRPCProtocolVersionKey = attribute.Key("jsonrpc.protocol.version") - - // JSONRPCRequestIDKey is the attribute Key conforming to the - // "jsonrpc.request.id" semantic conventions. It represents a string - // representation of the `id` property of the request and its corresponding - // response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "10", "request-7" - // Note: Under the [JSON-RPC specification], the `id` property may be a string, - // number, null, or omitted entirely. When omitted, the request is treated as a - // notification. Using `null` is not equivalent to omitting the `id`, but it is - // discouraged. - // Instrumentations SHOULD NOT capture this attribute when the `id` is `null` or - // omitted. - // - // [JSON-RPC specification]: https://www.jsonrpc.org/specification - JSONRPCRequestIDKey = attribute.Key("jsonrpc.request.id") -) - -// JSONRPCProtocolVersion returns an attribute KeyValue conforming to the -// "jsonrpc.protocol.version" semantic conventions. It represents the protocol -// version, as specified in the `jsonrpc` property of the request and its -// corresponding response. -func JSONRPCProtocolVersion(val string) attribute.KeyValue { - return JSONRPCProtocolVersionKey.String(val) -} - -// JSONRPCRequestID returns an attribute KeyValue conforming to the -// "jsonrpc.request.id" semantic conventions. It represents a string -// representation of the `id` property of the request and its corresponding -// response. -func JSONRPCRequestID(val string) attribute.KeyValue { - return JSONRPCRequestIDKey.String(val) -} - -// Namespace: k8s -const ( - // K8SClusterNameKey is the attribute Key conforming to the "k8s.cluster.name" - // semantic conventions. It represents the name of the cluster. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry-cluster" - K8SClusterNameKey = attribute.Key("k8s.cluster.name") - - // K8SClusterUIDKey is the attribute Key conforming to the "k8s.cluster.uid" - // semantic conventions. It represents a pseudo-ID for the cluster, set to the - // UID of the `kube-system` namespace. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" - // Note: K8s doesn't have support for obtaining a cluster ID. If this is ever - // added, we will recommend collecting the `k8s.cluster.uid` through the - // official APIs. In the meantime, we are able to use the `uid` of the - // `kube-system` namespace as a proxy for cluster ID. Read on for the - // rationale. - // - // Every object created in a K8s cluster is assigned a distinct UID. The - // `kube-system` namespace is used by Kubernetes itself and will exist - // for the lifetime of the cluster. Using the `uid` of the `kube-system` - // namespace is a reasonable proxy for the K8s ClusterID as it will only - // change if the cluster is rebuilt. Furthermore, Kubernetes UIDs are - // UUIDs as standardized by - // [ISO/IEC 9834-8 and ITU-T X.667]. - // Which states: - // - // > If generated according to one of the mechanisms defined in Rec. - // > ITU-T X.667 | ISO/IEC 9834-8, a UUID is either guaranteed to be - // > different from all other UUIDs generated before 3603 A.D., or is - // > extremely likely to be different (depending on the mechanism chosen). - // - // Therefore, UIDs between clusters should be extremely unlikely to - // conflict. - // - // [ISO/IEC 9834-8 and ITU-T X.667]: https://www.itu.int/ITU-T/studygroups/com17/oid.html - K8SClusterUIDKey = attribute.Key("k8s.cluster.uid") - - // K8SContainerNameKey is the attribute Key conforming to the - // "k8s.container.name" semantic conventions. It represents the name of the - // Container from Pod specification, must be unique within a Pod. Container - // runtime usually uses different globally unique name (`container.name`). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "redis" - K8SContainerNameKey = attribute.Key("k8s.container.name") - - // K8SContainerRestartCountKey is the attribute Key conforming to the - // "k8s.container.restart_count" semantic conventions. It represents the number - // of times the container was restarted. This attribute can be used to identify - // a particular container (running or stopped) within a container spec. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: - K8SContainerRestartCountKey = attribute.Key("k8s.container.restart_count") - - // K8SContainerStatusLastTerminatedReasonKey is the attribute Key conforming to - // the "k8s.container.status.last_terminated_reason" semantic conventions. It - // represents the last terminated reason of the Container. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Evicted", "Error" - K8SContainerStatusLastTerminatedReasonKey = attribute.Key("k8s.container.status.last_terminated_reason") - - // K8SContainerStatusReasonKey is the attribute Key conforming to the - // "k8s.container.status.reason" semantic conventions. It represents the reason - // for the container state. Corresponds to the `reason` field of the: - // [K8s ContainerStateWaiting] or [K8s ContainerStateTerminated]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ContainerCreating", "CrashLoopBackOff", - // "CreateContainerConfigError", "ErrImagePull", "ImagePullBackOff", - // "OOMKilled", "Completed", "Error", "ContainerCannotRun" - // - // [K8s ContainerStateWaiting]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstatewaiting-v1-core - // [K8s ContainerStateTerminated]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstateterminated-v1-core - K8SContainerStatusReasonKey = attribute.Key("k8s.container.status.reason") - - // K8SContainerStatusStateKey is the attribute Key conforming to the - // "k8s.container.status.state" semantic conventions. It represents the state of - // the container. [K8s ContainerState]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "terminated", "running", "waiting" - // - // [K8s ContainerState]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#containerstate-v1-core - K8SContainerStatusStateKey = attribute.Key("k8s.container.status.state") - - // K8SCronJobNameKey is the attribute Key conforming to the "k8s.cronjob.name" - // semantic conventions. It represents the name of the CronJob. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SCronJobNameKey = attribute.Key("k8s.cronjob.name") - - // K8SCronJobUIDKey is the attribute Key conforming to the "k8s.cronjob.uid" - // semantic conventions. It represents the UID of the CronJob. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SCronJobUIDKey = attribute.Key("k8s.cronjob.uid") - - // K8SDaemonSetNameKey is the attribute Key conforming to the - // "k8s.daemonset.name" semantic conventions. It represents the name of the - // DaemonSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SDaemonSetNameKey = attribute.Key("k8s.daemonset.name") - - // K8SDaemonSetUIDKey is the attribute Key conforming to the "k8s.daemonset.uid" - // semantic conventions. It represents the UID of the DaemonSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SDaemonSetUIDKey = attribute.Key("k8s.daemonset.uid") - - // K8SDeploymentNameKey is the attribute Key conforming to the - // "k8s.deployment.name" semantic conventions. It represents the name of the - // Deployment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SDeploymentNameKey = attribute.Key("k8s.deployment.name") - - // K8SDeploymentUIDKey is the attribute Key conforming to the - // "k8s.deployment.uid" semantic conventions. It represents the UID of the - // Deployment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SDeploymentUIDKey = attribute.Key("k8s.deployment.uid") - - // K8SHPAMetricTypeKey is the attribute Key conforming to the - // "k8s.hpa.metric.type" semantic conventions. It represents the type of metric - // source for the horizontal pod autoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Resource", "ContainerResource" - // Note: This attribute reflects the `type` field of spec.metrics[] in the HPA. - K8SHPAMetricTypeKey = attribute.Key("k8s.hpa.metric.type") - - // K8SHPANameKey is the attribute Key conforming to the "k8s.hpa.name" semantic - // conventions. It represents the name of the horizontal pod autoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry" - K8SHPANameKey = attribute.Key("k8s.hpa.name") - - // K8SHPAScaletargetrefAPIVersionKey is the attribute Key conforming to the - // "k8s.hpa.scaletargetref.api_version" semantic conventions. It represents the - // API version of the target resource to scale for the HorizontalPodAutoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "apps/v1", "autoscaling/v2" - // Note: This maps to the `apiVersion` field in the `scaleTargetRef` of the HPA - // spec. - K8SHPAScaletargetrefAPIVersionKey = attribute.Key("k8s.hpa.scaletargetref.api_version") - - // K8SHPAScaletargetrefKindKey is the attribute Key conforming to the - // "k8s.hpa.scaletargetref.kind" semantic conventions. It represents the kind of - // the target resource to scale for the HorizontalPodAutoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Deployment", "StatefulSet" - // Note: This maps to the `kind` field in the `scaleTargetRef` of the HPA spec. - K8SHPAScaletargetrefKindKey = attribute.Key("k8s.hpa.scaletargetref.kind") - - // K8SHPAScaletargetrefNameKey is the attribute Key conforming to the - // "k8s.hpa.scaletargetref.name" semantic conventions. It represents the name of - // the target resource to scale for the HorizontalPodAutoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-deployment", "my-statefulset" - // Note: This maps to the `name` field in the `scaleTargetRef` of the HPA spec. - K8SHPAScaletargetrefNameKey = attribute.Key("k8s.hpa.scaletargetref.name") - - // K8SHPAUIDKey is the attribute Key conforming to the "k8s.hpa.uid" semantic - // conventions. It represents the UID of the horizontal pod autoscaler. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SHPAUIDKey = attribute.Key("k8s.hpa.uid") - - // K8SHugepageSizeKey is the attribute Key conforming to the "k8s.hugepage.size" - // semantic conventions. It represents the size (identifier) of the K8s huge - // page. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2Mi" - K8SHugepageSizeKey = attribute.Key("k8s.hugepage.size") - - // K8SJobNameKey is the attribute Key conforming to the "k8s.job.name" semantic - // conventions. It represents the name of the Job. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SJobNameKey = attribute.Key("k8s.job.name") - - // K8SJobUIDKey is the attribute Key conforming to the "k8s.job.uid" semantic - // conventions. It represents the UID of the Job. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SJobUIDKey = attribute.Key("k8s.job.uid") - - // K8SNamespaceNameKey is the attribute Key conforming to the - // "k8s.namespace.name" semantic conventions. It represents the name of the - // namespace that the pod is running in. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "default" - K8SNamespaceNameKey = attribute.Key("k8s.namespace.name") - - // K8SNamespacePhaseKey is the attribute Key conforming to the - // "k8s.namespace.phase" semantic conventions. It represents the phase of the - // K8s namespace. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "active", "terminating" - // Note: This attribute aligns with the `phase` field of the - // [K8s NamespaceStatus] - // - // [K8s NamespaceStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#namespacestatus-v1-core - K8SNamespacePhaseKey = attribute.Key("k8s.namespace.phase") - - // K8SNodeConditionStatusKey is the attribute Key conforming to the - // "k8s.node.condition.status" semantic conventions. It represents the status of - // the condition, one of True, False, Unknown. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "true", "false", "unknown" - // Note: This attribute aligns with the `status` field of the - // [NodeCondition] - // - // [NodeCondition]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#nodecondition-v1-core - K8SNodeConditionStatusKey = attribute.Key("k8s.node.condition.status") - - // K8SNodeConditionTypeKey is the attribute Key conforming to the - // "k8s.node.condition.type" semantic conventions. It represents the condition - // type of a K8s Node. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Ready", "DiskPressure" - // Note: K8s Node conditions as described - // by [K8s documentation]. - // - // This attribute aligns with the `type` field of the - // [NodeCondition] - // - // The set of possible values is not limited to those listed here. Managed - // Kubernetes environments, - // or custom controllers MAY introduce additional node condition types. - // When this occurs, the exact value as reported by the Kubernetes API SHOULD be - // used. - // - // [K8s documentation]: https://v1-32.docs.kubernetes.io/docs/reference/node/node-status/#condition - // [NodeCondition]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#nodecondition-v1-core - K8SNodeConditionTypeKey = attribute.Key("k8s.node.condition.type") - - // K8SNodeNameKey is the attribute Key conforming to the "k8s.node.name" - // semantic conventions. It represents the name of the Node. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "node-1" - K8SNodeNameKey = attribute.Key("k8s.node.name") - - // K8SNodeUIDKey is the attribute Key conforming to the "k8s.node.uid" semantic - // conventions. It represents the UID of the Node. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "1eb3a0c6-0477-4080-a9cb-0cb7db65c6a2" - K8SNodeUIDKey = attribute.Key("k8s.node.uid") - - // K8SPodHostnameKey is the attribute Key conforming to the "k8s.pod.hostname" - // semantic conventions. It represents the specifies the hostname of the Pod. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "collector-gateway" - // Note: The K8s Pod spec has an optional hostname field, which can be used to - // specify a hostname. - // Refer to [K8s docs] - // for more information about this field. - // - // This attribute aligns with the `hostname` field of the - // [K8s PodSpec]. - // - // [K8s docs]: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-hostname-and-subdomain-field - // [K8s PodSpec]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podspec-v1-core - K8SPodHostnameKey = attribute.Key("k8s.pod.hostname") - - // K8SPodIPKey is the attribute Key conforming to the "k8s.pod.ip" semantic - // conventions. It represents the IP address allocated to the Pod. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "172.18.0.2" - // Note: This attribute aligns with the `podIP` field of the - // [K8s PodStatus]. - // - // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podstatus-v1-core - K8SPodIPKey = attribute.Key("k8s.pod.ip") - - // K8SPodNameKey is the attribute Key conforming to the "k8s.pod.name" semantic - // conventions. It represents the name of the Pod. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry-pod-autoconf" - K8SPodNameKey = attribute.Key("k8s.pod.name") - - // K8SPodStartTimeKey is the attribute Key conforming to the - // "k8s.pod.start_time" semantic conventions. It represents the start timestamp - // of the Pod. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "2025-12-04T08:41:03Z" - // Note: Date and time at which the object was acknowledged by the Kubelet. - // This is before the Kubelet pulled the container image(s) for the pod. - // - // This attribute aligns with the `startTime` field of the - // [K8s PodStatus], - // in ISO 8601 (RFC 3339 compatible) format. - // - // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podstatus-v1-core - K8SPodStartTimeKey = attribute.Key("k8s.pod.start_time") - - // K8SPodStatusPhaseKey is the attribute Key conforming to the - // "k8s.pod.status.phase" semantic conventions. It represents the phase for the - // pod. Corresponds to the `phase` field of the: [K8s PodStatus]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Pending", "Running" - // - // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.33/#podstatus-v1-core - K8SPodStatusPhaseKey = attribute.Key("k8s.pod.status.phase") - - // K8SPodStatusReasonKey is the attribute Key conforming to the - // "k8s.pod.status.reason" semantic conventions. It represents the reason for - // the pod state. Corresponds to the `reason` field of the: [K8s PodStatus]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Evicted", "NodeAffinity" - // - // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.33/#podstatus-v1-core - K8SPodStatusReasonKey = attribute.Key("k8s.pod.status.reason") - - // K8SPodUIDKey is the attribute Key conforming to the "k8s.pod.uid" semantic - // conventions. It represents the UID of the Pod. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SPodUIDKey = attribute.Key("k8s.pod.uid") - - // K8SReplicaSetNameKey is the attribute Key conforming to the - // "k8s.replicaset.name" semantic conventions. It represents the name of the - // ReplicaSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SReplicaSetNameKey = attribute.Key("k8s.replicaset.name") - - // K8SReplicaSetUIDKey is the attribute Key conforming to the - // "k8s.replicaset.uid" semantic conventions. It represents the UID of the - // ReplicaSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SReplicaSetUIDKey = attribute.Key("k8s.replicaset.uid") - - // K8SReplicationControllerNameKey is the attribute Key conforming to the - // "k8s.replicationcontroller.name" semantic conventions. It represents the name - // of the replication controller. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry" - K8SReplicationControllerNameKey = attribute.Key("k8s.replicationcontroller.name") - - // K8SReplicationControllerUIDKey is the attribute Key conforming to the - // "k8s.replicationcontroller.uid" semantic conventions. It represents the UID - // of the replication controller. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SReplicationControllerUIDKey = attribute.Key("k8s.replicationcontroller.uid") - - // K8SResourceQuotaNameKey is the attribute Key conforming to the - // "k8s.resourcequota.name" semantic conventions. It represents the name of the - // resource quota. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry" - K8SResourceQuotaNameKey = attribute.Key("k8s.resourcequota.name") - - // K8SResourceQuotaResourceNameKey is the attribute Key conforming to the - // "k8s.resourcequota.resource_name" semantic conventions. It represents the - // name of the K8s resource a resource quota defines. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "count/replicationcontrollers" - // Note: The value for this attribute can be either the full - // `count/[.]` string (e.g., count/deployments.apps, - // count/pods), or, for certain core Kubernetes resources, just the resource - // name (e.g., pods, services, configmaps). Both forms are supported by - // Kubernetes for object count quotas. See - // [Kubernetes Resource Quotas documentation] for more details. - // - // [Kubernetes Resource Quotas documentation]: https://kubernetes.io/docs/concepts/policy/resource-quotas/#quota-on-object-count - K8SResourceQuotaResourceNameKey = attribute.Key("k8s.resourcequota.resource_name") - - // K8SResourceQuotaUIDKey is the attribute Key conforming to the - // "k8s.resourcequota.uid" semantic conventions. It represents the UID of the - // resource quota. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SResourceQuotaUIDKey = attribute.Key("k8s.resourcequota.uid") - - // K8SStatefulSetNameKey is the attribute Key conforming to the - // "k8s.statefulset.name" semantic conventions. It represents the name of the - // StatefulSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "opentelemetry" - K8SStatefulSetNameKey = attribute.Key("k8s.statefulset.name") - - // K8SStatefulSetUIDKey is the attribute Key conforming to the - // "k8s.statefulset.uid" semantic conventions. It represents the UID of the - // StatefulSet. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Alpha - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - K8SStatefulSetUIDKey = attribute.Key("k8s.statefulset.uid") - - // K8SStorageclassNameKey is the attribute Key conforming to the - // "k8s.storageclass.name" semantic conventions. It represents the name of K8s - // [StorageClass] object. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "gold.storageclass.storage.k8s.io" - // - // [StorageClass]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#storageclass-v1-storage-k8s-io - K8SStorageclassNameKey = attribute.Key("k8s.storageclass.name") - - // K8SVolumeNameKey is the attribute Key conforming to the "k8s.volume.name" - // semantic conventions. It represents the name of the K8s volume. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "volume0" - K8SVolumeNameKey = attribute.Key("k8s.volume.name") - - // K8SVolumeTypeKey is the attribute Key conforming to the "k8s.volume.type" - // semantic conventions. It represents the type of the K8s volume. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "emptyDir", "persistentVolumeClaim" - K8SVolumeTypeKey = attribute.Key("k8s.volume.type") -) - -// K8SClusterName returns an attribute KeyValue conforming to the -// "k8s.cluster.name" semantic conventions. It represents the name of the -// cluster. -func K8SClusterName(val string) attribute.KeyValue { - return K8SClusterNameKey.String(val) -} - -// K8SClusterUID returns an attribute KeyValue conforming to the -// "k8s.cluster.uid" semantic conventions. It represents a pseudo-ID for the -// cluster, set to the UID of the `kube-system` namespace. -func K8SClusterUID(val string) attribute.KeyValue { - return K8SClusterUIDKey.String(val) -} - -// K8SContainerName returns an attribute KeyValue conforming to the -// "k8s.container.name" semantic conventions. It represents the name of the -// Container from Pod specification, must be unique within a Pod. Container -// runtime usually uses different globally unique name (`container.name`). -func K8SContainerName(val string) attribute.KeyValue { - return K8SContainerNameKey.String(val) -} - -// K8SContainerRestartCount returns an attribute KeyValue conforming to the -// "k8s.container.restart_count" semantic conventions. It represents the number -// of times the container was restarted. This attribute can be used to identify a -// particular container (running or stopped) within a container spec. -func K8SContainerRestartCount(val int) attribute.KeyValue { - return K8SContainerRestartCountKey.Int(val) -} - -// K8SContainerStatusLastTerminatedReason returns an attribute KeyValue -// conforming to the "k8s.container.status.last_terminated_reason" semantic -// conventions. It represents the last terminated reason of the Container. -func K8SContainerStatusLastTerminatedReason(val string) attribute.KeyValue { - return K8SContainerStatusLastTerminatedReasonKey.String(val) -} - -// K8SCronJobAnnotation returns an attribute KeyValue conforming to the -// "k8s.cronjob.annotation" semantic conventions. It represents the cronjob -// annotation placed on the CronJob, the `` being the annotation name, the -// value being the annotation value. -func K8SCronJobAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.cronjob.annotation."+key, val) -} - -// K8SCronJobLabel returns an attribute KeyValue conforming to the -// "k8s.cronjob.label" semantic conventions. It represents the label placed on -// the CronJob, the `` being the label name, the value being the label -// value. -func K8SCronJobLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.cronjob.label."+key, val) -} - -// K8SCronJobName returns an attribute KeyValue conforming to the -// "k8s.cronjob.name" semantic conventions. It represents the name of the -// CronJob. -func K8SCronJobName(val string) attribute.KeyValue { - return K8SCronJobNameKey.String(val) -} - -// K8SCronJobUID returns an attribute KeyValue conforming to the -// "k8s.cronjob.uid" semantic conventions. It represents the UID of the CronJob. -func K8SCronJobUID(val string) attribute.KeyValue { - return K8SCronJobUIDKey.String(val) -} - -// K8SDaemonSetAnnotation returns an attribute KeyValue conforming to the -// "k8s.daemonset.annotation" semantic conventions. It represents the annotation -// placed on the DaemonSet, the `` being the annotation name, the value -// being the annotation value, even if the value is empty. -func K8SDaemonSetAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.daemonset.annotation."+key, val) -} - -// K8SDaemonSetLabel returns an attribute KeyValue conforming to the -// "k8s.daemonset.label" semantic conventions. It represents the label placed on -// the DaemonSet, the `` being the label name, the value being the label -// value, even if the value is empty. -func K8SDaemonSetLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.daemonset.label."+key, val) -} - -// K8SDaemonSetName returns an attribute KeyValue conforming to the -// "k8s.daemonset.name" semantic conventions. It represents the name of the -// DaemonSet. -func K8SDaemonSetName(val string) attribute.KeyValue { - return K8SDaemonSetNameKey.String(val) -} - -// K8SDaemonSetUID returns an attribute KeyValue conforming to the -// "k8s.daemonset.uid" semantic conventions. It represents the UID of the -// DaemonSet. -func K8SDaemonSetUID(val string) attribute.KeyValue { - return K8SDaemonSetUIDKey.String(val) -} - -// K8SDeploymentAnnotation returns an attribute KeyValue conforming to the -// "k8s.deployment.annotation" semantic conventions. It represents the annotation -// placed on the Deployment, the `` being the annotation name, the value -// being the annotation value, even if the value is empty. -func K8SDeploymentAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.deployment.annotation."+key, val) -} - -// K8SDeploymentLabel returns an attribute KeyValue conforming to the -// "k8s.deployment.label" semantic conventions. It represents the label placed on -// the Deployment, the `` being the label name, the value being the label -// value, even if the value is empty. -func K8SDeploymentLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.deployment.label."+key, val) -} - -// K8SDeploymentName returns an attribute KeyValue conforming to the -// "k8s.deployment.name" semantic conventions. It represents the name of the -// Deployment. -func K8SDeploymentName(val string) attribute.KeyValue { - return K8SDeploymentNameKey.String(val) -} - -// K8SDeploymentUID returns an attribute KeyValue conforming to the -// "k8s.deployment.uid" semantic conventions. It represents the UID of the -// Deployment. -func K8SDeploymentUID(val string) attribute.KeyValue { - return K8SDeploymentUIDKey.String(val) -} - -// K8SHPAMetricType returns an attribute KeyValue conforming to the -// "k8s.hpa.metric.type" semantic conventions. It represents the type of metric -// source for the horizontal pod autoscaler. -func K8SHPAMetricType(val string) attribute.KeyValue { - return K8SHPAMetricTypeKey.String(val) -} - -// K8SHPAName returns an attribute KeyValue conforming to the "k8s.hpa.name" -// semantic conventions. It represents the name of the horizontal pod autoscaler. -func K8SHPAName(val string) attribute.KeyValue { - return K8SHPANameKey.String(val) -} - -// K8SHPAScaletargetrefAPIVersion returns an attribute KeyValue conforming to the -// "k8s.hpa.scaletargetref.api_version" semantic conventions. It represents the -// API version of the target resource to scale for the HorizontalPodAutoscaler. -func K8SHPAScaletargetrefAPIVersion(val string) attribute.KeyValue { - return K8SHPAScaletargetrefAPIVersionKey.String(val) -} - -// K8SHPAScaletargetrefKind returns an attribute KeyValue conforming to the -// "k8s.hpa.scaletargetref.kind" semantic conventions. It represents the kind of -// the target resource to scale for the HorizontalPodAutoscaler. -func K8SHPAScaletargetrefKind(val string) attribute.KeyValue { - return K8SHPAScaletargetrefKindKey.String(val) -} - -// K8SHPAScaletargetrefName returns an attribute KeyValue conforming to the -// "k8s.hpa.scaletargetref.name" semantic conventions. It represents the name of -// the target resource to scale for the HorizontalPodAutoscaler. -func K8SHPAScaletargetrefName(val string) attribute.KeyValue { - return K8SHPAScaletargetrefNameKey.String(val) -} - -// K8SHPAUID returns an attribute KeyValue conforming to the "k8s.hpa.uid" -// semantic conventions. It represents the UID of the horizontal pod autoscaler. -func K8SHPAUID(val string) attribute.KeyValue { - return K8SHPAUIDKey.String(val) -} - -// K8SHugepageSize returns an attribute KeyValue conforming to the -// "k8s.hugepage.size" semantic conventions. It represents the size (identifier) -// of the K8s huge page. -func K8SHugepageSize(val string) attribute.KeyValue { - return K8SHugepageSizeKey.String(val) -} - -// K8SJobAnnotation returns an attribute KeyValue conforming to the -// "k8s.job.annotation" semantic conventions. It represents the annotation placed -// on the Job, the `` being the annotation name, the value being the -// annotation value, even if the value is empty. -func K8SJobAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.job.annotation."+key, val) -} - -// K8SJobLabel returns an attribute KeyValue conforming to the "k8s.job.label" -// semantic conventions. It represents the label placed on the Job, the `` -// being the label name, the value being the label value, even if the value is -// empty. -func K8SJobLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.job.label."+key, val) -} - -// K8SJobName returns an attribute KeyValue conforming to the "k8s.job.name" -// semantic conventions. It represents the name of the Job. -func K8SJobName(val string) attribute.KeyValue { - return K8SJobNameKey.String(val) -} - -// K8SJobUID returns an attribute KeyValue conforming to the "k8s.job.uid" -// semantic conventions. It represents the UID of the Job. -func K8SJobUID(val string) attribute.KeyValue { - return K8SJobUIDKey.String(val) -} - -// K8SNamespaceAnnotation returns an attribute KeyValue conforming to the -// "k8s.namespace.annotation" semantic conventions. It represents the annotation -// placed on the Namespace, the `` being the annotation name, the value -// being the annotation value, even if the value is empty. -func K8SNamespaceAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.namespace.annotation."+key, val) -} - -// K8SNamespaceLabel returns an attribute KeyValue conforming to the -// "k8s.namespace.label" semantic conventions. It represents the label placed on -// the Namespace, the `` being the label name, the value being the label -// value, even if the value is empty. -func K8SNamespaceLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.namespace.label."+key, val) -} - -// K8SNamespaceName returns an attribute KeyValue conforming to the -// "k8s.namespace.name" semantic conventions. It represents the name of the -// namespace that the pod is running in. -func K8SNamespaceName(val string) attribute.KeyValue { - return K8SNamespaceNameKey.String(val) -} - -// K8SNodeAnnotation returns an attribute KeyValue conforming to the -// "k8s.node.annotation" semantic conventions. It represents the annotation -// placed on the Node, the `` being the annotation name, the value being the -// annotation value, even if the value is empty. -func K8SNodeAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.node.annotation."+key, val) -} - -// K8SNodeLabel returns an attribute KeyValue conforming to the "k8s.node.label" -// semantic conventions. It represents the label placed on the Node, the `` -// being the label name, the value being the label value, even if the value is -// empty. -func K8SNodeLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.node.label."+key, val) -} - -// K8SNodeName returns an attribute KeyValue conforming to the "k8s.node.name" -// semantic conventions. It represents the name of the Node. -func K8SNodeName(val string) attribute.KeyValue { - return K8SNodeNameKey.String(val) -} - -// K8SNodeUID returns an attribute KeyValue conforming to the "k8s.node.uid" -// semantic conventions. It represents the UID of the Node. -func K8SNodeUID(val string) attribute.KeyValue { - return K8SNodeUIDKey.String(val) -} - -// K8SPodAnnotation returns an attribute KeyValue conforming to the -// "k8s.pod.annotation" semantic conventions. It represents the annotation placed -// on the Pod, the `` being the annotation name, the value being the -// annotation value. -func K8SPodAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.pod.annotation."+key, val) -} - -// K8SPodHostname returns an attribute KeyValue conforming to the -// "k8s.pod.hostname" semantic conventions. It represents the specifies the -// hostname of the Pod. -func K8SPodHostname(val string) attribute.KeyValue { - return K8SPodHostnameKey.String(val) -} - -// K8SPodIP returns an attribute KeyValue conforming to the "k8s.pod.ip" semantic -// conventions. It represents the IP address allocated to the Pod. -func K8SPodIP(val string) attribute.KeyValue { - return K8SPodIPKey.String(val) -} - -// K8SPodLabel returns an attribute KeyValue conforming to the "k8s.pod.label" -// semantic conventions. It represents the label placed on the Pod, the `` -// being the label name, the value being the label value. -func K8SPodLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.pod.label."+key, val) -} - -// K8SPodName returns an attribute KeyValue conforming to the "k8s.pod.name" -// semantic conventions. It represents the name of the Pod. -func K8SPodName(val string) attribute.KeyValue { - return K8SPodNameKey.String(val) -} - -// K8SPodStartTime returns an attribute KeyValue conforming to the -// "k8s.pod.start_time" semantic conventions. It represents the start timestamp -// of the Pod. -func K8SPodStartTime(val string) attribute.KeyValue { - return K8SPodStartTimeKey.String(val) -} - -// K8SPodUID returns an attribute KeyValue conforming to the "k8s.pod.uid" -// semantic conventions. It represents the UID of the Pod. -func K8SPodUID(val string) attribute.KeyValue { - return K8SPodUIDKey.String(val) -} - -// K8SReplicaSetAnnotation returns an attribute KeyValue conforming to the -// "k8s.replicaset.annotation" semantic conventions. It represents the annotation -// placed on the ReplicaSet, the `` being the annotation name, the value -// being the annotation value, even if the value is empty. -func K8SReplicaSetAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.replicaset.annotation."+key, val) -} - -// K8SReplicaSetLabel returns an attribute KeyValue conforming to the -// "k8s.replicaset.label" semantic conventions. It represents the label placed on -// the ReplicaSet, the `` being the label name, the value being the label -// value, even if the value is empty. -func K8SReplicaSetLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.replicaset.label."+key, val) -} - -// K8SReplicaSetName returns an attribute KeyValue conforming to the -// "k8s.replicaset.name" semantic conventions. It represents the name of the -// ReplicaSet. -func K8SReplicaSetName(val string) attribute.KeyValue { - return K8SReplicaSetNameKey.String(val) -} - -// K8SReplicaSetUID returns an attribute KeyValue conforming to the -// "k8s.replicaset.uid" semantic conventions. It represents the UID of the -// ReplicaSet. -func K8SReplicaSetUID(val string) attribute.KeyValue { - return K8SReplicaSetUIDKey.String(val) -} - -// K8SReplicationControllerName returns an attribute KeyValue conforming to the -// "k8s.replicationcontroller.name" semantic conventions. It represents the name -// of the replication controller. -func K8SReplicationControllerName(val string) attribute.KeyValue { - return K8SReplicationControllerNameKey.String(val) -} - -// K8SReplicationControllerUID returns an attribute KeyValue conforming to the -// "k8s.replicationcontroller.uid" semantic conventions. It represents the UID of -// the replication controller. -func K8SReplicationControllerUID(val string) attribute.KeyValue { - return K8SReplicationControllerUIDKey.String(val) -} - -// K8SResourceQuotaName returns an attribute KeyValue conforming to the -// "k8s.resourcequota.name" semantic conventions. It represents the name of the -// resource quota. -func K8SResourceQuotaName(val string) attribute.KeyValue { - return K8SResourceQuotaNameKey.String(val) -} - -// K8SResourceQuotaResourceName returns an attribute KeyValue conforming to the -// "k8s.resourcequota.resource_name" semantic conventions. It represents the name -// of the K8s resource a resource quota defines. -func K8SResourceQuotaResourceName(val string) attribute.KeyValue { - return K8SResourceQuotaResourceNameKey.String(val) -} - -// K8SResourceQuotaUID returns an attribute KeyValue conforming to the -// "k8s.resourcequota.uid" semantic conventions. It represents the UID of the -// resource quota. -func K8SResourceQuotaUID(val string) attribute.KeyValue { - return K8SResourceQuotaUIDKey.String(val) -} - -// K8SStatefulSetAnnotation returns an attribute KeyValue conforming to the -// "k8s.statefulset.annotation" semantic conventions. It represents the -// annotation placed on the StatefulSet, the `` being the annotation name, -// the value being the annotation value, even if the value is empty. -func K8SStatefulSetAnnotation(key string, val string) attribute.KeyValue { - return attribute.String("k8s.statefulset.annotation."+key, val) -} - -// K8SStatefulSetLabel returns an attribute KeyValue conforming to the -// "k8s.statefulset.label" semantic conventions. It represents the label placed -// on the StatefulSet, the `` being the label name, the value being the -// label value, even if the value is empty. -func K8SStatefulSetLabel(key string, val string) attribute.KeyValue { - return attribute.String("k8s.statefulset.label."+key, val) -} - -// K8SStatefulSetName returns an attribute KeyValue conforming to the -// "k8s.statefulset.name" semantic conventions. It represents the name of the -// StatefulSet. -func K8SStatefulSetName(val string) attribute.KeyValue { - return K8SStatefulSetNameKey.String(val) -} - -// K8SStatefulSetUID returns an attribute KeyValue conforming to the -// "k8s.statefulset.uid" semantic conventions. It represents the UID of the -// StatefulSet. -func K8SStatefulSetUID(val string) attribute.KeyValue { - return K8SStatefulSetUIDKey.String(val) -} - -// K8SStorageclassName returns an attribute KeyValue conforming to the -// "k8s.storageclass.name" semantic conventions. It represents the name of K8s -// [StorageClass] object. -// -// [StorageClass]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#storageclass-v1-storage-k8s-io -func K8SStorageclassName(val string) attribute.KeyValue { - return K8SStorageclassNameKey.String(val) -} - -// K8SVolumeName returns an attribute KeyValue conforming to the -// "k8s.volume.name" semantic conventions. It represents the name of the K8s -// volume. -func K8SVolumeName(val string) attribute.KeyValue { - return K8SVolumeNameKey.String(val) -} - -// Enum values for k8s.container.status.reason -var ( - // The container is being created. - // Stability: development - K8SContainerStatusReasonContainerCreating = K8SContainerStatusReasonKey.String("ContainerCreating") - // The container is in a crash loop back off state. - // Stability: development - K8SContainerStatusReasonCrashLoopBackOff = K8SContainerStatusReasonKey.String("CrashLoopBackOff") - // There was an error creating the container configuration. - // Stability: development - K8SContainerStatusReasonCreateContainerConfigError = K8SContainerStatusReasonKey.String("CreateContainerConfigError") - // There was an error pulling the container image. - // Stability: development - K8SContainerStatusReasonErrImagePull = K8SContainerStatusReasonKey.String("ErrImagePull") - // The container image pull is in back off state. - // Stability: development - K8SContainerStatusReasonImagePullBackOff = K8SContainerStatusReasonKey.String("ImagePullBackOff") - // The container was killed due to out of memory. - // Stability: development - K8SContainerStatusReasonOomKilled = K8SContainerStatusReasonKey.String("OOMKilled") - // The container has completed execution. - // Stability: development - K8SContainerStatusReasonCompleted = K8SContainerStatusReasonKey.String("Completed") - // There was an error with the container. - // Stability: development - K8SContainerStatusReasonError = K8SContainerStatusReasonKey.String("Error") - // The container cannot run. - // Stability: development - K8SContainerStatusReasonContainerCannotRun = K8SContainerStatusReasonKey.String("ContainerCannotRun") -) - -// Enum values for k8s.container.status.state -var ( - // The container has terminated. - // Stability: development - K8SContainerStatusStateTerminated = K8SContainerStatusStateKey.String("terminated") - // The container is running. - // Stability: development - K8SContainerStatusStateRunning = K8SContainerStatusStateKey.String("running") - // The container is waiting. - // Stability: development - K8SContainerStatusStateWaiting = K8SContainerStatusStateKey.String("waiting") -) - -// Enum values for k8s.namespace.phase -var ( - // Active namespace phase as described by [K8s API] - // Stability: development - // - // [K8s API]: https://pkg.go.dev/k8s.io/api@v0.31.3/core/v1#NamespacePhase - K8SNamespacePhaseActive = K8SNamespacePhaseKey.String("active") - // Terminating namespace phase as described by [K8s API] - // Stability: development - // - // [K8s API]: https://pkg.go.dev/k8s.io/api@v0.31.3/core/v1#NamespacePhase - K8SNamespacePhaseTerminating = K8SNamespacePhaseKey.String("terminating") -) - -// Enum values for k8s.node.condition.status -var ( - // condition_true - // Stability: development - K8SNodeConditionStatusConditionTrue = K8SNodeConditionStatusKey.String("true") - // condition_false - // Stability: development - K8SNodeConditionStatusConditionFalse = K8SNodeConditionStatusKey.String("false") - // condition_unknown - // Stability: development - K8SNodeConditionStatusConditionUnknown = K8SNodeConditionStatusKey.String("unknown") -) - -// Enum values for k8s.node.condition.type -var ( - // The node is healthy and ready to accept pods - // Stability: development - K8SNodeConditionTypeReady = K8SNodeConditionTypeKey.String("Ready") - // Pressure exists on the disk size—that is, if the disk capacity is low - // Stability: development - K8SNodeConditionTypeDiskPressure = K8SNodeConditionTypeKey.String("DiskPressure") - // Pressure exists on the node memory—that is, if the node memory is low - // Stability: development - K8SNodeConditionTypeMemoryPressure = K8SNodeConditionTypeKey.String("MemoryPressure") - // Pressure exists on the processes—that is, if there are too many processes - // on the node - // Stability: development - K8SNodeConditionTypePIDPressure = K8SNodeConditionTypeKey.String("PIDPressure") - // The network for the node is not correctly configured - // Stability: development - K8SNodeConditionTypeNetworkUnavailable = K8SNodeConditionTypeKey.String("NetworkUnavailable") -) - -// Enum values for k8s.pod.status.phase -var ( - // The pod has been accepted by the system, but one or more of the containers - // has not been started. This includes time before being bound to a node, as - // well as time spent pulling images onto the host. - // - // Stability: development - K8SPodStatusPhasePending = K8SPodStatusPhaseKey.String("Pending") - // The pod has been bound to a node and all of the containers have been started. - // At least one container is still running or is in the process of being - // restarted. - // - // Stability: development - K8SPodStatusPhaseRunning = K8SPodStatusPhaseKey.String("Running") - // All containers in the pod have voluntarily terminated with a container exit - // code of 0, and the system is not going to restart any of these containers. - // - // Stability: development - K8SPodStatusPhaseSucceeded = K8SPodStatusPhaseKey.String("Succeeded") - // All containers in the pod have terminated, and at least one container has - // terminated in a failure (exited with a non-zero exit code or was stopped by - // the system). - // - // Stability: development - K8SPodStatusPhaseFailed = K8SPodStatusPhaseKey.String("Failed") - // For some reason the state of the pod could not be obtained, typically due to - // an error in communicating with the host of the pod. - // - // Stability: development - K8SPodStatusPhaseUnknown = K8SPodStatusPhaseKey.String("Unknown") -) - -// Enum values for k8s.pod.status.reason -var ( - // The pod is evicted. - // Stability: development - K8SPodStatusReasonEvicted = K8SPodStatusReasonKey.String("Evicted") - // The pod is in a status because of its node affinity - // Stability: development - K8SPodStatusReasonNodeAffinity = K8SPodStatusReasonKey.String("NodeAffinity") - // The reason on a pod when its state cannot be confirmed as kubelet is - // unresponsive on the node it is (was) running. - // - // Stability: development - K8SPodStatusReasonNodeLost = K8SPodStatusReasonKey.String("NodeLost") - // The node is shutdown - // Stability: development - K8SPodStatusReasonShutdown = K8SPodStatusReasonKey.String("Shutdown") - // The pod was rejected admission to the node because of an error during - // admission that could not be categorized. - // - // Stability: development - K8SPodStatusReasonUnexpectedAdmissionError = K8SPodStatusReasonKey.String("UnexpectedAdmissionError") -) - -// Enum values for k8s.volume.type -var ( - // A [persistentVolumeClaim] volume - // Stability: development - // - // [persistentVolumeClaim]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#persistentvolumeclaim - K8SVolumeTypePersistentVolumeClaim = K8SVolumeTypeKey.String("persistentVolumeClaim") - // A [configMap] volume - // Stability: development - // - // [configMap]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#configmap - K8SVolumeTypeConfigMap = K8SVolumeTypeKey.String("configMap") - // A [downwardAPI] volume - // Stability: development - // - // [downwardAPI]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#downwardapi - K8SVolumeTypeDownwardAPI = K8SVolumeTypeKey.String("downwardAPI") - // An [emptyDir] volume - // Stability: development - // - // [emptyDir]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#emptydir - K8SVolumeTypeEmptyDir = K8SVolumeTypeKey.String("emptyDir") - // A [secret] volume - // Stability: development - // - // [secret]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#secret - K8SVolumeTypeSecret = K8SVolumeTypeKey.String("secret") - // A [local] volume - // Stability: development - // - // [local]: https://v1-30.docs.kubernetes.io/docs/concepts/storage/volumes/#local - K8SVolumeTypeLocal = K8SVolumeTypeKey.String("local") -) - -// Namespace: log -const ( - // LogFileNameKey is the attribute Key conforming to the "log.file.name" - // semantic conventions. It represents the basename of the file. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "audit.log" - LogFileNameKey = attribute.Key("log.file.name") - - // LogFileNameResolvedKey is the attribute Key conforming to the - // "log.file.name_resolved" semantic conventions. It represents the basename of - // the file, with symlinks resolved. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "uuid.log" - LogFileNameResolvedKey = attribute.Key("log.file.name_resolved") - - // LogFilePathKey is the attribute Key conforming to the "log.file.path" - // semantic conventions. It represents the full path to the file. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/var/log/mysql/audit.log" - LogFilePathKey = attribute.Key("log.file.path") - - // LogFilePathResolvedKey is the attribute Key conforming to the - // "log.file.path_resolved" semantic conventions. It represents the full path to - // the file, with symlinks resolved. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/var/lib/docker/uuid.log" - LogFilePathResolvedKey = attribute.Key("log.file.path_resolved") - - // LogIostreamKey is the attribute Key conforming to the "log.iostream" semantic - // conventions. It represents the stream associated with the log. See below for - // a list of well-known values. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - LogIostreamKey = attribute.Key("log.iostream") - - // LogRecordOriginalKey is the attribute Key conforming to the - // "log.record.original" semantic conventions. It represents the complete - // original Log Record. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "77 <86>1 2015-08-06T21:58:59.694Z 192.168.2.133 inactive - - - - // Something happened", "[INFO] 8/3/24 12:34:56 Something happened" - // Note: This value MAY be added when processing a Log Record which was - // originally transmitted as a string or equivalent data type AND the Body field - // of the Log Record does not contain the same value. (e.g. a syslog or a log - // record read from a file.) - LogRecordOriginalKey = attribute.Key("log.record.original") - - // LogRecordUIDKey is the attribute Key conforming to the "log.record.uid" - // semantic conventions. It represents a unique identifier for the Log Record. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "01ARZ3NDEKTSV4RRFFQ69G5FAV" - // Note: If an id is provided, other log records with the same id will be - // considered duplicates and can be removed safely. This means, that two - // distinguishable log records MUST have different values. - // The id MAY be an - // [Universally Unique Lexicographically Sortable Identifier (ULID)], but other - // identifiers (e.g. UUID) may be used as needed. - // - // [Universally Unique Lexicographically Sortable Identifier (ULID)]: https://github.com/ulid/spec - LogRecordUIDKey = attribute.Key("log.record.uid") -) - -// LogFileName returns an attribute KeyValue conforming to the "log.file.name" -// semantic conventions. It represents the basename of the file. -func LogFileName(val string) attribute.KeyValue { - return LogFileNameKey.String(val) -} - -// LogFileNameResolved returns an attribute KeyValue conforming to the -// "log.file.name_resolved" semantic conventions. It represents the basename of -// the file, with symlinks resolved. -func LogFileNameResolved(val string) attribute.KeyValue { - return LogFileNameResolvedKey.String(val) -} - -// LogFilePath returns an attribute KeyValue conforming to the "log.file.path" -// semantic conventions. It represents the full path to the file. -func LogFilePath(val string) attribute.KeyValue { - return LogFilePathKey.String(val) -} - -// LogFilePathResolved returns an attribute KeyValue conforming to the -// "log.file.path_resolved" semantic conventions. It represents the full path to -// the file, with symlinks resolved. -func LogFilePathResolved(val string) attribute.KeyValue { - return LogFilePathResolvedKey.String(val) -} - -// LogRecordOriginal returns an attribute KeyValue conforming to the -// "log.record.original" semantic conventions. It represents the complete -// original Log Record. -func LogRecordOriginal(val string) attribute.KeyValue { - return LogRecordOriginalKey.String(val) -} - -// LogRecordUID returns an attribute KeyValue conforming to the "log.record.uid" -// semantic conventions. It represents a unique identifier for the Log Record. -func LogRecordUID(val string) attribute.KeyValue { - return LogRecordUIDKey.String(val) -} - -// Enum values for log.iostream -var ( - // Logs from stdout stream - // Stability: development - LogIostreamStdout = LogIostreamKey.String("stdout") - // Events from stderr stream - // Stability: development - LogIostreamStderr = LogIostreamKey.String("stderr") -) - -// Namespace: mainframe -const ( - // MainframeLparNameKey is the attribute Key conforming to the - // "mainframe.lpar.name" semantic conventions. It represents the name of the - // logical partition that hosts a systems with a mainframe operating system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "LPAR01" - MainframeLparNameKey = attribute.Key("mainframe.lpar.name") -) - -// MainframeLparName returns an attribute KeyValue conforming to the -// "mainframe.lpar.name" semantic conventions. It represents the name of the -// logical partition that hosts a systems with a mainframe operating system. -func MainframeLparName(val string) attribute.KeyValue { - return MainframeLparNameKey.String(val) -} - -// Namespace: mcp -const ( - // McpMethodNameKey is the attribute Key conforming to the "mcp.method.name" - // semantic conventions. It represents the name of the request or notification - // method. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - McpMethodNameKey = attribute.Key("mcp.method.name") - - // McpProtocolVersionKey is the attribute Key conforming to the - // "mcp.protocol.version" semantic conventions. It represents the [version] of - // the Model Context Protocol used. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2025-06-18" - // - // [version]: https://modelcontextprotocol.io/specification/versioning - McpProtocolVersionKey = attribute.Key("mcp.protocol.version") - - // McpResourceURIKey is the attribute Key conforming to the "mcp.resource.uri" - // semantic conventions. It represents the value of the resource uri. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "postgres://database/customers/schema", - // "file:///home/user/documents/report.pdf" - // Note: This is a URI of the resource provided in the following requests or - // notifications: `resources/read`, `resources/subscribe`, - // `resources/unsubscribe`, or `notifications/resources/updated`. - McpResourceURIKey = attribute.Key("mcp.resource.uri") - - // McpSessionIDKey is the attribute Key conforming to the "mcp.session.id" - // semantic conventions. It represents the identifies [MCP session]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "191c4850af6c49e08843a3f6c80e5046" - // - // [MCP session]: https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#session-management - McpSessionIDKey = attribute.Key("mcp.session.id") -) - -// McpProtocolVersion returns an attribute KeyValue conforming to the -// "mcp.protocol.version" semantic conventions. It represents the [version] of -// the Model Context Protocol used. -// -// [version]: https://modelcontextprotocol.io/specification/versioning -func McpProtocolVersion(val string) attribute.KeyValue { - return McpProtocolVersionKey.String(val) -} - -// McpResourceURI returns an attribute KeyValue conforming to the -// "mcp.resource.uri" semantic conventions. It represents the value of the -// resource uri. -func McpResourceURI(val string) attribute.KeyValue { - return McpResourceURIKey.String(val) -} - -// McpSessionID returns an attribute KeyValue conforming to the "mcp.session.id" -// semantic conventions. It represents the identifies [MCP session]. -// -// [MCP session]: https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#session-management -func McpSessionID(val string) attribute.KeyValue { - return McpSessionIDKey.String(val) -} - -// Enum values for mcp.method.name -var ( - // Notification cancelling a previously-issued request. - // - // Stability: development - McpMethodNameNotificationsCancelled = McpMethodNameKey.String("notifications/cancelled") - // Request to initialize the MCP client. - // - // Stability: development - McpMethodNameInitialize = McpMethodNameKey.String("initialize") - // Notification indicating that the MCP client has been initialized. - // - // Stability: development - McpMethodNameNotificationsInitialized = McpMethodNameKey.String("notifications/initialized") - // Notification indicating the progress for a long-running operation. - // - // Stability: development - McpMethodNameNotificationsProgress = McpMethodNameKey.String("notifications/progress") - // Request to check that the other party is still alive. - // - // Stability: development - McpMethodNamePing = McpMethodNameKey.String("ping") - // Request to list resources available on server. - // - // Stability: development - McpMethodNameResourcesList = McpMethodNameKey.String("resources/list") - // Request to list resource templates available on server. - // - // Stability: development - McpMethodNameResourcesTemplatesList = McpMethodNameKey.String("resources/templates/list") - // Request to read a resource. - // - // Stability: development - McpMethodNameResourcesRead = McpMethodNameKey.String("resources/read") - // Notification indicating that the list of resources has changed. - // - // Stability: development - McpMethodNameNotificationsResourcesListChanged = McpMethodNameKey.String("notifications/resources/list_changed") - // Request to subscribe to a resource. - // - // Stability: development - McpMethodNameResourcesSubscribe = McpMethodNameKey.String("resources/subscribe") - // Request to unsubscribe from resource updates. - // - // Stability: development - McpMethodNameResourcesUnsubscribe = McpMethodNameKey.String("resources/unsubscribe") - // Notification indicating that a resource has been updated. - // - // Stability: development - McpMethodNameNotificationsResourcesUpdated = McpMethodNameKey.String("notifications/resources/updated") - // Request to list prompts available on server. - // - // Stability: development - McpMethodNamePromptsList = McpMethodNameKey.String("prompts/list") - // Request to get a prompt. - // - // Stability: development - McpMethodNamePromptsGet = McpMethodNameKey.String("prompts/get") - // Notification indicating that the list of prompts has changed. - // - // Stability: development - McpMethodNameNotificationsPromptsListChanged = McpMethodNameKey.String("notifications/prompts/list_changed") - // Request to list tools available on server. - // - // Stability: development - McpMethodNameToolsList = McpMethodNameKey.String("tools/list") - // Request to call a tool. - // - // Stability: development - McpMethodNameToolsCall = McpMethodNameKey.String("tools/call") - // Notification indicating that the list of tools has changed. - // - // Stability: development - McpMethodNameNotificationsToolsListChanged = McpMethodNameKey.String("notifications/tools/list_changed") - // Request to set the logging level. - // - // Stability: development - McpMethodNameLoggingSetLevel = McpMethodNameKey.String("logging/setLevel") - // Notification indicating that a message has been received. - // - // Stability: development - McpMethodNameNotificationsMessage = McpMethodNameKey.String("notifications/message") - // Request to create a sampling message. - // - // Stability: development - McpMethodNameSamplingCreateMessage = McpMethodNameKey.String("sampling/createMessage") - // Request to complete a prompt. - // - // Stability: development - McpMethodNameCompletionComplete = McpMethodNameKey.String("completion/complete") - // Request to list roots available on server. - // - // Stability: development - McpMethodNameRootsList = McpMethodNameKey.String("roots/list") - // Notification indicating that the list of roots has changed. - // - // Stability: development - McpMethodNameNotificationsRootsListChanged = McpMethodNameKey.String("notifications/roots/list_changed") - // Request from the server to elicit additional information from the user via - // the client - // - // Stability: development - McpMethodNameElicitationCreate = McpMethodNameKey.String("elicitation/create") -) - -// Namespace: messaging -const ( - // MessagingBatchMessageCountKey is the attribute Key conforming to the - // "messaging.batch.message_count" semantic conventions. It represents the - // number of messages sent, received, or processed in the scope of the batching - // operation. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 0, 1, 2 - // Note: Instrumentations SHOULD NOT set `messaging.batch.message_count` on - // spans that operate with a single message. When a messaging client library - // supports both batch and single-message API for the same operation, - // instrumentations SHOULD use `messaging.batch.message_count` for batching APIs - // and SHOULD NOT use it for single-message APIs. - MessagingBatchMessageCountKey = attribute.Key("messaging.batch.message_count") - - // MessagingClientIDKey is the attribute Key conforming to the - // "messaging.client.id" semantic conventions. It represents a unique identifier - // for the client that consumes or produces a message. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "client-5", "myhost@8742@s8083jm" - MessagingClientIDKey = attribute.Key("messaging.client.id") - - // MessagingConsumerGroupNameKey is the attribute Key conforming to the - // "messaging.consumer.group.name" semantic conventions. It represents the name - // of the consumer group with which a consumer is associated. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-group", "indexer" - // Note: Semantic conventions for individual messaging systems SHOULD document - // whether `messaging.consumer.group.name` is applicable and what it means in - // the context of that system. - MessagingConsumerGroupNameKey = attribute.Key("messaging.consumer.group.name") - - // MessagingDestinationAnonymousKey is the attribute Key conforming to the - // "messaging.destination.anonymous" semantic conventions. It represents a - // boolean that is true if the message destination is anonymous (could be - // unnamed or have auto-generated name). - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - MessagingDestinationAnonymousKey = attribute.Key("messaging.destination.anonymous") - - // MessagingDestinationNameKey is the attribute Key conforming to the - // "messaging.destination.name" semantic conventions. It represents the message - // destination name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MyQueue", "MyTopic" - // Note: Destination name SHOULD uniquely identify a specific queue, topic or - // other entity within the broker. If - // the broker doesn't have such notion, the destination name SHOULD uniquely - // identify the broker. - MessagingDestinationNameKey = attribute.Key("messaging.destination.name") - - // MessagingDestinationPartitionIDKey is the attribute Key conforming to the - // "messaging.destination.partition.id" semantic conventions. It represents the - // identifier of the partition messages are sent to or received from, unique - // within the `messaging.destination.name`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1 - MessagingDestinationPartitionIDKey = attribute.Key("messaging.destination.partition.id") - - // MessagingDestinationSubscriptionNameKey is the attribute Key conforming to - // the "messaging.destination.subscription.name" semantic conventions. It - // represents the name of the destination subscription from which a message is - // consumed. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "subscription-a" - // Note: Semantic conventions for individual messaging systems SHOULD document - // whether `messaging.destination.subscription.name` is applicable and what it - // means in the context of that system. - MessagingDestinationSubscriptionNameKey = attribute.Key("messaging.destination.subscription.name") - - // MessagingDestinationTemplateKey is the attribute Key conforming to the - // "messaging.destination.template" semantic conventions. It represents the low - // cardinality representation of the messaging destination name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/customers/{customerId}" - // Note: Destination names could be constructed from templates. An example would - // be a destination name involving a user name or product id. Although the - // destination name in this case is of high cardinality, the underlying template - // is of low cardinality and can be effectively used for grouping and - // aggregation. - MessagingDestinationTemplateKey = attribute.Key("messaging.destination.template") - - // MessagingDestinationTemporaryKey is the attribute Key conforming to the - // "messaging.destination.temporary" semantic conventions. It represents a - // boolean that is true if the message destination is temporary and might not - // exist anymore after messages are processed. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - MessagingDestinationTemporaryKey = attribute.Key("messaging.destination.temporary") - - // MessagingEventHubsMessageEnqueuedTimeKey is the attribute Key conforming to - // the "messaging.eventhubs.message.enqueued_time" semantic conventions. It - // represents the UTC epoch seconds at which the message has been accepted and - // stored in the entity. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingEventHubsMessageEnqueuedTimeKey = attribute.Key("messaging.eventhubs.message.enqueued_time") - - // MessagingGCPPubSubMessageAckDeadlineKey is the attribute Key conforming to - // the "messaging.gcp_pubsub.message.ack_deadline" semantic conventions. It - // represents the ack deadline in seconds set for the modify ack deadline - // request. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingGCPPubSubMessageAckDeadlineKey = attribute.Key("messaging.gcp_pubsub.message.ack_deadline") - - // MessagingGCPPubSubMessageAckIDKey is the attribute Key conforming to the - // "messaging.gcp_pubsub.message.ack_id" semantic conventions. It represents the - // ack id for a given message. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: ack_id - MessagingGCPPubSubMessageAckIDKey = attribute.Key("messaging.gcp_pubsub.message.ack_id") - - // MessagingGCPPubSubMessageDeliveryAttemptKey is the attribute Key conforming - // to the "messaging.gcp_pubsub.message.delivery_attempt" semantic conventions. - // It represents the delivery attempt for a given message. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingGCPPubSubMessageDeliveryAttemptKey = attribute.Key("messaging.gcp_pubsub.message.delivery_attempt") - - // MessagingGCPPubSubMessageOrderingKeyKey is the attribute Key conforming to - // the "messaging.gcp_pubsub.message.ordering_key" semantic conventions. It - // represents the ordering key for a given message. If the attribute is not - // present, the message does not have an ordering key. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: ordering_key - MessagingGCPPubSubMessageOrderingKeyKey = attribute.Key("messaging.gcp_pubsub.message.ordering_key") - - // MessagingKafkaMessageKeyKey is the attribute Key conforming to the - // "messaging.kafka.message.key" semantic conventions. It represents the message - // keys in Kafka are used for grouping alike messages to ensure they're - // processed on the same partition. They differ from `messaging.message.id` in - // that they're not unique. If the key is `null`, the attribute MUST NOT be set. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: myKey - // Note: If the key type is not string, it's string representation has to be - // supplied for the attribute. If the key has no unambiguous, canonical string - // form, don't include its value. - MessagingKafkaMessageKeyKey = attribute.Key("messaging.kafka.message.key") - - // MessagingKafkaMessageTombstoneKey is the attribute Key conforming to the - // "messaging.kafka.message.tombstone" semantic conventions. It represents a - // boolean that is true if the message is a tombstone. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - MessagingKafkaMessageTombstoneKey = attribute.Key("messaging.kafka.message.tombstone") - - // MessagingKafkaOffsetKey is the attribute Key conforming to the - // "messaging.kafka.offset" semantic conventions. It represents the offset of a - // record in the corresponding Kafka partition. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingKafkaOffsetKey = attribute.Key("messaging.kafka.offset") - - // MessagingMessageBodySizeKey is the attribute Key conforming to the - // "messaging.message.body.size" semantic conventions. It represents the size of - // the message body in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Note: This can refer to both the compressed or uncompressed body size. If - // both sizes are known, the uncompressed - // body size should be used. - MessagingMessageBodySizeKey = attribute.Key("messaging.message.body.size") - - // MessagingMessageConversationIDKey is the attribute Key conforming to the - // "messaging.message.conversation_id" semantic conventions. It represents the - // conversation ID identifying the conversation to which the message belongs, - // represented as a string. Sometimes called "Correlation ID". - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: MyConversationId - MessagingMessageConversationIDKey = attribute.Key("messaging.message.conversation_id") - - // MessagingMessageEnvelopeSizeKey is the attribute Key conforming to the - // "messaging.message.envelope.size" semantic conventions. It represents the - // size of the message body and metadata in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Note: This can refer to both the compressed or uncompressed size. If both - // sizes are known, the uncompressed - // size should be used. - MessagingMessageEnvelopeSizeKey = attribute.Key("messaging.message.envelope.size") - - // MessagingMessageIDKey is the attribute Key conforming to the - // "messaging.message.id" semantic conventions. It represents a value used by - // the messaging system as an identifier for the message, represented as a - // string. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 452a7c7c7c7048c2f887f61572b18fc2 - MessagingMessageIDKey = attribute.Key("messaging.message.id") - - // MessagingOperationNameKey is the attribute Key conforming to the - // "messaging.operation.name" semantic conventions. It represents the - // system-specific name of the messaging operation. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ack", "nack", "send" - MessagingOperationNameKey = attribute.Key("messaging.operation.name") - - // MessagingOperationTypeKey is the attribute Key conforming to the - // "messaging.operation.type" semantic conventions. It represents a string - // identifying the type of the messaging operation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: If a custom value is used, it MUST be of low cardinality. - MessagingOperationTypeKey = attribute.Key("messaging.operation.type") - - // MessagingRabbitMQDestinationRoutingKeyKey is the attribute Key conforming to - // the "messaging.rabbitmq.destination.routing_key" semantic conventions. It - // represents the rabbitMQ message routing key. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: myKey - MessagingRabbitMQDestinationRoutingKeyKey = attribute.Key("messaging.rabbitmq.destination.routing_key") - - // MessagingRabbitMQMessageDeliveryTagKey is the attribute Key conforming to the - // "messaging.rabbitmq.message.delivery_tag" semantic conventions. It represents - // the rabbitMQ message delivery tag. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingRabbitMQMessageDeliveryTagKey = attribute.Key("messaging.rabbitmq.message.delivery_tag") - - // MessagingRocketMQConsumptionModelKey is the attribute Key conforming to the - // "messaging.rocketmq.consumption_model" semantic conventions. It represents - // the model of message consumption. This only applies to consumer spans. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - MessagingRocketMQConsumptionModelKey = attribute.Key("messaging.rocketmq.consumption_model") - - // MessagingRocketMQMessageDelayTimeLevelKey is the attribute Key conforming to - // the "messaging.rocketmq.message.delay_time_level" semantic conventions. It - // represents the delay time level for delay message, which determines the - // message delay time. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingRocketMQMessageDelayTimeLevelKey = attribute.Key("messaging.rocketmq.message.delay_time_level") - - // MessagingRocketMQMessageDeliveryTimestampKey is the attribute Key conforming - // to the "messaging.rocketmq.message.delivery_timestamp" semantic conventions. - // It represents the timestamp in milliseconds that the delay message is - // expected to be delivered to consumer. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingRocketMQMessageDeliveryTimestampKey = attribute.Key("messaging.rocketmq.message.delivery_timestamp") - - // MessagingRocketMQMessageGroupKey is the attribute Key conforming to the - // "messaging.rocketmq.message.group" semantic conventions. It represents the it - // is essential for FIFO message. Messages that belong to the same message group - // are always processed one by one within the same consumer group. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: myMessageGroup - MessagingRocketMQMessageGroupKey = attribute.Key("messaging.rocketmq.message.group") - - // MessagingRocketMQMessageKeysKey is the attribute Key conforming to the - // "messaging.rocketmq.message.keys" semantic conventions. It represents the - // key(s) of message, another way to mark message besides message id. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "keyA", "keyB" - MessagingRocketMQMessageKeysKey = attribute.Key("messaging.rocketmq.message.keys") - - // MessagingRocketMQMessageTagKey is the attribute Key conforming to the - // "messaging.rocketmq.message.tag" semantic conventions. It represents the - // secondary classifier of message besides topic. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: tagA - MessagingRocketMQMessageTagKey = attribute.Key("messaging.rocketmq.message.tag") - - // MessagingRocketMQMessageTypeKey is the attribute Key conforming to the - // "messaging.rocketmq.message.type" semantic conventions. It represents the - // type of message. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - MessagingRocketMQMessageTypeKey = attribute.Key("messaging.rocketmq.message.type") - - // MessagingRocketMQNamespaceKey is the attribute Key conforming to the - // "messaging.rocketmq.namespace" semantic conventions. It represents the - // namespace of RocketMQ resources, resources in different namespaces are - // individual. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: myNamespace - MessagingRocketMQNamespaceKey = attribute.Key("messaging.rocketmq.namespace") - - // MessagingServiceBusDispositionStatusKey is the attribute Key conforming to - // the "messaging.servicebus.disposition_status" semantic conventions. It - // represents the describes the [settlement type]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [settlement type]: https://learn.microsoft.com/azure/service-bus-messaging/message-transfers-locks-settlement#peeklock - MessagingServiceBusDispositionStatusKey = attribute.Key("messaging.servicebus.disposition_status") - - // MessagingServiceBusMessageDeliveryCountKey is the attribute Key conforming to - // the "messaging.servicebus.message.delivery_count" semantic conventions. It - // represents the number of deliveries that have been attempted for this - // message. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingServiceBusMessageDeliveryCountKey = attribute.Key("messaging.servicebus.message.delivery_count") - - // MessagingServiceBusMessageEnqueuedTimeKey is the attribute Key conforming to - // the "messaging.servicebus.message.enqueued_time" semantic conventions. It - // represents the UTC epoch seconds at which the message has been accepted and - // stored in the entity. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - MessagingServiceBusMessageEnqueuedTimeKey = attribute.Key("messaging.servicebus.message.enqueued_time") - - // MessagingSystemKey is the attribute Key conforming to the "messaging.system" - // semantic conventions. It represents the messaging system as identified by the - // client instrumentation. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The actual messaging system may differ from the one known by the - // client. For example, when using Kafka client libraries to communicate with - // Azure Event Hubs, the `messaging.system` is set to `kafka` based on the - // instrumentation's best knowledge. - MessagingSystemKey = attribute.Key("messaging.system") -) - -// MessagingBatchMessageCount returns an attribute KeyValue conforming to the -// "messaging.batch.message_count" semantic conventions. It represents the number -// of messages sent, received, or processed in the scope of the batching -// operation. -func MessagingBatchMessageCount(val int) attribute.KeyValue { - return MessagingBatchMessageCountKey.Int(val) -} - -// MessagingClientID returns an attribute KeyValue conforming to the -// "messaging.client.id" semantic conventions. It represents a unique identifier -// for the client that consumes or produces a message. -func MessagingClientID(val string) attribute.KeyValue { - return MessagingClientIDKey.String(val) -} - -// MessagingConsumerGroupName returns an attribute KeyValue conforming to the -// "messaging.consumer.group.name" semantic conventions. It represents the name -// of the consumer group with which a consumer is associated. -func MessagingConsumerGroupName(val string) attribute.KeyValue { - return MessagingConsumerGroupNameKey.String(val) -} - -// MessagingDestinationAnonymous returns an attribute KeyValue conforming to the -// "messaging.destination.anonymous" semantic conventions. It represents a -// boolean that is true if the message destination is anonymous (could be unnamed -// or have auto-generated name). -func MessagingDestinationAnonymous(val bool) attribute.KeyValue { - return MessagingDestinationAnonymousKey.Bool(val) -} - -// MessagingDestinationName returns an attribute KeyValue conforming to the -// "messaging.destination.name" semantic conventions. It represents the message -// destination name. -func MessagingDestinationName(val string) attribute.KeyValue { - return MessagingDestinationNameKey.String(val) -} - -// MessagingDestinationPartitionID returns an attribute KeyValue conforming to -// the "messaging.destination.partition.id" semantic conventions. It represents -// the identifier of the partition messages are sent to or received from, unique -// within the `messaging.destination.name`. -func MessagingDestinationPartitionID(val string) attribute.KeyValue { - return MessagingDestinationPartitionIDKey.String(val) -} - -// MessagingDestinationSubscriptionName returns an attribute KeyValue conforming -// to the "messaging.destination.subscription.name" semantic conventions. It -// represents the name of the destination subscription from which a message is -// consumed. -func MessagingDestinationSubscriptionName(val string) attribute.KeyValue { - return MessagingDestinationSubscriptionNameKey.String(val) -} - -// MessagingDestinationTemplate returns an attribute KeyValue conforming to the -// "messaging.destination.template" semantic conventions. It represents the low -// cardinality representation of the messaging destination name. -func MessagingDestinationTemplate(val string) attribute.KeyValue { - return MessagingDestinationTemplateKey.String(val) -} - -// MessagingDestinationTemporary returns an attribute KeyValue conforming to the -// "messaging.destination.temporary" semantic conventions. It represents a -// boolean that is true if the message destination is temporary and might not -// exist anymore after messages are processed. -func MessagingDestinationTemporary(val bool) attribute.KeyValue { - return MessagingDestinationTemporaryKey.Bool(val) -} - -// MessagingEventHubsMessageEnqueuedTime returns an attribute KeyValue conforming -// to the "messaging.eventhubs.message.enqueued_time" semantic conventions. It -// represents the UTC epoch seconds at which the message has been accepted and -// stored in the entity. -func MessagingEventHubsMessageEnqueuedTime(val int) attribute.KeyValue { - return MessagingEventHubsMessageEnqueuedTimeKey.Int(val) -} - -// MessagingGCPPubSubMessageAckDeadline returns an attribute KeyValue conforming -// to the "messaging.gcp_pubsub.message.ack_deadline" semantic conventions. It -// represents the ack deadline in seconds set for the modify ack deadline -// request. -func MessagingGCPPubSubMessageAckDeadline(val int) attribute.KeyValue { - return MessagingGCPPubSubMessageAckDeadlineKey.Int(val) -} - -// MessagingGCPPubSubMessageAckID returns an attribute KeyValue conforming to the -// "messaging.gcp_pubsub.message.ack_id" semantic conventions. It represents the -// ack id for a given message. -func MessagingGCPPubSubMessageAckID(val string) attribute.KeyValue { - return MessagingGCPPubSubMessageAckIDKey.String(val) -} - -// MessagingGCPPubSubMessageDeliveryAttempt returns an attribute KeyValue -// conforming to the "messaging.gcp_pubsub.message.delivery_attempt" semantic -// conventions. It represents the delivery attempt for a given message. -func MessagingGCPPubSubMessageDeliveryAttempt(val int) attribute.KeyValue { - return MessagingGCPPubSubMessageDeliveryAttemptKey.Int(val) -} - -// MessagingGCPPubSubMessageOrderingKey returns an attribute KeyValue conforming -// to the "messaging.gcp_pubsub.message.ordering_key" semantic conventions. It -// represents the ordering key for a given message. If the attribute is not -// present, the message does not have an ordering key. -func MessagingGCPPubSubMessageOrderingKey(val string) attribute.KeyValue { - return MessagingGCPPubSubMessageOrderingKeyKey.String(val) -} - -// MessagingKafkaMessageKey returns an attribute KeyValue conforming to the -// "messaging.kafka.message.key" semantic conventions. It represents the message -// keys in Kafka are used for grouping alike messages to ensure they're processed -// on the same partition. They differ from `messaging.message.id` in that they're -// not unique. If the key is `null`, the attribute MUST NOT be set. -func MessagingKafkaMessageKey(val string) attribute.KeyValue { - return MessagingKafkaMessageKeyKey.String(val) -} - -// MessagingKafkaMessageTombstone returns an attribute KeyValue conforming to the -// "messaging.kafka.message.tombstone" semantic conventions. It represents a -// boolean that is true if the message is a tombstone. -func MessagingKafkaMessageTombstone(val bool) attribute.KeyValue { - return MessagingKafkaMessageTombstoneKey.Bool(val) -} - -// MessagingKafkaOffset returns an attribute KeyValue conforming to the -// "messaging.kafka.offset" semantic conventions. It represents the offset of a -// record in the corresponding Kafka partition. -func MessagingKafkaOffset(val int) attribute.KeyValue { - return MessagingKafkaOffsetKey.Int(val) -} - -// MessagingMessageBodySize returns an attribute KeyValue conforming to the -// "messaging.message.body.size" semantic conventions. It represents the size of -// the message body in bytes. -func MessagingMessageBodySize(val int) attribute.KeyValue { - return MessagingMessageBodySizeKey.Int(val) -} - -// MessagingMessageConversationID returns an attribute KeyValue conforming to the -// "messaging.message.conversation_id" semantic conventions. It represents the -// conversation ID identifying the conversation to which the message belongs, -// represented as a string. Sometimes called "Correlation ID". -func MessagingMessageConversationID(val string) attribute.KeyValue { - return MessagingMessageConversationIDKey.String(val) -} - -// MessagingMessageEnvelopeSize returns an attribute KeyValue conforming to the -// "messaging.message.envelope.size" semantic conventions. It represents the size -// of the message body and metadata in bytes. -func MessagingMessageEnvelopeSize(val int) attribute.KeyValue { - return MessagingMessageEnvelopeSizeKey.Int(val) -} - -// MessagingMessageID returns an attribute KeyValue conforming to the -// "messaging.message.id" semantic conventions. It represents a value used by the -// messaging system as an identifier for the message, represented as a string. -func MessagingMessageID(val string) attribute.KeyValue { - return MessagingMessageIDKey.String(val) -} - -// MessagingOperationName returns an attribute KeyValue conforming to the -// "messaging.operation.name" semantic conventions. It represents the -// system-specific name of the messaging operation. -func MessagingOperationName(val string) attribute.KeyValue { - return MessagingOperationNameKey.String(val) -} - -// MessagingRabbitMQDestinationRoutingKey returns an attribute KeyValue -// conforming to the "messaging.rabbitmq.destination.routing_key" semantic -// conventions. It represents the rabbitMQ message routing key. -func MessagingRabbitMQDestinationRoutingKey(val string) attribute.KeyValue { - return MessagingRabbitMQDestinationRoutingKeyKey.String(val) -} - -// MessagingRabbitMQMessageDeliveryTag returns an attribute KeyValue conforming -// to the "messaging.rabbitmq.message.delivery_tag" semantic conventions. It -// represents the rabbitMQ message delivery tag. -func MessagingRabbitMQMessageDeliveryTag(val int) attribute.KeyValue { - return MessagingRabbitMQMessageDeliveryTagKey.Int(val) -} - -// MessagingRocketMQMessageDelayTimeLevel returns an attribute KeyValue -// conforming to the "messaging.rocketmq.message.delay_time_level" semantic -// conventions. It represents the delay time level for delay message, which -// determines the message delay time. -func MessagingRocketMQMessageDelayTimeLevel(val int) attribute.KeyValue { - return MessagingRocketMQMessageDelayTimeLevelKey.Int(val) -} - -// MessagingRocketMQMessageDeliveryTimestamp returns an attribute KeyValue -// conforming to the "messaging.rocketmq.message.delivery_timestamp" semantic -// conventions. It represents the timestamp in milliseconds that the delay -// message is expected to be delivered to consumer. -func MessagingRocketMQMessageDeliveryTimestamp(val int) attribute.KeyValue { - return MessagingRocketMQMessageDeliveryTimestampKey.Int(val) -} - -// MessagingRocketMQMessageGroup returns an attribute KeyValue conforming to the -// "messaging.rocketmq.message.group" semantic conventions. It represents the it -// is essential for FIFO message. Messages that belong to the same message group -// are always processed one by one within the same consumer group. -func MessagingRocketMQMessageGroup(val string) attribute.KeyValue { - return MessagingRocketMQMessageGroupKey.String(val) -} - -// MessagingRocketMQMessageKeys returns an attribute KeyValue conforming to the -// "messaging.rocketmq.message.keys" semantic conventions. It represents the -// key(s) of message, another way to mark message besides message id. -func MessagingRocketMQMessageKeys(val ...string) attribute.KeyValue { - return MessagingRocketMQMessageKeysKey.StringSlice(val) -} - -// MessagingRocketMQMessageTag returns an attribute KeyValue conforming to the -// "messaging.rocketmq.message.tag" semantic conventions. It represents the -// secondary classifier of message besides topic. -func MessagingRocketMQMessageTag(val string) attribute.KeyValue { - return MessagingRocketMQMessageTagKey.String(val) -} - -// MessagingRocketMQNamespace returns an attribute KeyValue conforming to the -// "messaging.rocketmq.namespace" semantic conventions. It represents the -// namespace of RocketMQ resources, resources in different namespaces are -// individual. -func MessagingRocketMQNamespace(val string) attribute.KeyValue { - return MessagingRocketMQNamespaceKey.String(val) -} - -// MessagingServiceBusMessageDeliveryCount returns an attribute KeyValue -// conforming to the "messaging.servicebus.message.delivery_count" semantic -// conventions. It represents the number of deliveries that have been attempted -// for this message. -func MessagingServiceBusMessageDeliveryCount(val int) attribute.KeyValue { - return MessagingServiceBusMessageDeliveryCountKey.Int(val) -} - -// MessagingServiceBusMessageEnqueuedTime returns an attribute KeyValue -// conforming to the "messaging.servicebus.message.enqueued_time" semantic -// conventions. It represents the UTC epoch seconds at which the message has been -// accepted and stored in the entity. -func MessagingServiceBusMessageEnqueuedTime(val int) attribute.KeyValue { - return MessagingServiceBusMessageEnqueuedTimeKey.Int(val) -} - -// Enum values for messaging.operation.type -var ( - // A message is created. "Create" spans always refer to a single message and are - // used to provide a unique creation context for messages in batch sending - // scenarios. - // - // Stability: development - MessagingOperationTypeCreate = MessagingOperationTypeKey.String("create") - // One or more messages are provided for sending to an intermediary. If a single - // message is sent, the context of the "Send" span can be used as the creation - // context and no "Create" span needs to be created. - // - // Stability: development - MessagingOperationTypeSend = MessagingOperationTypeKey.String("send") - // One or more messages are requested by a consumer. This operation refers to - // pull-based scenarios, where consumers explicitly call methods of messaging - // SDKs to receive messages. - // - // Stability: development - MessagingOperationTypeReceive = MessagingOperationTypeKey.String("receive") - // One or more messages are processed by a consumer. - // - // Stability: development - MessagingOperationTypeProcess = MessagingOperationTypeKey.String("process") - // One or more messages are settled. - // - // Stability: development - MessagingOperationTypeSettle = MessagingOperationTypeKey.String("settle") -) - -// Enum values for messaging.rocketmq.consumption_model -var ( - // Clustering consumption model - // Stability: development - MessagingRocketMQConsumptionModelClustering = MessagingRocketMQConsumptionModelKey.String("clustering") - // Broadcasting consumption model - // Stability: development - MessagingRocketMQConsumptionModelBroadcasting = MessagingRocketMQConsumptionModelKey.String("broadcasting") -) - -// Enum values for messaging.rocketmq.message.type -var ( - // Normal message - // Stability: development - MessagingRocketMQMessageTypeNormal = MessagingRocketMQMessageTypeKey.String("normal") - // FIFO message - // Stability: development - MessagingRocketMQMessageTypeFifo = MessagingRocketMQMessageTypeKey.String("fifo") - // Delay message - // Stability: development - MessagingRocketMQMessageTypeDelay = MessagingRocketMQMessageTypeKey.String("delay") - // Transaction message - // Stability: development - MessagingRocketMQMessageTypeTransaction = MessagingRocketMQMessageTypeKey.String("transaction") -) - -// Enum values for messaging.servicebus.disposition_status -var ( - // Message is completed - // Stability: development - MessagingServiceBusDispositionStatusComplete = MessagingServiceBusDispositionStatusKey.String("complete") - // Message is abandoned - // Stability: development - MessagingServiceBusDispositionStatusAbandon = MessagingServiceBusDispositionStatusKey.String("abandon") - // Message is sent to dead letter queue - // Stability: development - MessagingServiceBusDispositionStatusDeadLetter = MessagingServiceBusDispositionStatusKey.String("dead_letter") - // Message is deferred - // Stability: development - MessagingServiceBusDispositionStatusDefer = MessagingServiceBusDispositionStatusKey.String("defer") -) - -// Enum values for messaging.system -var ( - // Apache ActiveMQ - // Stability: development - MessagingSystemActiveMQ = MessagingSystemKey.String("activemq") - // Amazon Simple Notification Service (SNS) - // Stability: development - MessagingSystemAWSSNS = MessagingSystemKey.String("aws.sns") - // Amazon Simple Queue Service (SQS) - // Stability: development - MessagingSystemAWSSQS = MessagingSystemKey.String("aws_sqs") - // Azure Event Grid - // Stability: development - MessagingSystemEventGrid = MessagingSystemKey.String("eventgrid") - // Azure Event Hubs - // Stability: development - MessagingSystemEventHubs = MessagingSystemKey.String("eventhubs") - // Azure Service Bus - // Stability: development - MessagingSystemServiceBus = MessagingSystemKey.String("servicebus") - // Google Cloud Pub/Sub - // Stability: development - MessagingSystemGCPPubSub = MessagingSystemKey.String("gcp_pubsub") - // Java Message Service - // Stability: development - MessagingSystemJMS = MessagingSystemKey.String("jms") - // Apache Kafka - // Stability: development - MessagingSystemKafka = MessagingSystemKey.String("kafka") - // RabbitMQ - // Stability: development - MessagingSystemRabbitMQ = MessagingSystemKey.String("rabbitmq") - // Apache RocketMQ - // Stability: development - MessagingSystemRocketMQ = MessagingSystemKey.String("rocketmq") - // Apache Pulsar - // Stability: development - MessagingSystemPulsar = MessagingSystemKey.String("pulsar") -) - -// Namespace: network -const ( - // NetworkCarrierICCKey is the attribute Key conforming to the - // "network.carrier.icc" semantic conventions. It represents the ISO 3166-1 - // alpha-2 2-character country code associated with the mobile carrier network. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: DE - NetworkCarrierICCKey = attribute.Key("network.carrier.icc") - - // NetworkCarrierMCCKey is the attribute Key conforming to the - // "network.carrier.mcc" semantic conventions. It represents the mobile carrier - // country code. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 310 - NetworkCarrierMCCKey = attribute.Key("network.carrier.mcc") - - // NetworkCarrierMNCKey is the attribute Key conforming to the - // "network.carrier.mnc" semantic conventions. It represents the mobile carrier - // network code. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 001 - NetworkCarrierMNCKey = attribute.Key("network.carrier.mnc") - - // NetworkCarrierNameKey is the attribute Key conforming to the - // "network.carrier.name" semantic conventions. It represents the name of the - // mobile carrier. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: sprint - NetworkCarrierNameKey = attribute.Key("network.carrier.name") - - // NetworkConnectionStateKey is the attribute Key conforming to the - // "network.connection.state" semantic conventions. It represents the state of - // network connection. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "close_wait" - // Note: Connection states are defined as part of the [rfc9293] - // - // [rfc9293]: https://datatracker.ietf.org/doc/html/rfc9293#section-3.3.2 - NetworkConnectionStateKey = attribute.Key("network.connection.state") - - // NetworkConnectionSubtypeKey is the attribute Key conforming to the - // "network.connection.subtype" semantic conventions. It represents the this - // describes more details regarding the connection.type. It may be the type of - // cell technology connection, but it could be used for describing details about - // a wifi connection. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: LTE - NetworkConnectionSubtypeKey = attribute.Key("network.connection.subtype") - - // NetworkConnectionTypeKey is the attribute Key conforming to the - // "network.connection.type" semantic conventions. It represents the internet - // connection type. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: wifi - NetworkConnectionTypeKey = attribute.Key("network.connection.type") - - // NetworkInterfaceNameKey is the attribute Key conforming to the - // "network.interface.name" semantic conventions. It represents the network - // interface name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "lo", "eth0" - NetworkInterfaceNameKey = attribute.Key("network.interface.name") - - // NetworkIODirectionKey is the attribute Key conforming to the - // "network.io.direction" semantic conventions. It represents the network IO - // operation direction. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "transmit" - NetworkIODirectionKey = attribute.Key("network.io.direction") - - // NetworkLocalAddressKey is the attribute Key conforming to the - // "network.local.address" semantic conventions. It represents the local address - // of the network connection - IP address or Unix domain socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "10.1.2.80", "/tmp/my.sock" - NetworkLocalAddressKey = attribute.Key("network.local.address") - - // NetworkLocalPortKey is the attribute Key conforming to the - // "network.local.port" semantic conventions. It represents the local port - // number of the network connection. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 65123 - NetworkLocalPortKey = attribute.Key("network.local.port") - - // NetworkPeerAddressKey is the attribute Key conforming to the - // "network.peer.address" semantic conventions. It represents the peer address - // of the network connection - IP address or Unix domain socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "10.1.2.80", "/tmp/my.sock" - NetworkPeerAddressKey = attribute.Key("network.peer.address") - - // NetworkPeerPortKey is the attribute Key conforming to the "network.peer.port" - // semantic conventions. It represents the peer port number of the network - // connection. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 65123 - NetworkPeerPortKey = attribute.Key("network.peer.port") - - // NetworkProtocolNameKey is the attribute Key conforming to the - // "network.protocol.name" semantic conventions. It represents the - // [OSI application layer] or non-OSI equivalent. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "amqp", "http", "mqtt" - // Note: The value SHOULD be normalized to lowercase. - // - // [OSI application layer]: https://wikipedia.org/wiki/Application_layer - NetworkProtocolNameKey = attribute.Key("network.protocol.name") - - // NetworkProtocolVersionKey is the attribute Key conforming to the - // "network.protocol.version" semantic conventions. It represents the actual - // version of the protocol used for network communication. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "1.1", "2" - // Note: If protocol version is subject to negotiation (for example using [ALPN] - // ), this attribute SHOULD be set to the negotiated version. If the actual - // protocol version is not known, this attribute SHOULD NOT be set. - // - // [ALPN]: https://www.rfc-editor.org/rfc/rfc7301.html - NetworkProtocolVersionKey = attribute.Key("network.protocol.version") - - // NetworkTransportKey is the attribute Key conforming to the - // "network.transport" semantic conventions. It represents the - // [OSI transport layer] or [inter-process communication method]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "tcp", "udp" - // Note: The value SHOULD be normalized to lowercase. - // - // Consider always setting the transport when setting a port number, since - // a port number is ambiguous without knowing the transport. For example - // different processes could be listening on TCP port 12345 and UDP port 12345. - // - // [OSI transport layer]: https://wikipedia.org/wiki/Transport_layer - // [inter-process communication method]: https://wikipedia.org/wiki/Inter-process_communication - NetworkTransportKey = attribute.Key("network.transport") - - // NetworkTypeKey is the attribute Key conforming to the "network.type" semantic - // conventions. It represents the [OSI network layer] or non-OSI equivalent. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "ipv4", "ipv6" - // Note: The value SHOULD be normalized to lowercase. - // - // [OSI network layer]: https://wikipedia.org/wiki/Network_layer - NetworkTypeKey = attribute.Key("network.type") -) - -// NetworkCarrierICC returns an attribute KeyValue conforming to the -// "network.carrier.icc" semantic conventions. It represents the ISO 3166-1 -// alpha-2 2-character country code associated with the mobile carrier network. -func NetworkCarrierICC(val string) attribute.KeyValue { - return NetworkCarrierICCKey.String(val) -} - -// NetworkCarrierMCC returns an attribute KeyValue conforming to the -// "network.carrier.mcc" semantic conventions. It represents the mobile carrier -// country code. -func NetworkCarrierMCC(val string) attribute.KeyValue { - return NetworkCarrierMCCKey.String(val) -} - -// NetworkCarrierMNC returns an attribute KeyValue conforming to the -// "network.carrier.mnc" semantic conventions. It represents the mobile carrier -// network code. -func NetworkCarrierMNC(val string) attribute.KeyValue { - return NetworkCarrierMNCKey.String(val) -} - -// NetworkCarrierName returns an attribute KeyValue conforming to the -// "network.carrier.name" semantic conventions. It represents the name of the -// mobile carrier. -func NetworkCarrierName(val string) attribute.KeyValue { - return NetworkCarrierNameKey.String(val) -} - -// NetworkInterfaceName returns an attribute KeyValue conforming to the -// "network.interface.name" semantic conventions. It represents the network -// interface name. -func NetworkInterfaceName(val string) attribute.KeyValue { - return NetworkInterfaceNameKey.String(val) -} - -// NetworkLocalAddress returns an attribute KeyValue conforming to the -// "network.local.address" semantic conventions. It represents the local address -// of the network connection - IP address or Unix domain socket name. -func NetworkLocalAddress(val string) attribute.KeyValue { - return NetworkLocalAddressKey.String(val) -} - -// NetworkLocalPort returns an attribute KeyValue conforming to the -// "network.local.port" semantic conventions. It represents the local port number -// of the network connection. -func NetworkLocalPort(val int) attribute.KeyValue { - return NetworkLocalPortKey.Int(val) -} - -// NetworkPeerAddress returns an attribute KeyValue conforming to the -// "network.peer.address" semantic conventions. It represents the peer address of -// the network connection - IP address or Unix domain socket name. -func NetworkPeerAddress(val string) attribute.KeyValue { - return NetworkPeerAddressKey.String(val) -} - -// NetworkPeerPort returns an attribute KeyValue conforming to the -// "network.peer.port" semantic conventions. It represents the peer port number -// of the network connection. -func NetworkPeerPort(val int) attribute.KeyValue { - return NetworkPeerPortKey.Int(val) -} - -// NetworkProtocolName returns an attribute KeyValue conforming to the -// "network.protocol.name" semantic conventions. It represents the -// [OSI application layer] or non-OSI equivalent. -// -// [OSI application layer]: https://wikipedia.org/wiki/Application_layer -func NetworkProtocolName(val string) attribute.KeyValue { - return NetworkProtocolNameKey.String(val) -} - -// NetworkProtocolVersion returns an attribute KeyValue conforming to the -// "network.protocol.version" semantic conventions. It represents the actual -// version of the protocol used for network communication. -func NetworkProtocolVersion(val string) attribute.KeyValue { - return NetworkProtocolVersionKey.String(val) -} - -// Enum values for network.connection.state -var ( - // closed - // Stability: development - NetworkConnectionStateClosed = NetworkConnectionStateKey.String("closed") - // close_wait - // Stability: development - NetworkConnectionStateCloseWait = NetworkConnectionStateKey.String("close_wait") - // closing - // Stability: development - NetworkConnectionStateClosing = NetworkConnectionStateKey.String("closing") - // established - // Stability: development - NetworkConnectionStateEstablished = NetworkConnectionStateKey.String("established") - // fin_wait_1 - // Stability: development - NetworkConnectionStateFinWait1 = NetworkConnectionStateKey.String("fin_wait_1") - // fin_wait_2 - // Stability: development - NetworkConnectionStateFinWait2 = NetworkConnectionStateKey.String("fin_wait_2") - // last_ack - // Stability: development - NetworkConnectionStateLastAck = NetworkConnectionStateKey.String("last_ack") - // listen - // Stability: development - NetworkConnectionStateListen = NetworkConnectionStateKey.String("listen") - // syn_received - // Stability: development - NetworkConnectionStateSynReceived = NetworkConnectionStateKey.String("syn_received") - // syn_sent - // Stability: development - NetworkConnectionStateSynSent = NetworkConnectionStateKey.String("syn_sent") - // time_wait - // Stability: development - NetworkConnectionStateTimeWait = NetworkConnectionStateKey.String("time_wait") -) - -// Enum values for network.connection.subtype -var ( - // GPRS - // Stability: development - NetworkConnectionSubtypeGprs = NetworkConnectionSubtypeKey.String("gprs") - // EDGE - // Stability: development - NetworkConnectionSubtypeEdge = NetworkConnectionSubtypeKey.String("edge") - // UMTS - // Stability: development - NetworkConnectionSubtypeUmts = NetworkConnectionSubtypeKey.String("umts") - // CDMA - // Stability: development - NetworkConnectionSubtypeCdma = NetworkConnectionSubtypeKey.String("cdma") - // EVDO Rel. 0 - // Stability: development - NetworkConnectionSubtypeEvdo0 = NetworkConnectionSubtypeKey.String("evdo_0") - // EVDO Rev. A - // Stability: development - NetworkConnectionSubtypeEvdoA = NetworkConnectionSubtypeKey.String("evdo_a") - // CDMA2000 1XRTT - // Stability: development - NetworkConnectionSubtypeCdma20001xrtt = NetworkConnectionSubtypeKey.String("cdma2000_1xrtt") - // HSDPA - // Stability: development - NetworkConnectionSubtypeHsdpa = NetworkConnectionSubtypeKey.String("hsdpa") - // HSUPA - // Stability: development - NetworkConnectionSubtypeHsupa = NetworkConnectionSubtypeKey.String("hsupa") - // HSPA - // Stability: development - NetworkConnectionSubtypeHspa = NetworkConnectionSubtypeKey.String("hspa") - // IDEN - // Stability: development - NetworkConnectionSubtypeIden = NetworkConnectionSubtypeKey.String("iden") - // EVDO Rev. B - // Stability: development - NetworkConnectionSubtypeEvdoB = NetworkConnectionSubtypeKey.String("evdo_b") - // LTE - // Stability: development - NetworkConnectionSubtypeLte = NetworkConnectionSubtypeKey.String("lte") - // EHRPD - // Stability: development - NetworkConnectionSubtypeEhrpd = NetworkConnectionSubtypeKey.String("ehrpd") - // HSPAP - // Stability: development - NetworkConnectionSubtypeHspap = NetworkConnectionSubtypeKey.String("hspap") - // GSM - // Stability: development - NetworkConnectionSubtypeGsm = NetworkConnectionSubtypeKey.String("gsm") - // TD-SCDMA - // Stability: development - NetworkConnectionSubtypeTdScdma = NetworkConnectionSubtypeKey.String("td_scdma") - // IWLAN - // Stability: development - NetworkConnectionSubtypeIwlan = NetworkConnectionSubtypeKey.String("iwlan") - // 5G NR (New Radio) - // Stability: development - NetworkConnectionSubtypeNr = NetworkConnectionSubtypeKey.String("nr") - // 5G NRNSA (New Radio Non-Standalone) - // Stability: development - NetworkConnectionSubtypeNrnsa = NetworkConnectionSubtypeKey.String("nrnsa") - // LTE CA - // Stability: development - NetworkConnectionSubtypeLteCa = NetworkConnectionSubtypeKey.String("lte_ca") -) - -// Enum values for network.connection.type -var ( - // wifi - // Stability: development - NetworkConnectionTypeWifi = NetworkConnectionTypeKey.String("wifi") - // wired - // Stability: development - NetworkConnectionTypeWired = NetworkConnectionTypeKey.String("wired") - // cell - // Stability: development - NetworkConnectionTypeCell = NetworkConnectionTypeKey.String("cell") - // unavailable - // Stability: development - NetworkConnectionTypeUnavailable = NetworkConnectionTypeKey.String("unavailable") - // unknown - // Stability: development - NetworkConnectionTypeUnknown = NetworkConnectionTypeKey.String("unknown") -) - -// Enum values for network.io.direction -var ( - // transmit - // Stability: development - NetworkIODirectionTransmit = NetworkIODirectionKey.String("transmit") - // receive - // Stability: development - NetworkIODirectionReceive = NetworkIODirectionKey.String("receive") -) - -// Enum values for network.transport -var ( - // TCP - // Stability: stable - NetworkTransportTCP = NetworkTransportKey.String("tcp") - // UDP - // Stability: stable - NetworkTransportUDP = NetworkTransportKey.String("udp") - // Named or anonymous pipe. - // Stability: stable - NetworkTransportPipe = NetworkTransportKey.String("pipe") - // Unix domain socket - // Stability: stable - NetworkTransportUnix = NetworkTransportKey.String("unix") - // QUIC - // Stability: stable - NetworkTransportQUIC = NetworkTransportKey.String("quic") -) - -// Enum values for network.type -var ( - // IPv4 - // Stability: stable - NetworkTypeIPv4 = NetworkTypeKey.String("ipv4") - // IPv6 - // Stability: stable - NetworkTypeIPv6 = NetworkTypeKey.String("ipv6") -) - -// Namespace: nfs -const ( - // NfsOperationNameKey is the attribute Key conforming to the - // "nfs.operation.name" semantic conventions. It represents the NFSv4+ operation - // name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "OPEN", "READ", "GETATTR" - NfsOperationNameKey = attribute.Key("nfs.operation.name") - - // NfsServerRepcacheStatusKey is the attribute Key conforming to the - // "nfs.server.repcache.status" semantic conventions. It represents the linux: - // one of "hit" (NFSD_STATS_RC_HITS), "miss" (NFSD_STATS_RC_MISSES), or - // "nocache" (NFSD_STATS_RC_NOCACHE -- uncacheable). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: hit - NfsServerRepcacheStatusKey = attribute.Key("nfs.server.repcache.status") -) - -// NfsOperationName returns an attribute KeyValue conforming to the -// "nfs.operation.name" semantic conventions. It represents the NFSv4+ operation -// name. -func NfsOperationName(val string) attribute.KeyValue { - return NfsOperationNameKey.String(val) -} - -// NfsServerRepcacheStatus returns an attribute KeyValue conforming to the -// "nfs.server.repcache.status" semantic conventions. It represents the linux: -// one of "hit" (NFSD_STATS_RC_HITS), "miss" (NFSD_STATS_RC_MISSES), or "nocache" -// (NFSD_STATS_RC_NOCACHE -- uncacheable). -func NfsServerRepcacheStatus(val string) attribute.KeyValue { - return NfsServerRepcacheStatusKey.String(val) -} - -// Namespace: oci -const ( - // OCIManifestDigestKey is the attribute Key conforming to the - // "oci.manifest.digest" semantic conventions. It represents the digest of the - // OCI image manifest. For container images specifically is the digest by which - // the container image is known. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "sha256:e4ca62c0d62f3e886e684806dfe9d4e0cda60d54986898173c1083856cfda0f4" - // Note: Follows [OCI Image Manifest Specification], and specifically the - // [Digest property]. - // An example can be found in [Example Image Manifest]. - // - // [OCI Image Manifest Specification]: https://github.com/opencontainers/image-spec/blob/main/manifest.md - // [Digest property]: https://github.com/opencontainers/image-spec/blob/main/descriptor.md#digests - // [Example Image Manifest]: https://github.com/opencontainers/image-spec/blob/main/manifest.md#example-image-manifest - OCIManifestDigestKey = attribute.Key("oci.manifest.digest") -) - -// OCIManifestDigest returns an attribute KeyValue conforming to the -// "oci.manifest.digest" semantic conventions. It represents the digest of the -// OCI image manifest. For container images specifically is the digest by which -// the container image is known. -func OCIManifestDigest(val string) attribute.KeyValue { - return OCIManifestDigestKey.String(val) -} - -// Namespace: onc_rpc -const ( - // OncRPCProcedureNameKey is the attribute Key conforming to the - // "onc_rpc.procedure.name" semantic conventions. It represents the ONC/Sun RPC - // procedure name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "OPEN", "READ", "GETATTR" - OncRPCProcedureNameKey = attribute.Key("onc_rpc.procedure.name") - - // OncRPCProcedureNumberKey is the attribute Key conforming to the - // "onc_rpc.procedure.number" semantic conventions. It represents the ONC/Sun - // RPC procedure number. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - OncRPCProcedureNumberKey = attribute.Key("onc_rpc.procedure.number") - - // OncRPCProgramNameKey is the attribute Key conforming to the - // "onc_rpc.program.name" semantic conventions. It represents the ONC/Sun RPC - // program name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "portmapper", "nfs" - OncRPCProgramNameKey = attribute.Key("onc_rpc.program.name") - - // OncRPCVersionKey is the attribute Key conforming to the "onc_rpc.version" - // semantic conventions. It represents the ONC/Sun RPC program version. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - OncRPCVersionKey = attribute.Key("onc_rpc.version") -) - -// OncRPCProcedureName returns an attribute KeyValue conforming to the -// "onc_rpc.procedure.name" semantic conventions. It represents the ONC/Sun RPC -// procedure name. -func OncRPCProcedureName(val string) attribute.KeyValue { - return OncRPCProcedureNameKey.String(val) -} - -// OncRPCProcedureNumber returns an attribute KeyValue conforming to the -// "onc_rpc.procedure.number" semantic conventions. It represents the ONC/Sun RPC -// procedure number. -func OncRPCProcedureNumber(val int) attribute.KeyValue { - return OncRPCProcedureNumberKey.Int(val) -} - -// OncRPCProgramName returns an attribute KeyValue conforming to the -// "onc_rpc.program.name" semantic conventions. It represents the ONC/Sun RPC -// program name. -func OncRPCProgramName(val string) attribute.KeyValue { - return OncRPCProgramNameKey.String(val) -} - -// OncRPCVersion returns an attribute KeyValue conforming to the -// "onc_rpc.version" semantic conventions. It represents the ONC/Sun RPC program -// version. -func OncRPCVersion(val int) attribute.KeyValue { - return OncRPCVersionKey.Int(val) -} - -// Namespace: openai -const ( - // OpenAIRequestServiceTierKey is the attribute Key conforming to the - // "openai.request.service_tier" semantic conventions. It represents the service - // tier requested. May be a specific tier, default, or auto. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "auto", "default" - OpenAIRequestServiceTierKey = attribute.Key("openai.request.service_tier") - - // OpenAIResponseServiceTierKey is the attribute Key conforming to the - // "openai.response.service_tier" semantic conventions. It represents the - // service tier used for the response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "scale", "default" - OpenAIResponseServiceTierKey = attribute.Key("openai.response.service_tier") - - // OpenAIResponseSystemFingerprintKey is the attribute Key conforming to the - // "openai.response.system_fingerprint" semantic conventions. It represents a - // fingerprint to track any eventual change in the Generative AI environment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "fp_44709d6fcb" - OpenAIResponseSystemFingerprintKey = attribute.Key("openai.response.system_fingerprint") -) - -// OpenAIResponseServiceTier returns an attribute KeyValue conforming to the -// "openai.response.service_tier" semantic conventions. It represents the service -// tier used for the response. -func OpenAIResponseServiceTier(val string) attribute.KeyValue { - return OpenAIResponseServiceTierKey.String(val) -} - -// OpenAIResponseSystemFingerprint returns an attribute KeyValue conforming to -// the "openai.response.system_fingerprint" semantic conventions. It represents a -// fingerprint to track any eventual change in the Generative AI environment. -func OpenAIResponseSystemFingerprint(val string) attribute.KeyValue { - return OpenAIResponseSystemFingerprintKey.String(val) -} - -// Enum values for openai.request.service_tier -var ( - // The system will utilize scale tier credits until they are exhausted. - // Stability: development - OpenAIRequestServiceTierAuto = OpenAIRequestServiceTierKey.String("auto") - // The system will utilize the default scale tier. - // Stability: development - OpenAIRequestServiceTierDefault = OpenAIRequestServiceTierKey.String("default") -) - -// Namespace: openshift -const ( - // OpenShiftClusterquotaNameKey is the attribute Key conforming to the - // "openshift.clusterquota.name" semantic conventions. It represents the name of - // the cluster quota. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "opentelemetry" - OpenShiftClusterquotaNameKey = attribute.Key("openshift.clusterquota.name") - - // OpenShiftClusterquotaUIDKey is the attribute Key conforming to the - // "openshift.clusterquota.uid" semantic conventions. It represents the UID of - // the cluster quota. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" - OpenShiftClusterquotaUIDKey = attribute.Key("openshift.clusterquota.uid") -) - -// OpenShiftClusterquotaName returns an attribute KeyValue conforming to the -// "openshift.clusterquota.name" semantic conventions. It represents the name of -// the cluster quota. -func OpenShiftClusterquotaName(val string) attribute.KeyValue { - return OpenShiftClusterquotaNameKey.String(val) -} - -// OpenShiftClusterquotaUID returns an attribute KeyValue conforming to the -// "openshift.clusterquota.uid" semantic conventions. It represents the UID of -// the cluster quota. -func OpenShiftClusterquotaUID(val string) attribute.KeyValue { - return OpenShiftClusterquotaUIDKey.String(val) -} - -// Namespace: opentracing -const ( - // OpenTracingRefTypeKey is the attribute Key conforming to the - // "opentracing.ref_type" semantic conventions. It represents the parent-child - // Reference type. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The causal relationship between a child Span and a parent Span. - OpenTracingRefTypeKey = attribute.Key("opentracing.ref_type") -) - -// Enum values for opentracing.ref_type -var ( - // The parent Span depends on the child Span in some capacity - // Stability: development - OpenTracingRefTypeChildOf = OpenTracingRefTypeKey.String("child_of") - // The parent Span doesn't depend in any way on the result of the child Span - // Stability: development - OpenTracingRefTypeFollowsFrom = OpenTracingRefTypeKey.String("follows_from") -) - -// Namespace: os -const ( - // OSBuildIDKey is the attribute Key conforming to the "os.build_id" semantic - // conventions. It represents the unique identifier for a particular build or - // compilation of the operating system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "TQ3C.230805.001.B2", "20E247", "22621" - OSBuildIDKey = attribute.Key("os.build_id") - - // OSDescriptionKey is the attribute Key conforming to the "os.description" - // semantic conventions. It represents the human readable (not intended to be - // parsed) OS version information, like e.g. reported by `ver` or - // `lsb_release -a` commands. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Microsoft Windows [Version 10.0.18363.778]", "Ubuntu 18.04.1 LTS" - OSDescriptionKey = attribute.Key("os.description") - - // OSNameKey is the attribute Key conforming to the "os.name" semantic - // conventions. It represents the human readable operating system name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "iOS", "Android", "Ubuntu" - OSNameKey = attribute.Key("os.name") - - // OSTypeKey is the attribute Key conforming to the "os.type" semantic - // conventions. It represents the operating system type. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - OSTypeKey = attribute.Key("os.type") - - // OSVersionKey is the attribute Key conforming to the "os.version" semantic - // conventions. It represents the version string of the operating system as - // defined in [Version Attributes]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "14.2.1", "18.04.1" - // - // [Version Attributes]: /docs/resource/README.md#version-attributes - OSVersionKey = attribute.Key("os.version") -) - -// OSBuildID returns an attribute KeyValue conforming to the "os.build_id" -// semantic conventions. It represents the unique identifier for a particular -// build or compilation of the operating system. -func OSBuildID(val string) attribute.KeyValue { - return OSBuildIDKey.String(val) -} - -// OSDescription returns an attribute KeyValue conforming to the "os.description" -// semantic conventions. It represents the human readable (not intended to be -// parsed) OS version information, like e.g. reported by `ver` or -// `lsb_release -a` commands. -func OSDescription(val string) attribute.KeyValue { - return OSDescriptionKey.String(val) -} - -// OSName returns an attribute KeyValue conforming to the "os.name" semantic -// conventions. It represents the human readable operating system name. -func OSName(val string) attribute.KeyValue { - return OSNameKey.String(val) -} - -// OSVersion returns an attribute KeyValue conforming to the "os.version" -// semantic conventions. It represents the version string of the operating system -// as defined in [Version Attributes]. -// -// [Version Attributes]: /docs/resource/README.md#version-attributes -func OSVersion(val string) attribute.KeyValue { - return OSVersionKey.String(val) -} - -// Enum values for os.type -var ( - // Microsoft Windows - // Stability: development - OSTypeWindows = OSTypeKey.String("windows") - // Linux - // Stability: development - OSTypeLinux = OSTypeKey.String("linux") - // Apple Darwin - // Stability: development - OSTypeDarwin = OSTypeKey.String("darwin") - // FreeBSD - // Stability: development - OSTypeFreeBSD = OSTypeKey.String("freebsd") - // NetBSD - // Stability: development - OSTypeNetBSD = OSTypeKey.String("netbsd") - // OpenBSD - // Stability: development - OSTypeOpenBSD = OSTypeKey.String("openbsd") - // DragonFly BSD - // Stability: development - OSTypeDragonflyBSD = OSTypeKey.String("dragonflybsd") - // HP-UX (Hewlett Packard Unix) - // Stability: development - OSTypeHPUX = OSTypeKey.String("hpux") - // AIX (Advanced Interactive eXecutive) - // Stability: development - OSTypeAIX = OSTypeKey.String("aix") - // SunOS, Oracle Solaris - // Stability: development - OSTypeSolaris = OSTypeKey.String("solaris") - // IBM z/OS - // Stability: development - OSTypeZOS = OSTypeKey.String("zos") -) - -// Namespace: otel -const ( - // OTelComponentNameKey is the attribute Key conforming to the - // "otel.component.name" semantic conventions. It represents a name uniquely - // identifying the instance of the OpenTelemetry component within its containing - // SDK instance. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "otlp_grpc_span_exporter/0", "custom-name" - // Note: Implementations SHOULD ensure a low cardinality for this attribute, - // even across application or SDK restarts. - // E.g. implementations MUST NOT use UUIDs as values for this attribute. - // - // Implementations MAY achieve these goals by following a - // `/` pattern, e.g. - // `batching_span_processor/0`. - // Hereby `otel.component.type` refers to the corresponding attribute value of - // the component. - // - // The value of `instance-counter` MAY be automatically assigned by the - // component and uniqueness within the enclosing SDK instance MUST be - // guaranteed. - // For example, `` MAY be implemented by using a monotonically - // increasing counter (starting with `0`), which is incremented every time an - // instance of the given component type is started. - // - // With this implementation, for example the first Batching Span Processor would - // have `batching_span_processor/0` - // as `otel.component.name`, the second one `batching_span_processor/1` and so - // on. - // These values will therefore be reused in the case of an application restart. - OTelComponentNameKey = attribute.Key("otel.component.name") - - // OTelComponentTypeKey is the attribute Key conforming to the - // "otel.component.type" semantic conventions. It represents a name identifying - // the type of the OpenTelemetry component. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "batching_span_processor", "com.example.MySpanExporter" - // Note: If none of the standardized values apply, implementations SHOULD use - // the language-defined name of the type. - // E.g. for Java the fully qualified classname SHOULD be used in this case. - OTelComponentTypeKey = attribute.Key("otel.component.type") - - // OTelEventNameKey is the attribute Key conforming to the "otel.event.name" - // semantic conventions. It represents the identifies the class / type of event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "browser.mouse.click", "device.app.lifecycle" - // Note: This attribute SHOULD be used by non-OTLP exporters when destination - // does not support `EventName` or equivalent field. This attribute MAY be used - // by applications using existing logging libraries so that it can be used to - // set the `EventName` field by Collector or SDK components. - OTelEventNameKey = attribute.Key("otel.event.name") - - // OTelScopeNameKey is the attribute Key conforming to the "otel.scope.name" - // semantic conventions. It represents the name of the instrumentation scope - ( - // `InstrumentationScope.Name` in OTLP). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "io.opentelemetry.contrib.mongodb" - OTelScopeNameKey = attribute.Key("otel.scope.name") - - // OTelScopeSchemaURLKey is the attribute Key conforming to the - // "otel.scope.schema_url" semantic conventions. It represents the schema URL of - // the instrumentation scope. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://opentelemetry.io/schemas/1.31.0" - OTelScopeSchemaURLKey = attribute.Key("otel.scope.schema_url") - - // OTelScopeVersionKey is the attribute Key conforming to the - // "otel.scope.version" semantic conventions. It represents the version of the - // instrumentation scope - (`InstrumentationScope.Version` in OTLP). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "1.0.0" - OTelScopeVersionKey = attribute.Key("otel.scope.version") - - // OTelSpanParentOriginKey is the attribute Key conforming to the - // "otel.span.parent.origin" semantic conventions. It represents the determines - // whether the span has a parent span, and if so, - // [whether it is a remote parent]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [whether it is a remote parent]: https://opentelemetry.io/docs/specs/otel/trace/api/#isremote - OTelSpanParentOriginKey = attribute.Key("otel.span.parent.origin") - - // OTelSpanSamplingResultKey is the attribute Key conforming to the - // "otel.span.sampling_result" semantic conventions. It represents the result - // value of the sampler for this span. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - OTelSpanSamplingResultKey = attribute.Key("otel.span.sampling_result") - - // OTelStatusCodeKey is the attribute Key conforming to the "otel.status_code" - // semantic conventions. It represents the name of the code, either "OK" or - // "ERROR". MUST NOT be set if the status code is UNSET. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: - OTelStatusCodeKey = attribute.Key("otel.status_code") - - // OTelStatusDescriptionKey is the attribute Key conforming to the - // "otel.status_description" semantic conventions. It represents the description - // of the Status if it has a value, otherwise not set. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "resource not found" - OTelStatusDescriptionKey = attribute.Key("otel.status_description") -) - -// OTelComponentName returns an attribute KeyValue conforming to the -// "otel.component.name" semantic conventions. It represents a name uniquely -// identifying the instance of the OpenTelemetry component within its containing -// SDK instance. -func OTelComponentName(val string) attribute.KeyValue { - return OTelComponentNameKey.String(val) -} - -// OTelEventName returns an attribute KeyValue conforming to the -// "otel.event.name" semantic conventions. It represents the identifies the class -// / type of event. -func OTelEventName(val string) attribute.KeyValue { - return OTelEventNameKey.String(val) -} - -// OTelScopeName returns an attribute KeyValue conforming to the -// "otel.scope.name" semantic conventions. It represents the name of the -// instrumentation scope - (`InstrumentationScope.Name` in OTLP). -func OTelScopeName(val string) attribute.KeyValue { - return OTelScopeNameKey.String(val) -} - -// OTelScopeSchemaURL returns an attribute KeyValue conforming to the -// "otel.scope.schema_url" semantic conventions. It represents the schema URL of -// the instrumentation scope. -func OTelScopeSchemaURL(val string) attribute.KeyValue { - return OTelScopeSchemaURLKey.String(val) -} - -// OTelScopeVersion returns an attribute KeyValue conforming to the -// "otel.scope.version" semantic conventions. It represents the version of the -// instrumentation scope - (`InstrumentationScope.Version` in OTLP). -func OTelScopeVersion(val string) attribute.KeyValue { - return OTelScopeVersionKey.String(val) -} - -// OTelStatusDescription returns an attribute KeyValue conforming to the -// "otel.status_description" semantic conventions. It represents the description -// of the Status if it has a value, otherwise not set. -func OTelStatusDescription(val string) attribute.KeyValue { - return OTelStatusDescriptionKey.String(val) -} - -// Enum values for otel.component.type -var ( - // The builtin SDK batching span processor - // - // Stability: development - OTelComponentTypeBatchingSpanProcessor = OTelComponentTypeKey.String("batching_span_processor") - // The builtin SDK simple span processor - // - // Stability: development - OTelComponentTypeSimpleSpanProcessor = OTelComponentTypeKey.String("simple_span_processor") - // The builtin SDK batching log record processor - // - // Stability: development - OTelComponentTypeBatchingLogProcessor = OTelComponentTypeKey.String("batching_log_processor") - // The builtin SDK simple log record processor - // - // Stability: development - OTelComponentTypeSimpleLogProcessor = OTelComponentTypeKey.String("simple_log_processor") - // OTLP span exporter over gRPC with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpGRPCSpanExporter = OTelComponentTypeKey.String("otlp_grpc_span_exporter") - // OTLP span exporter over HTTP with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPSpanExporter = OTelComponentTypeKey.String("otlp_http_span_exporter") - // OTLP span exporter over HTTP with JSON serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPJSONSpanExporter = OTelComponentTypeKey.String("otlp_http_json_span_exporter") - // Zipkin span exporter over HTTP - // - // Stability: development - OTelComponentTypeZipkinHTTPSpanExporter = OTelComponentTypeKey.String("zipkin_http_span_exporter") - // OTLP log record exporter over gRPC with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpGRPCLogExporter = OTelComponentTypeKey.String("otlp_grpc_log_exporter") - // OTLP log record exporter over HTTP with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPLogExporter = OTelComponentTypeKey.String("otlp_http_log_exporter") - // OTLP log record exporter over HTTP with JSON serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPJSONLogExporter = OTelComponentTypeKey.String("otlp_http_json_log_exporter") - // The builtin SDK periodically exporting metric reader - // - // Stability: development - OTelComponentTypePeriodicMetricReader = OTelComponentTypeKey.String("periodic_metric_reader") - // OTLP metric exporter over gRPC with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpGRPCMetricExporter = OTelComponentTypeKey.String("otlp_grpc_metric_exporter") - // OTLP metric exporter over HTTP with protobuf serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPMetricExporter = OTelComponentTypeKey.String("otlp_http_metric_exporter") - // OTLP metric exporter over HTTP with JSON serialization - // - // Stability: development - OTelComponentTypeOtlpHTTPJSONMetricExporter = OTelComponentTypeKey.String("otlp_http_json_metric_exporter") - // Prometheus metric exporter over HTTP with the default text-based format - // - // Stability: development - OTelComponentTypePrometheusHTTPTextMetricExporter = OTelComponentTypeKey.String("prometheus_http_text_metric_exporter") -) - -// Enum values for otel.span.parent.origin -var ( - // The span does not have a parent, it is a root span - // Stability: development - OTelSpanParentOriginNone = OTelSpanParentOriginKey.String("none") - // The span has a parent and the parent's span context [isRemote()] is false - // Stability: development - // - // [isRemote()]: https://opentelemetry.io/docs/specs/otel/trace/api/#isremote - OTelSpanParentOriginLocal = OTelSpanParentOriginKey.String("local") - // The span has a parent and the parent's span context [isRemote()] is true - // Stability: development - // - // [isRemote()]: https://opentelemetry.io/docs/specs/otel/trace/api/#isremote - OTelSpanParentOriginRemote = OTelSpanParentOriginKey.String("remote") -) - -// Enum values for otel.span.sampling_result -var ( - // The span is not sampled and not recording - // Stability: development - OTelSpanSamplingResultDrop = OTelSpanSamplingResultKey.String("DROP") - // The span is not sampled, but recording - // Stability: development - OTelSpanSamplingResultRecordOnly = OTelSpanSamplingResultKey.String("RECORD_ONLY") - // The span is sampled and recording - // Stability: development - OTelSpanSamplingResultRecordAndSample = OTelSpanSamplingResultKey.String("RECORD_AND_SAMPLE") -) - -// Enum values for otel.status_code -var ( - // The operation has been validated by an Application developer or Operator to - // have completed successfully. - // Stability: stable - OTelStatusCodeOk = OTelStatusCodeKey.String("OK") - // The operation contains an error. - // Stability: stable - OTelStatusCodeError = OTelStatusCodeKey.String("ERROR") -) - -// Namespace: pprof -const ( - // PprofLocationIsFoldedKey is the attribute Key conforming to the - // "pprof.location.is_folded" semantic conventions. It represents the provides - // an indication that multiple symbols map to this location's address, for - // example due to identical code folding by the linker. In that case the line - // information represents one of the multiple symbols. This field must be - // recomputed when the symbolization state of the profile changes. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - PprofLocationIsFoldedKey = attribute.Key("pprof.location.is_folded") - - // PprofMappingHasFilenamesKey is the attribute Key conforming to the - // "pprof.mapping.has_filenames" semantic conventions. It represents the - // indicates that there are filenames related to this mapping. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - PprofMappingHasFilenamesKey = attribute.Key("pprof.mapping.has_filenames") - - // PprofMappingHasFunctionsKey is the attribute Key conforming to the - // "pprof.mapping.has_functions" semantic conventions. It represents the - // indicates that there are functions related to this mapping. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - PprofMappingHasFunctionsKey = attribute.Key("pprof.mapping.has_functions") - - // PprofMappingHasInlineFramesKey is the attribute Key conforming to the - // "pprof.mapping.has_inline_frames" semantic conventions. It represents the - // indicates that there are inline frames related to this mapping. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - PprofMappingHasInlineFramesKey = attribute.Key("pprof.mapping.has_inline_frames") - - // PprofMappingHasLineNumbersKey is the attribute Key conforming to the - // "pprof.mapping.has_line_numbers" semantic conventions. It represents the - // indicates that there are line numbers related to this mapping. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - PprofMappingHasLineNumbersKey = attribute.Key("pprof.mapping.has_line_numbers") - - // PprofProfileCommentKey is the attribute Key conforming to the - // "pprof.profile.comment" semantic conventions. It represents the free-form - // text associated with the profile. This field should not be used to store any - // machine-readable information, it is only for human-friendly content. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "hello world", "bazinga" - PprofProfileCommentKey = attribute.Key("pprof.profile.comment") - - // PprofProfileDocURLKey is the attribute Key conforming to the - // "pprof.profile.doc_url" semantic conventions. It represents the documentation - // link for this profile type. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "http://pprof.example.com/cpu-profile.html" - // Note: The URL must be absolute and may be missing if the profile was - // generated by code that did not supply a link - PprofProfileDocURLKey = attribute.Key("pprof.profile.doc_url") - - // PprofProfileDropFramesKey is the attribute Key conforming to the - // "pprof.profile.drop_frames" semantic conventions. It represents the frames - // with Function.function_name fully matching the regexp will be dropped from - // the samples, along with their successors. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/foobar/" - PprofProfileDropFramesKey = attribute.Key("pprof.profile.drop_frames") - - // PprofProfileKeepFramesKey is the attribute Key conforming to the - // "pprof.profile.keep_frames" semantic conventions. It represents the frames - // with Function.function_name fully matching the regexp will be kept, even if - // it matches drop_frames. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/bazinga/" - PprofProfileKeepFramesKey = attribute.Key("pprof.profile.keep_frames") -) - -// PprofLocationIsFolded returns an attribute KeyValue conforming to the -// "pprof.location.is_folded" semantic conventions. It represents the provides an -// indication that multiple symbols map to this location's address, for example -// due to identical code folding by the linker. In that case the line information -// represents one of the multiple symbols. This field must be recomputed when the -// symbolization state of the profile changes. -func PprofLocationIsFolded(val bool) attribute.KeyValue { - return PprofLocationIsFoldedKey.Bool(val) -} - -// PprofMappingHasFilenames returns an attribute KeyValue conforming to the -// "pprof.mapping.has_filenames" semantic conventions. It represents the -// indicates that there are filenames related to this mapping. -func PprofMappingHasFilenames(val bool) attribute.KeyValue { - return PprofMappingHasFilenamesKey.Bool(val) -} - -// PprofMappingHasFunctions returns an attribute KeyValue conforming to the -// "pprof.mapping.has_functions" semantic conventions. It represents the -// indicates that there are functions related to this mapping. -func PprofMappingHasFunctions(val bool) attribute.KeyValue { - return PprofMappingHasFunctionsKey.Bool(val) -} - -// PprofMappingHasInlineFrames returns an attribute KeyValue conforming to the -// "pprof.mapping.has_inline_frames" semantic conventions. It represents the -// indicates that there are inline frames related to this mapping. -func PprofMappingHasInlineFrames(val bool) attribute.KeyValue { - return PprofMappingHasInlineFramesKey.Bool(val) -} - -// PprofMappingHasLineNumbers returns an attribute KeyValue conforming to the -// "pprof.mapping.has_line_numbers" semantic conventions. It represents the -// indicates that there are line numbers related to this mapping. -func PprofMappingHasLineNumbers(val bool) attribute.KeyValue { - return PprofMappingHasLineNumbersKey.Bool(val) -} - -// PprofProfileComment returns an attribute KeyValue conforming to the -// "pprof.profile.comment" semantic conventions. It represents the free-form text -// associated with the profile. This field should not be used to store any -// machine-readable information, it is only for human-friendly content. -func PprofProfileComment(val ...string) attribute.KeyValue { - return PprofProfileCommentKey.StringSlice(val) -} - -// PprofProfileDocURL returns an attribute KeyValue conforming to the -// "pprof.profile.doc_url" semantic conventions. It represents the documentation -// link for this profile type. -func PprofProfileDocURL(val string) attribute.KeyValue { - return PprofProfileDocURLKey.String(val) -} - -// PprofProfileDropFrames returns an attribute KeyValue conforming to the -// "pprof.profile.drop_frames" semantic conventions. It represents the frames -// with Function.function_name fully matching the regexp will be dropped from the -// samples, along with their successors. -func PprofProfileDropFrames(val string) attribute.KeyValue { - return PprofProfileDropFramesKey.String(val) -} - -// PprofProfileKeepFrames returns an attribute KeyValue conforming to the -// "pprof.profile.keep_frames" semantic conventions. It represents the frames -// with Function.function_name fully matching the regexp will be kept, even if it -// matches drop_frames. -func PprofProfileKeepFrames(val string) attribute.KeyValue { - return PprofProfileKeepFramesKey.String(val) -} - -// Namespace: process -const ( - // ProcessArgsCountKey is the attribute Key conforming to the - // "process.args_count" semantic conventions. It represents the length of the - // process.command_args array. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 4 - // Note: This field can be useful for querying or performing bucket analysis on - // how many arguments were provided to start a process. More arguments may be an - // indication of suspicious activity. - ProcessArgsCountKey = attribute.Key("process.args_count") - - // ProcessCommandKey is the attribute Key conforming to the "process.command" - // semantic conventions. It represents the command used to launch the process - // (i.e. the command name). On Linux based systems, can be set to the zeroth - // string in `proc/[pid]/cmdline`. On Windows, can be set to the first parameter - // extracted from `GetCommandLineW`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cmd/otelcol" - ProcessCommandKey = attribute.Key("process.command") - - // ProcessCommandArgsKey is the attribute Key conforming to the - // "process.command_args" semantic conventions. It represents the all the - // command arguments (including the command/executable itself) as received by - // the process. On Linux-based systems (and some other Unixoid systems - // supporting procfs), can be set according to the list of null-delimited - // strings extracted from `proc/[pid]/cmdline`. For libc-based executables, this - // would be the full argv vector passed to `main`. SHOULD NOT be collected by - // default unless there is sanitization that excludes sensitive data. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cmd/otecol", "--config=config.yaml" - ProcessCommandArgsKey = attribute.Key("process.command_args") - - // ProcessCommandLineKey is the attribute Key conforming to the - // "process.command_line" semantic conventions. It represents the full command - // used to launch the process as a single string representing the full command. - // On Windows, can be set to the result of `GetCommandLineW`. Do not set this if - // you have to assemble it just for monitoring; use `process.command_args` - // instead. SHOULD NOT be collected by default unless there is sanitization that - // excludes sensitive data. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "C:\cmd\otecol --config="my directory\config.yaml"" - ProcessCommandLineKey = attribute.Key("process.command_line") - - // ProcessContextSwitchTypeKey is the attribute Key conforming to the - // "process.context_switch.type" semantic conventions. It represents the - // specifies whether the context switches for this data point were voluntary or - // involuntary. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - ProcessContextSwitchTypeKey = attribute.Key("process.context_switch.type") - - // ProcessCreationTimeKey is the attribute Key conforming to the - // "process.creation.time" semantic conventions. It represents the date and time - // the process was created, in ISO 8601 format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2023-11-21T09:25:34.853Z" - ProcessCreationTimeKey = attribute.Key("process.creation.time") - - // ProcessExecutableBuildIDGNUKey is the attribute Key conforming to the - // "process.executable.build_id.gnu" semantic conventions. It represents the GNU - // build ID as found in the `.note.gnu.build-id` ELF section (hex string). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "c89b11207f6479603b0d49bf291c092c2b719293" - ProcessExecutableBuildIDGNUKey = attribute.Key("process.executable.build_id.gnu") - - // ProcessExecutableBuildIDGoKey is the attribute Key conforming to the - // "process.executable.build_id.go" semantic conventions. It represents the Go - // build ID as retrieved by `go tool buildid `. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "foh3mEXu7BLZjsN9pOwG/kATcXlYVCDEFouRMQed_/WwRFB1hPo9LBkekthSPG/x8hMC8emW2cCjXD0_1aY" - ProcessExecutableBuildIDGoKey = attribute.Key("process.executable.build_id.go") - - // ProcessExecutableBuildIDHtlhashKey is the attribute Key conforming to the - // "process.executable.build_id.htlhash" semantic conventions. It represents the - // profiling specific build ID for executables. See the OTel specification for - // Profiles for more information. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "600DCAFE4A110000F2BF38C493F5FB92" - ProcessExecutableBuildIDHtlhashKey = attribute.Key("process.executable.build_id.htlhash") - - // ProcessExecutableNameKey is the attribute Key conforming to the - // "process.executable.name" semantic conventions. It represents the name of the - // process executable. On Linux based systems, this SHOULD be set to the base - // name of the target of `/proc/[pid]/exe`. On Windows, this SHOULD be set to - // the base name of `GetProcessImageFileNameW`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "otelcol" - ProcessExecutableNameKey = attribute.Key("process.executable.name") - - // ProcessExecutablePathKey is the attribute Key conforming to the - // "process.executable.path" semantic conventions. It represents the full path - // to the process executable. On Linux based systems, can be set to the target - // of `proc/[pid]/exe`. On Windows, can be set to the result of - // `GetProcessImageFileNameW`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/usr/bin/cmd/otelcol" - ProcessExecutablePathKey = attribute.Key("process.executable.path") - - // ProcessExitCodeKey is the attribute Key conforming to the "process.exit.code" - // semantic conventions. It represents the exit code of the process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 127 - ProcessExitCodeKey = attribute.Key("process.exit.code") - - // ProcessExitTimeKey is the attribute Key conforming to the "process.exit.time" - // semantic conventions. It represents the date and time the process exited, in - // ISO 8601 format. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2023-11-21T09:26:12.315Z" - ProcessExitTimeKey = attribute.Key("process.exit.time") - - // ProcessGroupLeaderPIDKey is the attribute Key conforming to the - // "process.group_leader.pid" semantic conventions. It represents the PID of the - // process's group leader. This is also the process group ID (PGID) of the - // process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 23 - ProcessGroupLeaderPIDKey = attribute.Key("process.group_leader.pid") - - // ProcessInteractiveKey is the attribute Key conforming to the - // "process.interactive" semantic conventions. It represents the whether the - // process is connected to an interactive shell. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - ProcessInteractiveKey = attribute.Key("process.interactive") - - // ProcessLinuxCgroupKey is the attribute Key conforming to the - // "process.linux.cgroup" semantic conventions. It represents the control group - // associated with the process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1:name=systemd:/user.slice/user-1000.slice/session-3.scope", - // "0::/user.slice/user-1000.slice/user@1000.service/tmux-spawn-0267755b-4639-4a27-90ed-f19f88e53748.scope" - // Note: Control groups (cgroups) are a kernel feature used to organize and - // manage process resources. This attribute provides the path(s) to the - // cgroup(s) associated with the process, which should match the contents of the - // [/proc/[PID]/cgroup] file. - // - // [/proc/[PID]/cgroup]: https://man7.org/linux/man-pages/man7/cgroups.7.html - ProcessLinuxCgroupKey = attribute.Key("process.linux.cgroup") - - // ProcessOwnerKey is the attribute Key conforming to the "process.owner" - // semantic conventions. It represents the username of the user that owns the - // process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "root" - ProcessOwnerKey = attribute.Key("process.owner") - - // ProcessParentPIDKey is the attribute Key conforming to the - // "process.parent_pid" semantic conventions. It represents the parent Process - // identifier (PPID). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 111 - ProcessParentPIDKey = attribute.Key("process.parent_pid") - - // ProcessPIDKey is the attribute Key conforming to the "process.pid" semantic - // conventions. It represents the process identifier (PID). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1234 - ProcessPIDKey = attribute.Key("process.pid") - - // ProcessRealUserIDKey is the attribute Key conforming to the - // "process.real_user.id" semantic conventions. It represents the real user ID - // (RUID) of the process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1000 - ProcessRealUserIDKey = attribute.Key("process.real_user.id") - - // ProcessRealUserNameKey is the attribute Key conforming to the - // "process.real_user.name" semantic conventions. It represents the username of - // the real user of the process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "operator" - ProcessRealUserNameKey = attribute.Key("process.real_user.name") - - // ProcessRuntimeDescriptionKey is the attribute Key conforming to the - // "process.runtime.description" semantic conventions. It represents an - // additional description about the runtime of the process, for example a - // specific vendor customization of the runtime environment. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: Eclipse OpenJ9 Eclipse OpenJ9 VM openj9-0.21.0 - ProcessRuntimeDescriptionKey = attribute.Key("process.runtime.description") - - // ProcessRuntimeNameKey is the attribute Key conforming to the - // "process.runtime.name" semantic conventions. It represents the name of the - // runtime of this process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "OpenJDK Runtime Environment" - ProcessRuntimeNameKey = attribute.Key("process.runtime.name") - - // ProcessRuntimeVersionKey is the attribute Key conforming to the - // "process.runtime.version" semantic conventions. It represents the version of - // the runtime of this process, as returned by the runtime without modification. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 14.0.2 - ProcessRuntimeVersionKey = attribute.Key("process.runtime.version") - - // ProcessSavedUserIDKey is the attribute Key conforming to the - // "process.saved_user.id" semantic conventions. It represents the saved user ID - // (SUID) of the process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1002 - ProcessSavedUserIDKey = attribute.Key("process.saved_user.id") - - // ProcessSavedUserNameKey is the attribute Key conforming to the - // "process.saved_user.name" semantic conventions. It represents the username of - // the saved user. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "operator" - ProcessSavedUserNameKey = attribute.Key("process.saved_user.name") - - // ProcessSessionLeaderPIDKey is the attribute Key conforming to the - // "process.session_leader.pid" semantic conventions. It represents the PID of - // the process's session leader. This is also the session ID (SID) of the - // process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 14 - ProcessSessionLeaderPIDKey = attribute.Key("process.session_leader.pid") - - // ProcessStateKey is the attribute Key conforming to the "process.state" - // semantic conventions. It represents the process state, e.g., - // [Linux Process State Codes]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "running" - // - // [Linux Process State Codes]: https://man7.org/linux/man-pages/man1/ps.1.html#PROCESS_STATE_CODES - ProcessStateKey = attribute.Key("process.state") - - // ProcessTitleKey is the attribute Key conforming to the "process.title" - // semantic conventions. It represents the process title (proctitle). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cat /etc/hostname", "xfce4-session", "bash" - // Note: In many Unix-like systems, process title (proctitle), is the string - // that represents the name or command line of a running process, displayed by - // system monitoring tools like ps, top, and htop. - ProcessTitleKey = attribute.Key("process.title") - - // ProcessUserIDKey is the attribute Key conforming to the "process.user.id" - // semantic conventions. It represents the effective user ID (EUID) of the - // process. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1001 - ProcessUserIDKey = attribute.Key("process.user.id") - - // ProcessUserNameKey is the attribute Key conforming to the "process.user.name" - // semantic conventions. It represents the username of the effective user of the - // process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "root" - ProcessUserNameKey = attribute.Key("process.user.name") - - // ProcessVpidKey is the attribute Key conforming to the "process.vpid" semantic - // conventions. It represents the virtual process identifier. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 12 - // Note: The process ID within a PID namespace. This is not necessarily unique - // across all processes on the host but it is unique within the process - // namespace that the process exists within. - ProcessVpidKey = attribute.Key("process.vpid") - - // ProcessWorkingDirectoryKey is the attribute Key conforming to the - // "process.working_directory" semantic conventions. It represents the working - // directory of the process. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/root" - ProcessWorkingDirectoryKey = attribute.Key("process.working_directory") -) - -// ProcessArgsCount returns an attribute KeyValue conforming to the -// "process.args_count" semantic conventions. It represents the length of the -// process.command_args array. -func ProcessArgsCount(val int) attribute.KeyValue { - return ProcessArgsCountKey.Int(val) -} - -// ProcessCommand returns an attribute KeyValue conforming to the -// "process.command" semantic conventions. It represents the command used to -// launch the process (i.e. the command name). On Linux based systems, can be set -// to the zeroth string in `proc/[pid]/cmdline`. On Windows, can be set to the -// first parameter extracted from `GetCommandLineW`. -func ProcessCommand(val string) attribute.KeyValue { - return ProcessCommandKey.String(val) -} - -// ProcessCommandArgs returns an attribute KeyValue conforming to the -// "process.command_args" semantic conventions. It represents the all the command -// arguments (including the command/executable itself) as received by the -// process. On Linux-based systems (and some other Unixoid systems supporting -// procfs), can be set according to the list of null-delimited strings extracted -// from `proc/[pid]/cmdline`. For libc-based executables, this would be the full -// argv vector passed to `main`. SHOULD NOT be collected by default unless there -// is sanitization that excludes sensitive data. -func ProcessCommandArgs(val ...string) attribute.KeyValue { - return ProcessCommandArgsKey.StringSlice(val) -} - -// ProcessCommandLine returns an attribute KeyValue conforming to the -// "process.command_line" semantic conventions. It represents the full command -// used to launch the process as a single string representing the full command. -// On Windows, can be set to the result of `GetCommandLineW`. Do not set this if -// you have to assemble it just for monitoring; use `process.command_args` -// instead. SHOULD NOT be collected by default unless there is sanitization that -// excludes sensitive data. -func ProcessCommandLine(val string) attribute.KeyValue { - return ProcessCommandLineKey.String(val) -} - -// ProcessCreationTime returns an attribute KeyValue conforming to the -// "process.creation.time" semantic conventions. It represents the date and time -// the process was created, in ISO 8601 format. -func ProcessCreationTime(val string) attribute.KeyValue { - return ProcessCreationTimeKey.String(val) -} - -// ProcessEnvironmentVariable returns an attribute KeyValue conforming to the -// "process.environment_variable" semantic conventions. It represents the process -// environment variables, `` being the environment variable name, the value -// being the environment variable value. -func ProcessEnvironmentVariable(key string, val string) attribute.KeyValue { - return attribute.String("process.environment_variable."+key, val) -} - -// ProcessExecutableBuildIDGNU returns an attribute KeyValue conforming to the -// "process.executable.build_id.gnu" semantic conventions. It represents the GNU -// build ID as found in the `.note.gnu.build-id` ELF section (hex string). -func ProcessExecutableBuildIDGNU(val string) attribute.KeyValue { - return ProcessExecutableBuildIDGNUKey.String(val) -} - -// ProcessExecutableBuildIDGo returns an attribute KeyValue conforming to the -// "process.executable.build_id.go" semantic conventions. It represents the Go -// build ID as retrieved by `go tool buildid `. -func ProcessExecutableBuildIDGo(val string) attribute.KeyValue { - return ProcessExecutableBuildIDGoKey.String(val) -} - -// ProcessExecutableBuildIDHtlhash returns an attribute KeyValue conforming to -// the "process.executable.build_id.htlhash" semantic conventions. It represents -// the profiling specific build ID for executables. See the OTel specification -// for Profiles for more information. -func ProcessExecutableBuildIDHtlhash(val string) attribute.KeyValue { - return ProcessExecutableBuildIDHtlhashKey.String(val) -} - -// ProcessExecutableName returns an attribute KeyValue conforming to the -// "process.executable.name" semantic conventions. It represents the name of the -// process executable. On Linux based systems, this SHOULD be set to the base -// name of the target of `/proc/[pid]/exe`. On Windows, this SHOULD be set to the -// base name of `GetProcessImageFileNameW`. -func ProcessExecutableName(val string) attribute.KeyValue { - return ProcessExecutableNameKey.String(val) -} - -// ProcessExecutablePath returns an attribute KeyValue conforming to the -// "process.executable.path" semantic conventions. It represents the full path to -// the process executable. On Linux based systems, can be set to the target of -// `proc/[pid]/exe`. On Windows, can be set to the result of -// `GetProcessImageFileNameW`. -func ProcessExecutablePath(val string) attribute.KeyValue { - return ProcessExecutablePathKey.String(val) -} - -// ProcessExitCode returns an attribute KeyValue conforming to the -// "process.exit.code" semantic conventions. It represents the exit code of the -// process. -func ProcessExitCode(val int) attribute.KeyValue { - return ProcessExitCodeKey.Int(val) -} - -// ProcessExitTime returns an attribute KeyValue conforming to the -// "process.exit.time" semantic conventions. It represents the date and time the -// process exited, in ISO 8601 format. -func ProcessExitTime(val string) attribute.KeyValue { - return ProcessExitTimeKey.String(val) -} - -// ProcessGroupLeaderPID returns an attribute KeyValue conforming to the -// "process.group_leader.pid" semantic conventions. It represents the PID of the -// process's group leader. This is also the process group ID (PGID) of the -// process. -func ProcessGroupLeaderPID(val int) attribute.KeyValue { - return ProcessGroupLeaderPIDKey.Int(val) -} - -// ProcessInteractive returns an attribute KeyValue conforming to the -// "process.interactive" semantic conventions. It represents the whether the -// process is connected to an interactive shell. -func ProcessInteractive(val bool) attribute.KeyValue { - return ProcessInteractiveKey.Bool(val) -} - -// ProcessLinuxCgroup returns an attribute KeyValue conforming to the -// "process.linux.cgroup" semantic conventions. It represents the control group -// associated with the process. -func ProcessLinuxCgroup(val string) attribute.KeyValue { - return ProcessLinuxCgroupKey.String(val) -} - -// ProcessOwner returns an attribute KeyValue conforming to the "process.owner" -// semantic conventions. It represents the username of the user that owns the -// process. -func ProcessOwner(val string) attribute.KeyValue { - return ProcessOwnerKey.String(val) -} - -// ProcessParentPID returns an attribute KeyValue conforming to the -// "process.parent_pid" semantic conventions. It represents the parent Process -// identifier (PPID). -func ProcessParentPID(val int) attribute.KeyValue { - return ProcessParentPIDKey.Int(val) -} - -// ProcessPID returns an attribute KeyValue conforming to the "process.pid" -// semantic conventions. It represents the process identifier (PID). -func ProcessPID(val int) attribute.KeyValue { - return ProcessPIDKey.Int(val) -} - -// ProcessRealUserID returns an attribute KeyValue conforming to the -// "process.real_user.id" semantic conventions. It represents the real user ID -// (RUID) of the process. -func ProcessRealUserID(val int) attribute.KeyValue { - return ProcessRealUserIDKey.Int(val) -} - -// ProcessRealUserName returns an attribute KeyValue conforming to the -// "process.real_user.name" semantic conventions. It represents the username of -// the real user of the process. -func ProcessRealUserName(val string) attribute.KeyValue { - return ProcessRealUserNameKey.String(val) -} - -// ProcessRuntimeDescription returns an attribute KeyValue conforming to the -// "process.runtime.description" semantic conventions. It represents an -// additional description about the runtime of the process, for example a -// specific vendor customization of the runtime environment. -func ProcessRuntimeDescription(val string) attribute.KeyValue { - return ProcessRuntimeDescriptionKey.String(val) -} - -// ProcessRuntimeName returns an attribute KeyValue conforming to the -// "process.runtime.name" semantic conventions. It represents the name of the -// runtime of this process. -func ProcessRuntimeName(val string) attribute.KeyValue { - return ProcessRuntimeNameKey.String(val) -} - -// ProcessRuntimeVersion returns an attribute KeyValue conforming to the -// "process.runtime.version" semantic conventions. It represents the version of -// the runtime of this process, as returned by the runtime without modification. -func ProcessRuntimeVersion(val string) attribute.KeyValue { - return ProcessRuntimeVersionKey.String(val) -} - -// ProcessSavedUserID returns an attribute KeyValue conforming to the -// "process.saved_user.id" semantic conventions. It represents the saved user ID -// (SUID) of the process. -func ProcessSavedUserID(val int) attribute.KeyValue { - return ProcessSavedUserIDKey.Int(val) -} - -// ProcessSavedUserName returns an attribute KeyValue conforming to the -// "process.saved_user.name" semantic conventions. It represents the username of -// the saved user. -func ProcessSavedUserName(val string) attribute.KeyValue { - return ProcessSavedUserNameKey.String(val) -} - -// ProcessSessionLeaderPID returns an attribute KeyValue conforming to the -// "process.session_leader.pid" semantic conventions. It represents the PID of -// the process's session leader. This is also the session ID (SID) of the -// process. -func ProcessSessionLeaderPID(val int) attribute.KeyValue { - return ProcessSessionLeaderPIDKey.Int(val) -} - -// ProcessTitle returns an attribute KeyValue conforming to the "process.title" -// semantic conventions. It represents the process title (proctitle). -func ProcessTitle(val string) attribute.KeyValue { - return ProcessTitleKey.String(val) -} - -// ProcessUserID returns an attribute KeyValue conforming to the -// "process.user.id" semantic conventions. It represents the effective user ID -// (EUID) of the process. -func ProcessUserID(val int) attribute.KeyValue { - return ProcessUserIDKey.Int(val) -} - -// ProcessUserName returns an attribute KeyValue conforming to the -// "process.user.name" semantic conventions. It represents the username of the -// effective user of the process. -func ProcessUserName(val string) attribute.KeyValue { - return ProcessUserNameKey.String(val) -} - -// ProcessVpid returns an attribute KeyValue conforming to the "process.vpid" -// semantic conventions. It represents the virtual process identifier. -func ProcessVpid(val int) attribute.KeyValue { - return ProcessVpidKey.Int(val) -} - -// ProcessWorkingDirectory returns an attribute KeyValue conforming to the -// "process.working_directory" semantic conventions. It represents the working -// directory of the process. -func ProcessWorkingDirectory(val string) attribute.KeyValue { - return ProcessWorkingDirectoryKey.String(val) -} - -// Enum values for process.context_switch.type -var ( - // voluntary - // Stability: development - ProcessContextSwitchTypeVoluntary = ProcessContextSwitchTypeKey.String("voluntary") - // involuntary - // Stability: development - ProcessContextSwitchTypeInvoluntary = ProcessContextSwitchTypeKey.String("involuntary") -) - -// Enum values for process.state -var ( - // running - // Stability: development - ProcessStateRunning = ProcessStateKey.String("running") - // sleeping - // Stability: development - ProcessStateSleeping = ProcessStateKey.String("sleeping") - // stopped - // Stability: development - ProcessStateStopped = ProcessStateKey.String("stopped") - // defunct - // Stability: development - ProcessStateDefunct = ProcessStateKey.String("defunct") -) - -// Namespace: profile -const ( - // ProfileFrameTypeKey is the attribute Key conforming to the - // "profile.frame.type" semantic conventions. It represents the describes the - // interpreter or compiler of a single frame. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "cpython" - ProfileFrameTypeKey = attribute.Key("profile.frame.type") -) - -// Enum values for profile.frame.type -var ( - // [.NET] - // - // Stability: development - // - // [.NET]: https://wikipedia.org/wiki/.NET - ProfileFrameTypeDotnet = ProfileFrameTypeKey.String("dotnet") - // [JVM] - // - // Stability: development - // - // [JVM]: https://wikipedia.org/wiki/Java_virtual_machine - ProfileFrameTypeJVM = ProfileFrameTypeKey.String("jvm") - // [Kernel] - // - // Stability: development - // - // [Kernel]: https://wikipedia.org/wiki/Kernel_(operating_system) - ProfileFrameTypeKernel = ProfileFrameTypeKey.String("kernel") - // Can be one of but not limited to [C], [C++], [Go] or [Rust]. If possible, a - // more precise value MUST be used. - // - // Stability: development - // - // [C]: https://wikipedia.org/wiki/C_(programming_language) - // [C++]: https://wikipedia.org/wiki/C%2B%2B - // [Go]: https://wikipedia.org/wiki/Go_(programming_language) - // [Rust]: https://wikipedia.org/wiki/Rust_(programming_language) - ProfileFrameTypeNative = ProfileFrameTypeKey.String("native") - // [Perl] - // - // Stability: development - // - // [Perl]: https://wikipedia.org/wiki/Perl - ProfileFrameTypePerl = ProfileFrameTypeKey.String("perl") - // [PHP] - // - // Stability: development - // - // [PHP]: https://wikipedia.org/wiki/PHP - ProfileFrameTypePHP = ProfileFrameTypeKey.String("php") - // [Python] - // - // Stability: development - // - // [Python]: https://wikipedia.org/wiki/Python_(programming_language) - ProfileFrameTypeCpython = ProfileFrameTypeKey.String("cpython") - // [Ruby] - // - // Stability: development - // - // [Ruby]: https://wikipedia.org/wiki/Ruby_(programming_language) - ProfileFrameTypeRuby = ProfileFrameTypeKey.String("ruby") - // [V8JS] - // - // Stability: development - // - // [V8JS]: https://wikipedia.org/wiki/V8_(JavaScript_engine) - ProfileFrameTypeV8JS = ProfileFrameTypeKey.String("v8js") - // [Erlang] - // - // Stability: development - // - // [Erlang]: https://en.wikipedia.org/wiki/BEAM_(Erlang_virtual_machine) - ProfileFrameTypeBeam = ProfileFrameTypeKey.String("beam") - // [Go], - // - // Stability: development - // - // [Go]: https://wikipedia.org/wiki/Go_(programming_language) - ProfileFrameTypeGo = ProfileFrameTypeKey.String("go") - // [Rust] - // - // Stability: development - // - // [Rust]: https://wikipedia.org/wiki/Rust_(programming_language) - ProfileFrameTypeRust = ProfileFrameTypeKey.String("rust") -) - -// Namespace: rpc -const ( - // RPCMessageCompressedSizeKey is the attribute Key conforming to the - // "rpc.message.compressed_size" semantic conventions. It represents the - // compressed size of the message in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - RPCMessageCompressedSizeKey = attribute.Key("rpc.message.compressed_size") - - // RPCMessageIDKey is the attribute Key conforming to the "rpc.message.id" - // semantic conventions. It MUST be calculated as two different counters - // starting from `1` one for sent messages and one for received message.. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: This way we guarantee that the values will be consistent between - // different implementations. - RPCMessageIDKey = attribute.Key("rpc.message.id") - - // RPCMessageTypeKey is the attribute Key conforming to the "rpc.message.type" - // semantic conventions. It represents the whether this is a received or sent - // message. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - RPCMessageTypeKey = attribute.Key("rpc.message.type") - - // RPCMessageUncompressedSizeKey is the attribute Key conforming to the - // "rpc.message.uncompressed_size" semantic conventions. It represents the - // uncompressed size of the message in bytes. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - RPCMessageUncompressedSizeKey = attribute.Key("rpc.message.uncompressed_size") - - // RPCMethodKey is the attribute Key conforming to the "rpc.method" semantic - // conventions. It represents the fully-qualified logical name of the method - // from the RPC interface perspective. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "com.example.ExampleService/exampleMethod", "EchoService/Echo", - // "_OTHER" - // Note: The method name MAY have unbounded cardinality in edge or error cases. - // - // Some RPC frameworks or libraries provide a fixed set of recognized methods - // for client stubs and server implementations. Instrumentations for such - // frameworks MUST set this attribute to the original method name only - // when the method is recognized by the framework or library. - // - // When the method is not recognized, for example, when the server receives - // a request for a method that is not predefined on the server, or when - // instrumentation is not able to reliably detect if the method is predefined, - // the attribute MUST be set to `_OTHER`. In such cases, tracing - // instrumentations MUST also set `rpc.method_original` attribute to - // the original method value. - // - // If the RPC instrumentation could end up converting valid RPC methods to - // `_OTHER`, then it SHOULD provide a way to configure the list of recognized - // RPC methods. - // - // The `rpc.method` can be different from the name of any implementing - // method/function. - // The `code.function.name` attribute may be used to record the fully-qualified - // method actually executing the call on the server side, or the - // RPC client stub method on the client side. - RPCMethodKey = attribute.Key("rpc.method") - - // RPCMethodOriginalKey is the attribute Key conforming to the - // "rpc.method_original" semantic conventions. It represents the original name - // of the method used by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "com.myservice.EchoService/catchAll", - // "com.myservice.EchoService/unknownMethod", "InvalidMethod" - RPCMethodOriginalKey = attribute.Key("rpc.method_original") - - // RPCResponseStatusCodeKey is the attribute Key conforming to the - // "rpc.response.status_code" semantic conventions. It represents the status - // code of the RPC returned by the RPC server or generated by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "OK", "DEADLINE_EXCEEDED", "-32602" - // Note: Usually it represents an error code, but may also represent partial - // success, warning, or differentiate between various types of successful - // outcomes. - // Semantic conventions for individual RPC frameworks SHOULD document what - // `rpc.response.status_code` means in the context of that system and which - // values are considered to represent errors. - RPCResponseStatusCodeKey = attribute.Key("rpc.response.status_code") - - // RPCSystemNameKey is the attribute Key conforming to the "rpc.system.name" - // semantic conventions. It represents the Remote Procedure Call (RPC) system. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: The client and server RPC systems may differ for the same RPC - // interaction. For example, a client may use Apache Dubbo or Connect RPC to - // communicate with a server that uses gRPC since both protocols provide - // compatibility with gRPC. - RPCSystemNameKey = attribute.Key("rpc.system.name") -) - -// RPCMessageCompressedSize returns an attribute KeyValue conforming to the -// "rpc.message.compressed_size" semantic conventions. It represents the -// compressed size of the message in bytes. -func RPCMessageCompressedSize(val int) attribute.KeyValue { - return RPCMessageCompressedSizeKey.Int(val) -} - -// RPCMessageID returns an attribute KeyValue conforming to the "rpc.message.id" -// semantic conventions. It MUST be calculated as two different counters starting -// from `1` one for sent messages and one for received message.. -func RPCMessageID(val int) attribute.KeyValue { - return RPCMessageIDKey.Int(val) -} - -// RPCMessageUncompressedSize returns an attribute KeyValue conforming to the -// "rpc.message.uncompressed_size" semantic conventions. It represents the -// uncompressed size of the message in bytes. -func RPCMessageUncompressedSize(val int) attribute.KeyValue { - return RPCMessageUncompressedSizeKey.Int(val) -} - -// RPCMethod returns an attribute KeyValue conforming to the "rpc.method" -// semantic conventions. It represents the fully-qualified logical name of the -// method from the RPC interface perspective. -func RPCMethod(val string) attribute.KeyValue { - return RPCMethodKey.String(val) -} - -// RPCMethodOriginal returns an attribute KeyValue conforming to the -// "rpc.method_original" semantic conventions. It represents the original name of -// the method used by the client. -func RPCMethodOriginal(val string) attribute.KeyValue { - return RPCMethodOriginalKey.String(val) -} - -// RPCRequestMetadata returns an attribute KeyValue conforming to the -// "rpc.request.metadata" semantic conventions. It represents the RPC request -// metadata, `` being the normalized RPC metadata key (lowercase), the value -// being the metadata values. -func RPCRequestMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.request.metadata."+key, val) -} - -// RPCResponseMetadata returns an attribute KeyValue conforming to the -// "rpc.response.metadata" semantic conventions. It represents the RPC response -// metadata, `` being the normalized RPC metadata key (lowercase), the value -// being the metadata values. -func RPCResponseMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.response.metadata."+key, val) -} - -// RPCResponseStatusCode returns an attribute KeyValue conforming to the -// "rpc.response.status_code" semantic conventions. It represents the status code -// of the RPC returned by the RPC server or generated by the client. -func RPCResponseStatusCode(val string) attribute.KeyValue { - return RPCResponseStatusCodeKey.String(val) -} - -// Enum values for rpc.message.type -var ( - // sent - // Stability: development - RPCMessageTypeSent = RPCMessageTypeKey.String("SENT") - // received - // Stability: development - RPCMessageTypeReceived = RPCMessageTypeKey.String("RECEIVED") -) - -// Enum values for rpc.system.name -var ( - // [gRPC] - // Stability: development - // - // [gRPC]: https://grpc.io/ - RPCSystemNameGRPC = RPCSystemNameKey.String("grpc") - // [Apache Dubbo] - // Stability: development - // - // [Apache Dubbo]: https://dubbo.apache.org/ - RPCSystemNameDubbo = RPCSystemNameKey.String("dubbo") - // [Connect RPC] - // Stability: development - // - // [Connect RPC]: https://connectrpc.com/ - RPCSystemNameConnectrpc = RPCSystemNameKey.String("connectrpc") - // [JSON-RPC] - // Stability: development - // - // [JSON-RPC]: https://www.jsonrpc.org/ - RPCSystemNameJSONRPC = RPCSystemNameKey.String("jsonrpc") -) - -// Namespace: security_rule -const ( - // SecurityRuleCategoryKey is the attribute Key conforming to the - // "security_rule.category" semantic conventions. It represents a categorization - // value keyword used by the entity using the rule for detection of this event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Attempted Information Leak" - SecurityRuleCategoryKey = attribute.Key("security_rule.category") - - // SecurityRuleDescriptionKey is the attribute Key conforming to the - // "security_rule.description" semantic conventions. It represents the - // description of the rule generating the event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Block requests to public DNS over HTTPS / TLS protocols" - SecurityRuleDescriptionKey = attribute.Key("security_rule.description") - - // SecurityRuleLicenseKey is the attribute Key conforming to the - // "security_rule.license" semantic conventions. It represents the name of the - // license under which the rule used to generate this event is made available. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Apache 2.0" - SecurityRuleLicenseKey = attribute.Key("security_rule.license") - - // SecurityRuleNameKey is the attribute Key conforming to the - // "security_rule.name" semantic conventions. It represents the name of the rule - // or signature generating the event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "BLOCK_DNS_over_TLS" - SecurityRuleNameKey = attribute.Key("security_rule.name") - - // SecurityRuleReferenceKey is the attribute Key conforming to the - // "security_rule.reference" semantic conventions. It represents the reference - // URL to additional information about the rule used to generate this event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://en.wikipedia.org/wiki/DNS_over_TLS" - // Note: The URL can point to the vendor’s documentation about the rule. If - // that’s not available, it can also be a link to a more general page - // describing this type of alert. - SecurityRuleReferenceKey = attribute.Key("security_rule.reference") - - // SecurityRuleRulesetNameKey is the attribute Key conforming to the - // "security_rule.ruleset.name" semantic conventions. It represents the name of - // the ruleset, policy, group, or parent category in which the rule used to - // generate this event is a member. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Standard_Protocol_Filters" - SecurityRuleRulesetNameKey = attribute.Key("security_rule.ruleset.name") - - // SecurityRuleUUIDKey is the attribute Key conforming to the - // "security_rule.uuid" semantic conventions. It represents a rule ID that is - // unique within the scope of a set or group of agents, observers, or other - // entities using the rule for detection of this event. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "550e8400-e29b-41d4-a716-446655440000", "1100110011" - SecurityRuleUUIDKey = attribute.Key("security_rule.uuid") - - // SecurityRuleVersionKey is the attribute Key conforming to the - // "security_rule.version" semantic conventions. It represents the version / - // revision of the rule being used for analysis. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1.0.0" - SecurityRuleVersionKey = attribute.Key("security_rule.version") -) - -// SecurityRuleCategory returns an attribute KeyValue conforming to the -// "security_rule.category" semantic conventions. It represents a categorization -// value keyword used by the entity using the rule for detection of this event. -func SecurityRuleCategory(val string) attribute.KeyValue { - return SecurityRuleCategoryKey.String(val) -} - -// SecurityRuleDescription returns an attribute KeyValue conforming to the -// "security_rule.description" semantic conventions. It represents the -// description of the rule generating the event. -func SecurityRuleDescription(val string) attribute.KeyValue { - return SecurityRuleDescriptionKey.String(val) -} - -// SecurityRuleLicense returns an attribute KeyValue conforming to the -// "security_rule.license" semantic conventions. It represents the name of the -// license under which the rule used to generate this event is made available. -func SecurityRuleLicense(val string) attribute.KeyValue { - return SecurityRuleLicenseKey.String(val) -} - -// SecurityRuleName returns an attribute KeyValue conforming to the -// "security_rule.name" semantic conventions. It represents the name of the rule -// or signature generating the event. -func SecurityRuleName(val string) attribute.KeyValue { - return SecurityRuleNameKey.String(val) -} - -// SecurityRuleReference returns an attribute KeyValue conforming to the -// "security_rule.reference" semantic conventions. It represents the reference -// URL to additional information about the rule used to generate this event. -func SecurityRuleReference(val string) attribute.KeyValue { - return SecurityRuleReferenceKey.String(val) -} - -// SecurityRuleRulesetName returns an attribute KeyValue conforming to the -// "security_rule.ruleset.name" semantic conventions. It represents the name of -// the ruleset, policy, group, or parent category in which the rule used to -// generate this event is a member. -func SecurityRuleRulesetName(val string) attribute.KeyValue { - return SecurityRuleRulesetNameKey.String(val) -} - -// SecurityRuleUUID returns an attribute KeyValue conforming to the -// "security_rule.uuid" semantic conventions. It represents a rule ID that is -// unique within the scope of a set or group of agents, observers, or other -// entities using the rule for detection of this event. -func SecurityRuleUUID(val string) attribute.KeyValue { - return SecurityRuleUUIDKey.String(val) -} - -// SecurityRuleVersion returns an attribute KeyValue conforming to the -// "security_rule.version" semantic conventions. It represents the version / -// revision of the rule being used for analysis. -func SecurityRuleVersion(val string) attribute.KeyValue { - return SecurityRuleVersionKey.String(val) -} - -// Namespace: server -const ( - // ServerAddressKey is the attribute Key conforming to the "server.address" - // semantic conventions. It represents the server domain name if available - // without reverse DNS lookup; otherwise, IP address or Unix domain socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "example.com", "10.1.2.80", "/tmp/my.sock" - // Note: When observed from the client side, and when communicating through an - // intermediary, `server.address` SHOULD represent the server address behind any - // intermediaries, for example proxies, if it's available. - ServerAddressKey = attribute.Key("server.address") - - // ServerPortKey is the attribute Key conforming to the "server.port" semantic - // conventions. It represents the server port number. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: 80, 8080, 443 - // Note: When observed from the client side, and when communicating through an - // intermediary, `server.port` SHOULD represent the server port behind any - // intermediaries, for example proxies, if it's available. - ServerPortKey = attribute.Key("server.port") -) - -// ServerAddress returns an attribute KeyValue conforming to the "server.address" -// semantic conventions. It represents the server domain name if available -// without reverse DNS lookup; otherwise, IP address or Unix domain socket name. -func ServerAddress(val string) attribute.KeyValue { - return ServerAddressKey.String(val) -} - -// ServerPort returns an attribute KeyValue conforming to the "server.port" -// semantic conventions. It represents the server port number. -func ServerPort(val int) attribute.KeyValue { - return ServerPortKey.Int(val) -} - -// Namespace: service -const ( - // ServiceInstanceIDKey is the attribute Key conforming to the - // "service.instance.id" semantic conventions. It represents the string ID of - // the service instance. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "627cc493-f310-47de-96bd-71410b7dec09" - // Note: MUST be unique for each instance of the same - // `service.namespace,service.name` pair (in other words - // `service.namespace,service.name,service.instance.id` triplet MUST be globally - // unique). The ID helps to - // distinguish instances of the same service that exist at the same time (e.g. - // instances of a horizontally scaled - // service). - // - // Implementations, such as SDKs, are recommended to generate a random Version 1 - // or Version 4 [RFC - // 4122] UUID, but are free to use an inherent unique ID as - // the source of - // this value if stability is desirable. In that case, the ID SHOULD be used as - // source of a UUID Version 5 and - // SHOULD use the following UUID as the namespace: - // `4d63009a-8d0f-11ee-aad7-4c796ed8e320`. - // - // UUIDs are typically recommended, as only an opaque value for the purposes of - // identifying a service instance is - // needed. Similar to what can be seen in the man page for the - // [`/etc/machine-id`] file, the underlying - // data, such as pod name and namespace should be treated as confidential, being - // the user's choice to expose it - // or not via another resource attribute. - // - // For applications running behind an application server (like unicorn), we do - // not recommend using one identifier - // for all processes participating in the application. Instead, it's recommended - // each division (e.g. a worker - // thread in unicorn) to have its own instance.id. - // - // It's not recommended for a Collector to set `service.instance.id` if it can't - // unambiguously determine the - // service instance that is generating that telemetry. For instance, creating an - // UUID based on `pod.name` will - // likely be wrong, as the Collector might not know from which container within - // that pod the telemetry originated. - // However, Collectors can set the `service.instance.id` if they can - // unambiguously determine the service instance - // for that telemetry. This is typically the case for scraping receivers, as - // they know the target address and - // port. - // - // [RFC - // 4122]: https://www.ietf.org/rfc/rfc4122.txt - // [`/etc/machine-id`]: https://www.freedesktop.org/software/systemd/man/latest/machine-id.html - ServiceInstanceIDKey = attribute.Key("service.instance.id") - - // ServiceNameKey is the attribute Key conforming to the "service.name" semantic - // conventions. It represents the logical name of the service. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "shoppingcart" - // Note: MUST be the same for all instances of horizontally scaled services. If - // the value was not specified, SDKs MUST fallback to `unknown_service:` - // concatenated with [`process.executable.name`], e.g. `unknown_service:bash`. - // If `process.executable.name` is not available, the value MUST be set to - // `unknown_service`. - // - // [`process.executable.name`]: process.md - ServiceNameKey = attribute.Key("service.name") - - // ServiceNamespaceKey is the attribute Key conforming to the - // "service.namespace" semantic conventions. It represents a namespace for - // `service.name`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Shop" - // Note: A string value having a meaning that helps to distinguish a group of - // services, for example the team name that owns a group of services. - // `service.name` is expected to be unique within the same namespace. If - // `service.namespace` is not specified in the Resource then `service.name` is - // expected to be unique for all services that have no explicit namespace - // defined (so the empty/unspecified namespace is simply one more valid - // namespace). Zero-length namespace string is assumed equal to unspecified - // namespace. - ServiceNamespaceKey = attribute.Key("service.namespace") - - // ServicePeerNameKey is the attribute Key conforming to the "service.peer.name" - // semantic conventions. It represents the logical name of the service on the - // other side of the connection. SHOULD be equal to the actual [`service.name`] - // resource attribute of the remote service if any. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "shoppingcart" - // - // [`service.name`]: /docs/resource/README.md#service - ServicePeerNameKey = attribute.Key("service.peer.name") - - // ServicePeerNamespaceKey is the attribute Key conforming to the - // "service.peer.namespace" semantic conventions. It represents the logical - // namespace of the service on the other side of the connection. SHOULD be equal - // to the actual [`service.namespace`] resource attribute of the remote service - // if any. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Shop" - // - // [`service.namespace`]: /docs/resource/README.md#service - ServicePeerNamespaceKey = attribute.Key("service.peer.namespace") - - // ServiceVersionKey is the attribute Key conforming to the "service.version" - // semantic conventions. It represents the version string of the service - // component. The format is not defined by these conventions. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "2.0.0", "a01dbef8a" - ServiceVersionKey = attribute.Key("service.version") -) - -// ServiceInstanceID returns an attribute KeyValue conforming to the -// "service.instance.id" semantic conventions. It represents the string ID of the -// service instance. -func ServiceInstanceID(val string) attribute.KeyValue { - return ServiceInstanceIDKey.String(val) -} - -// ServiceName returns an attribute KeyValue conforming to the "service.name" -// semantic conventions. It represents the logical name of the service. -func ServiceName(val string) attribute.KeyValue { - return ServiceNameKey.String(val) -} - -// ServiceNamespace returns an attribute KeyValue conforming to the -// "service.namespace" semantic conventions. It represents a namespace for -// `service.name`. -func ServiceNamespace(val string) attribute.KeyValue { - return ServiceNamespaceKey.String(val) -} - -// ServicePeerName returns an attribute KeyValue conforming to the -// "service.peer.name" semantic conventions. It represents the logical name of -// the service on the other side of the connection. SHOULD be equal to the actual -// [`service.name`] resource attribute of the remote service if any. -// -// [`service.name`]: /docs/resource/README.md#service -func ServicePeerName(val string) attribute.KeyValue { - return ServicePeerNameKey.String(val) -} - -// ServicePeerNamespace returns an attribute KeyValue conforming to the -// "service.peer.namespace" semantic conventions. It represents the logical -// namespace of the service on the other side of the connection. SHOULD be equal -// to the actual [`service.namespace`] resource attribute of the remote service -// if any. -// -// [`service.namespace`]: /docs/resource/README.md#service -func ServicePeerNamespace(val string) attribute.KeyValue { - return ServicePeerNamespaceKey.String(val) -} - -// ServiceVersion returns an attribute KeyValue conforming to the -// "service.version" semantic conventions. It represents the version string of -// the service component. The format is not defined by these conventions. -func ServiceVersion(val string) attribute.KeyValue { - return ServiceVersionKey.String(val) -} - -// Namespace: session -const ( - // SessionIDKey is the attribute Key conforming to the "session.id" semantic - // conventions. It represents a unique id to identify a session. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 00112233-4455-6677-8899-aabbccddeeff - SessionIDKey = attribute.Key("session.id") - - // SessionPreviousIDKey is the attribute Key conforming to the - // "session.previous_id" semantic conventions. It represents the previous - // `session.id` for this user, when known. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 00112233-4455-6677-8899-aabbccddeeff - SessionPreviousIDKey = attribute.Key("session.previous_id") -) - -// SessionID returns an attribute KeyValue conforming to the "session.id" -// semantic conventions. It represents a unique id to identify a session. -func SessionID(val string) attribute.KeyValue { - return SessionIDKey.String(val) -} - -// SessionPreviousID returns an attribute KeyValue conforming to the -// "session.previous_id" semantic conventions. It represents the previous -// `session.id` for this user, when known. -func SessionPreviousID(val string) attribute.KeyValue { - return SessionPreviousIDKey.String(val) -} - -// Namespace: signalr -const ( - // SignalRConnectionStatusKey is the attribute Key conforming to the - // "signalr.connection.status" semantic conventions. It represents the signalR - // HTTP connection closure status. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "app_shutdown", "timeout" - SignalRConnectionStatusKey = attribute.Key("signalr.connection.status") - - // SignalRTransportKey is the attribute Key conforming to the - // "signalr.transport" semantic conventions. It represents the - // [SignalR transport type]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "web_sockets", "long_polling" - // - // [SignalR transport type]: https://github.com/dotnet/aspnetcore/blob/main/src/SignalR/docs/specs/TransportProtocols.md - SignalRTransportKey = attribute.Key("signalr.transport") -) - -// Enum values for signalr.connection.status -var ( - // The connection was closed normally. - // Stability: stable - SignalRConnectionStatusNormalClosure = SignalRConnectionStatusKey.String("normal_closure") - // The connection was closed due to a timeout. - // Stability: stable - SignalRConnectionStatusTimeout = SignalRConnectionStatusKey.String("timeout") - // The connection was closed because the app is shutting down. - // Stability: stable - SignalRConnectionStatusAppShutdown = SignalRConnectionStatusKey.String("app_shutdown") -) - -// Enum values for signalr.transport -var ( - // ServerSentEvents protocol - // Stability: stable - SignalRTransportServerSentEvents = SignalRTransportKey.String("server_sent_events") - // LongPolling protocol - // Stability: stable - SignalRTransportLongPolling = SignalRTransportKey.String("long_polling") - // WebSockets protocol - // Stability: stable - SignalRTransportWebSockets = SignalRTransportKey.String("web_sockets") -) - -// Namespace: source -const ( - // SourceAddressKey is the attribute Key conforming to the "source.address" - // semantic conventions. It represents the source address - domain name if - // available without reverse DNS lookup; otherwise, IP address or Unix domain - // socket name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "source.example.com", "10.1.2.80", "/tmp/my.sock" - // Note: When observed from the destination side, and when communicating through - // an intermediary, `source.address` SHOULD represent the source address behind - // any intermediaries, for example proxies, if it's available. - SourceAddressKey = attribute.Key("source.address") - - // SourcePortKey is the attribute Key conforming to the "source.port" semantic - // conventions. It represents the source port number. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 3389, 2888 - SourcePortKey = attribute.Key("source.port") -) - -// SourceAddress returns an attribute KeyValue conforming to the "source.address" -// semantic conventions. It represents the source address - domain name if -// available without reverse DNS lookup; otherwise, IP address or Unix domain -// socket name. -func SourceAddress(val string) attribute.KeyValue { - return SourceAddressKey.String(val) -} - -// SourcePort returns an attribute KeyValue conforming to the "source.port" -// semantic conventions. It represents the source port number. -func SourcePort(val int) attribute.KeyValue { - return SourcePortKey.Int(val) -} - -// Namespace: system -const ( - // SystemDeviceKey is the attribute Key conforming to the "system.device" - // semantic conventions. It represents the device identifier. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "(identifier)" - SystemDeviceKey = attribute.Key("system.device") - - // SystemFilesystemModeKey is the attribute Key conforming to the - // "system.filesystem.mode" semantic conventions. It represents the filesystem - // mode. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "rw, ro" - SystemFilesystemModeKey = attribute.Key("system.filesystem.mode") - - // SystemFilesystemMountpointKey is the attribute Key conforming to the - // "system.filesystem.mountpoint" semantic conventions. It represents the - // filesystem mount path. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/mnt/data" - SystemFilesystemMountpointKey = attribute.Key("system.filesystem.mountpoint") - - // SystemFilesystemStateKey is the attribute Key conforming to the - // "system.filesystem.state" semantic conventions. It represents the filesystem - // state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "used" - SystemFilesystemStateKey = attribute.Key("system.filesystem.state") - - // SystemFilesystemTypeKey is the attribute Key conforming to the - // "system.filesystem.type" semantic conventions. It represents the filesystem - // type. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ext4" - SystemFilesystemTypeKey = attribute.Key("system.filesystem.type") - - // SystemMemoryLinuxSlabStateKey is the attribute Key conforming to the - // "system.memory.linux.slab.state" semantic conventions. It represents the - // Linux Slab memory state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "reclaimable", "unreclaimable" - SystemMemoryLinuxSlabStateKey = attribute.Key("system.memory.linux.slab.state") - - // SystemMemoryStateKey is the attribute Key conforming to the - // "system.memory.state" semantic conventions. It represents the memory state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "free", "cached" - SystemMemoryStateKey = attribute.Key("system.memory.state") - - // SystemPagingDirectionKey is the attribute Key conforming to the - // "system.paging.direction" semantic conventions. It represents the paging - // access direction. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "in" - SystemPagingDirectionKey = attribute.Key("system.paging.direction") - - // SystemPagingFaultTypeKey is the attribute Key conforming to the - // "system.paging.fault.type" semantic conventions. It represents the paging - // fault type. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "minor" - SystemPagingFaultTypeKey = attribute.Key("system.paging.fault.type") - - // SystemPagingStateKey is the attribute Key conforming to the - // "system.paging.state" semantic conventions. It represents the memory paging - // state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "free" - SystemPagingStateKey = attribute.Key("system.paging.state") -) - -// SystemDevice returns an attribute KeyValue conforming to the "system.device" -// semantic conventions. It represents the device identifier. -func SystemDevice(val string) attribute.KeyValue { - return SystemDeviceKey.String(val) -} - -// SystemFilesystemMode returns an attribute KeyValue conforming to the -// "system.filesystem.mode" semantic conventions. It represents the filesystem -// mode. -func SystemFilesystemMode(val string) attribute.KeyValue { - return SystemFilesystemModeKey.String(val) -} - -// SystemFilesystemMountpoint returns an attribute KeyValue conforming to the -// "system.filesystem.mountpoint" semantic conventions. It represents the -// filesystem mount path. -func SystemFilesystemMountpoint(val string) attribute.KeyValue { - return SystemFilesystemMountpointKey.String(val) -} - -// Enum values for system.filesystem.state -var ( - // used - // Stability: development - SystemFilesystemStateUsed = SystemFilesystemStateKey.String("used") - // free - // Stability: development - SystemFilesystemStateFree = SystemFilesystemStateKey.String("free") - // reserved - // Stability: development - SystemFilesystemStateReserved = SystemFilesystemStateKey.String("reserved") -) - -// Enum values for system.filesystem.type -var ( - // fat32 - // Stability: development - SystemFilesystemTypeFat32 = SystemFilesystemTypeKey.String("fat32") - // exfat - // Stability: development - SystemFilesystemTypeExfat = SystemFilesystemTypeKey.String("exfat") - // ntfs - // Stability: development - SystemFilesystemTypeNtfs = SystemFilesystemTypeKey.String("ntfs") - // refs - // Stability: development - SystemFilesystemTypeRefs = SystemFilesystemTypeKey.String("refs") - // hfsplus - // Stability: development - SystemFilesystemTypeHfsplus = SystemFilesystemTypeKey.String("hfsplus") - // ext4 - // Stability: development - SystemFilesystemTypeExt4 = SystemFilesystemTypeKey.String("ext4") -) - -// Enum values for system.memory.linux.slab.state -var ( - // reclaimable - // Stability: development - SystemMemoryLinuxSlabStateReclaimable = SystemMemoryLinuxSlabStateKey.String("reclaimable") - // unreclaimable - // Stability: development - SystemMemoryLinuxSlabStateUnreclaimable = SystemMemoryLinuxSlabStateKey.String("unreclaimable") -) - -// Enum values for system.memory.state -var ( - // Actual used virtual memory in bytes. - // Stability: development - SystemMemoryStateUsed = SystemMemoryStateKey.String("used") - // free - // Stability: development - SystemMemoryStateFree = SystemMemoryStateKey.String("free") - // buffers - // Stability: development - SystemMemoryStateBuffers = SystemMemoryStateKey.String("buffers") - // cached - // Stability: development - SystemMemoryStateCached = SystemMemoryStateKey.String("cached") -) - -// Enum values for system.paging.direction -var ( - // in - // Stability: development - SystemPagingDirectionIn = SystemPagingDirectionKey.String("in") - // out - // Stability: development - SystemPagingDirectionOut = SystemPagingDirectionKey.String("out") -) - -// Enum values for system.paging.fault.type -var ( - // major - // Stability: development - SystemPagingFaultTypeMajor = SystemPagingFaultTypeKey.String("major") - // minor - // Stability: development - SystemPagingFaultTypeMinor = SystemPagingFaultTypeKey.String("minor") -) - -// Enum values for system.paging.state -var ( - // used - // Stability: development - SystemPagingStateUsed = SystemPagingStateKey.String("used") - // free - // Stability: development - SystemPagingStateFree = SystemPagingStateKey.String("free") -) - -// Namespace: telemetry -const ( - // TelemetryDistroNameKey is the attribute Key conforming to the - // "telemetry.distro.name" semantic conventions. It represents the name of the - // auto instrumentation agent or distribution, if used. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "parts-unlimited-java" - // Note: Official auto instrumentation agents and distributions SHOULD set the - // `telemetry.distro.name` attribute to - // a string starting with `opentelemetry-`, e.g. - // `opentelemetry-java-instrumentation`. - TelemetryDistroNameKey = attribute.Key("telemetry.distro.name") - - // TelemetryDistroVersionKey is the attribute Key conforming to the - // "telemetry.distro.version" semantic conventions. It represents the version - // string of the auto instrumentation agent or distribution, if used. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1.2.3" - TelemetryDistroVersionKey = attribute.Key("telemetry.distro.version") - - // TelemetrySDKLanguageKey is the attribute Key conforming to the - // "telemetry.sdk.language" semantic conventions. It represents the language of - // the telemetry SDK. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: - TelemetrySDKLanguageKey = attribute.Key("telemetry.sdk.language") - - // TelemetrySDKNameKey is the attribute Key conforming to the - // "telemetry.sdk.name" semantic conventions. It represents the name of the - // telemetry SDK as defined above. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "opentelemetry" - // Note: The OpenTelemetry SDK MUST set the `telemetry.sdk.name` attribute to - // `opentelemetry`. - // If another SDK, like a fork or a vendor-provided implementation, is used, - // this SDK MUST set the - // `telemetry.sdk.name` attribute to the fully-qualified class or module name of - // this SDK's main entry point - // or another suitable identifier depending on the language. - // The identifier `opentelemetry` is reserved and MUST NOT be used in this case. - // All custom identifiers SHOULD be stable across different versions of an - // implementation. - TelemetrySDKNameKey = attribute.Key("telemetry.sdk.name") - - // TelemetrySDKVersionKey is the attribute Key conforming to the - // "telemetry.sdk.version" semantic conventions. It represents the version - // string of the telemetry SDK. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "1.2.3" - TelemetrySDKVersionKey = attribute.Key("telemetry.sdk.version") -) - -// TelemetryDistroName returns an attribute KeyValue conforming to the -// "telemetry.distro.name" semantic conventions. It represents the name of the -// auto instrumentation agent or distribution, if used. -func TelemetryDistroName(val string) attribute.KeyValue { - return TelemetryDistroNameKey.String(val) -} - -// TelemetryDistroVersion returns an attribute KeyValue conforming to the -// "telemetry.distro.version" semantic conventions. It represents the version -// string of the auto instrumentation agent or distribution, if used. -func TelemetryDistroVersion(val string) attribute.KeyValue { - return TelemetryDistroVersionKey.String(val) -} - -// TelemetrySDKName returns an attribute KeyValue conforming to the -// "telemetry.sdk.name" semantic conventions. It represents the name of the -// telemetry SDK as defined above. -func TelemetrySDKName(val string) attribute.KeyValue { - return TelemetrySDKNameKey.String(val) -} - -// TelemetrySDKVersion returns an attribute KeyValue conforming to the -// "telemetry.sdk.version" semantic conventions. It represents the version string -// of the telemetry SDK. -func TelemetrySDKVersion(val string) attribute.KeyValue { - return TelemetrySDKVersionKey.String(val) -} - -// Enum values for telemetry.sdk.language -var ( - // cpp - // Stability: stable - TelemetrySDKLanguageCPP = TelemetrySDKLanguageKey.String("cpp") - // dotnet - // Stability: stable - TelemetrySDKLanguageDotnet = TelemetrySDKLanguageKey.String("dotnet") - // erlang - // Stability: stable - TelemetrySDKLanguageErlang = TelemetrySDKLanguageKey.String("erlang") - // go - // Stability: stable - TelemetrySDKLanguageGo = TelemetrySDKLanguageKey.String("go") - // java - // Stability: stable - TelemetrySDKLanguageJava = TelemetrySDKLanguageKey.String("java") - // nodejs - // Stability: stable - TelemetrySDKLanguageNodejs = TelemetrySDKLanguageKey.String("nodejs") - // php - // Stability: stable - TelemetrySDKLanguagePHP = TelemetrySDKLanguageKey.String("php") - // python - // Stability: stable - TelemetrySDKLanguagePython = TelemetrySDKLanguageKey.String("python") - // ruby - // Stability: stable - TelemetrySDKLanguageRuby = TelemetrySDKLanguageKey.String("ruby") - // rust - // Stability: stable - TelemetrySDKLanguageRust = TelemetrySDKLanguageKey.String("rust") - // swift - // Stability: stable - TelemetrySDKLanguageSwift = TelemetrySDKLanguageKey.String("swift") - // webjs - // Stability: stable - TelemetrySDKLanguageWebJS = TelemetrySDKLanguageKey.String("webjs") -) - -// Namespace: test -const ( - // TestCaseNameKey is the attribute Key conforming to the "test.case.name" - // semantic conventions. It represents the fully qualified human readable name - // of the [test case]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "org.example.TestCase1.test1", "example/tests/TestCase1.test1", - // "ExampleTestCase1_test1" - // - // [test case]: https://wikipedia.org/wiki/Test_case - TestCaseNameKey = attribute.Key("test.case.name") - - // TestCaseResultStatusKey is the attribute Key conforming to the - // "test.case.result.status" semantic conventions. It represents the status of - // the actual test case result from test execution. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "pass", "fail" - TestCaseResultStatusKey = attribute.Key("test.case.result.status") - - // TestSuiteNameKey is the attribute Key conforming to the "test.suite.name" - // semantic conventions. It represents the human readable name of a [test suite] - // . - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "TestSuite1" - // - // [test suite]: https://wikipedia.org/wiki/Test_suite - TestSuiteNameKey = attribute.Key("test.suite.name") - - // TestSuiteRunStatusKey is the attribute Key conforming to the - // "test.suite.run.status" semantic conventions. It represents the status of the - // test suite run. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "success", "failure", "skipped", "aborted", "timed_out", - // "in_progress" - TestSuiteRunStatusKey = attribute.Key("test.suite.run.status") -) - -// TestCaseName returns an attribute KeyValue conforming to the "test.case.name" -// semantic conventions. It represents the fully qualified human readable name of -// the [test case]. -// -// [test case]: https://wikipedia.org/wiki/Test_case -func TestCaseName(val string) attribute.KeyValue { - return TestCaseNameKey.String(val) -} - -// TestSuiteName returns an attribute KeyValue conforming to the -// "test.suite.name" semantic conventions. It represents the human readable name -// of a [test suite]. -// -// [test suite]: https://wikipedia.org/wiki/Test_suite -func TestSuiteName(val string) attribute.KeyValue { - return TestSuiteNameKey.String(val) -} - -// Enum values for test.case.result.status -var ( - // pass - // Stability: development - TestCaseResultStatusPass = TestCaseResultStatusKey.String("pass") - // fail - // Stability: development - TestCaseResultStatusFail = TestCaseResultStatusKey.String("fail") -) - -// Enum values for test.suite.run.status -var ( - // success - // Stability: development - TestSuiteRunStatusSuccess = TestSuiteRunStatusKey.String("success") - // failure - // Stability: development - TestSuiteRunStatusFailure = TestSuiteRunStatusKey.String("failure") - // skipped - // Stability: development - TestSuiteRunStatusSkipped = TestSuiteRunStatusKey.String("skipped") - // aborted - // Stability: development - TestSuiteRunStatusAborted = TestSuiteRunStatusKey.String("aborted") - // timed_out - // Stability: development - TestSuiteRunStatusTimedOut = TestSuiteRunStatusKey.String("timed_out") - // in_progress - // Stability: development - TestSuiteRunStatusInProgress = TestSuiteRunStatusKey.String("in_progress") -) - -// Namespace: thread -const ( - // ThreadIDKey is the attribute Key conforming to the "thread.id" semantic - // conventions. It represents the current "managed" thread ID (as opposed to OS - // thread ID). - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Note: - // Examples of where the value can be extracted from: - // - // | Language or platform | Source | - // | --- | --- | - // | JVM | `Thread.currentThread().threadId()` | - // | .NET | `Thread.CurrentThread.ManagedThreadId` | - // | Python | `threading.current_thread().ident` | - // | Ruby | `Thread.current.object_id` | - // | C++ | `std::this_thread::get_id()` | - // | Erlang | `erlang:self()` | - ThreadIDKey = attribute.Key("thread.id") - - // ThreadNameKey is the attribute Key conforming to the "thread.name" semantic - // conventions. It represents the current thread name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: main - // Note: - // Examples of where the value can be extracted from: - // - // | Language or platform | Source | - // | --- | --- | - // | JVM | `Thread.currentThread().getName()` | - // | .NET | `Thread.CurrentThread.Name` | - // | Python | `threading.current_thread().name` | - // | Ruby | `Thread.current.name` | - // | Erlang | `erlang:process_info(self(), registered_name)` | - ThreadNameKey = attribute.Key("thread.name") -) - -// ThreadID returns an attribute KeyValue conforming to the "thread.id" semantic -// conventions. It represents the current "managed" thread ID (as opposed to OS -// thread ID). -func ThreadID(val int) attribute.KeyValue { - return ThreadIDKey.Int(val) -} - -// ThreadName returns an attribute KeyValue conforming to the "thread.name" -// semantic conventions. It represents the current thread name. -func ThreadName(val string) attribute.KeyValue { - return ThreadNameKey.String(val) -} - -// Namespace: tls -const ( - // TLSCipherKey is the attribute Key conforming to the "tls.cipher" semantic - // conventions. It represents the string indicating the [cipher] used during the - // current connection. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", - // "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" - // Note: The values allowed for `tls.cipher` MUST be one of the `Descriptions` - // of the [registered TLS Cipher Suits]. - // - // [cipher]: https://datatracker.ietf.org/doc/html/rfc5246#appendix-A.5 - // [registered TLS Cipher Suits]: https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#table-tls-parameters-4 - TLSCipherKey = attribute.Key("tls.cipher") - - // TLSClientCertificateKey is the attribute Key conforming to the - // "tls.client.certificate" semantic conventions. It represents the PEM-encoded - // stand-alone certificate offered by the client. This is usually - // mutually-exclusive of `client.certificate_chain` since this value also exists - // in that list. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MII..." - TLSClientCertificateKey = attribute.Key("tls.client.certificate") - - // TLSClientCertificateChainKey is the attribute Key conforming to the - // "tls.client.certificate_chain" semantic conventions. It represents the array - // of PEM-encoded certificates that make up the certificate chain offered by the - // client. This is usually mutually-exclusive of `client.certificate` since that - // value should be the first certificate in the chain. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MII...", "MI..." - TLSClientCertificateChainKey = attribute.Key("tls.client.certificate_chain") - - // TLSClientHashMd5Key is the attribute Key conforming to the - // "tls.client.hash.md5" semantic conventions. It represents the certificate - // fingerprint using the MD5 digest of DER-encoded version of certificate - // offered by the client. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC" - TLSClientHashMd5Key = attribute.Key("tls.client.hash.md5") - - // TLSClientHashSha1Key is the attribute Key conforming to the - // "tls.client.hash.sha1" semantic conventions. It represents the certificate - // fingerprint using the SHA1 digest of DER-encoded version of certificate - // offered by the client. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9E393D93138888D288266C2D915214D1D1CCEB2A" - TLSClientHashSha1Key = attribute.Key("tls.client.hash.sha1") - - // TLSClientHashSha256Key is the attribute Key conforming to the - // "tls.client.hash.sha256" semantic conventions. It represents the certificate - // fingerprint using the SHA256 digest of DER-encoded version of certificate - // offered by the client. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0" - TLSClientHashSha256Key = attribute.Key("tls.client.hash.sha256") - - // TLSClientIssuerKey is the attribute Key conforming to the "tls.client.issuer" - // semantic conventions. It represents the distinguished name of [subject] of - // the issuer of the x.509 certificate presented by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CN=Example Root CA, OU=Infrastructure Team, DC=example, DC=com" - // - // [subject]: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 - TLSClientIssuerKey = attribute.Key("tls.client.issuer") - - // TLSClientJa3Key is the attribute Key conforming to the "tls.client.ja3" - // semantic conventions. It represents a hash that identifies clients based on - // how they perform an SSL/TLS handshake. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "d4e5b18d6b55c71272893221c96ba240" - TLSClientJa3Key = attribute.Key("tls.client.ja3") - - // TLSClientNotAfterKey is the attribute Key conforming to the - // "tls.client.not_after" semantic conventions. It represents the date/Time - // indicating when client certificate is no longer considered valid. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T00:00:00.000Z" - TLSClientNotAfterKey = attribute.Key("tls.client.not_after") - - // TLSClientNotBeforeKey is the attribute Key conforming to the - // "tls.client.not_before" semantic conventions. It represents the date/Time - // indicating when client certificate is first considered valid. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1970-01-01T00:00:00.000Z" - TLSClientNotBeforeKey = attribute.Key("tls.client.not_before") - - // TLSClientSubjectKey is the attribute Key conforming to the - // "tls.client.subject" semantic conventions. It represents the distinguished - // name of subject of the x.509 certificate presented by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CN=myclient, OU=Documentation Team, DC=example, DC=com" - TLSClientSubjectKey = attribute.Key("tls.client.subject") - - // TLSClientSupportedCiphersKey is the attribute Key conforming to the - // "tls.client.supported_ciphers" semantic conventions. It represents the array - // of ciphers offered by the client during the client hello. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - // "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" - TLSClientSupportedCiphersKey = attribute.Key("tls.client.supported_ciphers") - - // TLSCurveKey is the attribute Key conforming to the "tls.curve" semantic - // conventions. It represents the string indicating the curve used for the given - // cipher, when applicable. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "secp256r1" - TLSCurveKey = attribute.Key("tls.curve") - - // TLSEstablishedKey is the attribute Key conforming to the "tls.established" - // semantic conventions. It represents the boolean flag indicating if the TLS - // negotiation was successful and transitioned to an encrypted tunnel. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: true - TLSEstablishedKey = attribute.Key("tls.established") - - // TLSNextProtocolKey is the attribute Key conforming to the "tls.next_protocol" - // semantic conventions. It represents the string indicating the protocol being - // tunneled. Per the values in the [IANA registry], this string should be lower - // case. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "http/1.1" - // - // [IANA registry]: https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids - TLSNextProtocolKey = attribute.Key("tls.next_protocol") - - // TLSProtocolNameKey is the attribute Key conforming to the "tls.protocol.name" - // semantic conventions. It represents the normalized lowercase protocol name - // parsed from original string of the negotiated [SSL/TLS protocol version]. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [SSL/TLS protocol version]: https://docs.openssl.org/1.1.1/man3/SSL_get_version/#return-values - TLSProtocolNameKey = attribute.Key("tls.protocol.name") - - // TLSProtocolVersionKey is the attribute Key conforming to the - // "tls.protocol.version" semantic conventions. It represents the numeric part - // of the version parsed from the original string of the negotiated - // [SSL/TLS protocol version]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1.2", "3" - // - // [SSL/TLS protocol version]: https://docs.openssl.org/1.1.1/man3/SSL_get_version/#return-values - TLSProtocolVersionKey = attribute.Key("tls.protocol.version") - - // TLSResumedKey is the attribute Key conforming to the "tls.resumed" semantic - // conventions. It represents the boolean flag indicating if this TLS connection - // was resumed from an existing TLS negotiation. - // - // Type: boolean - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: true - TLSResumedKey = attribute.Key("tls.resumed") - - // TLSServerCertificateKey is the attribute Key conforming to the - // "tls.server.certificate" semantic conventions. It represents the PEM-encoded - // stand-alone certificate offered by the server. This is usually - // mutually-exclusive of `server.certificate_chain` since this value also exists - // in that list. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MII..." - TLSServerCertificateKey = attribute.Key("tls.server.certificate") - - // TLSServerCertificateChainKey is the attribute Key conforming to the - // "tls.server.certificate_chain" semantic conventions. It represents the array - // of PEM-encoded certificates that make up the certificate chain offered by the - // server. This is usually mutually-exclusive of `server.certificate` since that - // value should be the first certificate in the chain. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "MII...", "MI..." - TLSServerCertificateChainKey = attribute.Key("tls.server.certificate_chain") - - // TLSServerHashMd5Key is the attribute Key conforming to the - // "tls.server.hash.md5" semantic conventions. It represents the certificate - // fingerprint using the MD5 digest of DER-encoded version of certificate - // offered by the server. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC" - TLSServerHashMd5Key = attribute.Key("tls.server.hash.md5") - - // TLSServerHashSha1Key is the attribute Key conforming to the - // "tls.server.hash.sha1" semantic conventions. It represents the certificate - // fingerprint using the SHA1 digest of DER-encoded version of certificate - // offered by the server. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9E393D93138888D288266C2D915214D1D1CCEB2A" - TLSServerHashSha1Key = attribute.Key("tls.server.hash.sha1") - - // TLSServerHashSha256Key is the attribute Key conforming to the - // "tls.server.hash.sha256" semantic conventions. It represents the certificate - // fingerprint using the SHA256 digest of DER-encoded version of certificate - // offered by the server. For consistency with other hash values, this value - // should be formatted as an uppercase hash. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0" - TLSServerHashSha256Key = attribute.Key("tls.server.hash.sha256") - - // TLSServerIssuerKey is the attribute Key conforming to the "tls.server.issuer" - // semantic conventions. It represents the distinguished name of [subject] of - // the issuer of the x.509 certificate presented by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CN=Example Root CA, OU=Infrastructure Team, DC=example, DC=com" - // - // [subject]: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 - TLSServerIssuerKey = attribute.Key("tls.server.issuer") - - // TLSServerJa3sKey is the attribute Key conforming to the "tls.server.ja3s" - // semantic conventions. It represents a hash that identifies servers based on - // how they perform an SSL/TLS handshake. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "d4e5b18d6b55c71272893221c96ba240" - TLSServerJa3sKey = attribute.Key("tls.server.ja3s") - - // TLSServerNotAfterKey is the attribute Key conforming to the - // "tls.server.not_after" semantic conventions. It represents the date/Time - // indicating when server certificate is no longer considered valid. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2021-01-01T00:00:00.000Z" - TLSServerNotAfterKey = attribute.Key("tls.server.not_after") - - // TLSServerNotBeforeKey is the attribute Key conforming to the - // "tls.server.not_before" semantic conventions. It represents the date/Time - // indicating when server certificate is first considered valid. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "1970-01-01T00:00:00.000Z" - TLSServerNotBeforeKey = attribute.Key("tls.server.not_before") - - // TLSServerSubjectKey is the attribute Key conforming to the - // "tls.server.subject" semantic conventions. It represents the distinguished - // name of subject of the x.509 certificate presented by the server. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "CN=myserver, OU=Documentation Team, DC=example, DC=com" - TLSServerSubjectKey = attribute.Key("tls.server.subject") -) - -// TLSCipher returns an attribute KeyValue conforming to the "tls.cipher" -// semantic conventions. It represents the string indicating the [cipher] used -// during the current connection. -// -// [cipher]: https://datatracker.ietf.org/doc/html/rfc5246#appendix-A.5 -func TLSCipher(val string) attribute.KeyValue { - return TLSCipherKey.String(val) -} - -// TLSClientCertificate returns an attribute KeyValue conforming to the -// "tls.client.certificate" semantic conventions. It represents the PEM-encoded -// stand-alone certificate offered by the client. This is usually -// mutually-exclusive of `client.certificate_chain` since this value also exists -// in that list. -func TLSClientCertificate(val string) attribute.KeyValue { - return TLSClientCertificateKey.String(val) -} - -// TLSClientCertificateChain returns an attribute KeyValue conforming to the -// "tls.client.certificate_chain" semantic conventions. It represents the array -// of PEM-encoded certificates that make up the certificate chain offered by the -// client. This is usually mutually-exclusive of `client.certificate` since that -// value should be the first certificate in the chain. -func TLSClientCertificateChain(val ...string) attribute.KeyValue { - return TLSClientCertificateChainKey.StringSlice(val) -} - -// TLSClientHashMd5 returns an attribute KeyValue conforming to the -// "tls.client.hash.md5" semantic conventions. It represents the certificate -// fingerprint using the MD5 digest of DER-encoded version of certificate offered -// by the client. For consistency with other hash values, this value should be -// formatted as an uppercase hash. -func TLSClientHashMd5(val string) attribute.KeyValue { - return TLSClientHashMd5Key.String(val) -} - -// TLSClientHashSha1 returns an attribute KeyValue conforming to the -// "tls.client.hash.sha1" semantic conventions. It represents the certificate -// fingerprint using the SHA1 digest of DER-encoded version of certificate -// offered by the client. For consistency with other hash values, this value -// should be formatted as an uppercase hash. -func TLSClientHashSha1(val string) attribute.KeyValue { - return TLSClientHashSha1Key.String(val) -} - -// TLSClientHashSha256 returns an attribute KeyValue conforming to the -// "tls.client.hash.sha256" semantic conventions. It represents the certificate -// fingerprint using the SHA256 digest of DER-encoded version of certificate -// offered by the client. For consistency with other hash values, this value -// should be formatted as an uppercase hash. -func TLSClientHashSha256(val string) attribute.KeyValue { - return TLSClientHashSha256Key.String(val) -} - -// TLSClientIssuer returns an attribute KeyValue conforming to the -// "tls.client.issuer" semantic conventions. It represents the distinguished name -// of [subject] of the issuer of the x.509 certificate presented by the client. -// -// [subject]: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 -func TLSClientIssuer(val string) attribute.KeyValue { - return TLSClientIssuerKey.String(val) -} - -// TLSClientJa3 returns an attribute KeyValue conforming to the "tls.client.ja3" -// semantic conventions. It represents a hash that identifies clients based on -// how they perform an SSL/TLS handshake. -func TLSClientJa3(val string) attribute.KeyValue { - return TLSClientJa3Key.String(val) -} - -// TLSClientNotAfter returns an attribute KeyValue conforming to the -// "tls.client.not_after" semantic conventions. It represents the date/Time -// indicating when client certificate is no longer considered valid. -func TLSClientNotAfter(val string) attribute.KeyValue { - return TLSClientNotAfterKey.String(val) -} - -// TLSClientNotBefore returns an attribute KeyValue conforming to the -// "tls.client.not_before" semantic conventions. It represents the date/Time -// indicating when client certificate is first considered valid. -func TLSClientNotBefore(val string) attribute.KeyValue { - return TLSClientNotBeforeKey.String(val) -} - -// TLSClientSubject returns an attribute KeyValue conforming to the -// "tls.client.subject" semantic conventions. It represents the distinguished -// name of subject of the x.509 certificate presented by the client. -func TLSClientSubject(val string) attribute.KeyValue { - return TLSClientSubjectKey.String(val) -} - -// TLSClientSupportedCiphers returns an attribute KeyValue conforming to the -// "tls.client.supported_ciphers" semantic conventions. It represents the array -// of ciphers offered by the client during the client hello. -func TLSClientSupportedCiphers(val ...string) attribute.KeyValue { - return TLSClientSupportedCiphersKey.StringSlice(val) -} - -// TLSCurve returns an attribute KeyValue conforming to the "tls.curve" semantic -// conventions. It represents the string indicating the curve used for the given -// cipher, when applicable. -func TLSCurve(val string) attribute.KeyValue { - return TLSCurveKey.String(val) -} - -// TLSEstablished returns an attribute KeyValue conforming to the -// "tls.established" semantic conventions. It represents the boolean flag -// indicating if the TLS negotiation was successful and transitioned to an -// encrypted tunnel. -func TLSEstablished(val bool) attribute.KeyValue { - return TLSEstablishedKey.Bool(val) -} - -// TLSNextProtocol returns an attribute KeyValue conforming to the -// "tls.next_protocol" semantic conventions. It represents the string indicating -// the protocol being tunneled. Per the values in the [IANA registry], this -// string should be lower case. -// -// [IANA registry]: https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids -func TLSNextProtocol(val string) attribute.KeyValue { - return TLSNextProtocolKey.String(val) -} - -// TLSProtocolVersion returns an attribute KeyValue conforming to the -// "tls.protocol.version" semantic conventions. It represents the numeric part of -// the version parsed from the original string of the negotiated -// [SSL/TLS protocol version]. -// -// [SSL/TLS protocol version]: https://docs.openssl.org/1.1.1/man3/SSL_get_version/#return-values -func TLSProtocolVersion(val string) attribute.KeyValue { - return TLSProtocolVersionKey.String(val) -} - -// TLSResumed returns an attribute KeyValue conforming to the "tls.resumed" -// semantic conventions. It represents the boolean flag indicating if this TLS -// connection was resumed from an existing TLS negotiation. -func TLSResumed(val bool) attribute.KeyValue { - return TLSResumedKey.Bool(val) -} - -// TLSServerCertificate returns an attribute KeyValue conforming to the -// "tls.server.certificate" semantic conventions. It represents the PEM-encoded -// stand-alone certificate offered by the server. This is usually -// mutually-exclusive of `server.certificate_chain` since this value also exists -// in that list. -func TLSServerCertificate(val string) attribute.KeyValue { - return TLSServerCertificateKey.String(val) -} - -// TLSServerCertificateChain returns an attribute KeyValue conforming to the -// "tls.server.certificate_chain" semantic conventions. It represents the array -// of PEM-encoded certificates that make up the certificate chain offered by the -// server. This is usually mutually-exclusive of `server.certificate` since that -// value should be the first certificate in the chain. -func TLSServerCertificateChain(val ...string) attribute.KeyValue { - return TLSServerCertificateChainKey.StringSlice(val) -} - -// TLSServerHashMd5 returns an attribute KeyValue conforming to the -// "tls.server.hash.md5" semantic conventions. It represents the certificate -// fingerprint using the MD5 digest of DER-encoded version of certificate offered -// by the server. For consistency with other hash values, this value should be -// formatted as an uppercase hash. -func TLSServerHashMd5(val string) attribute.KeyValue { - return TLSServerHashMd5Key.String(val) -} - -// TLSServerHashSha1 returns an attribute KeyValue conforming to the -// "tls.server.hash.sha1" semantic conventions. It represents the certificate -// fingerprint using the SHA1 digest of DER-encoded version of certificate -// offered by the server. For consistency with other hash values, this value -// should be formatted as an uppercase hash. -func TLSServerHashSha1(val string) attribute.KeyValue { - return TLSServerHashSha1Key.String(val) -} - -// TLSServerHashSha256 returns an attribute KeyValue conforming to the -// "tls.server.hash.sha256" semantic conventions. It represents the certificate -// fingerprint using the SHA256 digest of DER-encoded version of certificate -// offered by the server. For consistency with other hash values, this value -// should be formatted as an uppercase hash. -func TLSServerHashSha256(val string) attribute.KeyValue { - return TLSServerHashSha256Key.String(val) -} - -// TLSServerIssuer returns an attribute KeyValue conforming to the -// "tls.server.issuer" semantic conventions. It represents the distinguished name -// of [subject] of the issuer of the x.509 certificate presented by the client. -// -// [subject]: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 -func TLSServerIssuer(val string) attribute.KeyValue { - return TLSServerIssuerKey.String(val) -} - -// TLSServerJa3s returns an attribute KeyValue conforming to the -// "tls.server.ja3s" semantic conventions. It represents a hash that identifies -// servers based on how they perform an SSL/TLS handshake. -func TLSServerJa3s(val string) attribute.KeyValue { - return TLSServerJa3sKey.String(val) -} - -// TLSServerNotAfter returns an attribute KeyValue conforming to the -// "tls.server.not_after" semantic conventions. It represents the date/Time -// indicating when server certificate is no longer considered valid. -func TLSServerNotAfter(val string) attribute.KeyValue { - return TLSServerNotAfterKey.String(val) -} - -// TLSServerNotBefore returns an attribute KeyValue conforming to the -// "tls.server.not_before" semantic conventions. It represents the date/Time -// indicating when server certificate is first considered valid. -func TLSServerNotBefore(val string) attribute.KeyValue { - return TLSServerNotBeforeKey.String(val) -} - -// TLSServerSubject returns an attribute KeyValue conforming to the -// "tls.server.subject" semantic conventions. It represents the distinguished -// name of subject of the x.509 certificate presented by the server. -func TLSServerSubject(val string) attribute.KeyValue { - return TLSServerSubjectKey.String(val) -} - -// Enum values for tls.protocol.name -var ( - // ssl - // Stability: development - TLSProtocolNameSsl = TLSProtocolNameKey.String("ssl") - // tls - // Stability: development - TLSProtocolNameTLS = TLSProtocolNameKey.String("tls") -) - -// Namespace: url -const ( - // URLDomainKey is the attribute Key conforming to the "url.domain" semantic - // conventions. It represents the domain extracted from the `url.full`, such as - // "opentelemetry.io". - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "www.foo.bar", "opentelemetry.io", "3.12.167.2", - // "[1080:0:0:0:8:800:200C:417A]" - // Note: In some cases a URL may refer to an IP and/or port directly, without a - // domain name. In this case, the IP address would go to the domain field. If - // the URL contains a [literal IPv6 address] enclosed by `[` and `]`, the `[` - // and `]` characters should also be captured in the domain field. - // - // [literal IPv6 address]: https://www.rfc-editor.org/rfc/rfc2732#section-2 - URLDomainKey = attribute.Key("url.domain") - - // URLExtensionKey is the attribute Key conforming to the "url.extension" - // semantic conventions. It represents the file extension extracted from the - // `url.full`, excluding the leading dot. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "png", "gz" - // Note: The file extension is only set if it exists, as not every url has a - // file extension. When the file name has multiple extensions `example.tar.gz`, - // only the last one should be captured `gz`, not `tar.gz`. - URLExtensionKey = attribute.Key("url.extension") - - // URLFragmentKey is the attribute Key conforming to the "url.fragment" semantic - // conventions. It represents the [URI fragment] component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "SemConv" - // - // [URI fragment]: https://www.rfc-editor.org/rfc/rfc3986#section-3.5 - URLFragmentKey = attribute.Key("url.fragment") - - // URLFullKey is the attribute Key conforming to the "url.full" semantic - // conventions. It represents the absolute URL describing a network resource - // according to [RFC3986]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "https://www.foo.bar/search?q=OpenTelemetry#SemConv", "//localhost" - // Note: For network calls, URL usually has - // `scheme://host[:port][path][?query][#fragment]` format, where the fragment - // is not transmitted over HTTP, but if it is known, it SHOULD be included - // nevertheless. - // - // `url.full` MUST NOT contain credentials passed via URL in form of - // `https://username:password@www.example.com/`. - // In such case username and password SHOULD be redacted and attribute's value - // SHOULD be `https://REDACTED:REDACTED@www.example.com/`. - // - // `url.full` SHOULD capture the absolute URL when it is available (or can be - // reconstructed). - // - // Sensitive content provided in `url.full` SHOULD be scrubbed when - // instrumentations can identify it. - // - // - // Query string values for the following keys SHOULD be redacted by default and - // replaced by the - // value `REDACTED`: - // - // - [`AWSAccessKeyId`] - // - [`Signature`] - // - [`sig`] - // - [`X-Goog-Signature`] - // - // This list is subject to change over time. - // - // When a query string value is redacted, the query string key SHOULD still be - // preserved, e.g. - // `https://www.example.com/path?color=blue&sig=REDACTED`. - // - // [RFC3986]: https://www.rfc-editor.org/rfc/rfc3986 - // [`AWSAccessKeyId`]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationQueryStringAuth - // [`Signature`]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationQueryStringAuth - // [`sig`]: https://learn.microsoft.com/azure/storage/common/storage-sas-overview#sas-token - // [`X-Goog-Signature`]: https://cloud.google.com/storage/docs/access-control/signed-urls - URLFullKey = attribute.Key("url.full") - - // URLOriginalKey is the attribute Key conforming to the "url.original" semantic - // conventions. It represents the unmodified original URL as seen in the event - // source. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "https://www.foo.bar/search?q=OpenTelemetry#SemConv", - // "search?q=OpenTelemetry" - // Note: In network monitoring, the observed URL may be a full URL, whereas in - // access logs, the URL is often just represented as a path. This field is meant - // to represent the URL as it was observed, complete or not. - // `url.original` might contain credentials passed via URL in form of - // `https://username:password@www.example.com/`. In such case password and - // username SHOULD NOT be redacted and attribute's value SHOULD remain the same. - URLOriginalKey = attribute.Key("url.original") - - // URLPathKey is the attribute Key conforming to the "url.path" semantic - // conventions. It represents the [URI path] component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "/search" - // Note: Sensitive content provided in `url.path` SHOULD be scrubbed when - // instrumentations can identify it. - // - // [URI path]: https://www.rfc-editor.org/rfc/rfc3986#section-3.3 - URLPathKey = attribute.Key("url.path") - - // URLPortKey is the attribute Key conforming to the "url.port" semantic - // conventions. It represents the port extracted from the `url.full`. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 443 - URLPortKey = attribute.Key("url.port") - - // URLQueryKey is the attribute Key conforming to the "url.query" semantic - // conventions. It represents the [URI query] component. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "q=OpenTelemetry" - // Note: Sensitive content provided in `url.query` SHOULD be scrubbed when - // instrumentations can identify it. - // - // - // Query string values for the following keys SHOULD be redacted by default and - // replaced by the value `REDACTED`: - // - // - [`AWSAccessKeyId`] - // - [`Signature`] - // - [`sig`] - // - [`X-Goog-Signature`] - // - // This list is subject to change over time. - // - // When a query string value is redacted, the query string key SHOULD still be - // preserved, e.g. - // `q=OpenTelemetry&sig=REDACTED`. - // - // [URI query]: https://www.rfc-editor.org/rfc/rfc3986#section-3.4 - // [`AWSAccessKeyId`]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationQueryStringAuth - // [`Signature`]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationQueryStringAuth - // [`sig`]: https://learn.microsoft.com/azure/storage/common/storage-sas-overview#sas-token - // [`X-Goog-Signature`]: https://cloud.google.com/storage/docs/access-control/signed-urls - URLQueryKey = attribute.Key("url.query") - - // URLRegisteredDomainKey is the attribute Key conforming to the - // "url.registered_domain" semantic conventions. It represents the highest - // registered url domain, stripped of the subdomain. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "example.com", "foo.co.uk" - // Note: This value can be determined precisely with the [public suffix list]. - // For example, the registered domain for `foo.example.com` is `example.com`. - // Trying to approximate this by simply taking the last two labels will not work - // well for TLDs such as `co.uk`. - // - // [public suffix list]: https://publicsuffix.org/ - URLRegisteredDomainKey = attribute.Key("url.registered_domain") - - // URLSchemeKey is the attribute Key conforming to the "url.scheme" semantic - // conventions. It represents the [URI scheme] component identifying the used - // protocol. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "https", "ftp", "telnet" - // - // [URI scheme]: https://www.rfc-editor.org/rfc/rfc3986#section-3.1 - URLSchemeKey = attribute.Key("url.scheme") - - // URLSubdomainKey is the attribute Key conforming to the "url.subdomain" - // semantic conventions. It represents the subdomain portion of a fully - // qualified domain name includes all of the names except the host name under - // the registered_domain. In a partially qualified domain, or if the - // qualification level of the full name cannot be determined, subdomain contains - // all of the names below the registered domain. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "east", "sub2.sub1" - // Note: The subdomain portion of `www.east.mydomain.co.uk` is `east`. If the - // domain has multiple levels of subdomain, such as `sub2.sub1.example.com`, the - // subdomain field should contain `sub2.sub1`, with no trailing period. - URLSubdomainKey = attribute.Key("url.subdomain") - - // URLTemplateKey is the attribute Key conforming to the "url.template" semantic - // conventions. It represents the low-cardinality template of an - // [absolute path reference]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "/users/{id}", "/users/:id", "/users?id={id}" - // - // [absolute path reference]: https://www.rfc-editor.org/rfc/rfc3986#section-4.2 - URLTemplateKey = attribute.Key("url.template") - - // URLTopLevelDomainKey is the attribute Key conforming to the - // "url.top_level_domain" semantic conventions. It represents the effective top - // level domain (eTLD), also known as the domain suffix, is the last part of the - // domain name. For example, the top level domain for example.com is `com`. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "com", "co.uk" - // Note: This value can be determined precisely with the [public suffix list]. - // - // [public suffix list]: https://publicsuffix.org/ - URLTopLevelDomainKey = attribute.Key("url.top_level_domain") -) - -// URLDomain returns an attribute KeyValue conforming to the "url.domain" -// semantic conventions. It represents the domain extracted from the `url.full`, -// such as "opentelemetry.io". -func URLDomain(val string) attribute.KeyValue { - return URLDomainKey.String(val) -} - -// URLExtension returns an attribute KeyValue conforming to the "url.extension" -// semantic conventions. It represents the file extension extracted from the -// `url.full`, excluding the leading dot. -func URLExtension(val string) attribute.KeyValue { - return URLExtensionKey.String(val) -} - -// URLFragment returns an attribute KeyValue conforming to the "url.fragment" -// semantic conventions. It represents the [URI fragment] component. -// -// [URI fragment]: https://www.rfc-editor.org/rfc/rfc3986#section-3.5 -func URLFragment(val string) attribute.KeyValue { - return URLFragmentKey.String(val) -} - -// URLFull returns an attribute KeyValue conforming to the "url.full" semantic -// conventions. It represents the absolute URL describing a network resource -// according to [RFC3986]. -// -// [RFC3986]: https://www.rfc-editor.org/rfc/rfc3986 -func URLFull(val string) attribute.KeyValue { - return URLFullKey.String(val) -} - -// URLOriginal returns an attribute KeyValue conforming to the "url.original" -// semantic conventions. It represents the unmodified original URL as seen in the -// event source. -func URLOriginal(val string) attribute.KeyValue { - return URLOriginalKey.String(val) -} - -// URLPath returns an attribute KeyValue conforming to the "url.path" semantic -// conventions. It represents the [URI path] component. -// -// [URI path]: https://www.rfc-editor.org/rfc/rfc3986#section-3.3 -func URLPath(val string) attribute.KeyValue { - return URLPathKey.String(val) -} - -// URLPort returns an attribute KeyValue conforming to the "url.port" semantic -// conventions. It represents the port extracted from the `url.full`. -func URLPort(val int) attribute.KeyValue { - return URLPortKey.Int(val) -} - -// URLQuery returns an attribute KeyValue conforming to the "url.query" semantic -// conventions. It represents the [URI query] component. -// -// [URI query]: https://www.rfc-editor.org/rfc/rfc3986#section-3.4 -func URLQuery(val string) attribute.KeyValue { - return URLQueryKey.String(val) -} - -// URLRegisteredDomain returns an attribute KeyValue conforming to the -// "url.registered_domain" semantic conventions. It represents the highest -// registered url domain, stripped of the subdomain. -func URLRegisteredDomain(val string) attribute.KeyValue { - return URLRegisteredDomainKey.String(val) -} - -// URLScheme returns an attribute KeyValue conforming to the "url.scheme" -// semantic conventions. It represents the [URI scheme] component identifying the -// used protocol. -// -// [URI scheme]: https://www.rfc-editor.org/rfc/rfc3986#section-3.1 -func URLScheme(val string) attribute.KeyValue { - return URLSchemeKey.String(val) -} - -// URLSubdomain returns an attribute KeyValue conforming to the "url.subdomain" -// semantic conventions. It represents the subdomain portion of a fully qualified -// domain name includes all of the names except the host name under the -// registered_domain. In a partially qualified domain, or if the qualification -// level of the full name cannot be determined, subdomain contains all of the -// names below the registered domain. -func URLSubdomain(val string) attribute.KeyValue { - return URLSubdomainKey.String(val) -} - -// URLTemplate returns an attribute KeyValue conforming to the "url.template" -// semantic conventions. It represents the low-cardinality template of an -// [absolute path reference]. -// -// [absolute path reference]: https://www.rfc-editor.org/rfc/rfc3986#section-4.2 -func URLTemplate(val string) attribute.KeyValue { - return URLTemplateKey.String(val) -} - -// URLTopLevelDomain returns an attribute KeyValue conforming to the -// "url.top_level_domain" semantic conventions. It represents the effective top -// level domain (eTLD), also known as the domain suffix, is the last part of the -// domain name. For example, the top level domain for example.com is `com`. -func URLTopLevelDomain(val string) attribute.KeyValue { - return URLTopLevelDomainKey.String(val) -} - -// Namespace: user -const ( - // UserEmailKey is the attribute Key conforming to the "user.email" semantic - // conventions. It represents the user email address. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "a.einstein@example.com" - UserEmailKey = attribute.Key("user.email") - - // UserFullNameKey is the attribute Key conforming to the "user.full_name" - // semantic conventions. It represents the user's full name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Albert Einstein" - UserFullNameKey = attribute.Key("user.full_name") - - // UserHashKey is the attribute Key conforming to the "user.hash" semantic - // conventions. It represents the unique user hash to correlate information for - // a user in anonymized form. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "364fc68eaf4c8acec74a4e52d7d1feaa" - // Note: Useful if `user.id` or `user.name` contain confidential information and - // cannot be used. - UserHashKey = attribute.Key("user.hash") - - // UserIDKey is the attribute Key conforming to the "user.id" semantic - // conventions. It represents the unique identifier of the user. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "S-1-5-21-202424912787-2692429404-2351956786-1000" - UserIDKey = attribute.Key("user.id") - - // UserNameKey is the attribute Key conforming to the "user.name" semantic - // conventions. It represents the short name or login/username of the user. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "a.einstein" - UserNameKey = attribute.Key("user.name") - - // UserRolesKey is the attribute Key conforming to the "user.roles" semantic - // conventions. It represents the array of user roles at the time of the event. - // - // Type: string[] - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "admin", "reporting_user" - UserRolesKey = attribute.Key("user.roles") -) - -// UserEmail returns an attribute KeyValue conforming to the "user.email" -// semantic conventions. It represents the user email address. -func UserEmail(val string) attribute.KeyValue { - return UserEmailKey.String(val) -} - -// UserFullName returns an attribute KeyValue conforming to the "user.full_name" -// semantic conventions. It represents the user's full name. -func UserFullName(val string) attribute.KeyValue { - return UserFullNameKey.String(val) -} - -// UserHash returns an attribute KeyValue conforming to the "user.hash" semantic -// conventions. It represents the unique user hash to correlate information for a -// user in anonymized form. -func UserHash(val string) attribute.KeyValue { - return UserHashKey.String(val) -} - -// UserID returns an attribute KeyValue conforming to the "user.id" semantic -// conventions. It represents the unique identifier of the user. -func UserID(val string) attribute.KeyValue { - return UserIDKey.String(val) -} - -// UserName returns an attribute KeyValue conforming to the "user.name" semantic -// conventions. It represents the short name or login/username of the user. -func UserName(val string) attribute.KeyValue { - return UserNameKey.String(val) -} - -// UserRoles returns an attribute KeyValue conforming to the "user.roles" -// semantic conventions. It represents the array of user roles at the time of the -// event. -func UserRoles(val ...string) attribute.KeyValue { - return UserRolesKey.StringSlice(val) -} - -// Namespace: user_agent -const ( - // UserAgentNameKey is the attribute Key conforming to the "user_agent.name" - // semantic conventions. It represents the name of the user-agent extracted from - // original. Usually refers to the browser's name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Safari", "YourApp" - // Note: [Example] of extracting browser's name from original string. In the - // case of using a user-agent for non-browser products, such as microservices - // with multiple names/versions inside the `user_agent.original`, the most - // significant name SHOULD be selected. In such a scenario it should align with - // `user_agent.version` - // - // [Example]: https://uaparser.dev/#demo - UserAgentNameKey = attribute.Key("user_agent.name") - - // UserAgentOriginalKey is the attribute Key conforming to the - // "user_agent.original" semantic conventions. It represents the value of the - // [HTTP User-Agent] header sent by the client. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Stable - // - // Examples: "CERN-LineMode/2.15 libwww/2.17b3", "Mozilla/5.0 (iPhone; CPU - // iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) - // Version/14.1.2 Mobile/15E148 Safari/604.1", "YourApp/1.0.0 - // grpc-java-okhttp/1.27.2" - // - // [HTTP User-Agent]: https://www.rfc-editor.org/rfc/rfc9110.html#field.user-agent - UserAgentOriginalKey = attribute.Key("user_agent.original") - - // UserAgentOSNameKey is the attribute Key conforming to the - // "user_agent.os.name" semantic conventions. It represents the human readable - // operating system name. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "iOS", "Android", "Ubuntu" - // Note: For mapping user agent strings to OS names, libraries such as - // [ua-parser] can be utilized. - // - // [ua-parser]: https://github.com/ua-parser - UserAgentOSNameKey = attribute.Key("user_agent.os.name") - - // UserAgentOSVersionKey is the attribute Key conforming to the - // "user_agent.os.version" semantic conventions. It represents the version - // string of the operating system as defined in [Version Attributes]. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "14.2.1", "18.04.1" - // Note: For mapping user agent strings to OS versions, libraries such as - // [ua-parser] can be utilized. - // - // [Version Attributes]: /docs/resource/README.md#version-attributes - // [ua-parser]: https://github.com/ua-parser - UserAgentOSVersionKey = attribute.Key("user_agent.os.version") - - // UserAgentSyntheticTypeKey is the attribute Key conforming to the - // "user_agent.synthetic.type" semantic conventions. It represents the specifies - // the category of synthetic traffic, such as tests or bots. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // Note: This attribute MAY be derived from the contents of the - // `user_agent.original` attribute. Components that populate the attribute are - // responsible for determining what they consider to be synthetic bot or test - // traffic. This attribute can either be set for self-identification purposes, - // or on telemetry detected to be generated as a result of a synthetic request. - // This attribute is useful for distinguishing between genuine client traffic - // and synthetic traffic generated by bots or tests. - UserAgentSyntheticTypeKey = attribute.Key("user_agent.synthetic.type") - - // UserAgentVersionKey is the attribute Key conforming to the - // "user_agent.version" semantic conventions. It represents the version of the - // user-agent extracted from original. Usually refers to the browser's version. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "14.1.2", "1.0.0" - // Note: [Example] of extracting browser's version from original string. In the - // case of using a user-agent for non-browser products, such as microservices - // with multiple names/versions inside the `user_agent.original`, the most - // significant version SHOULD be selected. In such a scenario it should align - // with `user_agent.name` - // - // [Example]: https://uaparser.dev/#demo - UserAgentVersionKey = attribute.Key("user_agent.version") -) - -// UserAgentName returns an attribute KeyValue conforming to the -// "user_agent.name" semantic conventions. It represents the name of the -// user-agent extracted from original. Usually refers to the browser's name. -func UserAgentName(val string) attribute.KeyValue { - return UserAgentNameKey.String(val) -} - -// UserAgentOriginal returns an attribute KeyValue conforming to the -// "user_agent.original" semantic conventions. It represents the value of the -// [HTTP User-Agent] header sent by the client. -// -// [HTTP User-Agent]: https://www.rfc-editor.org/rfc/rfc9110.html#field.user-agent -func UserAgentOriginal(val string) attribute.KeyValue { - return UserAgentOriginalKey.String(val) -} - -// UserAgentOSName returns an attribute KeyValue conforming to the -// "user_agent.os.name" semantic conventions. It represents the human readable -// operating system name. -func UserAgentOSName(val string) attribute.KeyValue { - return UserAgentOSNameKey.String(val) -} - -// UserAgentOSVersion returns an attribute KeyValue conforming to the -// "user_agent.os.version" semantic conventions. It represents the version string -// of the operating system as defined in [Version Attributes]. -// -// [Version Attributes]: /docs/resource/README.md#version-attributes -func UserAgentOSVersion(val string) attribute.KeyValue { - return UserAgentOSVersionKey.String(val) -} - -// UserAgentVersion returns an attribute KeyValue conforming to the -// "user_agent.version" semantic conventions. It represents the version of the -// user-agent extracted from original. Usually refers to the browser's version. -func UserAgentVersion(val string) attribute.KeyValue { - return UserAgentVersionKey.String(val) -} - -// Enum values for user_agent.synthetic.type -var ( - // Bot source. - // Stability: development - UserAgentSyntheticTypeBot = UserAgentSyntheticTypeKey.String("bot") - // Synthetic test source. - // Stability: development - UserAgentSyntheticTypeTest = UserAgentSyntheticTypeKey.String("test") -) - -// Namespace: vcs -const ( - // VCSChangeIDKey is the attribute Key conforming to the "vcs.change.id" - // semantic conventions. It represents the ID of the change (pull request/merge - // request/changelist) if applicable. This is usually a unique (within - // repository) identifier generated by the VCS system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "123" - VCSChangeIDKey = attribute.Key("vcs.change.id") - - // VCSChangeStateKey is the attribute Key conforming to the "vcs.change.state" - // semantic conventions. It represents the state of the change (pull - // request/merge request/changelist). - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "open", "closed", "merged" - VCSChangeStateKey = attribute.Key("vcs.change.state") - - // VCSChangeTitleKey is the attribute Key conforming to the "vcs.change.title" - // semantic conventions. It represents the human readable title of the change - // (pull request/merge request/changelist). This title is often a brief summary - // of the change and may get merged in to a ref as the commit summary. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Fixes broken thing", "feat: add my new feature", "[chore] update - // dependency" - VCSChangeTitleKey = attribute.Key("vcs.change.title") - - // VCSLineChangeTypeKey is the attribute Key conforming to the - // "vcs.line_change.type" semantic conventions. It represents the type of line - // change being measured on a branch or change. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "added", "removed" - VCSLineChangeTypeKey = attribute.Key("vcs.line_change.type") - - // VCSOwnerNameKey is the attribute Key conforming to the "vcs.owner.name" - // semantic conventions. It represents the group owner within the version - // control system. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-org", "myteam", "business-unit" - VCSOwnerNameKey = attribute.Key("vcs.owner.name") - - // VCSProviderNameKey is the attribute Key conforming to the "vcs.provider.name" - // semantic conventions. It represents the name of the version control system - // provider. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "github", "gitlab", "gitea", "bitbucket" - VCSProviderNameKey = attribute.Key("vcs.provider.name") - - // VCSRefBaseNameKey is the attribute Key conforming to the "vcs.ref.base.name" - // semantic conventions. It represents the name of the [reference] such as - // **branch** or **tag** in the repository. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-feature-branch", "tag-1-test" - // Note: `base` refers to the starting point of a change. For example, `main` - // would be the base reference of type branch if you've created a new - // reference of type branch from it and created new commits. - // - // [reference]: https://git-scm.com/docs/gitglossary#def_ref - VCSRefBaseNameKey = attribute.Key("vcs.ref.base.name") - - // VCSRefBaseRevisionKey is the attribute Key conforming to the - // "vcs.ref.base.revision" semantic conventions. It represents the revision, - // literally [revised version], The revision most often refers to a commit - // object in Git, or a revision number in SVN. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9d59409acf479dfa0df1aa568182e43e43df8bbe28d60fcf2bc52e30068802cc", - // "main", "123", "HEAD" - // Note: `base` refers to the starting point of a change. For example, `main` - // would be the base reference of type branch if you've created a new - // reference of type branch from it and created new commits. The - // revision can be a full [hash value (see - // glossary)], - // of the recorded change to a ref within a repository pointing to a - // commit [commit] object. It does - // not necessarily have to be a hash; it can simply define a [revision - // number] - // which is an integer that is monotonically increasing. In cases where - // it is identical to the `ref.base.name`, it SHOULD still be included. - // It is up to the implementer to decide which value to set as the - // revision based on the VCS system and situational context. - // - // [revised version]: https://www.merriam-webster.com/dictionary/revision - // [hash value (see - // glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf - // [commit]: https://git-scm.com/docs/git-commit - // [revision - // number]: https://svnbook.red-bean.com/en/1.7/svn.tour.revs.specifiers.html - VCSRefBaseRevisionKey = attribute.Key("vcs.ref.base.revision") - - // VCSRefBaseTypeKey is the attribute Key conforming to the "vcs.ref.base.type" - // semantic conventions. It represents the type of the [reference] in the - // repository. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "branch", "tag" - // Note: `base` refers to the starting point of a change. For example, `main` - // would be the base reference of type branch if you've created a new - // reference of type branch from it and created new commits. - // - // [reference]: https://git-scm.com/docs/gitglossary#def_ref - VCSRefBaseTypeKey = attribute.Key("vcs.ref.base.type") - - // VCSRefHeadNameKey is the attribute Key conforming to the "vcs.ref.head.name" - // semantic conventions. It represents the name of the [reference] such as - // **branch** or **tag** in the repository. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "my-feature-branch", "tag-1-test" - // Note: `head` refers to where you are right now; the current reference at a - // given time. - // - // [reference]: https://git-scm.com/docs/gitglossary#def_ref - VCSRefHeadNameKey = attribute.Key("vcs.ref.head.name") - - // VCSRefHeadRevisionKey is the attribute Key conforming to the - // "vcs.ref.head.revision" semantic conventions. It represents the revision, - // literally [revised version], The revision most often refers to a commit - // object in Git, or a revision number in SVN. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "9d59409acf479dfa0df1aa568182e43e43df8bbe28d60fcf2bc52e30068802cc", - // "main", "123", "HEAD" - // Note: `head` refers to where you are right now; the current reference at a - // given time.The revision can be a full [hash value (see - // glossary)], - // of the recorded change to a ref within a repository pointing to a - // commit [commit] object. It does - // not necessarily have to be a hash; it can simply define a [revision - // number] - // which is an integer that is monotonically increasing. In cases where - // it is identical to the `ref.head.name`, it SHOULD still be included. - // It is up to the implementer to decide which value to set as the - // revision based on the VCS system and situational context. - // - // [revised version]: https://www.merriam-webster.com/dictionary/revision - // [hash value (see - // glossary)]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf - // [commit]: https://git-scm.com/docs/git-commit - // [revision - // number]: https://svnbook.red-bean.com/en/1.7/svn.tour.revs.specifiers.html - VCSRefHeadRevisionKey = attribute.Key("vcs.ref.head.revision") - - // VCSRefHeadTypeKey is the attribute Key conforming to the "vcs.ref.head.type" - // semantic conventions. It represents the type of the [reference] in the - // repository. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "branch", "tag" - // Note: `head` refers to where you are right now; the current reference at a - // given time. - // - // [reference]: https://git-scm.com/docs/gitglossary#def_ref - VCSRefHeadTypeKey = attribute.Key("vcs.ref.head.type") - - // VCSRefTypeKey is the attribute Key conforming to the "vcs.ref.type" semantic - // conventions. It represents the type of the [reference] in the repository. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "branch", "tag" - // - // [reference]: https://git-scm.com/docs/gitglossary#def_ref - VCSRefTypeKey = attribute.Key("vcs.ref.type") - - // VCSRepositoryNameKey is the attribute Key conforming to the - // "vcs.repository.name" semantic conventions. It represents the human readable - // name of the repository. It SHOULD NOT include any additional identifier like - // Group/SubGroup in GitLab or organization in GitHub. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "semantic-conventions", "my-cool-repo" - // Note: Due to it only being the name, it can clash with forks of the same - // repository if collecting telemetry across multiple orgs or groups in - // the same backends. - VCSRepositoryNameKey = attribute.Key("vcs.repository.name") - - // VCSRepositoryURLFullKey is the attribute Key conforming to the - // "vcs.repository.url.full" semantic conventions. It represents the - // [canonical URL] of the repository providing the complete HTTP(S) address in - // order to locate and identify the repository through a browser. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // "https://github.com/opentelemetry/open-telemetry-collector-contrib", - // "https://gitlab.com/my-org/my-project/my-projects-project/repo" - // Note: In Git Version Control Systems, the canonical URL SHOULD NOT include - // the `.git` extension. - // - // [canonical URL]: https://support.google.com/webmasters/answer/10347851 - VCSRepositoryURLFullKey = attribute.Key("vcs.repository.url.full") - - // VCSRevisionDeltaDirectionKey is the attribute Key conforming to the - // "vcs.revision_delta.direction" semantic conventions. It represents the type - // of revision comparison. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "ahead", "behind" - VCSRevisionDeltaDirectionKey = attribute.Key("vcs.revision_delta.direction") -) - -// VCSChangeID returns an attribute KeyValue conforming to the "vcs.change.id" -// semantic conventions. It represents the ID of the change (pull request/merge -// request/changelist) if applicable. This is usually a unique (within -// repository) identifier generated by the VCS system. -func VCSChangeID(val string) attribute.KeyValue { - return VCSChangeIDKey.String(val) -} - -// VCSChangeTitle returns an attribute KeyValue conforming to the -// "vcs.change.title" semantic conventions. It represents the human readable -// title of the change (pull request/merge request/changelist). This title is -// often a brief summary of the change and may get merged in to a ref as the -// commit summary. -func VCSChangeTitle(val string) attribute.KeyValue { - return VCSChangeTitleKey.String(val) -} - -// VCSOwnerName returns an attribute KeyValue conforming to the "vcs.owner.name" -// semantic conventions. It represents the group owner within the version control -// system. -func VCSOwnerName(val string) attribute.KeyValue { - return VCSOwnerNameKey.String(val) -} - -// VCSRefBaseName returns an attribute KeyValue conforming to the -// "vcs.ref.base.name" semantic conventions. It represents the name of the -// [reference] such as **branch** or **tag** in the repository. -// -// [reference]: https://git-scm.com/docs/gitglossary#def_ref -func VCSRefBaseName(val string) attribute.KeyValue { - return VCSRefBaseNameKey.String(val) -} - -// VCSRefBaseRevision returns an attribute KeyValue conforming to the -// "vcs.ref.base.revision" semantic conventions. It represents the revision, -// literally [revised version], The revision most often refers to a commit object -// in Git, or a revision number in SVN. -// -// [revised version]: https://www.merriam-webster.com/dictionary/revision -func VCSRefBaseRevision(val string) attribute.KeyValue { - return VCSRefBaseRevisionKey.String(val) -} - -// VCSRefHeadName returns an attribute KeyValue conforming to the -// "vcs.ref.head.name" semantic conventions. It represents the name of the -// [reference] such as **branch** or **tag** in the repository. -// -// [reference]: https://git-scm.com/docs/gitglossary#def_ref -func VCSRefHeadName(val string) attribute.KeyValue { - return VCSRefHeadNameKey.String(val) -} - -// VCSRefHeadRevision returns an attribute KeyValue conforming to the -// "vcs.ref.head.revision" semantic conventions. It represents the revision, -// literally [revised version], The revision most often refers to a commit object -// in Git, or a revision number in SVN. -// -// [revised version]: https://www.merriam-webster.com/dictionary/revision -func VCSRefHeadRevision(val string) attribute.KeyValue { - return VCSRefHeadRevisionKey.String(val) -} - -// VCSRepositoryName returns an attribute KeyValue conforming to the -// "vcs.repository.name" semantic conventions. It represents the human readable -// name of the repository. It SHOULD NOT include any additional identifier like -// Group/SubGroup in GitLab or organization in GitHub. -func VCSRepositoryName(val string) attribute.KeyValue { - return VCSRepositoryNameKey.String(val) -} - -// VCSRepositoryURLFull returns an attribute KeyValue conforming to the -// "vcs.repository.url.full" semantic conventions. It represents the -// [canonical URL] of the repository providing the complete HTTP(S) address in -// order to locate and identify the repository through a browser. -// -// [canonical URL]: https://support.google.com/webmasters/answer/10347851 -func VCSRepositoryURLFull(val string) attribute.KeyValue { - return VCSRepositoryURLFullKey.String(val) -} - -// Enum values for vcs.change.state -var ( - // Open means the change is currently active and under review. It hasn't been - // merged into the target branch yet, and it's still possible to make changes or - // add comments. - // Stability: development - VCSChangeStateOpen = VCSChangeStateKey.String("open") - // WIP (work-in-progress, draft) means the change is still in progress and not - // yet ready for a full review. It might still undergo significant changes. - // Stability: development - VCSChangeStateWip = VCSChangeStateKey.String("wip") - // Closed means the merge request has been closed without merging. This can - // happen for various reasons, such as the changes being deemed unnecessary, the - // issue being resolved in another way, or the author deciding to withdraw the - // request. - // Stability: development - VCSChangeStateClosed = VCSChangeStateKey.String("closed") - // Merged indicates that the change has been successfully integrated into the - // target codebase. - // Stability: development - VCSChangeStateMerged = VCSChangeStateKey.String("merged") -) - -// Enum values for vcs.line_change.type -var ( - // How many lines were added. - // Stability: development - VCSLineChangeTypeAdded = VCSLineChangeTypeKey.String("added") - // How many lines were removed. - // Stability: development - VCSLineChangeTypeRemoved = VCSLineChangeTypeKey.String("removed") -) - -// Enum values for vcs.provider.name -var ( - // [GitHub] - // Stability: development - // - // [GitHub]: https://github.com - VCSProviderNameGithub = VCSProviderNameKey.String("github") - // [GitLab] - // Stability: development - // - // [GitLab]: https://gitlab.com - VCSProviderNameGitlab = VCSProviderNameKey.String("gitlab") - // [Gitea] - // Stability: development - // - // [Gitea]: https://gitea.io - VCSProviderNameGitea = VCSProviderNameKey.String("gitea") - // [Bitbucket] - // Stability: development - // - // [Bitbucket]: https://bitbucket.org - VCSProviderNameBitbucket = VCSProviderNameKey.String("bitbucket") -) - -// Enum values for vcs.ref.base.type -var ( - // [branch] - // Stability: development - // - // [branch]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefbranchabranch - VCSRefBaseTypeBranch = VCSRefBaseTypeKey.String("branch") - // [tag] - // Stability: development - // - // [tag]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftagatag - VCSRefBaseTypeTag = VCSRefBaseTypeKey.String("tag") -) - -// Enum values for vcs.ref.head.type -var ( - // [branch] - // Stability: development - // - // [branch]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefbranchabranch - VCSRefHeadTypeBranch = VCSRefHeadTypeKey.String("branch") - // [tag] - // Stability: development - // - // [tag]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftagatag - VCSRefHeadTypeTag = VCSRefHeadTypeKey.String("tag") -) - -// Enum values for vcs.ref.type -var ( - // [branch] - // Stability: development - // - // [branch]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefbranchabranch - VCSRefTypeBranch = VCSRefTypeKey.String("branch") - // [tag] - // Stability: development - // - // [tag]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftagatag - VCSRefTypeTag = VCSRefTypeKey.String("tag") -) - -// Enum values for vcs.revision_delta.direction -var ( - // How many revisions the change is behind the target ref. - // Stability: development - VCSRevisionDeltaDirectionBehind = VCSRevisionDeltaDirectionKey.String("behind") - // How many revisions the change is ahead of the target ref. - // Stability: development - VCSRevisionDeltaDirectionAhead = VCSRevisionDeltaDirectionKey.String("ahead") -) - -// Namespace: webengine -const ( - // WebEngineDescriptionKey is the attribute Key conforming to the - // "webengine.description" semantic conventions. It represents the additional - // description of the web engine (e.g. detailed version and edition - // information). - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "WildFly Full 21.0.0.Final (WildFly Core 13.0.1.Final) - - // 2.2.2.Final" - WebEngineDescriptionKey = attribute.Key("webengine.description") - - // WebEngineNameKey is the attribute Key conforming to the "webengine.name" - // semantic conventions. It represents the name of the web engine. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "WildFly" - WebEngineNameKey = attribute.Key("webengine.name") - - // WebEngineVersionKey is the attribute Key conforming to the - // "webengine.version" semantic conventions. It represents the version of the - // web engine. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "21.0.0" - WebEngineVersionKey = attribute.Key("webengine.version") -) - -// WebEngineDescription returns an attribute KeyValue conforming to the -// "webengine.description" semantic conventions. It represents the additional -// description of the web engine (e.g. detailed version and edition information). -func WebEngineDescription(val string) attribute.KeyValue { - return WebEngineDescriptionKey.String(val) -} - -// WebEngineName returns an attribute KeyValue conforming to the "webengine.name" -// semantic conventions. It represents the name of the web engine. -func WebEngineName(val string) attribute.KeyValue { - return WebEngineNameKey.String(val) -} - -// WebEngineVersion returns an attribute KeyValue conforming to the -// "webengine.version" semantic conventions. It represents the version of the web -// engine. -func WebEngineVersion(val string) attribute.KeyValue { - return WebEngineVersionKey.String(val) -} - -// Namespace: zos -const ( - // ZOSSmfIDKey is the attribute Key conforming to the "zos.smf.id" semantic - // conventions. It represents the System Management Facility (SMF) Identifier - // uniquely identified a z/OS system within a SYSPLEX or mainframe environment - // and is used for system and performance analysis. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "SYS1" - ZOSSmfIDKey = attribute.Key("zos.smf.id") - - // ZOSSysplexNameKey is the attribute Key conforming to the "zos.sysplex.name" - // semantic conventions. It represents the name of the SYSPLEX to which the z/OS - // system belongs too. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "SYSPLEX1" - ZOSSysplexNameKey = attribute.Key("zos.sysplex.name") -) - -// ZOSSmfID returns an attribute KeyValue conforming to the "zos.smf.id" semantic -// conventions. It represents the System Management Facility (SMF) Identifier -// uniquely identified a z/OS system within a SYSPLEX or mainframe environment -// and is used for system and performance analysis. -func ZOSSmfID(val string) attribute.KeyValue { - return ZOSSmfIDKey.String(val) -} - -// ZOSSysplexName returns an attribute KeyValue conforming to the -// "zos.sysplex.name" semantic conventions. It represents the name of the SYSPLEX -// to which the z/OS system belongs too. -func ZOSSysplexName(val string) attribute.KeyValue { - return ZOSSysplexNameKey.String(val) -} diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go deleted file mode 100644 index 852362ef770..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -// Package semconv implements OpenTelemetry semantic conventions. -// -// OpenTelemetry semantic conventions are agreed standardized naming -// patterns for OpenTelemetry things. This package represents the v1.39.0 -// version of the OpenTelemetry semantic conventions. -package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go deleted file mode 100644 index 84cf636a727..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" - -import ( - "reflect" - - "go.opentelemetry.io/otel/attribute" -) - -// ErrorType returns an [attribute.KeyValue] identifying the error type of err. -// -// If err is nil, the returned attribute has the default value -// [ErrorTypeOther]. -// -// If err's type has the method -// -// ErrorType() string -// -// then the returned attribute has the value of err.ErrorType(). Otherwise, the -// returned attribute has a value derived from the concrete type of err. -// -// The key of the returned attribute is [ErrorTypeKey]. -func ErrorType(err error) attribute.KeyValue { - if err == nil { - return ErrorTypeOther - } - - return ErrorTypeKey.String(errorType(err)) -} - -func errorType(err error) string { - var s string - if et, ok := err.(interface{ ErrorType() string }); ok { - // Prioritize the ErrorType method if available. - s = et.ErrorType() - } - if s == "" { - // Fallback to reflection if the ErrorType method is not supported or - // returns an empty value. - - t := reflect.TypeOf(err) - pkg, name := t.PkgPath(), t.Name() - if pkg != "" && name != "" { - s = pkg + "." + name - } else { - // The type has no package path or name (predeclared, not-defined, - // or alias for a not-defined type). - // - // This is not guaranteed to be unique, but is a best effort. - s = t.String() - } - } - return s -} diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go deleted file mode 100644 index 7b688ecc33d..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" - -const ( - // ExceptionEventName is the name of the Span event representing an exception. - ExceptionEventName = "exception" -) diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go b/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go deleted file mode 100644 index e1a199d89bf..00000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" - -// SchemaURL is the schema URL that matches the version of the semantic conventions -// that this package defines. Semconv packages starting from v1.4.0 must declare -// non-empty schema URL in the form https://opentelemetry.io/schemas/ -const SchemaURL = "https://opentelemetry.io/schemas/1.39.0" diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/httpconv/metric.go b/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/httpconv/metric.go similarity index 97% rename from vendor/go.opentelemetry.io/otel/semconv/v1.39.0/httpconv/metric.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.40.0/httpconv/metric.go index fa67d197172..013629dc3b6 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/httpconv/metric.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/httpconv/metric.go @@ -157,6 +157,9 @@ func (m ClientActiveRequests) Add( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr, metric.WithAttributes( attribute.String("server.address", serverAddress), @@ -175,7 +178,7 @@ func (m ClientActiveRequests) Add( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), )..., @@ -187,6 +190,9 @@ func (m ClientActiveRequests) Add( // AddSet adds incr to the existing count for set. func (m ClientActiveRequests) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -300,6 +306,9 @@ func (m ClientConnectionDuration) Record( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("server.address", serverAddress), @@ -318,7 +327,7 @@ func (m ClientConnectionDuration) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), )..., @@ -330,6 +339,9 @@ func (m ClientConnectionDuration) Record( // RecordSet records val to the current distribution for set. func (m ClientConnectionDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -446,6 +458,9 @@ func (m ClientOpenConnections) Add( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr, metric.WithAttributes( attribute.String("http.connection.state", string(connectionState)), @@ -465,7 +480,7 @@ func (m ClientOpenConnections) Add( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.connection.state", string(connectionState)), attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), @@ -478,6 +493,9 @@ func (m ClientOpenConnections) Add( // AddSet adds incr to the existing count for set. func (m ClientOpenConnections) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -599,6 +617,9 @@ func (m ClientRequestBodySize) Record( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -618,7 +639,7 @@ func (m ClientRequestBodySize) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), @@ -638,6 +659,9 @@ func (m ClientRequestBodySize) Record( // // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length func (m ClientRequestBodySize) RecordSet(ctx context.Context, val int64, set attribute.Set) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Histogram.Record(ctx, val) return @@ -779,6 +803,9 @@ func (m ClientRequestDuration) Record( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -798,7 +825,7 @@ func (m ClientRequestDuration) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), @@ -811,6 +838,9 @@ func (m ClientRequestDuration) Record( // RecordSet records val to the current distribution for set. func (m ClientRequestDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -959,6 +989,9 @@ func (m ClientResponseBodySize) Record( serverPort int, attrs ...attribute.KeyValue, ) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -978,7 +1011,7 @@ func (m ClientResponseBodySize) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("server.address", serverAddress), attribute.Int("server.port", serverPort), @@ -998,6 +1031,9 @@ func (m ClientResponseBodySize) Record( // // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length func (m ClientResponseBodySize) RecordSet(ctx context.Context, val int64, set attribute.Set) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Histogram.Record(ctx, val) return @@ -1137,6 +1173,9 @@ func (m ServerActiveRequests) Add( urlScheme string, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -1155,7 +1194,7 @@ func (m ServerActiveRequests) Add( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("url.scheme", urlScheme), )..., @@ -1167,6 +1206,9 @@ func (m ServerActiveRequests) Add( // AddSet adds incr to the existing count for set. func (m ServerActiveRequests) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -1276,6 +1318,9 @@ func (m ServerRequestBodySize) Record( urlScheme string, attrs ...attribute.KeyValue, ) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -1294,7 +1339,7 @@ func (m ServerRequestBodySize) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("url.scheme", urlScheme), )..., @@ -1313,6 +1358,9 @@ func (m ServerRequestBodySize) Record( // // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length func (m ServerRequestBodySize) RecordSet(ctx context.Context, val int64, set attribute.Set) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Histogram.Record(ctx, val) return @@ -1463,6 +1511,9 @@ func (m ServerRequestDuration) Record( urlScheme string, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -1481,7 +1532,7 @@ func (m ServerRequestDuration) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("url.scheme", urlScheme), )..., @@ -1493,6 +1544,9 @@ func (m ServerRequestDuration) Record( // RecordSet records val to the current distribution for set. func (m ServerRequestDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -1649,6 +1703,9 @@ func (m ServerResponseBodySize) Record( urlScheme string, attrs ...attribute.KeyValue, ) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Histogram.Record(ctx, val, metric.WithAttributes( attribute.String("http.request.method", string(requestMethod)), @@ -1667,7 +1724,7 @@ func (m ServerResponseBodySize) Record( *o, metric.WithAttributes( append( - attrs, + attrs[:len(attrs):len(attrs)], attribute.String("http.request.method", string(requestMethod)), attribute.String("url.scheme", urlScheme), )..., @@ -1686,6 +1743,9 @@ func (m ServerResponseBodySize) Record( // // [Content-Length]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length func (m ServerResponseBodySize) RecordSet(ctx context.Context, val int64, set attribute.Set) { + if !m.Int64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Histogram.Record(ctx, val) return diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/otelconv/metric.go b/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go similarity index 98% rename from vendor/go.opentelemetry.io/otel/semconv/v1.39.0/otelconv/metric.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go index 2ec60d9cbf7..ba9d29e9d1c 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/otelconv/metric.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go @@ -195,6 +195,9 @@ func (m SDKExporterLogExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -226,6 +229,9 @@ func (m SDKExporterLogExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterLogExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -345,6 +351,9 @@ func (m SDKExporterLogInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -371,6 +380,9 @@ func (m SDKExporterLogInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterLogInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -489,6 +501,9 @@ func (m SDKExporterMetricDataPointExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -520,6 +535,9 @@ func (m SDKExporterMetricDataPointExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterMetricDataPointExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -641,6 +659,9 @@ func (m SDKExporterMetricDataPointInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -667,6 +688,9 @@ func (m SDKExporterMetricDataPointInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterMetricDataPointInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -786,6 +810,9 @@ func (m SDKExporterOperationDuration) Record( val float64, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val) return @@ -819,6 +846,9 @@ func (m SDKExporterOperationDuration) Record( // [http]: https://github.com/open-telemetry/opentelemetry-proto/blob/v1.5.0/docs/specification.md#full-success-1 // [grpc]: https://github.com/open-telemetry/opentelemetry-proto/blob/v1.5.0/docs/specification.md#full-success func (m SDKExporterOperationDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -957,6 +987,9 @@ func (m SDKExporterSpanExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -988,6 +1021,9 @@ func (m SDKExporterSpanExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterSpanExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1107,6 +1143,9 @@ func (m SDKExporterSpanInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -1133,6 +1172,9 @@ func (m SDKExporterSpanInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterSpanInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -1235,6 +1277,9 @@ func (SDKLogCreated) Description() string { // Add adds incr to the existing count for attrs. func (m SDKLogCreated) Add(ctx context.Context, incr int64, attrs ...attribute.KeyValue) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1252,6 +1297,9 @@ func (m SDKLogCreated) Add(ctx context.Context, incr int64, attrs ...attribute.K // AddSet adds incr to the existing count for set. func (m SDKLogCreated) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1341,6 +1389,9 @@ func (m SDKMetricReaderCollectionDuration) Record( val float64, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val) return @@ -1370,6 +1421,9 @@ func (m SDKMetricReaderCollectionDuration) Record( // while others fail. In that case `error.type` SHOULD be set to any of the // failure causes. func (m SDKMetricReaderCollectionDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -1479,6 +1533,9 @@ func (m SDKProcessorLogProcessed) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1508,6 +1565,9 @@ func (m SDKProcessorLogProcessed) Add( // considered to be processed already when it has been submitted to the exporter, // not when the corresponding export call has finished. func (m SDKProcessorLogProcessed) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1766,6 +1826,9 @@ func (m SDKProcessorSpanProcessed) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1795,6 +1858,9 @@ func (m SDKProcessorSpanProcessed) Add( // processed already when it has been submitted to the exporter, not when the // corresponding export call has finished. func (m SDKProcessorSpanProcessed) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -2047,6 +2113,9 @@ func (m SDKSpanLive) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -2070,6 +2139,9 @@ func (m SDKSpanLive) Add( // AddSet adds incr to the existing count for set. func (m SDKSpanLive) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -2161,6 +2233,9 @@ func (m SDKSpanStarted) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -2187,6 +2262,9 @@ func (m SDKSpanStarted) Add( // Implementations MUST record this metric for all spans, even for non-recording // ones. func (m SDKSpanStarted) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return diff --git a/vendor/modules.txt b/vendor/modules.txt index 5818a5e8e13..577da8a672d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -240,7 +240,7 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url github.com/aws/aws-sdk-go-v2/service/signin github.com/aws/aws-sdk-go-v2/service/signin/internal/endpoints github.com/aws/aws-sdk-go-v2/service/signin/types -# github.com/aws/aws-sdk-go-v2/service/sns v1.39.15 +# github.com/aws/aws-sdk-go-v2/service/sns v1.39.17 ## explicit; go 1.24 github.com/aws/aws-sdk-go-v2/service/sns github.com/aws/aws-sdk-go-v2/service/sns/internal/endpoints @@ -327,13 +327,13 @@ github.com/cncf/xds/go/xds/data/orca/v3 github.com/cncf/xds/go/xds/service/orca/v3 github.com/cncf/xds/go/xds/type/matcher/v3 github.com/cncf/xds/go/xds/type/v3 -# github.com/coder/quartz v0.3.0 +# github.com/coder/quartz v0.3.1 ## explicit; go 1.23.9 github.com/coder/quartz # github.com/coreos/go-semver v0.3.0 ## explicit github.com/coreos/go-semver/semver -# github.com/coreos/go-systemd/v22 v22.6.0 +# github.com/coreos/go-systemd/v22 v22.7.0 ## explicit; go 1.23 github.com/coreos/go-systemd/v22/activation github.com/coreos/go-systemd/v22/journal @@ -431,8 +431,8 @@ github.com/felixge/fgprof # github.com/felixge/httpsnoop v1.0.4 ## explicit; go 1.13 github.com/felixge/httpsnoop -# github.com/fsnotify/fsnotify v1.9.0 -## explicit; go 1.17 +# github.com/fsnotify/fsnotify v1.10.0 +## explicit; go 1.23 github.com/fsnotify/fsnotify github.com/fsnotify/fsnotify/internal # github.com/go-chi/chi/v5 v5.2.4 @@ -479,8 +479,8 @@ github.com/go-openapi/jsonreference/internal # github.com/go-openapi/loads v0.23.3 ## explicit; go 1.24.0 github.com/go-openapi/loads -# github.com/go-openapi/runtime v0.29.3 -## explicit; go 1.24.0 +# github.com/go-openapi/runtime v0.29.4 +## explicit; go 1.25.0 github.com/go-openapi/runtime github.com/go-openapi/runtime/client github.com/go-openapi/runtime/flagext @@ -498,20 +498,20 @@ github.com/go-openapi/spec ## explicit; go 1.25.0 github.com/go-openapi/strfmt github.com/go-openapi/strfmt/internal/bsonlite -# github.com/go-openapi/swag v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag -# github.com/go-openapi/swag/cmdutils v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/cmdutils v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/cmdutils # github.com/go-openapi/swag/conv v0.26.1 ## explicit; go 1.25.0 github.com/go-openapi/swag/conv -# github.com/go-openapi/swag/fileutils v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/fileutils v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/fileutils -# github.com/go-openapi/swag/jsonname v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/jsonname v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/jsonname # github.com/go-openapi/swag/jsonutils v0.26.1 ## explicit; go 1.25.0 @@ -519,23 +519,23 @@ github.com/go-openapi/swag/jsonutils github.com/go-openapi/swag/jsonutils/adapters github.com/go-openapi/swag/jsonutils/adapters/ifaces github.com/go-openapi/swag/jsonutils/adapters/stdlib/json -# github.com/go-openapi/swag/loading v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/loading v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/loading -# github.com/go-openapi/swag/mangling v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/mangling v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/mangling -# github.com/go-openapi/swag/netutils v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/netutils v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/netutils -# github.com/go-openapi/swag/stringutils v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/stringutils v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/stringutils # github.com/go-openapi/swag/typeutils v0.26.1 ## explicit; go 1.25.0 github.com/go-openapi/swag/typeutils -# github.com/go-openapi/swag/yamlutils v0.25.5 -## explicit; go 1.24.0 +# github.com/go-openapi/swag/yamlutils v0.26.0 +## explicit; go 1.25.0 github.com/go-openapi/swag/yamlutils # github.com/go-openapi/validate v0.25.2 ## explicit; go 1.24.0 @@ -944,8 +944,8 @@ github.com/parquet-go/parquet-go/variant # github.com/philhofer/fwd v1.2.0 ## explicit; go 1.20 github.com/philhofer/fwd -# github.com/pierrec/lz4/v4 v4.1.22 -## explicit; go 1.14 +# github.com/pierrec/lz4/v4 v4.1.26 +## explicit; go 1.17 github.com/pierrec/lz4/v4 github.com/pierrec/lz4/v4/internal/lz4block github.com/pierrec/lz4/v4/internal/lz4errors @@ -980,7 +980,7 @@ github.com/prometheus-community/parquet-common/util # github.com/prometheus-community/prom-label-proxy v0.11.1 ## explicit; go 1.23.0 github.com/prometheus-community/prom-label-proxy/injectproxy -# github.com/prometheus/alertmanager v0.32.1 +# github.com/prometheus/alertmanager v0.33.0 ## explicit; go 1.25.0 github.com/prometheus/alertmanager/alert github.com/prometheus/alertmanager/api @@ -1005,9 +1005,13 @@ github.com/prometheus/alertmanager/cluster/clusterpb github.com/prometheus/alertmanager/config github.com/prometheus/alertmanager/config/common github.com/prometheus/alertmanager/dispatch +github.com/prometheus/alertmanager/eventrecorder +github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb github.com/prometheus/alertmanager/featurecontrol github.com/prometheus/alertmanager/inhibit +github.com/prometheus/alertmanager/kafka github.com/prometheus/alertmanager/limit +github.com/prometheus/alertmanager/marker github.com/prometheus/alertmanager/matcher/compat github.com/prometheus/alertmanager/matcher/parse github.com/prometheus/alertmanager/nflog @@ -1074,8 +1078,8 @@ github.com/prometheus/common/model github.com/prometheus/common/promslog github.com/prometheus/common/route github.com/prometheus/common/version -# github.com/prometheus/exporter-toolkit v0.15.1 -## explicit; go 1.24.0 +# github.com/prometheus/exporter-toolkit v0.16.0 +## explicit; go 1.25.0 github.com/prometheus/exporter-toolkit/web # github.com/prometheus/otlptranslator v1.0.0 => github.com/prometheus/otlptranslator v1.0.0 ## explicit; go 1.23.0 @@ -1317,6 +1321,22 @@ github.com/tinylib/msgp/msgp/setof # github.com/tjhop/slog-gokit v0.2.0 ## explicit; go 1.21 github.com/tjhop/slog-gokit +# github.com/twmb/franz-go v1.21.2 +## explicit; go 1.25.0 +github.com/twmb/franz-go/pkg/kbin +github.com/twmb/franz-go/pkg/kerr +github.com/twmb/franz-go/pkg/kgo +github.com/twmb/franz-go/pkg/kgo/internal/sticky +github.com/twmb/franz-go/pkg/kgo/internal/xsync +github.com/twmb/franz-go/pkg/kversion +github.com/twmb/franz-go/pkg/sasl +# github.com/twmb/franz-go/pkg/kmsg v1.13.1 +## explicit; go 1.24.0 +github.com/twmb/franz-go/pkg/kmsg +github.com/twmb/franz-go/pkg/kmsg/internal/kbin +# github.com/twmb/franz-go/plugin/kslog v1.0.0 +## explicit; go 1.21.0 +github.com/twmb/franz-go/plugin/kslog # github.com/twpayne/go-geom v1.6.1 ## explicit; go 1.22 github.com/twpayne/go-geom @@ -1476,12 +1496,12 @@ go.opentelemetry.io/contrib/detectors/gcp ## explicit; go 1.23.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal -# go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.66.0 -## explicit; go 1.24.0 +# go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 +## explicit; go 1.25.0 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace/internal/semconv -# go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0 -## explicit; go 1.24.0 +# go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 +## explicit; go 1.25.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv @@ -1517,10 +1537,9 @@ go.opentelemetry.io/otel/semconv/v1.21.0 go.opentelemetry.io/otel/semconv/v1.24.0 go.opentelemetry.io/otel/semconv/v1.30.0 go.opentelemetry.io/otel/semconv/v1.37.0 -go.opentelemetry.io/otel/semconv/v1.39.0 -go.opentelemetry.io/otel/semconv/v1.39.0/httpconv -go.opentelemetry.io/otel/semconv/v1.39.0/otelconv go.opentelemetry.io/otel/semconv/v1.40.0 +go.opentelemetry.io/otel/semconv/v1.40.0/httpconv +go.opentelemetry.io/otel/semconv/v1.40.0/otelconv go.opentelemetry.io/otel/semconv/v1.41.0 go.opentelemetry.io/otel/semconv/v1.41.0/otelconv # go.opentelemetry.io/otel/bridge/opentracing v1.44.0 @@ -1541,8 +1560,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/observ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/otlpconfig go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/retry go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/x -# go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 -## explicit; go 1.24.0 +# go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 +## explicit; go 1.25.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/counter