Add bundle bitmap for per-field telemetry

Taskfile.yml

@@ -513,6 +513,9 @@ tasks:
 
   test-acc:
     desc: Run acceptance tests
+    # Refresh the embedded bitmap schema before running so bundle/bitmap tests
+    # assert against an up-to-date schema.
+    deps: ['generate-bitmap-schema']
     # Sources mirror `build` (acceptance_test.go builds the CLI in-process via BuildCLI)
     # plus acceptance/**. For test-acc the checked-in out.* files are golden inputs:
     # changing them must re-run the test. test-update* excludes out.* because they
@@ -552,6 +555,8 @@ tasks:
 
   test-update:
     desc: Update acceptance test output (local)
+    # Refresh the embedded bitmap schema before regenerating golden output.
+    deps: ['generate-bitmap-schema']
     # Excludes out* because the task rewrites them; keeping them in sources would
     # invalidate the checksum and force a re-run on every invocation.
     # Note: output.txt / output.*.txt are outputs here (Phase 0 regenerates them
@@ -806,6 +811,7 @@ tasks:
       # Refreshes acceptance/bundle/refschema/out.fields.txt, which feeds
       # generate-direct-apitypes and generate-direct-resources below.
       - task: generate-refschema
+      - task: generate-bitmap-schema
       - task: generate-schema
       - task: generate-validation
       - task: generate-direct
@@ -827,6 +833,7 @@ tasks:
     cmds:
       - task: generate-cligen
       - task: generate-refschema
+      - task: generate-bitmap-schema
       - task: generate-schema
       - task: generate-validation
       - task: generate-direct
@@ -941,6 +948,33 @@ tasks:
     cmds:
       - go test ./acceptance -run TestAccept/bundle/refschema -update
 
+  # Appends any newly added bundle config fields to bundle/bitmap/schema.txt, the
+  # embedded bitmap schema (see cmd/bundle/bitmap). The schema is append-only, so
+  # update-schema reads the currently embedded schema and only appends. Output is
+  # written to a temp file first so the redirect does not truncate schema.txt
+  # before `go run` embeds it at compile time. The acceptance goldens embed the
+  # schema (out.schema.txt) and depend on bit positions (out.bitmap-text.txt,
+  # the encoded bitmap), so regenerate them here too — otherwise they go stale
+  # whenever the schema changes.
+  generate-bitmap-schema:
+    desc: Regenerate bundle/bitmap/schema.txt and the bitmap acceptance goldens
+    sources:
+      - bundle/**/*.go
+      - exclude: bundle/**/*_test.go
+      - acceptance/bundle/bitmap/script
+      - acceptance/bundle/bitmap/test.toml
+      - acceptance/bundle/bitmap/databricks.yml
+      - go.mod
+      - go.sum
+      - "{{.EMBED_SOURCES}}"
+    generates:
+      - bundle/bitmap/schema.txt
+      - acceptance/bundle/bitmap/out.schema.txt
+    cmds:
+      - go run . bundle bitmap update-schema > bundle/bitmap/schema.txt.tmp
+      - mv bundle/bitmap/schema.txt.tmp bundle/bitmap/schema.txt
+      - go test ./acceptance -run TestAccept/bundle/bitmap -update
+
   # Upstream field documentation comes from the checked-in .codegen/cli.json;
   # bundle/internal/schema/annotations.yml carries the CLI-owned docs and
   # overrides and is rewritten in place (synced with the config structure).

acceptance/bin/decode_bitmap.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Decode a base64 bundle bitmap and print '0/1 field_path' per schema line.
+
+Usage: decode_bitmap.py SCHEMA_FILE < ENCODED_BITMAP
+
+The output matches `databricks bundle bitmap bitmap-text`, so a test can assert
+the encoded bitmap round-trips to the same per-field view.
+
+Wire format (see bundle/bitmap/bitmap.go): raw-DEFLATE of
+  magic "DBTB" (4) | size uint32 BE | context uint16 BE | bitmap ceil(size/8) bytes
+with bit i (MSB-first within each byte) corresponding to schema line i.
+"""
+
+import base64
+import struct
+import sys
+import zlib
+
+MAGIC = b"DBTB"
+HEADER_SIZE = 10
+
+
+def main():
+    schema_file = sys.argv[1]
+    with open(schema_file) as f:
+        schema = f.read().splitlines()
+
+    encoded = sys.stdin.read().strip()
+    # -15 selects raw DEFLATE (no zlib/gzip wrapper), matching compress/flate.
+    raw = zlib.decompress(base64.b64decode(encoded), -15)
+
+    if raw[:4] != MAGIC:
+        sys.exit(f"bad magic: {raw[:4]!r}")
+    (size,) = struct.unpack(">I", raw[4:8])
+    if len(raw) != HEADER_SIZE + (size + 7) // 8:
+        sys.exit(f"payload size mismatch: {size} bits, {len(raw)} bytes")
+    if size != len(schema):
+        sys.exit(f"schema has {len(schema)} lines but bitmap has {size} bits")
+
+    bitmap = raw[HEADER_SIZE:]
+    for i, field in enumerate(schema):
+        bit = (bitmap[i // 8] >> (7 - i % 8)) & 1
+        print(f"{bit} {field}")
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/bitmap/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle
+
+variables:
+  myvar:
+    description: a variable
+    default: abc
+
+presets:
+  name_prefix: dev_
+
+resources:
+  jobs:
+    my_job:
+      name: My Job
+      tags:
+        team: data