From 63c0692c771d9f31d503b95588e009bb4033f872 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 14:59:55 -0700 Subject: [PATCH 1/5] Add GCP support for RP SQL (BYOC) --- .../pages/sql/sql-statements/copy-to.adoc | 14 ++++- .../sql/sql-statements/create-storage.adoc | 57 +++++++++++++++++-- .../pages/connect-to-sql/authenticate.adoc | 2 +- .../pages/get-started/deploy-sql-cluster.adoc | 16 ++++-- .../sql/pages/get-started/sql-quickstart.adoc | 2 +- 5 files changed, 78 insertions(+), 13 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/copy-to.adoc b/modules/reference/pages/sql/sql-statements/copy-to.adoc index 5e68d7c47..3b5ec63bd 100644 --- a/modules/reference/pages/sql/sql-statements/copy-to.adoc +++ b/modules/reference/pages/sql/sql-statements/copy-to.adoc @@ -162,6 +162,9 @@ WITH (AWS_CRED(AWS_REGION 'us-west-1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'acc ==== Google Cloud Storage +// TODO: This example passes a file path to GCS_CRED, but GCS_CRED (like CREATE STORAGE's +// service_account_key) accepts the JSON key file's *contents*, not a path. Verify and fix +// in a follow-up pass on this PR. Pass the path to your credentials JSON file: [source,sql] @@ -176,7 +179,14 @@ If you cannot use a path to the credentials file, pass its contents as a string: COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' WITH (GCS_CRED('')); ---- -You can also use `AWS_CRED` with GCS by setting the endpoint: +`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE: + +[source,sql] +---- +COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file'; +---- + +`GCS_CRED` has no option to override the GCS endpoint. To target a custom endpoint (for example, a GCS emulator or the XML API), use `AWS_CRED` instead, since Google Cloud Storage supports an S3-compatible API: [source,sql] ---- @@ -186,7 +196,7 @@ WITH (AWS_CRED(AWS_REGION 'region1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'acces [TIP] ==== -For Google Cloud Storage, use HMAC keys for authentication. See the link:https://cloud.google.com/storage/docs/authentication/hmackeys[HMAC keys - Cloud Storage^] page for details. +The `AWS_CRED` approach requires HMAC keys for authentication. See the link:https://cloud.google.com/storage/docs/authentication/hmackeys[HMAC keys - Cloud Storage^] page for details. ==== ==== Azure Blob Storage diff --git a/modules/reference/pages/sql/sql-statements/create-storage.adoc b/modules/reference/pages/sql/sql-statements/create-storage.adoc index cbd100f34..25f88dfb7 100644 --- a/modules/reference/pages/sql/sql-statements/create-storage.adoc +++ b/modules/reference/pages/sql/sql-statements/create-storage.adoc @@ -2,22 +2,22 @@ :description: The CREATE STORAGE statement creates a connection to external object storage for use with Redpanda catalogs. :page-topic-type: reference -The `CREATE STORAGE` statement creates a named connection to external object storage such as Amazon S3. +The `CREATE STORAGE` statement creates a named connection to external object storage such as Amazon S3 or Google Cloud Storage (GCS). == Syntax [source,sql] ---- CREATE STORAGE [IF NOT EXISTS] storage_name -TYPE = S3 +TYPE = S3 | GCS WITH (option = 'value' [, ...]); ---- * `storage_name`: Name for the new storage connection. -* `TYPE`: Storage type. Redpanda SQL currently supports only `S3`. +* `TYPE`: Storage type. Redpanda SQL supports `S3` and `GCS`. * `IF NOT EXISTS`: Optional. Prevents an error if a storage connection with the same name already exists. -== Options +== S3 options [cols="<30%,<15%,<10%,<45%",options="header"] |=== @@ -39,8 +39,32 @@ WITH (option = 'value' [, ...]); |AWS secret access key. |=== +== GCS options + +[cols="<30%,<15%,<10%,<45%",options="header"] +|=== +|Option |Type |Required |Description + +|`url` +|STRING +|No +|GCS bucket URL, in the form `gs:///`. If omitted, the connection holds credentials only and has no associated bucket. + +|`service_account_key` +|STRING +|No +|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE. + +|`endpoint` +|STRING +|No +|Override the default GCS endpoint. +|=== + == Examples +=== Create an S3 storage connection + [source,sql] ---- CREATE STORAGE archive_storage @@ -51,3 +75,28 @@ WITH ( secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' ); ---- + +=== Create a GCS storage connection with a service account key + +[source,sql] +---- +CREATE STORAGE archive_storage +TYPE = GCS +WITH ( + url = 'gs://archive-bucket/redpanda-sql', + service_account_key = '{"type": "service_account", "project_id": "my-project", ...}' +); +---- + +=== Create a GCS storage connection using Application Default Credentials + +Omit `service_account_key` to authenticate with Application Default Credentials (ADC). On GKE, this includes Workload Identity Federation. + +[source,sql] +---- +CREATE STORAGE archive_storage +TYPE = GCS +WITH ( + url = 'gs://archive-bucket/redpanda-sql' +); +---- diff --git a/modules/sql/pages/connect-to-sql/authenticate.adoc b/modules/sql/pages/connect-to-sql/authenticate.adoc index 99b6a2019..af23d49e6 100644 --- a/modules/sql/pages/connect-to-sql/authenticate.adoc +++ b/modules/sql/pages/connect-to-sql/authenticate.adoc @@ -14,7 +14,7 @@ After completing these steps, you will be able to: == Prerequisites -* A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. +* A Redpanda BYOC cluster on AWS or GCP with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * https://www.postgresql.org/download/[`psql`^] or another PostgreSQL-compatible client. * A user or service account assigned a role with the *SQL: Access* or *SQL: Manage* permission in Redpanda Cloud's data-plane RBAC. Redpanda Cloud provisions a corresponding user in the SQL engine when the role is assigned. To assign roles, go to *Organization IAM > Roles*; SQL permissions are under the *Data Plane* tab when you create or edit a role. See xref:security:authorization/rbac/rbac_dp.adoc[Configure RBAC in the Data Plane]. diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index 22bd946de..19df2b039 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -6,7 +6,9 @@ :learning-objective-2: Scale or disable the SQL engine :learning-objective-3: Verify that the SQL engine is running and ready to accept connections -Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. For Iceberg-enabled topics, queries can span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. +// TODO: Drop the "On AWS" scoping once RPSQL Deployment with Iceberg on GCP +// (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. +Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. On AWS, Iceberg-enabled topics also support queries that span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. After reading this page, you will be able to: @@ -14,7 +16,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is currently available only on BYOC clusters running on AWS. +NOTE: Redpanda SQL is currently available on BYOC clusters running on AWS and GCP. TIP: If you manage your infrastructure with Terraform, see xref:manage:terraform-provider.adoc#enable-redpanda-sql-on-a-byoc-cluster[Enable Redpanda SQL on a BYOC cluster] in the Terraform provider guide. @@ -43,7 +45,7 @@ Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. -. Start creating a new BYOC cluster on AWS. For details and prerequisites, see xref:get-started:cluster-types/byoc/aws/create-byoc-cluster-aws.adoc[]. +. Start creating a new BYOC cluster on AWS or GCP. For details and prerequisites, see xref:get-started:cluster-types/byoc/aws/create-byoc-cluster-aws.adoc[] or xref:get-started:cluster-types/byoc/gcp/create-byoc-cluster-gcp.adoc[]. . In the cluster creation form, locate the *Redpanda SQL engine* card. Toggle the engine on and use the *RPU* slider to set the compute size. + For more on RPUs, compute, and cost calculations, see xref:billing:billing.adoc#redpanda-sql-billing-metrics[Redpanda SQL billing metrics]. @@ -64,7 +66,7 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ -d '{ "cluster": { "name": "", - "cloud_provider": "CLOUD_PROVIDER_AWS", + "cloud_provider": "", "type": "TYPE_BYOC", "region": "", "zones": [ ], @@ -79,8 +81,9 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ }' ---- + -Replace the `rpsql` placeholders with your own values: +Replace the placeholders with your own values: + +* ``: `CLOUD_PROVIDER_AWS` or `CLOUD_PROVIDER_GCP`. * ``: Set the initial number of SQL compute nodes (minimum 1). * `` (optional): For multi-AZ clusters, specify which of the cluster's availability zones to deploy for the SQL engine. Provide a single zone string from the cluster's `zones` list. + @@ -90,6 +93,9 @@ Replace the `rpsql` placeholders with your own values: === On an existing cluster +// TODO: Confirm with SME whether GCP BYOC clusters created before Redpanda SQL launched on GCP +// (M7/CIAINFRA-3420) need an equivalent agent-permission refresh step before enabling SQL, similar +// to the AWS `rpk cloud byoc aws apply` requirement below. NOTE: If your BYOC cluster on AWS was created before May 28, 2026, rerun `rpk cloud byoc aws apply --redpanda-id=` to update the agent's IAM permissions before enabling Redpanda SQL. [tabs] diff --git a/modules/sql/pages/get-started/sql-quickstart.adoc b/modules/sql/pages/get-started/sql-quickstart.adoc index 989b2807d..703a1ddd8 100644 --- a/modules/sql/pages/get-started/sql-quickstart.adoc +++ b/modules/sql/pages/get-started/sql-quickstart.adoc @@ -14,7 +14,7 @@ After reading this page, you will be able to: == Prerequisites -* A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[]. +* A Redpanda BYOC cluster on AWS or GCP with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[]. * Admin access to your cluster in the Redpanda Cloud Console, or a role with the *SQL: Manage* permission. You need one of these to view SQL connection details and to create catalogs, tables, and grants in the SQL engine. For more information on authorization in Redpanda Cloud, see xref:security:authorization/rbac/index.adoc[]. * A Redpanda topic with a schema registered in Schema Registry. If you don't have one, follow the optional <> section below to create a sample `orders` topic. * xref:manage:rpk/rpk-install.adoc[`rpk` v26.1.6] or later installed on your local machine to generate an authentication token. From 2ea7fb709d1392203a2ea26ea98d67d48a03c8eb Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 15:00:28 -0700 Subject: [PATCH 2/5] TODOs for Iceberg/BigLake --- .../pages/sql/sql-statements/create-iceberg-catalog.adoc | 3 +++ modules/reference/pages/sql/system-virtual-tables.adoc | 2 ++ modules/sql/pages/query-data/query-iceberg-topics.adoc | 2 ++ 3 files changed, 7 insertions(+) diff --git a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc index 749fd12ae..fc96ecb65 100644 --- a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc +++ b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc @@ -36,6 +36,9 @@ NOTE: Catalogs are created in the current schema (`public` by default). To creat |No |Iceberg warehouse identifier or location. +// TODO: Add `gcp` auth_type (keyless/Workload Identity, requires gcp_project_id) once BigLake +// REST catalog support ships (OXLA-9522, cloudv2 PR #27525). Add a corresponding example +// and a `gcp_project_id` option row below. |`auth_type` |STRING |No diff --git a/modules/reference/pages/sql/system-virtual-tables.adoc b/modules/reference/pages/sql/system-virtual-tables.adoc index bbaad3d8b..7232123af 100644 --- a/modules/reference/pages/sql/system-virtual-tables.adoc +++ b/modules/reference/pages/sql/system-virtual-tables.adoc @@ -76,6 +76,8 @@ Iceberg catalog connections, with their REST endpoint and authentication details |`name` |`text` |No |Iceberg catalog name. |`uri` |`text` |No |REST catalog endpoint URI. |`warehouse` |`text` |No |Iceberg warehouse identifier or location. +// TODO: Add `gcp` to the auth_type list once BigLake REST catalog support ships +// (OXLA-9522, cloudv2 PR #27525) — see also create-iceberg-catalog.adoc auth_type option. |`auth_type` |`text` |No |Authentication type for the REST catalog. One of `oauth2`, `basic`, `aws_sigv4`, or empty if the catalog connects without authentication. |`namespace_name` |`text` |No |Schema containing the catalog. |`database_name` |`text` |No |Database containing the catalog. diff --git a/modules/sql/pages/query-data/query-iceberg-topics.adoc b/modules/sql/pages/query-data/query-iceberg-topics.adoc index d028c8ae5..af1c25fd5 100644 --- a/modules/sql/pages/query-data/query-iceberg-topics.adoc +++ b/modules/sql/pages/query-data/query-iceberg-topics.adoc @@ -18,6 +18,8 @@ After reading this page, you will be able to: == Prerequisites +// TODO: Update prerequisites to include GCP once RPSQL Deployment with Iceberg on GCP +// (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. * A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * The cluster's xref:reference:properties/cluster-properties.adoc#iceberg_catalog_type[`iceberg_catalog_type`] property is set to `rest`. The `object_storage` catalog type does not support Iceberg queries from Redpanda SQL. * The cluster's Iceberg REST catalog is configured. See xref:manage:iceberg/rest-catalog/index.adoc[Integrate with REST Catalogs] for the supported REST catalog options (AWS Glue, Snowflake, Databricks Unity, Polaris, and others) and their configuration steps. From 9a826db20af0bf668f5f635183a37e17fb197984 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 15:15:02 -0700 Subject: [PATCH 3/5] Review pass --- modules/reference/pages/sql/sql-statements/copy-to.adoc | 2 +- .../reference/pages/sql/sql-statements/create-storage.adoc | 2 +- modules/sql/pages/get-started/deploy-sql-cluster.adoc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/copy-to.adoc b/modules/reference/pages/sql/sql-statements/copy-to.adoc index 3b5ec63bd..fdb25f424 100644 --- a/modules/reference/pages/sql/sql-statements/copy-to.adoc +++ b/modules/reference/pages/sql/sql-statements/copy-to.adoc @@ -179,7 +179,7 @@ If you cannot use a path to the credentials file, pass its contents as a string: COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' WITH (GCS_CRED('')); ---- -`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE: +`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on Google Kubernetes Engine (GKE): [source,sql] ---- diff --git a/modules/reference/pages/sql/sql-statements/create-storage.adoc b/modules/reference/pages/sql/sql-statements/create-storage.adoc index 25f88dfb7..b622d744f 100644 --- a/modules/reference/pages/sql/sql-statements/create-storage.adoc +++ b/modules/reference/pages/sql/sql-statements/create-storage.adoc @@ -53,7 +53,7 @@ WITH (option = 'value' [, ...]); |`service_account_key` |STRING |No -|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE. +|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on Google Kubernetes Engine (GKE). |`endpoint` |STRING diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index 19df2b039..7a6272631 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -8,7 +8,7 @@ // TODO: Drop the "On AWS" scoping once RPSQL Deployment with Iceberg on GCP // (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. -Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. On AWS, Iceberg-enabled topics also support queries that span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. +Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can also run queries that span both the streaming topic and its Iceberg history for Iceberg-enabled topics. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. After reading this page, you will be able to: @@ -16,7 +16,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is currently available on BYOC clusters running on AWS and GCP. +NOTE: Redpanda SQL is available for BYOC clusters on AWS and GCP. TIP: If you manage your infrastructure with Terraform, see xref:manage:terraform-provider.adoc#enable-redpanda-sql-on-a-byoc-cluster[Enable Redpanda SQL on a BYOC cluster] in the Terraform provider guide. From 80528b7a080a15edd0f722727022d5ba3ad20196 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 22 Jul 2026 14:45:08 -0700 Subject: [PATCH 4/5] Update from source verification --- .../pages/sql/sql-statements/copy-to.adoc | 24 ++++++------------ .../create-iceberg-catalog.adoc | 25 ++++++++++++++++--- .../pages/sql/system-virtual-tables.adoc | 4 +-- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/copy-to.adoc b/modules/reference/pages/sql/sql-statements/copy-to.adoc index fdb25f424..29adc35d0 100644 --- a/modules/reference/pages/sql/sql-statements/copy-to.adoc +++ b/modules/reference/pages/sql/sql-statements/copy-to.adoc @@ -155,42 +155,32 @@ To export data to cloud storage, use the `COPY TO` command with the appropriate [source,sql] ---- -COPY default_redpanda_catalog=>film TO 's3://your-bucket/file_name' -WITH (AWS_CRED(AWS_REGION 'us-west-1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'access_key', ENDPOINT 's3.us-west-1.amazonaws.com'), +COPY default_redpanda_catalog=>film TO 's3:///' +WITH (AWS_CRED(AWS_REGION 'us-west-1', AWS_KEY_ID '', AWS_PRIVATE_KEY '', ENDPOINT 's3.us-west-1.amazonaws.com'), FORMAT CSV, HEADER ON, NULL 'unknown'); ---- ==== Google Cloud Storage -// TODO: This example passes a file path to GCS_CRED, but GCS_CRED (like CREATE STORAGE's -// service_account_key) accepts the JSON key file's *contents*, not a path. Verify and fix -// in a follow-up pass on this PR. -Pass the path to your credentials JSON file: +Pass the contents of your service account key JSON file to `GCS_CRED`: [source,sql] ---- -COPY default_redpanda_catalog=>film TO 'gs://your-bucket/file_name' WITH (GCS_CRED('/path/to/credentials.json')); ----- - -If you cannot use a path to the credentials file, pass its contents as a string: - -[source,sql] ----- -COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' WITH (GCS_CRED('')); +COPY default_redpanda_catalog=>film TO 'gs:///' WITH (GCS_CRED('')); ---- `GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on Google Kubernetes Engine (GKE): [source,sql] ---- -COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file'; +COPY default_redpanda_catalog=>project TO 'gs:///'; ---- `GCS_CRED` has no option to override the GCS endpoint. To target a custom endpoint (for example, a GCS emulator or the XML API), use `AWS_CRED` instead, since Google Cloud Storage supports an S3-compatible API: [source,sql] ---- -COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' +COPY default_redpanda_catalog=>project TO 'gs:///' WITH (AWS_CRED(AWS_REGION 'region1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'access_key', ENDPOINT 'https://storage.googleapis.com')); ---- @@ -207,6 +197,6 @@ The `AWS_CRED` approach requires HMAC keys for authentication. See the link:http [source,sql] ---- -COPY default_redpanda_catalog=>taxi_data TO 'wasbs://container-name/your_blob' +COPY default_redpanda_catalog=>taxi_data TO 'wasbs:///' WITH (AZURE_CRED(TENANT_ID 'your_tenant_id' CLIENT_ID 'your_client_id', CLIENT_SECRET 'your_client_secret')); ---- diff --git a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc index fc96ecb65..f5c8a0636 100644 --- a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc +++ b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc @@ -36,13 +36,15 @@ NOTE: Catalogs are created in the current schema (`public` by default). To creat |No |Iceberg warehouse identifier or location. -// TODO: Add `gcp` auth_type (keyless/Workload Identity, requires gcp_project_id) once BigLake -// REST catalog support ships (OXLA-9522, cloudv2 PR #27525). Add a corresponding example -// and a `gcp_project_id` option row below. |`auth_type` |STRING |No -|Authentication type for the REST catalog. One of `oauth2`, `basic`, or `aws_sigv4`. If omitted, the catalog connects without authentication. Providing an auth-specific option (such as `username` or `aws_region`) without `auth_type` is rejected. +|Authentication type for the REST catalog. One of `oauth2`, `basic`, `aws_sigv4`, or `gcp`. If omitted, the catalog connects without authentication. Providing an auth-specific option (such as `username`, `aws_region`, or `gcp_project_id`) without `auth_type` is rejected. + +|`gcp_project_id` +|STRING +|Required when `auth_type = 'gcp'` +|GCP project ID for the BigLake catalog. Attributes API calls for billing and quota. |`oauth2_client_id` |STRING @@ -218,6 +220,21 @@ CREATE ICEBERG CATALOG glue_cat STORAGE iceberg_storage ); ---- +=== Create an Iceberg catalog for BigLake on GCP + +Use `auth_type = 'gcp'` for a Google Cloud BigLake REST catalog. Redpanda SQL authenticates with Application Default Credentials (ADC), which on Google Kubernetes Engine (GKE) resolves to Workload Identity, so no static key is required. The `gcp_project_id` option is required. Set `uri` and `warehouse` to the values for your BigLake catalog. + +[source,sql] +---- +CREATE ICEBERG CATALOG biglake_cat STORAGE iceberg_storage + WITH ( + uri = '', + warehouse = '', + auth_type = 'gcp', + gcp_project_id = '' + ); +---- + === Create an Iceberg catalog scoped to specific namespaces (nested backend) For backends that support nested namespaces (such as Polaris), use multi-segment dotted paths in `allowed_namespaces`. diff --git a/modules/reference/pages/sql/system-virtual-tables.adoc b/modules/reference/pages/sql/system-virtual-tables.adoc index 7232123af..0719a3a70 100644 --- a/modules/reference/pages/sql/system-virtual-tables.adoc +++ b/modules/reference/pages/sql/system-virtual-tables.adoc @@ -76,9 +76,7 @@ Iceberg catalog connections, with their REST endpoint and authentication details |`name` |`text` |No |Iceberg catalog name. |`uri` |`text` |No |REST catalog endpoint URI. |`warehouse` |`text` |No |Iceberg warehouse identifier or location. -// TODO: Add `gcp` to the auth_type list once BigLake REST catalog support ships -// (OXLA-9522, cloudv2 PR #27525) — see also create-iceberg-catalog.adoc auth_type option. -|`auth_type` |`text` |No |Authentication type for the REST catalog. One of `oauth2`, `basic`, `aws_sigv4`, or empty if the catalog connects without authentication. +|`auth_type` |`text` |No |Authentication type for the REST catalog. One of `oauth2`, `basic`, `aws_sigv4`, `gcp`, or empty if the catalog connects without authentication. |`namespace_name` |`text` |No |Schema containing the catalog. |`database_name` |`text` |No |Database containing the catalog. |=== From 1b20cb3ff5cfcac4e1fd6f25e295d9e283a3fbbd Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 22 Jul 2026 15:12:24 -0700 Subject: [PATCH 5/5] Apply suggestions from automated review --- .../pages/sql/sql-statements/create-iceberg-catalog.adoc | 2 +- .../reference/pages/sql/sql-statements/create-storage.adoc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc index f5c8a0636..8bccc3672 100644 --- a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc +++ b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc @@ -44,7 +44,7 @@ NOTE: Catalogs are created in the current schema (`public` by default). To creat |`gcp_project_id` |STRING |Required when `auth_type = 'gcp'` -|GCP project ID for the BigLake catalog. Attributes API calls for billing and quota. +|GCP project that BigLake API requests are billed and quota-attributed to. |`oauth2_client_id` |STRING diff --git a/modules/reference/pages/sql/sql-statements/create-storage.adoc b/modules/reference/pages/sql/sql-statements/create-storage.adoc index b622d744f..08f680cfa 100644 --- a/modules/reference/pages/sql/sql-statements/create-storage.adoc +++ b/modules/reference/pages/sql/sql-statements/create-storage.adoc @@ -80,7 +80,7 @@ WITH ( [source,sql] ---- -CREATE STORAGE archive_storage +CREATE STORAGE gcs_key_storage TYPE = GCS WITH ( url = 'gs://archive-bucket/redpanda-sql', @@ -94,7 +94,7 @@ Omit `service_account_key` to authenticate with Application Default Credentials [source,sql] ---- -CREATE STORAGE archive_storage +CREATE STORAGE gcs_adc_storage TYPE = GCS WITH ( url = 'gs://archive-bucket/redpanda-sql'