From fcc5560a0230efdc7f688897c62289b458f73db9 Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Mon, 13 Apr 2026 11:35:58 -0400 Subject: [PATCH 1/7] Add example for creating a Lakebase Autoscaling Postgres project --- knowledge_base/postgres_autoscaling/README.md | 53 +++++++++++++++++++ .../postgres_autoscaling/databricks.yml | 47 ++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 knowledge_base/postgres_autoscaling/README.md create mode 100644 knowledge_base/postgres_autoscaling/databricks.yml diff --git a/knowledge_base/postgres_autoscaling/README.md b/knowledge_base/postgres_autoscaling/README.md new file mode 100644 index 0000000..b070b94 --- /dev/null +++ b/knowledge_base/postgres_autoscaling/README.md @@ -0,0 +1,53 @@ +# Lakebase Autoscaling (Postgres) project with branching + +This example demonstrates how to define a Lakebase Autoscaling project with a non-default branch in a Declarative Automation Bundle. + +It includes and deploys: +- A Lakebase Autoscaling project with configurable min and max compute units (CUs) +- A non-default `development` branch for isolated dev/test workflows +- A read-only replica endpoint on the default production branch + +Lakebase Autoscaling is Databricks' managed PostgreSQL service with autoscaling compute, Git-like branching, scale-to-zero, and instant point-in-time restore. + +For more information about Lakebase Autoscaling, see the [documentation](https://docs.databricks.com/aws/en/oltp/projects/). + +## Prerequisites + +* Databricks CLI v0.287.0 or above +* `psql` client version 14 or above (Optional, only needed to run the demo queries) + +## Usage + +Modify `databricks.yml`: +* Update the `host` field under `workspace` to the Databricks workspace to deploy to +* Adjust `autoscaling_limit_min_cu` and `autoscaling_limit_max_cu` to fit your workload (valid range: 0.5-32 CU, max minus min cannot exceed 16 CU) +* Adjust `suspend_timeout_duration` to control scale-to-zero behavior (minimum: 60 seconds) + +Run `databricks bundle deploy` to deploy the bundle. + +Please note that after this bundle gets deployed, the project and its computes start running, which incurs cost. Computes with scale-to-zero enabled will suspend after the configured `suspend_timeout_duration`. + +Run the following queries against the **production** branch to populate your database with sample data: + +```bash +# Create a demo table: +databricks psql --project my-autoscaling-project -- -d databricks_postgres -c "CREATE TABLE IF NOT EXISTS hello_world (id SERIAL PRIMARY KEY, message TEXT, number INTEGER, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);" + +# Insert 100 rows of demo data: +databricks psql --project my-autoscaling-project -- -d databricks_postgres -c "INSERT INTO hello_world (message, number) SELECT 'Hello World #' || generate_series, generate_series FROM generate_series(1, 100);" + +# Show generated rows: +databricks psql --project my-autoscaling-project -- -d databricks_postgres -c "SELECT * FROM hello_world;" +``` + +Run the following queries against the **development** branch to verify the branched data: + +**Important:** Prior to running the following command, the `development` branch MUST be 'reset' via the web UI. This will re-sync the child branch with its parent. More information regarding resetting a branch can be found [here](https://docs.databricks.com/aws/en/oltp/projects/manage-branches?#reset-branch-from-parent). + +```bash +# Query the development branch: +databricks psql --project my-autoscaling-project --branch development -- -d databricks_postgres -c "SELECT count(*) FROM hello_world;" +``` + +## Clean up +To remove the provisioned project, branches, and endpoints run `databricks bundle destroy` diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml new file mode 100644 index 0000000..b649d31 --- /dev/null +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -0,0 +1,47 @@ +bundle: + name: postgres-autoscaling-example + +# workspace: +# host: https://myworkspace.cloud.databricks.com + +resources: + # Top-level project container for Lakebase Autoscaling. + # Creating a project automatically provisions: + # - A default branch named "production" + # - A primary READ_WRITE endpoint on the default branch + # - A "databricks_postgres" database + postgres_projects: + my_project: + project_id: my-autoscaling-project + display_name: "My Autoscaling Project" + pg_version: 17 + default_endpoint_settings: + # All newly created endpoints will inherit these settings + # Changing these will not affect existing endpoints + autoscaling_limit_min_cu: 1 + autoscaling_limit_max_cu: 4 + suspend_timeout_duration: "300s" # Scale-to-zero after 5 minutes + + # (Optional) Read replica on the default (production) branch. + postgres_endpoints: + prod_read_replica: + parent: ${resources.postgres_projects.my_project.id}/branches/production + endpoint_id: ep-production-read-replica + endpoint_type: ENDPOINT_TYPE_READ_ONLY + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 2 + + # (Optional) Non-default branch for development/testing. + # Note: Branch is al + # Note: READ_WRITE endpoints are currently created by default + postgres_branches: + dev: + parent: ${resources.postgres_projects.my_project.id} + branch_id: development # The name for the branch + no_expiry: true + +# Targets allow you to deploy the same bundle to different Databricks workspaces. +targets: + dev: + default: true + mode: development From 140358571bfb82a770c11b595cfd4f01c7a273aa Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Mon, 13 Apr 2026 11:46:33 -0400 Subject: [PATCH 2/7] Update comment and documentation for clarity --- knowledge_base/postgres_autoscaling/README.md | 4 ++-- knowledge_base/postgres_autoscaling/databricks.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/knowledge_base/postgres_autoscaling/README.md b/knowledge_base/postgres_autoscaling/README.md index b070b94..07d660b 100644 --- a/knowledge_base/postgres_autoscaling/README.md +++ b/knowledge_base/postgres_autoscaling/README.md @@ -25,7 +25,7 @@ Modify `databricks.yml`: Run `databricks bundle deploy` to deploy the bundle. -Please note that after this bundle gets deployed, the project and its computes start running, which incurs cost. Computes with scale-to-zero enabled will suspend after the configured `suspend_timeout_duration`. +Please note that after this bundle gets deployed, the project and its compute endpoints start running, which incurs cost. Endpoints with scale-to-zero enabled will suspend after the configured `suspend_timeout_duration`. Run the following queries against the **production** branch to populate your database with sample data: @@ -42,7 +42,7 @@ databricks psql --project my-autoscaling-project -- -d databricks_postgres -c "S Run the following queries against the **development** branch to verify the branched data: -**Important:** Prior to running the following command, the `development` branch MUST be 'reset' via the web UI. This will re-sync the child branch with its parent. More information regarding resetting a branch can be found [here](https://docs.databricks.com/aws/en/oltp/projects/manage-branches?#reset-branch-from-parent). +**Important:** Prior to running the following command, the `development` branch MUST be 'reset' via the web UI. This will re-sync the child branch with its parent. More information regarding resetting a branch can be found [here](https://docs.databricks.com/aws/en/oltp/projects/manage-branches#reset-branch-from-parent). ```bash # Query the development branch: diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml index b649d31..2392b38 100644 --- a/knowledge_base/postgres_autoscaling/databricks.yml +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -32,7 +32,6 @@ resources: autoscaling_limit_max_cu: 2 # (Optional) Non-default branch for development/testing. - # Note: Branch is al # Note: READ_WRITE endpoints are currently created by default postgres_branches: dev: From 295f8052951b486cb998bb36cfdadd83ad48c67e Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Tue, 26 May 2026 14:59:19 -0400 Subject: [PATCH 3/7] Add clarification as to what "databricks_postgres" referred to here --- knowledge_base/postgres_autoscaling/databricks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml index 2392b38..386043c 100644 --- a/knowledge_base/postgres_autoscaling/databricks.yml +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -9,7 +9,7 @@ resources: # Creating a project automatically provisions: # - A default branch named "production" # - A primary READ_WRITE endpoint on the default branch - # - A "databricks_postgres" database + # - A database with the name "databricks_postgres" postgres_projects: my_project: project_id: my-autoscaling-project From 80ac3575e633d66209a62afc7026e1e6d1adf2c3 Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Wed, 27 May 2026 14:42:50 -0400 Subject: [PATCH 4/7] Move workspace host setting to targets, add a prod workspace target --- knowledge_base/postgres_autoscaling/databricks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml index 386043c..932782c 100644 --- a/knowledge_base/postgres_autoscaling/databricks.yml +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -1,9 +1,6 @@ bundle: name: postgres-autoscaling-example -# workspace: -# host: https://myworkspace.cloud.databricks.com - resources: # Top-level project container for Lakebase Autoscaling. # Creating a project automatically provisions: @@ -44,3 +41,8 @@ targets: dev: default: true mode: development + workspace: + host: https://my-dev-workspace.cloud.databricks.com + prod: + workspace: + host: https://my-prod-workspace.cloud.databricks.com \ No newline at end of file From e08df2e8f6c69d4164868b636dbfdd7f08196ae7 Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Wed, 27 May 2026 14:45:04 -0400 Subject: [PATCH 5/7] Update comments to add clarification --- knowledge_base/postgres_autoscaling/databricks.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml index 932782c..bef17a1 100644 --- a/knowledge_base/postgres_autoscaling/databricks.yml +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -25,11 +25,14 @@ resources: parent: ${resources.postgres_projects.my_project.id}/branches/production endpoint_id: ep-production-read-replica endpoint_type: ENDPOINT_TYPE_READ_ONLY + # Note: The below two options override the ones set in + # "default_endpoint_settings" above autoscaling_limit_min_cu: 0.5 autoscaling_limit_max_cu: 2 # (Optional) Non-default branch for development/testing. - # Note: READ_WRITE endpoints are currently created by default + # Note: A READ_WRITE endpoint is currently created by default + # when provisioning a new branch postgres_branches: dev: parent: ${resources.postgres_projects.my_project.id} From b395b4684743a18c7649064522a45632914672e2 Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Wed, 27 May 2026 14:48:43 -0400 Subject: [PATCH 6/7] Clarify note regarding default_endpoint_settings --- knowledge_base/postgres_autoscaling/databricks.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/knowledge_base/postgres_autoscaling/databricks.yml b/knowledge_base/postgres_autoscaling/databricks.yml index bef17a1..d8b182e 100644 --- a/knowledge_base/postgres_autoscaling/databricks.yml +++ b/knowledge_base/postgres_autoscaling/databricks.yml @@ -13,8 +13,9 @@ resources: display_name: "My Autoscaling Project" pg_version: 17 default_endpoint_settings: - # All newly created endpoints will inherit these settings - # Changing these will not affect existing endpoints + # Newly created endpoints inherit these settings unless overridden + # at the endpoint level (see "prod_read_replica" below). + # Changing these will not affect existing endpoints. autoscaling_limit_min_cu: 1 autoscaling_limit_max_cu: 4 suspend_timeout_duration: "300s" # Scale-to-zero after 5 minutes From acb1522ce0b1943549d3ac98c230310e23b33712 Mon Sep 17 00:00:00 2001 From: Ridge Coffman Date: Wed, 27 May 2026 14:49:54 -0400 Subject: [PATCH 7/7] Update README to include instructions on the 'prod' bundle target --- knowledge_base/postgres_autoscaling/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/knowledge_base/postgres_autoscaling/README.md b/knowledge_base/postgres_autoscaling/README.md index 07d660b..33dbcfc 100644 --- a/knowledge_base/postgres_autoscaling/README.md +++ b/knowledge_base/postgres_autoscaling/README.md @@ -19,11 +19,11 @@ For more information about Lakebase Autoscaling, see the [documentation](https:/ ## Usage Modify `databricks.yml`: -* Update the `host` field under `workspace` to the Databricks workspace to deploy to +* Update the `host` field under each target's `workspace` block (`dev` and `prod`) to point at the Databricks workspace you want to deploy to * Adjust `autoscaling_limit_min_cu` and `autoscaling_limit_max_cu` to fit your workload (valid range: 0.5-32 CU, max minus min cannot exceed 16 CU) * Adjust `suspend_timeout_duration` to control scale-to-zero behavior (minimum: 60 seconds) -Run `databricks bundle deploy` to deploy the bundle. +The bundle defines two targets: `dev` (the default) and `prod`. Run `databricks bundle deploy` to deploy to `dev`, or `databricks bundle deploy --target prod` to deploy to `prod`. Please note that after this bundle gets deployed, the project and its compute endpoints start running, which incurs cost. Endpoints with scale-to-zero enabled will suspend after the configured `suspend_timeout_duration`.