Add postgres_data_apis bundle resource

.nextchanges/bundles/postgres-data-apis.md

@@ -0,0 +1 @@
+* Add the `postgres_data_apis` resource (Private Preview) for enabling and configuring the Data API of a Lakebase Postgres database. ([#7160](https://github.com/databricks/cli/pull/7160))

acceptance/bundle/deployment/bind/postgres_data_api/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle
+
+resources:
+  postgres_data_apis:
+    data_api1:
+      parent: projects/test-project/branches/production/databases/test-database
+      db_schemas:
+        - public
+      db_max_rows: 500

acceptance/bundle/deployment/bind/postgres_data_api/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/deployment/bind/postgres_data_api/output.txt

@@ -0,0 +1,94 @@
+
+=== Enable the Data API on a database outside the bundle
+=== Deploy fails because the Data API is already enabled
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for this database (409 ALREADY_EXISTS)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-project/branches/production/databases/test-database/data-api
+HTTP Status: 409 Conflict
+API error_code: ALREADY_EXISTS
+API message: Data API is already enabled for this database
+
+Files: 2 uploaded, 0 deleted
+
+=== Bind adopts the existing Data API
+>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve
+Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  Postgres Data APIs:
+    data_api1:
+      Name: projects/test-project/branches/production/databases/test-database/data-api
+
+>>> [CLI] bundle plan
+update postgres_data_apis.data_api1
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Updated postgres_data_apis.data_api1
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api
+{
+  "name": "projects/test-project/branches/production/databases/test-database/data-api",
+  "status": {
+    "db_schemas": [
+      "public"
+    ],
+    "db_max_rows": 500
+  }
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Unbind leaves the Data API enabled
+>>> [CLI] bundle deployment unbind data_api1
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  Postgres Data APIs:
+    data_api1:
+      Name: 
+
+>>> [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api
+{
+  "name": "projects/test-project/branches/production/databases/test-database/data-api"
+}
+
+=== Bind again; destroy turns off the Data API
+>>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve
+Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_data_apis.data_api1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Destroy: 1 deleted
+
+>>> musterr [CLI] postgres get-data-api projects/test-project/branches/production/databases/test-database/data-api
+Error: Data API is not enabled for this database
+
+>>> [CLI] postgres get-database projects/test-project/branches/production/databases/test-database
+{
+  "name": "projects/test-project/branches/production/databases/test-database"
+}

acceptance/bundle/deployment/bind/postgres_data_api/script

@@ -0,0 +1,31 @@
+DATABASE_NAME="projects/test-project/branches/production/databases/test-database"
+DATA_API_NAME="${DATABASE_NAME}/data-api"
+
+title "Enable the Data API on a database outside the bundle"
+$CLI postgres create-project test-project --json '{"spec": {"display_name": "Test Project", "pg_version": 16}}' &> LOG.setup
+$CLI postgres create-database projects/test-project/branches/production --database-id test-database --json '{"spec": {"postgres_database": "app_db", "role": "projects/test-project/branches/production/roles/owner"}}' >> LOG.setup 2>&1
+$CLI postgres create-data-api "${DATABASE_NAME}" --json '{"spec": {"db_schemas": ["public"]}}' >> LOG.setup 2>&1
+
+title "Deploy fails because the Data API is already enabled"
+trace musterr $CLI bundle deploy
+
+title "Bind adopts the existing Data API"
+# Bind records the config as state. The plan compares it with the spec read back
+# from status, so the configured db_max_rows is applied on the next deploy.
+trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve
+trace $CLI bundle summary
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name, status: (.status | {db_schemas, db_max_rows})}'
+trace $CLI bundle plan
+
+title "Unbind leaves the Data API enabled"
+trace $CLI bundle deployment unbind data_api1
+trace $CLI bundle summary
+trace $CLI postgres get-data-api "${DATA_API_NAME}" | jq '{name}'
+
+title "Bind again; destroy turns off the Data API"
+trace $CLI bundle deployment bind data_api1 "${DATA_API_NAME}" --auto-approve
+trace $CLI bundle destroy --auto-approve
+trace musterr $CLI postgres get-data-api "${DATA_API_NAME}"
+trace $CLI postgres get-database "${DATABASE_NAME}" | jq '{name}'

acceptance/bundle/deployment/bind/postgres_data_api/test.toml

@@ -0,0 +1,9 @@
+Cloud = false
+
+# Deployment history has no "postgres_data_apis." resource-key prefix yet. See dmsResourceTypes in
+# libs/testserver/bundledeployments.go, which mirrors Universe.
+EnvMatrixExclude.no_data_api_dms = ["DMS=true"]
+
+Ignore = [
+    ".databricks"
+]

acceptance/bundle/invariant/auto-migrate/test.toml

@@ -43,6 +43,8 @@ EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"]
 EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"]
 # Features are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"]
+# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it.
+EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/configs/postgres_data_api.yml.tmpl

@@ -0,0 +1,45 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  postgres_projects:
+    project:
+      project_id: test-pg-project-$UNIQUE_NAME
+      display_name: Test Postgres Project
+
+  # The Data API needs a read-write endpoint on the branch. Take over the
+  # "production" branch created with the project, which comes with one.
+  postgres_branches:
+    production:
+      parent: ${resources.postgres_projects.project.name}
+      branch_id: production
+      replace_existing: true
+      no_expiry: true
+
+  postgres_roles:
+    owner:
+      parent: ${resources.postgres_branches.production.name}
+      role_id: test-role-$UNIQUE_NAME
+      postgres_role: app_role
+
+  postgres_databases:
+    database:
+      parent: ${resources.postgres_branches.production.name}
+      database_id: test-database-$UNIQUE_NAME
+      postgres_database: app_db
+      role: ${resources.postgres_roles.owner.name}
+
+  postgres_data_apis:
+    foo:
+      parent: ${resources.postgres_databases.database.name}
+      db_schemas:
+        - public
+      db_max_rows: 1000 # CAN_REMOVE
+      server_cors_allowed_origins: # CAN_REMOVE
+        - https://app.example.com # CAN_REMOVE
+      jwt_cache_max_lifetime: 300s # CAN_REMOVE
+      openapi_mode: OPEN_API_MODE_IGNORE_PRIVILEGES # CAN_REMOVE
+      # Removing a bool converges like the other fields: the server keeps the
+      # last value and status reports it.
+      db_aggregates_enabled: false # CAN_REMOVE
+      server_timing_enabled: true # CAN_REMOVE

acceptance/bundle/invariant/continue_293/test.toml

@@ -43,6 +43,9 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml
 # secret resource is not supported on v0.293.0
 EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"]
 
+# postgres_data_apis resource is not supported on v0.293.0
+EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"]
+
 # $${...} escaping is not supported on v0.293.0: the seed deploy fails with
 # 'invalid dependency "${batch_id}", no such node ""' (issue #6480).
 EnvMatrixExclude.no_escaped_refs = ["INPUT_CONFIG=job_escaped_refs.yml.tmpl"]

acceptance/bundle/invariant/delete_idempotent/out.test.toml

@@ -42,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [
   "pipeline_config_dots.yml.tmpl",
   "postgres_branch.yml.tmpl",
   "postgres_catalog.yml.tmpl",
+  "postgres_data_api.yml.tmpl",
   "postgres_database.yml.tmpl",
   "postgres_endpoint.yml.tmpl",
   "postgres_project.yml.tmpl",

acceptance/bundle/invariant/destroy_idempotent/out.test.toml

@@ -42,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [
   "pipeline_config_dots.yml.tmpl",
   "postgres_branch.yml.tmpl",
   "postgres_catalog.yml.tmpl",
+  "postgres_data_api.yml.tmpl",
   "postgres_database.yml.tmpl",
   "postgres_endpoint.yml.tmpl",
   "postgres_project.yml.tmpl",

acceptance/bundle/invariant/migrate/test.toml

@@ -43,6 +43,8 @@ EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"]
 EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"]
 # Features are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_feature = ["INPUT_CONFIG=feature.yml.tmpl"]
+# The Postgres Data API is direct-only; the terraform deploy that seeds the migration fails for it.
+EnvMatrixExclude.no_postgres_data_api = ["INPUT_CONFIG=postgres_data_api.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/no_drift/out.test.toml

@@ -42,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [
   "pipeline_config_dots.yml.tmpl",
   "postgres_branch.yml.tmpl",
   "postgres_catalog.yml.tmpl",
+  "postgres_data_api.yml.tmpl",
   "postgres_database.yml.tmpl",
   "postgres_endpoint.yml.tmpl",
   "postgres_project.yml.tmpl",

acceptance/bundle/invariant/test.toml

@@ -56,6 +56,7 @@ EnvMatrix.INPUT_CONFIG = [
     "pipeline_config_dots.yml.tmpl",
     "postgres_branch.yml.tmpl",
     "postgres_catalog.yml.tmpl",
+    "postgres_data_api.yml.tmpl",
     "postgres_database.yml.tmpl",
     "postgres_endpoint.yml.tmpl",
     "postgres_project.yml.tmpl",
@@ -95,6 +96,7 @@ EnvMatrixExclude.no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CON
 EnvMatrixExclude.no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"]
 EnvMatrixExclude.no_postgres_role_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_role.yml.tmpl"]
 EnvMatrixExclude.no_postgres_database_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_database.yml.tmpl"]
+EnvMatrixExclude.no_postgres_data_api_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"]
 # Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP,
 # so exclude these from all cloud runs (same as the postgres resources above).
 EnvMatrixExclude.no_database_instance_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_instance.yml.tmpl"]
@@ -123,6 +125,8 @@ EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT
 # Deployment history has no "features." resource-key prefix, so the deploy fails before the feature
 # is created; see dmsResourceTypes in libs/testserver/bundledeployments.go.
 EnvMatrixExclude.no_feature_dms = ["DMS=true", "INPUT_CONFIG=feature.yml.tmpl"]
+# Same for the "postgres_data_apis." prefix.
+EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"]
 # Fake SQL endpoint for local tests
 
 [[Server]]

acceptance/bundle/refschema/output/postgres_data_apis.txt

@@ -0,0 +1,40 @@
+resources.postgres_data_apis.*.create_time	*time.Time	REMOTE
+resources.postgres_data_apis.*.db_aggregates_enabled	bool	ALL
+resources.postgres_data_apis.*.db_extra_search_path	[]string	ALL
+resources.postgres_data_apis.*.db_extra_search_path[*]	string	ALL
+resources.postgres_data_apis.*.db_max_rows	int	ALL
+resources.postgres_data_apis.*.db_schemas	[]string	ALL
+resources.postgres_data_apis.*.db_schemas[*]	string	ALL
+resources.postgres_data_apis.*.id	string	INPUT
+resources.postgres_data_apis.*.jwt_cache_max_lifetime	*duration.Duration	ALL
+resources.postgres_data_apis.*.jwt_role_claim_key	string	ALL
+resources.postgres_data_apis.*.lifecycle	resources.Lifecycle	INPUT
+resources.postgres_data_apis.*.lifecycle.depends_on	[]resources.DependsOn	INPUT
+resources.postgres_data_apis.*.lifecycle.depends_on[*]	resources.DependsOn	INPUT
+resources.postgres_data_apis.*.lifecycle.depends_on[*].resource	string	INPUT
+resources.postgres_data_apis.*.lifecycle.prevent_destroy	bool	INPUT
+resources.postgres_data_apis.*.modified_status	string	INPUT
+resources.postgres_data_apis.*.name	string	REMOTE
+resources.postgres_data_apis.*.openapi_mode	postgres.OpenApiMode	ALL
+resources.postgres_data_apis.*.parent	string	ALL
+resources.postgres_data_apis.*.server_cors_allowed_origins	[]string	ALL
+resources.postgres_data_apis.*.server_cors_allowed_origins[*]	string	ALL
+resources.postgres_data_apis.*.server_timing_enabled	bool	ALL
+resources.postgres_data_apis.*.status	*postgres.DataApiDataApiStatus	REMOTE
+resources.postgres_data_apis.*.status.available_schemas	[]string	REMOTE
+resources.postgres_data_apis.*.status.available_schemas[*]	string	REMOTE
+resources.postgres_data_apis.*.status.db_aggregates_enabled	bool	REMOTE
+resources.postgres_data_apis.*.status.db_extra_search_path	[]string	REMOTE
+resources.postgres_data_apis.*.status.db_extra_search_path[*]	string	REMOTE
+resources.postgres_data_apis.*.status.db_max_rows	int	REMOTE
+resources.postgres_data_apis.*.status.db_schemas	[]string	REMOTE
+resources.postgres_data_apis.*.status.db_schemas[*]	string	REMOTE
+resources.postgres_data_apis.*.status.jwt_cache_max_lifetime	*duration.Duration	REMOTE
+resources.postgres_data_apis.*.status.jwt_role_claim_key	string	REMOTE
+resources.postgres_data_apis.*.status.openapi_mode	postgres.OpenApiMode	REMOTE
+resources.postgres_data_apis.*.status.server_cors_allowed_origins	[]string	REMOTE
+resources.postgres_data_apis.*.status.server_cors_allowed_origins[*]	string	REMOTE
+resources.postgres_data_apis.*.status.server_timing_enabled	bool	REMOTE
+resources.postgres_data_apis.*.status.url	string	REMOTE
+resources.postgres_data_apis.*.update_time	*time.Time	REMOTE
+resources.postgres_data_apis.*.url	string	INPUT

acceptance/bundle/resources/postgres_data_apis/basic/databricks.yml.tmpl

@@ -0,0 +1,41 @@
+bundle:
+  name: deploy-pg-data-api-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Test Project for Data API"
+      pg_version: 16
+
+  # The Data API needs a read-write endpoint on the branch. Take over the
+  # "production" branch created with the project, which comes with one.
+  postgres_branches:
+    production:
+      parent: ${resources.postgres_projects.my_project.id}
+      branch_id: production
+      replace_existing: true
+      no_expiry: true
+
+  postgres_roles:
+    owner:
+      parent: ${resources.postgres_branches.production.id}
+      role_id: app-owner
+      postgres_role: app_owner
+
+  postgres_databases:
+    my_database:
+      parent: ${resources.postgres_branches.production.id}
+      database_id: my-database
+      postgres_database: app_db
+      role: ${resources.postgres_roles.owner.id}
+
+  postgres_data_apis:
+    my_data_api:
+      parent: ${resources.postgres_databases.my_database.name}
+      db_schemas:
+        - public
+      db_max_rows: 100