Fix Databricks provider version (#329)

* Upgrade Databricks provider and use account-level provider for metastore resources * Fix creating dfs private link * Revert accidental commit * Pass unity catalog config via spark secret * Fix fail on dbks secret unity catalog if none specified * Fix dev container for ARM --------- Co-authored-by: James Griffin <[email protected]>
SAFEHR-data · Sep 20, 2023 · 11c830e · 11c830e
1 parent 59a9fdd
commit 11c830e
Show file tree

Hide file tree

Showing 12 changed files with 107 additions and 96 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -36,10 +36,7 @@
 		"ghcr.io/stuartleeks/dev-container-features/shell-history:0": {},
 		"ghcr.io/devcontainers-contrib/features/pre-commit:2": {},
 		"ghcr.io/lukewiwa/features/shellcheck:0": {},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:1": {
-			"jqVersion": "1.6",
-			"yqVersion": "4"
-		},
+		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {},
 		"ghcr.io/jlaundry/devcontainer-features/mssql-odbc-driver:1": {
 			"version": "18"
 		},

diff --git a/apps/app/aad_app/.terraform.lock.hcl b/apps/app/aad_app/.terraform.lock.hcl
diff --git a/infrastructure/core/locals.tf b/infrastructure/core/locals.tf
@@ -24,15 +24,16 @@ locals {
   databricks_container_address_space = local.subnet_address_spaces[2]
   serve_webapps_address_space        = local.subnet_address_spaces[3]
 
-  create_dns_zones = var.private_dns_zones_rg == null
-  datalake_enabled = try(var.transform.datalake, null) != null
+  create_dns_zones      = var.private_dns_zones_rg == null
+  datalake_enabled      = try(var.transform.datalake, null) != null
+  unity_catalog_enabled = try(var.transform.unity_catalog, null) != null
   required_private_dns_zones = merge({
     blob       = "privatelink.blob.core.windows.net"
     keyvault   = "privatelink.vaultcore.azure.net"
     cosmos     = "privatelink.documents.azure.com"
     databricks = "privatelink.azuredatabricks.net"
     sql        = "privatelink.database.windows.net"
-    }, local.datalake_enabled ? {
+    }, local.datalake_enabled || local.unity_catalog_enabled ? {
     dfs = "privatelink.dfs.core.windows.net"
   } : {})
 }
diff --git a/infrastructure/transform/.terraform.lock.hcl b/infrastructure/transform/.terraform.lock.hcl
diff --git a/infrastructure/transform/databricks.tf b/infrastructure/transform/databricks.tf
@@ -127,6 +127,10 @@ resource "databricks_cluster" "cluster" {
     tomap({ for secret_name, secret_value in var.transform.databricks_secrets :
       "spark.secret.${secret_name}" => "{{secrets/${databricks_secret_scope.secrets.name}/${secret_name}}}"
     }),
+    local.unity_catalog_enabled ? tomap({
+      "spark.secret.unity-catalog-catalog-name" = "{{secrets/${databricks_secret_scope.secrets.name}/${databricks_secret.unity_catalog_catalog_name[0].key}}}"
+      "spark.secret.unity-catalog-schema-name"  = "{{secrets/${databricks_secret_scope.secrets.name}/${databricks_secret.unity_catalog_schema_name[0].key}}}"
+    }) : tomap({}),
     # Any values set in the config
     var.transform.spark_config,
     # Special config if in single node configuration

diff --git a/infrastructure/transform/datalake/.terraform.lock.hcl b/infrastructure/transform/datalake/.terraform.lock.hcl
diff --git a/infrastructure/transform/pipeline.tf b/infrastructure/transform/pipeline.tf
@@ -21,11 +21,7 @@ resource "azurerm_data_factory_pipeline" "pipeline" {
   parameters = merge(
     tomap({
       for param_name, param in each.value.properties.parameters : param_name => param.defaultValue
-    }),
-    local.unity_catalog_enabled ? tomap({
-      "catalog_name" = module.unity_catalog[0].catalog_name,
-      "schema_name"  = module.unity_catalog[0].schema_name
-  }) : tomap({}))
+  }))
 
   depends_on = [
     azurerm_data_factory_linked_service_azure_databricks.msi_linked

diff --git a/infrastructure/transform/secrets.tf b/infrastructure/transform/secrets.tf
@@ -92,3 +92,17 @@ resource "databricks_secret" "databricks_config_secret" {
   string_value = each.value
   scope        = databricks_secret_scope.secrets.id
 }
+
+resource "databricks_secret" "unity_catalog_catalog_name" {
+  count        = local.unity_catalog_enabled ? 1 : 0
+  key          = "unity-catalog-catalog-name"
+  string_value = module.unity_catalog[0].catalog_name
+  scope        = databricks_secret_scope.secrets.id
+}
+
+resource "databricks_secret" "unity_catalog_schema_name" {
+  count        = local.unity_catalog_enabled ? 1 : 0
+  key          = "unity-catalog-schema-name"
+  string_value = module.unity_catalog[0].schema_name
+  scope        = databricks_secret_scope.secrets.id
+}
diff --git a/infrastructure/transform/unity-catalog-metastore/.terraform.lock.hcl b/infrastructure/transform/unity-catalog-metastore/.terraform.lock.hcl
diff --git a/infrastructure/transform/unity-catalog/.terraform.lock.hcl b/infrastructure/transform/unity-catalog/.terraform.lock.hcl
diff --git a/infrastructure/transform/unity-catalog/workspace-assignment.tf b/infrastructure/transform/unity-catalog/workspace-assignment.tf
@@ -13,13 +13,13 @@
 #  limitations under the License.
 
 resource "databricks_metastore_assignment" "workspace_assignment" {
+  provider     = databricks.accounts
   workspace_id = data.azurerm_databricks_workspace.workspace.workspace_id
   metastore_id = var.metastore_id
-
-  depends_on = [databricks_metastore_data_access.metastore_data_access]
 }
 
 resource "databricks_metastore_data_access" "metastore_data_access" {
+  provider     = databricks.accounts
   metastore_id = var.metastore_id
   name         = "dbks-metastore-access-${var.naming_suffix}"
 
@@ -28,4 +28,6 @@ resource "databricks_metastore_data_access" "metastore_data_access" {
   }
 
   is_default = var.metastore_created
+
+  depends_on = [databricks_metastore_assignment.workspace_assignment]
 }
diff --git a/providers.hcl b/providers.hcl
@@ -60,7 +60,7 @@ EOF
   required_provider_databricks = <<EOF
 databricks = {
       source = "databricks/databricks"
-      version = "1.9.1"
+      version = "1.25.1"
       configuration_aliases = [ databricks.accounts ]
     }
 EOF