Merge branch 'master' into master

Author: masoncusack

databricks/resource_databricks_azure_blob_mount.go

@@ -139,7 +139,8 @@ func resourceAzureBlobMountRead(d *schema.ResourceData, m interface{}) error {
 	url, err := blobMount.Read(client, clusterID)
 	if err != nil {
 		//Reset id in case of inability to find mount
-		if strings.Contains(err.Error(), "Unable to find mount point!") {
+		if strings.Contains(err.Error(), "Unable to find mount point!") ||
+			strings.Contains(err.Error(), fmt.Sprintf("/mnt/%s does not exist.", mountName)) {
 			d.SetId("")
 			return nil
 		}

databricks/resource_databricks_azure_blob_mount_test.go

@@ -0,0 +1,159 @@
+package databricks
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/databrickslabs/databricks-terraform/client/model"
+	"github.com/databrickslabs/databricks-terraform/client/service"
+	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
+	"github.com/hashicorp/terraform-plugin-sdk/terraform"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAccAzureBlobMount_correctly_mounts(t *testing.T) {
+	terraformToApply := testAccAzureBlobMount_correctly_mounts()
+	var clusterInfo model.ClusterInfo
+	var azureBlobMount AzureBlobMount
+
+	resource.Test(t, resource.TestCase{
+		Providers: testAccProviders,
+		Steps: []resource.TestStep{
+			{
+				Config: terraformToApply,
+				Check: resource.ComposeTestCheckFunc(
+					testAccAzureBlobMount_cluster_exists("databricks_cluster.cluster", &clusterInfo),
+					testAccAzureBlobMount_mount_exists("databricks_azure_blob_mount.mount", &azureBlobMount, &clusterInfo),
+				),
+			},
+			{
+				PreConfig: func() {
+					client := testAccProvider.Meta().(service.DBApiClient)
+					err := azureBlobMount.Delete(client, clusterInfo.ClusterID)
+					assert.NoError(t, err, "TestAccAzureBlobMount_correctly_mounts: Failed to remove the mount.")
+				},
+				Config: terraformToApply,
+				Check: resource.ComposeTestCheckFunc(
+					testAccAzureBlobMount_mount_exists("databricks_azure_blob_mount.mount", &azureBlobMount, &clusterInfo),
+				),
+			},
+		},
+	})
+}
+
+func testAccAzureBlobMount_cluster_exists(n string, clusterInfo *model.ClusterInfo) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		// find the corresponding state object
+		rs, ok := s.RootModule().Resources[n]
+		if !ok {
+			return fmt.Errorf("Not found: %s", n)
+		}
+
+		// retrieve the configured client from the test setup
+		client := testAccProvider.Meta().(service.DBApiClient)
+		resp, err := client.Clusters().Get(rs.Primary.ID)
+		if err != nil {
+			return err
+		}
+
+		// If no error, assign the response Widget attribute to the widget pointer
+		*clusterInfo = resp
+		return nil
+	}
+}
+
+func testAccAzureBlobMount_mount_exists(n string, azureBlobMount *AzureBlobMount, clusterInfo *model.ClusterInfo) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		// find the corresponding state object
+		rs, ok := s.RootModule().Resources[n]
+		if !ok {
+			return fmt.Errorf("Not found in tfstate: %s", n)
+		}
+
+		authType := rs.Primary.Attributes["auth_type"]
+		containerName := rs.Primary.Attributes["container_name"]
+		storageAccountName := rs.Primary.Attributes["storage_account_name"]
+		directory := rs.Primary.Attributes["directory"]
+		mountName := rs.Primary.Attributes["mount_name"]
+		tokenSecretScope := rs.Primary.Attributes["token_secret_scope"]
+		tokenSecretKey := rs.Primary.Attributes["token_secret_key"]
+
+		blobMount := NewAzureBlobMount(containerName, storageAccountName, directory, mountName, authType,
+			tokenSecretScope, tokenSecretKey)
+
+		client := testAccProvider.Meta().(service.DBApiClient)
+		cluster_id := clusterInfo.ClusterID
+
+		message, err := blobMount.Read(client, cluster_id)
+		if err != nil {
+			return fmt.Errorf("Error reading the mount %s: error %s", message, err)
+		}
+
+		*azureBlobMount = *blobMount
+		return nil
+
+	}
+}
+
+func testAccAzureBlobMount_correctly_mounts() string {
+	clientID := os.Getenv("ARM_CLIENT_ID")
+	clientSecret := os.Getenv("ARM_CLIENT_SECRET")
+	tenantID := os.Getenv("ARM_TENANT_ID")
+	subscriptionID := os.Getenv("ARM_SUBSCRIPTION_ID")
+	workspaceName := os.Getenv("TEST_WORKSPACE_NAME")
+	resourceGroupName := os.Getenv("TEST_RESOURCE_GROUP")
+	managedResourceGroupName := os.Getenv("TEST_MANAGED_RESOURCE_GROUP")
+	location := os.Getenv("TEST_LOCATION")
+	blobAccountKey := os.Getenv("TEST_STORAGE_ACCOUNT_KEY")
+	blobAccountName := os.Getenv("TEST_STORAGE_ACCOUNT_NAME")
+
+	definition := fmt.Sprintf(`
+	provider "databricks" {
+	  azure_auth = {
+		client_id              = "%[1]s"
+		client_secret          = "%[2]s"
+		tenant_id              = "%[3]s"
+		subscription_id        = "%[4]s"
+
+		workspace_name         = "%[5]s"
+		resource_group         = "%[6]s"
+		managed_resource_group = "%[7]s"
+		azure_region           = "%[8]s"
+	  }
+	}
+
+	resource "databricks_cluster" "cluster" {
+		num_workers = 1
+		spark_version = "6.4.x-scala2.11"
+		node_type_id = "Standard_D3_v2"
+		# Don't spend too much, turn off cluster after 15mins
+		autotermination_minutes = 15
+	} 
+
+	resource "databricks_secret_scope" "terraform" {
+	  # Add the cluster ID into the secret scope to ensure 
+	  # it doesn't clash with one used by another test
+	  name                     = "terraform${databricks_cluster.cluster.cluster_id}"
+	  initial_manage_principal = "users"
+	}
+	
+	resource "databricks_secret" "storage_key" {
+	  key          = "blob_storage_key"
+	  string_value = "%[10]s"
+	  scope        = databricks_secret_scope.terraform.name
+	}
+	
+	resource "databricks_azure_blob_mount" "mount" {
+		cluster_id           = databricks_cluster.cluster.id
+		container_name       = "dev" # Created by prereqs.tf
+		storage_account_name = "%[9]s"
+		mount_name           = "dev"
+		auth_type            = "ACCESS_KEY"
+		token_secret_scope   = databricks_secret_scope.terraform.name
+		token_secret_key     = databricks_secret.storage_key.key
+	}
+
+`, clientID, clientSecret, tenantID, subscriptionID, workspaceName, resourceGroupName, managedResourceGroupName, location, blobAccountName, blobAccountKey)
+	return definition
+}

integration-environment-azure/prereqs.tf

@@ -26,8 +26,8 @@ resource "azurerm_databricks_workspace" "example" {
   managed_resource_group_name = "workspace${random_string.naming.result}"
 }
 
-# Create container in storage acc and container for use by mount tests
-resource "azurerm_storage_account" "account" {
+# Create container in storage acc and container for use by adls gen2 mount tests
+resource "azurerm_storage_account" "adlsaccount" {
   name                     = "${random_string.naming.result}datalake"
   resource_group_name      = azurerm_resource_group.example.name
   location                 = azurerm_resource_group.example.location
@@ -37,19 +37,35 @@ resource "azurerm_storage_account" "account" {
   is_hns_enabled           = "true"
 }
 
+# Create container in storage acc and container for use by blob mount tests
+resource "azurerm_storage_account" "blobaccount" {
+  name                     = "${random_string.naming.result}blob"
+  resource_group_name      = azurerm_resource_group.example.name
+  location                 = azurerm_resource_group.example.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  account_kind             = "StorageV2"
+}
+
 data "azurerm_client_config" "current" {
 }
 
 resource "azurerm_role_assignment" "datalake" {
-  scope = azurerm_storage_account.account.id
+  scope = azurerm_storage_account.adlsaccount.id
   #https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor
   role_definition_name = "Storage Blob Data Contributor"
   principal_id         = data.azurerm_client_config.current.object_id
 }
 
-resource "azurerm_storage_container" "example" {
+resource "azurerm_storage_container" "adlsexample" {
+  name                  = "dev"
+  storage_account_name  = azurerm_storage_account.adlsaccount.name
+  container_access_type = "private"
+}
+
+resource "azurerm_storage_container" "blobexample" {
   name                  = "dev"
-  storage_account_name  = azurerm_storage_account.account.name
+  storage_account_name  = azurerm_storage_account.blobaccount.name
   container_access_type = "private"
 }
 
@@ -62,13 +78,22 @@ output "workspace_name" {
 }
 
 output "gen2_adal_name" {
-  value = azurerm_storage_account.account.name
+  value = azurerm_storage_account.adlsaccount.name
 }
 
 output "location" {
-  value = azurerm_storage_account.account.location
+  value = azurerm_storage_account.adlsaccount.location
 }
 
 output "rg_name" {
   value = azurerm_resource_group.example.name
 }
+
+output "blob_storage_key" {
+  value     = azurerm_storage_account.blobaccount.primary_access_key
+  sensitive = true
+}
+
+output "blob_storage_name" {
+  value = azurerm_storage_account.blobaccount.name
+}

integration-environment-azure/run.sh

@@ -37,7 +37,8 @@ export TEST_WORKSPACE_NAME=$(terraform output workspace_name)
 export TEST_GEN2_ADAL_NAME=$(terraform output gen2_adal_name)
 export TEST_MANAGED_RESOURCE_GROUP=$(terraform output workspace_managed_rg_name)
 export TEST_LOCATION=$(terraform output location)
-
+export TEST_STORAGE_ACCOUNT_KEY=$(terraform output blob_storage_key)
+export TEST_STORAGE_ACCOUNT_NAME=$(terraform output blob_storage_name)
 
 echo -e "----> Running Azure Acceptance Tests \n\n"
 # Output debug log to file while tests run