Skip to content

Commit fe1f43a

Browse files
stikkireddystikkireddy
authored
Merge pull request #88 from databrickslabs/multiple-workspace-apis
multiple workspaces api implementation
2 parents 6f6fc0b + 9b3ea1b commit fe1f43a

File tree

49 files changed

+3577
-177
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+3577
-177
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -329,3 +329,4 @@ website/public/**
329329

330330
.vscode/private.env
331331
tf.log
332+
*.env

‎Makefile

+6-20
Original file line numberDiff line numberDiff line change
@@ -39,25 +39,6 @@ fmt: lint
3939
@echo "==> Formatting source code with gofmt..."
4040
@go fmt ./...
4141

42-
43-
python-setup:
44-
@echo "==> Setting up virtual env and installing python libraries..."
45-
@python -m pip install virtualenv
46-
@cd docs && python -m virtualenv venv
47-
@cd docs && source venv/bin/activate && python -m pip install -r requirements.txt
48-
49-
docs: python-setup
50-
@echo "==> Building Docs ..."
51-
@cd docs && source venv/bin/activate && make clean && make html
52-
53-
opendocs: python-setup docs
54-
@echo "==> Opening Docs ..."
55-
@cd docs && open build/html/index.html
56-
57-
singlehtmldocs: python-setup
58-
@echo "==> Building Docs ..."
59-
@cd docs && source venv/bin/activate && make clean && make singlehtml
60-
6142
vendor:
6243
@echo "==> Filling vendor folder with library code..."
6344
@go mod vendor
@@ -70,7 +51,12 @@ terraform-acc-azure: fmt
7051
# INTEGRATION TESTING WITH AWS
7152
terraform-acc-aws: fmt
7253
@echo "==> Running Terraform Acceptance Tests for AWS..."
73-
@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -tags=aws -short -coverprofile=coverage.out ./...
54+
@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -short -coverprofile=coverage.out -run 'TestAccAws' ./...
55+
56+
# INTEGRATION TESTING WITH AWS
57+
terraform-acc-mws: fmt
58+
@echo "==> Running Terraform Acceptance Tests for Multiple Workspace APIs on AWS..."
59+
@/bin/bash integration-environment-mws/run.sh
7460

7561
terraform-setup: build
7662
@echo "==> Initializing Terraform..."

‎client/model/mws.go

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package model
2+
3+
// StsRole is the object that contains cross account role arn and external app id
4+
type StsRole struct {
5+
RoleArn string `json:"role_arn,omitempty"`
6+
ExternalID string `json:"external_id,omitempty"`
7+
}
8+
9+
// AwsCredentials is the object that points to the cross account role
10+
type AwsCredentials struct {
11+
StsRole *StsRole `json:"sts_role,omitempty"`
12+
}
13+
14+
// MWSCredentials is the object that contains all the information for the credentials to create a workspace
15+
type MWSCredentials struct {
16+
CredentialsID string `json:"credentials_id,omitempty"`
17+
CredentialsName string `json:"credentials_name,omitempty"`
18+
AwsCredentials *AwsCredentials `json:"aws_credentials,omitempty"`
19+
AccountID string `json:"account_id,omitempty"`
20+
CreationTime int64 `json:"creation_time,omitempty"`
21+
}
22+
23+
// RootBucketInfo points to a bucket name
24+
type RootBucketInfo struct {
25+
BucketName string `json:"bucket_name,omitempty"`
26+
}
27+
28+
// MWSStorageConfigurations is the object that contains all the information for the root storage bucket
29+
type MWSStorageConfigurations struct {
30+
StorageConfigurationID string `json:"storage_configuration_id,omitempty"`
31+
StorageConfigurationName string `json:"storage_configuration_name,omitempty"`
32+
RootBucketInfo *RootBucketInfo `json:"root_bucket_info,omitempty"`
33+
AccountID string `json:"account_id,omitempty"`
34+
CreationTime int64 `json:"creation_time,omitempty"`
35+
}
36+
37+
// NetworkHealth is the object that contains all the error message when attaching a network to workspace
38+
type NetworkHealth struct {
39+
ErrorType string `json:"error_type,omitempty"`
40+
ErrorMessage string `json:"error_message,omitempty"`
41+
}
42+
43+
// MWSNetwork is the object that contains all the information for BYOVPC
44+
type MWSNetwork struct {
45+
NetworkID string `json:"network_id,omitempty"`
46+
NetworkName string `json:"network_name,omitempty"`
47+
VPCID string `json:"vpc_id,omitempty"`
48+
SubnetIds []string `json:"subnet_ids,omitempty"`
49+
SecurityGroupIds []string `json:"security_group_ids,omitempty"`
50+
VPCStatus string `json:"vpc_status,omitempty"`
51+
ErrorMessages []NetworkHealth `json:"error_messages,omitempty"`
52+
WorkspaceID int64 `json:"workspace_id,omitempty"`
53+
AccountID string `json:"account_id,omitempty"`
54+
CreationTime int64 `json:"creation_time,omitempty"`
55+
}
56+
57+
// AwsKeyInfo has information about the KMS key for BYOK
58+
type AwsKeyInfo struct {
59+
KeyArn string `json:"key_arn,omitempty"`
60+
KeyAlias string `json:"key_alias,omitempty"`
61+
KeyRegion string `json:"key_region,omitempty"`
62+
}
63+
64+
// MWSCustomerManagedKey contains key information and metadata for BYOK for E2
65+
type MWSCustomerManagedKey struct {
66+
CustomerManagedKeyID string `json:"customer_managed_key_id,omitempty"`
67+
AwsKeyInfo *AwsKeyInfo `json:"aws_key_info,omitempty"`
68+
AccountID string `json:"account_id,omitempty"`
69+
CreationTime int64 `json:"creation_time,omitempty"`
70+
}
71+
72+
// List of workspace statuses for provisioning the workspace
73+
const (
74+
WorkspaceStatusNotProvisioned = "NOT_PROVISIONED"
75+
WorkspaceStatusProvisioning = "PROVISIONING"
76+
WorkspaceStatusRunning = "RUNNING"
77+
WorkspaceStatusFailed = "FAILED"
78+
WorkspaceStatusCanceled = "CANCELLED"
79+
)
80+
81+
// WorkspaceStatusesNonRunnable is a list of statuses in which the workspace is not runnable
82+
var WorkspaceStatusesNonRunnable = []string{WorkspaceStatusCanceled, WorkspaceStatusFailed}
83+
84+
// ContainsWorkspaceState given a list of workspaceStates and the search state
85+
// it will return true if it found the search state
86+
func ContainsWorkspaceState(workspaceStates []string, searchState string) bool {
87+
for _, state := range workspaceStates {
88+
if state == searchState {
89+
return true
90+
}
91+
}
92+
return false
93+
}
94+
95+
// MWSWorkspace is the object that contains all the information for deploying a E2 workspace
96+
type MWSWorkspace struct {
97+
WorkspaceID int64 `json:"workspace_id,omitempty"`
98+
WorkspaceName string `json:"workspace_name,omitempty"`
99+
DeploymentName string `json:"deployment_name,omitempty"`
100+
AwsRegion string `json:"aws_region,omitempty"`
101+
CredentialsID string `json:"credentials_id,omitempty"`
102+
StorageConfigurationID string `json:"storage_configuration_id,omitempty"`
103+
NetworkID string `json:"network_id,omitempty"`
104+
CustomerManagedKeyID string `json:"customer_managed_key_id,omitempty"`
105+
IsNoPublicIpEnabled bool `json:"is_no_public_ip_enabled,omitempty"`
106+
AccountID string `json:"account_id,omitempty"`
107+
WorkspaceStatus string `json:"workspace_status,omitempty"`
108+
WorkspaceStatusMessage string `json:"workspace_status_message,omitempty"`
109+
CreationTime int64 `json:"creation_time,omitempty"`
110+
}

‎client/service/apis.go

+25
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,31 @@ func (c *DBApiClient) Commands() CommandsAPI {
8686
return CommandsAPI{Client: c}
8787
}
8888

89+
// MWSCredentials returns an instance of MWSCredentialsAPI
90+
func (c *DBApiClient) MWSCredentials() MWSCredentialsAPI {
91+
return MWSCredentialsAPI{Client: c}
92+
}
93+
94+
// MWSStorageConfigurations returns an instance of MWSStorageConfigurationsAPI
95+
func (c *DBApiClient) MWSStorageConfigurations() MWSStorageConfigurationsAPI {
96+
return MWSStorageConfigurationsAPI{Client: c}
97+
}
98+
99+
// MWSWorkspaces returns an instance of MWSWorkspacesAPI
100+
func (c *DBApiClient) MWSWorkspaces() MWSWorkspacesAPI {
101+
return MWSWorkspacesAPI{Client: c}
102+
}
103+
104+
// MWSNetworks returns an instance of MWSNetworksAPI
105+
func (c *DBApiClient) MWSNetworks() MWSNetworksAPI {
106+
return MWSNetworksAPI{Client: c}
107+
}
108+
109+
// MWSCustomerManagedKeys returns an instance of MWSCustomerManagedKeysAPI
110+
func (c *DBApiClient) MWSCustomerManagedKeys() MWSCustomerManagedKeysAPI {
111+
return MWSCustomerManagedKeysAPI{Client: c}
112+
}
113+
89114
func (c *DBApiClient) performQuery(method, path string, apiVersion string, headers map[string]string, data interface{}, secretsMask *SecretsMask) ([]byte, error) {
90115
err := c.Config.getOrCreateToken()
91116
if err != nil {

‎client/service/main_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package service
22

33
import (
4+
"encoding/base64"
45
"encoding/json"
56
"fmt"
67
"github.com/joho/godotenv"
@@ -51,6 +52,18 @@ func GetIntegrationDBAPIClient() *DBApiClient {
5152
return &c
5253
}
5354

55+
func GetIntegrationMWSAPIClient() *DBApiClient {
56+
var config DBApiClientConfig
57+
tokenUnB64 := fmt.Sprintf("%s:%s", os.Getenv("DATABRICKS_USERNAME"), os.Getenv("DATABRICKS_PASSWORD"))
58+
config.AuthType = BasicAuth
59+
config.Token = base64.StdEncoding.EncodeToString([]byte(tokenUnB64))
60+
config.Host = os.Getenv("DATABRICKS_MWS_HOST")
61+
62+
var c DBApiClient
63+
c.SetConfig(&config)
64+
return &c
65+
}
66+
5467
func GetCloudInstanceType(c *DBApiClient) string {
5568
if strings.Contains(c.Config.Host, "azure") {
5669
return "Standard_DS3_v2"

‎client/service/mws_credentials.go

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package service
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"github.com/databrickslabs/databricks-terraform/client/model"
7+
"net/http"
8+
)
9+
10+
// MWSCredentialsAPI exposes the mws credentials API
11+
type MWSCredentialsAPI struct {
12+
Client *DBApiClient
13+
}
14+
15+
// Create creates a set of MWS Credentials for the cross account role
16+
func (a MWSCredentialsAPI) Create(mwsAcctId, credentialsName string, roleArn string) (model.MWSCredentials, error) {
17+
var mwsCreds model.MWSCredentials
18+
19+
credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials", mwsAcctId)
20+
21+
mwsCredentialsRequest := model.MWSCredentials{
22+
CredentialsName: credentialsName,
23+
AwsCredentials: &model.AwsCredentials{
24+
StsRole: &model.StsRole{
25+
RoleArn: roleArn,
26+
},
27+
},
28+
}
29+
30+
resp, err := a.Client.performQuery(http.MethodPost, credentialsAPIPath, "2.0", nil, mwsCredentialsRequest, nil)
31+
if err != nil {
32+
return mwsCreds, err
33+
}
34+
35+
err = json.Unmarshal(resp, &mwsCreds)
36+
return mwsCreds, err
37+
}
38+
39+
// Read returns the credentials object along with metadata
40+
func (a MWSCredentialsAPI) Read(mwsAcctId, credentialsID string) (model.MWSCredentials, error) {
41+
var mwsCreds model.MWSCredentials
42+
43+
credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials/%s", mwsAcctId, credentialsID)
44+
45+
resp, err := a.Client.performQuery(http.MethodGet, credentialsAPIPath, "2.0", nil, nil, nil)
46+
if err != nil {
47+
return mwsCreds, err
48+
}
49+
50+
err = json.Unmarshal(resp, &mwsCreds)
51+
return mwsCreds, err
52+
}
53+
54+
// Delete deletes the credentials object given a credentials id
55+
func (a MWSCredentialsAPI) Delete(mwsAcctId, credentialsID string) error {
56+
57+
credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials/%s", mwsAcctId, credentialsID)
58+
59+
_, err := a.Client.performQuery(http.MethodDelete, credentialsAPIPath, "2.0", nil, nil, nil)
60+
61+
return err
62+
}
63+
64+
// List lists all the available credentials object in the mws account
65+
func (a MWSCredentialsAPI) List(mwsAcctId string) ([]model.MWSCredentials, error) {
66+
var mwsCredsList []model.MWSCredentials
67+
68+
credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials", mwsAcctId)
69+
70+
resp, err := a.Client.performQuery(http.MethodGet, credentialsAPIPath, "2.0", nil, nil, nil)
71+
if err != nil {
72+
return mwsCredsList, err
73+
}
74+
75+
err = json.Unmarshal(resp, &mwsCredsList)
76+
return mwsCredsList, err
77+
}

‎client/service/mws_credentials_integration_test.go

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package service
2+
3+
import (
4+
"github.com/stretchr/testify/assert"
5+
"os"
6+
"testing"
7+
)
8+
9+
func TestMWSCreds(t *testing.T) {
10+
if testing.Short() {
11+
t.Skip("skipping integration test in short mode.")
12+
}
13+
acctId := os.Getenv("DATABRICKS_MWS_ACCT_ID")
14+
client := GetIntegrationMWSAPIClient()
15+
credsList, err := client.MWSCredentials().List(acctId)
16+
assert.NoError(t, err, err)
17+
t.Log(credsList)
18+
19+
myCreds, err := client.MWSCredentials().Create(acctId, "sri-mws-terraform-automation-role", "arn:aws:iam::997819999999:role/sri-e2-terraform-automation-role")
20+
assert.NoError(t, err, err)
21+
22+
myCredsFull, err := client.MWSCredentials().Read(acctId, myCreds.CredentialsID)
23+
assert.NoError(t, err, err)
24+
t.Log(myCredsFull.AwsCredentials.StsRole.ExternalID)
25+
26+
defer func() {
27+
err = client.MWSCredentials().Delete(acctId, myCreds.CredentialsID)
28+
assert.NoError(t, err, err)
29+
}()
30+
}

‎client/service/mws_customer_managed_keys.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package service
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"fmt"
7+
"github.com/databrickslabs/databricks-terraform/client/model"
8+
"net/http"
9+
)
10+
11+
// MWSCustomerManagedKeysAPI exposes the mws customerManagedKeys API
12+
type MWSCustomerManagedKeysAPI struct {
13+
Client *DBApiClient
14+
}
15+
16+
// Create creates a set of MWS CustomerManagedKeys for the BYOVPC
17+
func (a MWSCustomerManagedKeysAPI) Create(mwsAcctId, keyArn, keyAlias, keyRegion string) (model.MWSCustomerManagedKey, error) {
18+
var mwsCustomerManagedKey model.MWSCustomerManagedKey
19+
20+
customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys", mwsAcctId)
21+
mwsCustomerManagedKeysRequest := model.MWSCustomerManagedKey{
22+
AwsKeyInfo: &model.AwsKeyInfo{
23+
KeyArn: keyArn,
24+
KeyAlias: keyAlias,
25+
KeyRegion: keyRegion,
26+
},
27+
}
28+
resp, err := a.Client.performQuery(http.MethodPost, customerManagedKeysAPIPath, "2.0", nil, mwsCustomerManagedKeysRequest, nil)
29+
if err != nil {
30+
return mwsCustomerManagedKey, err
31+
}
32+
err = json.Unmarshal(resp, &mwsCustomerManagedKey)
33+
return mwsCustomerManagedKey, err
34+
}
35+
36+
// Read returns the customer managed key object along with metadata
37+
func (a MWSCustomerManagedKeysAPI) Read(mwsAcctId, customerManagedKeysID string) (model.MWSCustomerManagedKey, error) {
38+
var mwsCustomerManagedKey model.MWSCustomerManagedKey
39+
customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys/%s", mwsAcctId, customerManagedKeysID)
40+
resp, err := a.Client.performQuery(http.MethodGet, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
41+
if err != nil {
42+
return mwsCustomerManagedKey, err
43+
}
44+
err = json.Unmarshal(resp, &mwsCustomerManagedKey)
45+
return mwsCustomerManagedKey, err
46+
}
47+
48+
// Delete deletes the customer managed key object given a network id
49+
func (a MWSCustomerManagedKeysAPI) Delete(customerManagedKeysID string) error {
50+
//customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys/%s", a.Client.Config.E2AcctID, customerManagedKeysID)
51+
//_, err := a.Client.performQuery(http.MethodDelete, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
52+
//return err
53+
return errors.New("delete is not yet supported")
54+
}
55+
56+
// List lists all the available customer managed key objects in the mws account
57+
func (a MWSCustomerManagedKeysAPI) List(mwsAcctId string) ([]model.MWSCustomerManagedKey, error) {
58+
var mwsCustomerManagedKeyList []model.MWSCustomerManagedKey
59+
60+
customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys", mwsAcctId)
61+
62+
resp, err := a.Client.performQuery(http.MethodGet, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
63+
if err != nil {
64+
return mwsCustomerManagedKeyList, err
65+
}
66+
67+
err = json.Unmarshal(resp, &mwsCustomerManagedKeyList)
68+
return mwsCustomerManagedKeyList, err
69+
}

0 commit comments

Comments
 (0)