Merge pull request #3153 from jingyih/bqcc

feat: allow user-specified connection ID in BigQueryConnection Connection

Author: yuwenma

apis/bigqueryconnection/v1alpha1/connection_types.go

@@ -38,8 +38,10 @@ type Parent struct {
 type BigQueryConnectionConnectionSpec struct {
 	Parent `json:",inline"`
 
-	// The BigQuery ConnectionID. This is a server-generated ID in the UUID format.
-	// If not provided, ConfigConnector will create a new Connection and store the UUID in `status.serviceGeneratedID` field.
+	// Immutable. Optional.
+	// The BigQuery Connection ID used for resource creation or acquisition.
+	// For creation: If specified, this value is used as the connection ID. If not provided, a UUID will be generated and assigned as the connection ID.
+	// For acquisition: This field must be provided to identify the connection resource to acquire.
 	ResourceID *string `json:"resourceID,omitempty"`
 
 	// User provided display name for the connection.

apis/bigqueryconnection/v1beta1/connection_reference.go

@@ -22,7 +22,6 @@ import (
 	refsv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
-	"github.com/google/uuid"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
@@ -44,46 +43,33 @@ func NewBigQueryConnectionConnectionRef(ctx context.Context, reader client.Reade
 	// Get location
 	location := obj.Spec.Location
 
-	// Get desired service-generated ID from spec
-	desiredServiceID := direct.ValueOf(obj.Spec.ResourceID)
-	if desiredServiceID != "" {
-		if _, err := uuid.Parse(desiredServiceID); err != nil {
-			return nil, fmt.Errorf("spec.resourceID should be in a UUID format, got %s ", desiredServiceID)
-		}
-	}
+	// Get desired connection ID from spec
+	desiredID := direct.ValueOf(obj.Spec.ResourceID)
 
-	// Get externalReference
+	// Validate status.externalRef
 	externalRef := direct.ValueOf(obj.Status.ExternalRef)
 	if externalRef != "" {
-		tokens := strings.Split(externalRef, "/")
-
-		if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "connections" {
-			return nil, fmt.Errorf("externalRef should be projects/<project>/locations/<location>/connections/<Connection>, got %s", externalRef)
+		actualProject, actualLocation, actualID, err := parseExternal(externalRef)
+		if err != nil {
+			return nil, err
 		}
-		id.parent = "projects/" + tokens[1] + "/locations/" + tokens[3]
-
 		// Validate spec parent and resourceID field if the resource is already reconcilied with a GCP Connection resource.
-		if tokens[1] != projectID {
-			return nil, fmt.Errorf("spec.projectRef changed, expect %s, got %s",
-				tokens[1], projectID)
+		if projectID != actualProject {
+			return nil, fmt.Errorf("spec.projectRef changed, expect %s, got %s", projectID, actualProject)
 		}
-		if tokens[3] != location {
-			return nil, fmt.Errorf("spec.location changed, expect %s, got %s",
-				tokens[3], location)
+		if location != actualLocation {
+			return nil, fmt.Errorf("spec.location changed, expect %s, got %s", location, actualLocation)
 		}
-		if desiredServiceID != "" && tokens[5] != desiredServiceID {
-			// Service generated ID shall not be reset in the same BigQueryConnectionConnection.
+		if desiredID != "" && desiredID != actualID {
+			// Connection ID shall not be reset in the same BigQueryConnectionConnection.
 			// TODO: what if multiple BigQueryConnectionConnection points to the same GCP Connection?
 			return nil, fmt.Errorf("cannot reset `spec.resourceID` to %s, since it has already acquired the Connection %s",
-				desiredServiceID, tokens[5])
+				desiredID, actualID)
 		}
 		id.External = externalRef
 		return id, nil
 	}
-	id.parent = "projects/" + projectID + "/locations/" + location
-	if desiredServiceID != "" {
-		id.External = id.parent + "/connections/" + desiredServiceID
-	}
+	id.External = "projects/" + projectID + "/locations/" + location + "/connections/" + desiredID
 	return id, nil
 }
 
@@ -100,26 +86,40 @@ type BigQueryConnectionConnectionRef struct {
 	Name string `json:"name,omitempty"`
 	// The `namespace` of a `BigQueryConnectionConnection` resource.
 	Namespace string `json:"namespace,omitempty"`
+}
 
-	parent string
+func parseExternal(external string) (string, string, string, error) {
+	tokens := strings.Split(external, "/")
+	if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "connections" {
+		return "", "", "", fmt.Errorf("external should be projects/<project>/locations/<location>/connections/<Connection>, got %s", external)
+	}
+	return tokens[1], tokens[3], tokens[5], nil
 }
 
 func (r *BigQueryConnectionConnectionRef) Parent() (string, error) {
-	if r.parent != "" {
-		return r.parent, nil
-	}
 	if r.External != "" {
 		r.External = strings.TrimPrefix(r.External, "/")
-		tokens := strings.Split(r.External, "/")
-		if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "connections" {
-			return "", fmt.Errorf("format of BigQueryConnectionConnection external=%q was not known (use projects/<projectId>/locations/<location>/connections/<connectionID>)", r.External)
+		project, location, _, err := parseExternal(r.External)
+		if err != nil {
+			return "", err
 		}
-		r.parent = "projects/" + tokens[1] + "/locations/" + tokens[3]
-		return r.parent, nil
+		return "projects/" + project + "/locations/" + location, nil
 	}
 	return "", fmt.Errorf("BigQueryConnectionConnectionRef not normalized to External form or not created from `New()`")
 }
 
+// ConnectionID returns the connection ID, a boolean indicating whether the connection ID is specified by user (or generated by service), and an error.
+func (r *BigQueryConnectionConnectionRef) ConnectionID() (string, bool, error) {
+	if r.External != "" {
+		_, _, id, err := parseExternal(r.External)
+		if err != nil {
+			return "", false, err
+		}
+		return id, id != "", nil
+	}
+	return "", false, fmt.Errorf("BigQueryConnectionConnectionRef not normalized to External form or not created from `New()`")
+}
+
 // NormalizedExternal provision the "External" value.
 // If the "External" comes from the ConfigConnector object, it has to acquire or reconcile with the GCP resource already.
 func (r *BigQueryConnectionConnectionRef) NormalizedExternal(ctx context.Context, reader client.Reader, othernamespace string) (string, error) {

apis/bigqueryconnection/v1beta1/connection_types.go

@@ -38,8 +38,10 @@ type Parent struct {
 type BigQueryConnectionConnectionSpec struct {
 	Parent `json:",inline"`
 
-	// The BigQuery ConnectionID. This is a server-generated ID in the UUID format.
-	// If not provided, ConfigConnector will create a new Connection and store the UUID in `status.serviceGeneratedID` field.
+	// Immutable. Optional.
+	// The BigQuery Connection ID used for resource creation or acquisition.
+	// For creation: If specified, this value is used as the connection ID. If not provided, a UUID will be generated and assigned as the connection ID.
+	// For acquisition: This field must be provided to identify the connection resource to acquire.
 	ResourceID *string `json:"resourceID,omitempty"`
 
 	// User provided display name for the connection.

config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigqueryconnectionconnections.bigqueryconnection.cnrm.cloud.google.com.yaml

@@ -305,10 +305,12 @@ spec:
                     type: string
                 type: object
               resourceID:
-                description: The BigQuery ConnectionID. This is a server-generated
-                  ID in the UUID format. If not provided, ConfigConnector will create
-                  a new Connection and store the UUID in `status.serviceGeneratedID`
-                  field.
+                description: 'Immutable. Optional. The BigQuery Connection ID used
+                  for resource creation or acquisition. For creation: If specified,
+                  this value is used as the connection ID. If not provided, a UUID
+                  will be generated and assigned as the connection ID. For acquisition:
+                  This field must be provided to identify the connection resource
+                  to acquire.'
                 type: string
               spark:
                 description: Spark properties.
@@ -800,10 +802,12 @@ spec:
                     type: string
                 type: object
               resourceID:
-                description: The BigQuery ConnectionID. This is a server-generated
-                  ID in the UUID format. If not provided, ConfigConnector will create
-                  a new Connection and store the UUID in `status.serviceGeneratedID`
-                  field.
+                description: 'Immutable. Optional. The BigQuery Connection ID used
+                  for resource creation or acquisition. For creation: If specified,
+                  this value is used as the connection ID. If not provided, a UUID
+                  will be generated and assigned as the connection ID. For acquisition:
+                  This field must be provided to identify the connection resource
+                  to acquire.'
                 type: string
               spark:
                 description: Spark properties.

pkg/clients/generated/apis/bigqueryconnection/v1beta1/bigqueryconnectionconnection_types.go

@@ -176,8 +176,11 @@ func (a *Adapter) Find(ctx context.Context) (bool, error) {
 
 	log.V(2).Info("getting BigQueryConnectionConnection", "name", a.id.External)
 
-	if a.id.External == "" {
-		// Cannot retrieve the Connection without ServiceGeneratedID, expecting to create a new Connection.
+	_, idIsSet, err := a.id.ConnectionID()
+	if err != nil {
+		return false, err
+	}
+	if !idIsSet { // resource is not yet created
 		return false, nil
 	}
 	req := &bigqueryconnectionpb.GetConnectionRequest{Name: a.id.External}
@@ -211,12 +214,23 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati
 
 	parent, err := a.id.Parent()
 	if err != nil {
-		return fmt.Errorf("get BigQueryConnectionConnection parent %s: %w", a.id.External, err)
+		return err
 	}
 	req := &bigqueryconnectionpb.CreateConnectionRequest{
 		Parent:     parent,
 		Connection: resource,
 	}
+	id, isIsSet, err := a.id.ConnectionID()
+	if err != nil {
+		return err
+	}
+	if isIsSet { // during "Create", this means user has specified connection ID in `spec.ResourceID` field.
+		req = &bigqueryconnectionpb.CreateConnectionRequest{
+			Parent:       parent,
+			ConnectionId: id,
+			Connection:   resource,
+		}
+	}
 	created, err := a.gcpClient.CreateConnection(ctx, req)
 	if err != nil {
 		return fmt.Errorf("creating Connection %s: %w", a.id.External, err)

pkg/controller/direct/bigqueryconnection/connection_controller.go

@@ -0,0 +1,56 @@
+GET https://bigqueryconnection.googleapis.com/v1/projects/${projectId}/locations/us-central1/connections/test-bqcc-acquisition?%24alt=json%3Benum-encoding%3Dint
+Content-Type: application/json
+User-Agent: kcc/controller-manager
+x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2Fconnections%2Ftest-bqcc-acquisition
+
+404 Not Found
+Cache-Control: private
+Content-Type: application/json; charset=UTF-8
+Server: ESF
+Vary: Origin
+Vary: X-Origin
+Vary: Referer
+X-Content-Type-Options: nosniff
+X-Frame-Options: SAMEORIGIN
+X-Xss-Protection: 0
+
+{
+  "error": {
+    "code": 404,
+    "message": "Not found: Connection projects/${projectNumber}/locations/us-central1/connections/test-bqcc-acquisition",
+    "status": "NOT_FOUND"
+  }
+}
+
+---
+
+POST https://bigqueryconnection.googleapis.com/v1/projects/${projectId}/locations/us-central1/connections?%24alt=json%3Benum-encoding%3Dint&connectionId=test-bqcc-acquisition
+Content-Type: application/json
+User-Agent: kcc/controller-manager
+x-goog-request-params: parent=projects%2F${projectId}%2Flocations%2Fus-central1
+
+{
+  "cloudResource": {},
+  "description": "BigQueryConnection Connection resource for acquisition"
+}
+
+200 OK
+Cache-Control: private
+Content-Type: application/json; charset=UTF-8
+Server: ESF
+Vary: Origin
+Vary: X-Origin
+Vary: Referer
+X-Content-Type-Options: nosniff
+X-Frame-Options: SAMEORIGIN
+X-Xss-Protection: 0
+
+{
+  "cloudResource": {
+    "serviceAccountId": "bqcx-projects/${projectId}/locations/[email protected]"
+  },
+  "creationTime": "1731379730",
+  "description": "BigQueryConnection Connection resource for acquisition",
+  "lastModifiedTime": "1731379730",
+  "name": "projects/${projectNumber}/locations/us-central1/connections/test-bqcc-acquisition"
+}