INTMDB-805: [Terraform] Create new TF Data Lake Pipelines Run data sources only (not resource)

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/mwielbut/pointy v1.1.0
 	github.com/spf13/cast v1.5.0
 	github.com/zclconf/go-cty v1.13.1
-	go.mongodb.org/atlas v0.26.2-0.20230518140002-aee9ff7710a0
+	go.mongodb.org/atlas v0.28.0
 	go.mongodb.org/realm v0.1.0
 	golang.org/x/exp v0.0.0-20221208152030-732eee02a75a
 )

go.sum

@@ -482,10 +482,8 @@ github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc
 github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 go.mongodb.org/atlas v0.12.0/go.mod h1:wVCnHcm/7/IfTjEB6K8K35PLG70yGz8BdkRwX0oK9/M=
-go.mongodb.org/atlas v0.26.2-0.20230517121323-88420f887c30 h1:bsNzJPOJ7T7An+6HLbyIyBioE9LnJneLqD63uBK+rpY=
-go.mongodb.org/atlas v0.26.2-0.20230517121323-88420f887c30/go.mod h1:L4BKwVx/OeEhOVjCSdgo90KJm4469iv7ZLzQms/EPTg=
-go.mongodb.org/atlas v0.26.2-0.20230518140002-aee9ff7710a0 h1:e6pAkoOnAOCjQHzRHX6RopDvRF09apl84Vpltfn0HYA=
-go.mongodb.org/atlas v0.26.2-0.20230518140002-aee9ff7710a0/go.mod h1:L4BKwVx/OeEhOVjCSdgo90KJm4469iv7ZLzQms/EPTg=
+go.mongodb.org/atlas v0.28.0 h1:CelAXtmiM36tdifSDwWdDH1nNbdvq0M2XfUR8208JxA=
+go.mongodb.org/atlas v0.28.0/go.mod h1:L4BKwVx/OeEhOVjCSdgo90KJm4469iv7ZLzQms/EPTg=
 go.mongodb.org/realm v0.1.0 h1:zJiXyLaZrznQ+Pz947ziSrDKUep39DO4SfA0Fzx8M4M=
 go.mongodb.org/realm v0.1.0/go.mod h1:4Vj6iy+Puo1TDERcoh4XZ+pjtwbOzPpzqy3Cwe8ZmDM=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

mongodbatlas/data_source_mongodbatlas_data_lake_pipeline_run.go

@@ -0,0 +1,168 @@
+package mongodbatlas
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	matlas "go.mongodb.org/atlas/mongodbatlas"
+)
+
+const errorDataLakePipelineRunRead = "error reading MongoDB Atlas DataLake Run (%s): %s"
+
+func dataSourceMongoDBAtlasDataLakePipelineRun() *schema.Resource {
+	return &schema.Resource{
+		ReadContext: dataSourceMongoDBAtlasDataLakeRunRead,
+		Schema: map[string]*schema.Schema{
+			"project_id": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"pipeline_name": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"pipeline_run_id": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"id": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"created_date": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"last_updated_date": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"state": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"dataset_name": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"phase": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"pipeline_id": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"snapshot_id": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"backup_frequency_type": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"stats": {
+				Type:     schema.TypeList,
+				Computed: true,
+				MaxItems: 0,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"bytes_exported": {
+							Type:     schema.TypeInt,
+							Computed: true,
+						},
+						"num_docs": {
+							Type:     schema.TypeInt,
+							Computed: true,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func dataSourceMongoDBAtlasDataLakeRunRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
+	conn := meta.(*MongoDBClient).Atlas
+	projectID := d.Get("project_id").(string)
+	name := d.Get("pipeline_name").(string)
+	pipelineRunID := d.Get("pipeline_run_id").(string)
+
+	dataLakeRun, resp, err := conn.DataLakePipeline.GetRun(ctx, projectID, name, pipelineRunID)
+	if err != nil {
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			d.SetId("")
+			return nil
+		}
+
+		return diag.FromErr(fmt.Errorf(errorDataLakePipelineRunRead, name, err))
+	}
+
+	if err := d.Set("id", dataLakeRun.ID); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "hostnames", name, err))
+	}
+
+	if err := d.Set("project_id", dataLakeRun.GroupID); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "state", name, err))
+	}
+
+	if err := d.Set("created_date", dataLakeRun.CreatedDate); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_databases", name, err))
+	}
+
+	if err := d.Set("last_updated_date", dataLakeRun.LastUpdatedDate); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_databases", name, err))
+	}
+
+	if err := d.Set("state", dataLakeRun.State); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_databases", name, err))
+	}
+
+	if err := d.Set("phase", dataLakeRun.Phase); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_databases", name, err))
+	}
+
+	if err := d.Set("pipeline_id", dataLakeRun.PipelineID); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_stores", name, err))
+	}
+
+	if err := d.Set("dataset_name", dataLakeRun.DatasetName); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_stores", name, err))
+	}
+
+	if err := d.Set("snapshot_id", dataLakeRun.SnapshotID); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_stores", name, err))
+	}
+
+	if err := d.Set("backup_frequency_type", dataLakeRun.BackupFrequencyType); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_stores", name, err))
+	}
+
+	if err := d.Set("stats", flattenDataLakePipelineRunStats(dataLakeRun.Stats)); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "storage_stores", name, err))
+	}
+
+	d.SetId(encodeStateID(map[string]string{
+		"project_id":      projectID,
+		"pipeline_name":   name,
+		"pipeline_run_id": pipelineRunID,
+	}))
+
+	return nil
+}
+
+func flattenDataLakePipelineRunStats(datalakeRunStats *matlas.DataLakePipelineRunStats) []map[string]interface{} {
+	if datalakeRunStats == nil {
+		return nil
+	}
+
+	maps := make([]map[string]interface{}, 1)
+	maps[0] = map[string]interface{}{
+		"bytes_exported": datalakeRunStats.BytesExported,
+		"num_docs":       datalakeRunStats.NumDocs,
+	}
+	return maps
+}

mongodbatlas/data_source_mongodbatlas_data_lake_pipeline_run_test.go

@@ -0,0 +1,49 @@
+package mongodbatlas
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+)
+
+func TestAccBackupDSDataLakePipelineRun_basic(t *testing.T) {
+	testCheckDataLakePipelineRun(t)
+	var (
+		dataSourceName = "data.mongodbatlas_data_lake_pipeline_run.test"
+		projectID      = os.Getenv("MONGODB_ATLAS_PROJECT_ID")
+		pipelineName   = os.Getenv("MONGODB_ATLAS_DATA_LAKE_PIPELINE_NAME")
+		runID          = os.Getenv("MONGODB_ATLAS_DATA_LAKE_PIPELINE_RUN_ID")
+	)
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:          func() { testAccPreCheck(t) },
+		ProviderFactories: testAccProviderFactories,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccMongoDBAtlasDataLakeDataSourcePipelineRunConfig(projectID, pipelineName, runID),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttrSet(dataSourceName, "project_id"),
+					resource.TestCheckResourceAttr(dataSourceName, "pipeline_name", pipelineName),
+					resource.TestCheckResourceAttrSet(dataSourceName, "id"),
+					resource.TestCheckResourceAttrSet(dataSourceName, "state"),
+					resource.TestCheckResourceAttrSet(dataSourceName, "phase"),
+					resource.TestCheckResourceAttrSet(dataSourceName, "pipeline_id"),
+					resource.TestCheckResourceAttrSet(dataSourceName, "dataset_name"),
+				),
+			},
+		},
+	})
+}
+
+func testAccMongoDBAtlasDataLakeDataSourcePipelineRunConfig(projectID, pipelineName, runID string) string {
+	return fmt.Sprintf(`
+
+data "mongodbatlas_data_lake_pipeline_run" "test" {
+  project_id           = %[1]q
+  pipeline_name        = %[2]q
+  pipeline_run_id      = %[3]q
+}
+	`, projectID, pipelineName, runID)
+}

mongodbatlas/data_source_mongodbatlas_data_lake_pipeline_runs.go

@@ -0,0 +1,134 @@
+package mongodbatlas
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/id"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	matlas "go.mongodb.org/atlas/mongodbatlas"
+)
+
+const errorDataLakePipelineRunList = "error reading MongoDB Atlas DataLake Runs (%s): %s"
+
+func dataSourceMongoDBAtlasDataLakePipelineRuns() *schema.Resource {
+	return &schema.Resource{
+		ReadContext: dataSourceMongoDBAtlasDataLakeRunsRead,
+		Schema: map[string]*schema.Schema{
+			"project_id": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"pipeline_name": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"results": {
+				Type:     schema.TypeList,
+				Computed: true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"pipeline_run_id": {
+							Type:     schema.TypeString,
+							Required: true,
+						},
+						"id": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"created_date": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"last_updated_date": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"state": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"phase": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"pipeline_id": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"snapshot_id": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"backup_frequency_type": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"stats": {
+							Type:     schema.TypeList,
+							Computed: true,
+							MaxItems: 0,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"bytes_exported": {
+										Type:     schema.TypeInt,
+										Computed: true,
+									},
+									"num_docs": {
+										Type:     schema.TypeInt,
+										Computed: true,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func dataSourceMongoDBAtlasDataLakeRunsRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
+	conn := meta.(*MongoDBClient).Atlas
+	projectID := d.Get("project_id").(string)
+	name := d.Get("pipeline_name").(string)
+
+	dataLakeRuns, _, err := conn.DataLakePipeline.ListRuns(ctx, projectID, name)
+	if err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakePipelineRunList, projectID, err))
+	}
+
+	if err := d.Set("results", flattenDataLakePipelineRunResult(dataLakeRuns.Results)); err != nil {
+		return diag.FromErr(fmt.Errorf(errorDataLakeSetting, "results", projectID, err))
+	}
+
+	d.SetId(id.UniqueId())
+
+	return nil
+}
+
+func flattenDataLakePipelineRunResult(datalakePipelineRuns []*matlas.DataLakePipelineRun) []map[string]interface{} {
+	var results []map[string]interface{}
+
+	if len(datalakePipelineRuns) == 0 {
+		return results
+	}
+
+	results = make([]map[string]interface{}, len(datalakePipelineRuns))
+
+	for k, run := range datalakePipelineRuns {
+		results[k] = map[string]interface{}{
+			"id":                    run.ID,
+			"created_date":          run.CreatedDate,
+			"last_updated_date":     run.LastUpdatedDate,
+			"state":                 run.State,
+			"pipeline_id":           run.PipelineID,
+			"snapshot_id":           run.SnapshotID,
+			"backup_frequency_type": run.BackupFrequencyType,
+			"stats":                 flattenDataLakePipelineRunStats(run.Stats),
+		}
+	}
+
+	return results
+}