Supports aws_db_instance (RDS) (#81)

Author: obierlaire

README.md

@@ -27,11 +27,11 @@ It can estimate Carbon Emissions of:
 - Amazon Web Services
   - [x] EC2 (including inline root, elastic, and ephemeral block storages)
   - [x] EBS Volumes
+  - [x] RDS
 
 The following will also be supported soon:
 
 - Amazon Web Services
-  - [ ] RDS
   - [ ] AutoScaling Group
 - Azure
   - [ ] Virtual Machines

doc/scope.md

@@ -43,13 +43,15 @@ Data resources:
 |---|---|---|
 | `aws_instance`| No GPU | |
 | `aws_ebs_volume`| if size set, or if snapshot declared as data resource | |
+| `aws_db_instance` | | |
 
 Data resources:
 
 | Resource | Limitations  | Comment |
 |---|---|---|
 | `aws_ami`| `ebs.volume_size` can be set, otherwise get it from image only if AWS credentials are provided| |
 | `aws_ebs_snapshot`| `volume_size` can be set, otherwise get it from image only if AWS credentials are provided| |
+| `aws_db_snapshot`|  get it only if AWS credentials are provided| |
 
 
 _more to be implemented_

internal/plan/defaults.go

@@ -9,7 +9,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 )
 
+var defaultRegion *string
+
 func getDefaultRegion() *string {
+	if defaultRegion != nil {
+		return defaultRegion
+	}
 	var region interface{}
 	if region == nil {
 		if os.Getenv("AWS_DEFAULT_REGION") != "" {

internal/plan/json_getters.go

@@ -85,7 +85,7 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 	for _, pathRaw := range paths {
 		path := pathRaw
 		if strings.Contains(pathRaw, "${") {
-			path, err = resolvePlaceholders(path, context.ParentContext)
+			path, err = resolvePlaceholders(pathRaw, context.ParentContext)
 			if err != nil {
 				return nil, err
 			}
@@ -98,7 +98,7 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 		if len(jsonResults) == 0 && TfPlan != nil {
 			jsonResults, err = utils.GetJSON(path, *TfPlan)
 			if err != nil {
-				return nil, errors.Wrapf(err, "Cannot get item: %v", path)
+				return nil, errors.Wrapf(err, "Cannot get item in full plan: %v", path)
 			}
 		}
 		for _, jsonResultsI := range jsonResults {
@@ -108,7 +108,9 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 				if err != nil {
 					return nil, err
 				}
-				results = append(results, result)
+				if result != nil {
+					results = append(results, result)
+				}
 			case []interface{}:
 				for _, jsonResultI := range jsonResults {
 					jsonResultI, ok := jsonResultI.(map[string]interface{})
@@ -119,7 +121,9 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 					if err != nil {
 						return nil, err
 					}
-					results = append(results, result)
+					if result != nil {
+						results = append(results, result)
+					}
 				}
 			default:
 				return nil, errors.Errorf("Not an map or an array of maps: %T", jsonResultsI)
@@ -146,7 +150,9 @@ func getItem(context tfContext, itemMappingProperties *ResourceMapping, jsonResu
 		if err != nil {
 			return nil, err
 		}
-		result[key] = property
+		if property != nil {
+			result[key] = property
+		}
 	}
 	return result, nil
 }
@@ -198,7 +204,7 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 	for _, propertyMapping := range propertiesMappings {
 		paths, err := readPaths(propertyMapping.Paths)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "Cannot get paths for %v", context.ResourceAddress)
 		}
 		unit := propertyMapping.Unit
 
@@ -208,25 +214,27 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 			}
 			path := pathRaw
 			if strings.Contains(pathRaw, "${") {
+				fmt.Println("pathRaw: ", pathRaw)
 				path, err = resolvePlaceholders(path, context)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrapf(err, "Cannot resolve placeholders for %v", path)
 				}
+				fmt.Println("path: ", path)
 			}
 			valueFounds, err := utils.GetJSON(path, context.Resource)
 			if err != nil {
-				return nil, err
+				return nil, errors.Wrapf(err, "Cannot get value for %v", path)
 			}
 			if len(valueFounds) == 0 && TfPlan != nil {
 				// Try to resolve it against the whole plan
 				valueFounds, err = utils.GetJSON(path, *TfPlan)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrapf(err, "Cannot get value in the whole plan for %v", path)
 				}
 			}
 			if len(valueFounds) > 0 {
 				if len(valueFounds) > 1 {
-					return nil, fmt.Errorf("Found more than one value for property %v of resource type %v", key, context.ResourceAddress)
+					return nil, errors.Errorf("Found more than one value for property %v of resource type %v", key, context.ResourceAddress)
 				}
 				if valueFounds[0] == nil {
 					continue
@@ -240,14 +248,14 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 			if ok {
 				valueFound, err = applyRegex(valueFoundStr, &propertyMapping, context)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrapf(err, "Cannot apply regex for %v", valueFoundStr)
 				}
 			}
 			valueFoundStr, ok = valueFound.(string)
 			if ok {
 				valueFound, err = applyReference(valueFoundStr, &propertyMapping, context)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrapf(err, "Cannot apply reference for %v", valueFoundStr)
 				}
 			}
 		}
@@ -257,7 +265,7 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 		if ok {
 			valueFound, err = getValueOfExpression(valueFoundMap, context)
 			if err != nil {
-				return nil, err
+				return nil, errors.Wrapf(err, "Cannot get value of expression for %v", valueFoundMap)
 			}
 		}
 
@@ -272,7 +280,7 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 	if valueFound == nil {
 		defaultValue, err := getDefaultValue(key, context)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "Cannot get default value for %v", key)
 		}
 
 		if defaultValue != nil {
@@ -303,7 +311,13 @@ func resolvePlaceholders(input string, context *tfContext) (string, error) {
 		if err != nil {
 			return input, err
 		}
-		resolvedExpressions[placeholder] = resolved
+		if resolved == nil {
+			// It's ok to not find a value for a placeholder
+			resolvedExpressions[placeholder] = ".not_found"
+		} else {
+			resolvedExpressions[placeholder] = *resolved
+		}
+
 	}
 
 	// Replace placeholders in the input string with resolved expressions
@@ -317,32 +331,37 @@ func resolvePlaceholders(input string, context *tfContext) (string, error) {
 	return resolvedString, nil
 }
 
-func resolvePlaceholder(expression string, context *tfContext) (string, error) {
-	result := ""
+func resolvePlaceholder(expression string, context *tfContext) (*string, error) {
 	if strings.HasPrefix(expression, "this.") {
 		thisProperty := strings.TrimPrefix(expression, "this")
 		resource := context.Resource
 		value, err := utils.GetJSON(thisProperty, resource)
 		if err != nil {
-			return "", errors.Wrapf(err, "Cannot get value for variable %s", expression)
+			return nil, errors.Wrapf(err, "Cannot get value for variable %s", expression)
 		}
-		if value == nil {
-			return "", errors.Errorf("No value found for variable %s", expression)
+		if len(value) == 0 {
+			return nil, nil
+		}
+		if value[0] == nil {
+			return nil, nil
 		}
-		return fmt.Sprintf("%v", value[0]), err
+		valueStr := fmt.Sprintf("%v", value[0])
+		return &valueStr, err
 	} else if strings.HasPrefix(expression, "config.") {
 		configProperty := strings.TrimPrefix(expression, "config.")
 		value := viper.GetFloat64(configProperty)
-		return fmt.Sprintf("%v", value), nil
+		valueStr := fmt.Sprintf("%v", value)
+		return &valueStr, nil
 	}
 	variable, err := getVariable(expression, context)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	if variable != nil {
-		result = fmt.Sprintf("%v", variable)
+		valueStr := fmt.Sprintf("%v", variable)
+		return &valueStr, nil
 	}
-	return result, nil
+	return nil, nil
 }
 
 func getDefaultValue(key string, context *tfContext) (*valueWithUnit, error) {
@@ -400,10 +419,10 @@ func getVariable(name string, context *tfContext) (interface{}, error) {
 	}
 	value, err := getValue(name, &variableContext)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "Cannot get variable %v", name)
 	}
 	if value == nil {
-		return nil, fmt.Errorf("Cannot get variable : %v", name)
+		return nil, nil
 	}
 	return value.Value, nil
 

internal/plan/mapping.go

@@ -13,7 +13,7 @@ import (
 var globalMappings *Mappings
 
 // GetMapping returns the mapping of the terraform resources
-func getMapping() (*Mappings, error) {
+func GetMapping() (*Mappings, error) {
 	if globalMappings != nil {
 		return globalMappings, nil
 	}

internal/plan/mappings/aws/general.yaml

@@ -13,4 +13,5 @@ general:
     json_data:
       aws_instances : "aws_instances.json"
     ignored_resources: 
-      - "aws_vpc"
+      - "aws_vpc"
+      - "aws_volume_attachment"

internal/plan/mappings/aws/rds_instance.yaml

@@ -0,0 +1,75 @@
+compute_resource:
+  aws_db_instance: 
+    paths: 
+      - .planned_values.root_module.resources[] | select(.type == "aws_db_instance")
+    type: resource
+    variables:
+      properties:
+        replicate_source_db:
+          - paths:
+            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.replicate_source_db?.references[]? | select(endswith("id")) | gsub("\\.id$"; "")'
+            reference:
+              paths:
+                - .planned_values.root_module.resources[] | select(.address == "${key}")
+                - .planned_values.root_module.child_modules[] | select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
+                - .prior_state.values.root_module.resources[] | select(.address == "${key}")
+              return_path: true
+    properties:
+      name:
+        - paths: ".name"
+      address:
+        - paths: ".address"
+      type:
+        - paths: ".type"
+      zone:
+        - paths: ".values.availability_zone"
+      region:
+        - paths: ".values.availability_zone"
+          regex:
+            pattern: '^(.+-\d+)[a-z]+'
+            group: 1
+        - paths: ".configuration.provider_config.aws.expressions.region"
+      replication_factor:
+        - paths: '.values| if .multi_az then 2 else 1 end'
+      vCPUs:
+        - paths: ".values.instance_class"
+          regex:
+            pattern: '^db\.(.+)'
+            group: 1
+          reference:
+            json_file: aws_instances
+            property: "VCPU"
+      memory:
+        - paths: ".values.instance_class"
+          unit: mb
+          regex:
+            pattern: '^db\.(.+)'
+            group: 1
+          reference:
+            json_file: aws_instances
+            property: "MemoryMb"
+      storage:
+        - type: list
+          item:
+            - paths: 
+              - '.values | select(.allocated_storage)'
+              - '${replicate_source_db}.values | select(.allocated_storage)'
+              properties:
+                size:
+                  - paths: ".allocated_storage"
+                    unit: gb
+                type:
+                  - paths: ".storage_type"
+                    default: gp2
+                    reference:
+                      general: disk_types
+            - paths: '.prior_state.values.root_module.resources[] | select(.values.db_snapshot_identifier == "${this.values.snapshot_identifier}")'
+              properties:
+                size:
+                  - paths: ".values.allocated_storage"
+                    unit: gb
+                type:
+                  - paths: "values.storage_type"
+                    default: gp2
+                    reference:
+                      general: disk_types

internal/plan/resources.go

@@ -34,7 +34,7 @@ func GetResources(tfplan *map[string]interface{}) (map[string]resources.Resource
 	resourcesMap := map[string]resources.Resource{}
 
 	// Get compute resources
-	mapping, err := getMapping()
+	mapping, err := GetMapping()
 	if err != nil {
 		errW := errors.Wrap(err, "Cannot get mapping")
 		return nil, errW
@@ -118,7 +118,7 @@ func getResourcesOfType(resourceType string, mapping *ResourceMapping) ([]resour
 		}
 		log.Debugf("  Found %d resources of type '%s'", len(resourcesFound), resourceType)
 		for _, resourceI := range resourcesFound {
-			resourcesResultGot, err := getComputeResource(resourceI, mapping, resourcesResult)
+			resourcesResultGot, err := GetComputeResource(resourceI, mapping, resourcesResult)
 			if err != nil {
 				errW := errors.Wrapf(err, "Cannot get compute resource for path %v", path)
 				return nil, errW
@@ -133,7 +133,7 @@ func getResourcesOfType(resourceType string, mapping *ResourceMapping) ([]resour
 
 }
 
-func getComputeResource(resourceI interface{}, resourceMapping *ResourceMapping, resourcesResult []resources.Resource) ([]resources.Resource, error) {
+func GetComputeResource(resourceI interface{}, resourceMapping *ResourceMapping, resourcesResult []resources.Resource) ([]resources.Resource, error) {
 	resource := resourceI.(map[string]interface{})
 	resourceAddress := resource["address"].(string)
 	providerName, ok := resource["provider_name"].(string)
@@ -293,6 +293,9 @@ func getComputeResource(resourceI interface{}, resourceMapping *ResourceMapping,
 	}
 
 	for i, storageI := range storages {
+		if storageI == nil {
+			continue
+		}
 		storage, err := getStorage(storageI.(map[string]interface{}))
 		if err != nil {
 			return nil, errors.Wrapf(err, "Cannot get storage[%v] for %v", i, resourceAddress)
@@ -331,7 +334,19 @@ func getGPU(gpu map[string]interface{}) ([]string, error) {
 }
 
 func getStorage(storageMap map[string]interface{}) (*storage, error) {
-	storageSize := storageMap["size"].(*valueWithUnit)
+	storageSize, ok := storageMap["size"].(*valueWithUnit)
+	if !ok {
+		// It can happen there is no size but type as been set by default. In this case, we ignore the storage
+		// Can be fixed in the mapping by selecting only if the size property is set (example '.values | select(.allocated_storage)')
+		log.Warnf("Cannot find storage size in storageMap '%v': %T", storageMap, storageMap)
+		return nil, nil
+	}
+	if storageSize == nil {
+		return nil, errors.Errorf("Storage size is nil '%v': %T", storageSize, storageSize)
+	}
+	if storageSize.Value == nil {
+		return nil, errors.Errorf("Storage size value is nil '%v': %T", storageSize, storageSize)
+	}
 	storageSizeGb, err := decimal.NewFromString(fmt.Sprintf("%v", storageSize.Value))
 	if err != nil {
 		log.Fatal(err)