[connector/signaltometrics] Introduce optional config for attributes

.chloggen/optional-signaltometrics.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: signaltometricsconnector
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow to configure `optional` attributes which are copied as-is to the output metric.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [38600]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

connector/signaltometricsconnector/README.md

@@ -146,12 +146,38 @@ attributes:
   - key: datapoint.foo
   - key: datapoint.bar
     default_value: bar
+  - key: datapoint.baz
+    optional: true
 ```
 
 If attributes are specified then a separate metric will be generated for each unique
-set of attribute values. Optionally, a `default_value` can be used to always include
-the attribute with the value of the attribute defaulting to the value specified in
-`default_value` if the incoming data is missing that attribute.
+set of attribute values. There are three behaviors that can be configured for an
+attribute:
+
+- Without any extra parameters: `datapoint.foo` in the above yaml is an example
+  of such configuration. In this configuration, only the signals which have the said
+  attribute are processed with the attribute's value as one of the attributes for
+  the output metric. If the attribute is missing then the signal is not processed.
+- With `default_value`: `datapoint.bar` in the above yaml is an example of such
+  configuration. In this configuration all the signals are processed irrespective
+  of the attribute being present or not in the input signal. The output metric
+  is categorized as per the incoming value of the attribute and an extra bucket
+  exists with the attribute set to the configured default value for all the signals
+  that were missing the configured attribute.
+- With `optional` set to `true`: `datapoint.baz` in the above yaml is an example
+  of such configuration. If the attribute is configured with `optional` and present
+  in the incoming signal then it will be added directly to the output metric. If
+  it is absent then a new metric with missing attributes will be created. In addition,
+  the `optional` attribute will not impact the decision i.e. even if the `optional`
+  attributes are not present in the incoming signal, the signal will be processed
+  and will produce a metric given all other non-optional attributes are present
+  or have a default value defined.
+
+Note that resource attributes are handled differently, check the resource attributes
+section for more details on this. Think of `attributes` as conditional filters for
+choosing which attributes should be included in the output metric whereas
+`include_resource_attributes` is an include list for customizing resource attributes
+of the output metric.
 
 ### Conditions
 
@@ -201,16 +227,23 @@ include_resource_attributes:
   - key: resource.foo # Include resource.foo attribute if present
   - key: resource.bar # Always include resource.bar attribute, default to bar
     default_value: bar
+  - key: resource.baz # Optional resource.baz attribute is added if present
+    optional: true
 ```
 
-With the above configuration the produced metrics would only have the couple of
-resource attributes specified in the list:
+With the above configuration the produced metrics would have the following
+resource attributes:
 
 - `resource.foo` will be present for the produced metrics if the incoming data also
-  has the attribute defined.
+  has the attribute defined. If the attribute is missing in the incoming data the
+  output metric will be produced without the said attribute.
 - `resource.bar` will always be present because of the `default_value`. If the incoming
   data does not have a resource attribute with name `resource.bar` then the configured
   `default_value` of `bar` will be used.
+- `resource.baz` will behave exactly same as `resource.foo`. Since resource attributes
+  are basically an include list, the `optional` option is a no-op i.e. the resource
+  attributes with `optional` set to `true` behaves identical to an attribute configured
+  without `default_value` or `optional`.
 
 ### Single writer
 

connector/signaltometricsconnector/config/config.go

@@ -123,6 +123,7 @@ func (c *Config) Unmarshal(collectorCfg *confmap.Conf) error {
 
 type Attribute struct {
 	Key          string `mapstructure:"key"`
+	Optional     bool   `mapstructure:"optional"`
 	DefaultValue any    `mapstructure:"default_value"`
 }
 
@@ -185,6 +186,9 @@ func (mi *MetricInfo) validateAttributes() error {
 		if attr.Key == "" {
 			return fmt.Errorf("attribute key missing")
 		}
+		if attr.DefaultValue != nil && attr.Optional {
+			return errors.New("only one of default_value or optional should be set")
+		}
 		if _, ok := duplicate[attr.Key]; ok {
 			return fmt.Errorf("duplicate key found in attributes config: %s", attr.Key)
 		}

connector/signaltometricsconnector/config/config_test.go

@@ -51,6 +51,14 @@ func TestConfig(t *testing.T) {
 				fullErrorForSignal(t, "logs", "attributes validation failed"),
 			},
 		},
+		{
+			path: "with_optional_and_default_value",
+			errorMsgs: []string{
+				fullErrorForSignal(t, "spans", "attributes validation failed: only one of default_value or optional should be set"),
+				fullErrorForSignal(t, "datapoints", "attributes validation failed: only one of default_value or optional should be set"),
+				fullErrorForSignal(t, "logs", "attributes validation failed: only one of default_value or optional should be set"),
+			},
+		},
 		{
 			path: "invalid_histogram",
 			errorMsgs: []string{
@@ -108,7 +116,10 @@ func TestConfig(t *testing.T) {
 						Description:               "Exponential histogram",
 						Unit:                      "us",
 						IncludeResourceAttributes: []Attribute{{Key: "key.1", DefaultValue: "foo"}},
-						Attributes:                []Attribute{{Key: "key.2", DefaultValue: "bar"}},
+						Attributes: []Attribute{
+							{Key: "key.2", DefaultValue: "bar"},
+							{Key: "key.3", Optional: true},
+						},
 						Conditions: []string{
 							`attributes["some.optional.1"] != nil`,
 							`resource.attributes["some.optional.2"] != nil`,
@@ -124,7 +135,10 @@ func TestConfig(t *testing.T) {
 						Description:               "Histogram",
 						Unit:                      "us",
 						IncludeResourceAttributes: []Attribute{{Key: "key.1", DefaultValue: "foo"}},
-						Attributes:                []Attribute{{Key: "key.2", DefaultValue: "bar"}},
+						Attributes: []Attribute{
+							{Key: "key.2", DefaultValue: "bar"},
+							{Key: "key.3", Optional: true},
+						},
 						Conditions: []string{
 							`attributes["some.optional.1"] != nil`,
 							`resource.attributes["some.optional.2"] != nil`,
@@ -142,7 +156,10 @@ func TestConfig(t *testing.T) {
 						Description:               "Sum",
 						Unit:                      "ms",
 						IncludeResourceAttributes: []Attribute{{Key: "key.1", DefaultValue: "foo"}},
-						Attributes:                []Attribute{{Key: "key.2", DefaultValue: "bar"}},
+						Attributes: []Attribute{
+							{Key: "key.2", DefaultValue: "bar"},
+							{Key: "key.3", Optional: true},
+						},
 						Conditions: []string{
 							`attributes["some.optional.1"] != nil`,
 							`IsDouble(attributes["some.optional.1"])`,
@@ -158,7 +175,10 @@ func TestConfig(t *testing.T) {
 						Description:               "Sum",
 						Unit:                      "1",
 						IncludeResourceAttributes: []Attribute{{Key: "key.1", DefaultValue: "foo"}},
-						Attributes:                []Attribute{{Key: "key.2", DefaultValue: "bar"}},
+						Attributes: []Attribute{
+							{Key: "key.2", DefaultValue: "bar"},
+							{Key: "key.3", Optional: true},
+						},
 						Conditions: []string{
 							`attributes["some.optional.1"] != nil`,
 						},

connector/signaltometricsconnector/go.mod

@@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/connector/signa
 go 1.23.0
 
 require (
+	github.com/google/go-cmp v0.7.0
 	github.com/lightstep/go-expohisto v1.0.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.123.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.123.0

connector/signaltometricsconnector/internal/model/model.go

@@ -16,6 +16,7 @@ import (
 
 type AttributeKeyValue struct {
 	Key          string
+	Optional     bool
 	DefaultValue pcommon.Value
 }
 
@@ -199,8 +200,7 @@ func (md *MetricDef[K]) FilterResourceAttributes(
 func (md *MetricDef[K]) FilterAttributes(attrs pcommon.Map) (pcommon.Map, bool) {
 	// Figure out if all the attributes are available, saves allocation
 	for _, filter := range md.Attributes {
-		if filter.DefaultValue.Type() != pcommon.ValueTypeEmpty {
-			// will always add an attribute
+		if filter.DefaultValue.Type() != pcommon.ValueTypeEmpty || filter.Optional {
 			continue
 		}
 		if _, ok := attrs.Get(filter.Key); !ok {
@@ -235,6 +235,7 @@ func parseAttributeConfigs(cfgs []config.Attribute) ([]AttributeKeyValue, error)
 		}
 		kvs[i] = AttributeKeyValue{
 			Key:          attr.Key,
+			Optional:     attr.Optional,
 			DefaultValue: val,
 		}
 	}