Push host metadata on startup (#65)

* Push host metadata on startup

* Have the same behavior as the Datadog Agent with tags
Only env should be added to host tags if set.

* Apply suggestions from code review

Co-authored-by: Albert Vaca Cintora <[email protected]>

* Run on separate goroutine and add exponential backoff

* Set User-Agent for zorkian's client and for the host metadata

* Disable Fast Fallback

Co-authored-by: Albert Vaca Cintora <[email protected]>

Author: mx-psi

exporter/datadogexporter/config.go

@@ -98,23 +98,11 @@ type TagsConfig struct {
 }
 
 // GetTags gets the default tags extracted from the configuration
-func (t *TagsConfig) GetTags(addHost bool) []string {
-	tags := make([]string, 0, 4)
+func (t *TagsConfig) GetTags() []string {
+	tags := make([]string, 0, len(t.Tags)+1)
 
-	vars := map[string]string{
-		"env":     t.Env,
-		"service": t.Service,
-		"version": t.Version,
-	}
-
-	if addHost {
-		vars["host"] = t.Hostname
-	}
-
-	for name, val := range vars {
-		if val != "" {
-			tags = append(tags, fmt.Sprintf("%s:%s", name, val))
-		}
+	if t.Env != "none" {
+		tags = append(tags, fmt.Sprintf("env:%s", t.Env))
 	}
 
 	tags = append(tags, t.Tags...)

exporter/datadogexporter/config_test.go

@@ -84,24 +84,27 @@ func TestLoadConfig(t *testing.T) {
 
 func TestTags(t *testing.T) {
 	tc := TagsConfig{
-		Hostname: "customhost",
-		Env:      "customenv",
-		Service:  "customservice",
-		Version:  "customversion",
-		Tags:     []string{"key1:val1", "key2:val2"},
+		// environment should be picked up if it is not 'none'
+		Env: "customenv",
+
+		// these should be ignored;
+		// they are used only on trace translation
+		Service: "customservice",
+		Version: "customversion",
+		Tags:    []string{"key1:val1", "key2:val2"},
 	}
 
 	assert.ElementsMatch(t,
 		[]string{
-			"host:customhost",
 			"env:customenv",
-			"service:customservice",
-			"version:customversion",
 			"key1:val1",
 			"key2:val2",
 		},
-		tc.GetTags(true), // get host
+		tc.GetTags(),
 	)
+
+	tc.Env = "none"
+	assert.ElementsMatch(t, tc.GetTags(), tc.Tags)
 }
 
 // TestOverrideMetricsURL tests that the metrics URL is overridden

exporter/datadogexporter/factory.go

@@ -15,11 +15,13 @@ package datadogexporter
 
 import (
 	"context"
+	"time"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/configmodels"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
+	"go.uber.org/zap"
 )
 
 const (
@@ -28,6 +30,9 @@ const (
 
 	// DefaultSite is the default site of the Datadog intake to send data to
 	DefaultSite = "datadoghq.com"
+
+	// maxRetries is the maximum number of retries for pushing host metadata
+	maxRetries = 5
 )
 
 // NewFactory creates a Datadog exporter factory
@@ -86,6 +91,31 @@ func createMetricsExporter(
 		return nil, err
 	}
 
+	go func() {
+		// Send host metadata
+		var sent bool
+		wait := 1 * time.Second
+		metadata := getHostMetadata(cfg)
+		for i := 0; i < maxRetries; i++ {
+			err := exp.pushHostMetadata(metadata)
+			if err != nil {
+				params.Logger.Warn("Sending host metadata failed", zap.Error(err))
+			} else {
+				sent = true
+				params.Logger.Info("Sent host metadata", zap.Int("numRetries", i))
+				break
+			}
+
+			time.Sleep(wait)
+			wait = 2 * wait
+		}
+
+		if !sent {
+			// log and continue without metadata
+			params.Logger.Error("Could not send host metadata", zap.Int("numRetries", maxRetries))
+		}
+	}()
+
 	return exporterhelper.NewMetricsExporter(
 		cfg,
 		exp.PushMetricsData,

exporter/datadogexporter/host.go

@@ -14,7 +14,19 @@
 
 package datadogexporter
 
-import "os"
+import (
+	"fmt"
+	"os"
+)
+
+const (
+	opentelemetryFlavor  = "opentelemetry-collector"
+	opentelemetryVersion = "alpha"
+)
+
+var (
+	userAgent = fmt.Sprintf("%s/%s", opentelemetryFlavor, opentelemetryVersion)
+)
 
 // GetHost gets the hostname according to configuration.
 // It gets the configuration hostname and if
@@ -30,3 +42,63 @@ func GetHost(cfg *Config) *string {
 	}
 	return &host
 }
+
+// hostMetadata includes metadata about the host tags,
+// host aliases and identifies the host as an OpenTelemetry host
+type hostMetadata struct {
+	// Meta includes metadata about the host.
+	Meta *meta `json:"meta"`
+
+	// InternalHostname is the canonical hostname
+	InternalHostname string `json:"internalHostname"`
+
+	// Version is the OpenTelemetry Collector version.
+	// This is used for correctly identifying the Collector in the backend,
+	// and for telemetry purposes.
+	Version string `json:"otel_version"`
+
+	// Flavor is always set to "opentelemetry-collector".
+	// It is used for telemetry purposes in the backend.
+	Flavor string `json:"agent-flavor"`
+
+	// Tags includes the host tags
+	Tags *hostTags `json:"host-tags"`
+}
+
+// hostTags are the host tags.
+// Currently only system (configuration) tags are considered.
+type hostTags struct {
+	// System are host tags set in the configuration
+	System []string `json:"system,omitempty"`
+}
+
+// meta includes metadata about the host aliases
+type meta struct {
+	// InstanceID is the EC2 instance id the Collector is running on, if available
+	InstanceID string `json:"instance-id,omitempty"`
+
+	// EC2Hostname is the hostname from the EC2 metadata API
+	EC2Hostname string `json:"ec2-hostname,omitempty"`
+
+	// Hostname is the canonical hostname
+	Hostname string `json:"hostname"`
+
+	// SocketHostname is the OS hostname
+	SocketHostname string `json:"socket-hostname"`
+
+	// HostAliases are other available host names
+	HostAliases []string `json:"host-aliases,omitempty"`
+}
+
+func getHostMetadata(cfg *Config) hostMetadata {
+	host := *GetHost(cfg)
+	return hostMetadata{
+		InternalHostname: host,
+		Flavor:           opentelemetryFlavor,
+		Version:          opentelemetryVersion,
+		Tags:             &hostTags{cfg.TagsConfig.GetTags()},
+		Meta: &meta{
+			Hostname: host,
+		},
+	}
+}

exporter/datadogexporter/metrics_exporter.go

@@ -15,7 +15,13 @@
 package datadogexporter
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
 
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.uber.org/zap"
@@ -26,23 +32,65 @@ type metricsExporter struct {
 	logger *zap.Logger
 	cfg    *Config
 	client *datadog.Client
-	tags   []string
+}
+
+func newHTTPClient() *http.Client {
+	return &http.Client{
+		Timeout: 20 * time.Second,
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				// Disable RFC 6555 Fast Fallback ("Happy Eyeballs")
+				FallbackDelay: -1 * time.Nanosecond,
+			}).DialContext,
+			MaxIdleConns: 100,
+			// Not supported by intake
+			ForceAttemptHTTP2: false,
+		},
+	}
 }
 
 func newMetricsExporter(logger *zap.Logger, cfg *Config) (*metricsExporter, error) {
 	client := datadog.NewClient(cfg.API.Key, "")
+	client.ExtraHeader["User-Agent"] = userAgent
 	client.SetBaseUrl(cfg.Metrics.TCPAddr.Endpoint)
+	client.HttpClient = newHTTPClient()
 
-	// Calculate tags at startup
-	tags := cfg.TagsConfig.GetTags(false)
+	return &metricsExporter{logger, cfg, client}, nil
+}
+
+// pushHostMetadata sends a host metadata payload to the "/intake" endpoint
+func (exp *metricsExporter) pushHostMetadata(metadata hostMetadata) error {
+	path := exp.cfg.Metrics.TCPAddr.Endpoint + "/intake"
+	buf, _ := json.Marshal(metadata)
+	req, _ := http.NewRequest(http.MethodPost, path, bytes.NewBuffer(buf))
+	req.Header.Set("DD-API-KEY", exp.cfg.API.Key)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", userAgent)
+	client := newHTTPClient()
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return err
+	}
 
-	return &metricsExporter{logger, cfg, client, tags}, nil
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf(
+			"'%d - %s' error when sending metadata payload to %s",
+			resp.StatusCode,
+			resp.Status,
+			path,
+		)
+	}
+
+	return nil
 }
 
 func (exp *metricsExporter) processMetrics(metrics []datadog.Metric) {
 	addNamespace := exp.cfg.Metrics.Namespace != ""
 	overrideHostname := exp.cfg.Hostname != ""
-	addTags := len(exp.tags) > 0
 
 	for i := range metrics {
 		if addNamespace {
@@ -53,11 +101,6 @@ func (exp *metricsExporter) processMetrics(metrics []datadog.Metric) {
 		if overrideHostname || metrics[i].GetHost() == "" {
 			metrics[i].Host = GetHost(exp.cfg)
 		}
-
-		if addTags {
-			metrics[i].Tags = append(metrics[i].Tags, exp.tags...)
-		}
-
 	}
 }
 

exporter/datadogexporter/metrics_exporter_test.go

@@ -58,14 +58,14 @@ func TestProcessMetrics(t *testing.T) {
 			0,
 			[]string{"key2:val2"},
 		),
-	}	 
+	}
 
 	exp.processMetrics(metrics)
 
 	assert.Equal(t, "test_host", *metrics[0].Host)
 	assert.Equal(t, "test.metric_name", *metrics[0].Metric)
 	assert.ElementsMatch(t,
-		[]string{"key:val", "env:test_env", "key2:val2"},
+		[]string{"key2:val2"},
 		metrics[0].Tags,
 	)