Add Splunk HEC metrics example, taking metrics from telegraf (open-telemetry#64)

Author: atoulme

examples/splunk-hec-metrics/README.md

@@ -0,0 +1,16 @@
+# Splunk HEC Metrics Example
+
+This example showcases how the collector can send data to a Splunk Enterprise deployment from a Telegraf instance.
+
+The example runs as a Docker Compose deployment. The collector can be configured to send metrics to Splunk Enterprise.
+
+Splunk is configured to receive data from the OpenTelemetry Collector using the HTTP Event collector. To learn more about HEC, visit [our guide](https://dev.splunk.com/enterprise/docs/dataapps/httpeventcollector/).
+
+To deploy the example, check out this git repository, open a terminal and in this directory type:
+```bash
+$> docker-compose up
+```
+
+Splunk will become available on port 18000. You can login on [http://localhost:18000](http://localhost:18000) with `admin` and `changeme`.
+
+Once logged in, visit the [analytics workspace](http://localhost:18000/en-US/app/search/analytics_workspace) to see the metrics collected by Splunk.

examples/splunk-hec-metrics/docker-compose.yml

@@ -0,0 +1,44 @@
+version: "3"
+services:
+  telegraf:
+    image: telegraf:1.17.0
+    container_name: telegraf
+    restart: always
+    environment:
+      HOST_PROC: /rootfs/proc
+      HOST_SYS: /rootfs/sys
+      HOST_ETC: /rootfs/etc
+    volumes:
+      - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /sys:/rootfs/sys:ro
+      - /proc:/rootfs/proc:ro
+      - /etc:/rootfs/etc:ro
+  # Splunk Enterprise server:
+  splunk:
+    image: splunk/splunk:latest
+    container_name: splunk
+    environment:
+      - SPLUNK_START_ARGS=--accept-license
+      - SPLUNK_HEC_TOKEN=00000000-0000-0000-0000-0000000000000
+      - SPLUNK_PASSWORD=changeme
+    ports:
+      - 18000:8000
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8000']
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    volumes:
+      - ./splunk.yml:/tmp/defaults/default.yml
+      - /opt/splunk/var
+      - /opt/splunk/etc
+  # OpenTelemetry Collector
+  otelcollector:
+    image:  otel/opentelemetry-collector-contrib-dev:latest #quay.io/signalfx/splunk-otel-collector:0.4.0
+    container_name: otelcollector
+    command: ["--config=/etc/otel-collector-config.yml", "--log-level=DEBUG"]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
+    depends_on:
+      - splunk

examples/splunk-hec-metrics/otel-collector-config.yml

@@ -0,0 +1,43 @@
+receivers:
+    splunk_hec:
+
+exporters:
+    splunk_hec/metrics:
+        # Splunk HTTP Event Collector token.
+        token: "00000000-0000-0000-0000-0000000000000"
+        # URL to a Splunk instance to send data to.
+        endpoint: "https://splunk:8088/services/collector"
+        # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
+        source: "app"
+        # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
+        sourcetype: "telegraf"
+        # Splunk index, optional name of the Splunk index targeted.
+        index: "metrics"
+        # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
+        max_connections: 20
+        # Whether to disable gzip compression over HTTP. Defaults to false.
+        disable_compression: false
+        # HTTP timeout when sending data. Defaults to 10s.
+        timeout: 10s
+        # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
+        # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
+        insecure_skip_verify: true
+
+processors:
+    batch:
+    queued_retry:
+
+extensions:
+    health_check:
+    pprof:
+      endpoint: :1888
+    zpages:
+      endpoint: :55679
+
+service:
+    extensions: [pprof, zpages, health_check]
+    pipelines:
+      metrics:
+        receivers: [splunk_hec]
+        processors: [batch, queued_retry]
+        exporters: [splunk_hec/metrics]

examples/splunk-hec-metrics/splunk.yml

@@ -0,0 +1,11 @@
+splunk:
+    conf:
+      indexes:
+        directory: /opt/splunk/etc/apps/search/local
+        content:
+          metrics:
+            coldPath: $SPLUNK_DB/metrics/colddb
+            datatype: metric
+            homePath: $SPLUNK_DB/metrics/db
+            maxTotalDataSizeMB: 512000
+            thawedPath: $SPLUNK_DB/metrics/thaweddb

examples/splunk-hec-metrics/telegraf.conf

@@ -0,0 +1,49 @@
+[agent]
+  ## Default data collection interval for all inputs
+  interval = "10s"
+
+[[inputs.cpu]]
+  percpu = true
+  totalcpu = true
+  collect_cpu_time = false
+  report_active = false
+    [inputs.cpu.tags]
+    index = "metrics"
+    event = "metric"
+
+
+[[outputs.http]]
+   ## URL is the address to send metrics to
+   url = "http://otelcollector:8088/services/collector"
+
+   ## Timeout for HTTP message
+   # timeout = "5s"
+
+   ## HTTP method, one of: "POST" or "PUT"
+   # method = "POST"
+
+   ## HTTP Basic Auth credentials
+   # username = "username"
+   # password = "pa$$word"
+
+   ## Optional TLS Config
+   # tls_ca = "/etc/telegraf/ca.pem"
+   # tls_cert = "/etc/telegraf/cert.pem"
+   # tls_key = "/etc/telegraf/key.pem"
+   ## Use TLS but skip chain & host verification
+   insecure_skip_verify = true
+
+   ## Data format to output.
+   ## Each data format has it's own unique set of configuration options, read
+   ## more about them here:
+   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+   data_format = "splunkmetric"
+    ## Provides time, index, source overrides for the HEC
+   splunkmetric_hec_routing = true
+   splunkmetric_multimetric = true
+
+   ## Additional HTTP headers
+    [outputs.http.headers]
+   # Should be set manually to "application/json" for json data_format
+      Content-Type = "application/json"
+      Authorization = "Splunk 00000000-0000-0000-0000-0000000000000"