Add k8s container logs collection (open-telemetry#36)

Fixes open-telemetry/opentelemetry-collector-contrib#2536

Author: andrzej-stencel

charts/opentelemetry-collector/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: opentelemetry-collector
-version: 0.4.3
+version: 0.5.0
 description: OpenTelemetry Collector Helm chart for Kubernetes
 type: application
 home: https://opentelemetry.io/
@@ -13,4 +13,4 @@ maintainers:
   - name: pjanotti
   - name: tigrannajaryan
 icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png
-appVersion: 0.20.0
+appVersion: 0.22.0

charts/opentelemetry-collector/README.md

@@ -25,9 +25,9 @@ helm install my-opentelemetry-collector open-telemetry/opentelemetry-collector
 
 ### Default configuration
 
-By default this chart will deploy an OpenTelemetry Collector as daemonset with two pipelines (traces and metrics)
+By default this chart will deploy an OpenTelemetry Collector as daemonset with three pipelines (logs, metrics and traces)
 and logging exporter enabled by default. Besides daemonset (agent), it can be also installed as standalone deployment.
-Both modes can be enabled together, in that case metrics and traces will be flowing from agents to standalone collectors.
+Both modes can be enabled together, in that case logs, metrics and traces will be flowing from agents to standalone collectors.
 
 *Example*: Install collector as a standalone deployment, and do not run it as an agent.
 
@@ -42,6 +42,7 @@ By default collector has the following receivers enabled:
 
 - **metrics**: OTLP and prometheus. Prometheus is configured only for scraping collector's own metrics.
 - **traces**: OTLP, zipkin and jaeger (thrift and grpc).
+- **logs**: OTLP (to enable container logs, see [Configuration for Kubernetes container logs](#configuration-for-kubernetes-container-logs)).
 
 There are two ways to configure collector pipelines, which can be used together as well.
 
@@ -107,6 +108,76 @@ agentCollector:
     mountPropagation: HostToContainer
 ```
 
+### Configuration for Kubernetes container logs
+
+The collector can be used to collect logs sent to standard output by Kubernetes containers.
+This feature is disabled by default. It has the following requirements:
+
+- It needs agent collector to be deployed, which means it will not work if only standalone collector is enabled.
+- It requires the [contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) version
+of the collector image.
+
+To enable this feature, set the  `agentCollector.containerLogs.enabled` property to `true` and replace the collector image.
+Here is an example `values.yaml`:
+
+```yaml
+agentCollector:
+  containerLogs:
+    enabled: true
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+
+command:
+  name: otelcontribcol
+```
+
+The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured
+to read the files where Kubernetes container runtime writes all containers' console output to.
+
+#### :warning: Warning: Risk of looping the exported logs back into the receiver, causing "log explosion"
+
+The container logs pipeline uses the `logging` console exporter by default.
+Paired with the default `filelog` receiver that receives all containers' console output,
+it is easy to accidentally feed the exported logs back into the receiver.
+
+Also note that using the `--log-level=debug` option for the `logging` exporter causes it to output
+multiple lines per single received log, which when looped, would amplify the logs exponentially.
+
+To prevent the looping, the default configuration of the receiver excludes logs from the collector's containers.
+
+If you want to include the collector's logs, make sure to replace the `logging` exporter
+with an exporter that does not send logs to collector's standard output.
+
+Here's an example `values.yaml` file that replaces the default `logging` exporter on the `logs` pipeline
+with an `otlphttp` exporter that sends the container logs to `https://example.com:55681` endpoint.
+It also clears the `filelog` receiver's `exclude` property, for collector logs to be included in the pipeline.
+
+```yaml
+agentCollector:
+  containerLogs:
+    enabled: true
+
+  configOverride:
+    exporters:
+      otlphttp:
+        endpoint: https://example.com:55681
+    receivers:
+      filelog:
+        exclude: []
+    service:
+      pipelines:
+        logs:
+          exporters:
+            - otlphttp
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+
+command:
+  name: otelcontribcol
+```
+
 ### Other configuration options
 
 The [values.yaml](./values.yaml) file contains information about all other configuration

charts/opentelemetry-collector/templates/_config.tpl

@@ -32,6 +32,7 @@ Build config file for agent OpenTelemetry Collector
 {{- $values := deepCopy .Values.agentCollector | mustMergeOverwrite (deepCopy .Values)  }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }}
+{{- $config := include "opentelemetry-collector.agent.containerLogsConfig" $data | fromYaml | mustMergeOverwrite $config }}
 {{- $config := include "opentelemetry-collector.agentConfigOverride" $data | fromYaml | mustMergeOverwrite $config }}
 {{- .Values.agentCollector.configOverride | mustMergeOverwrite $config | toYaml }}
 {{- end }}
@@ -115,9 +116,90 @@ exporters:
 {{- if .Values.standaloneCollector.enabled }}
 service:
   pipelines:
+    logs:
+      exporters: [otlp]
     metrics:
       exporters: [otlp]
     traces:
       exporters: [otlp]
 {{- end }}
 {{- end }}
+
+{{- define "opentelemetry-collector.agent.containerLogsConfig" -}}
+{{- if .Values.agentCollector.containerLogs.enabled }}
+receivers:
+  filelog:
+    include: [ /var/log/pods/*/*/*.log ]
+    # Exclude collector container's logs. The file format is /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<run_id>.log
+    exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}*_*/{{ .Chart.Name }}/*.log ]
+    start_at: beginning
+    include_file_path: true
+    include_file_name: false
+    operators:
+      # Find out which format is used by kubernetes
+      - type: router
+        id: get-format
+        routes:
+          - output: parser-docker
+            expr: '$$record matches "^\\{"'
+          - output: parser-crio
+            expr: '$$record matches "^[^ Z]+ "'
+          - output: parser-containerd
+            expr: '$$record matches "^[^ Z]+Z"'
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.000000000-07:00'
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Parse Docker format
+      - type: json_parser
+        id: parser-docker
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract_metadata_from_filepath
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        parse_from: $$attributes.file_path
+      # Move out attributes to Attributes
+      - type: metadata
+        labels:
+          stream: 'EXPR($.stream)'
+          k8s.container.name: 'EXPR($.container_name)'
+          k8s.namespace.name: 'EXPR($.namespace)'
+          k8s.pod.name: 'EXPR($.pod_name)'
+          run_id: 'EXPR($.run_id)'
+          k8s.pod.uid: 'EXPR($.uid)'
+      # Clean up log record
+      - type: restructure
+        id: clean-up-log-record
+        ops:
+          - remove: logtag
+          - remove: stream
+          - remove: container_name
+          - remove: namespace
+          - remove: pod_name
+          - remove: run_id
+          - remove: uid
+service:
+  pipelines:
+    logs:
+      receivers:
+        - filelog
+        - otlp
+{{- end }}
+{{- end }}

charts/opentelemetry-collector/templates/_pod.tpl

@@ -61,6 +61,14 @@ containers:
         mountPropagation: {{ .mountPropagation }}
         {{- end }}
       {{- end }}
+      {{- if and $.isAgent .Values.agentCollector.containerLogs.enabled }}
+      - name: varlogpods
+        mountPath: /var/log/pods
+        readOnly: true
+      - name: varlibdockercontainers
+        mountPath: /var/lib/docker/containers
+        readOnly: true
+      {{- end }}
 volumes:
   - name: {{ .Chart.Name }}-configmap
     configMap:
@@ -73,6 +81,14 @@ volumes:
     hostPath:
       path: {{ .hostPath }}
   {{- end }}
+  {{- if and $.isAgent .Values.agentCollector.containerLogs.enabled }}
+  - name: varlogpods
+    hostPath:
+      path: /var/log/pods
+  - name: varlibdockercontainers
+    hostPath:
+      path: /var/lib/docker/containers
+  {{- end }}
 {{- with .Values.nodeSelector }}
 nodeSelector:
   {{- toYaml . | nindent 2 }}

charts/opentelemetry-collector/values.yaml

@@ -39,6 +39,14 @@ config:
     extensions:
       - health_check
     pipelines:
+      logs:
+        exporters:
+          - logging
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          - otlp
       metrics:
         exporters:
           - logging
@@ -63,8 +71,8 @@ config:
 # Can be overridden here or for any component independently using the same keys.
 
 image:
+  # If you want to use the contrib image `otel/opentelemetry-collector-contrib`, you also need to change `command.name` value to `otelcontribcol`.
   repository: otel/opentelemetry-collector
-  # repository: otel/opentelemetry-collector-contrib
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
@@ -73,7 +81,6 @@ imagePullSecrets: []
 # OpenTelemetry Collector executable
 command:
   name: otelcol
-  # name: otelcontribcol
   extraArgs: []
 
 serviceAccount:
@@ -127,6 +134,9 @@ ports:
 agentCollector:
   enabled: true
 
+  containerLogs:
+    enabled: false
+
   resources:
     limits:
       cpu: 256m