Merge pull request #442 from logzio/logzio-fluentd-0.29.1

Logzio fluentd 0.29.1

Author: yotamloe

charts/fluentd/Chart.yaml

@@ -4,10 +4,8 @@ description: A Helm chart for shipping Kubernetes logs via Fluentd.
 keywords:
   - logging
   - fluentd
-version: 0.29.0
+version: 0.29.1
 appVersion: 1.5.1
 maintainers:
-  - name: Miri Bar
-    email: [email protected]
-  - name: Tamir Michaeli
-    email: [email protected]
+  - name: Yotam loewenbach
+    email: [email protected]

charts/fluentd/README.md

@@ -91,6 +91,7 @@ helm install -n monitoring \
 | `daemonset.auditLogFormat` | Match Fluentd's format for kube-apiserver audit logs. Set to `audit-json` if your audit logs are in json format. | `audit` |
 | `daemonset.containerdRuntime` | **Deprecated from chart version 0.1.0.** Determines whether to use a configuration for a Containerd runtime. Set to `false` if your cluster doesn't use Containerd as CRI. | `true` |
 | `daemonset.cri` | Container runtime interface of the cluster. Used to determine which configuration to use when concatenating partial logs. Valid options are: `docker`, `containerd`. | `containerd` |
+| `daemonset.LogFileRefreshInterval` | The interval of refreshing the list of watch file for log files.. | `60s` |
 | `daemonset.logzioBufferType` | Specifies which plugin to use as the backend. | `file` |
 | `daemonset.logzioBufferPath` | Path of the buffer. | `/var/log/fluentd-buffers/stackdriver.buffer` |
 | `daemonset.logzioOverflowAction` | Controls the behavior when the queue becomes full. | `block` |
@@ -120,6 +121,7 @@ helm install -n monitoring \
 | `windowsDaemonset.auditLogFormat` | Match Fluentd's format for kube-apiserver audit logs. Set to `audit-json` if your audit logs are in json format. (windows) | `audit` |
 | `windowsDaemonset.containerdRuntime` | **Deprecated from chart version 0.1.0.** Determines whether to use a configuration for a Containerd runtime. Set to `false` if your cluster doesn't use Containerd as CRI. (windows) | `true` |
 | `windowsDaemonset.cri` | Container runtime interface of the cluster. Used to determine which configuration to use when concatenating partial logs (windows). Valid options are: `docker`, `containerd`. | `containerd` |
+| `windowsDaemonset.LogFileRefreshInterval` | The interval of refreshing the list of watch file for log files.. | `60s` |
 | `windowsDaemonset.logzioBufferType` | Specifies which plugin to use as the backend. | `file` |
 | `windowsDaemonset.logzioBufferPath` | Path of the buffer. (windows) | `/var/log/fluentd-buffers/stackdriver.buffer` |
 | `windowsDaemonset.logzioOverflowAction` | Controls the behavior when the queue becomes full. (windows) | `block` |
@@ -301,6 +303,9 @@ If needed, the fluentd image can be changed to support windows server 2022 with
 
 
 ## Change log
+ - **0.29.1**:
+   - Added `enabled` value, to conditianly control the deployment of this chart by a parent chart.
+   - Added `daemonset.LogFileRefreshInterval` and `windowsDaemonset.LogFileRefreshInterval` values, to control list of watched log files refresh interval.
  - **0.29.0**:
    - EKS Fargate logging:
     - Send logs to port `8070` in logzio listener (instead of port `5050`)

charts/fluentd/templates/aws-observability-namespace.yaml

@@ -1,8 +1,10 @@
+{{ if .Values.enabled }}
 {{- if .Values.fargateLogRouter.enabled }}
 kind: Namespace
 apiVersion: v1
 metadata:
   name: aws-observability
   labels:
     aws-observability: enabled
-{{- end }}
+{{- end }}
+{{ end }}

charts/fluentd/templates/clusterrole.yaml

@@ -1,8 +1,10 @@
+{{ if .Values.enabled }}
 {{- if .Values.isRBAC }}
 apiVersion: {{ .Values.apiVersions.clusterRole }}
 kind: ClusterRole
 metadata:
   name: {{ include "fluentd.fullname" . }}
   namespace: {{ .Release.Namespace }}
 rules: {{ toYaml .Values.clusterRole.rules | nindent 2 -}}
-{{- end }}
+{{- end }}
+{{ end }}

charts/fluentd/templates/clusterrolebinding.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 {{- if .Values.isRBAC }}
 apiVersion: {{ .Values.apiVersions.clusterRoleBinding }}
 kind: ClusterRoleBinding
@@ -11,4 +12,5 @@ subjects:
 - kind: ServiceAccount
   name: {{ template "fluentd.serviceAccount" . }}
   namespace: {{ .Release.Namespace }}
-{{- end }}
+{{- end }}
+{{ end }}

charts/fluentd/templates/configmap-monitoring.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 {{- if or .Values.daemonset.fluentdPrometheusConf .Values.windowsDaemonset.fluentdPrometheusConf }}
 apiVersion: {{ .Values.apiVersions.configmap }}
 kind: ConfigMap
@@ -8,4 +9,5 @@ metadata:
     k8s-app: {{ .Values.k8sApp }}
 data:
   prometheus.conf : {{ toYaml .Values.configmap.prometheus | indent 2 }}
-{{- end }}
+{{- end }}
+{{ end }}

charts/fluentd/templates/configmap.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 apiVersion: {{ .Values.apiVersions.configmap }}
 kind: ConfigMap
 metadata:
@@ -43,3 +44,4 @@ data:
 {{ $value | indent 4 -}}
 {{- end -}}
 {{- end }}
+{{ end }}

charts/fluentd/templates/daemonset.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 apiVersion: {{ .Values.apiVersions.daemonset }}
 kind: DaemonSet
 metadata:
@@ -87,6 +88,8 @@ spec:
             secretKeyRef:
               name: {{ .Values.secretName }}
               key: logzio-log-listener
+        - name: LOG_FILE_REFRESH_INTERVAL
+          value: {{ .Values.daemonset.LogFileRefreshInterval | quote }}
         - name: LOGZIO_BUFFER_TYPE
           value: {{ .Values.daemonset.logzioBufferType | quote }}
         - name: LOGZIO_BUFFER_PATH
@@ -236,6 +239,8 @@ spec:
             secretKeyRef:
               name: {{ .Values.secretName }}
               key: logzio-log-listener
+        - name: LOG_FILE_REFRESH_INTERVAL
+          value: {{ .Values.windowsDaemonset.LogFileRefreshInterval | quote }}
         - name: LOGZIO_BUFFER_TYPE
           value: {{ .Values.windowsDaemonset.logzioBufferType | quote }}
         - name: LOGZIO_BUFFER_PATH
@@ -316,3 +321,4 @@ spec:
   updateStrategy: {{ toYaml .Values.windowsDaemonset.updateStrategy | nindent 4 }}
   {{- end }}
 {{- end }}
+{{ end }}

charts/fluentd/templates/fargate-logging-configmap.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 {{ if .Values.fargateLogRouter.enabled }}
 kind: ConfigMap
 apiVersion: v1
@@ -44,4 +45,5 @@ data:
       Port  8070
       Retry_Limit no_retries
       Index logzioCustomerIndex
+{{ end }}
 {{ end }}

charts/fluentd/templates/secret.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.enabled }}
 {{- if .Values.secrets.enabled -}}
 apiVersion: {{ .Values.apiVersions.secret }}
 kind: Secret
@@ -9,3 +10,4 @@ stringData:
   logzio-log-shipping-token: {{ required "Logzio shipping token is required!" .Values.secrets.logzioShippingToken }}
   logzio-log-listener: {{ template "logzio.listenerHost" . }}
 {{- end }}
+{{ end }}

charts/fluentd/templates/serviceaccount.yaml

@@ -1,7 +1,9 @@
+{{ if .Values.enabled }}
 {{- if .Values.isRBAC }}
 apiVersion: {{ .Values.apiVersions.serviceAccount }}
 kind: ServiceAccount
 metadata:
   name: {{ template "fluentd.serviceAccount" . }}
   namespace: {{ .Release.Namespace }}
-{{- end }}
+{{- end }}
+{{ end }}

charts/fluentd/values.yaml

@@ -1,3 +1,4 @@
+enabled: true
 # Fluentd image in daemonset
 image: logzio/logzio-fluentd
 # Fluentd image tag
@@ -87,6 +88,8 @@ daemonset:
   # Container runtime interface of the cluster. Used to determine which configuration to use when concatenating partial logs.
   # Valid options are: docker, containerd
   cri: containerd
+  # The interval of refreshing the list of watched log files.
+  LogFileRefreshInterval: 60s
   # Specifies which plugin to use as the backend
   logzioBufferType: file
   # Path of the buffer
@@ -170,6 +173,8 @@ windowsDaemonset:
   # Container runtime interface of the cluster. Used to determine which configuration to use when concatenating partial logs.
   # Valid options are: docker, containerd
   cri: containerd
+  # The interval of refreshing the list of watch file for log files.
+  LogFileRefreshInterval: 60s
   # Specifies which plugin to use as the backend
   logzioBufferType: file
   # Path of the buffer
@@ -304,6 +309,7 @@ configmap:
       exclude_path "#{ENV['EXCLUDE_PATH'].split(',')}"
       tag logzio.kubernetes.*
       read_from_head true
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type multi_format
         <pattern>
@@ -329,6 +335,7 @@ configmap:
       path /var/log/salt/minion
       pos_file /var/log/fluentd-salt.pos
       tag logzio.salt
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type regexp
         expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
@@ -342,6 +349,7 @@ configmap:
       path /var/log/startupscript.log
       pos_file /var/log/fluentd-startupscript.log.pos
       tag logzio.startupscript
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type syslog
       </parse>
@@ -353,6 +361,7 @@ configmap:
       path /var/log/docker.log
       pos_file /var/log/fluentd-docker.log.pos
       tag logzio.docker
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type regexp
         expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
@@ -365,6 +374,7 @@ configmap:
       path /var/log/etcd.log
       pos_file /var/log/fluentd-etcd.log.pos
       tag logzio.etcd
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type none
       </parse>
@@ -377,6 +387,7 @@ configmap:
       path /var/log/kubelet.log
       pos_file /var/log/fluentd-kubelet.log.pos
       tag logzio.kubelet
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -389,6 +400,7 @@ configmap:
       path /var/log/kube-proxy.log
       pos_file /var/log/fluentd-kube-proxy.log.pos
       tag logzio.kube-proxy
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -401,6 +413,7 @@ configmap:
       path /var/log/kube-apiserver.log
       pos_file /var/log/fluentd-kube-apiserver.log.pos
       tag logzio.kube-apiserver
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -413,6 +426,7 @@ configmap:
       path /var/log/kube-controller-manager.log
       pos_file /var/log/fluentd-kube-controller-manager.log.pos
       tag logzio.kube-controller-manager
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -425,6 +439,7 @@ configmap:
       path /var/log/kube-scheduler.log
       pos_file /var/log/fluentd-kube-scheduler.log.pos
       tag logzio.kube-scheduler
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -437,6 +452,7 @@ configmap:
       path /var/log/rescheduler.log
       pos_file /var/log/fluentd-rescheduler.log.pos
       tag logzio.rescheduler
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -449,6 +465,7 @@ configmap:
       path /var/log/glbc.log
       pos_file /var/log/fluentd-glbc.log.pos
       tag logzio.glbc
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -461,6 +478,7 @@ configmap:
       path /var/log/cluster-autoscaler.log
       pos_file /var/log/fluentd-cluster-autoscaler.log.pos
       tag logzio.cluster-autoscaler
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type kubernetes
       </parse>
@@ -586,6 +604,7 @@ configmap:
       path /var/log/kubernetes/kube-apiserver-audit.log
       pos_file /var/log/kube-apiserver-audit.log.pos
       tag logzio.kube-apiserver-audit
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type multiline
         format_firstline /^\S+\s+AUDIT:/
@@ -609,6 +628,7 @@ configmap:
       path /var/log/kubernetes/kube-apiserver-audit.log
       pos_file /var/log/kube-apiserver-audit.log.pos
       tag logzio.kube-apiserver-audit
+      refresh_insterval "#{ENV['LOG_FILE_REFRESH_INTERVAL']}"
       <parse>
         @type json
         keep_time_key true