Logzio logs collector 1.0.1

charts/logzio-logs-collector/README.md

@@ -142,5 +142,8 @@ Multi line logs configuration
 The collector supports by default various log formats (including multiline logs) such as `CRI-O` `CRI-Containerd` `Docker` formats. You can configure the chart to parse custom multiline logs pattern according to your needs, please read [Customizing Multiline Log Handling](./examples/multiline.md) guide for more details.
 
 ## Change log
+* 1.0.1
+  - Update multiline parsing
+  - Update error detection in logs
 * 1.0.0
   - kubernetes logs collection agent for logz.io based on opentelemetry collector

charts/logzio-logs-collector/examples/multiline.md

@@ -22,7 +22,7 @@ Creating Custom Formats for Multiline Logs
 
 To configure custom formats, you must understand your logs' structure to accurately use `is_first_entry` or `is_last_entry` expressions. Regular expressions (regex) are powerful tools in matching specific log patterns, allowing you to identify the start or end of a multiline log entry effectively.
 
-Custom multiline `recombine` operators should be added before `move from attributes.log to body`:
+Custom multiline `recombine` operators should be added after `move from attributes.log to body`:
 ```yaml
   # Update body field after finishing all parsing
   - from: attributes.log
@@ -41,7 +41,7 @@ config:
       operators:
       - id: get-format
         routes:
-        - expr: body matches "^\\{"
+        - expr: body matches "^{.*}$"
           output: parser-docker
         - expr: body matches "^[^ Z]+ "
           output: parser-crio
@@ -105,17 +105,19 @@ config:
       - from: attributes.uid
         to: resource["k8s.pod.uid"]
         type: move
+      - id: parser-json
+        type: json_parser
+      # Update body field after finishing all parsing
+      - from: attributes.log
+        to: body
+        type: move
       # Add custom multiline parsers here. Add more `type: recombine` operators for custom multiline formats
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/recombine.md
       - type: recombine
         id: stack-errors-recombine
         combine_field: body
         is_first_entry: body matches "^[^\\s]"
         source_identifier: attributes["log.file.path"]
-      # Update body field after finishing all parsing
-      - from: attributes.log
-        to: body
-        type: move
 ```
 ### Examples
 
@@ -138,15 +140,16 @@ config:
     filelog:
       operators:
         # previous operators
-        - type: recombine
-          id: Java-Stack-Trace-Errors
-          combine_field: body
-          is_first_entry: body matches "^[\\w]+(Exception|Error)"
-          combine_with: "\n"
         # Update body field after finishing all parsing
         - from: attributes.log
           to: body
           type: move
+        # custom multiline recombine
+        - type: recombine
+          id: Java-Stack-Trace-Errors
+          combine_field: body
+          is_first_entry: body matches "^[\\w]+(Exception|Error)"
+          source_identifier: attributes["log.file.path"]
 ```
 
 #### Python Tracebacks
@@ -169,15 +172,16 @@ config:
     filelog:
       operators:
         # previous operators
-        - type: recombine
-          id: Python-Tracebacks
-          combine_field: body
-          is_first_entry: body matches "^Traceback"
-          combine_with: "\n"
         # Update body field after finishing all parsing
         - from: attributes.log
           to: body
           type: move
+        # custom multiline recombine
+        - type: recombine
+          id: Python-Tracebacks
+          combine_field: body
+          is_first_entry: body matches "^Traceback"
+          source_identifier: attributes["log.file.path"]
 ```
 
 #### Custom Multiline Log Format
@@ -199,13 +203,14 @@ config:
     filelog:
       operators:
         # previous operators
-        - type: recombine
-          id: custom-multiline
-          combine_field: body
-          is_first_entry: body matches "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"
-          combine_with: "\n"
         # Update body field after finishing all parsing
         - from: attributes.log
           to: body
           type: move
+        # custom multiline recombine
+        - type: recombine
+          id: custom-multiline
+          combine_field: body
+          is_first_entry: body matches "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"
+          source_identifier: attributes["log.file.path"]
 ```

charts/logzio-logs-collector/values.yaml

@@ -25,7 +25,7 @@ secrets:
   # environment indentifier attribute that will be added to all logs
   env_id: "my_env"
   # defualt log type field
-  logType: "k8s"
+  logType: "test"
   # Secret with your logzio logs shipping token
   logzioLogsToken: "token"
   # Secret with your logzio region
@@ -61,7 +61,7 @@ config:
             - set(attributes["log.level"], "INFO")
             - set(attributes["log.level"], "DEBUG") where (IsMatch(body, ".*\\b(?i:debug)\\b.*"))
             - set(attributes["log.level"], "WARNING") where (IsMatch(body, ".*\\b(?i:warning|warn)\\b.*"))
-            - set(attributes["log.level"], "ERROR") where (IsMatch(body, ".*\\b(?i:error|failure|failed|exception|panic)\\b.*"))
+            - set(attributes["log.level"], "ERROR") where (IsMatch(body, ".*(?i:(?:error|fail|failure|exception|panic)).*"))
     transform/log_type:
       error_mode: ignore
       log_statements:
@@ -132,7 +132,7 @@ config:
       # Find out which format is used by kubernetes
       - id: get-format
         routes:
-        - expr: body matches "^\\{"
+        - expr: body matches "^{.*"
           output: parser-docker
         - expr: body matches "^[^ Z]+ "
           output: parser-crio
@@ -201,17 +201,22 @@ config:
       - from: attributes.uid
         to: resource["k8s.pod.uid"]
         type: move
+      # Update body field after finishing all parsing
+      - from: attributes.log
+        to: body
+        type: move
+      # conditional json parser
+      - type: json_parser
+        id: json
+        parse_from: body
+        if: 'body matches "^{.*}$"'
       # multiline parsers. add more `type: recombine` operators for custom multiline formats
       # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/recombine.md
       - type: recombine
         id: stack-errors-recombine
         combine_field: body
         is_first_entry: body matches "^[^\\s]"
         source_identifier: attributes["log.file.path"]
-      # Update body field after finishing all parsing
-      - from: attributes.log
-        to: body
-        type: move
     otlp:
       protocols:
         grpc: