fix(loki sink, observability): Drop non-fatal template render errors to warnings (#17746)

If a render error doesn't result in a dropped event, it seems more like
a warning than an error.

For the places that currently emit template errors with `drop_event:
false`:

* `loki` sink: skips inserting label if key or value fails to render;
falls back to `None` for
  partitioning using `tenant_id`
* `throttle` transform: falls back to `None` for throttle key
* `log_to_metric` transform: skips tag addition
* `papertrail` sink: falls back to `vector` for the `process` field
* `splunk_hec_logs` sink: falls back to `None` for partition keys
(source, sourcetype, index)
* `splunk_hec_metrics` sink: falls back to `None` for source,
sourcetype, index

Fixes: #17487

Signed-off-by: Jesse Szwedko <[email protected]>

Signed-off-by: Jesse Szwedko <[email protected]>

Author: jszwedko

src/internal_events/template.rs

@@ -19,25 +19,33 @@ impl<'a> InternalEvent for TemplateRenderingError<'a> {
         }
         msg.push('.');
 
-        error!(
-            message = %msg,
-            error = %self.error,
-            error_type = error_type::TEMPLATE_FAILED,
-            stage = error_stage::PROCESSING,
-            internal_log_rate_limit = true,
-        );
+        if self.drop_event {
+            error!(
+                message = %msg,
+                error = %self.error,
+                error_type = error_type::TEMPLATE_FAILED,
+                stage = error_stage::PROCESSING,
+                internal_log_rate_limit = true,
+            );
 
-        counter!(
-            "component_errors_total", 1,
-            "error_type" => error_type::TEMPLATE_FAILED,
-            "stage" => error_stage::PROCESSING,
-        );
+            counter!(
+                "component_errors_total", 1,
+                "error_type" => error_type::TEMPLATE_FAILED,
+                "stage" => error_stage::PROCESSING,
+            );
 
-        if self.drop_event {
             emit!(ComponentEventsDropped::<UNINTENTIONAL> {
                 count: 1,
                 reason: "Failed to render template.",
             });
+        } else {
+            warn!(
+                message = %msg,
+                error = %self.error,
+                error_type = error_type::TEMPLATE_FAILED,
+                stage = error_stage::PROCESSING,
+                internal_log_rate_limit = true,
+            );
         }
     }
 }