enhancement(unit tests): Allow objects and arrays in log_fields test input (#22406)

* enhancement(unit tests): Allow objects and arrays in log_fields

By making vrl Value a configurable type

Fixes: #9386

* doc: Add doc comment to vrl::value::Value::to_value

Co-authored-by: Pavlos Rontidis <[email protected]>

---------

Co-authored-by: Pavlos Rontidis <[email protected]>

Author: tmccombs

changelog.d/log-fields-additional-types.enhancement.md

@@ -0,0 +1,4 @@
+Allow additional types to be used in `tests.inputs.log_fields` values, including
+nested objects and arrays.
+
+authors: tmccombs

lib/vector-config/src/external/vrl.rs

@@ -1,7 +1,7 @@
 use std::cell::RefCell;
 
 use serde_json::Value;
-use vrl::{compiler::VrlRuntime, datadog_search_syntax::QueryNode};
+use vrl::{compiler::VrlRuntime, datadog_search_syntax::QueryNode, value::Value as VrlValue};
 
 use crate::{
     schema::{generate_string_schema, SchemaGenerator, SchemaObject},
@@ -36,3 +36,33 @@ impl Configurable for QueryNode {
         Ok(generate_string_schema())
     }
 }
+
+impl Configurable for VrlValue {
+    fn is_optional() -> bool {
+        true
+    }
+
+    fn metadata() -> Metadata {
+        Metadata::with_transparent(true)
+    }
+
+    fn generate_schema(_: &RefCell<SchemaGenerator>) -> Result<SchemaObject, GenerateError> {
+        // We don't have any constraints on the inputs
+        Ok(SchemaObject::default())
+    }
+}
+
+impl ToValue for VrlValue {
+    /// Converts a `VrlValue` into a `serde_json::Value`.
+    ///
+    /// This conversion should always succeed, though it may result in a loss
+    /// of type information for some value types.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if serialization fails, which is not expected
+    /// under normal circumstances.
+    fn to_value(&self) -> Value {
+        serde_json::to_value(self).expect("Unable to serialize VRL value")
+    }
+}

src/config/mod.rs

@@ -8,7 +8,13 @@ use std::{
     time::Duration,
 };
 
-use crate::{conditions, event::Metric, secrets::SecretBackends, serde::OneOrMany};
+use crate::{
+    conditions,
+    event::{Metric, Value},
+    secrets::SecretBackends,
+    serde::OneOrMany,
+};
+
 use indexmap::IndexMap;
 use serde::Serialize;
 
@@ -473,24 +479,6 @@ impl TestDefinition<OutputId> {
     }
 }
 
-/// Value for a log field.
-#[configurable_component]
-#[derive(Clone, Debug)]
-#[serde(untagged)]
-pub enum TestInputValue {
-    /// A string.
-    String(String),
-
-    /// An integer.
-    Integer(i64),
-
-    /// A floating-point number.
-    Float(f64),
-
-    /// A boolean.
-    Boolean(bool),
-}
-
 /// A unit test input.
 ///
 /// An input describes not only the type of event to insert, but also which transform within the
@@ -522,7 +510,7 @@ pub struct TestInput {
     /// The set of log fields to use when creating a log input event.
     ///
     /// Only relevant when `type` is `log`.
-    pub log_fields: Option<IndexMap<String, TestInputValue>>,
+    pub log_fields: Option<IndexMap<String, Value>>,
 
     /// The metric to use as an input event.
     ///

src/config/unit_test/mod.rs

@@ -18,7 +18,6 @@ use std::{
 
 use futures_util::{stream::FuturesUnordered, StreamExt};
 use indexmap::IndexMap;
-use ordered_float::NotNan;
 use tokio::sync::{
     oneshot::{self, Receiver},
     Mutex,
@@ -39,9 +38,9 @@ use crate::{
     conditions::Condition,
     config::{
         self, loading, ComponentKey, Config, ConfigBuilder, ConfigPath, SinkOuter, SourceOuter,
-        TestDefinition, TestInput, TestInputValue, TestOutput,
+        TestDefinition, TestInput, TestOutput,
     },
-    event::{Event, EventMetadata, LogEvent, Value},
+    event::{Event, EventMetadata, LogEvent},
     signal,
     topology::{builder::TopologyPieces, RunningTopology},
 };
@@ -622,16 +621,8 @@ fn build_input_event(input: &TestInput) -> Result<Event, String> {
             if let Some(log_fields) = &input.log_fields {
                 let mut event = LogEvent::from_str_legacy("");
                 for (path, value) in log_fields {
-                    let value: Value = match value {
-                        TestInputValue::String(s) => Value::from(s.to_owned()),
-                        TestInputValue::Boolean(b) => Value::from(*b),
-                        TestInputValue::Integer(i) => Value::from(*i),
-                        TestInputValue::Float(f) => Value::from(
-                            NotNan::new(*f).map_err(|_| "NaN value not supported".to_string())?,
-                        ),
-                    };
                     event
-                        .parse_path_and_insert(path, value)
+                        .parse_path_and_insert(path, value.clone())
                         .map_err(|e| e.to_string())?;
                 }
                 Ok(event.into())

src/config/unit_test/tests.rs

@@ -805,6 +805,9 @@ async fn test_log_input() {
                 message = "this is the message"
                 int_val = 5
                 bool_val = true
+                arr_val = [1, 2, "hi", false]
+                obj_val = { a =  true, b = "b", c = 5 }
+
 
             [[tests.outputs]]
               extract_from = "foo"
@@ -816,6 +819,10 @@ async fn test_log_input() {
                     assert_eq!(.message, "this is the message")
                     assert!(.bool_val)
                     assert_eq!(.int_val, 5)
+                    assert_eq!(.arr_val, [1, 2, "hi", false])
+                    assert!(.obj_val.a)
+                    assert_eq!(.obj_val.b, "b")
+                    assert_eq!(.obj_val.c, 5)
                 """
       "#})
     .unwrap();