chore(config): Convert top-level transforms enum to typetag (#16572)

This replaces the top-level `enum Transforms` into a boxed transform type.
Serialization and deserialization are handled by `typetag`, and the
`configurable_component` macro is enhanced to build out the necessary table
entries to generate the schema bits dynamically from all of the components that
are compiled into the current configuration.

Author: bruceg

Cargo.lock

@@ -9,6 +9,7 @@ proc-macro = true
 
 [dependencies]
 darling = { version = "0.13", default-features = false, features = ["suggestions"] }
+itertools = { version = "0.10.5", default-features = false }
 proc-macro2 = { version = "1.0", default-features = false }
 quote = { version = "1.0", default-features = false }
 serde_derive_internals = "0.26"

lib/vector-config-macros/Cargo.toml

@@ -1,4 +1,5 @@
 use darling::{Error, FromMeta};
+use itertools::Itertools as _;
 use proc_macro::TokenStream;
 use proc_macro2::{Ident, Span};
 use quote::{quote, quote_spanned};
@@ -88,6 +89,7 @@ struct TypedComponent {
     span: Span,
     component_type: ComponentType,
     component_name: Option<LitStr>,
+    description: Option<LitStr>,
 }
 
 impl TypedComponent {
@@ -102,6 +104,7 @@ impl TypedComponent {
                 span: path.span(),
                 component_type,
                 component_name: None,
+                description: None,
             })
     }
 
@@ -110,18 +113,24 @@ impl TypedComponent {
     /// If the meta list does not have a path that matches a known component type, `None` is
     /// returned. Otherwise, `Some(...)` is returned with a valid `TypedComponent`.
     fn from_meta_list(ml: &MetaList) -> Option<Self> {
+        let mut items = ml.nested.iter();
         ComponentType::try_from(&ml.path)
             .ok()
-            .map(|component_type| match ml.nested.first() {
-                Some(NestedMeta::Lit(Lit::Str(component_name))) => {
-                    (component_type, Some(component_name.clone()))
+            .map(|component_type| {
+                let component_name = match items.next() {
+                    Some(NestedMeta::Lit(Lit::Str(component_name))) => Some(component_name.clone()),
+                    _ => None,
+                };
+                let description = match items.next() {
+                    Some(NestedMeta::Lit(Lit::Str(description))) => Some(description.clone()),
+                    _ => None,
+                };
+                Self {
+                    span: ml.span(),
+                    component_type,
+                    component_name,
+                    description,
                 }
-                _ => (component_type, None),
-            })
-            .map(|(component_type, component_name)| Self {
-                span: ml.span(),
-                component_type,
-                component_name,
             })
     }
 
@@ -159,9 +168,28 @@ impl TypedComponent {
                 }
             };
 
+            // Derive the label from the component name, but capitalized.
+            let label = capitalize_words(&component_name.value());
+
+            // Derive the logical name from the config type, with the trailing "Config" dropped.
+            let logical_name = config_ty.to_string();
+            let logical_name = logical_name.strip_suffix("Config").unwrap_or(&logical_name);
+
+            // TODO: Make this an `expect` once all component types have been converted.
+            let description = self
+                .description
+                .as_ref()
+                .map(LitStr::value)
+                .unwrap_or_else(|| "This component is missing a description.".into());
+
             quote! {
                 ::inventory::submit! {
-                    #desc_ty::new::<#config_ty>(#component_name)
+                    #desc_ty::new::<#config_ty>(
+                        #component_name,
+                        #label,
+                        #logical_name,
+                        #description,
+                    )
                 }
             }
         })
@@ -371,3 +399,15 @@ pub fn configurable_component_impl(args: TokenStream, item: TokenStream) -> Toke
 
     derived.into()
 }
+
+fn capitalize(s: &str) -> String {
+    let mut iter = s.chars();
+    match iter.next() {
+        None => String::new(),
+        Some(first) => first.to_uppercase().collect::<String>() + iter.as_str(),
+    }
+}
+
+fn capitalize_words(s: &str) -> String {
+    s.split('_').map(capitalize).join(" ")
+}

lib/vector-config-macros/src/configurable_component.rs

@@ -1,9 +1,12 @@
-use std::marker::PhantomData;
+use std::{cell::RefCell, marker::PhantomData};
 
 use snafu::Snafu;
 use toml::Value;
+use vector_config_common::attributes::CustomAttribute;
 
 use super::{ComponentMarker, GenerateConfig};
+use crate::schema::{SchemaGenerator, SchemaObject};
+use crate::{schema, Configurable, ConfigurableRef, GenerateError, Metadata};
 
 #[derive(Debug, Snafu, Clone, PartialEq, Eq)]
 pub enum ExampleError {
@@ -17,7 +20,11 @@ pub enum ExampleError {
 /// Description of a component.
 pub struct ComponentDescription<T: ComponentMarker + Sized> {
     component_name: &'static str,
+    description: &'static str,
+    label: &'static str,
+    logical_name: &'static str,
     example_value: fn() -> Option<Value>,
+    config: ConfigurableRef,
     _component_type: PhantomData<T>,
 }
 
@@ -33,11 +40,21 @@ where
     /// type `T` and the component name. As such, if `T` is `SourceComponent`, and the name is
     /// `stdin`, you would say that the component is a "source called `stdin`".
     ///
-    /// The type parameter `C` must be the component's configuration type that implements `GenerateConfig`.
-    pub const fn new<C: GenerateConfig>(component_name: &'static str) -> Self {
+    /// The type parameter `C` must be the component's configuration type that implements
+    /// `Configurable` and `GenerateConfig`.
+    pub const fn new<C: GenerateConfig + Configurable + 'static>(
+        component_name: &'static str,
+        label: &'static str,
+        logical_name: &'static str,
+        description: &'static str,
+    ) -> Self {
         ComponentDescription {
             component_name,
+            description,
+            label,
+            logical_name,
             example_value: || Some(C::generate_config()),
+            config: ConfigurableRef::new::<C>(),
             _component_type: PhantomData,
         }
     }
@@ -67,4 +84,46 @@ where
         types.sort_unstable();
         types
     }
+
+    /// Generate a schema object covering all the descriptions of this type.
+    pub fn generate_schemas(gen: &RefCell<SchemaGenerator>) -> Result<SchemaObject, GenerateError> {
+        let mut descriptions: Vec<_> = inventory::iter::<Self>.into_iter().collect();
+        descriptions.sort_unstable_by_key(|desc| desc.component_name);
+        let subschemas: Vec<SchemaObject> = descriptions
+            .into_iter()
+            .map(|description| description.generate_schema(gen))
+            .collect::<Result<_, _>>()?;
+        Ok(schema::generate_one_of_schema(&subschemas))
+    }
+
+    /// Generate a schema object for this description.
+    fn generate_schema(
+        &self,
+        gen: &RefCell<SchemaGenerator>,
+    ) -> Result<SchemaObject, GenerateError> {
+        let mut tag_subschema =
+            schema::generate_const_string_schema(self.component_name.to_string());
+        let variant_tag_metadata = Metadata::with_description(self.description);
+        schema::apply_base_metadata(&mut tag_subschema, variant_tag_metadata);
+
+        let tag_schema =
+            schema::generate_internal_tagged_variant_schema("type".to_string(), tag_subschema);
+        let flattened_subschemas = vec![tag_schema];
+
+        let mut field_metadata = Metadata::default();
+        field_metadata.set_transparent();
+        let mut subschema =
+            schema::get_or_generate_schema(&self.config, gen, Some(field_metadata))?;
+
+        schema::convert_to_flattened_schema(&mut subschema, flattened_subschemas);
+
+        let mut variant_metadata = Metadata::default();
+        variant_metadata.set_description(self.description);
+        variant_metadata.add_custom_attribute(CustomAttribute::kv("docs::label", self.label));
+        variant_metadata
+            .add_custom_attribute(CustomAttribute::kv("logical_name", self.logical_name));
+        schema::apply_base_metadata(&mut subschema, variant_metadata);
+
+        Ok(subschema)
+    }
 }

lib/vector-config/src/component/description.rs

@@ -17,15 +17,15 @@ use crate::{
 /// `Configurable` provides the machinery to allow describing and encoding the shape of a type, recursively, so that by
 /// instrumenting all transitive types of the configuration, the schema can be discovered by generating the schema from
 /// some root type.
-pub trait Configurable
-where
-    Self: Sized,
-{
+pub trait Configurable {
     /// Gets the referenceable name of this value, if any.
     ///
     /// When specified, this implies the value is both complex and standardized, and should be
     /// reused within any generated schema it is present in.
-    fn referenceable_name() -> Option<&'static str> {
+    fn referenceable_name() -> Option<&'static str>
+    where
+        Self: Sized,
+    {
         None
     }
 
@@ -38,12 +38,18 @@ where
     /// Maps, by definition, are inherently free-form, and thus inherently optional. Thus, this
     /// method should likely not be overridden except for implementing `Configurable` for map
     /// types. If you're using it for something else, you are expected to know what you're doing.
-    fn is_optional() -> bool {
+    fn is_optional() -> bool
+    where
+        Self: Sized,
+    {
         false
     }
 
     /// Gets the metadata for this value.
-    fn metadata() -> Metadata {
+    fn metadata() -> Metadata
+    where
+        Self: Sized,
+    {
         Metadata::default()
     }
 
@@ -54,7 +60,10 @@ where
     /// numeric types, there is a limited amount of validation that can occur within the
     /// `Configurable` derive macro, and additional validation must happen at runtime when the
     /// `Configurable` trait is being used, which this method allows for.
-    fn validate_metadata(_metadata: &Metadata) -> Result<(), GenerateError> {
+    fn validate_metadata(_metadata: &Metadata) -> Result<(), GenerateError>
+    where
+        Self: Sized,
+    {
         Ok(())
     }
 
@@ -64,12 +73,14 @@ where
     ///
     /// If an error occurs while generating the schema, an error variant will be returned describing
     /// the issue.
-    fn generate_schema(gen: &RefCell<SchemaGenerator>) -> Result<SchemaObject, GenerateError>;
+    fn generate_schema(gen: &RefCell<SchemaGenerator>) -> Result<SchemaObject, GenerateError>
+    where
+        Self: Sized;
 
     /// Create a new configurable reference table.
     fn as_configurable_ref() -> ConfigurableRef
     where
-        Self: 'static,
+        Self: Sized + 'static,
     {
         ConfigurableRef::new::<Self>()
     }
@@ -98,7 +109,7 @@ pub struct ConfigurableRef {
 
 impl ConfigurableRef {
     /// Create a new configurable reference table.
-    pub const fn new<T: Configurable + ?Sized + 'static>() -> Self {
+    pub const fn new<T: Configurable + 'static>() -> Self {
         Self {
             type_name: std::any::type_name::<T>,
             referenceable_name: T::referenceable_name,

lib/vector-config/src/configurable.rs

@@ -20,7 +20,7 @@ use crate::{
         filter::{self, filter_items},
         relay, sort,
     },
-    config::{ComponentKey, Config, TransformConfig},
+    config::{ComponentKey, Config},
     filter_check,
 };
 use crate::{config::SourceConfig, topology::schema::merged_definition};

src/api/schema/components/mod.rs

@@ -6,7 +6,7 @@ mod test_case;
 pub mod util;
 mod validators;
 
-use crate::{sinks::Sinks, sources::Sources, transforms::Transforms};
+use crate::{config::BoxedTransform, sinks::Sinks, sources::Sources};
 
 pub use self::resources::*;
 #[cfg(feature = "component-validation-runner")]
@@ -43,7 +43,7 @@ pub enum ComponentConfiguration {
     Source(Sources),
 
     /// A transform component.
-    Transform(Transforms),
+    Transform(BoxedTransform),
 
     /// A sink component.
     Sink(Sinks),
@@ -78,7 +78,7 @@ impl ValidationConfiguration {
     }
 
     /// Creates a new `ValidationConfiguration` for a transform.
-    pub fn from_transform<C: Into<Transforms>>(component_name: &'static str, config: C) -> Self {
+    pub fn from_transform(component_name: &'static str, config: impl Into<BoxedTransform>) -> Self {
         Self {
             component_name,
             component_type: ComponentType::Source,

src/components/validation/mod.rs

@@ -4,11 +4,10 @@ use crate::{
         util::GrpcAddress,
         ComponentConfiguration, ComponentType, ValidationConfiguration,
     },
-    config::ConfigBuilder,
+    config::{BoxedTransform, ConfigBuilder},
     sinks::{vector::VectorConfig as VectorSinkConfig, Sinks},
     sources::{vector::VectorConfig as VectorSourceConfig, Sources},
     test_util::next_addr,
-    transforms::Transforms,
 };
 
 use super::{
@@ -57,7 +56,7 @@ impl TopologyBuilder {
         }
     }
 
-    fn from_transform(transform: Transforms) -> Self {
+    fn from_transform(transform: BoxedTransform) -> Self {
         let (input_edge, input_source) = build_input_edge();
         let (output_edge, output_sink) = build_output_edge();
 

src/components/validation/runner/config.rs

@@ -10,16 +10,16 @@ use vector_core::config::GlobalOptions;
 
 use crate::{
     enrichment_tables::EnrichmentTables, providers::Providers, secrets::SecretBackends,
-    sinks::Sinks, sources::Sources, transforms::Transforms,
+    sinks::Sinks, sources::Sources,
 };
 
 #[cfg(feature = "api")]
 use super::api;
 #[cfg(feature = "enterprise")]
 use super::enterprise;
 use super::{
-    compiler, schema, ComponentKey, Config, EnrichmentTableOuter, HealthcheckOptions, SinkOuter,
-    SourceOuter, TestDefinition, TransformOuter,
+    compiler, schema, BoxedTransform, ComponentKey, Config, EnrichmentTableOuter,
+    HealthcheckOptions, SinkOuter, SourceOuter, TestDefinition, TransformOuter,
 };
 
 /// A complete Vector configuration.
@@ -275,11 +275,11 @@ impl ConfigBuilder {
 
     // For some feature sets, no transforms are compiled, which leads to no callers using this
     // method, and in turn, annoying errors about unused variables.
-    pub fn add_transform<K: Into<String>, T: Into<Transforms>>(
+    pub fn add_transform(
         &mut self,
-        key: K,
+        key: impl Into<String>,
         inputs: &[&str],
-        transform: T,
+        transform: impl Into<BoxedTransform>,
     ) {
         let inputs = inputs
             .iter()

src/config/builder.rs

@@ -2,7 +2,7 @@ use indexmap::IndexSet;
 
 use super::{
     builder::ConfigBuilder, graph::Graph, id::Inputs, schema, validation, Config, OutputId,
-    SourceConfig, TransformConfig,
+    SourceConfig,
 };
 
 pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec<String>), Vec<String>> {