Merge 1907a7a into aa9d210

Author: ModProg

Cargo.toml

@@ -21,6 +21,7 @@ syn = { version = "2.0.15", features = ["full", "parsing", "extra-traits"] }
 thiserror = "1.0.37"
 syn_derive = "0.1.6"
 proc-macro2-diagnostics = { version = "0.10", default-features = false }
+derive-where = "1.2.5"
 
 [dev-dependencies]
 proc-macro2 = {version = "1.0.47", features = ["span-locations"]}

benches/bench.rs

@@ -22,27 +22,29 @@ fn criterion_benchmark(c: &mut Criterion) {
 
     c.bench_function("rstml::parse2(simple)", |b| {
         b.iter(|| {
-            let config = rstml::ParserConfig::new().always_self_closed_elements(
-                vec![
-                    "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
-                    "param", "source", "track", "wbr",
-                ]
-                .into_iter()
-                .collect(),
-            );
+            let config: rstml::ParserConfig = rstml::ParserConfig::new()
+                .always_self_closed_elements(
+                    vec![
+                        "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
+                        "param", "source", "track", "wbr",
+                    ]
+                    .into_iter()
+                    .collect(),
+                );
             rstml::Parser::new(config).parse_simple(tokens.clone())
         })
     });
     c.bench_function("rstml::parse2(rust_site)", |b| {
         b.iter(|| {
-            let config = rstml::ParserConfig::new().always_self_closed_elements(
-                vec![
-                    "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
-                    "param", "source", "track", "wbr",
-                ]
-                .into_iter()
-                .collect(),
-            );
+            let config: rstml::ParserConfig = rstml::ParserConfig::new()
+                .always_self_closed_elements(
+                    vec![
+                        "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
+                        "param", "source", "track", "wbr",
+                    ]
+                    .into_iter()
+                    .collect(),
+                );
             rstml::Parser::new(config).parse_simple(tokens.clone())
         })
     });

examples/html-to-string-macro/src/lib.rs

@@ -111,6 +111,7 @@ fn walk_nodes<'a>(empty_elements: &HashSet<&str>, nodes: &'a Vec<Node>) -> WalkN
                 out.static_format.push_str("{}");
                 out.values.push(block.to_token_stream());
             }
+            Node::Custom(custom) => match *custom {},
         }
     }
 

src/config.rs

@@ -1,19 +1,20 @@
-use std::{collections::HashSet, fmt::Debug, rc::Rc};
+use std::{collections::HashSet, convert::Infallible, fmt::Debug, marker::PhantomData, rc::Rc};
 
+use derive_where::derive_where;
 use proc_macro2::TokenStream;
 use syn::{parse::ParseStream, Result};
 
 use crate::{
     atoms::{CloseTag, OpenTag},
-    node::NodeType,
+    node::{CustomNode, NodeType},
 };
 
 pub type TransformBlockFn = dyn Fn(ParseStream) -> Result<Option<TokenStream>>;
 pub type ElementWildcardFn = dyn Fn(&OpenTag, &CloseTag) -> bool;
 
 /// Configures the `Parser` behavior
-#[derive(Default, Clone)]
-pub struct ParserConfig {
+#[derive_where(Clone)]
+pub struct ParserConfig<C = Infallible> {
     pub(crate) flat_tree: bool,
     pub(crate) number_of_top_level_nodes: Option<usize>,
     pub(crate) type_of_top_level_nodes: Option<NodeType>,
@@ -22,9 +23,26 @@ pub struct ParserConfig {
     pub(crate) always_self_closed_elements: HashSet<&'static str>,
     pub(crate) raw_text_elements: HashSet<&'static str>,
     pub(crate) element_close_wildcard: Option<Rc<ElementWildcardFn>>,
+    custom_node: PhantomData<C>,
 }
 
-impl Debug for ParserConfig {
+impl Default for ParserConfig {
+    fn default() -> Self {
+        Self {
+            flat_tree: Default::default(),
+            number_of_top_level_nodes: Default::default(),
+            type_of_top_level_nodes: Default::default(),
+            transform_block: Default::default(),
+            recover_block: Default::default(),
+            always_self_closed_elements: Default::default(),
+            raw_text_elements: Default::default(),
+            element_close_wildcard: Default::default(),
+            custom_node: Default::default(),
+        }
+    }
+}
+
+impl<C> Debug for ParserConfig<C> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ParserConfig")
             .field("flat_tree", &self.flat_tree)
@@ -49,7 +67,9 @@ impl ParserConfig {
     pub fn new() -> ParserConfig {
         ParserConfig::default()
     }
+}
 
+impl<C> ParserConfig<C> {
     /// Return flat tree instead of nested tree
     pub fn flat_tree(mut self) -> Self {
         self.flat_tree = true;
@@ -188,4 +208,20 @@ impl ParserConfig {
             close_tag.name.is_wildcard() && (!open_tag_should_be_block || open_tag.name.is_block())
         })
     }
+
+    /// Enables parsing for [`Node::Custom`] using a type implementing
+    /// [`CustomNode`].
+    pub fn custom_node<CN: CustomNode>(self) -> ParserConfig<CN> {
+        ParserConfig {
+            flat_tree: self.flat_tree,
+            number_of_top_level_nodes: self.number_of_top_level_nodes,
+            type_of_top_level_nodes: self.type_of_top_level_nodes,
+            transform_block: self.transform_block,
+            recover_block: self.recover_block,
+            always_self_closed_elements: self.always_self_closed_elements,
+            raw_text_elements: self.raw_text_elements,
+            element_close_wildcard: self.element_close_wildcard,
+            custom_node: Default::default(),
+        }
+    }
 }

src/lib.rs

@@ -203,14 +203,17 @@
 //! configuration.
 //!
 //!   One highlight with regards to customization is the [`transform_block`]
-//!   configuration, which takes a closure that receives raw block content as
-//!   `ParseStream` and lets you optionally convert it to a `TokenStream`. That
-//! makes it   possible to have custom syntax in blocks. More details in [#9]
+//! configuration, which takes a closure that receives raw block content as
+//! `ParseStream` and lets you optionally convert it to a `TokenStream`. That
+//! makes it possible to have custom syntax in blocks. More details in [#9].
 //!
+//!   Additionally, [`CustomNode`] can be used to implement fully custom
+//! parsing.
 //!
 //! [`syn`]: /syn
 //! [`TokenStream`]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html
 //! [`Node`]: enum.Node.html
+//! [`CustomNode`]: trait.CustomNode.html
 //! [`Span::join`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.join
 //! [`Span::source_text`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.source_text
 //! [`ParserConfig`]: struct.ParserConfig.html
@@ -231,7 +234,7 @@ mod parser;
 pub use config::ParserConfig;
 pub use error::Error;
 pub use node::atoms;
-use node::Node;
+use node::{CustomNode, Node};
 pub use parser::{recoverable, recoverable::ParsingResult, Parser};
 
 /// Parse the given [`proc-macro::TokenStream`] into a [`Node`] tree.
@@ -249,10 +252,10 @@ pub fn parse(tokens: proc_macro::TokenStream) -> Result<Vec<Node>> {
 /// [`Node`]: struct.Node.html
 /// [`ParserConfig`]: struct.ParserConfig.html
 #[deprecated(since = "0.10.2", note = "use rstml::Parser::parse_simple instead")]
-pub fn parse_with_config(
+pub fn parse_with_config<C: CustomNode>(
     tokens: proc_macro::TokenStream,
-    config: ParserConfig,
-) -> Result<Vec<Node>> {
+    config: ParserConfig<C>,
+) -> Result<Vec<Node<C>>> {
     Parser::new(config).parse_simple(tokens)
 }
 /// Parse the given [`proc-macro2::TokenStream`] into a [`Node`] tree.
@@ -270,9 +273,9 @@ pub fn parse2(tokens: proc_macro2::TokenStream) -> Result<Vec<Node>> {
 /// [`Node`]: struct.Node.html
 /// [`ParserConfig`]: struct.ParserConfig.html
 #[deprecated(since = "0.10.2", note = "use rstml::Parser::parse_simple instead")]
-pub fn parse2_with_config(
+pub fn parse2_with_config<C: CustomNode>(
     tokens: proc_macro2::TokenStream,
-    config: ParserConfig,
-) -> Result<Vec<Node>> {
+    config: ParserConfig<C>,
+) -> Result<Vec<Node<C>>> {
     Parser::new(config).parse_simple(tokens)
 }

src/node/mod.rs

@@ -1,16 +1,17 @@
 //! Tree of nodes.
 
-use std::fmt;
+use std::{convert::Infallible, fmt};
 
 use atoms::{tokens, FragmentClose, FragmentOpen};
-use proc_macro2::Ident;
-use syn::{ExprPath, LitStr, Token};
+use proc_macro2::{Ident, TokenStream};
+use syn::{parse::ParseStream, ExprPath, LitStr, Token};
 
 pub mod atoms;
 mod attribute;
 mod node_name;
 mod node_value;
 pub mod parse;
+mod parser_ext;
 mod raw_text;
 
 pub use attribute::{
@@ -20,6 +21,7 @@ pub use node_name::{NodeName, NodeNameFragment};
 pub use node_value::NodeBlock;
 
 pub use self::raw_text::RawText;
+use crate::recoverable::RecoverableContext;
 
 /// Node types.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -31,6 +33,7 @@ pub enum NodeType {
     Block,
     Fragment,
     RawText,
+    Custom,
 }
 
 impl fmt::Display for NodeType {
@@ -46,24 +49,26 @@ impl fmt::Display for NodeType {
                 Self::Doctype => "NodeType::Doctype",
                 Self::Block => "NodeType::Block",
                 Self::Fragment => "NodeType::Fragment",
+                Self::Custom => "NodeType::Custom",
             }
         )
     }
 }
 
 /// Node in the tree.
 #[derive(Clone, Debug, syn_derive::ToTokens)]
-pub enum Node {
+pub enum Node<C: CustomNode = Infallible> {
     Comment(NodeComment),
     Doctype(NodeDoctype),
-    Fragment(NodeFragment),
-    Element(NodeElement),
+    Fragment(NodeFragment<C>),
+    Element(NodeElement<C>),
     Block(NodeBlock),
     Text(NodeText),
     RawText(RawText),
+    Custom(C),
 }
 
-impl Node {
+impl<C: CustomNode> Node<C> {
     pub fn flatten(mut self) -> Vec<Self> {
         let children = self
             .children_mut()
@@ -86,11 +91,12 @@ impl Node {
             Self::Block(_) => NodeType::Block,
             Self::Fragment(_) => NodeType::Fragment,
             Self::RawText(_) => NodeType::RawText,
+            Self::Custom(_) => NodeType::Custom,
         }
     }
 
     /// Get node children.
-    pub fn children(&self) -> Option<&Vec<Node>> {
+    pub fn children(&self) -> Option<&Vec<Self>> {
         match self {
             Self::Fragment(NodeFragment { children, .. })
             | Self::Element(NodeElement { children, .. }) => Some(children),
@@ -99,7 +105,7 @@ impl Node {
     }
 
     /// Get mutable node children.
-    pub fn children_mut(&mut self) -> Option<&mut Vec<Node>> {
+    pub fn children_mut(&mut self) -> Option<&mut Vec<Self>> {
         match self {
             Self::Fragment(NodeFragment { children, .. })
             | Self::Element(NodeElement { children, .. }) => Some(children),
@@ -113,14 +119,14 @@ impl Node {
 /// A HTMLElement tag, with optional children and attributes.
 /// Potentially selfclosing. Any tag name is valid.
 #[derive(Clone, Debug, syn_derive::ToTokens)]
-pub struct NodeElement {
+pub struct NodeElement<C: CustomNode = Infallible> {
     pub open_tag: atoms::OpenTag,
     #[to_tokens(parse::to_tokens_array)]
-    pub children: Vec<Node>,
+    pub children: Vec<Node<C>>,
     pub close_tag: Option<atoms::CloseTag>,
 }
 
-impl NodeElement {
+impl<C: CustomNode> NodeElement<C> {
     pub fn name(&self) -> &NodeName {
         &self.open_tag.name
     }
@@ -177,12 +183,12 @@ pub struct NodeDoctype {
 ///
 /// Fragment: `<></>`
 #[derive(Clone, Debug, syn_derive::ToTokens)]
-pub struct NodeFragment {
+pub struct NodeFragment<C: CustomNode = Infallible> {
     /// Open fragment token
     pub tag_open: FragmentOpen,
     /// Children of the fragment node.
     #[to_tokens(parse::to_tokens_array)]
-    pub children: Vec<Node>,
+    pub children: Vec<Node<C>>,
     /// Close fragment token
     pub tag_close: Option<FragmentClose>,
 }
@@ -195,3 +201,35 @@ fn path_to_string(expr: &ExprPath) -> String {
         .collect::<Vec<String>>()
         .join("::")
 }
+
+pub trait CustomNode: Sized {
+    /// Should correspond to [`ToTokens::to_tokens`].
+    ///
+    /// [`ToTokens::to_tokens`]: quote::ToTokens::to_tokens
+    fn to_tokens(&self, tokens: &mut TokenStream);
+    /// Peeks the token stream to decide whether this node should be parsed.
+    ///
+    /// Recieves a [`ParseStream::fork`].
+    ///
+    /// [`ParseStream::fork`]: syn::parse::ParseBuffer::fork
+    fn peek_element(input: ParseStream) -> bool;
+    /// Parses the custom node, only called when [`peek_element`] returns
+    /// `true`.
+    ///
+    /// [`peek_element`]: Self::peek_element
+    fn parse_element(parser: &mut RecoverableContext, input: ParseStream) -> Option<Self>;
+}
+
+impl CustomNode for Infallible {
+    fn to_tokens(&self, _tokens: &mut TokenStream) {
+        match *self {}
+    }
+
+    fn peek_element(_input: ParseStream) -> bool {
+        false
+    }
+
+    fn parse_element(_parser: &mut RecoverableContext, _input: ParseStream) -> Option<Self> {
+        unreachable!("Infallible::peek_element returns false")
+    }
+}