diff --git a/src/lazy/any_encoding.rs b/src/lazy/any_encoding.rs index 4311df28..7055ea1d 100644 --- a/src/lazy/any_encoding.rs +++ b/src/lazy/any_encoding.rs @@ -12,7 +12,7 @@ use crate::lazy::binary::raw::value::LazyRawBinaryValue; use crate::lazy::decoder::private::{LazyContainerPrivate, LazyRawValuePrivate}; use crate::lazy::decoder::{ LazyDecoder, LazyRawFieldExpr, LazyRawReader, LazyRawSequence, LazyRawStruct, LazyRawValue, - LazyRawValueExpr, + LazyRawValueExpr, RawFieldExpr, RawValueExpr, }; use crate::lazy::encoding::{BinaryEncoding_1_0, Never, TextEncoding_1_0, TextEncoding_1_1}; use crate::lazy::expanded::macro_evaluator::MacroInvocation; @@ -114,11 +114,11 @@ impl<'data> Iterator for LazyRawAnyMacroArgsIterator<'data> { fn next(&mut self) -> Option { match self.encoding { LazyRawAnyMacroArgsIteratorKind::Text_1_1(mut iter) => match iter.next() { - Some(Ok(LazyRawValueExpr::ValueLiteral(value))) => Some(Ok( - LazyRawValueExpr::ValueLiteral(LazyRawAnyValue::from(value)), - )), - Some(Ok(LazyRawValueExpr::MacroInvocation(invocation))) => Some(Ok( - LazyRawValueExpr::MacroInvocation(LazyRawAnyMacroInvocation { + Some(Ok(RawValueExpr::ValueLiteral(value))) => { + Some(Ok(RawValueExpr::ValueLiteral(LazyRawAnyValue::from(value)))) + } + Some(Ok(RawValueExpr::MacroInvocation(invocation))) => Some(Ok( + RawValueExpr::MacroInvocation(LazyRawAnyMacroInvocation { encoding: LazyRawAnyMacroInvocationKind::Text_1_1(invocation), }), )), @@ -217,12 +217,12 @@ impl<'data> From> { fn from(value: LazyRawValueExpr<'data, TextEncoding_1_0>) -> Self { match value { - LazyRawValueExpr::ValueLiteral(v) => LazyRawValueExpr::ValueLiteral(v.into()), - LazyRawValueExpr::MacroInvocation(m) => { + RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), + RawValueExpr::MacroInvocation(m) => { let invocation = LazyRawAnyMacroInvocation { encoding: LazyRawAnyMacroInvocationKind::Text_1_0(m), }; - LazyRawValueExpr::MacroInvocation(invocation) + RawValueExpr::MacroInvocation(invocation) } } } @@ -233,12 +233,12 @@ impl<'data> From> { fn from(value: LazyRawValueExpr<'data, BinaryEncoding_1_0>) -> Self { match value { - LazyRawValueExpr::ValueLiteral(v) => LazyRawValueExpr::ValueLiteral(v.into()), - LazyRawValueExpr::MacroInvocation(m) => { + RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), + RawValueExpr::MacroInvocation(m) => { let invocation = LazyRawAnyMacroInvocation { encoding: LazyRawAnyMacroInvocationKind::Binary_1_0(m), }; - LazyRawValueExpr::MacroInvocation(invocation) + RawValueExpr::MacroInvocation(invocation) } } } @@ -249,12 +249,12 @@ impl<'data> From> { fn from(value: LazyRawValueExpr<'data, TextEncoding_1_1>) -> Self { match value { - LazyRawValueExpr::ValueLiteral(v) => LazyRawValueExpr::ValueLiteral(v.into()), - LazyRawValueExpr::MacroInvocation(m) => { + RawValueExpr::ValueLiteral(v) => RawValueExpr::ValueLiteral(v.into()), + RawValueExpr::MacroInvocation(m) => { let invocation = LazyRawAnyMacroInvocation { encoding: LazyRawAnyMacroInvocationKind::Text_1_1(m), }; - LazyRawValueExpr::MacroInvocation(invocation) + RawValueExpr::MacroInvocation(invocation) } } } @@ -731,11 +731,13 @@ impl<'data> From> { fn from(text_field: LazyRawFieldExpr<'data, TextEncoding_1_0>) -> Self { let (name, value) = match text_field { - LazyRawFieldExpr::NameValuePair(name, value) => (name, value), - LazyRawFieldExpr::MacroInvocation(_) => unreachable!("macro invocation in Ion 1.0"), + RawFieldExpr::NameValuePair(name, value) => (name, value), + RawFieldExpr::MacroInvocation(_) => { + unreachable!("macro invocation in Ion 1.0") + } }; // Convert the text-encoded value into an any-encoded value - LazyRawFieldExpr::NameValuePair(name, value.into()) + RawFieldExpr::NameValuePair(name, value.into()) } } @@ -744,11 +746,13 @@ impl<'data> From> { fn from(binary_field: LazyRawFieldExpr<'data, BinaryEncoding_1_0>) -> Self { let (name, value) = match binary_field { - LazyRawFieldExpr::NameValuePair(name, value) => (name, value), - LazyRawFieldExpr::MacroInvocation(_) => unreachable!("macro invocation in Ion 1.0"), + RawFieldExpr::NameValuePair(name, value) => (name, value), + RawFieldExpr::MacroInvocation(_) => { + unreachable!("macro invocation in Ion 1.0") + } }; // Convert the binary-encoded value into an any-encoded value - LazyRawFieldExpr::NameValuePair(name, value.into()) + RawFieldExpr::NameValuePair(name, value.into()) } } @@ -756,8 +760,8 @@ impl<'data> From> for LazyRawFieldExpr<'data, AnyEncoding> { fn from(text_field: LazyRawFieldExpr<'data, TextEncoding_1_1>) -> Self { - use LazyRawFieldExpr::{MacroInvocation as FieldMacroInvocation, NameValuePair}; - use LazyRawValueExpr::{MacroInvocation as ValueMacroInvocation, ValueLiteral}; + use RawFieldExpr::{MacroInvocation as FieldMacroInvocation, NameValuePair}; + use RawValueExpr::{MacroInvocation as ValueMacroInvocation, ValueLiteral}; match text_field { NameValuePair(name, ValueLiteral(value)) => { NameValuePair(name, ValueLiteral(value.into())) diff --git a/src/lazy/binary/raw/sequence.rs b/src/lazy/binary/raw/sequence.rs index bc2b5a0b..455d453b 100644 --- a/src/lazy/binary/raw/sequence.rs +++ b/src/lazy/binary/raw/sequence.rs @@ -3,7 +3,7 @@ use crate::lazy::binary::raw::annotations_iterator::RawBinaryAnnotationsIterator use crate::lazy::binary::raw::reader::DataSource; use crate::lazy::binary::raw::value::LazyRawBinaryValue; use crate::lazy::decoder::private::LazyContainerPrivate; -use crate::lazy::decoder::{LazyRawSequence, LazyRawValueExpr}; +use crate::lazy::decoder::{LazyRawSequence, LazyRawValueExpr, RawValueExpr}; use crate::lazy::encoding::BinaryEncoding_1_0; use crate::{IonResult, IonType}; use std::fmt::{Debug, Formatter}; @@ -145,7 +145,7 @@ impl<'data> Iterator for RawBinarySequenceIterator<'data> { .source .try_parse_next(ImmutableBuffer::peek_sequence_value) { - Ok(Some(value)) => Some(Ok(LazyRawValueExpr::ValueLiteral(value))), + Ok(Some(value)) => Some(Ok(RawValueExpr::ValueLiteral(value))), Ok(None) => None, Err(e) => Some(Err(e)), } diff --git a/src/lazy/binary/raw/struct.rs b/src/lazy/binary/raw/struct.rs index ebaa9a50..d7c9a103 100644 --- a/src/lazy/binary/raw/struct.rs +++ b/src/lazy/binary/raw/struct.rs @@ -8,7 +8,9 @@ use crate::lazy::binary::raw::value::LazyRawBinaryValue; use crate::lazy::decoder::private::{ LazyContainerPrivate, LazyRawFieldPrivate, LazyRawValuePrivate, }; -use crate::lazy::decoder::{LazyRawField, LazyRawFieldExpr, LazyRawStruct, LazyRawValueExpr}; +use crate::lazy::decoder::{ + LazyRawField, LazyRawFieldExpr, LazyRawStruct, RawFieldExpr, RawValueExpr, +}; use crate::lazy::encoding::BinaryEncoding_1_0; use crate::{IonResult, RawSymbolTokenRef}; @@ -87,9 +89,9 @@ impl<'data> Iterator for RawBinaryStructIterator<'data> { fn next(&mut self) -> Option { match self.source.try_parse_next(ImmutableBuffer::peek_field) { - Ok(Some(lazy_raw_value)) => Some(Ok(LazyRawFieldExpr::NameValuePair( + Ok(Some(lazy_raw_value)) => Some(Ok(RawFieldExpr::NameValuePair( lazy_raw_value.field_name().unwrap(), - LazyRawValueExpr::ValueLiteral(lazy_raw_value), + RawValueExpr::ValueLiteral(lazy_raw_value), ))), Ok(None) => None, Err(e) => Some(Err(e)), diff --git a/src/lazy/decoder.rs b/src/lazy/decoder.rs index 8a54eed9..a6836378 100644 --- a/src/lazy/decoder.rs +++ b/src/lazy/decoder.rs @@ -35,35 +35,45 @@ where type MacroInvocation: MacroInvocation<'data, Self>; } -/// An item found in value position within an Ion data stream. -/// This item may be either a value literal or a macro invocation. +/// An expression found in value position in either serialized Ion or a template. /// If it is a value literal, it is considered a stream with exactly one Ion value. /// If it is a macro invocation, it is a stream with zero or more Ion values. #[derive(Copy, Clone, Debug)] -pub enum LazyRawValueExpr<'data, D: LazyDecoder<'data>> { - /// A text Ion 1.1 value literal. For example: `5`, `foo`, or `"hello"` - ValueLiteral(D::Value), - /// A text Ion 1.1 macro invocation. For example: `(:employee 12345 "Sarah" "Gonzalez")` - MacroInvocation(D::MacroInvocation), +pub enum RawValueExpr { + /// A value literal. For example: `5`, `foo`, or `"hello"` in text. + ValueLiteral(V), + /// An Ion 1.1+ macro invocation. For example: `(:employee 12345 "Sarah" "Gonzalez")` in text. + MacroInvocation(M), } -impl<'data, D: LazyDecoder<'data>> LazyRawValueExpr<'data, D> { - pub fn expect_value(self) -> IonResult { +// `RawValueExpr` above has no ties to a particular encoding. The `LazyRawValueExpr` type alias +// below uses the `Value` and `MacroInvocation` associated types from the decoder `D`. In most +// places, this is a helpful constraint; we can talk about the value expression in terms of the +// LazyDecoder it's associated with. However, in some places (primarily when expanding template +// values that don't have a LazyDecoder) we need to be able to use it without constraints. + +/// An item found in value position within an Ion data stream written in the encoding represented +/// by the LazyDecoder `D`. This item may be either a value literal or a macro invocation. +pub type LazyRawValueExpr<'data, D> = + RawValueExpr<>::Value, >::MacroInvocation>; + +impl RawValueExpr { + pub fn expect_value(self) -> IonResult { match self { - LazyRawValueExpr::ValueLiteral(v) => Ok(v), - LazyRawValueExpr::MacroInvocation(_m) => { - IonResult::decoding_error("expected a value literal, but found macro invocation") - } + RawValueExpr::ValueLiteral(v) => Ok(v), + RawValueExpr::MacroInvocation(_m) => IonResult::decoding_error( + "expected a value literal, but found a macro invocation ({:?})", + ), } } - pub fn expect_macro(self) -> IonResult { + pub fn expect_macro(self) -> IonResult { match self { - LazyRawValueExpr::ValueLiteral(v) => IonResult::decoding_error(format!( - "expected a macro invocation but found a value literal {:?}", + RawValueExpr::ValueLiteral(v) => IonResult::decoding_error(format!( + "expected a macro invocation but found a value literal ({:?})", v )), - LazyRawValueExpr::MacroInvocation(m) => Ok(m), + RawValueExpr::MacroInvocation(m) => Ok(m), } } } @@ -74,15 +84,27 @@ impl<'data, D: LazyDecoder<'data>> LazyRawValueExpr<'data, D> { /// * a name/e-expression pair /// * an e-expression #[derive(Debug)] -pub enum LazyRawFieldExpr<'data, D: LazyDecoder<'data>> { - NameValuePair(RawSymbolTokenRef<'data>, LazyRawValueExpr<'data, D>), - MacroInvocation(D::MacroInvocation), +pub enum RawFieldExpr<'name, V, M> { + NameValuePair(RawSymbolTokenRef<'name>, RawValueExpr), + MacroInvocation(M), } -impl<'data, D: LazyDecoder<'data>> LazyRawFieldExpr<'data, D> { - pub fn expect_name_value(self) -> IonResult<(RawSymbolTokenRef<'data>, D::Value)> { +// As with the `RawValueExpr`/`LazyRawValueExpr` type pair, a `RawFieldExpr` has no constraints +// on the types used for values or macros, while the `LazyRawFieldExpr` type alias below uses the +// value and macro types associated with the decoder `D`. + +/// An item found in struct field position an Ion data stream written in the encoding represented +/// by the LazyDecoder `D`. +pub type LazyRawFieldExpr<'data, D> = RawFieldExpr< + 'data, + >::Value, + >::MacroInvocation, +>; + +impl<'name, V: Debug, M: Debug> RawFieldExpr<'name, V, M> { + pub fn expect_name_value(self) -> IonResult<(RawSymbolTokenRef<'name>, V)> { match self { - LazyRawFieldExpr::NameValuePair(name, LazyRawValueExpr::ValueLiteral(value)) => { + RawFieldExpr::NameValuePair(name, RawValueExpr::ValueLiteral(value)) => { Ok((name, value)) } _ => IonResult::decoding_error(format!( @@ -92,12 +114,11 @@ impl<'data, D: LazyDecoder<'data>> LazyRawFieldExpr<'data, D> { } } - pub fn expect_name_macro(self) -> IonResult<(RawSymbolTokenRef<'data>, D::MacroInvocation)> { + pub fn expect_name_macro(self) -> IonResult<(RawSymbolTokenRef<'name>, M)> { match self { - LazyRawFieldExpr::NameValuePair( - name, - LazyRawValueExpr::MacroInvocation(invocation), - ) => Ok((name, invocation)), + RawFieldExpr::NameValuePair(name, RawValueExpr::MacroInvocation(invocation)) => { + Ok((name, invocation)) + } _ => IonResult::decoding_error(format!( "expected a name/macro pair but found {:?}", self @@ -105,9 +126,9 @@ impl<'data, D: LazyDecoder<'data>> LazyRawFieldExpr<'data, D> { } } - pub fn expect_macro(self) -> IonResult { + pub fn expect_macro(self) -> IonResult { match self { - LazyRawFieldExpr::MacroInvocation(invocation) => Ok(invocation), + RawFieldExpr::MacroInvocation(invocation) => Ok(invocation), _ => IonResult::decoding_error(format!( "expected a macro invocation but found {:?}", self @@ -125,6 +146,7 @@ impl<'data, D: LazyDecoder<'data>> LazyRawFieldExpr<'data, D> { // internal code that is defined in terms of `LazyRawField` to call the private `into_value()` // function while also preventing users from seeing or depending on it. pub(crate) mod private { + use crate::lazy::encoding::RawValueLiteral; use crate::{IonResult, RawSymbolTokenRef}; use super::LazyDecoder; @@ -143,7 +165,7 @@ pub(crate) mod private { fn from_value(value: D::Value) -> Self; } - pub trait LazyRawValuePrivate<'data> { + pub trait LazyRawValuePrivate<'data>: RawValueLiteral { /// Returns the field name associated with this value. If the value is not inside a struct, /// returns `IllegalOperation`. fn field_name(&self) -> IonResult>; diff --git a/src/lazy/encoding.rs b/src/lazy/encoding.rs index 49cecc6a..7c24f39d 100644 --- a/src/lazy/encoding.rs +++ b/src/lazy/encoding.rs @@ -1,5 +1,6 @@ #![allow(non_camel_case_types)] +use crate::lazy::any_encoding::LazyRawAnyValue; use crate::lazy::binary::raw::annotations_iterator::RawBinaryAnnotationsIterator; use crate::lazy::binary::raw::r#struct::LazyRawBinaryStruct; use crate::lazy::binary::raw::reader::LazyRawBinaryReader; @@ -131,3 +132,21 @@ impl<'data> LazyDecoder<'data> for TextEncoding_1_1 { type AnnotationsIterator = RawTextAnnotationsIterator<'data>; type MacroInvocation = RawTextMacroInvocation<'data>; } + +/// Marker trait for types that represent value literals in an Ion stream of some encoding. +// This trait is used to provide generic conversion implementation of types used as a +// `LazyDecoder::Value` to `ExpandedValueSource`. That is: +// +// impl<'top, 'data, V: RawValueLiteral, D: LazyDecoder<'data, Value = V>> From +// for ExpandedValueSource<'top, 'data, D> +// +// If we do not confine the implementation to types with a marker trait, rustc complains that +// someone may someday use `ExpandedValueSource` as a `LazyDecoder::Value`, and then the +// the implementation will conflict with the core `impl From for T` implementation. +pub trait RawValueLiteral {} + +impl<'data> RawValueLiteral for MatchedRawTextValue<'data> {} +impl<'data> RawValueLiteral for LazyRawTextValue_1_0<'data> {} +impl<'data> RawValueLiteral for LazyRawTextValue_1_1<'data> {} +impl<'data> RawValueLiteral for LazyRawBinaryValue<'data> {} +impl<'data> RawValueLiteral for LazyRawAnyValue<'data> {} diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index c8de1ee5..b1d87a81 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -1,6 +1,6 @@ //! Types and traits representing an e-expression in an Ion stream. -use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr}; +use crate::lazy::decoder::{LazyDecoder, LazyRawValueExpr, RawValueExpr}; use crate::lazy::expanded::macro_evaluator::{ArgumentKind, ToArgumentKind}; use crate::lazy::expanded::{EncodingContext, ExpandedValueSource, LazyExpandedValue}; @@ -21,15 +21,11 @@ impl<'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, D::MacroInvocation> // Because e-expressions appear in the data stream (and not in a template), there is no // environment of named variables. We do not attempt to resolve symbols as though they // were variable names and instead pass them along as value literals. - LazyRawValueExpr::ValueLiteral(value) => { - ArgumentKind::ValueLiteral(LazyExpandedValue { - context, - source: ExpandedValueSource::ValueLiteral(value), - }) - } - LazyRawValueExpr::MacroInvocation(invocation) => { - ArgumentKind::MacroInvocation(invocation) - } + RawValueExpr::ValueLiteral(value) => ArgumentKind::ValueLiteral(LazyExpandedValue { + context, + source: ExpandedValueSource::ValueLiteral(value), + }), + RawValueExpr::MacroInvocation(invocation) => ArgumentKind::MacroInvocation(invocation), } } } diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index 7c04b26a..6157d618 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -45,7 +45,7 @@ pub trait MacroInvocation<'data, D: LazyDecoder<'data>>: Copy + Clone + Debug { } /// A single expression appearing in argument position within a macro invocation. -pub enum ArgumentKind<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { +pub enum ArgumentKind<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { /// An Ion value that requires no further evaluation. ValueLiteral(LazyExpandedValue<'top, 'data, D>), /// A variable name that requires expansion. @@ -98,7 +98,8 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MacroExpansion< context: EncodingContext<'top>, ) -> IonResult> where - M: 'data + 'top, + 'data: 'top, + M: 'top, { use MacroExpansionKind::*; // Delegate the call to `next()` based on the macro kind. @@ -112,8 +113,7 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MacroExpansion< } /// Represents a single step in the process of evaluating a macro. -pub enum MacroExpansionStep<'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> -{ +pub enum MacroExpansionStep<'top, 'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> { /// The next value produced by continuing the macro evaluation. ExpandedValue(LazyExpandedValue<'top, 'data, D>), /// Another macro that will need to be evaluated before an expanded value can be returned. @@ -143,7 +143,7 @@ pub struct MacroEvaluator< > { // A stack of all of the macro invocations currently being evaluated. macro_stack: S, - spooky: PhantomData<(&'data D, &'data M)>, + spooky: PhantomData<(&'data D, M)>, } impl< @@ -164,7 +164,7 @@ impl< /// Constructs a `MacroEvaluator` with a lifetime tied to the current [`EncodingContext`]. pub fn new_transient<'top>( context: EncodingContext<'top>, - ) -> MacroEvaluator<'data, D, M, BumpVec>> { + ) -> MacroEvaluator<'data, D, M, BumpVec<'top, MacroExpansion<'data, D, M>>> { MacroEvaluator { macro_stack: BumpVec::new_in(context.allocator), spooky: PhantomData, @@ -254,6 +254,7 @@ impl< ) -> IonResult>> where 'data: 'top, + M: 'top, { debug_assert!( self.stack_depth() >= depth_to_exhaust, @@ -302,11 +303,11 @@ impl< /// Attempts to resolve the provided `invocation` in the specified `context`. Upon success, /// returns an iterator that lazily computes the expansion of the macro invocation and yields /// its values. - pub(crate) fn evaluate<'top>( - &mut self, + pub(crate) fn evaluate<'iter, 'top>( + &'iter mut self, context: EncodingContext<'top>, invocation: M, - ) -> IonResult> { + ) -> IonResult> { self.push(context, invocation)?; Ok(EvaluatingIterator::new(self, context)) } @@ -354,9 +355,16 @@ pub type TransientEExpEvaluator<'top, 'data, D> = MacroEvaluator< /// A [`MacroEvaluator`] for expanding macro invocations found in a template body, all in the context /// of a data stream in the format `D`. -pub type TdlTemplateEvaluator<'top, 'data, D> = +pub type TdlMacroEvaluator<'top, 'data, D> = MacroEvaluator<'data, D, &'top Sequence, Vec>>; +pub type TransientTdlMacroEvaluator<'top, 'data, D> = MacroEvaluator< + 'data, + D, + &'top Sequence, + BumpVec<'top, MacroExpansion<'data, D, &'top Sequence>>, +>; + /// Yields the values produced by incrementally evaluating the macro that was at the top of the /// evaluator's stack when the iterator was created. pub struct EvaluatingIterator< @@ -370,12 +378,11 @@ pub struct EvaluatingIterator< evaluator: &'iter mut MacroEvaluator<'data, D, M, S>, context: EncodingContext<'top>, initial_stack_depth: usize, - spooky: PhantomData<&'data D>, } impl< 'iter, - 'top: 'iter, + 'top, 'data: 'top, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>, @@ -391,7 +398,6 @@ impl< evaluator, context, initial_stack_depth, - spooky: PhantomData, } } } @@ -401,7 +407,7 @@ impl< 'top, 'data: 'top, D: LazyDecoder<'data>, - M: MacroInvocation<'data, D>, + M: MacroInvocation<'data, D> + 'top, S: Stack>, > Iterator for EvaluatingIterator<'iter, 'top, 'data, D, M, S> { @@ -448,6 +454,7 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> ValuesExpansion context: EncodingContext<'top>, ) -> IonResult> where + 'data: 'top, M: 'top, { // We visit the argument expressions in the invocation in order from left to right. @@ -510,7 +517,8 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan context: EncodingContext<'top>, ) -> IonResult> where - M: 'data + 'top, + 'data: 'top, + M: 'top, { // `make_string` always produces a single value. Once that value has been returned, it needs // to report `Complete` on the following call to `next()`. @@ -527,7 +535,9 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan // itself. Instead, we use the bump allocator the make a transient macro evaluator // whose resources can be trivially reclaimed when the expansion is done. let mut evaluator = - MacroEvaluator::<'data, D, M, BumpVec<'top, MacroExpansion>>::new(); + MacroEvaluator::<'data, D, M, BumpVec<'top, MacroExpansion>>::new_transient( + context, + ); for arg in self.arguments.by_ref() { let arg_expr: ArgumentKind = arg?.to_arg_expr(context); @@ -600,15 +610,12 @@ impl<'data, D: LazyDecoder<'data>, M: MacroInvocation<'data, D>> MakeStringExpan mod tests { use bumpalo::Bump as BumpAllocator; - use crate::lazy::decoder::LazyRawReader; use crate::lazy::encoding::TextEncoding_1_1; - use crate::lazy::expanded::macro_evaluator::TdlTemplateEvaluator; + use crate::lazy::expanded::macro_evaluator::TdlMacroEvaluator; use crate::lazy::expanded::macro_table::MacroTable; use crate::lazy::expanded::EncodingContext; - use crate::lazy::expanded::ExpandedStreamItem; - use crate::lazy::expanded::LazyExpandingReader; use crate::lazy::reader::LazyTextReader_1_1; - use crate::lazy::text::raw::v1_1::reader::LazyRawTextReader_1_1; + use crate::lazy::value::LazyValue; use crate::{Element, ElementReader, IonResult, SymbolTable}; /// Reads `input` and `expected` using an expanding reader and asserts that their output @@ -640,21 +647,18 @@ mod tests { let symbol_table = SymbolTable::new(); let allocator = BumpAllocator::new(); let context = EncodingContext::new(¯o_table, &symbol_table, &allocator); - let mut evaluator = TdlTemplateEvaluator::::new(); + let mut evaluator = TdlMacroEvaluator::::new(); let invocation = Element::read_one(invocation)?; let actuals = evaluator.evaluate(context, invocation.expect_sexp()?)?; - let raw_reader = LazyRawTextReader_1_1::new(expected.as_ref()); - let mut expected_reader = LazyExpandingReader::::new(raw_reader); - for actual in actuals { + let mut expected_reader = LazyTextReader_1_1::new(expected.as_bytes())?; + for actual_result in actuals { // Read the next expected value as a raw value, then wrap it in an `ExpandedRawValueRef` // so it can be directly compared to the actual. - let expected = expected_reader.next(context)?.expect_value()?.read()?; - assert_eq!(actual?.read()?, expected); + let expected: Element = expected_reader.next()?.unwrap().read()?.try_into()?; + let actual: Element = LazyValue::from(actual_result?).try_into()?; + assert_eq!(actual, expected); } - assert!(matches!( - expected_reader.next(context), - Ok(ExpandedStreamItem::EndOfStream) - )); + assert!(matches!(expected_reader.next(), Ok(None))); Ok(()) } @@ -722,6 +726,27 @@ mod tests { Ok(()) } + #[test] + fn macros_inside_a_tdl_list() -> IonResult<()> { + eval_tdl_template_invocation( + r#" + (values [ + 1, + 2, + (values 3 4), + 5, + (void), + (void), + 6, + (make_string "foo" "bar" "baz"), + 7 + ]) + "#, + "[1, 2, 3, 4, 5, 6, \"foobarbaz\", 7]", + )?; + Ok(()) + } + #[test] fn e_expressions_inside_a_sexp() -> IonResult<()> { eval_enc_expr( @@ -731,6 +756,10 @@ mod tests { Ok(()) } + // TODO: macros_inside_a_tdl_sexp() + // This requires an implementation of TDL's `(make_sexp)` or `(quote)`. Without these, + // a sexp is always considered a TDL macro invocation. + #[test] fn e_expressions_inside_a_struct() -> IonResult<()> { eval_enc_expr( @@ -779,4 +808,55 @@ mod tests { )?; Ok(()) } + + #[test] + fn macros_inside_a_tdl_struct() -> IonResult<()> { + eval_tdl_template_invocation( + r#" + (values { + a: 1, + + // When a macro in field value position produces more than one value, + // a field will be emitted for each value. The same field name will be used for + // each one. + b: (values 2 3), + + c: 4, + + // If the value-position-macro doesn't produce any values, the field will not + // appear in the expansion. + d: (void), + + // If a single value is produced, a single field with that value will appear in the + // output. + e: (make_string "foo" "bar" "baz"), + + // Nested struct to demonstrate recursive expansion + f: { + quux: 5, + quuz: (values true false), + }, + + g: 6 + }) + "#, + r#" + { + a: 1, + b: 2, + b: 3, + c: 4, + // no 'd', + e: "foobarbaz", + f: { + quux: 5, + quuz: true, + quuz: false, + }, + g: 6, + } + "#, + )?; + Ok(()) + } } diff --git a/src/lazy/expanded/mod.rs b/src/lazy/expanded/mod.rs index 7e4151b1..cb0fb818 100644 --- a/src/lazy/expanded/mod.rs +++ b/src/lazy/expanded/mod.rs @@ -43,12 +43,16 @@ use sequence::{LazyExpandedList, LazyExpandedSExp}; use crate::element::iterators::SymbolsIterator; use crate::lazy::bytes_ref::BytesRef; use crate::lazy::decoder::{LazyDecoder, LazyRawReader, LazyRawValue}; +use crate::lazy::encoding::RawValueLiteral; use crate::lazy::expanded::macro_evaluator::EExpEvaluator; use crate::lazy::expanded::macro_table::MacroTable; use crate::lazy::expanded::r#struct::LazyExpandedStruct; +use crate::lazy::r#struct::LazyStruct; use crate::lazy::raw_stream_item::RawStreamItem; use crate::lazy::raw_value_ref::RawValueRef; +use crate::lazy::sequence::{LazyList, LazySExp}; use crate::lazy::str_ref::StrRef; +use crate::lazy::value::LazyValue; use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; use crate::result::IonFailure; use crate::{ @@ -205,6 +209,25 @@ pub enum ExpandedValueSource<'top, 'data, D: LazyDecoder<'data>> { ), } +// Converts the raw value literal types associated with each format decoder (e.g. LazyRawTextValue_1_1) +// into an ExpandedValueSource. +impl<'top, 'data, V: RawValueLiteral, D: LazyDecoder<'data, Value = V>> From + for ExpandedValueSource<'top, 'data, D> +{ + fn from(value: V) -> Self { + ExpandedValueSource::ValueLiteral(value) + } +} + +// Converts an Element from the body of a template into an ExpandedValueSource. +impl<'top, 'data, D: LazyDecoder<'data>> From<&'top Element> + for ExpandedValueSource<'top, 'data, D> +{ + fn from(element: &'top Element) -> Self { + ExpandedValueSource::Template(element) + } +} + /// A value produced by expanding the 'raw' view of the input data. #[derive(Clone)] pub struct LazyExpandedValue<'top, 'data, D: LazyDecoder<'data>> { @@ -212,13 +235,27 @@ pub struct LazyExpandedValue<'top, 'data, D: LazyDecoder<'data>> { pub(crate) source: ExpandedValueSource<'top, 'data, D>, } -impl<'top, 'data: 'top, D: LazyDecoder<'data>> Debug for LazyExpandedValue<'top, 'data, D> { +impl<'top, 'data, D: LazyDecoder<'data>> Debug for LazyExpandedValue<'top, 'data, D> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self.source) } } impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> { + pub(crate) fn from_value(context: EncodingContext<'top>, value: D::Value) -> Self { + Self { + context, + source: ExpandedValueSource::ValueLiteral(value), + } + } + + pub(crate) fn from_template(context: EncodingContext<'top>, element: &'top Element) -> Self { + Self { + context, + source: ExpandedValueSource::Template(element), + } + } + pub fn ion_type(&self) -> IonType { use ExpandedValueSource::*; match &self.source { @@ -272,6 +309,38 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedValue<'top, 'data, D> } } +impl<'top, 'data, D: LazyDecoder<'data>> From> + for LazyValue<'top, 'data, D> +{ + fn from(expanded_value: LazyExpandedValue<'top, 'data, D>) -> Self { + LazyValue { expanded_value } + } +} + +impl<'top, 'data, D: LazyDecoder<'data>> From> + for LazyStruct<'top, 'data, D> +{ + fn from(expanded_struct: LazyExpandedStruct<'top, 'data, D>) -> Self { + LazyStruct { expanded_struct } + } +} + +impl<'top, 'data, D: LazyDecoder<'data>> From> + for LazySExp<'top, 'data, D> +{ + fn from(expanded_sexp: LazyExpandedSExp<'top, 'data, D>) -> Self { + LazySExp { expanded_sexp } + } +} + +impl<'top, 'data, D: LazyDecoder<'data>> From> + for LazyList<'top, 'data, D> +{ + fn from(expanded_list: LazyExpandedList<'top, 'data, D>) -> Self { + LazyList { expanded_list } + } +} + pub enum ExpandedAnnotationsSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral(D::AnnotationsIterator), Template(SymbolsIterator<'top>), diff --git a/src/lazy/expanded/sequence.rs b/src/lazy/expanded/sequence.rs index 1a6a6752..2d350ee3 100644 --- a/src/lazy/expanded/sequence.rs +++ b/src/lazy/expanded/sequence.rs @@ -1,5 +1,5 @@ -use crate::lazy::decoder::{LazyDecoder, LazyRawSequence, LazyRawValueExpr}; -use crate::lazy::expanded::macro_evaluator::TransientEExpEvaluator; +use crate::lazy::decoder::{LazyDecoder, LazyRawSequence, LazyRawValueExpr, RawValueExpr}; +use crate::lazy::expanded::macro_evaluator::{TransientEExpEvaluator, TransientTdlMacroEvaluator}; use crate::lazy::expanded::template::TemplateSequenceIterator; use crate::lazy::expanded::{ EncodingContext, ExpandedAnnotationsIterator, ExpandedAnnotationsSource, ExpandedValueSource, @@ -62,6 +62,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedList<'top, 'data, D> { ExpandedListSource::Template(_annotations, sequence) => { ExpandedListIteratorSource::Template(TemplateSequenceIterator::new( self.context, + TransientTdlMacroEvaluator::new_transient(self.context), sequence, )) } @@ -73,14 +74,14 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedList<'top, 'data, D> { } } -pub enum ExpandedListIteratorSource<'top, 'data: 'top, D: LazyDecoder<'data>> { +pub enum ExpandedListIteratorSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral( // Giving the list iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. TransientEExpEvaluator<'top, 'data, D>, >::Iterator, ), - Template(TemplateSequenceIterator<'top>), + Template(TemplateSequenceIterator<'top, 'data, D>), // TODO: Constructed } @@ -97,12 +98,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedListIterator<'top, ExpandedListIteratorSource::ValueLiteral(evaluator, iter) => { expand_next_sequence_value(self.context, evaluator, iter) } - ExpandedListIteratorSource::Template(iter) => iter.next().map(|element| { - Ok(LazyExpandedValue { - source: ExpandedValueSource::Template(element), - context: self.context, - }) - }), + ExpandedListIteratorSource::Template(iter) => iter.next(), } } } @@ -115,8 +111,8 @@ pub enum ExpandedSExpSource<'top, 'data, D: LazyDecoder<'data>> { #[derive(Clone)] pub struct LazyExpandedSExp<'top, 'data, D: LazyDecoder<'data>> { - source: ExpandedSExpSource<'top, 'data, D>, - context: EncodingContext<'top>, + pub(crate) source: ExpandedSExpSource<'top, 'data, D>, + pub(crate) context: EncodingContext<'top>, } impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { @@ -144,6 +140,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { ExpandedSExpSource::Template(_annotations, sequence) => { ExpandedSExpIteratorSource::Template(TemplateSequenceIterator::new( self.context, + TransientTdlMacroEvaluator::new_transient(self.context), sequence, )) } @@ -172,14 +169,14 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyExpandedSExp<'top, 'data, D> { } } -pub enum ExpandedSExpIteratorSource<'top, 'data: 'top, D: LazyDecoder<'data>> { +pub enum ExpandedSExpIteratorSource<'top, 'data, D: LazyDecoder<'data>> { ValueLiteral( // Giving the sexp iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. TransientEExpEvaluator<'top, 'data, D>, >::Iterator, ), - Template(TemplateSequenceIterator<'top>), + Template(TemplateSequenceIterator<'top, 'data, D>), // TODO: Constructed } @@ -196,12 +193,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedSExpIterator<'top, ExpandedSExpIteratorSource::ValueLiteral(evaluator, iter) => { expand_next_sequence_value(self.context, evaluator, iter) } - ExpandedSExpIteratorSource::Template(iter) => iter.next().map(|element| { - Ok(LazyExpandedValue { - source: ExpandedValueSource::Template(element), - context: self.context, - }) - }), + ExpandedSExpIteratorSource::Template(iter) => iter.next(), } } } @@ -227,13 +219,13 @@ fn expand_next_sequence_value<'top, 'data, D: LazyDecoder<'data>>( match iter.next() { None => return None, - Some(Ok(LazyRawValueExpr::ValueLiteral(value))) => { + Some(Ok(RawValueExpr::ValueLiteral(value))) => { return Some(Ok(LazyExpandedValue { source: ExpandedValueSource::ValueLiteral(value), context, })) } - Some(Ok(LazyRawValueExpr::MacroInvocation(invocation))) => { + Some(Ok(RawValueExpr::MacroInvocation(invocation))) => { let begin_expansion_result = evaluator.push(context, invocation); if let Err(e) = begin_expansion_result { return Some(Err(e)); diff --git a/src/lazy/expanded/struct.rs b/src/lazy/expanded/struct.rs index a3fbe6b1..a544e818 100644 --- a/src/lazy/expanded/struct.rs +++ b/src/lazy/expanded/struct.rs @@ -1,5 +1,12 @@ -use crate::lazy::decoder::{LazyDecoder, LazyRawFieldExpr, LazyRawStruct, LazyRawValueExpr}; -use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, TransientEExpEvaluator}; +use std::ops::ControlFlow; + +use crate::lazy::decoder::{LazyDecoder, LazyRawStruct, RawFieldExpr, RawValueExpr}; +use crate::lazy::expanded::macro_evaluator::{ + MacroEvaluator, MacroExpansion, MacroInvocation, TransientEExpEvaluator, + TransientTdlMacroEvaluator, +}; +use crate::lazy::expanded::stack::Stack; +use crate::lazy::expanded::template::TemplateStructRawFieldsIterator; use crate::lazy::expanded::{ EncodingContext, ExpandedAnnotationsIterator, ExpandedAnnotationsSource, ExpandedValueRef, ExpandedValueSource, LazyExpandedValue, @@ -37,8 +44,8 @@ pub enum ExpandedStructSource<'top, 'data, D: LazyDecoder<'data>> { #[derive(Clone)] pub struct LazyExpandedStruct<'top, 'data, D: LazyDecoder<'data>> { - context: EncodingContext<'top>, - source: ExpandedStructSource<'top, 'data, D>, + pub(crate) context: EncodingContext<'top>, + pub(crate) source: ExpandedStructSource<'top, 'data, D>, } impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D> { @@ -82,8 +89,11 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D raw_struct.iter(), ) } - ExpandedStructSource::Template(_, _) => { - todo!("iterate over struct from template") + ExpandedStructSource::Template(_annotations, struct_) => { + ExpandedStructIteratorSource::Template( + TransientTdlMacroEvaluator::new_transient(self.context), + TemplateStructRawFieldsIterator::new(struct_), + ) } }; ExpandedStructIterator { @@ -97,7 +107,7 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D self.context.allocator.alloc_with(|| self.iter()) } - fn find(&self, name: &str) -> IonResult>> { + pub fn find(&self, name: &str) -> IonResult>> { for field_result in self.iter() { let field = field_result?; if field.name() == name.as_raw_symbol_token_ref() { @@ -107,11 +117,11 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D Ok(None) } - fn get(&self, name: &str) -> IonResult>> { + pub fn get(&self, name: &str) -> IonResult>> { self.find(name)?.map(|f| f.read()).transpose() } - fn get_expected(&self, name: &str) -> IonResult> { + pub fn get_expected(&self, name: &str) -> IonResult> { if let Some(value) = self.get(name)? { Ok(value) } else { @@ -120,14 +130,21 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyExpandedStruct<'top, 'data, D } } -pub enum ExpandedStructIteratorSource<'top, 'data: 'top, D: LazyDecoder<'data>> { +pub enum ExpandedStructIteratorSource<'top, 'data, D: LazyDecoder<'data>> { + // The struct we're iterating over is a literal in the data stream. It may contain + // e-expressions that need to be evaluated. ValueLiteral( // Giving the struct iterator its own evaluator means that we can abandon the iterator // at any time without impacting the evaluation state of its parent container. TransientEExpEvaluator<'top, 'data, D>, >::Iterator, ), - // TODO: Template + // The struct we're iterating over is a value in a TDL template. It may contain macro + // invocations that need to be evaluated. + Template( + TransientTdlMacroEvaluator<'top, 'data, D>, + TemplateStructRawFieldsIterator<'top>, + ), // TODO: Constructed } @@ -140,7 +157,7 @@ pub struct ExpandedStructIterator<'top, 'data, D: LazyDecoder<'data>> { /// Ion 1.1's struct is very versatile, and supports a variety of expansion operations. This /// types indicates which operation is in the process of being carried out. -enum ExpandedStructIteratorState<'top, 'data, D: LazyDecoder<'data>> { +enum ExpandedStructIteratorState<'top, 'data: 'top, D: LazyDecoder<'data>> { // The iterator is not performing any operations. It is ready to pull the next field from its // source. ReadingFieldFromSource, @@ -167,7 +184,7 @@ enum ExpandedStructIteratorState<'top, 'data, D: LazyDecoder<'data>> { ), } -impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedStructIterator<'top, 'data, D> { +impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator for ExpandedStructIterator<'top, 'data, D> { type Item = IonResult>; fn next(&mut self) -> Option { @@ -177,101 +194,212 @@ impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ExpandedStructIterator<'to ref mut state, } = *self; match source { - ExpandedStructIteratorSource::ValueLiteral(evaluator, iter) => { - loop { - use ExpandedStructIteratorState::*; - match state { - ReadingFieldFromSource => { - match iter.next()? { - Err(e) => { - return Some( - Err::, IonError>(e), - ); - } - // Plain (name, value literal) pair. For example: `foo: 1` - Ok(LazyRawFieldExpr::NameValuePair( - name, - LazyRawValueExpr::ValueLiteral(value), - )) => { - return Some(Ok(LazyExpandedField::new( - name, - LazyExpandedValue { - context, - source: ExpandedValueSource::ValueLiteral(value), - }, - ))); - } - // (name, macro invocation) pair. For example: `foo: (:bar)` - Ok(LazyRawFieldExpr::NameValuePair( - name, - LazyRawValueExpr::MacroInvocation(invocation), - )) => { - if let Err(e) = evaluator.push(context, invocation) { - return Some(Err(e)); - }; - *state = ExpandingValueExpr(name); - continue; - } - // Macro invocation in field name position. - Ok(LazyRawFieldExpr::MacroInvocation(invocation)) => { - // The next item was a macro. We expect it to expand to a single - // struct whose fields will be merged into the one we're iterating - // over. For example: - // {a: 1, (:make_struct b 2 c 3), d: 4} - // expands to: - // {a: 1, b: 2, c: 3, d: 4} - let mut evaluation = - match evaluator.evaluate(context, invocation) { - Ok(iter) => iter, - Err(e) => return Some(Err(e)), - }; - let expanded_value = match evaluation.next() { - Some(Ok(item)) => item, - Some(Err(e)) => return Some(Err(e)), - None => return Some(IonResult::decoding_error(format!("macros in field name position must produce a single struct; '{:?}' produced nothing", invocation))), - }; - let struct_ = match expanded_value.read() { - Ok(ExpandedValueRef::Struct(s)) => s, - Ok(other) => return Some(IonResult::decoding_error(format!("macros in field name position must produce structs; '{:?}' produced: {:?}", invocation, other))), - Err(e) => return Some(Err(e)), - }; - let iter: &'top mut ExpandedStructIterator<'top, 'data, D> = - struct_.bump_iter(); - *state = InliningAStruct(struct_, iter); - continue; - } - }; - } - InliningAStruct(_struct, struct_iter) => { - if let Some(inlined_field) = struct_iter.next() { - // We pulled another field from the struct we're inlining. - return Some(inlined_field); - } else { - // We're done inlining this struct. Switch back to reading from the source. - *state = ReadingFieldFromSource; - } + ExpandedStructIteratorSource::Template(tdl_macro_evaluator, template_iterator) => { + Self::next_field_from(context, state, tdl_macro_evaluator, template_iterator) + } + ExpandedStructIteratorSource::ValueLiteral(e_exp_evaluator, iter) => { + Self::next_field_from(context, state, e_exp_evaluator, iter) + } + } + } +} + +// Struct expansion is rather complex, and we need to perform it in text Ion, binary Ion, and in +// the body of templates. This implementation covers all of those use cases, but involves some +// potentially intimidating generics as a result. We'll walk through them as they're introduced. +// +// 'top: The lifetime associated with the top-level value we're currently reading at some depth. +// 'data: The lifetime associated with the byte array containing the Ion we're reading from. +// D: The decoder being used to read the Ion data stream. For example: `TextEncoding_1_1` +impl<'top, 'data: 'top, D: LazyDecoder<'data>> ExpandedStructIterator<'top, 'data, D> { + /// Pulls the next expanded field from the raw source struct. The field returned may correspond + /// to a `(name, value literal)` pair in the raw struct, or it may be the product of a macro + /// evaluation. + fn next_field_from< + // The lifetime of this method invocation. + 'a, + // The lifetime of the field name that we return; it needs to live at least as long as + // `top -- the amount of time that the reader will be parked on this top level value. + 'name: 'top, + // The syntactic element that represents a macro invocation in this context. For + // example: a `RawTextMacroInvocation` when reading text Ion 1.1 or a `&'top Sequence` when + // evaluating a TDL macro. + M: MacroInvocation<'data, D> + 'top, + // We have an iterator (see `I` below) that gives us raw fields from an input struct. + // This type, `V`, is the type of value in that raw field. For example: `LazyRawTextValue_1_1` + // when reading text Ion 1.1, or `&'top Element` when evaluating a TDL macro. + V: Into>, + // The type of backing storage used by our macro evaluator. If struct we're iterating over is + // at the top level of the data stream, the evaluator will use a `Vec` for its stack to have + // storage that can persist across top level values. If this is a nested struct or part of + // a template, this will be a transient `BumpVec` with a lifetime tied to the top level. + S: Stack>, + // An iterator over the struct we're expanding. It may be the fields iterator from a + // LazyRawStruct, or it could be a `TemplateStructRawFieldsIterator`. + I: Iterator>>, + >( + context: EncodingContext<'top>, + state: &'a mut ExpandedStructIteratorState<'top, 'data, D>, + evaluator: &'a mut MacroEvaluator<'data, D, M, S>, + iter: &'a mut I, + ) -> Option>> { + // This method begins by pulling raw field expressions from the source iterator. + // If the expression is a (name, value literal) pair, we can wrap it in an LazyExpandedField + // and return it immediately. However, if it is a (name, macro) pair or (macro) expression, + // then an unknown amount of evaluation will need to happen before we can return our next + // field. + loop { + use ControlFlow::{Break, Continue}; + use ExpandedStructIteratorState::*; + match state { + // This is the initial state. We're reading a raw field expression from our source + // iterator. + ReadingFieldFromSource => { + // We'll see what kind of expression it is. + match Self::next_from_iterator(context, state, evaluator, iter) { + // The iterator found a (name, value literal) pair. + Break(maybe_result) => return maybe_result, + // The iterator found a (name, macro) pair or a macro; further evaluation + // is needed to yield a (name, value) pair. + Continue(_) => continue, + } + } + // The iterator previously encountered a macro in field-name position. That macro + // yielded a struct, and now we're merging that expanded struct's fields into our + // own one at a time. + InliningAStruct(_struct, struct_iter) => { + if let Some(inlined_field) = struct_iter.next() { + // We pulled another field from the struct we're inlining. + return Some(inlined_field); + } else { + // We're done inlining this struct. Switch back to reading from the source. + *state = ReadingFieldFromSource; + continue; + } + } + // The iterator previously encountered a (name, macro) pair. We're evaluating the + // macro in field value position, emitting (name, value) pairs for each value + // in the expansion, one at a time. + ExpandingValueExpr(field_name) => { + match evaluator.next(context, 0) { + Err(e) => return Some(Err(e)), + Ok(Some(next_value)) => { + // We got another value from the macro we're evaluating. Emit + // it as another field using the same field_name. + return Some(Ok(LazyExpandedField::new( + field_name.clone(), + next_value, + ))); } - ExpandingValueExpr(field_name) => { - match evaluator.next(context, 0) { - Err(e) => return Some(Err(e)), - Ok(Some(next_value)) => { - // We got another value from the macro we're evaluating. Emit - // it as another field using the same field_name. - return Some(Ok(LazyExpandedField::new( - field_name.clone(), - next_value, - ))); - } - Ok(None) => { - // The macro in the value position is no longer emitting values. Switch - // back to reading from the source. - *state = ReadingFieldFromSource; - } - } + Ok(None) => { + // The macro in the value position is no longer emitting values. Switch + // back to reading from the source. + *state = ReadingFieldFromSource; } } } } } } + + /// Pulls a single raw field expression from the source iterator and sets `state` according to + /// the expression's kind. + fn next_from_iterator< + // These generics are all carried over from the function above. + 'a, + 'name: 'top, + M: MacroInvocation<'data, D> + 'top, + V: Into>, + S: Stack>, + I: Iterator>>, + >( + context: EncodingContext<'top>, + state: &mut ExpandedStructIteratorState<'top, 'data, D>, + evaluator: &mut MacroEvaluator<'data, D, M, S>, + iter: &mut I, + ) -> ControlFlow>>> { + // Because this helper function is always being invoked from within a loop, it uses + // the `ControlFlow` enum to signal whether its return value should cause the loop to + // terminate (`ControlFlow::Break`) or continue (`ControlFlow::Continue`). + use ControlFlow::*; + + // If the iterator is empty, we're done. + let field_expr_result = match iter.next() { + Some(result) => result, + None => return Break(None), + }; + + return match field_expr_result { + Err(e) => Break(Some(Err::, IonError>(e))), + // Plain (name, value literal) pair. For example: `foo: 1` + Ok(RawFieldExpr::NameValuePair(name, RawValueExpr::ValueLiteral(value))) => { + Break(Some(Ok(LazyExpandedField::new( + name, + LazyExpandedValue { + context, + source: value.into(), + }, + )))) + } + // (name, macro invocation) pair. For example: `foo: (:bar)` + Ok(RawFieldExpr::NameValuePair(name, RawValueExpr::MacroInvocation(invocation))) => { + if let Err(e) = evaluator.push(context, invocation) { + return Break(Some(Err(e))); + }; + *state = ExpandedStructIteratorState::ExpandingValueExpr(name); + // We've pushed the macro invocation onto the evaluator's stack, but further evaluation + // is needed to get our next field. + Continue(()) + } + // Macro invocation in field name position. + Ok(RawFieldExpr::MacroInvocation(invocation)) => { + // The next expression from the iterator was a macro. We expect it to expand to a + // single struct whose fields will be merged into the one we're iterating over. For example: + // {a: 1, (:make_struct b 2 c 3), d: 4} + // expands to: + // {a: 1, b: 2, c: 3, d: 4} + match Self::begin_inlining_struct_from_macro(context, state, evaluator, invocation) + { + // If the macro expanded to a struct as expected, continue the evaluation + // until we get a field to return. + Ok(_) => Continue(()), + // If something went wrong, surface the error. + Err(e) => Break(Some(Err(e))), + } + } + }; + } + + /// Pulls the next value from the evaluator, confirms that it's a struct, and then switches + /// the iterator state to `InliningAStruct` so it can begin merging its fields. + fn begin_inlining_struct_from_macro< + 'a, + 'name: 'top, + M: MacroInvocation<'data, D> + 'top, + S: Stack>, + >( + context: EncodingContext<'top>, + state: &mut ExpandedStructIteratorState<'top, 'data, D>, + evaluator: &mut MacroEvaluator<'data, D, M, S>, + invocation: M, + ) -> IonResult<()> { + let mut evaluation = evaluator.evaluate(context, invocation)?; + let expanded_value = match evaluation.next() { + Some(Ok(item)) => item, + Some(Err(e)) => return Err(e), + None => return IonResult::decoding_error(format!("macros in field name position must produce a single struct; '{:?}' produced nothing", invocation)), + }; + let struct_ = match expanded_value.read()? { + ExpandedValueRef::Struct(s) => s, + other => { + return IonResult::decoding_error(format!( + "macros in field name position must produce structs; '{:?}' produced: {:?}", + invocation, other + )) + } + }; + let iter: &'top mut ExpandedStructIterator<'top, 'data, D> = struct_.bump_iter(); + *state = ExpandedStructIteratorState::InliningAStruct(struct_, iter); + Ok(()) + } } diff --git a/src/lazy/expanded/tdl_macro.rs b/src/lazy/expanded/tdl_macro.rs index 36d73627..b9f2a6b0 100644 --- a/src/lazy/expanded/tdl_macro.rs +++ b/src/lazy/expanded/tdl_macro.rs @@ -71,7 +71,7 @@ impl<'element, 'data, D: LazyDecoder<'data>> ToArgumentKind<'data, D, &'element context: EncodingContext<'top>, ) -> ArgumentKind<'top, 'data, D, &'element Sequence> where - Self: 'top, + 'element: 'top, { // In this implementation, we are reading the arguments to a template macro invocation. // For example: diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index a37be653..0e06a40d 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -1,67 +1,101 @@ -use bumpalo::collections::Vec as BumpVec; - -use crate::lazy::expanded::EncodingContext; -use crate::{Element, Sequence}; +use crate::lazy::decoder::{LazyDecoder, RawFieldExpr, RawValueExpr}; +use crate::lazy::expanded::macro_evaluator::TransientTdlMacroEvaluator; +use crate::lazy::expanded::{EncodingContext, ExpandedValueSource, LazyExpandedValue}; +use crate::raw_symbol_token_ref::AsRawSymbolTokenRef; +use crate::{Element, IonResult, Sequence, Struct, Value}; pub type TdlMacroInvocation<'top> = &'top Element; -pub struct TemplateSequenceIterator<'top> { +pub struct TemplateSequenceIterator<'top, 'data, D: LazyDecoder<'data>> { + context: EncodingContext<'top>, + evaluator: TransientTdlMacroEvaluator<'top, 'data, D>, // The list element over which we're iterating sequence: &'top Sequence, index: usize, - macro_stack: BumpVec<'top, TdlMacroInvocation<'top>>, } -impl<'top> TemplateSequenceIterator<'top> { - pub fn new(context: EncodingContext<'top>, sequence: &'top Sequence) -> Self { +impl<'top, 'data, D: LazyDecoder<'data>> TemplateSequenceIterator<'top, 'data, D> { + pub fn new( + context: EncodingContext<'top>, + evaluator: TransientTdlMacroEvaluator<'top, 'data, D>, + sequence: &'top Sequence, + ) -> Self { Self { sequence, index: 0, - macro_stack: BumpVec::new_in(context.allocator), + context, + evaluator, } } } -impl<'top> Iterator for TemplateSequenceIterator<'top> { - type Item = &'top Element; +impl<'top, 'data, D: LazyDecoder<'data>> Iterator for TemplateSequenceIterator<'top, 'data, D> { + type Item = IonResult>; fn next(&mut self) -> Option { - match self.sequence.get(self.index) { - Some(element) => { - self.index += 1; - Some(element) + loop { + // If the evaluator's stack is not empty, give it the opportunity to yield a value. + if self.evaluator.stack_depth() > 0 { + match self.evaluator.next(self.context, 0).transpose() { + Some(value) => return Some(value), + None => { + // The stack did not produce values and is empty, pull + // the next expression from `self.sequence` + } + } + } + // We didn't get a value from the evaluator, so pull the next expression from the + // sequence. + let element = self.sequence.get(self.index)?; + self.index += 1; + // If the expression is an s-expression... + if let Value::SExp(sequence) = element.value() { + // ...it's a TDL macro invocation. Push it onto the evaluator's stack and return + // to the top of the loop. + match self.evaluator.push(self.context, sequence) { + Ok(_) => continue, + Err(e) => return Some(Err(e)), + } } - None => None, + // Otherwise, it's our next value. + return Some(Ok(LazyExpandedValue { + context: self.context, + source: ExpandedValueSource::Template(element), + })); } } } -#[cfg(test)] -mod tests { - use bumpalo::Bump; +// An iterator that pulls values from a template body and wraps them in a `RawFieldExpr` to +// mimic reading them from input. The LazyExpandedStruct handles evaluating any macros that this +// yields. +pub struct TemplateStructRawFieldsIterator<'top> { + // The struct element over whose fields we're iterating + struct_: &'top Struct, + index: usize, +} + +impl<'top> TemplateStructRawFieldsIterator<'top> { + pub fn new(struct_: &'top Struct) -> Self { + Self { struct_, index: 0 } + } +} - use crate::lazy::expanded::macro_table::MacroTable; - use crate::lazy::expanded::template::TemplateSequenceIterator; - use crate::lazy::expanded::EncodingContext; - use crate::{Element, IonResult, SymbolTable}; +impl<'top> Iterator for TemplateStructRawFieldsIterator<'top> { + type Item = IonResult>; - #[test] - fn template_list() -> IonResult<()> { - let data = "[1, (values 2 3 4), 5]"; - let element = Element::read_one(data)?; - let sequence = element.as_list().expect("list"); - let macro_table = MacroTable::new(); - let symtab = SymbolTable::new(); - let allocator = Bump::new(); - let context = EncodingContext { - macro_table: ¯o_table, - symbol_table: &symtab, - allocator: &allocator, - }; - let iter = TemplateSequenceIterator::new(context, sequence); - for value in iter { - println!("{:?}", value); + fn next(&mut self) -> Option { + if let Some((name, element)) = self.struct_.get_index(self.index) { + self.index += 1; + let name = name.as_raw_symbol_token_ref(); + let value = if let Value::SExp(sequence) = element.value() { + RawValueExpr::MacroInvocation(sequence) + } else { + RawValueExpr::ValueLiteral(element) + }; + Some(Ok(RawFieldExpr::NameValuePair(name, value))) + } else { + None } - Ok(()) } } diff --git a/src/lazy/sequence.rs b/src/lazy/sequence.rs index 13edde1e..2bc12232 100644 --- a/src/lazy/sequence.rs +++ b/src/lazy/sequence.rs @@ -8,7 +8,7 @@ use crate::lazy::expanded::sequence::{ }; use crate::lazy::value::{AnnotationsIterator, LazyValue}; use crate::{Annotations, Element, IntoAnnotatedElement, Sequence, Value}; -use crate::{IonError, IonResult, SymbolTable}; +use crate::{IonError, IonResult}; /// A list in a binary Ion stream whose header has been parsed but whose body /// (i.e. its child values) have not. A `LazyList` is immutable; its data can be read any @@ -20,7 +20,7 @@ use crate::{IonError, IonResult, SymbolTable}; /// /// // Construct an Element and serialize it as binary Ion. /// use ion_rs::{Element, ion_list}; -/// use ion_rs::lazy::reader::LazyBinaryReader;; +/// use ion_rs::lazy::reader::LazyBinaryReader; /// /// let element: Element = ion_list! [10, 20, 30].into(); /// let binary_ion = element.to_binary()?; @@ -50,7 +50,6 @@ use crate::{IonError, IonResult, SymbolTable}; /// ``` pub struct LazyList<'top, 'data, D: LazyDecoder<'data>> { pub(crate) expanded_list: LazyExpandedList<'top, 'data, D>, - pub(crate) symbol_table: &'top SymbolTable, } pub type LazyBinarySequence<'top, 'data> = LazyList<'top, 'data, BinaryEncoding_1_0>; @@ -60,7 +59,6 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyList<'top, 'data, D> { pub fn iter(&self) -> ListIterator<'top, 'data, D> { ListIterator { expanded_list_iter: self.expanded_list.iter(), - symbol_table: self.symbol_table, } } @@ -95,7 +93,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazyList<'top, 'data, D> { pub fn annotations(&self) -> AnnotationsIterator<'top, 'data, D> { AnnotationsIterator { expanded_annotations: self.expanded_list.annotations(), - symbol_table: self.symbol_table, + symbol_table: self.expanded_list.context.symbol_table, } } } @@ -135,7 +133,6 @@ impl<'a, 'top, 'data: 'top, D: LazyDecoder<'data>> IntoIterator for &'a LazyList pub struct ListIterator<'top, 'data, D: LazyDecoder<'data>> { expanded_list_iter: ExpandedListIterator<'top, 'data, D>, - symbol_table: &'top SymbolTable, } impl<'top, 'data, D: LazyDecoder<'data>> Iterator for ListIterator<'top, 'data, D> { @@ -169,7 +166,6 @@ impl<'top, 'data, D: LazyDecoder<'data>> Debug for LazyList<'top, 'data, D> { pub struct LazySExp<'top, 'data, D: LazyDecoder<'data>> { pub(crate) expanded_sexp: LazyExpandedSExp<'top, 'data, D>, - pub(crate) symbol_table: &'top SymbolTable, } impl<'top, 'data, D: LazyDecoder<'data>> Debug for LazySExp<'top, 'data, D> { @@ -189,7 +185,6 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazySExp<'top, 'data, D> { pub fn iter(&self) -> SExpIterator<'top, 'data, D> { SExpIterator { expanded_sexp_iter: self.expanded_sexp.iter(), - symbol_table: self.symbol_table, } } @@ -224,7 +219,7 @@ impl<'top, 'data, D: LazyDecoder<'data>> LazySExp<'top, 'data, D> { pub fn annotations(&self) -> AnnotationsIterator<'top, 'data, D> { AnnotationsIterator { expanded_annotations: self.expanded_sexp.annotations(), - symbol_table: self.symbol_table, + symbol_table: self.expanded_sexp.context.symbol_table, } } } @@ -264,7 +259,6 @@ impl<'a, 'top, 'data: 'top, D: LazyDecoder<'data>> IntoIterator for &'a LazySExp pub struct SExpIterator<'top, 'data, D: LazyDecoder<'data>> { expanded_sexp_iter: ExpandedSExpIterator<'top, 'data, D>, - symbol_table: &'top SymbolTable, } impl<'top, 'data, D: LazyDecoder<'data>> Iterator for SExpIterator<'top, 'data, D> { diff --git a/src/lazy/struct.rs b/src/lazy/struct.rs index 4ed15c0a..dc9abbac 100644 --- a/src/lazy/struct.rs +++ b/src/lazy/struct.rs @@ -1,3 +1,6 @@ +use std::fmt; +use std::fmt::{Debug, Formatter}; + use crate::element::builders::StructBuilder; use crate::lazy::decoder::LazyDecoder; use crate::lazy::encoding::BinaryEncoding_1_0; @@ -9,10 +12,8 @@ use crate::lazy::value_ref::ValueRef; use crate::result::IonFailure; use crate::{ Annotations, Element, IntoAnnotatedElement, IonError, IonResult, RawSymbolTokenRef, Struct, - SymbolRef, SymbolTable, + SymbolRef, }; -use std::fmt; -use std::fmt::{Debug, Formatter}; /// An as-of-yet unread binary Ion struct. `LazyStruct` is immutable; its fields and annotations /// can be read any number of times. @@ -46,7 +47,6 @@ use std::fmt::{Debug, Formatter}; #[derive(Clone)] pub struct LazyStruct<'top, 'data, D: LazyDecoder<'data>> { pub(crate) expanded_struct: LazyExpandedStruct<'top, 'data, D>, - pub(crate) symbol_table: &'top SymbolTable, } pub type LazyBinaryStruct<'top, 'data> = LazyStruct<'top, 'data, BinaryEncoding_1_0>; @@ -73,7 +73,6 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyStruct<'top, 'data, D> { pub fn iter(&self) -> StructIterator<'top, 'data, D> { StructIterator { expanded_struct_iter: self.expanded_struct.iter(), - symbol_table: self.symbol_table, } } @@ -234,7 +233,7 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyStruct<'top, 'data, D> { pub fn annotations(&self) -> AnnotationsIterator<'top, 'data, D> { AnnotationsIterator { expanded_annotations: self.expanded_struct.annotations(), - symbol_table: self.symbol_table, + symbol_table: self.expanded_struct.context.symbol_table, } } } @@ -288,7 +287,6 @@ where pub struct StructIterator<'top, 'data, D: LazyDecoder<'data>> { pub(crate) expanded_struct_iter: ExpandedStructIterator<'top, 'data, D>, - pub(crate) symbol_table: &'top SymbolTable, } impl<'top, 'data, D: LazyDecoder<'data>> Iterator for StructIterator<'top, 'data, D> { @@ -345,10 +343,11 @@ impl<'a, 'top, 'data: 'top, D: LazyDecoder<'data>> IntoIterator for &'a LazyStru #[cfg(test)] mod tests { - use super::*; use crate::lazy::binary::test_utilities::to_binary_ion; use crate::lazy::reader::LazyBinaryReader; + use super::*; + #[test] fn find() -> IonResult<()> { let ion_data = to_binary_ion("{foo: 1, bar: 2, baz: 3}")?; diff --git a/src/lazy/system_reader.rs b/src/lazy/system_reader.rs index 61e18365..1dc7d77c 100644 --- a/src/lazy/system_reader.rs +++ b/src/lazy/system_reader.rs @@ -193,7 +193,6 @@ impl<'data, D: LazyDecoder<'data>> LazySystemReader<'data, D> { Self::process_symbol_table(pending_lst, &lazy_expanded_value)?; let lazy_struct = LazyStruct { expanded_struct: lazy_expanded_value.read()?.expect_struct()?, - symbol_table: context.symbol_table, }; return Ok(SystemStreamItem::SymbolTable(lazy_struct)); } diff --git a/src/lazy/text/buffer.rs b/src/lazy/text/buffer.rs index cd7eec11..356c3cb2 100644 --- a/src/lazy/text/buffer.rs +++ b/src/lazy/text/buffer.rs @@ -21,7 +21,7 @@ use nom::sequence::{delimited, pair, preceded, separated_pair, terminated, tuple use nom::{AsBytes, CompareResult, IResult, InputLength, InputTake, Needed, Parser}; use crate::lazy::decoder::private::LazyRawValuePrivate; -use crate::lazy::decoder::{LazyRawFieldExpr, LazyRawValueExpr}; +use crate::lazy::decoder::{LazyRawFieldExpr, LazyRawValueExpr, RawFieldExpr, RawValueExpr}; use crate::lazy::encoding::{TextEncoding_1_0, TextEncoding_1_1}; use crate::lazy::raw_stream_item::RawStreamItem; use crate::lazy::text::encoded_value::EncodedTextValue; @@ -337,10 +337,10 @@ impl<'data> TextBufferView<'data> { alt(( whitespace_and_then( Self::match_e_expression - .map(|matched| Some(LazyRawValueExpr::MacroInvocation(matched))), + .map(|matched| Some(RawValueExpr::MacroInvocation(matched))), ), Self::match_sexp_value.map(|maybe_matched| { - maybe_matched.map(|matched| LazyRawValueExpr::ValueLiteral(matched.into())) + maybe_matched.map(|matched| RawValueExpr::ValueLiteral(matched.into())) }), )) .parse(self) @@ -416,7 +416,7 @@ impl<'data> TextBufferView<'data> { Self::match_e_expression, whitespace_and_then(alt((tag(","), peek(tag("}"))))), ) - .map(|invocation| Ok(Some(LazyRawFieldExpr::MacroInvocation(invocation)))), + .map(|invocation| Ok(Some(RawFieldExpr::MacroInvocation(invocation)))), Self::match_struct_field_name_and_e_expression_1_1.map( |((matched_name, name_span), invocation)| { // TODO: We're discarding the name encoding information here. When we revise our field name @@ -433,9 +433,9 @@ impl<'data> TextBufferView<'data> { return Err(nom::Err::Error(IonParseError::Invalid(error))); } }; - Ok(Some(LazyRawFieldExpr::NameValuePair( + Ok(Some(RawFieldExpr::NameValuePair( name, - LazyRawValueExpr::MacroInvocation(invocation), + RawValueExpr::MacroInvocation(invocation), ))) }, ), @@ -460,9 +460,9 @@ impl<'data> TextBufferView<'data> { } }; let field_value = LazyRawTextValue_1_1 { matched: value }; - Ok(Some(LazyRawFieldExpr::NameValuePair( + Ok(Some(RawFieldExpr::NameValuePair( field_name, - LazyRawValueExpr::ValueLiteral(field_value), + RawValueExpr::ValueLiteral(field_value), ))) }, ), @@ -937,10 +937,10 @@ impl<'data> TextBufferView<'data> { Self::match_e_expression, Self::match_delimiter_after_list_value, ) - .map(|matched| Some(LazyRawValueExpr::MacroInvocation(matched))), + .map(|matched| Some(RawValueExpr::MacroInvocation(matched))), ), Self::match_list_value.map(|maybe_matched| { - maybe_matched.map(|matched| LazyRawValueExpr::ValueLiteral(matched.into())) + maybe_matched.map(|matched| RawValueExpr::ValueLiteral(matched.into())) }), )) .parse(self) diff --git a/src/lazy/text/raw/sequence.rs b/src/lazy/text/raw/sequence.rs index 1ce9b039..8fbd28cd 100644 --- a/src/lazy/text/raw/sequence.rs +++ b/src/lazy/text/raw/sequence.rs @@ -6,7 +6,7 @@ use std::ops::Range; use nom::character::streaming::satisfy; use crate::lazy::decoder::private::LazyContainerPrivate; -use crate::lazy::decoder::{LazyRawSequence, LazyRawValue, LazyRawValueExpr}; +use crate::lazy::decoder::{LazyRawSequence, LazyRawValue, LazyRawValueExpr, RawValueExpr}; use crate::lazy::encoding::TextEncoding_1_0; use crate::lazy::text::buffer::TextBufferView; use crate::lazy::text::parse_result::AddContext; @@ -146,7 +146,7 @@ impl<'data> Iterator for RawTextListIterator_1_0<'data> { match self.input.match_list_value() { Ok((remaining, Some(value))) => { self.input = remaining; - let value = LazyRawValueExpr::ValueLiteral(LazyRawTextValue_1_0::from(value)); + let value = RawValueExpr::ValueLiteral(LazyRawTextValue_1_0::from(value)); Some(Ok(value)) } Ok((_remaining, None)) => { @@ -243,9 +243,9 @@ impl<'data> Iterator for RawTextSExpIterator_1_0<'data> { match self.input.match_sexp_value() { Ok((remaining, Some(value))) => { self.input = remaining; - Some(Ok(LazyRawValueExpr::ValueLiteral( - LazyRawTextValue_1_0::from(value), - ))) + Some(Ok(RawValueExpr::ValueLiteral(LazyRawTextValue_1_0::from( + value, + )))) } Ok((_remaining, None)) => None, Err(e) => { diff --git a/src/lazy/text/raw/struct.rs b/src/lazy/text/raw/struct.rs index f46d5519..f9dcaee5 100644 --- a/src/lazy/text/raw/struct.rs +++ b/src/lazy/text/raw/struct.rs @@ -1,11 +1,12 @@ #![allow(non_camel_case_types)] + use std::ops::Range; use nom::character::streaming::satisfy; use crate::lazy::decoder::private::{LazyContainerPrivate, LazyRawFieldPrivate}; use crate::lazy::decoder::{ - LazyRawField, LazyRawFieldExpr, LazyRawStruct, LazyRawValue, LazyRawValueExpr, + LazyRawField, LazyRawFieldExpr, LazyRawStruct, LazyRawValue, RawFieldExpr, RawValueExpr, }; use crate::lazy::encoding::TextEncoding_1_0; use crate::lazy::text::buffer::TextBufferView; @@ -35,7 +36,7 @@ impl<'data> RawTextStructIterator_1_0<'data> { let value = match field_result? { LazyRawFieldExpr::::NameValuePair( _name, - LazyRawValueExpr::ValueLiteral(value), + RawValueExpr::ValueLiteral(value), ) => value, _ => unreachable!("struct field with macro invocation in Ion 1.0"), }; @@ -76,9 +77,9 @@ impl<'data> Iterator for RawTextStructIterator_1_0<'data> { match self.input.match_struct_field() { Ok((remaining_input, Some(field))) => { self.input = remaining_input; - Some(Ok(LazyRawFieldExpr::NameValuePair( + Some(Ok(RawFieldExpr::NameValuePair( field.name(), - LazyRawValueExpr::ValueLiteral(field.value), + RawValueExpr::ValueLiteral(field.value), ))) } Ok((_, None)) => None, diff --git a/src/lazy/text/raw/v1_1/reader.rs b/src/lazy/text/raw/v1_1/reader.rs index 506c28e8..c2d577ca 100644 --- a/src/lazy/text/raw/v1_1/reader.rs +++ b/src/lazy/text/raw/v1_1/reader.rs @@ -7,7 +7,8 @@ use nom::character::streaming::satisfy; use crate::lazy::decoder::private::LazyContainerPrivate; use crate::lazy::decoder::{ - LazyRawFieldExpr, LazyRawReader, LazyRawSequence, LazyRawStruct, LazyRawValue, LazyRawValueExpr, + LazyRawFieldExpr, LazyRawReader, LazyRawSequence, LazyRawStruct, LazyRawValue, + LazyRawValueExpr, RawFieldExpr, RawValueExpr, }; use crate::lazy::encoding::TextEncoding_1_1; use crate::lazy::raw_stream_item::RawStreamItem; @@ -149,11 +150,11 @@ impl<'data> RawTextListIterator_1_1<'data> { let value_expr = value_expr_result?; // ...the input slice that follows the last sequence value... match value_expr { - LazyRawValueExpr::ValueLiteral(value) => value + RawValueExpr::ValueLiteral(value) => value .matched .input .slice_to_end(value.matched.encoded_value.total_length()), - LazyRawValueExpr::MacroInvocation(invocation) => { + RawValueExpr::MacroInvocation(invocation) => { let end_of_expr = invocation.input.offset() + invocation.input.len(); let remaining = self.input.slice_to_end(end_of_expr - self.input.offset()); remaining @@ -316,11 +317,11 @@ impl<'data> RawTextSExpIterator_1_1<'data> { let value_expr = value_expr_result?; // ...the input slice that follows the last sequence value... match value_expr { - LazyRawValueExpr::ValueLiteral(value) => value + RawValueExpr::ValueLiteral(value) => value .matched .input .slice_to_end(value.matched.encoded_value.total_length()), - LazyRawValueExpr::MacroInvocation(invocation) => { + RawValueExpr::MacroInvocation(invocation) => { let end_of_expr = invocation.input.offset() + invocation.input.len(); let remaining = self.input.slice_to_end(end_of_expr - self.input.offset()); remaining @@ -444,14 +445,14 @@ impl<'data> RawTextStructIterator_1_1<'data> { let input_after_last = if let Some(field_result) = self.last() { // If there are any field expressions, we need to isolate the input slice that follows // the last one. - use LazyRawFieldExpr::*; + use RawFieldExpr::*; match field_result? { // foo: bar - NameValuePair(_name, LazyRawValueExpr::ValueLiteral(value)) => { + NameValuePair(_name, RawValueExpr::ValueLiteral(value)) => { value.matched.input.slice_to_end(value.matched.encoded_value.total_length()) }, // foo: (:bar ...) - NameValuePair(_, LazyRawValueExpr::MacroInvocation(invocation)) + NameValuePair(_, RawValueExpr::MacroInvocation(invocation)) // (:foo) | MacroInvocation(invocation) => { self.input.slice_to_end(invocation.input.len()) diff --git a/src/lazy/value.rs b/src/lazy/value.rs index 3fd1014d..d1343974 100644 --- a/src/lazy/value.rs +++ b/src/lazy/value.rs @@ -192,24 +192,15 @@ impl<'top, 'data: 'top, D: LazyDecoder<'data>> LazyValue<'top, 'data, D> { Blob(b) => ValueRef::Blob(b), Clob(c) => ValueRef::Clob(c), SExp(s) => { - let lazy_sexp = LazySExp { - expanded_sexp: s, - symbol_table: self.expanded_value.context.symbol_table, - }; + let lazy_sexp = LazySExp { expanded_sexp: s }; ValueRef::SExp(lazy_sexp) } List(l) => { - let lazy_sequence = LazyList { - expanded_list: l, - symbol_table: self.expanded_value.context.symbol_table, - }; + let lazy_sequence = LazyList { expanded_list: l }; ValueRef::List(lazy_sequence) } Struct(s) => { - let lazy_struct = LazyStruct { - expanded_struct: s, - symbol_table: self.expanded_value.context.symbol_table, - }; + let lazy_struct = LazyStruct { expanded_struct: s }; ValueRef::Struct(lazy_struct) } }; @@ -230,14 +221,11 @@ impl<'top, 'data, D: LazyDecoder<'data>> TryFrom> for /// Iterates over a slice of bytes, lazily reading them as a sequence of symbol tokens encoded /// using the format described by generic type parameter `D`. pub struct AnnotationsIterator<'top, 'data, D: LazyDecoder<'data>> { - pub(crate) symbol_table: &'top SymbolTable, pub(crate) expanded_annotations: ExpandedAnnotationsIterator<'top, 'data, D>, + pub(crate) symbol_table: &'top SymbolTable, } -impl<'top, 'data, D: LazyDecoder<'data>> AnnotationsIterator<'top, 'data, D> -where - 'data: 'top, -{ +impl<'top, 'data: 'top, D: LazyDecoder<'data>> AnnotationsIterator<'top, 'data, D> { /// Returns `Ok(true)` if this annotations iterator matches the provided sequence exactly, or /// `Ok(false)` if not. If a decoding error occurs while visiting and resolving each annotation, /// returns an `Err(IonError)`. @@ -320,10 +308,7 @@ where } } -impl<'top, 'data, D: LazyDecoder<'data>> Iterator for AnnotationsIterator<'top, 'data, D> -where - 'data: 'top, -{ +impl<'top, 'data: 'top, D: LazyDecoder<'data>> Iterator for AnnotationsIterator<'top, 'data, D> { type Item = IonResult>; fn next(&mut self) -> Option {