Skip to content

Commit 83d6d1b

Browse files
stefnotchstefnotch
committed
Convert more parsing
1 parent b434809 commit 83d6d1b

File tree

6 files changed

+335
-128
lines changed

6 files changed

+335
-128
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ rustc-hash = "1.1.0"
2929
unicode-ident = "1"
3030
once_cell = "1.17.0"
3131
indexmap = "2"
32-
winnow = "0.6.7"
32+
winnow = { version = "0.6.7" }
3333

3434
[dev-dependencies]
3535
wgpu = { version = "0.19.0", features = ["naga-ir"] }

‎src/compose/comment_strip_iter.rs

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ fn parse_source(input: &mut Located<&str>) -> PResult<SourceCode> {
4343
}
4444
Ok(SourceCode { parts })
4545
}
46-
fn quoted_string(input: &mut Located<&str>) -> PResult<Range<usize>> {
46+
pub fn quoted_string(input: &mut Located<&str>) -> PResult<Range<usize>> {
4747
// See https://docs.rs/winnow/latest/winnow/_topic/json/index.html
4848
preceded(
4949
'\"',
@@ -62,12 +62,13 @@ fn string_character(input: &mut Located<&str>) -> PResult<()> {
6262
}
6363
Ok(())
6464
}
65-
fn single_line_comment(input: &mut Located<&str>) -> PResult<Range<usize>> {
65+
pub fn single_line_comment(input: &mut Located<&str>) -> PResult<Range<usize>> {
6666
let start_span = "//".span().parse_next(input)?;
67+
// TODO: Use the rules from https://www.w3.org/TR/WGSL/#line-break instead of till_line_ending
6768
let text_span = till_line_ending.span().parse_next(input)?;
6869
Ok(start_span.start..text_span.end)
6970
}
70-
fn multi_line_comment(input: &mut Located<&str>) -> PResult<Range<usize>> {
71+
pub fn multi_line_comment(input: &mut Located<&str>) -> PResult<Range<usize>> {
7172
let start_span = "/*".span().parse_next(input)?;
7273
loop {
7374
if let Some(end_span) = opt("*/".span()).parse_next(input)? {
@@ -108,7 +109,7 @@ impl<'a> Iterator for CommentReplaceIter<'a> {
108109
.map(|i| line_start + i + 1)
109110
.unwrap_or_else(|| self.text.len());
110111
self.text_index = line_end;
111-
let original = self.text[line_start..line_end].trim_end();
112+
let original = self.text[line_start..line_end].trim_end_matches('\n');
112113

113114
let mut parts = Vec::new();
114115
for (i, (code_part, span)) in self.parsed.parts.iter().enumerate().skip(self.parsed_index) {
@@ -128,7 +129,7 @@ impl<'a> Iterator for CommentReplaceIter<'a> {
128129
if parts.len() == 1 {
129130
match parts.into_iter().next().unwrap() {
130131
(CodePart::Text | CodePart::QuotedText, span) => {
131-
return Some((Cow::Borrowed(&self.text[span].trim_end()), original));
132+
return Some((Cow::Borrowed(self.text[span].trim_end()), original));
132133
}
133134
(CodePart::SingleLineComment | CodePart::MultiLineComment, _) => {
134135
let spaces = " ".repeat(original.len());
@@ -147,11 +148,15 @@ impl<'a> Iterator for CommentReplaceIter<'a> {
147148
output.push_str(&self.text[span]);
148149
}
149150
CodePart::SingleLineComment | CodePart::MultiLineComment => {
150-
// TODO: This technically changes the length of a line (\n)
151151
output.extend(std::iter::repeat(' ').take(span.len()));
152152
}
153153
}
154154
}
155+
// Limit the length of output to the length of the original line
156+
let max_len = original.len();
157+
if output.len() > max_len {
158+
output.truncate(max_len);
159+
}
155160

156161
assert!(last_end == line_end);
157162
Some((Cow::Owned(output), original))

‎src/compose/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ pub mod comment_strip_iter;
144144
pub mod error;
145145
pub mod parse_imports;
146146
pub mod preprocess;
147+
pub mod preprocess1;
147148
mod test;
148149
pub mod tokenizer;
149150
pub mod util;

‎src/compose/parse_imports.rs

Lines changed: 43 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1+
use std::ops::Range;
2+
13
use indexmap::IndexMap;
4+
use winnow::{Located, Parser};
5+
6+
use crate::compose::preprocess1::{import_directive, ImportTree};
27

38
use super::{
49
tokenizer::{Token, Tokenizer},
@@ -9,106 +14,49 @@ pub fn parse_imports<'a>(
914
input: &'a str,
1015
declared_imports: &mut IndexMap<String, Vec<String>>,
1116
) -> Result<(), (&'a str, usize)> {
12-
let mut tokens = Tokenizer::new(input, false).peekable();
13-
14-
match tokens.next() {
15-
Some(Token::Other('#', _)) => (),
16-
Some(other) => return Err(("expected `#import`", other.pos())),
17-
None => return Err(("expected #import", input.len())),
18-
};
19-
match tokens.next() {
20-
Some(Token::Identifier("import", _)) => (),
21-
Some(other) => return Err(("expected `#import`", other.pos())),
22-
None => return Err(("expected `#import`", input.len())),
23-
};
24-
25-
let mut stack = Vec::default();
26-
let mut current = String::default();
27-
let mut as_name = None;
28-
let mut is_deprecated_itemlist = false;
29-
30-
loop {
31-
match tokens.peek() {
32-
Some(Token::Identifier(ident, _)) => {
33-
current.push_str(ident);
34-
tokens.next();
35-
36-
if tokens.peek().and_then(Token::identifier) == Some("as") {
37-
let pos = tokens.next().unwrap().pos();
38-
let Some(Token::Identifier(name, _)) = tokens.next() else {
39-
return Err(("expected identifier after `as`", pos));
40-
};
41-
42-
as_name = Some(name);
43-
}
44-
45-
// support deprecated #import mod item
46-
if let Some(Token::Identifier(..)) = tokens.peek() {
47-
#[cfg(not(feature = "allow_deprecated"))]
48-
tracing::warn!("item list imports are deprecated, please use `rust::style::item_imports` (or use feature `allow_deprecated`)`\n| {}", input);
49-
50-
is_deprecated_itemlist = true;
51-
stack.push(format!("{}::", current));
52-
current = String::default();
53-
as_name = None;
54-
}
55-
56-
continue;
57-
}
58-
Some(Token::Other('{', pos)) => {
59-
if !current.ends_with("::") {
60-
return Err(("open brace must follow `::`", *pos));
61-
}
62-
stack.push(current);
63-
current = String::default();
64-
as_name = None;
65-
}
66-
Some(Token::Other(',', _))
67-
| Some(Token::Other('}', _))
68-
| Some(Token::Other('\n', _))
69-
| None => {
70-
if !current.is_empty() {
71-
let used_name = as_name.map(ToString::to_string).unwrap_or_else(|| {
72-
current
73-
.rsplit_once("::")
74-
.map(|(_, name)| name.to_owned())
75-
.unwrap_or(current.clone())
76-
});
77-
declared_imports.entry(used_name).or_default().push(format!(
78-
"{}{}",
79-
stack.join(""),
80-
current
81-
));
82-
current = String::default();
83-
as_name = None;
84-
}
85-
86-
if let Some(Token::Other('}', pos)) = tokens.peek() {
87-
if stack.pop().is_none() {
88-
return Err(("close brace without open", *pos));
89-
}
90-
}
17+
let input = input.trim();
18+
let imports = import_directive.parse(Located::new(input)).map_err(|_v| {
19+
// panic!("{:#?}", _v);
20+
("failed to parse imports", 0)
21+
})?;
22+
23+
fn to_stack<'a>(input: &'a str, ranges: &[Range<usize>]) -> Vec<&'a str> {
24+
ranges
25+
.iter()
26+
.map(|range| &input[range.clone()])
27+
.collect::<Vec<_>>()
28+
}
9129

92-
if tokens.peek().is_none() {
93-
break;
30+
fn walk_import_tree<'a>(
31+
input: &'a str,
32+
tree: &ImportTree,
33+
stack: &[&'a str],
34+
declared_imports: &mut IndexMap<String, Vec<String>>,
35+
) {
36+
let (name_range, path_ranges) = match tree {
37+
ImportTree::Path(path_ranges) => (path_ranges.last().unwrap().clone(), path_ranges),
38+
ImportTree::Alias {
39+
path: path_ranges,
40+
alias: alias_range,
41+
} => (alias_range.clone(), path_ranges),
42+
ImportTree::Children { path, children } => {
43+
let extended_stack = [stack, &to_stack(input, path)].concat();
44+
for child in children {
45+
walk_import_tree(input, child, &extended_stack, declared_imports);
9446
}
47+
return;
9548
}
96-
Some(Token::Other(';', _)) => {
97-
tokens.next();
98-
if let Some(token) = tokens.peek() {
99-
return Err(("unexpected token after ';'", token.pos()));
100-
}
101-
}
102-
Some(Token::Other(_, pos)) => return Err(("unexpected token", *pos)),
103-
Some(Token::Whitespace(..)) => unreachable!(),
104-
}
105-
106-
tokens.next();
49+
};
50+
51+
let name = input[name_range].to_string();
52+
let extended_stack = [stack, &to_stack(input, &path_ranges)].concat();
53+
declared_imports
54+
.entry(name)
55+
.or_default()
56+
.push(extended_stack.join("::"));
10757
}
10858

109-
if !(stack.is_empty() || is_deprecated_itemlist && stack.len() == 1) {
110-
return Err(("missing close brace", input.len()));
111-
}
59+
walk_import_tree(input, &imports.tree, &[], declared_imports);
11260

11361
Ok(())
11462
}

‎src/compose/preprocess.rs

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,28 @@ use std::collections::{HashMap, HashSet};
22

33
use indexmap::IndexMap;
44
use regex::Regex;
5+
use winnow::{Located, Parser};
56

67
use super::{
78
comment_strip_iter::replace_comments,
89
parse_imports::{parse_imports, substitute_identifiers},
9-
ComposerErrorInner, ImportDefWithOffset, ShaderDefValue,
10+
preprocess1, ComposerErrorInner, ImportDefWithOffset, ShaderDefValue,
1011
};
1112

1213
#[derive(Debug)]
1314
pub struct Preprocessor {
14-
version_regex: Regex,
1515
ifdef_regex: Regex,
1616
ifndef_regex: Regex,
1717
ifop_regex: Regex,
1818
else_regex: Regex,
1919
endif_regex: Regex,
2020
def_regex: Regex,
2121
def_regex_delimited: Regex,
22-
import_regex: Regex,
23-
define_import_path_regex: Regex,
24-
define_shader_def_regex: Regex,
2522
}
2623

2724
impl Default for Preprocessor {
2825
fn default() -> Self {
2926
Self {
30-
version_regex: Regex::new(r"^\s*#version\s+([0-9]+)").unwrap(),
3127
ifdef_regex: Regex::new(r"^\s*#\s*(else\s+)?\s*ifdef\s+([\w|\d|_]+)").unwrap(),
3228
ifndef_regex: Regex::new(r"^\s*#\s*(else\s+)?\s*ifndef\s+([\w|\d|_]+)").unwrap(),
3329
ifop_regex: Regex::new(
@@ -38,10 +34,6 @@ impl Default for Preprocessor {
3834
endif_regex: Regex::new(r"^\s*#\s*endif").unwrap(),
3935
def_regex: Regex::new(r"#\s*([\w|\d|_]+)").unwrap(),
4036
def_regex_delimited: Regex::new(r"#\s*\{([\w|\d|_]+)\}").unwrap(),
41-
import_regex: Regex::new(r"^\s*#\s*import\s").unwrap(),
42-
define_import_path_regex: Regex::new(r"^\s*#\s*define_import_path\s+([^\s]+)").unwrap(),
43-
define_shader_def_regex: Regex::new(r"^\s*#\s*define\s+([\w|\d|_]+)\s*([-\w|\d]+)?")
44-
.unwrap(),
4537
}
4638
}
4739
}
@@ -250,20 +242,26 @@ impl Preprocessor {
250242
while let Some((mut line, original_line)) = lines.next() {
251243
let mut output = false;
252244

253-
if let Some(cap) = self.version_regex.captures(&line) {
254-
let v = cap.get(1).unwrap().as_str();
255-
if v != "440" && v != "450" {
245+
if let Some(cap) = {
246+
let a = preprocess1::version.parse(Located::new(line.trim())).ok();
247+
a
248+
} {
249+
if cap.version_number != 440 && cap.version_number != 450 {
256250
return Err(ComposerErrorInner::GlslInvalidVersion(offset));
257251
}
258252
} else if self
259253
.check_scope(shader_defs, &line, Some(&mut scope), offset)?
260254
.0
261-
|| self.define_import_path_regex.captures(&line).is_some()
262-
|| self.define_shader_def_regex.captures(&line).is_some()
255+
|| preprocess1::define_import_path
256+
.parse(Located::new(line.trim()))
257+
.is_ok()
258+
|| preprocess1::define_shader_def
259+
.parse(Located::new(line.trim()))
260+
.is_ok()
263261
{
264262
// ignore
265263
} else if scope.active() {
266-
if self.import_regex.is_match(&line) {
264+
if preprocess1::import_start.parse(Located::new(&line)).is_ok() {
267265
let mut import_lines = String::default();
268266
let mut open_count = 0;
269267
let initial_offset = offset;
@@ -406,7 +404,7 @@ impl Preprocessor {
406404
if let Some(def) = def {
407405
effective_defs.insert(def.to_owned());
408406
}
409-
} else if self.import_regex.is_match(&line) {
407+
} else if preprocess1::import_start.parse(Located::new(&line)).is_ok() {
410408
let mut import_lines = String::default();
411409
let mut open_count = 0;
412410
let initial_offset = offset;
@@ -441,19 +439,28 @@ impl Preprocessor {
441439
)
442440
},
443441
)?;
444-
} else if let Some(cap) = self.define_import_path_regex.captures(&line) {
445-
name = Some(cap.get(1).unwrap().as_str().to_string());
446-
} else if let Some(cap) = self.define_shader_def_regex.captures(&line) {
442+
} else if let Some(cap) = {
443+
let a = preprocess1::define_import_path
444+
.parse(Located::new(line.trim()))
445+
.ok();
446+
a
447+
} {
448+
name = Some(line[cap.path].to_string());
449+
} else if let Some(cap) = {
450+
let a = preprocess1::define_shader_def
451+
.parse(Located::new(line.trim()))
452+
.ok();
453+
a
454+
} {
447455
if allow_defines {
448-
let def = cap.get(1).unwrap();
449-
let name = def.as_str().to_string();
456+
let name = line[cap.name].to_string();
450457

451-
let value = if let Some(val) = cap.get(2) {
452-
if let Ok(val) = val.as_str().parse::<u32>() {
458+
let value = if let Some(val) = cap.value.map(|v| &line[v]) {
459+
if let Ok(val) = val.parse::<u32>() {
453460
ShaderDefValue::UInt(val)
454-
} else if let Ok(val) = val.as_str().parse::<i32>() {
461+
} else if let Ok(val) = val.parse::<i32>() {
455462
ShaderDefValue::Int(val)
456-
} else if let Ok(val) = val.as_str().parse::<bool>() {
463+
} else if let Ok(val) = val.parse::<bool>() {
457464
ShaderDefValue::Bool(val)
458465
} else {
459466
ShaderDefValue::Bool(false) // this error will get picked up when we fully preprocess the module

0 commit comments

Comments
 (0)