Merge remote-tracking branch 'refs/remotes/origin/main'

Author: hikalium

app/browser-rs/Makefile

@@ -29,4 +29,4 @@ clippy :
 	$(CARGO) clippy $(CARGO_ARGS)
 
 test:
-	$(CARGO) test $(CARGO_ARGS) --tests -- --test-threads=1
+	$(CARGO) test $(CARGO_ARGS) --tests -- --test-threads=1 --nocapture

app/browser-rs/src/rendering.rs

@@ -10,6 +10,5 @@ pub fn render(html: String) {
     println!("===== rendering start ===== ");
     println!("{}", html);
 
-    let tokenizer = Tokenizer::new(html).tokens();
-    println!("{:?}", tokenizer);
+    let _tokenizer = Tokenizer::new(html);
 }

app/browser-rs/src/rendering/tokenizer.rs

@@ -3,6 +3,9 @@
 
 use alloc::string::String;
 use alloc::vec::Vec;
+use core::iter::Iterator;
+#[allow(unused_imports)]
+use liumlib::*;
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -16,95 +19,76 @@ pub enum State {
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum TokenType {
+pub enum Token {
     Doctype,
-    StartTag,
-    EndTag,
-    Char,
+    StartTag { tag: String, self_closing: bool },
+    EndTag { tag: String, self_closing: bool },
+    Char(char),
     Eof,
 }
 
-#[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Token {
-    token_type: TokenType,
-    tag: String,
-    self_closing: bool,
-    data: String,
-}
-
-impl Token {
-    pub fn new(token_type: TokenType, tag: String, self_closing: bool, data: String) -> Self {
-        Self {
-            token_type,
-            tag,
-            self_closing,
-            data,
-        }
-    }
-}
-
-#[derive(Debug)]
 pub struct Tokenizer {
     state: State,
     pos: usize,
-    length: usize,
-    html: String,
-    tokens: Vec<Token>,
+    input: Vec<char>,
 }
 
 impl Tokenizer {
     pub fn new(html: String) -> Self {
         Self {
             state: State::Data,
             pos: 0,
-            length: html.len(),
-            html,
-            tokens: Vec::new(),
+            input: html.chars().collect(),
         }
     }
 
-    pub fn tokens(&mut self) -> Vec<Token> {
+    fn switch_to(&mut self, s: State) {
+        self.state = s;
+    }
+
+    /// Consume the next input character.
+    fn consume_next_input(&mut self) -> char {
+        let c = self.input[self.pos];
+        self.pos += 1;
+        c
+    }
+}
+
+impl Iterator for Tokenizer {
+    type Item = Token;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.pos >= self.input.len() {
+            return None;
+        }
+
         loop {
-            // https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
+            let c = self.consume_next_input();
 
             match self.state {
                 State::Data => {
-                    if self.html[self.pos..].starts_with("<") {
-                        self.state = State::TagOpen;
+                    // https://html.spec.whatwg.org/multipage/parsing.html#data-state
+                    if c == '<' {
+                        self.switch_to(State::TagOpen);
                         continue;
                     }
-                    if self.pos == self.length {
-                        self.append_eof();
-                        return self.tokens.clone();
+
+                    if self.pos > self.input.len() {
+                        return Some(Token::Eof);
                     }
-                    self.append_char();
-                    break;
+
+                    return Some(Token::Char(c));
+                }
+                State::TagOpen => {
+                    // https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
                 }
                 _ => {}
             }
 
-            self.pos += 1;
+            if self.pos == self.input.len() {
+                return Some(Token::Eof);
+            }
         }
-
-        self.tokens.clone()
-    }
-
-    pub fn append_eof(&mut self) {
-        self.tokens.push(Token::new(
-            TokenType::Eof,
-            String::new(),
-            false,
-            String::new(),
-        ));
-    }
-
-    fn append_char(&mut self) {
-        self.tokens.push(Token::new(
-            TokenType::Char,
-            String::new(),
-            false,
-            String::new(),
-        ));
     }
 }

app/browser-rs/tests/tokenizer.rs

@@ -47,13 +47,34 @@ fn main() {
 #[test_case]
 fn no_input() {
     let mut t = Tokenizer::new(String::new());
-    let mut tokens = Vec::new();
-    tokens.push(Token::new(
-        TokenType::Eof,
-        String::new(),
-        false,
-        String::new(),
-    ));
-
-    assert_eq!(t.tokens(), tokens);
+    assert_eq!(t.next(), None);
 }
+
+#[test_case]
+fn chars() {
+    let mut t = Tokenizer::new(String::from("foo"));
+
+    let mut expected = Vec::new();
+    expected.push(Token::Char('f'));
+    expected.push(Token::Char('o'));
+    expected.push(Token::Char('o'));
+
+    for e in expected {
+        assert_eq!(t.next().expect("toknizer should have a next Token"), e);
+    }
+}
+
+/*
+#[test_case]
+fn body() {
+    let mut t = Tokenizer::new(String::from("<body></body>"));
+
+    let mut expected = Vec::new();
+    expected.push(Token::StartTag({ tag: String::from("body"), self_closing: false }));
+    expected.push(Token::EndTag({ tag: String::from("body"), self_closing: false }));
+
+    for e in expected {
+        assert_eq!(t.next().expect("toknizer should have a next Token"), e);
+    }
+}
+*/

app/browser/tokenize.c

@@ -61,7 +61,7 @@ void Tokenize(char *html) {
     }
 
     switch (state) {
-      case DATA:
+        case DATA:
         // https://html.spec.whatwg.org/multipage/parsing.html#data-state
         if (*html == '<') {
           // U+003C LESS-THAN SIGN (<)

app/liumlib/src/lib.rs

@@ -281,7 +281,7 @@ trait MutableAllocator {
     fn dealloc(&mut self, _ptr: *mut u8, _layout: Layout);
 }
 
-const ALLOCATOR_BUF_SIZE: usize = 0x2000;
+const ALLOCATOR_BUF_SIZE: usize = 0x100000;
 pub struct WaterMarkAllocator {
     buf: [u8; ALLOCATOR_BUF_SIZE],
     used_bytes: usize,