feat(brill): train and use Brill tagger (#1344)

Co-authored-by: hippietrail <[email protected]>

Author: elijah-potter

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace]
-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst" , "harper-stats"]
+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst" , "harper-stats", "harper-pos-utils", "harper-brill"]
 resolver = "2"
 
 # Comment out the below lines if you plan to use a debugger.

Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "harper-brill"
+version = "0.42.0"
+edition = "2024"
+
+[dependencies]
+harper-pos-utils = { path = "../harper-pos-utils/", version = "0.42.0" }
+lazy_static = "1.5.0"
+rs-conllu = "0.3.0"
+serde = "1.0.219"
+serde_json = "1.0.140"
+
+[build-dependencies]
+rs-conllu = "0.3.0"
+serde = "1.0.219"
+serde_json = "1.0.140"

harper-brill/Cargo.toml

@@ -0,0 +1,32 @@
+use lazy_static::lazy_static;
+use std::sync::Arc;
+
+pub use harper_pos_utils::{BrillChunker, BrillTagger, Chunker, FreqDict, Tagger, UPOS};
+
+const BRILL_TAGGER_SOURCE: &str = include_str!("../trained_tagger_model.json");
+
+lazy_static! {
+    static ref BRILL_TAGGER: Arc<BrillTagger<FreqDict>> = Arc::new(uncached_brill_tagger());
+}
+
+fn uncached_brill_tagger() -> BrillTagger<FreqDict> {
+    serde_json::from_str(BRILL_TAGGER_SOURCE).unwrap()
+}
+
+pub fn brill_tagger() -> Arc<BrillTagger<FreqDict>> {
+    (*BRILL_TAGGER).clone()
+}
+
+const BRILL_CHUNKER_SOURCE: &str = include_str!("../trained_chunker_model.json");
+
+lazy_static! {
+    static ref BRILL_CHUNKER: Arc<BrillChunker> = Arc::new(uncached_brill_chunker());
+}
+
+fn uncached_brill_chunker() -> BrillChunker {
+    serde_json::from_str(BRILL_CHUNKER_SOURCE).unwrap()
+}
+
+pub fn brill_chunker() -> Arc<BrillChunker> {
+    (*BRILL_CHUNKER).clone()
+}