Added from_str method to Tokenizer

Author: ankane

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.2 (unreleased)
+
+- Added `from_str` method to `Tokenizer`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

ext/tokenizers/src/lib.rs

@@ -42,6 +42,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
 
     let class = module.define_class("Tokenizer", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbTokenizer::from_model, 1))?;
+    class.define_singleton_method("from_str", function!(RbTokenizer::from_str, 1))?;
     class.define_singleton_method("from_file", function!(RbTokenizer::from_file, 1))?;
     class.define_method(
         "add_special_tokens",

ext/tokenizers/src/tokenizer.rs

@@ -1,9 +1,10 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::str::FromStr;
 
 use magnus::prelude::*;
-use magnus::{exception, Error, RArray, RHash, Symbol, TryConvert, Value};
+use magnus::{exception, Error, RArray, RHash, RString, Symbol, TryConvert, Value};
 use tk::tokenizer::{
     Model, PaddingDirection, PaddingParams, PaddingStrategy,
     TruncationDirection, TruncationParams, TruncationStrategy, TokenizerImpl
@@ -203,6 +204,14 @@ impl RbTokenizer {
         RbTokenizer::new(TokenizerImpl::new(model.clone()))
     }
 
+    pub fn from_str(json: RString) -> RbResult<Self> {
+        Tokenizer::from_str(unsafe { json.as_str()? })
+            .map(|v| RbTokenizer {
+                tokenizer: RefCell::new(v),
+            })
+            .map_err(RbError::from)
+    }
+
     pub fn from_file(path: PathBuf) -> RbResult<Self> {
         Tokenizer::from_file(path)
             .map(|v| RbTokenizer {