Updated Tokenizers to 0.20.2

ankane · ankane · commit 035dc522ac55 · 2024-11-04T10:04:39.000-08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## 0.5.4 (unreleased)
 
-- Updated Tokenizers to 0.20.1
+- Updated Tokenizers to 0.20.2
 
 ## 0.5.3 (2024-09-17)
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ext/tokenizers/Cargo.toml b/ext/tokenizers/Cargo.toml
@@ -16,6 +16,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.20.1" # also update in from_pretrained.rb
+version = "=0.20.2" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]
diff --git a/lib/tokenizers/from_pretrained.rb b/lib/tokenizers/from_pretrained.rb
@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.20.1"
+    TOKENIZERS_VERSION = "0.20.2"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
