Update to new released versions

Author: torymur

.github/workflows/build_wheels.yml

@@ -16,7 +16,10 @@ jobs:
     strategy:
       matrix:
         platform:
-          # older ubuntu to avoid messing with glibc version
+          # Older ubuntu to avoid messing with glibc version since glibc is forward compatible,
+          # but not backwards compatible (since about glibc 2.2 or 2.4)
+          # All programs built against an older glibc work on a system with a newer glibc than
+          # it was built with. The reverse is not true.
           - runner: ubuntu-22.04
             target: x86_64
             manylinux: auto

Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 description = "Structured Generation"
 license = "Apache-2.0"
 repository = "https://github.com/dottxt-ai/outlines-core"
-rust-version = "1.71.1"
+rust-version = "1.85.0"
 
 [dependencies]
 once_cell = "1.20"
@@ -15,7 +15,7 @@ regex = "1.10.6"
 serde-pyobject = { version = "0.5.0", optional = true }
 serde_json = { version = "1.0", features = ["preserve_order"] }
 serde = {version = "1.0", features = ["derive"]}
-bincode = "2.0.0-rc.3"
+bincode = "2.0.1"
 rustc-hash = "2.1.0"
 regex-automata = "0.4.9"
 
@@ -26,10 +26,7 @@ features = ["ureq", "rustls-tls"]
 default-features = false
 
 [dependencies.tokenizers]
-git = "https://github.com/huggingface/tokenizers.git"
-# This is a version > 0.21.0 before 0.21.1, we're looking for rustls-tls feature
-# Once released could be pinned to 0.21.1+
-rev = "4f1a810aa258d287e6936315e63fbf58bde2a980"
+version = "=0.21.1"
 features = ["http", "rustls-tls"]
 
 [features]

README.md

@@ -2,18 +2,14 @@
 
 <img src="./docs/assets/images/logo.png" alt="Outlines-core Logo" width=500></img>
 
-[![Latest Version]][crates.io] [![License]][github] ![MSRV]
+[![Latest Version]][crates.io] [![License]][github] ![MSRV][version]
 
 [Latest Version]: https://img.shields.io/crates/v/outlines-core.svg
 [crates.io]: https://crates.io/crates/outlines-core
 [License]: https://img.shields.io/github/license/dottxt-ai/outlines-core.svg?color=blue&cachedrop
 [github]: https://github.com/dottxt-ai/outlines-core/blob/main/LICENSE
-[MSRV]: https://img.shields.io/badge/MSRV-1.71.1-brightgreen
-
-<!---
-Once it uploaded to crates.io badge could be generated like:
- [version]: https://img.shields.io/crates/msrv/outlines-core.svg?label=msrv&color=lightgrayy
- -->
+[MSRV]: MSRV
+[version]: https://img.shields.io/crates/msrv/outlines-core.svg?label=msrv&color=lightgrayy
 
 *Structured generation (in Rust).*
 

src/vocabulary/mod.rs

@@ -160,7 +160,7 @@ impl Vocabulary {
                 NormalizerWrapper::Sequence(normalization_sequence) => {
                     let new_sequence = Sequence::new(
                         normalization_sequence
-                            .as_ref()
+                            .get_normalizers()
                             .iter()
                             .filter_map(|normalizer| match normalizer {
                                 NormalizerWrapper::Prepend(_) => None,
@@ -465,7 +465,7 @@ mod tests {
             if let Some(n) = normalized_t.get_normalizer() {
                 match n {
                     NormalizerWrapper::Sequence(seq) => {
-                        for n in seq.as_ref() {
+                        for n in seq.get_normalizers() {
                             if let NormalizerWrapper::Prepend(_) = n {
                                 unreachable!()
                             }