Tue, 17 Nov 2026 12:45:57 UTC Log in
Implementation of today's most used tokenizers in Rust 159.30 KB built 2026-10-21 11:40:41 GMT

Description

Provides an implementation of today's most used tokenizers, with a focus on performances and versatility. This package contains library source intended for building other packages which use the "tokenizers" crate.

Dependencies

38 total
Requires35
  • (crate(ahash/default) >= 0.8.11 with crate(ahash/default) < 0.9.0~)
  • (crate(ahash/serde) >= 0.8.11 with crate(ahash/serde) < 0.9.0~)
  • (crate(compact_str/default) >= 0.9.0 with crate(compact_str/default) < 0.10.0~)
  • (crate(compact_str/serde) >= 0.9.0 with crate(compact_str/serde) < 0.10.0~)
  • (crate(daachorse/default) >= 1.0.1 with crate(daachorse/default) < 2.0.0~)
  • (crate(dary_heap/default) >= 0.3.6 with crate(dary_heap/default) < 0.4.0~)
  • (crate(dary_heap/serde) >= 0.3.6 with crate(dary_heap/serde) < 0.4.0~)
  • (crate(derive_builder/default) >= 0.20.0 with crate(derive_builder/default) < 0.21.0~)
  • (crate(esaxx-rs) >= 0.1.10 with crate(esaxx-rs) < 0.2.0~)
  • (crate(getrandom/default) >= 0.3.0 with crate(getrandom/default) < 0.4.0~)
  • (crate(itertools/default) >= 0.14.0 with crate(itertools/default) < 0.15.0~)
  • (crate(log/default) >= 0.4.0 with crate(log/default) < 0.5.0~)
  • (crate(macro_rules_attribute/default) >= 0.2.0 with crate(macro_rules_attribute/default) < 0.3.0~)
  • (crate(monostate/default) >= 1.0.0 with crate(monostate/default) < 2.0.0~)
  • (crate(paste/default) >= 1.0.14 with crate(paste/default) < 2.0.0~)
  • (crate(rand/default) >= 0.9.0 with crate(rand/default) < 0.10.0~)
  • (crate(rayon-cond/default) >= 0.4.0 with crate(rayon-cond/default) < 0.5.0~)
  • (crate(rayon/default) >= 1.10.0 with crate(rayon/default) < 2.0.0~)
  • (crate(regex-syntax/default) >= 0.8.0 with crate(regex-syntax/default) < 0.9.0~)
  • (crate(regex/default) >= 1.10.0 with crate(regex/default) < 2.0.0~)
  • (crate(serde/default) >= 1.0.0 with crate(serde/default) < 2.0.0~)
  • (crate(serde/derive) >= 1.0.0 with crate(serde/derive) < 2.0.0~)
  • (crate(serde_json/default) >= 1.0.0 with crate(serde_json/default) < 2.0.0~)
  • (crate(spm_precompiled/default) >= 0.1.3 with crate(spm_precompiled/default) < 0.2.0~)
  • (crate(thiserror/default) >= 2.0.0 with crate(thiserror/default) < 3.0.0~)
  • (crate(unicode-normalization-alignments/default) >= 0.1.0 with crate(unicode-normalization-alignments/default) < 0.2.0~)
  • (crate(unicode-segmentation/default) >= 1.11.0 with crate(unicode-segmentation/default) < 2.0.0~)
  • (crate(unicode_categories/default) >= 0.1.0 with crate(unicode_categories/default) < 0.2.0~)
  • cargo
  • rpmlib(CompressedFileNames) <= 3.0.4-1
  • rpmlib(FileDigests) <= 4.6.0-1
  • rpmlib(PayloadFilesHavePrefix) <= 4.0-1
  • rpmlib(PayloadIsZstd) <= 5.4.18-1
  • rpmlib(RichDependencies) <= 4.12.0-1
  • rpmlib(TildeInVersions) <= 4.10.0-1
Provides3
  • crate(tokenizers) = 0.23.1
  • purl(pkg:cargo/tokenizers@0.23.1)
  • rust-tokenizers-devel = 0.23.1-1.fc45
ObsoletesNo Obsoletes
ConflictsNo Conflicts
RecommendsNo Recommends
SuggestsNo Suggests
SupplementsNo Supplements
EnhancesNo Enhances

Files

118
Name ascending sort Size
/usr/share/cargo/registry/tokenizers-0.23.1 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/.cargo-checksum.json 26.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/CHANGELOG.md 10.65 KB
/usr/share/cargo/registry/tokenizers-0.23.1/Cargo.toml 4.73 KB
/usr/share/cargo/registry/tokenizers-0.23.1/LICENSE 11.09 KB
/usr/share/cargo/registry/tokenizers-0.23.1/Makefile 2.27 KB
/usr/share/cargo/registry/tokenizers-0.23.1/README.md 5.04 KB
/usr/share/cargo/registry/tokenizers-0.23.1/README.tpl 589.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/benches 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/benches/added_vocab_deserialize.rs 2.74 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/bert_benchmark.rs 4.83 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/bpe_benchmark.rs 4.40 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/ci_benchmark.rs 10.10 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/common 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/benches/common/mod.rs 2.11 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/layout_benchmark.rs 2.52 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/llama3_benchmark.rs 3.58 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/truncation_benchmark.rs 5.39 KB
/usr/share/cargo/registry/tokenizers-0.23.1/benches/unigram_benchmark.rs 2.05 KB
/usr/share/cargo/registry/tokenizers-0.23.1/examples 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/examples/encode_batch.rs 414.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/examples/serialization.rs 781.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/bpe.rs 955.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/byte_fallback.rs 3.32 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/ctc.rs 6.26 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/fuse.rs 1005.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/mod.rs 9.20 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/sequence.rs 1.43 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/strip.rs 2.23 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/decoders/wordpiece.rs 2.20 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/lib.rs 5.71 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe/mod.rs 2.28 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe/model.rs 37.93 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe/serialization.rs 8.69 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe/trainer.rs 29.17 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/bpe/word.rs 11.14 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/mod.rs 12.63 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/lattice.rs 22.87 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/mod.rs 180.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/model.rs 22.71 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/serialization.rs 3.53 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/trainer.rs 29.42 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/unigram/trie.rs 1.90 KB
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/wordlevel 0.00 B
/usr/share/cargo/registry/tokenizers-0.23.1/src/models/wordlevel/mod.rs 6.85 KB
Files 1–50 of 118 Next →
of 3

Component of

buildroots that installed this RPM

No Buildroots