List of all items

Structs

• decoders::bpe::BPEDecoder
• decoders::byte_fallback::ByteFallback
• decoders::ctc::CTC
• decoders::fuse::Fuse
• decoders::sequence::Sequence
• decoders::strip::Strip
• decoders::wordpiece::WordPiece
• models::bpe::BPE
• models::bpe::BpeBuilder
• models::bpe::trainer::BpeTrainer
• models::bpe::trainer::BpeTrainerBuilder
• models::unigram::Lattice
• models::unigram::Node
• models::unigram::Unigram
• models::unigram::UnigramIterator
• models::unigram::UnigramTrainer
• models::unigram::UnigramTrainerBuilder
• models::wordlevel::WordLevel
• models::wordlevel::WordLevelBuilder
• models::wordlevel::WordLevelTrainer
• models::wordlevel::WordLevelTrainerBuilder
• models::wordpiece::WordPiece
• models::wordpiece::WordPieceBuilder
• models::wordpiece::WordPieceTrainer
• models::wordpiece::WordPieceTrainerBuilder
• normalizers::Precompiled
• normalizers::bert::BertNormalizer
• normalizers::precompiled::Precompiled
• normalizers::prepend::Prepend
• normalizers::replace::Replace
• normalizers::strip::Strip
• normalizers::strip::StripAccents
• normalizers::unicode::NFC
• normalizers::unicode::NFD
• normalizers::unicode::NFKC
• normalizers::unicode::NFKD
• normalizers::unicode::Nmt
• normalizers::utils::Lowercase
• normalizers::utils::Sequence
• pre_tokenizers::bert::BertPreTokenizer
• pre_tokenizers::byte_level::ByteLevel
• pre_tokenizers::delimiter::CharDelimiterSplit
• pre_tokenizers::digits::Digits
• pre_tokenizers::metaspace::Metaspace
• pre_tokenizers::punctuation::Punctuation
• pre_tokenizers::sequence::Sequence
• pre_tokenizers::split::Split
• pre_tokenizers::unicode_scripts::UnicodeScripts
• pre_tokenizers::whitespace::Whitespace
• pre_tokenizers::whitespace::WhitespaceSplit
• processors::bert::BertProcessing
• processors::roberta::RobertaProcessing
• processors::sequence::Sequence
• processors::template::SpecialToken
• processors::template::Template
• processors::template::TemplateProcessing
• processors::template::TemplateProcessingBuilder
• processors::template::Tokens
• tokenizer::AddedToken
• tokenizer::BuilderError
• tokenizer::Encoding
• tokenizer::Token
• tokenizer::Tokenizer
• tokenizer::TokenizerBuilder
• tokenizer::TokenizerImpl
• tokenizer::TruncationParamError
• tokenizer::normalizer::NormalizedString
• tokenizer::pattern::Invert
• tokenizer::pre_tokenizer::PreTokenizedString
• tokenizer::pre_tokenizer::Split
• utils::SysRegex
• utils::iter::Lines
• utils::iter::ResultShunt
• utils::padding::PaddingParams
• utils::truncation::TruncationParams

Enums

• decoders::DecoderWrapper
• models::ModelWrapper
• models::TrainerWrapper
• models::bpe::Error
• models::unigram::UnigramError
• models::unigram::UnigramTrainerBuilderError
• models::unigram::UnigramTrainerError
• models::wordlevel::Error
• models::wordlevel::WordLevelTrainerBuilderError
• models::wordpiece::Error
• normalizers::NormalizerWrapper
• normalizers::replace::ReplacePattern
• pre_tokenizers::PreTokenizerWrapper
• pre_tokenizers::metaspace::PrependScheme
• pre_tokenizers::split::SplitPattern
• processors::PostProcessorWrapper
• processors::bert::BertProcessorError
• processors::template::Piece
• processors::template::Sequence
• processors::template::TemplateProcessingBuilderError
• tokenizer::EncodeInput
• tokenizer::InputSequence
• tokenizer::ProcessorError
• tokenizer::normalizer::OffsetReferential
• tokenizer::normalizer::Range
• tokenizer::normalizer::SplitDelimiterBehavior
• tokenizer::pre_tokenizer::OffsetType
• utils::padding::PaddingDirection
• utils::padding::PaddingStrategy
• utils::truncation::TruncationDirection
• utils::truncation::TruncationError
• utils::truncation::TruncationStrategy

Traits

• tokenizer::Decoder
• tokenizer::Model
• tokenizer::Normalizer
• tokenizer::PostProcessor
• tokenizer::PreTokenizer
• tokenizer::Trainer
• tokenizer::pattern::Pattern
• utils::iter::LinesWithEnding
• utils::parallelism::MaybeParallelBridge
• utils::parallelism::MaybeParallelIterator
• utils::parallelism::MaybeParallelRefIterator
• utils::parallelism::MaybeParallelRefMutIterator
• utils::parallelism::MaybeParallelSlice

Macros

• impl_serde_type

Attribute Macros

• utils::macro_rules_attribute

Functions

• decoders::wordpiece::cleanup
• pre_tokenizers::byte_level::process_offsets
• tokenizer::normalizer::bytes_to_char
• tokenizer::normalizer::char_to_bytes
• tokenizer::normalizer::get_range_of
• utils::padding::pad_encodings
• utils::parallelism::current_num_threads
• utils::parallelism::get_parallelism
• utils::parallelism::has_parallelism_been_used
• utils::parallelism::is_parallelism_configured
• utils::parallelism::set_parallelism
• utils::truncation::truncate_encodings

Type Aliases

• models::bpe::MergeMap
• models::bpe::Merges
• models::bpe::Vocab
• tokenizer::Error
• tokenizer::Offsets
• tokenizer::Result

Constants

• utils::parallelism::ENV_VARIABLE