Enum tokenizers::models::ModelWrapper

pub enum ModelWrapper {
    BPE(BPE),
    WordPiece(WordPiece),
    WordLevel(WordLevel),
    Unigram(Unigram),
}

Variants

BPE(BPE)

WordPiece(WordPiece)

WordLevel(WordLevel)

Unigram(Unigram)

Trait Implementations

impl Clone for ModelWrapper

fn clone(&self) -> ModelWrapper

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelWrapper

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for ModelWrapper

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<BPE> for ModelWrapper

fn from(from: BPE) -> Self

Converts to this type from the input type.

impl From<Unigram> for ModelWrapper

fn from(from: Unigram) -> Self

Converts to this type from the input type.

impl From<WordLevel> for ModelWrapper

fn from(from: WordLevel) -> Self

Converts to this type from the input type.

impl From<WordPiece> for ModelWrapper

fn from(from: WordPiece) -> Self

Converts to this type from the input type.

impl Model for ModelWrapper

type Trainer = TrainerWrapper

fn tokenize(&self, tokens: &str) -> Result<Vec<Token>>

Tokenize the given sequence into multiple underlying Token. The offsets on the Token are expected to be relative to the given sequence.

fn token_to_id(&self, token: &str) -> Option<u32>

Find the ID associated to a string token

fn id_to_token(&self, id: u32) -> Option<String>

Find the string token associated to an ID

fn get_vocab(&self) -> HashMap<String, u32>

Retrieve the entire vocabulary mapping (token -> ID)

fn get_vocab_size(&self) -> usize

Retrieve the size of the vocabulary

fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>

Save the current Model in the given folder, using the given prefix for the various files that need to be saved.

fn get_trainer(&self) -> Self::Trainer

Get an instance of a Trainer capable of training this Model

impl PartialEq for ModelWrapper

fn eq(&self, other: &ModelWrapper) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for ModelWrapper

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl StructuralPartialEq for ModelWrapper