Enum tokenizers::models::TrainerWrapper

pub enum TrainerWrapper {
    BpeTrainer(BpeTrainer),
    WordPieceTrainer(WordPieceTrainer),
    WordLevelTrainer(WordLevelTrainer),
    UnigramTrainer(UnigramTrainer),
}

Variants

BpeTrainer(BpeTrainer)

WordPieceTrainer(WordPieceTrainer)

WordLevelTrainer(WordLevelTrainer)

UnigramTrainer(UnigramTrainer)

Trait Implementations

impl Clone for TrainerWrapper

fn clone(&self) -> TrainerWrapper

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'de> Deserialize<'de> for TrainerWrapper

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<BpeTrainer> for TrainerWrapper

fn from(from: BpeTrainer) -> Self

Converts to this type from the input type.

impl From<UnigramTrainer> for TrainerWrapper

fn from(from: UnigramTrainer) -> Self

Converts to this type from the input type.

impl From<WordLevelTrainer> for TrainerWrapper

fn from(from: WordLevelTrainer) -> Self

Converts to this type from the input type.

impl From<WordPieceTrainer> for TrainerWrapper

fn from(from: WordPieceTrainer) -> Self

Converts to this type from the input type.

impl Serialize for TrainerWrapper

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Trainer for TrainerWrapper

type Model = ModelWrapper

fn should_show_progress(&self) -> bool

Whether we should show progress during the training.

fn train(&self, model: &mut ModelWrapper) -> Result<Vec<AddedToken>>

The actual training method. This will return a new trained Model as well as a list of special_tokens to be added directly to the tokenizer along with the model.

fn feed<I, S, F>(&mut self, iterator: I, process: F) -> Result<()>

where I: Iterator<Item = S> + Send, S: AsRef<str> + Send, F: Fn(&str) -> Result<Vec<String>> + Sync,

Process an iterator of sequences, calling process for each of them in order to pre-process the said sequence as relevant.