Struct tokenizers::models::wordpiece::WordPieceTrainer

pub struct WordPieceTrainer { /* private fields */ }

Trains a WordPiece model.

Implementations

impl WordPieceTrainer

pub fn min_frequency(&self) -> u64

pub fn set_min_frequency(&mut self, freq: u64)

pub fn vocab_size(&self) -> usize

pub fn set_vocab_size(&mut self, size: usize)

pub fn show_progress(&self) -> bool

pub fn set_show_progress(&mut self, show_progress: bool)

pub fn special_tokens(&self) -> &[AddedToken]

pub fn set_special_tokens(&mut self, special_tokens: Vec<AddedToken>)

pub fn limit_alphabet(&self) -> Option<usize>

pub fn set_limit_alphabet(&mut self, limit: Option<usize>)

pub fn initial_alphabet(&self) -> &HashSet<char>

pub fn set_initial_alphabet(&mut self, alphabet: HashSet<char>)

pub fn continuing_subword_prefix(&self) -> &Option<String>

pub fn set_continuing_subword_prefix(&mut self, prefix: Option<String>)

pub fn end_of_word_suffix(&self) -> &Option<String>

pub fn set_end_of_word_suffix(&mut self, suffix: Option<String>)

pub fn builder() -> WordPieceTrainerBuilder

pub fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>>

Trait Implementations

impl Clone for WordPieceTrainer

fn clone(&self) -> WordPieceTrainer

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for WordPieceTrainer

fn default() -> WordPieceTrainer

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for WordPieceTrainer

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<WordPieceTrainer> for TrainerWrapper

fn from(from: WordPieceTrainer) -> Self

Converts to this type from the input type.

impl Serialize for WordPieceTrainer

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Trainer for WordPieceTrainer

type Model = WordPiece

fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>>

The actual training method. This will return a new trained Model as well as a list of special_tokens to be added directly to the tokenizer along with the model.

fn should_show_progress(&self) -> bool

Whether we should show progress during the training.

fn feed<I, S, F>(&mut self, iterator: I, process: F) -> Result<()>

where I: Iterator<Item = S> + Send, S: AsRef<str> + Send, F: Fn(&str) -> Result<Vec<String>> + Sync,

Process an iterator of sequences, calling process for each of them in order to pre-process the said sequence as relevant.