Struct tokenizers::models::wordpiece::WordPieceTrainerBuilder

pub struct WordPieceTrainerBuilder { /* private fields */ }

A WordPieceTrainerBuilder can be used to create a WordPieceTrainer with a custom configuration.

Implementations

impl WordPieceTrainerBuilder

pub fn new() -> Self

Constructs a new WordPieceTrainerBuilder

pub fn min_frequency(self, frequency: u64) -> Self

Set the expected minimum frequency

pub fn vocab_size(self, size: usize) -> Self

Set the vocabulary size

pub fn show_progress(self, show: bool) -> Self

Set whether to show progress

pub fn special_tokens(self, tokens: Vec<AddedToken>) -> Self

Set the special tokens

pub fn limit_alphabet(self, limit: usize) -> Self

Set whether to limit the alphabet

pub fn initial_alphabet(self, alphabet: HashSet<char>) -> Self

Set the initial alphabet

pub fn continuing_subword_prefix(self, prefix: String) -> Self

Set the continuing_subword_prefix

pub fn end_of_word_suffix(self, suffix: String) -> Self

Set the end_of_word_suffix

pub fn build(self) -> WordPieceTrainer

Constructs the final BpeTrainer

Trait Implementations

impl Default for WordPieceTrainerBuilder

fn default() -> Self

Returns the “default value” for a type.