Struct tokenizers::models::bpe::trainer::BpeTrainerBuilder

pub struct BpeTrainerBuilder { /* private fields */ }

A BpeTrainerBuilder can be used to create a BpeTrainer with a custom configuration.

Implementations

impl BpeTrainerBuilder

pub fn new() -> Self

Constructs a new BpeTrainerBuilder

pub fn min_frequency(self, frequency: u64) -> Self

Set the expected minimum frequency

pub fn vocab_size(self, size: usize) -> Self

Set the vocabulary size

pub fn show_progress(self, show: bool) -> Self

Set whether to show progress

pub fn special_tokens(self, tokens: Vec<AddedToken>) -> Self

Set the special tokens

pub fn limit_alphabet(self, limit: usize) -> Self

Set whether to limit the alphabet

pub fn initial_alphabet(self, alphabet: HashSet<char>) -> Self

Set the initial alphabet

pub fn continuing_subword_prefix(self, prefix: String) -> Self

Set the continuing_subword_prefix

pub fn end_of_word_suffix(self, suffix: String) -> Self

Set the end_of_word_suffix

pub fn max_token_length(self, max_token_length: Option<usize>) -> Self

Set max_token_length

pub fn build(self) -> BpeTrainer

Constructs the final BpeTrainer

Trait Implementations

impl Default for BpeTrainerBuilder

fn default() -> Self

Returns the “default value” for a type.