Struct tokenizers::models::bpe::BPE

pub struct BPE {
    pub dropout: Option<f32>,
    pub unk_token: Option<String>,
    pub continuing_subword_prefix: Option<String>,
    pub end_of_word_suffix: Option<String>,
    pub fuse_unk: bool,
    pub byte_fallback: bool,
    /* private fields */
}

A Byte Pair Encoding model.

Fields

dropout: Option<f32>

Dropout probability for merges. 0 = no dropout is the default. At 1.0, tokenization will perform no merges, so the result will just be characters.

unk_token: Option<String>

The unknown token to be used when we encounter an unknown char

continuing_subword_prefix: Option<String>

An optional prefix to use on any subword that exist only behind another one

end_of_word_suffix: Option<String>

An optional suffix to caracterize and end-of-word subword

fuse_unk: bool

Do multiple unk tokens get fused

byte_fallback: bool

Byte fallback from sentence pieces, instead of UNK, uses "<0x00>" for each byte in the unk token

Implementations

impl BPE

pub fn builder() -> BpeBuilder

Initialize a BpeBuilder.

pub fn new(vocab: Vocab, merges: Merges) -> Self

Create a new BPE model with the given vocab and merges.

pub fn from_file(vocab: &str, merges: &str) -> BpeBuilder

Initialize a BpeBuilder model from vocab and merges files

pub fn read_file(vocab: &str, merges: &str) -> Result<(Vocab, Merges)>

Read the given files to extract the vocab and merges

pub fn clear_cache(&self)

Reset the cache.

pub fn get_vocab(&self) -> Vocab

pub fn get_unk_token(&self) -> &Option<String>

pub fn get_continuing_subword_prefix(&self) -> &Option<String>

Trait Implementations

impl Clone for BPE

fn clone(&self) -> Self

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BPE

fn fmt(&self, fmt: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for BPE

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for BPE

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<BPE> for ModelWrapper

fn from(from: BPE) -> Self

Converts to this type from the input type.

impl Model for BPE

type Trainer = BpeTrainer

fn get_vocab(&self) -> HashMap<String, u32>

Retrieve the entire vocabulary mapping (token -> ID)

fn get_vocab_size(&self) -> usize

Retrieve the size of the vocabulary

fn tokenize(&self, sequence: &str) -> Result<Vec<Token>>

Tokenize the given sequence into multiple underlying Token. The offsets on the Token are expected to be relative to the given sequence.

fn token_to_id(&self, token: &str) -> Option<u32>

Find the ID associated to a string token

fn id_to_token(&self, id: u32) -> Option<String>

Find the string token associated to an ID

fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>

Save the current Model in the given folder, using the given prefix for the various files that need to be saved.

fn get_trainer(&self) -> BpeTrainer

Get an instance of a Trainer capable of training this Model

impl PartialEq for BPE

fn eq(&self, other: &BPE) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for BPE

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl StructuralPartialEq for BPE