Struct tokenizers::models::wordpiece::WordPiece

pub struct WordPiece {
    pub unk_token: String,
    pub continuing_subword_prefix: String,
    pub max_input_chars_per_word: usize,
    /* private fields */
}

A WordPiece model.

Fields

unk_token: String

continuing_subword_prefix: String

max_input_chars_per_word: usize

Implementations

impl WordPiece

pub fn builder() -> WordPieceBuilder

Get a WordPieceBuilder.

pub fn read_file(vocab: &str) -> Result<HashMap<String, u32>>

Read the given files to extract the vocab

pub fn from_file(vocab: &str) -> WordPieceBuilder

Initialize a WordPiece model from a vocab mapping file.

pub fn from_bpe(bpe: &BPE) -> Self

Create a WordPiece model from a BPE model.

Trait Implementations

impl Clone for WordPiece

fn clone(&self) -> WordPiece

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for WordPiece

fn fmt(&self, fmt: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for WordPiece

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for WordPiece

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<WordPiece> for ModelWrapper

fn from(from: WordPiece) -> Self

Converts to this type from the input type.

impl Model for WordPiece

type Trainer = WordPieceTrainer

fn get_vocab(&self) -> HashMap<String, u32>

Retrieve the entire vocabulary mapping (token -> ID)

fn get_vocab_size(&self) -> usize

Retrieve the size of the vocabulary

fn tokenize(&self, sequence: &str) -> Result<Vec<Token>>

Tokenize the given sequence into multiple underlying Token. The offsets on the Token are expected to be relative to the given sequence.

fn token_to_id(&self, token: &str) -> Option<u32>

Find the ID associated to a string token

fn id_to_token(&self, id: u32) -> Option<String>

Find the string token associated to an ID

fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>

Save the current Model in the given folder, using the given prefix for the various files that need to be saved.

fn get_trainer(&self) -> Self::Trainer

Get an instance of a Trainer capable of training this Model

impl PartialEq for WordPiece

fn eq(&self, other: &WordPiece) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for WordPiece

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for WordPiece

impl StructuralPartialEq for WordPiece