Struct tokenizers::models::wordlevel::WordLevel

pub struct WordLevel {
    pub unk_token: String,
    /* private fields */
}

Fields

unk_token: String

Implementations

impl WordLevel

pub fn builder() -> WordLevelBuilder

pub fn read_file(vocab_path: &str) -> Result<HashMap<String, u32>>

pub fn from_file(vocab_path: &str, unk_token: String) -> Result<WordLevel>

Initialize a WordLevel model from vocab and merges file.

Trait Implementations

impl Clone for WordLevel

fn clone(&self) -> WordLevel

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for WordLevel

fn fmt(&self, fmt: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for WordLevel

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for WordLevel

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<WordLevel> for ModelWrapper

fn from(from: WordLevel) -> Self

Converts to this type from the input type.

impl Model for WordLevel

type Trainer = WordLevelTrainer

fn tokenize(&self, token: &str) -> Result<Vec<Token>>

Tokenize the given sequence into multiple underlying Token. The offsets on the Token are expected to be relative to the given sequence.

fn token_to_id(&self, token: &str) -> Option<u32>

Find the ID associated to a string token

fn id_to_token(&self, id: u32) -> Option<String>

Find the string token associated to an ID

fn get_vocab(&self) -> HashMap<String, u32>

Retrieve the entire vocabulary mapping (token -> ID)

fn get_vocab_size(&self) -> usize

Retrieve the size of the vocabulary

fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>

Save the current Model in the given folder, using the given prefix for the various files that need to be saved.

fn get_trainer(&self) -> Self::Trainer

Get an instance of a Trainer capable of training this Model

impl PartialEq for WordLevel

fn eq(&self, other: &WordLevel) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for WordLevel

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for WordLevel

impl StructuralPartialEq for WordLevel