Struct tokenizers::tokenizer::Encoding

pub struct Encoding { /* private fields */ }

Represents the output of a Tokenizer.

Implementations

impl Encoding

pub fn new( ids: Vec<u32>, type_ids: Vec<u32>, tokens: Vec<String>, words: Vec<Option<u32>>, offsets: Vec<Offsets>, special_tokens_mask: Vec<u32>, attention_mask: Vec<u32>, overflowing: Vec<Self>, sequence_ranges: HashMap<usize, Range<usize>> ) -> Self

pub fn with_capacity(len: usize) -> Self

pub fn from_tokens(tokens: Vec<Token>, type_id: u32) -> Self

pub fn is_empty(&self) -> bool

Whether this Encoding is empty

pub fn len(&self) -> usize

Return the total length of this Encoding

pub fn n_sequences(&self) -> usize

Return the number of sequences combined in this Encoding

pub fn set_sequence_id(&mut self, sequence_id: usize)

Set the given sequence id for the whole range of tokens contained in this Encoding

pub fn get_tokens(&self) -> &[String]

pub fn get_word_ids(&self) -> &[Option<u32>]

pub fn get_word_ids_mut(&mut self) -> &mut [Option<u32>]

pub fn get_sequence_ids(&self) -> Vec<Option<usize>>

pub fn get_ids(&self) -> &[u32]

pub fn get_type_ids(&self) -> &[u32]

pub fn set_type_ids(&mut self, type_ids: Vec<u32>)

pub fn get_offsets(&self) -> &[Offsets]

pub fn get_offsets_mut(&mut self) -> &mut [Offsets]

pub fn get_special_tokens_mask(&self) -> &[u32]

pub fn get_attention_mask(&self) -> &[u32]

pub fn get_overflowing(&self) -> &Vec<Encoding>

pub fn set_overflowing(&mut self, overflowing: Vec<Encoding>)

pub fn get_overflowing_mut(&mut self) -> &mut Vec<Encoding>

pub fn take_overflowing(&mut self) -> Vec<Encoding>

pub fn token_to_sequence(&self, token: usize) -> Option<usize>

Returns the index of the sequence containing the given token

pub fn word_to_tokens( &self, word: u32, sequence_id: usize ) -> Option<(usize, usize)>

Get the encoded tokens corresponding to the word at the given index in the input sequence, with the form (start_token, end_token + 1)

pub fn word_to_chars(&self, word: u32, sequence_id: usize) -> Option<Offsets>

Get the offsets of the word at the given index in the input sequence.

pub fn token_to_chars(&self, token: usize) -> Option<(usize, Offsets)>

Get the offsets of the token at the given index.

pub fn token_to_word(&self, token: usize) -> Option<(usize, u32)>

Get the word that contains the token at the given index.

pub fn char_to_token(&self, pos: usize, sequence_id: usize) -> Option<usize>

Get the token that contains the given char.

pub fn char_to_word(&self, pos: usize, sequence_id: usize) -> Option<u32>

Get the word that contains the given char.

pub fn truncate( &mut self, max_len: usize, stride: usize, direction: TruncationDirection )

Truncate the current Encoding.

Panics if stride >= max_len

pub fn merge<I: IntoIterator<Item = Encoding>>( encodings: I, growing_offsets: bool ) -> Self

Merge all Encodings together

pub fn merge_with(&mut self, pair: Encoding, growing_offsets: bool)

Merge ourself with the given Encoding. Happens in place.

pub fn pad( &mut self, target_length: usize, pad_id: u32, pad_type_id: u32, pad_token: &str, direction: PaddingDirection )

Trait Implementations

impl Clone for Encoding

fn clone(&self) -> Encoding

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Encoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Encoding

fn default() -> Encoding

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for Encoding

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl FromIterator<(u32, String, (usize, usize), Option<u32>, u32)> for Encoding

fn from_iter<I: IntoIterator<Item = (u32, String, (usize, usize), Option<u32>, u32)>>( iter: I ) -> Self

Creates a value from an iterator.

impl FromIterator<Encoding> for Encoding

fn from_iter<I: IntoIterator<Item = Encoding>>(iter: I) -> Self

Creates a value from an iterator.

impl PartialEq for Encoding

fn eq(&self, other: &Encoding) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for Encoding

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl StructuralPartialEq for Encoding