Struct tokenizers::pre_tokenizers::metaspace::Metaspace

pub struct Metaspace {
    pub add_prefix_space: bool,
    pub prepend_scheme: PrependScheme,
    /* private fields */
}

Replaces all the whitespaces by the provided meta character and then splits on this character

Fields

add_prefix_space: bool

prepend_scheme: PrependScheme

Implementations

impl Metaspace

pub fn new(replacement: char, add_prefix_space: bool) -> Self

pub fn new_with_prepend_scheme( replacement: char, add_prefix_space: bool, prepend_scheme: PrependScheme ) -> Self

pub fn get_replacement(&self) -> char

pub fn set_replacement(&mut self, replacement: char)

pub fn get_prepend_scheme(&self) -> PrependScheme

pub fn set_prepend_scheme(&mut self, scheme: PrependScheme)

Trait Implementations

impl Clone for Metaspace

fn clone(&self) -> Metaspace

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Metaspace

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Decoder for Metaspace

fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>>

fn decode(&self, tokens: Vec<String>) -> Result<String>

impl Default for Metaspace

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for Metaspace

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<Metaspace> for DecoderWrapper

fn from(from: Metaspace) -> Self

Converts to this type from the input type.

impl From<Metaspace> for PreTokenizerWrapper

fn from(from: Metaspace) -> Self

Converts to this type from the input type.

impl PartialEq for Metaspace

fn eq(&self, other: &Metaspace) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PreTokenizer for Metaspace

fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()>

impl Serialize for Metaspace

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for Metaspace

impl StructuralPartialEq for Metaspace