Enum tokenizers::decoders::DecoderWrapper

pub enum DecoderWrapper {
    BPE(BPEDecoder),
    ByteLevel(ByteLevel),
    WordPiece(WordPiece),
    Metaspace(Metaspace),
    CTC(CTC),
    Sequence(Sequence),
    Replace(Replace),
    Fuse(Fuse),
    Strip(Strip),
    ByteFallback(ByteFallback),
}

Variants

BPE(BPEDecoder)

ByteLevel(ByteLevel)

WordPiece(WordPiece)

Metaspace(Metaspace)

CTC(CTC)

Sequence(Sequence)

Replace(Replace)

Fuse(Fuse)

Strip(Strip)

ByteFallback(ByteFallback)

Trait Implementations

impl Clone for DecoderWrapper

fn clone(&self) -> DecoderWrapper

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DecoderWrapper

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Decoder for DecoderWrapper

fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>>

fn decode(&self, tokens: Vec<String>) -> Result<String>

impl<'de> Deserialize<'de> for DecoderWrapper

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<BPEDecoder> for DecoderWrapper

fn from(from: BPEDecoder) -> Self

Converts to this type from the input type.

impl From<ByteFallback> for DecoderWrapper

fn from(from: ByteFallback) -> Self

Converts to this type from the input type.

impl From<ByteLevel> for DecoderWrapper

fn from(from: ByteLevel) -> Self

Converts to this type from the input type.

impl From<CTC> for DecoderWrapper

fn from(from: CTC) -> Self

Converts to this type from the input type.

impl From<Fuse> for DecoderWrapper

fn from(from: Fuse) -> Self

Converts to this type from the input type.

impl From<Metaspace> for DecoderWrapper

fn from(from: Metaspace) -> Self

Converts to this type from the input type.

impl From<Replace> for DecoderWrapper

fn from(from: Replace) -> Self

Converts to this type from the input type.

impl From<Sequence> for DecoderWrapper

fn from(from: Sequence) -> Self

Converts to this type from the input type.

impl From<Strip> for DecoderWrapper

fn from(from: Strip) -> Self

Converts to this type from the input type.

impl From<WordPiece> for DecoderWrapper

fn from(from: WordPiece) -> Self

Converts to this type from the input type.

impl Serialize for DecoderWrapper

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.