Struct tokenizers::models::unigram::Lattice

pub struct Lattice<'a> { /* private fields */ }

Structure to implement Viterbi algorithm to find the best encoding, or sample from all possible encodings of a given sentence.

Implementations

impl<'a> Lattice<'a>

pub fn from(sentence: &'a str, bos_id: usize, eos_id: usize) -> Self

pub fn insert(&mut self, pos: usize, length: usize, score: f64, id: usize)

pub fn viterbi(&mut self) -> Vec<Rc<RefCell<Node>>>

pub fn piece(&self, node: &Node) -> String

pub fn tokens(&mut self) -> Vec<String>

pub fn nbest(&mut self, n: usize) -> Vec<Vec<Rc<RefCell<Node>>>>

pub fn nbest_tokens(&mut self, n: usize) -> Vec<Vec<String>>

pub fn len(&self) -> usize

pub fn is_empty(&self) -> bool

pub fn bos_node(&self) -> Rc<RefCell<Node>>

pub fn eos_node(&self) -> Rc<RefCell<Node>>

pub fn surface(&self, n: usize) -> &str

pub fn sentence(&self) -> &str

pub fn populate_marginal(&self, freq: f64, expected: &mut [f64]) -> f64

pub fn sample(&self, theta: f64) -> Vec<Rc<RefCell<Node>>>

pub fn sample_token(&self, theta: f64) -> Vec<String>

Trait Implementations

impl<'a> Debug for Lattice<'a>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Display for Lattice<'_>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.