Function tokenizers::pre_tokenizers::byte_level::process_offsets
source · pub fn process_offsets(encoding: &mut Encoding, add_prefix_space: bool)pub fn process_offsets(encoding: &mut Encoding, add_prefix_space: bool)