Struct Decode 

pub(crate) struct Decode {
    cur: u32,
    drafts: Vec<u32>,
    n_draft: usize,
    sampler: Option<Sampler>,
    eos: Option<[u32; 2]>,
    limit: usize,
    pub tokens: Vec<u32>,
    pub finish_reason: Option<&'static str>,
    pub steps: usize,
    pub accepted: usize,
}

Fields

cur: u32

drafts: Vec<u32>

n_draft: usize

sampler: Option<Sampler>

eos: Option<[u32; 2]>

limit: usize

tokens: Vec<u32>

finish_reason: Option<&'static str>

steps: usize

accepted: usize

Implementations

impl Decode

pub(crate) fn rng(&self) -> Option<StdRng>

pub(crate) fn new( seed: PrefillResume, ids: &[u32], tok: &ChatTokenizer, options: &Options, limit: usize, rng: Option<StdRng>, ) -> Result<Self>

fn is_eos(&self, token: u32) -> bool

fn emit( &mut self, token: u32, emit: &mut dyn FnMut(u32) -> Result<()>, ) -> Result<()>

pub(crate) fn step( &mut self, gpu: &mut Gpu<'_>, cpu: Option<&CpuModel<'_>>, cpu_state: Option<&mut State>, emit: &mut dyn FnMut(u32) -> Result<()>, ) -> Result<()>

The step returns after committed trunk/MTP work, so GPU scratch can be reused.