Struct PrefixCache 

pub(crate) struct PrefixCache {
    entries: VecDeque<Entry>,
    capacity: usize,
    used: usize,
    evictions: u64,
    max_entries: usize,
    idle: Option<Duration>,
}

Fields

entries: VecDeque<Entry>

capacity: usize

used: usize

evictions: u64

max_entries: usize

idle: Option<Duration>

Implementations

impl PrefixCache

pub(crate) fn new( capacity: usize, max_entries: usize, idle_seconds: u64, ) -> Self

fn evict_oldest(&mut self)

pub(crate) fn expire(&mut self)

pub(crate) fn stats(&self) -> (usize, usize, u64)

fn store( &mut self, gpu: &Gpu<'_>, ids: &[u32], seed: &PrefillResume, drafting: bool, )

pub(crate) fn begin( &mut self, gpu: &mut Gpu<'_>, ids: &[u32], stable_boundaries: &[usize], options: &Options, ) -> Result<Prefill>