cherenkov/qwen4_exp/mod.rs
1//! qwen4-exp (model_type qwen4_exp_text): 48-layer hybrid of Gated
2//! DeltaNet and sparse attention, 512-expert MoE per layer, four-stream
3//! gated residual ("hyper connections"), a hashed n-gram embedding injected
4//! at one layer, and a one-layer MTP head. 125B parameters, 6B active.
5//!
6//! Weights come from the packed layout written by `pack`. Metal wires whole buffers, so
7//! every expert record must be its own page-aligned buffer, and the MLX
8//! shards are not even 4-byte aligned on disk.
9
10pub mod cpu;
11pub mod gpu;
12pub mod lowbit;
13pub mod pack;
14pub mod packed;
15
16mod config;
17mod manifest;
18pub use config::{Qwen4ExpConfig, RopeParams};
19pub use manifest::{DenseEntry, ExpertLayout, Manifest, NgramLayout, PAGE};