Humanizer-12B-Draftfit-DFlash / mlx-runtime.patch
jayyun98's picture
Publish Humanizer family draft and verified MLX comparison
8795749 verified
Raw History Blame Contribute Delete
29 kB
diff --git a/src/mlx_dspark/config.py b/src/mlx_dspark/config.py
index f3ea4cb..4bddbf2 100644
--- a/src/mlx_dspark/config.py
+++ b/src/mlx_dspark/config.py
@@ -109,6 +109,7 @@ def _translate_speculators(c: dict, path) -> dict:
lt = tlc.get("layer_types") or []
if "sliding_attention" in lt and tlc.get("use_sliding_window") and tlc.get("sliding_window"):
out["sliding_window"] = int(tlc["sliding_window"])
+ out["layer_types"] = list(lt)
# sliding_window_non_causal=False => the window is causal (DFlash query-causal). Absent
# defaults to causal, the DFlash norm for a sliding backbone.
out["dflash_query_causal"] = not bool(c.get("sliding_window_non_causal", False))
@@ -209,6 +210,7 @@ class DSparkConfig:
# heads (bidirectional full-attention block), so they default off and change nothing there.
causal_block: bool = False
sliding_window: int | None = None
+ layer_types: tuple[str, ...] = ()
attention_sink: bool = False
# has_lm_head=false drafters ship only embed_tokens and reuse the TARGET's lm_head (like
# DFlash's bind); generate binds it once at run time. True = the drafter has its own head.
@@ -306,7 +308,7 @@ class DSparkConfig:
# `dflash_config` dict tagged `projector_type: "dspark"` β€” the discriminator (absent
# from every other packaging; the fork-labelled heads carry a dflash_config too, but
# never that tag). Three things follow from the reference code these repos ship:
- # (1) target_layer_ids / mask_token_id live only inside dflash_config β€” hoist them;
+ # (1) target taps and head options can live only inside dflash_config β€” hoist them;
# (2) the DSpark flavor samples the ANCHOR slot (anchor-as-pos0, the DeepSpec
# convention its markov/confidence code is adapted from): a block_size-7 head
# proposes 7 tokens, and the card's own "verify width 8 including the bonus" is
@@ -324,10 +326,12 @@ class DSparkConfig:
_sf = (c.get("dflash_config") or {})
specforge = _sf.get("projector_type") == "dspark"
if specforge:
- for k in ("target_layer_ids", "mask_token_id"):
+ for k in ("target_layer_ids", "mask_token_id", "markov_rank", "markov_head_type",
+ "enable_confidence_head", "confidence_head_with_markov",
+ "sample_from_anchor", "sliding_window_non_causal"):
if k not in c and _sf.get(k) is not None:
c[k] = _sf[k]
- c.setdefault("logits_start", 0)
+ c.setdefault("logits_start", 0 if c.get("sample_from_anchor", True) else 1)
# LiquidAI LFM2.5-DSpark packaging (the fifth). A plain qwen3-backbone DSpark head
# (architectures ["Lfm2DSparkDraftModel"]) for an LFM2 conv+attention target. Like
@@ -375,6 +379,16 @@ class DSparkConfig:
f"does not look like a DeepSpec-format DSpark drafter checkpoint."
)
+ layer_types = tuple(c.get("layer_types") or ())
+ if layer_types and (len(layer_types) != c["num_hidden_layers"] or
+ set(layer_types) - {"sliding_attention", "full_attention"}):
+ raise ValueError(f"{path}: layer_types must name each draft layer as sliding or full")
+
+ # An explicitly declared full vocabulary uses identity token ids, not a d2t table.
+ draft_vocab_size = c.get("draft_vocab_size")
+ if draft_vocab_size == c["vocab_size"]:
+ draft_vocab_size = None
+
if family == "qwen3":
rp = c.get("rope_parameters") or {}
head_dim = c.get("head_dim", c["hidden_size"] // c["num_attention_heads"])
@@ -410,13 +424,17 @@ class DSparkConfig:
# the DFlash spelling of logits_start (false -> slot 0 is a pure anchor, predict
# from slot 1), the same convention the speculators path derives.
dflc = c.get("dflash_config") or {}
- causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False))
+ causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False)
+ or ("sliding_attention" in layer_types and not dflc.get(
+ "sliding_window_non_causal", c.get("sliding_window_non_causal", False))))
attn_sink = bool(c.get("attention_sink_bias", False))
swa = None
if causal_block or attn_sink or dflc:
w = c.get("sliding_window") or (
dflc.get("swa_window_size") if dflc.get("use_swa", False) else None)
swa = int(w) if w else None
+ if "sliding_attention" in layer_types and not swa:
+ raise ValueError(f"{path}: sliding_attention requires a sliding_window")
has_own_lm = bool(c.get("has_lm_head", True))
# YaRN rope β€” honored ONLY for the SpecForge packaging, where the reference builds
# transformers' Qwen3RotaryEmbedding from the head's own config and that class
@@ -456,7 +474,7 @@ class DSparkConfig:
return cls(
family="qwen3",
hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
- draft_vocab_size=c.get("draft_vocab_size"),
+ draft_vocab_size=draft_vocab_size,
num_hidden_layers=c["num_hidden_layers"],
intermediate_size=c["intermediate_size"],
rms_norm_eps=c.get("rms_norm_eps", 1e-6),
@@ -471,7 +489,7 @@ class DSparkConfig:
rope_traditional=rope_trad,
gated_q_proj=gated_q,
offset_rms_norm=offset_norms,
- causal_block=causal_block, sliding_window=swa,
+ causal_block=causal_block, sliding_window=swa, layer_types=layer_types,
attention_sink=attn_sink, has_own_lm_head=has_own_lm,
block_size=c["block_size"], mask_token_id=c["mask_token_id"],
logits_start=logits_start,
@@ -493,7 +511,7 @@ class DSparkConfig:
return cls(
family="gemma4",
hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
- draft_vocab_size=c.get("draft_vocab_size"),
+ draft_vocab_size=draft_vocab_size,
num_hidden_layers=c["num_hidden_layers"],
intermediate_size=c["intermediate_size"],
rms_norm_eps=c.get("rms_norm_eps", 1e-6),
diff --git a/src/mlx_dspark/dflash_model.py b/src/mlx_dspark/dflash_model.py
index a45ea26..b99627a 100644
--- a/src/mlx_dspark/dflash_model.py
+++ b/src/mlx_dspark/dflash_model.py
@@ -87,6 +87,7 @@ class DFlashConfig:
num_target_layers: int
mask_token_id: int = 0
rope_scaling: dict[str, Any] | None = None
+ rope_traditional: bool = False
layer_types: tuple[str, ...] = field(default_factory=tuple)
sliding_window: int | None = None
final_logit_softcapping: float | None = None
@@ -100,11 +101,11 @@ class DFlashConfig:
# reuse-head rule (raw lm_head only) the multiplier/softcap must be applied here.
-def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling):
+def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling, traditional):
return initialize_rope(
dims=head_dim,
base=rope_theta,
- traditional=False,
+ traditional=traditional,
scaling_config=rope_scaling,
max_position_embeddings=max_position_embeddings,
)
@@ -320,7 +321,8 @@ class DFlashDraftModel(nn.Module):
self.layers = [DFlashDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rope = _build_rope(
- config.head_dim, config.rope_theta, config.max_position_embeddings, config.rope_scaling
+ config.head_dim, config.rope_theta, config.max_position_embeddings,
+ config.rope_scaling, config.rope_traditional,
)
self.candidate_selector = (
CandidateSelector(config.hidden_size, config.vocab_size,
diff --git a/src/mlx_dspark/generate.py b/src/mlx_dspark/generate.py
index c0aae6a..acd6fdd 100644
--- a/src/mlx_dspark/generate.py
+++ b/src/mlx_dspark/generate.py
@@ -769,6 +769,18 @@ def dflash_generate(
st.update(out_ids)
while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+ remaining = max_new_tokens - len(out_ids)
+ if remaining == 1:
+ v_logits, _ = target_model.verify(mx.array([[pending]]), cache, tap)
+ nxt = _pick(v_logits[0, -1], temperature, top_p, top_k)
+ target_forwards += 1
+ accept_lengths.append(1)
+ if on_round is not None:
+ on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+ out_ids.append(nxt)
+ pending = nxt
+ st.update(out_ids)
+ break
# ---- free copy draft (a 4-gram+ suffix match extends an earlier occurrence): verify
# that continuation instead of running the DFlash drafter this round. DFlash caches
# ctx KV only (advanced inside the draft call from pending_ctx), so a copy round β€”
@@ -778,7 +790,7 @@ def dflash_generate(
# the target verifies every copied token; a bad copy just costs acceptance.
lk_draft = index.propose(
long_draft=lookup_long_draft if lk_gate.allowed else None,
- )[: max_new_tokens - len(out_ids)] if index is not None else []
+ )[: remaining - 1] if index is not None else []
if lk_draft:
lookup_rounds += 1
if temperature > 0.0:
@@ -804,6 +816,9 @@ def dflash_generate(
n = int(n_arr.item())
tt = tt_arr.tolist()
committed = lk_draft[:n] + [tt[n]]
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
lk_gate.update(len(lk_draft), n, max(1, lookup_max_draft))
target_forwards += 1
accept_lengths.append(len(committed))
@@ -831,6 +846,7 @@ def dflash_generate(
# any depth and ratchets up β€” measured 1.04x vs cap 3's 1.48x at 32k)
cap_controller.set_depth(len(ids) + len(out_ids))
cap = max(1, min(cap_controller.cap, cap_ceiling))
+ cap = min(cap, remaining - 1)
block = mx.array([[pending] + [mask_id] * (bs - 1)])
sel = getattr(drafter, "candidate_selector", None) # DFlash 2 path when present
if temperature > 0.0:
@@ -882,6 +898,9 @@ def dflash_generate(
drafted = draft_arr.tolist()
tt = tt_arr.tolist()
committed = drafted[:n] + [tt[n]] # accepted prefix + bonus
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if cap_controller is not None:
@@ -1223,12 +1242,30 @@ def speculative_generate(
st.update(out_ids)
while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+ remaining = max_new_tokens - len(out_ids)
lk_draft = []
if index is not None:
- # clamp to the remaining budget: no verify width wasted past max_new_tokens
+ # Reserve one position for the verifier's residual/bonus token.
lk_draft = index.propose(
long_draft=lookup_long_draft if lk_gate.allowed else None,
- )[:max_new_tokens - len(out_ids)]
+ )[:remaining - 1]
+ if remaining == 1:
+ # Consume the pending token through the tapped target path; its next token is
+ # the sole remaining output, with the target cache still one pending behind.
+ v_logits, v_fused = target_model.verify(mx.array([[pending]]), cache, tap)
+ nxt = _pick(pen.apply(v_logits[0], [])[0], temperature, top_p, top_k)
+ drafter.update_context(v_fused, ctx_offset=n_cached, ctx_caches=ctx_caches)
+ target_forwards += 1
+ accept_lengths.append(1)
+ if on_round is not None:
+ on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+ out_ids.append(nxt)
+ pen.add([nxt])
+ if lp_list is not None:
+ lp_list.extend(_logprobs_for_block(v_logits[0][:1], [nxt], logprobs))
+ pending = nxt
+ st.update(out_ids)
+ break
use_conf = confidence_threshold > 0.0 and drafter.confidence_head is not None
if cap_controller is not None and not lk_draft:
# live context depth -> the controller's verify pricing (see dflash_generate)
@@ -1236,6 +1273,7 @@ def speculative_generate(
# cap 0 = the controller parked speculation (live acceptance too low for this
# machine's verify slope): run a plain committed step; probe rounds re-enter.
cap = max(0, min(cap_controller.cap, cap_ceiling))
+ cap = min(cap, remaining - 1)
if not lk_draft and cap == 0 and not (temperature > 0.0 or pen.active
or lp_list is not None):
# ---- parked sprint ----
@@ -1406,6 +1444,9 @@ def speculative_generate(
draft = draft_arr.tolist()
tt = tt_arr.tolist()
committed = draft[:n] + [tt[n]] # accepted prefix + bonus
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if lk_draft:
diff --git a/src/mlx_dspark/load.py b/src/mlx_dspark/load.py
index eded611..376a05e 100644
--- a/src/mlx_dspark/load.py
+++ b/src/mlx_dspark/load.py
@@ -607,6 +607,7 @@ def load_dflash(repo_or_path: str, *, quantize: bool = True, bits: int = 4, grou
num_target_layers=cfg["num_target_layers"],
mask_token_id=dfc.get("mask_token_id", cfg.get("mask_token_id", 0)),
rope_scaling=cfg.get("rope_scaling"), layer_types=layer_types,
+ rope_traditional=not bool(cfg["rope_is_neox_style"]) if "rope_is_neox_style" in cfg else False,
sliding_window=cfg.get("sliding_window"),
# DFlash 2 nests these in dflash_config (muse); DFlash 1 gemma had softcap top-level
final_logit_softcapping=dfc.get("final_logit_softcapping",
diff --git a/src/mlx_dspark/lookup.py b/src/mlx_dspark/lookup.py
index f2b9ede..fb63573 100644
--- a/src/mlx_dspark/lookup.py
+++ b/src/mlx_dspark/lookup.py
@@ -207,7 +207,7 @@ def lookup_generate(
# clamp to the remaining budget: no verify width wasted past max_new_tokens
draft = index.propose(
long_draft=long_draft_tokens if gate.allowed else None,
- )[:max_new_tokens - len(out_ids)]
+ )[:max_new_tokens - len(out_ids) - 1]
if not draft:
# no confident match -> plain 1-token step (zero miss cost; verify() with no
@@ -237,6 +237,9 @@ def lookup_generate(
n = int(n_arr.item())
tt = tt_arr.tolist()
committed = draft[:n] + [tt[n]]
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if draft:
diff --git a/src/mlx_dspark/model.py b/src/mlx_dspark/model.py
index d293661..dcdb9ef 100644
--- a/src/mlx_dspark/model.py
+++ b/src/mlx_dspark/model.py
@@ -86,7 +86,7 @@ class CtxCache:
class DSparkAttention(nn.Module):
"""Cross-attention: Q from the draft block, K/V from [target_context, block]."""
- def __init__(self, config: DSparkConfig):
+ def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
super().__init__()
self.n_heads = config.num_attention_heads
self.head_dim = config.attn_head_dim
@@ -100,6 +100,10 @@ class DSparkAttention(nn.Module):
# (getattr defaults keep minimal config-like doubles working).
self.causal_block = getattr(config, "causal_block", False)
self.sliding_window = getattr(config, "sliding_window", None)
+ if layer_idx is not None and "sliding_attention" in getattr(config, "layer_types", ()):
+ if config.layer_types[layer_idx] == "full_attention":
+ self.causal_block = False
+ self.sliding_window = None
self.sink = getattr(config, "attention_sink", False)
h = config.hidden_size
@@ -164,10 +168,12 @@ class DSparkAttention(nn.Module):
# DFlash-lineage heads mask the block attention (causal within the block, sliding window
# over the context) and add a per-head sink logit. The default DSpark head does neither:
# mask stays None (full bidirectional block over the whole context) and sinks stays None,
- # so this is byte-identical there. The mask is only built for the single-sequence path
- # (int offset); the batched drafter path already supplies its own [B,1,k,L] mask.
- if mask is None and (self.causal_block or self.sliding_window) and isinstance(block_offset, int):
- mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+ # so this is byte-identical there. In a batch, combine the layer's attention
+ # constraint with the caller's context-padding mask.
+ if self.causal_block or self.sliding_window:
+ local_mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+ if local_mask is not None:
+ mask = local_mask if mask is None else mask & local_mask
sinks = self.attention_sink_bias if self.sink else None
# SDPA does GQA/MQA head broadcast internally β€” pass the n_kv-head K/V straight through.
@@ -197,22 +203,28 @@ class DSparkAttention(nn.Module):
W = self.sliding_window
if not self.causal_block and (W is None or total <= W):
return None
- qpos = block_offset + mx.arange(q_len)[:, None] # [q_len, 1]
- kpos = mx.arange(total)[None, :] # [1, total]
+ if isinstance(block_offset, int):
+ qpos = block_offset + mx.arange(q_len)[:, None]
+ kpos = mx.arange(total)[None, :]
+ else:
+ lengths = block_offset[:, None, None, None]
+ qpos = lengths + mx.arange(q_len)[None, None, :, None]
+ keys = mx.arange(total)[None, None, None, :]
+ kpos = mx.where(keys < ctx_len, keys, lengths + keys - ctx_len)
allowed = mx.ones((q_len, total), dtype=mx.bool_)
if self.causal_block:
allowed = allowed & (kpos <= qpos)
if W is not None:
allowed = allowed & (kpos > qpos - W)
- return allowed[None, None]
+ return allowed[None, None] if isinstance(block_offset, int) else allowed
class DSparkDecoderLayer(nn.Module):
- def __init__(self, config: DSparkConfig):
+ def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
super().__init__()
eps = config.rms_norm_eps
self.norm_style = config.norm_style
- self.self_attn = DSparkAttention(config)
+ self.self_attn = DSparkAttention(config, layer_idx)
self.mlp = MLP(config)
self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
@@ -314,7 +326,9 @@ class DSparkDrafter(nn.Module):
self.block_size = config.block_size
self.mask_token_id = config.mask_token_id
self.logits_start = config.logits_start
- self.causal_block = getattr(config, "causal_block", False)
+ layer_types = getattr(config, "layer_types", ())
+ mixed_attention = "sliding_attention" in layer_types and "full_attention" in layer_types
+ self.causal_block = getattr(config, "causal_block", False) and not mixed_attention
self.embed_scale = (float(config.hidden_size) ** 0.5) if config.family == "gemma4" else 1.0
self.softcap = config.final_logit_softcapping
@@ -328,7 +342,7 @@ class DSparkDrafter(nn.Module):
len(config.target_layer_ids) * config.hidden_size, config.hidden_size, bias=False
)
self.hidden_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.layers = [DSparkDecoderLayer(config) for _ in range(config.num_hidden_layers)]
+ self.layers = [DSparkDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# A has_lm_head=false drafter (Nemotron DSpark head) reuses the TARGET's lm_head β€” it
# ships no lm_head weight, so we don't build one; bind_lm_head() supplies it at run time.
@@ -381,15 +395,6 @@ class DSparkDrafter(nn.Module):
def backbone(self, noise_embedding, block_offset, ctx_caches, mask=None) -> mx.array:
h = noise_embedding
- if mask is None and isinstance(block_offset, int) and self.layers:
- # Every layer shares one attention config, so the block mask (a function of
- # block_offset / ctx_len / q_len only) is identical across them β€” build it once
- # here instead of 5x per round. attend() keeps its own fallback for direct
- # callers; None stays None for the default bidirectional DSpark head.
- attn = self.layers[0].self_attn
- if attn.causal_block or attn.sliding_window:
- mask = attn._block_mask(
- block_offset, ctx_len=ctx_caches[0].length, q_len=h.shape[1])
for layer, cache in zip(self.layers, ctx_caches):
h = layer(h, block_offset, cache, mask)
return self.norm(h)
diff --git a/tests/test_dflash2.py b/tests/test_dflash2.py
index 8b5b3d9..7d3bb9b 100644
--- a/tests/test_dflash2.py
+++ b/tests/test_dflash2.py
@@ -327,6 +327,18 @@ def test_load_dflash2_roundtrip_and_config_fields(tmp_path):
assert (cfg.selector_rank, cfg.selector_top_k) == (8, 4)
assert (cfg.conv_kernel_size, cfg.conv_group_size) == (2, 16)
assert cfg.output_multiplier == 1.0 and cfg.final_logit_softcapping is None
+ assert cfg.rope_traditional is False
+
+
+def test_lfm_dflash_interleaved_rope_matches_reference(tmp_path):
+ cfg_dict = {**DFLASH2_MIN, "rope_is_neox_style": False}
+ drafter, cfg = load_dflash(_write_dflash2_ckpt(tmp_path, cfg_dict), quantize=False)
+ assert cfg.rope_traditional is True
+ x = mx.arange(16, dtype=mx.float32).reshape(1, 1, 1, 16) / 16
+ expected = nn.RoPE(16, traditional=True, base=1e7)(x, offset=2)
+ assert mx.allclose(drafter.rope(x, offset=2), expected, atol=1e-6).item()
+ neox = nn.RoPE(16, traditional=False, base=1e7)(x, offset=2)
+ assert not mx.allclose(drafter.rope(x, offset=2), neox).item()
def test_load_dflash2_reads_muse_transform_fields_nested(tmp_path):
diff --git a/tests/test_formats.py b/tests/test_formats.py
index 33f206b..3fcbfb9 100644
--- a/tests/test_formats.py
+++ b/tests/test_formats.py
@@ -417,6 +417,15 @@ def test_speculators_dflash_lineage_causal_sliding_window(tmp_path):
assert base.causal_block is False and base.sliding_window is None
+def test_taichu_mixed_attention_config(tmp_path):
+ config = {**QWEN3_MIN, "num_hidden_layers": 2,
+ "layer_types": ["sliding_attention", "full_attention"],
+ "use_sliding_window": True, "sliding_window": 4,
+ "dflash_config": {"causal": True}}
+ parsed = DSparkConfig.from_json(_cfg_path(tmp_path, config))
+ assert parsed.layer_types == ("sliding_attention", "full_attention")
+
+
def test_specforge_dspark_config_parses(tmp_path):
"""SpecForge packaging: nested taps hoisted, anchor slot reserved, yarn rope carried."""
cfg = DSparkConfig.from_json(_cfg_path(tmp_path, RADIXARK_SPECFORGE))
@@ -546,6 +555,16 @@ def test_lfm2_dspark_reuse_both_roundtrips(tmp_path):
assert drafter.draft_width(2) == cfg.block_size # bidirectional: backbone stays full-width
+def test_lfm2_full_vocab_explicit_size_needs_no_mapping(tmp_path):
+ config = {**LFM2_DSPARK, "draft_vocab_size": LFM2_DSPARK["vocab_size"]}
+ weights = _reference_weights(config)
+ path = _write_drafter_ckpt(tmp_path, weights, config)
+ drafter, cfg = load_drafter(path, quantize=False)
+ assert cfg.draft_vocab_size is None
+ assert cfg.rope_traditional is True
+ assert drafter.to_target_ids(mx.array([0, 7, 31])).tolist() == [0, 7, 31]
+
+
def test_plain_qwen3_keeps_neox_rope_without_the_flag(tmp_path):
"""A config without rope_is_neox_style keeps the family default (neox / traditional=False),
so every existing head is untouched by the new interleaved-rope knob."""
@@ -660,6 +679,27 @@ def test_load_drafter_matching_checkpoint_loads(tmp_path):
assert cfg.family == "qwen3" and drafter.block_size == 4
+def test_specforge_nested_head_flags_strict_checkpoint(tmp_path):
+ # The trained Taichu export keeps these head flags only under dflash_config.
+ # Its checkpoint has Markov weights and no confidence tensors.
+ nested = {**RADIXARK_SPECFORGE["dflash_config"],
+ "markov_rank": 3, "enable_confidence_head": False,
+ "sample_from_anchor": False}
+ config = {k: v for k, v in RADIXARK_SPECFORGE.items()
+ if k not in ("markov_rank", "markov_head_type", "enable_confidence_head",
+ "confidence_head_with_markov", "sample_from_anchor")}
+ config["dflash_config"] = nested
+ weights = _reference_weights({**config, "markov_rank": 3,
+ "enable_confidence_head": False})
+ path = _write_drafter_ckpt(tmp_path, weights, cfg=config)
+ drafter, parsed = load_drafter(path, quantize=False)
+ assert parsed.markov_rank == 3 and parsed.enable_confidence_head is False
+ assert parsed.logits_start == 1 and drafter.confidence_head is None
+ assert parsed.confidence_head_with_markov is True
+ assert DSparkConfig.from_json(_cfg_path(tmp_path, {
+ **config, "enable_confidence_head": True})).enable_confidence_head is True
+
+
def test_load_drafter_mismatch_raises_by_default(tmp_path):
weights = _reference_weights()
weights["bogus.weight"] = weights.pop("lm_head.weight")
diff --git a/tests/test_model.py b/tests/test_model.py
index 63b1dab..f7d4811 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -140,6 +140,18 @@ def test_draft_width_full_for_bidirectional_truncated_for_causal():
assert [d.draft_width(c) for c in (1, 4, 99)] == [2, 5, 8]
+def test_mixed_sliding_full_layers_keep_distinct_masks():
+ d = _tiny_drafter(layer_types=("sliding_attention", "full_attention"),
+ sliding_window=4, causal_block=True)
+ sliding, full = (layer.self_attn for layer in d.layers)
+ assert sliding.causal_block and sliding.sliding_window == 4
+ assert not full.causal_block and full.sliding_window is None
+ assert sliding._block_mask(6, ctx_len=6, q_len=2).tolist() == [[[[False, False, False,
+ True, True, True, True, False], [False, False, False, False, True, True, True, True]]]]
+ assert full._block_mask(6, ctx_len=6, q_len=2) is None
+ assert d.draft_width(2) == 8 # the final bidirectional layer reads the whole block
+
+
def test_causal_backbone_truncation_matches_full_width_rows():
"""The load-bearing invariant behind draft_width: with a causal block mask, position i
attends only positions <= i, so running the backbone at a truncated width must reproduce
@@ -165,3 +177,13 @@ def test_causal_backbone_truncation_matches_full_width_rows():
full = d.backbone(noise, 10, ctx)
trunc = d.backbone(noise[:, :3, :], 10, ctx)
assert not mx.allclose(full[:, :3, :], trunc, atol=1e-5).item()
+
+
+def test_full_attention_keeps_explicit_causal_block():
+ d = _tiny_drafter(layer_types=('full_attention', 'full_attention'), causal_block=True)
+ assert d.causal_block
+ assert d.draft_width(2) == 2
+ for layer in d.layers:
+ mask = layer.self_attn._block_mask(4, ctx_len=4, q_len=2)
+ assert mask[0, 0, 0, -1].item() is False
+ assert mask[0, 0, 1, -1].item() is True