Transformers
Safetensors
English
Chinese
qwen3
draftfit
speculative-decoding
dflash
humanizer
text-rewriting
rewriting
paraphrase
style-transfer
gemma4
text-generation-inference
Instructions to use jayyun98/Humanizer-12B-Draftfit-DFlash with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use jayyun98/Humanizer-12B-Draftfit-DFlash with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModel tokenizer = AutoTokenizer.from_pretrained("jayyun98/Humanizer-12B-Draftfit-DFlash") model = AutoModel.from_pretrained("jayyun98/Humanizer-12B-Draftfit-DFlash", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 28,983 Bytes
8795749 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 | diff --git a/src/mlx_dspark/config.py b/src/mlx_dspark/config.py
index f3ea4cb..4bddbf2 100644
--- a/src/mlx_dspark/config.py
+++ b/src/mlx_dspark/config.py
@@ -109,6 +109,7 @@ def _translate_speculators(c: dict, path) -> dict:
lt = tlc.get("layer_types") or []
if "sliding_attention" in lt and tlc.get("use_sliding_window") and tlc.get("sliding_window"):
out["sliding_window"] = int(tlc["sliding_window"])
+ out["layer_types"] = list(lt)
# sliding_window_non_causal=False => the window is causal (DFlash query-causal). Absent
# defaults to causal, the DFlash norm for a sliding backbone.
out["dflash_query_causal"] = not bool(c.get("sliding_window_non_causal", False))
@@ -209,6 +210,7 @@ class DSparkConfig:
# heads (bidirectional full-attention block), so they default off and change nothing there.
causal_block: bool = False
sliding_window: int | None = None
+ layer_types: tuple[str, ...] = ()
attention_sink: bool = False
# has_lm_head=false drafters ship only embed_tokens and reuse the TARGET's lm_head (like
# DFlash's bind); generate binds it once at run time. True = the drafter has its own head.
@@ -306,7 +308,7 @@ class DSparkConfig:
# `dflash_config` dict tagged `projector_type: "dspark"` β the discriminator (absent
# from every other packaging; the fork-labelled heads carry a dflash_config too, but
# never that tag). Three things follow from the reference code these repos ship:
- # (1) target_layer_ids / mask_token_id live only inside dflash_config β hoist them;
+ # (1) target taps and head options can live only inside dflash_config β hoist them;
# (2) the DSpark flavor samples the ANCHOR slot (anchor-as-pos0, the DeepSpec
# convention its markov/confidence code is adapted from): a block_size-7 head
# proposes 7 tokens, and the card's own "verify width 8 including the bonus" is
@@ -324,10 +326,12 @@ class DSparkConfig:
_sf = (c.get("dflash_config") or {})
specforge = _sf.get("projector_type") == "dspark"
if specforge:
- for k in ("target_layer_ids", "mask_token_id"):
+ for k in ("target_layer_ids", "mask_token_id", "markov_rank", "markov_head_type",
+ "enable_confidence_head", "confidence_head_with_markov",
+ "sample_from_anchor", "sliding_window_non_causal"):
if k not in c and _sf.get(k) is not None:
c[k] = _sf[k]
- c.setdefault("logits_start", 0)
+ c.setdefault("logits_start", 0 if c.get("sample_from_anchor", True) else 1)
# LiquidAI LFM2.5-DSpark packaging (the fifth). A plain qwen3-backbone DSpark head
# (architectures ["Lfm2DSparkDraftModel"]) for an LFM2 conv+attention target. Like
@@ -375,6 +379,16 @@ class DSparkConfig:
f"does not look like a DeepSpec-format DSpark drafter checkpoint."
)
+ layer_types = tuple(c.get("layer_types") or ())
+ if layer_types and (len(layer_types) != c["num_hidden_layers"] or
+ set(layer_types) - {"sliding_attention", "full_attention"}):
+ raise ValueError(f"{path}: layer_types must name each draft layer as sliding or full")
+
+ # An explicitly declared full vocabulary uses identity token ids, not a d2t table.
+ draft_vocab_size = c.get("draft_vocab_size")
+ if draft_vocab_size == c["vocab_size"]:
+ draft_vocab_size = None
+
if family == "qwen3":
rp = c.get("rope_parameters") or {}
head_dim = c.get("head_dim", c["hidden_size"] // c["num_attention_heads"])
@@ -410,13 +424,17 @@ class DSparkConfig:
# the DFlash spelling of logits_start (false -> slot 0 is a pure anchor, predict
# from slot 1), the same convention the speculators path derives.
dflc = c.get("dflash_config") or {}
- causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False))
+ causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False)
+ or ("sliding_attention" in layer_types and not dflc.get(
+ "sliding_window_non_causal", c.get("sliding_window_non_causal", False))))
attn_sink = bool(c.get("attention_sink_bias", False))
swa = None
if causal_block or attn_sink or dflc:
w = c.get("sliding_window") or (
dflc.get("swa_window_size") if dflc.get("use_swa", False) else None)
swa = int(w) if w else None
+ if "sliding_attention" in layer_types and not swa:
+ raise ValueError(f"{path}: sliding_attention requires a sliding_window")
has_own_lm = bool(c.get("has_lm_head", True))
# YaRN rope β honored ONLY for the SpecForge packaging, where the reference builds
# transformers' Qwen3RotaryEmbedding from the head's own config and that class
@@ -456,7 +474,7 @@ class DSparkConfig:
return cls(
family="qwen3",
hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
- draft_vocab_size=c.get("draft_vocab_size"),
+ draft_vocab_size=draft_vocab_size,
num_hidden_layers=c["num_hidden_layers"],
intermediate_size=c["intermediate_size"],
rms_norm_eps=c.get("rms_norm_eps", 1e-6),
@@ -471,7 +489,7 @@ class DSparkConfig:
rope_traditional=rope_trad,
gated_q_proj=gated_q,
offset_rms_norm=offset_norms,
- causal_block=causal_block, sliding_window=swa,
+ causal_block=causal_block, sliding_window=swa, layer_types=layer_types,
attention_sink=attn_sink, has_own_lm_head=has_own_lm,
block_size=c["block_size"], mask_token_id=c["mask_token_id"],
logits_start=logits_start,
@@ -493,7 +511,7 @@ class DSparkConfig:
return cls(
family="gemma4",
hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
- draft_vocab_size=c.get("draft_vocab_size"),
+ draft_vocab_size=draft_vocab_size,
num_hidden_layers=c["num_hidden_layers"],
intermediate_size=c["intermediate_size"],
rms_norm_eps=c.get("rms_norm_eps", 1e-6),
diff --git a/src/mlx_dspark/dflash_model.py b/src/mlx_dspark/dflash_model.py
index a45ea26..b99627a 100644
--- a/src/mlx_dspark/dflash_model.py
+++ b/src/mlx_dspark/dflash_model.py
@@ -87,6 +87,7 @@ class DFlashConfig:
num_target_layers: int
mask_token_id: int = 0
rope_scaling: dict[str, Any] | None = None
+ rope_traditional: bool = False
layer_types: tuple[str, ...] = field(default_factory=tuple)
sliding_window: int | None = None
final_logit_softcapping: float | None = None
@@ -100,11 +101,11 @@ class DFlashConfig:
# reuse-head rule (raw lm_head only) the multiplier/softcap must be applied here.
-def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling):
+def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling, traditional):
return initialize_rope(
dims=head_dim,
base=rope_theta,
- traditional=False,
+ traditional=traditional,
scaling_config=rope_scaling,
max_position_embeddings=max_position_embeddings,
)
@@ -320,7 +321,8 @@ class DFlashDraftModel(nn.Module):
self.layers = [DFlashDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rope = _build_rope(
- config.head_dim, config.rope_theta, config.max_position_embeddings, config.rope_scaling
+ config.head_dim, config.rope_theta, config.max_position_embeddings,
+ config.rope_scaling, config.rope_traditional,
)
self.candidate_selector = (
CandidateSelector(config.hidden_size, config.vocab_size,
diff --git a/src/mlx_dspark/generate.py b/src/mlx_dspark/generate.py
index c0aae6a..acd6fdd 100644
--- a/src/mlx_dspark/generate.py
+++ b/src/mlx_dspark/generate.py
@@ -769,6 +769,18 @@ def dflash_generate(
st.update(out_ids)
while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+ remaining = max_new_tokens - len(out_ids)
+ if remaining == 1:
+ v_logits, _ = target_model.verify(mx.array([[pending]]), cache, tap)
+ nxt = _pick(v_logits[0, -1], temperature, top_p, top_k)
+ target_forwards += 1
+ accept_lengths.append(1)
+ if on_round is not None:
+ on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+ out_ids.append(nxt)
+ pending = nxt
+ st.update(out_ids)
+ break
# ---- free copy draft (a 4-gram+ suffix match extends an earlier occurrence): verify
# that continuation instead of running the DFlash drafter this round. DFlash caches
# ctx KV only (advanced inside the draft call from pending_ctx), so a copy round β
@@ -778,7 +790,7 @@ def dflash_generate(
# the target verifies every copied token; a bad copy just costs acceptance.
lk_draft = index.propose(
long_draft=lookup_long_draft if lk_gate.allowed else None,
- )[: max_new_tokens - len(out_ids)] if index is not None else []
+ )[: remaining - 1] if index is not None else []
if lk_draft:
lookup_rounds += 1
if temperature > 0.0:
@@ -804,6 +816,9 @@ def dflash_generate(
n = int(n_arr.item())
tt = tt_arr.tolist()
committed = lk_draft[:n] + [tt[n]]
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
lk_gate.update(len(lk_draft), n, max(1, lookup_max_draft))
target_forwards += 1
accept_lengths.append(len(committed))
@@ -831,6 +846,7 @@ def dflash_generate(
# any depth and ratchets up β measured 1.04x vs cap 3's 1.48x at 32k)
cap_controller.set_depth(len(ids) + len(out_ids))
cap = max(1, min(cap_controller.cap, cap_ceiling))
+ cap = min(cap, remaining - 1)
block = mx.array([[pending] + [mask_id] * (bs - 1)])
sel = getattr(drafter, "candidate_selector", None) # DFlash 2 path when present
if temperature > 0.0:
@@ -882,6 +898,9 @@ def dflash_generate(
drafted = draft_arr.tolist()
tt = tt_arr.tolist()
committed = drafted[:n] + [tt[n]] # accepted prefix + bonus
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if cap_controller is not None:
@@ -1223,12 +1242,30 @@ def speculative_generate(
st.update(out_ids)
while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+ remaining = max_new_tokens - len(out_ids)
lk_draft = []
if index is not None:
- # clamp to the remaining budget: no verify width wasted past max_new_tokens
+ # Reserve one position for the verifier's residual/bonus token.
lk_draft = index.propose(
long_draft=lookup_long_draft if lk_gate.allowed else None,
- )[:max_new_tokens - len(out_ids)]
+ )[:remaining - 1]
+ if remaining == 1:
+ # Consume the pending token through the tapped target path; its next token is
+ # the sole remaining output, with the target cache still one pending behind.
+ v_logits, v_fused = target_model.verify(mx.array([[pending]]), cache, tap)
+ nxt = _pick(pen.apply(v_logits[0], [])[0], temperature, top_p, top_k)
+ drafter.update_context(v_fused, ctx_offset=n_cached, ctx_caches=ctx_caches)
+ target_forwards += 1
+ accept_lengths.append(1)
+ if on_round is not None:
+ on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+ out_ids.append(nxt)
+ pen.add([nxt])
+ if lp_list is not None:
+ lp_list.extend(_logprobs_for_block(v_logits[0][:1], [nxt], logprobs))
+ pending = nxt
+ st.update(out_ids)
+ break
use_conf = confidence_threshold > 0.0 and drafter.confidence_head is not None
if cap_controller is not None and not lk_draft:
# live context depth -> the controller's verify pricing (see dflash_generate)
@@ -1236,6 +1273,7 @@ def speculative_generate(
# cap 0 = the controller parked speculation (live acceptance too low for this
# machine's verify slope): run a plain committed step; probe rounds re-enter.
cap = max(0, min(cap_controller.cap, cap_ceiling))
+ cap = min(cap, remaining - 1)
if not lk_draft and cap == 0 and not (temperature > 0.0 or pen.active
or lp_list is not None):
# ---- parked sprint ----
@@ -1406,6 +1444,9 @@ def speculative_generate(
draft = draft_arr.tolist()
tt = tt_arr.tolist()
committed = draft[:n] + [tt[n]] # accepted prefix + bonus
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if lk_draft:
diff --git a/src/mlx_dspark/load.py b/src/mlx_dspark/load.py
index eded611..376a05e 100644
--- a/src/mlx_dspark/load.py
+++ b/src/mlx_dspark/load.py
@@ -607,6 +607,7 @@ def load_dflash(repo_or_path: str, *, quantize: bool = True, bits: int = 4, grou
num_target_layers=cfg["num_target_layers"],
mask_token_id=dfc.get("mask_token_id", cfg.get("mask_token_id", 0)),
rope_scaling=cfg.get("rope_scaling"), layer_types=layer_types,
+ rope_traditional=not bool(cfg["rope_is_neox_style"]) if "rope_is_neox_style" in cfg else False,
sliding_window=cfg.get("sliding_window"),
# DFlash 2 nests these in dflash_config (muse); DFlash 1 gemma had softcap top-level
final_logit_softcapping=dfc.get("final_logit_softcapping",
diff --git a/src/mlx_dspark/lookup.py b/src/mlx_dspark/lookup.py
index f2b9ede..fb63573 100644
--- a/src/mlx_dspark/lookup.py
+++ b/src/mlx_dspark/lookup.py
@@ -207,7 +207,7 @@ def lookup_generate(
# clamp to the remaining budget: no verify width wasted past max_new_tokens
draft = index.propose(
long_draft=long_draft_tokens if gate.allowed else None,
- )[:max_new_tokens - len(out_ids)]
+ )[:max_new_tokens - len(out_ids) - 1]
if not draft:
# no confident match -> plain 1-token step (zero miss cost; verify() with no
@@ -237,6 +237,9 @@ def lookup_generate(
n = int(n_arr.item())
tt = tt_arr.tolist()
committed = draft[:n] + [tt[n]]
+ if any(tok in eos_ids for tok in committed):
+ committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+ n = min(n, len(committed) - 1)
target_forwards += 1
accept_lengths.append(len(committed))
if draft:
diff --git a/src/mlx_dspark/model.py b/src/mlx_dspark/model.py
index d293661..dcdb9ef 100644
--- a/src/mlx_dspark/model.py
+++ b/src/mlx_dspark/model.py
@@ -86,7 +86,7 @@ class CtxCache:
class DSparkAttention(nn.Module):
"""Cross-attention: Q from the draft block, K/V from [target_context, block]."""
- def __init__(self, config: DSparkConfig):
+ def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
super().__init__()
self.n_heads = config.num_attention_heads
self.head_dim = config.attn_head_dim
@@ -100,6 +100,10 @@ class DSparkAttention(nn.Module):
# (getattr defaults keep minimal config-like doubles working).
self.causal_block = getattr(config, "causal_block", False)
self.sliding_window = getattr(config, "sliding_window", None)
+ if layer_idx is not None and "sliding_attention" in getattr(config, "layer_types", ()):
+ if config.layer_types[layer_idx] == "full_attention":
+ self.causal_block = False
+ self.sliding_window = None
self.sink = getattr(config, "attention_sink", False)
h = config.hidden_size
@@ -164,10 +168,12 @@ class DSparkAttention(nn.Module):
# DFlash-lineage heads mask the block attention (causal within the block, sliding window
# over the context) and add a per-head sink logit. The default DSpark head does neither:
# mask stays None (full bidirectional block over the whole context) and sinks stays None,
- # so this is byte-identical there. The mask is only built for the single-sequence path
- # (int offset); the batched drafter path already supplies its own [B,1,k,L] mask.
- if mask is None and (self.causal_block or self.sliding_window) and isinstance(block_offset, int):
- mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+ # so this is byte-identical there. In a batch, combine the layer's attention
+ # constraint with the caller's context-padding mask.
+ if self.causal_block or self.sliding_window:
+ local_mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+ if local_mask is not None:
+ mask = local_mask if mask is None else mask & local_mask
sinks = self.attention_sink_bias if self.sink else None
# SDPA does GQA/MQA head broadcast internally β pass the n_kv-head K/V straight through.
@@ -197,22 +203,28 @@ class DSparkAttention(nn.Module):
W = self.sliding_window
if not self.causal_block and (W is None or total <= W):
return None
- qpos = block_offset + mx.arange(q_len)[:, None] # [q_len, 1]
- kpos = mx.arange(total)[None, :] # [1, total]
+ if isinstance(block_offset, int):
+ qpos = block_offset + mx.arange(q_len)[:, None]
+ kpos = mx.arange(total)[None, :]
+ else:
+ lengths = block_offset[:, None, None, None]
+ qpos = lengths + mx.arange(q_len)[None, None, :, None]
+ keys = mx.arange(total)[None, None, None, :]
+ kpos = mx.where(keys < ctx_len, keys, lengths + keys - ctx_len)
allowed = mx.ones((q_len, total), dtype=mx.bool_)
if self.causal_block:
allowed = allowed & (kpos <= qpos)
if W is not None:
allowed = allowed & (kpos > qpos - W)
- return allowed[None, None]
+ return allowed[None, None] if isinstance(block_offset, int) else allowed
class DSparkDecoderLayer(nn.Module):
- def __init__(self, config: DSparkConfig):
+ def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
super().__init__()
eps = config.rms_norm_eps
self.norm_style = config.norm_style
- self.self_attn = DSparkAttention(config)
+ self.self_attn = DSparkAttention(config, layer_idx)
self.mlp = MLP(config)
self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
@@ -314,7 +326,9 @@ class DSparkDrafter(nn.Module):
self.block_size = config.block_size
self.mask_token_id = config.mask_token_id
self.logits_start = config.logits_start
- self.causal_block = getattr(config, "causal_block", False)
+ layer_types = getattr(config, "layer_types", ())
+ mixed_attention = "sliding_attention" in layer_types and "full_attention" in layer_types
+ self.causal_block = getattr(config, "causal_block", False) and not mixed_attention
self.embed_scale = (float(config.hidden_size) ** 0.5) if config.family == "gemma4" else 1.0
self.softcap = config.final_logit_softcapping
@@ -328,7 +342,7 @@ class DSparkDrafter(nn.Module):
len(config.target_layer_ids) * config.hidden_size, config.hidden_size, bias=False
)
self.hidden_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.layers = [DSparkDecoderLayer(config) for _ in range(config.num_hidden_layers)]
+ self.layers = [DSparkDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# A has_lm_head=false drafter (Nemotron DSpark head) reuses the TARGET's lm_head β it
# ships no lm_head weight, so we don't build one; bind_lm_head() supplies it at run time.
@@ -381,15 +395,6 @@ class DSparkDrafter(nn.Module):
def backbone(self, noise_embedding, block_offset, ctx_caches, mask=None) -> mx.array:
h = noise_embedding
- if mask is None and isinstance(block_offset, int) and self.layers:
- # Every layer shares one attention config, so the block mask (a function of
- # block_offset / ctx_len / q_len only) is identical across them β build it once
- # here instead of 5x per round. attend() keeps its own fallback for direct
- # callers; None stays None for the default bidirectional DSpark head.
- attn = self.layers[0].self_attn
- if attn.causal_block or attn.sliding_window:
- mask = attn._block_mask(
- block_offset, ctx_len=ctx_caches[0].length, q_len=h.shape[1])
for layer, cache in zip(self.layers, ctx_caches):
h = layer(h, block_offset, cache, mask)
return self.norm(h)
diff --git a/tests/test_dflash2.py b/tests/test_dflash2.py
index 8b5b3d9..7d3bb9b 100644
--- a/tests/test_dflash2.py
+++ b/tests/test_dflash2.py
@@ -327,6 +327,18 @@ def test_load_dflash2_roundtrip_and_config_fields(tmp_path):
assert (cfg.selector_rank, cfg.selector_top_k) == (8, 4)
assert (cfg.conv_kernel_size, cfg.conv_group_size) == (2, 16)
assert cfg.output_multiplier == 1.0 and cfg.final_logit_softcapping is None
+ assert cfg.rope_traditional is False
+
+
+def test_lfm_dflash_interleaved_rope_matches_reference(tmp_path):
+ cfg_dict = {**DFLASH2_MIN, "rope_is_neox_style": False}
+ drafter, cfg = load_dflash(_write_dflash2_ckpt(tmp_path, cfg_dict), quantize=False)
+ assert cfg.rope_traditional is True
+ x = mx.arange(16, dtype=mx.float32).reshape(1, 1, 1, 16) / 16
+ expected = nn.RoPE(16, traditional=True, base=1e7)(x, offset=2)
+ assert mx.allclose(drafter.rope(x, offset=2), expected, atol=1e-6).item()
+ neox = nn.RoPE(16, traditional=False, base=1e7)(x, offset=2)
+ assert not mx.allclose(drafter.rope(x, offset=2), neox).item()
def test_load_dflash2_reads_muse_transform_fields_nested(tmp_path):
diff --git a/tests/test_formats.py b/tests/test_formats.py
index 33f206b..3fcbfb9 100644
--- a/tests/test_formats.py
+++ b/tests/test_formats.py
@@ -417,6 +417,15 @@ def test_speculators_dflash_lineage_causal_sliding_window(tmp_path):
assert base.causal_block is False and base.sliding_window is None
+def test_taichu_mixed_attention_config(tmp_path):
+ config = {**QWEN3_MIN, "num_hidden_layers": 2,
+ "layer_types": ["sliding_attention", "full_attention"],
+ "use_sliding_window": True, "sliding_window": 4,
+ "dflash_config": {"causal": True}}
+ parsed = DSparkConfig.from_json(_cfg_path(tmp_path, config))
+ assert parsed.layer_types == ("sliding_attention", "full_attention")
+
+
def test_specforge_dspark_config_parses(tmp_path):
"""SpecForge packaging: nested taps hoisted, anchor slot reserved, yarn rope carried."""
cfg = DSparkConfig.from_json(_cfg_path(tmp_path, RADIXARK_SPECFORGE))
@@ -546,6 +555,16 @@ def test_lfm2_dspark_reuse_both_roundtrips(tmp_path):
assert drafter.draft_width(2) == cfg.block_size # bidirectional: backbone stays full-width
+def test_lfm2_full_vocab_explicit_size_needs_no_mapping(tmp_path):
+ config = {**LFM2_DSPARK, "draft_vocab_size": LFM2_DSPARK["vocab_size"]}
+ weights = _reference_weights(config)
+ path = _write_drafter_ckpt(tmp_path, weights, config)
+ drafter, cfg = load_drafter(path, quantize=False)
+ assert cfg.draft_vocab_size is None
+ assert cfg.rope_traditional is True
+ assert drafter.to_target_ids(mx.array([0, 7, 31])).tolist() == [0, 7, 31]
+
+
def test_plain_qwen3_keeps_neox_rope_without_the_flag(tmp_path):
"""A config without rope_is_neox_style keeps the family default (neox / traditional=False),
so every existing head is untouched by the new interleaved-rope knob."""
@@ -660,6 +679,27 @@ def test_load_drafter_matching_checkpoint_loads(tmp_path):
assert cfg.family == "qwen3" and drafter.block_size == 4
+def test_specforge_nested_head_flags_strict_checkpoint(tmp_path):
+ # The trained Taichu export keeps these head flags only under dflash_config.
+ # Its checkpoint has Markov weights and no confidence tensors.
+ nested = {**RADIXARK_SPECFORGE["dflash_config"],
+ "markov_rank": 3, "enable_confidence_head": False,
+ "sample_from_anchor": False}
+ config = {k: v for k, v in RADIXARK_SPECFORGE.items()
+ if k not in ("markov_rank", "markov_head_type", "enable_confidence_head",
+ "confidence_head_with_markov", "sample_from_anchor")}
+ config["dflash_config"] = nested
+ weights = _reference_weights({**config, "markov_rank": 3,
+ "enable_confidence_head": False})
+ path = _write_drafter_ckpt(tmp_path, weights, cfg=config)
+ drafter, parsed = load_drafter(path, quantize=False)
+ assert parsed.markov_rank == 3 and parsed.enable_confidence_head is False
+ assert parsed.logits_start == 1 and drafter.confidence_head is None
+ assert parsed.confidence_head_with_markov is True
+ assert DSparkConfig.from_json(_cfg_path(tmp_path, {
+ **config, "enable_confidence_head": True})).enable_confidence_head is True
+
+
def test_load_drafter_mismatch_raises_by_default(tmp_path):
weights = _reference_weights()
weights["bogus.weight"] = weights.pop("lm_head.weight")
diff --git a/tests/test_model.py b/tests/test_model.py
index 63b1dab..f7d4811 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -140,6 +140,18 @@ def test_draft_width_full_for_bidirectional_truncated_for_causal():
assert [d.draft_width(c) for c in (1, 4, 99)] == [2, 5, 8]
+def test_mixed_sliding_full_layers_keep_distinct_masks():
+ d = _tiny_drafter(layer_types=("sliding_attention", "full_attention"),
+ sliding_window=4, causal_block=True)
+ sliding, full = (layer.self_attn for layer in d.layers)
+ assert sliding.causal_block and sliding.sliding_window == 4
+ assert not full.causal_block and full.sliding_window is None
+ assert sliding._block_mask(6, ctx_len=6, q_len=2).tolist() == [[[[False, False, False,
+ True, True, True, True, False], [False, False, False, False, True, True, True, True]]]]
+ assert full._block_mask(6, ctx_len=6, q_len=2) is None
+ assert d.draft_width(2) == 8 # the final bidirectional layer reads the whole block
+
+
def test_causal_backbone_truncation_matches_full_width_rows():
"""The load-bearing invariant behind draft_width: with a causal block mask, position i
attends only positions <= i, so running the backbone at a truncated width must reproduce
@@ -165,3 +177,13 @@ def test_causal_backbone_truncation_matches_full_width_rows():
full = d.backbone(noise, 10, ctx)
trunc = d.backbone(noise[:, :3, :], 10, ctx)
assert not mx.allclose(full[:, :3, :], trunc, atol=1e-5).item()
+
+
+def test_full_attention_keeps_explicit_causal_block():
+ d = _tiny_drafter(layer_types=('full_attention', 'full_attention'), causal_block=True)
+ assert d.causal_block
+ assert d.draft_width(2) == 2
+ for layer in d.layers:
+ mask = layer.self_attn._block_mask(4, ctx_len=4, q_len=2)
+ assert mask[0, 0, 0, -1].item() is False
+ assert mask[0, 0, 1, -1].item() is True
|