File size: 28,983 Bytes
8795749
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
diff --git a/src/mlx_dspark/config.py b/src/mlx_dspark/config.py
index f3ea4cb..4bddbf2 100644
--- a/src/mlx_dspark/config.py
+++ b/src/mlx_dspark/config.py
@@ -109,6 +109,7 @@ def _translate_speculators(c: dict, path) -> dict:
     lt = tlc.get("layer_types") or []
     if "sliding_attention" in lt and tlc.get("use_sliding_window") and tlc.get("sliding_window"):
         out["sliding_window"] = int(tlc["sliding_window"])
+        out["layer_types"] = list(lt)
         # sliding_window_non_causal=False => the window is causal (DFlash query-causal). Absent
         # defaults to causal, the DFlash norm for a sliding backbone.
         out["dflash_query_causal"] = not bool(c.get("sliding_window_non_causal", False))
@@ -209,6 +210,7 @@ class DSparkConfig:
     # heads (bidirectional full-attention block), so they default off and change nothing there.
     causal_block: bool = False
     sliding_window: int | None = None
+    layer_types: tuple[str, ...] = ()
     attention_sink: bool = False
     # has_lm_head=false drafters ship only embed_tokens and reuse the TARGET's lm_head (like
     # DFlash's bind); generate binds it once at run time. True = the drafter has its own head.
@@ -306,7 +308,7 @@ class DSparkConfig:
         # `dflash_config` dict tagged `projector_type: "dspark"` β€” the discriminator (absent
         # from every other packaging; the fork-labelled heads carry a dflash_config too, but
         # never that tag). Three things follow from the reference code these repos ship:
-        # (1) target_layer_ids / mask_token_id live only inside dflash_config β€” hoist them;
+        # (1) target taps and head options can live only inside dflash_config β€” hoist them;
         # (2) the DSpark flavor samples the ANCHOR slot (anchor-as-pos0, the DeepSpec
         #     convention its markov/confidence code is adapted from): a block_size-7 head
         #     proposes 7 tokens, and the card's own "verify width 8 including the bonus" is
@@ -324,10 +326,12 @@ class DSparkConfig:
         _sf = (c.get("dflash_config") or {})
         specforge = _sf.get("projector_type") == "dspark"
         if specforge:
-            for k in ("target_layer_ids", "mask_token_id"):
+            for k in ("target_layer_ids", "mask_token_id", "markov_rank", "markov_head_type",
+                      "enable_confidence_head", "confidence_head_with_markov",
+                      "sample_from_anchor", "sliding_window_non_causal"):
                 if k not in c and _sf.get(k) is not None:
                     c[k] = _sf[k]
-            c.setdefault("logits_start", 0)
+            c.setdefault("logits_start", 0 if c.get("sample_from_anchor", True) else 1)
 
         # LiquidAI LFM2.5-DSpark packaging (the fifth). A plain qwen3-backbone DSpark head
         # (architectures ["Lfm2DSparkDraftModel"]) for an LFM2 conv+attention target. Like
@@ -375,6 +379,16 @@ class DSparkConfig:
                 f"does not look like a DeepSpec-format DSpark drafter checkpoint."
             )
 
+        layer_types = tuple(c.get("layer_types") or ())
+        if layer_types and (len(layer_types) != c["num_hidden_layers"] or
+                            set(layer_types) - {"sliding_attention", "full_attention"}):
+            raise ValueError(f"{path}: layer_types must name each draft layer as sliding or full")
+
+        # An explicitly declared full vocabulary uses identity token ids, not a d2t table.
+        draft_vocab_size = c.get("draft_vocab_size")
+        if draft_vocab_size == c["vocab_size"]:
+            draft_vocab_size = None
+
         if family == "qwen3":
             rp = c.get("rope_parameters") or {}
             head_dim = c.get("head_dim", c["hidden_size"] // c["num_attention_heads"])
@@ -410,13 +424,17 @@ class DSparkConfig:
             # the DFlash spelling of logits_start (false -> slot 0 is a pure anchor, predict
             # from slot 1), the same convention the speculators path derives.
             dflc = c.get("dflash_config") or {}
-            causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False))
+            causal_block = bool(c.get("dflash_query_causal", False) or dflc.get("causal", False)
+                                or ("sliding_attention" in layer_types and not dflc.get(
+                                    "sliding_window_non_causal", c.get("sliding_window_non_causal", False))))
             attn_sink = bool(c.get("attention_sink_bias", False))
             swa = None
             if causal_block or attn_sink or dflc:
                 w = c.get("sliding_window") or (
                     dflc.get("swa_window_size") if dflc.get("use_swa", False) else None)
                 swa = int(w) if w else None
+            if "sliding_attention" in layer_types and not swa:
+                raise ValueError(f"{path}: sliding_attention requires a sliding_window")
             has_own_lm = bool(c.get("has_lm_head", True))
             # YaRN rope β€” honored ONLY for the SpecForge packaging, where the reference builds
             # transformers' Qwen3RotaryEmbedding from the head's own config and that class
@@ -456,7 +474,7 @@ class DSparkConfig:
             return cls(
                 family="qwen3",
                 hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
-                draft_vocab_size=c.get("draft_vocab_size"),
+                draft_vocab_size=draft_vocab_size,
                 num_hidden_layers=c["num_hidden_layers"],
                 intermediate_size=c["intermediate_size"],
                 rms_norm_eps=c.get("rms_norm_eps", 1e-6),
@@ -471,7 +489,7 @@ class DSparkConfig:
                 rope_traditional=rope_trad,
                 gated_q_proj=gated_q,
                 offset_rms_norm=offset_norms,
-                causal_block=causal_block, sliding_window=swa,
+                causal_block=causal_block, sliding_window=swa, layer_types=layer_types,
                 attention_sink=attn_sink, has_own_lm_head=has_own_lm,
                 block_size=c["block_size"], mask_token_id=c["mask_token_id"],
                 logits_start=logits_start,
@@ -493,7 +511,7 @@ class DSparkConfig:
         return cls(
             family="gemma4",
             hidden_size=c["hidden_size"], vocab_size=c["vocab_size"],
-            draft_vocab_size=c.get("draft_vocab_size"),
+            draft_vocab_size=draft_vocab_size,
             num_hidden_layers=c["num_hidden_layers"],
             intermediate_size=c["intermediate_size"],
             rms_norm_eps=c.get("rms_norm_eps", 1e-6),
diff --git a/src/mlx_dspark/dflash_model.py b/src/mlx_dspark/dflash_model.py
index a45ea26..b99627a 100644
--- a/src/mlx_dspark/dflash_model.py
+++ b/src/mlx_dspark/dflash_model.py
@@ -87,6 +87,7 @@ class DFlashConfig:
     num_target_layers: int
     mask_token_id: int = 0
     rope_scaling: dict[str, Any] | None = None
+    rope_traditional: bool = False
     layer_types: tuple[str, ...] = field(default_factory=tuple)
     sliding_window: int | None = None
     final_logit_softcapping: float | None = None
@@ -100,11 +101,11 @@ class DFlashConfig:
     # reuse-head rule (raw lm_head only) the multiplier/softcap must be applied here.
 
 
-def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling):
+def _build_rope(head_dim, rope_theta, max_position_embeddings, rope_scaling, traditional):
     return initialize_rope(
         dims=head_dim,
         base=rope_theta,
-        traditional=False,
+        traditional=traditional,
         scaling_config=rope_scaling,
         max_position_embeddings=max_position_embeddings,
     )
@@ -320,7 +321,8 @@ class DFlashDraftModel(nn.Module):
         self.layers = [DFlashDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
         self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
         self.rope = _build_rope(
-            config.head_dim, config.rope_theta, config.max_position_embeddings, config.rope_scaling
+            config.head_dim, config.rope_theta, config.max_position_embeddings,
+            config.rope_scaling, config.rope_traditional,
         )
         self.candidate_selector = (
             CandidateSelector(config.hidden_size, config.vocab_size,
diff --git a/src/mlx_dspark/generate.py b/src/mlx_dspark/generate.py
index c0aae6a..acd6fdd 100644
--- a/src/mlx_dspark/generate.py
+++ b/src/mlx_dspark/generate.py
@@ -769,6 +769,18 @@ def dflash_generate(
 
     st.update(out_ids)
     while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+        remaining = max_new_tokens - len(out_ids)
+        if remaining == 1:
+            v_logits, _ = target_model.verify(mx.array([[pending]]), cache, tap)
+            nxt = _pick(v_logits[0, -1], temperature, top_p, top_k)
+            target_forwards += 1
+            accept_lengths.append(1)
+            if on_round is not None:
+                on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+            out_ids.append(nxt)
+            pending = nxt
+            st.update(out_ids)
+            break
         # ---- free copy draft (a 4-gram+ suffix match extends an earlier occurrence): verify
         # that continuation instead of running the DFlash drafter this round. DFlash caches
         # ctx KV only (advanced inside the draft call from pending_ctx), so a copy round β€”
@@ -778,7 +790,7 @@ def dflash_generate(
         # the target verifies every copied token; a bad copy just costs acceptance.
         lk_draft = index.propose(
             long_draft=lookup_long_draft if lk_gate.allowed else None,
-        )[: max_new_tokens - len(out_ids)] if index is not None else []
+        )[: remaining - 1] if index is not None else []
         if lk_draft:
             lookup_rounds += 1
             if temperature > 0.0:
@@ -804,6 +816,9 @@ def dflash_generate(
                 n = int(n_arr.item())
                 tt = tt_arr.tolist()
                 committed = lk_draft[:n] + [tt[n]]
+            if any(tok in eos_ids for tok in committed):
+                committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+                n = min(n, len(committed) - 1)
             lk_gate.update(len(lk_draft), n, max(1, lookup_max_draft))
             target_forwards += 1
             accept_lengths.append(len(committed))
@@ -831,6 +846,7 @@ def dflash_generate(
             # any depth and ratchets up β€” measured 1.04x vs cap 3's 1.48x at 32k)
             cap_controller.set_depth(len(ids) + len(out_ids))
             cap = max(1, min(cap_controller.cap, cap_ceiling))
+        cap = min(cap, remaining - 1)
         block = mx.array([[pending] + [mask_id] * (bs - 1)])
         sel = getattr(drafter, "candidate_selector", None)   # DFlash 2 path when present
         if temperature > 0.0:
@@ -882,6 +898,9 @@ def dflash_generate(
             drafted = draft_arr.tolist()
             tt = tt_arr.tolist()
             committed = drafted[:n] + [tt[n]]            # accepted prefix + bonus
+        if any(tok in eos_ids for tok in committed):
+            committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+            n = min(n, len(committed) - 1)
         target_forwards += 1
         accept_lengths.append(len(committed))
         if cap_controller is not None:
@@ -1223,12 +1242,30 @@ def speculative_generate(
 
     st.update(out_ids)
     while len(out_ids) < max_new_tokens and pending not in eos_ids and not st.stopped:
+        remaining = max_new_tokens - len(out_ids)
         lk_draft = []
         if index is not None:
-            # clamp to the remaining budget: no verify width wasted past max_new_tokens
+            # Reserve one position for the verifier's residual/bonus token.
             lk_draft = index.propose(
                 long_draft=lookup_long_draft if lk_gate.allowed else None,
-            )[:max_new_tokens - len(out_ids)]
+            )[:remaining - 1]
+        if remaining == 1:
+            # Consume the pending token through the tapped target path; its next token is
+            # the sole remaining output, with the target cache still one pending behind.
+            v_logits, v_fused = target_model.verify(mx.array([[pending]]), cache, tap)
+            nxt = _pick(pen.apply(v_logits[0], [])[0], temperature, top_p, top_k)
+            drafter.update_context(v_fused, ctx_offset=n_cached, ctx_caches=ctx_caches)
+            target_forwards += 1
+            accept_lengths.append(1)
+            if on_round is not None:
+                on_round(drafted=0, accepted=0, committed=1, cap=0, source="plain")
+            out_ids.append(nxt)
+            pen.add([nxt])
+            if lp_list is not None:
+                lp_list.extend(_logprobs_for_block(v_logits[0][:1], [nxt], logprobs))
+            pending = nxt
+            st.update(out_ids)
+            break
         use_conf = confidence_threshold > 0.0 and drafter.confidence_head is not None
         if cap_controller is not None and not lk_draft:
             # live context depth -> the controller's verify pricing (see dflash_generate)
@@ -1236,6 +1273,7 @@ def speculative_generate(
             # cap 0 = the controller parked speculation (live acceptance too low for this
             # machine's verify slope): run a plain committed step; probe rounds re-enter.
             cap = max(0, min(cap_controller.cap, cap_ceiling))
+        cap = min(cap, remaining - 1)
         if not lk_draft and cap == 0 and not (temperature > 0.0 or pen.active
                                               or lp_list is not None):
             # ---- parked sprint ----
@@ -1406,6 +1444,9 @@ def speculative_generate(
                 draft = draft_arr.tolist()
                 tt = tt_arr.tolist()
                 committed = draft[:n] + [tt[n]]                # accepted prefix + bonus
+        if any(tok in eos_ids for tok in committed):
+            committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+            n = min(n, len(committed) - 1)
         target_forwards += 1
         accept_lengths.append(len(committed))
         if lk_draft:
diff --git a/src/mlx_dspark/load.py b/src/mlx_dspark/load.py
index eded611..376a05e 100644
--- a/src/mlx_dspark/load.py
+++ b/src/mlx_dspark/load.py
@@ -607,6 +607,7 @@ def load_dflash(repo_or_path: str, *, quantize: bool = True, bits: int = 4, grou
         num_target_layers=cfg["num_target_layers"],
         mask_token_id=dfc.get("mask_token_id", cfg.get("mask_token_id", 0)),
         rope_scaling=cfg.get("rope_scaling"), layer_types=layer_types,
+        rope_traditional=not bool(cfg["rope_is_neox_style"]) if "rope_is_neox_style" in cfg else False,
         sliding_window=cfg.get("sliding_window"),
         # DFlash 2 nests these in dflash_config (muse); DFlash 1 gemma had softcap top-level
         final_logit_softcapping=dfc.get("final_logit_softcapping",
diff --git a/src/mlx_dspark/lookup.py b/src/mlx_dspark/lookup.py
index f2b9ede..fb63573 100644
--- a/src/mlx_dspark/lookup.py
+++ b/src/mlx_dspark/lookup.py
@@ -207,7 +207,7 @@ def lookup_generate(
         # clamp to the remaining budget: no verify width wasted past max_new_tokens
         draft = index.propose(
             long_draft=long_draft_tokens if gate.allowed else None,
-        )[:max_new_tokens - len(out_ids)]
+        )[:max_new_tokens - len(out_ids) - 1]
 
         if not draft:
             # no confident match -> plain 1-token step (zero miss cost; verify() with no
@@ -237,6 +237,9 @@ def lookup_generate(
             n = int(n_arr.item())
             tt = tt_arr.tolist()
             committed = draft[:n] + [tt[n]]
+        if any(tok in eos_ids for tok in committed):
+            committed = committed[:next(i for i, tok in enumerate(committed) if tok in eos_ids) + 1]
+            n = min(n, len(committed) - 1)
         target_forwards += 1
         accept_lengths.append(len(committed))
         if draft:
diff --git a/src/mlx_dspark/model.py b/src/mlx_dspark/model.py
index d293661..dcdb9ef 100644
--- a/src/mlx_dspark/model.py
+++ b/src/mlx_dspark/model.py
@@ -86,7 +86,7 @@ class CtxCache:
 class DSparkAttention(nn.Module):
     """Cross-attention: Q from the draft block, K/V from [target_context, block]."""
 
-    def __init__(self, config: DSparkConfig):
+    def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
         super().__init__()
         self.n_heads = config.num_attention_heads
         self.head_dim = config.attn_head_dim
@@ -100,6 +100,10 @@ class DSparkAttention(nn.Module):
         # (getattr defaults keep minimal config-like doubles working).
         self.causal_block = getattr(config, "causal_block", False)
         self.sliding_window = getattr(config, "sliding_window", None)
+        if layer_idx is not None and "sliding_attention" in getattr(config, "layer_types", ()):
+            if config.layer_types[layer_idx] == "full_attention":
+                self.causal_block = False
+                self.sliding_window = None
         self.sink = getattr(config, "attention_sink", False)
 
         h = config.hidden_size
@@ -164,10 +168,12 @@ class DSparkAttention(nn.Module):
         # DFlash-lineage heads mask the block attention (causal within the block, sliding window
         # over the context) and add a per-head sink logit. The default DSpark head does neither:
         # mask stays None (full bidirectional block over the whole context) and sinks stays None,
-        # so this is byte-identical there. The mask is only built for the single-sequence path
-        # (int offset); the batched drafter path already supplies its own [B,1,k,L] mask.
-        if mask is None and (self.causal_block or self.sliding_window) and isinstance(block_offset, int):
-            mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+        # so this is byte-identical there. In a batch, combine the layer's attention
+        # constraint with the caller's context-padding mask.
+        if self.causal_block or self.sliding_window:
+            local_mask = self._block_mask(block_offset, ctx_len=k.shape[2] - q_len, q_len=q_len)
+            if local_mask is not None:
+                mask = local_mask if mask is None else mask & local_mask
         sinks = self.attention_sink_bias if self.sink else None
 
         # SDPA does GQA/MQA head broadcast internally β€” pass the n_kv-head K/V straight through.
@@ -197,22 +203,28 @@ class DSparkAttention(nn.Module):
         W = self.sliding_window
         if not self.causal_block and (W is None or total <= W):
             return None
-        qpos = block_offset + mx.arange(q_len)[:, None]        # [q_len, 1]
-        kpos = mx.arange(total)[None, :]                       # [1, total]
+        if isinstance(block_offset, int):
+            qpos = block_offset + mx.arange(q_len)[:, None]
+            kpos = mx.arange(total)[None, :]
+        else:
+            lengths = block_offset[:, None, None, None]
+            qpos = lengths + mx.arange(q_len)[None, None, :, None]
+            keys = mx.arange(total)[None, None, None, :]
+            kpos = mx.where(keys < ctx_len, keys, lengths + keys - ctx_len)
         allowed = mx.ones((q_len, total), dtype=mx.bool_)
         if self.causal_block:
             allowed = allowed & (kpos <= qpos)
         if W is not None:
             allowed = allowed & (kpos > qpos - W)
-        return allowed[None, None]
+        return allowed[None, None] if isinstance(block_offset, int) else allowed
 
 
 class DSparkDecoderLayer(nn.Module):
-    def __init__(self, config: DSparkConfig):
+    def __init__(self, config: DSparkConfig, layer_idx: int | None = None):
         super().__init__()
         eps = config.rms_norm_eps
         self.norm_style = config.norm_style
-        self.self_attn = DSparkAttention(config)
+        self.self_attn = DSparkAttention(config, layer_idx)
         self.mlp = MLP(config)
         self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
         self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=eps)
@@ -314,7 +326,9 @@ class DSparkDrafter(nn.Module):
         self.block_size = config.block_size
         self.mask_token_id = config.mask_token_id
         self.logits_start = config.logits_start
-        self.causal_block = getattr(config, "causal_block", False)
+        layer_types = getattr(config, "layer_types", ())
+        mixed_attention = "sliding_attention" in layer_types and "full_attention" in layer_types
+        self.causal_block = getattr(config, "causal_block", False) and not mixed_attention
         self.embed_scale = (float(config.hidden_size) ** 0.5) if config.family == "gemma4" else 1.0
         self.softcap = config.final_logit_softcapping
 
@@ -328,7 +342,7 @@ class DSparkDrafter(nn.Module):
             len(config.target_layer_ids) * config.hidden_size, config.hidden_size, bias=False
         )
         self.hidden_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.layers = [DSparkDecoderLayer(config) for _ in range(config.num_hidden_layers)]
+        self.layers = [DSparkDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
         self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
         # A has_lm_head=false drafter (Nemotron DSpark head) reuses the TARGET's lm_head β€” it
         # ships no lm_head weight, so we don't build one; bind_lm_head() supplies it at run time.
@@ -381,15 +395,6 @@ class DSparkDrafter(nn.Module):
 
     def backbone(self, noise_embedding, block_offset, ctx_caches, mask=None) -> mx.array:
         h = noise_embedding
-        if mask is None and isinstance(block_offset, int) and self.layers:
-            # Every layer shares one attention config, so the block mask (a function of
-            # block_offset / ctx_len / q_len only) is identical across them β€” build it once
-            # here instead of 5x per round. attend() keeps its own fallback for direct
-            # callers; None stays None for the default bidirectional DSpark head.
-            attn = self.layers[0].self_attn
-            if attn.causal_block or attn.sliding_window:
-                mask = attn._block_mask(
-                    block_offset, ctx_len=ctx_caches[0].length, q_len=h.shape[1])
         for layer, cache in zip(self.layers, ctx_caches):
             h = layer(h, block_offset, cache, mask)
         return self.norm(h)
diff --git a/tests/test_dflash2.py b/tests/test_dflash2.py
index 8b5b3d9..7d3bb9b 100644
--- a/tests/test_dflash2.py
+++ b/tests/test_dflash2.py
@@ -327,6 +327,18 @@ def test_load_dflash2_roundtrip_and_config_fields(tmp_path):
     assert (cfg.selector_rank, cfg.selector_top_k) == (8, 4)
     assert (cfg.conv_kernel_size, cfg.conv_group_size) == (2, 16)
     assert cfg.output_multiplier == 1.0 and cfg.final_logit_softcapping is None
+    assert cfg.rope_traditional is False
+
+
+def test_lfm_dflash_interleaved_rope_matches_reference(tmp_path):
+    cfg_dict = {**DFLASH2_MIN, "rope_is_neox_style": False}
+    drafter, cfg = load_dflash(_write_dflash2_ckpt(tmp_path, cfg_dict), quantize=False)
+    assert cfg.rope_traditional is True
+    x = mx.arange(16, dtype=mx.float32).reshape(1, 1, 1, 16) / 16
+    expected = nn.RoPE(16, traditional=True, base=1e7)(x, offset=2)
+    assert mx.allclose(drafter.rope(x, offset=2), expected, atol=1e-6).item()
+    neox = nn.RoPE(16, traditional=False, base=1e7)(x, offset=2)
+    assert not mx.allclose(drafter.rope(x, offset=2), neox).item()
 
 
 def test_load_dflash2_reads_muse_transform_fields_nested(tmp_path):
diff --git a/tests/test_formats.py b/tests/test_formats.py
index 33f206b..3fcbfb9 100644
--- a/tests/test_formats.py
+++ b/tests/test_formats.py
@@ -417,6 +417,15 @@ def test_speculators_dflash_lineage_causal_sliding_window(tmp_path):
     assert base.causal_block is False and base.sliding_window is None
 
 
+def test_taichu_mixed_attention_config(tmp_path):
+    config = {**QWEN3_MIN, "num_hidden_layers": 2,
+              "layer_types": ["sliding_attention", "full_attention"],
+              "use_sliding_window": True, "sliding_window": 4,
+              "dflash_config": {"causal": True}}
+    parsed = DSparkConfig.from_json(_cfg_path(tmp_path, config))
+    assert parsed.layer_types == ("sliding_attention", "full_attention")
+
+
 def test_specforge_dspark_config_parses(tmp_path):
     """SpecForge packaging: nested taps hoisted, anchor slot reserved, yarn rope carried."""
     cfg = DSparkConfig.from_json(_cfg_path(tmp_path, RADIXARK_SPECFORGE))
@@ -546,6 +555,16 @@ def test_lfm2_dspark_reuse_both_roundtrips(tmp_path):
     assert drafter.draft_width(2) == cfg.block_size  # bidirectional: backbone stays full-width
 
 
+def test_lfm2_full_vocab_explicit_size_needs_no_mapping(tmp_path):
+    config = {**LFM2_DSPARK, "draft_vocab_size": LFM2_DSPARK["vocab_size"]}
+    weights = _reference_weights(config)
+    path = _write_drafter_ckpt(tmp_path, weights, config)
+    drafter, cfg = load_drafter(path, quantize=False)
+    assert cfg.draft_vocab_size is None
+    assert cfg.rope_traditional is True
+    assert drafter.to_target_ids(mx.array([0, 7, 31])).tolist() == [0, 7, 31]
+
+
 def test_plain_qwen3_keeps_neox_rope_without_the_flag(tmp_path):
     """A config without rope_is_neox_style keeps the family default (neox / traditional=False),
     so every existing head is untouched by the new interleaved-rope knob."""
@@ -660,6 +679,27 @@ def test_load_drafter_matching_checkpoint_loads(tmp_path):
     assert cfg.family == "qwen3" and drafter.block_size == 4
 
 
+def test_specforge_nested_head_flags_strict_checkpoint(tmp_path):
+    # The trained Taichu export keeps these head flags only under dflash_config.
+    # Its checkpoint has Markov weights and no confidence tensors.
+    nested = {**RADIXARK_SPECFORGE["dflash_config"],
+              "markov_rank": 3, "enable_confidence_head": False,
+              "sample_from_anchor": False}
+    config = {k: v for k, v in RADIXARK_SPECFORGE.items()
+              if k not in ("markov_rank", "markov_head_type", "enable_confidence_head",
+                           "confidence_head_with_markov", "sample_from_anchor")}
+    config["dflash_config"] = nested
+    weights = _reference_weights({**config, "markov_rank": 3,
+                                  "enable_confidence_head": False})
+    path = _write_drafter_ckpt(tmp_path, weights, cfg=config)
+    drafter, parsed = load_drafter(path, quantize=False)
+    assert parsed.markov_rank == 3 and parsed.enable_confidence_head is False
+    assert parsed.logits_start == 1 and drafter.confidence_head is None
+    assert parsed.confidence_head_with_markov is True
+    assert DSparkConfig.from_json(_cfg_path(tmp_path, {
+        **config, "enable_confidence_head": True})).enable_confidence_head is True
+
+
 def test_load_drafter_mismatch_raises_by_default(tmp_path):
     weights = _reference_weights()
     weights["bogus.weight"] = weights.pop("lm_head.weight")
diff --git a/tests/test_model.py b/tests/test_model.py
index 63b1dab..f7d4811 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -140,6 +140,18 @@ def test_draft_width_full_for_bidirectional_truncated_for_causal():
     assert [d.draft_width(c) for c in (1, 4, 99)] == [2, 5, 8]
 
 
+def test_mixed_sliding_full_layers_keep_distinct_masks():
+    d = _tiny_drafter(layer_types=("sliding_attention", "full_attention"),
+                      sliding_window=4, causal_block=True)
+    sliding, full = (layer.self_attn for layer in d.layers)
+    assert sliding.causal_block and sliding.sliding_window == 4
+    assert not full.causal_block and full.sliding_window is None
+    assert sliding._block_mask(6, ctx_len=6, q_len=2).tolist() == [[[[False, False, False,
+        True, True, True, True, False], [False, False, False, False, True, True, True, True]]]]
+    assert full._block_mask(6, ctx_len=6, q_len=2) is None
+    assert d.draft_width(2) == 8  # the final bidirectional layer reads the whole block
+
+
 def test_causal_backbone_truncation_matches_full_width_rows():
     """The load-bearing invariant behind draft_width: with a causal block mask, position i
     attends only positions <= i, so running the backbone at a truncated width must reproduce
@@ -165,3 +177,13 @@ def test_causal_backbone_truncation_matches_full_width_rows():
     full = d.backbone(noise, 10, ctx)
     trunc = d.backbone(noise[:, :3, :], 10, ctx)
     assert not mx.allclose(full[:, :3, :], trunc, atol=1e-5).item()
+
+
+def test_full_attention_keeps_explicit_causal_block():
+    d = _tiny_drafter(layer_types=('full_attention', 'full_attention'), causal_block=True)
+    assert d.causal_block
+    assert d.draft_width(2) == 2
+    for layer in d.layers:
+        mask = layer.self_attn._block_mask(4, ctx_len=4, q_len=2)
+        assert mask[0, 0, 0, -1].item() is False
+        assert mask[0, 0, 1, -1].item() is True