"""Phase 1 spec test for P3b: S_kv tile sweep in the GQA kernels (ADR-0063 §A.2 + ADR-0060 §B "long/short context split"). The local one-shot partial in three kernels — ``_gqa_decode_long.py``, ``_gqa_decode_short.py``, ``_gqa_prefill_short.py`` — loads its rank's entire ``(d_head, S_local)`` / ``(S_local, d_head)`` KV slice in one shot, so per-rank scratch grows linearly with ``S_local``. The ``_attention_*`` baselines hit a TCM ceiling once ``S_local`` exceeds the 1 MiB pool divided by the per-rank intermediate footprint. P3b replaces the one-shot partial with a tile sweep: - Module-level ``TILE_S_KV = 1024`` per kernel file. - Tile 0 establishes the persistent ``(m_local, l_local, O_local)``. - Tiles 1..n_tiles-1 wrap their intermediates (K_T tile, V tile, scores, centered, exp_scores, partials) in ``tl.scratch_scope()`` and persist the merged ``(m, ℓ, O)`` to the persistent handles via ``tl.copy_to`` (ADR-0063 §D3 / §D3.1). When ``S_local ≤ TILE_S_KV`` the loop body never runs and the op_log is structurally identical to today's one-shot path — every existing validation-scale test continues to pass. ``_gqa_prefill_long.py`` is **out of scope** for P3b. Its inner step is IPCQ partial-recv (Ring KV rotation), not an HBM load; tiling it intersects the ring-step structure and is a separate phase. Phase 1 (this commit): tests only — production code lands in Phase 2. The four multi-tile tests fail today because the kernels never call ``copy_to`` outside the chain-reduce merges (so isolating to a chain-free config gives ``copy_to == 0`` today). The 128K test fails today with a ``TLContext`` scratch overflow. """ from __future__ import annotations from pathlib import Path from kernbench.benches._gqa_decode_long import gqa_decode_long_kernel # noqa: F401 from kernbench.benches._gqa_decode_short import gqa_decode_short_kernel # noqa: F401 from kernbench.benches._gqa_prefill_short import gqa_prefill_short_kernel # noqa: F401 from kernbench.ccl.install import load_ccl_config, resolve_algorithm_config from kernbench.ccl.sfr_config import configure_sfr_intercube_multisip from kernbench.policy.placement.dp import DPPolicy from kernbench.runtime_api.bench_runner import run_bench from kernbench.runtime_api.types import resolve_device from kernbench.sim_engine.engine import GraphEngine from kernbench.topology.builder import resolve_topology TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml" D_HEAD = 64 DTYPE = "f16" def _ccl_cfg(): return resolve_algorithm_config( load_ccl_config(), name="lrab_hierarchical_allreduce", ) def _engine_factory(t, d): return GraphEngine(getattr(t, "topology_obj", t), enable_data=True) def _count(op_log, name: str) -> int: return sum(1 for r in op_log if r.op_name == name) # ── decode_long runner (chain-free configs isolate tile-sweep behavior) ── def _run_decode_long(*, C: int, P: int, S_kv: int, h_q: int = 1, h_kv: int = 1): """Run the long-context decode kernel. For tile-sweep isolation, callers pass C=1, P=1 — this leaves both chain-reduce levels inactive so any ``copy_to`` in op_log comes solely from the tile-merge body. """ topo = resolve_topology(str(TOPOLOGY_DEFAULT)) def _bench_fn(ctx): configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) dp_full = DPPolicy(cube="replicate", pe="replicate", num_cubes=C, num_pes=P) dp_kv = DPPolicy(cube="row_wise" if C > 1 else "replicate", pe="row_wise" if P > 1 else "replicate", num_cubes=C, num_pes=P) q = ctx.zeros((1, h_q * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"q_tl_c{C}_p{P}_s{S_kv}") k = ctx.zeros((S_kv, h_kv * D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"k_tl_c{C}_p{P}_s{S_kv}") v = ctx.zeros((S_kv, h_kv * D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"v_tl_c{C}_p{P}_s{S_kv}") o = ctx.empty((1, h_q * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"o_tl_c{C}_p{P}_s{S_kv}") ctx.launch( f"gqa_decode_long_tile_c{C}_p{P}_s{S_kv}", gqa_decode_long_kernel, q, k, v, o, 1, S_kv, h_q, h_kv, D_HEAD, C, P, _auto_dim_remap=False, ) return run_bench( topology=topo, bench_fn=_bench_fn, device=resolve_device(None), engine_factory=_engine_factory, ) # ── decode_short / prefill_short runners ───────────────────────────── def _run_decode_short(*, kv_per_cube: int, C: int, P: int, S_kv: int, h_q: int = 8, h_kv: int = 8): """For tile-sweep isolation: kv_per_cube=P → group_size=1 (no chain).""" topo = resolve_topology(str(TOPOLOGY_DEFAULT)) def _bench_fn(ctx): configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) dp_full = DPPolicy(cube="replicate", pe="replicate", num_cubes=C, num_pes=P) dp_kv = DPPolicy(cube="row_wise" if C > 1 else "replicate", pe="row_wise", num_cubes=C, num_pes=P) q = ctx.zeros((1, h_q * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"q_dsh_s{S_kv}") k = ctx.zeros((h_kv * S_kv, D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"k_dsh_s{S_kv}") v = ctx.zeros((h_kv * S_kv, D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"v_dsh_s{S_kv}") o = ctx.empty((1, h_q * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"o_dsh_s{S_kv}") ctx.launch( f"gqa_decode_short_tile_s{S_kv}", gqa_decode_short_kernel, q, k, v, o, 1, S_kv, h_q, h_kv, D_HEAD, C, P, kv_per_cube, _auto_dim_remap=False, ) return run_bench( topology=topo, bench_fn=_bench_fn, device=resolve_device(None), engine_factory=_engine_factory, ) def _run_prefill_short(*, kv_per_cube: int, C: int, P: int, T_q: int, S_kv: int, h_kv: int = 8): topo = resolve_topology(str(TOPOLOGY_DEFAULT)) def _bench_fn(ctx): configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) dp_full = DPPolicy(cube="replicate", pe="replicate", num_cubes=C, num_pes=P) dp_kv = DPPolicy(cube="row_wise" if C > 1 else "replicate", pe="row_wise", num_cubes=C, num_pes=P) q = ctx.zeros((T_q, h_kv * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"q_psh_s{S_kv}") k = ctx.zeros((h_kv * S_kv, D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"k_psh_s{S_kv}") v = ctx.zeros((h_kv * S_kv, D_HEAD), dtype=DTYPE, dp=dp_kv, name=f"v_psh_s{S_kv}") o = ctx.empty((T_q, h_kv * D_HEAD), dtype=DTYPE, dp=dp_full, name=f"o_psh_s{S_kv}") ctx.launch( f"gqa_prefill_short_tile_s{S_kv}", gqa_prefill_short_kernel, q, k, v, o, T_q, S_kv, h_kv, D_HEAD, C, P, kv_per_cube, _auto_dim_remap=False, ) return run_bench( topology=topo, bench_fn=_bench_fn, device=resolve_device(None), engine_factory=_engine_factory, ) # ── T1: single-tile path is op_log-stable (regression guard) ───────── def test_decode_long_single_tile_path_unchanged(): """ADR-0063 §A.2: when S_local <= TILE_S_KV the tile-sweep loop body never runs and op_log is structurally identical to today's one-shot path. Passes today (one-shot path) AND after Phase 2 (n_tiles=1 skips the merge loop). The witness: with chain reduce inactive (C=1, P=1), there must be zero ``copy_to`` entries — neither today nor after Phase 2 should the kernel emit tile-merge writebacks here. """ result = _run_decode_long(C=1, P=1, S_kv=64) assert result.completion.ok, ( f"single-tile decode_long must complete; got {result.completion}" ) n_copy = _count(result.engine.op_log, "copy") assert n_copy == 0, ( f"S_local <= TILE_S_KV must not emit tile-merge copy_to; got {n_copy}" ) # ── T2: decode_long multi-tile emits tile-merge copy_to ────────────── def test_decode_long_multi_tile_emits_copy_to_merges(): """ADR-0063 §A.2 + §D3.1: when S_local > TILE_S_KV the kernel must wrap each subsequent tile's intermediates in ``scratch_scope`` and persist the merged ``(m, ℓ, O)`` via ``tl.copy_to``. Chain-free config (C=1, P=1) isolates the tile-merge body — any ``copy_to`` in op_log comes from the tile sweep, not chain reduce. S_kv=2048, C=1, P=1 → S_local=2048 → 2 tiles → 1 merge × 3 handles (m, ℓ, O) ⇒ 3 ``copy`` entries. Currently 0 because the kernel performs a one-shot partial. """ result = _run_decode_long(C=1, P=1, S_kv=2048) assert result.completion.ok, ( f"multi-tile decode_long must complete; got {result.completion}" ) n_copy = _count(result.engine.op_log, "copy") assert n_copy >= 3, ( f"decode_long multi-tile must emit >= 3 copy_to entries " f"(1 merge × 3 handles); got {n_copy}" ) # ── T3: decode_short multi-tile emits tile-merge copy_to ───────────── def test_decode_short_multi_tile_emits_copy_to_merges(): """Same property as T2 for the short decode kernel. kv_per_cube=8, P=8, C=1 → group_size=1 (no chain reduce); S_kv=2048 → S_local=2048 → 2 tiles per PE → 3 ``copy`` entries per PE × 8 PEs = 24 total. Currently 0 because the kernel performs a one-shot partial. """ result = _run_decode_short(kv_per_cube=8, C=1, P=8, S_kv=2048) assert result.completion.ok, ( f"multi-tile decode_short must complete; got {result.completion}" ) n_copy = _count(result.engine.op_log, "copy") assert n_copy >= 3 * 8, ( f"decode_short multi-tile must emit >= 24 copy_to entries " f"(1 merge × 3 handles × 8 PEs); got {n_copy}" ) # ── T4: prefill_short multi-tile emits tile-merge copy_to ──────────── def test_prefill_short_multi_tile_emits_copy_to_merges(): """Same property as T3 for the short prefill kernel. kv_per_cube=8, P=8, C=1, T_q=4, S_kv=2048 → group_size=1 (no chain), S_local=2048 → 2 tiles per PE → 3 ``copy`` entries per PE × 8 PEs = 24 total. Currently 0 because the kernel performs a one-shot partial. """ result = _run_prefill_short( kv_per_cube=8, C=1, P=8, T_q=4, S_kv=2048, ) assert result.completion.ok, ( f"multi-tile prefill_short must complete; got {result.completion}" ) n_copy = _count(result.engine.op_log, "copy") assert n_copy >= 3 * 8, ( f"prefill_short multi-tile must emit >= 24 copy_to entries " f"(1 merge × 3 handles × 8 PEs); got {n_copy}" ) # ── T5: decode_long at S_kv=256K completes (TCM ceiling lifted) ────── def test_decode_long_context_256k_completes(): """ADR-0063 §A.2 (test req 3): headline ceiling-lift. C=1 P=8 decode at S_kv=256K → S_local=32K. Today the per-rank score stack (3 ×M·S_local·2 bytes, M=G·T_q=8) is ~1.5 MB → exceeds the 1 MiB scratch pool → TLContext scratch overflow. After Phase 2 the tile sweep bounds per-tile score stack at 3·M·TILE_S_KV·2 ≈ 48 KB and the run completes regardless of S_kv. """ result = _run_decode_long(C=1, P=8, S_kv=262144, h_q=8, h_kv=1) assert result.completion.ok, ( f"decode_long at S_kv=256K must complete after tile sweep; " f"got {result.completion}" )