From 5672c8f3ef1b4dce6545ae8e26d5fb04224eb111 Mon Sep 17 00:00:00 2001 From: Mukesh Garg Date: Mon, 15 Jun 2026 14:59:58 -0700 Subject: [PATCH] =?UTF-8?q?gqa(decode-4cases):=20Case=202=20=E2=80=94=20Cu?= =?UTF-8?q?be-Repl=20=C3=97=20PE-TP=20(no=20comm;=208=C3=97=20memory)=20(5?= =?UTF-8?q?C.B)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second case in the GQA decode 4-cases comparative study per GQA_full_deck.pptx slide 11. Case 2 replicates K, V across all 8 cubes × 8 PEs (the slide-11 8 KB/tok/PE memory waste) and has zero inter-rank comm. For B=1 (single-user decode default), only PE 0 of CUBE 0 has work — the inherent PE-TP waste slide 11 calls out. Changes: - New kernel: src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py Simplest of the 4 cases. Active rank loads full Q/K/V from HBM, computes attention via S_kv tile sweep with online-softmax merge, writes O. All non-(0,0) ranks early-return. No tl.send/recv. - src/kernbench/benches/milestone_gqa_decode_4cases.py: - Add panel single_kv_group_decode_gqa_cube_repl_pe_tp (Case 2) to _PANELS and _PANEL_DISPATCH. - Add _run_decode_panel_cube_repl_pe_tp helper: DPPolicy K/V/Q/O = cube=replicate, pe=replicate (models 8× memory waste). - Extend _make_bench_fn to dispatch kind="decode_cube_repl_pe_tp" to the new runner. - tests/attention/test_milestone_gqa_decode_4cases.py: 4 new tests assert Case 2 contract: panel registered, smoke completion, zero ipcq_copy (no comm), single dma_write from cube 0. Verification: 8/8 tests pass (4 Case 4 anchor + 4 new Case 2). Co-Authored-By: Claude Opus 4.7 --- .../_gqa_attention_decode_cube_repl_pe_tp.py | 101 +++++++++++++++++ .../benches/milestone_gqa_decode_4cases.py | 55 ++++++++- .../test_milestone_gqa_decode_4cases.py | 107 ++++++++++++++++++ 3 files changed, 259 insertions(+), 4 deletions(-) create mode 100644 src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py diff --git a/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py b/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py new file mode 100644 index 0000000..91da063 --- /dev/null +++ b/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py @@ -0,0 +1,101 @@ +"""GQA decode kernel — Case 2 (Cube-Repl × PE-TP). + +Per GQA_full_deck.pptx slide 11: + - K, V replicated across all 8 cubes × 8 PEs (the 8 KB/tok/PE + memory waste — this is the inherent cost Case 2 demonstrates). + - PEs nominally split on the batch dim (PE-TP). For B=1 (single- + user decode, the slide-11 default), only PE 0 of CUBE 0 has + work; the other 63 ranks idle. + - NO inter-rank communication (each rank has full KV — slide 11 + lists comm cost as "none"). + +This kernel is the simplest of the 4 cases by design: one active +rank does the full attention locally; everyone else early-returns. +The DPPolicy at the call site models the cluster-wide 8× memory +waste even though only one rank reads from HBM. + +Tensor layout (B=1): + Q : (T_q, h_q · d_head) replicated on every rank; loaded as + (G · T_q, d_head) on the active rank. + K : (S_kv, h_kv · d_head) replicated on every rank. + V : (S_kv, h_kv · d_head) replicated on every rank. + O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores. + +Topology / SFR: + - configure_sfr_intercube_multisip is fine but not strictly required + (no inter-rank sends/recvs happen). +""" +from __future__ import annotations + + +TILE_S_KV = 1024 # match decode_long — per-tile S_kv width (ADR-0063 §A.2). + + +def gqa_attention_decode_cube_repl_pe_tp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-2 decode: single-rank attention; full KV per rank; no comm.""" + pe_id = tl.program_id(axis=0) + cube_id = tl.program_id(axis=1) + # B=1 single-user decode + PE-TP: only one rank has work. + # Slide-11 acknowledges this PE-TP waste at B=1. + if pe_id != 0 or cube_id != 0: + return + + G = h_q // h_kv + n_tiles = (S_kv + TILE_S_KV - 1) // TILE_S_KV + KV_ROW_BYTES = d_head * 2 # f16 + + # ── Load Q (full; replicated; M-folded for GQA reuse) ── + Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16") + + # ── Tile 0: bootstrap persistent (m, ℓ, O) ── + tile_s0 = min(TILE_S_KV, S_kv) + K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16") + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + centered = scores - m_local + exp_scores = tl.exp(centered) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + # ── Tiles 1..N: fold via online-softmax merge in scratch_scope ── + for tile_idx in range(1, n_tiles): + tile_start = tile_idx * TILE_S_KV + tile_s = min(TILE_S_KV, S_kv - tile_start) + with tl.scratch_scope(): + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + scores_t = tl.dot(Q, K_T_t) + m_tile = tl.max(scores_t, axis=-1) + centered_t = scores_t - m_tile + exp_scores_t = tl.exp(centered_t) + l_tile = tl.sum(exp_scores_t, axis=-1) + O_tile = tl.dot(exp_scores_t, V_t) + m_new = tl.maximum(m_local, m_tile) + scale_old = tl.exp(m_local - m_new) + scale_new = tl.exp(m_tile - m_new) + l_new = l_local * scale_old + l_tile * scale_new + O_new = O_local * scale_old + O_tile * scale_new + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Final normalise + store ── + O_final = O_local / l_local + tl.store(o_ptr, O_final) diff --git a/src/kernbench/benches/milestone_gqa_decode_4cases.py b/src/kernbench/benches/milestone_gqa_decode_4cases.py index 002b152..267fcb7 100644 --- a/src/kernbench/benches/milestone_gqa_decode_4cases.py +++ b/src/kernbench/benches/milestone_gqa_decode_4cases.py @@ -32,12 +32,17 @@ import json import os from pathlib import Path +from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import ( + gqa_attention_decode_cube_repl_pe_tp_kernel, +) from kernbench.benches.milestone_gqa_headline import ( _ccl_cfg, _run_decode_panel, _summarize_op_log, ) from kernbench.benches.registry import bench +from kernbench.ccl.sfr_config import configure_sfr_intercube_multisip +from kernbench.policy.placement.dp import DPPolicy _OUTPUT_DIR = ( Path(__file__).resolve().parent @@ -51,11 +56,12 @@ _SWEEP_JSON = _OUTPUT_DIR / "sweep.json" _PANELS = ( - "single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 - # Cases 1-3 to be added by 5C.A/B/C + "single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 ★ optimal + "single_kv_group_decode_gqa_cube_repl_pe_tp", # Case 2 (no comm; 8× memory) + # Cases 1, 3 to be added by 5C.A/C ) -# Each entry: (kind, panel-specific params for _run_decode_panel). +# Each entry: (kind, panel-specific params). # LLaMA-3.1-70B single-KV-head group target: # 1 KV head, h_q = 8 (G = 8 group), d_head = 128 # 8 cubes (head-parallel group), 8 PEs/cube @@ -68,22 +74,63 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = { "T_q": 1, "S_kv": 131_072, "d_head": 128, "h_q": 8, "h_kv": 1, }), + "single_kv_group_decode_gqa_cube_repl_pe_tp": ("decode_cube_repl_pe_tp", { + # Case 2: K, V replicated everywhere (8× memory waste); PEs TP + # on batch. For B=1 only one rank works (slide-11 PE-TP waste). + # No inter-rank communication. + "C": 8, "P": 8, + "T_q": 1, "S_kv": 131_072, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), } # ── Per-panel runner ───────────────────────────────────────────────── +def _run_decode_panel_cube_repl_pe_tp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 2 runner: K, V replicated everywhere; B=1 single-rank work. + + DPPolicy models the cluster-wide memory waste — every rank holds + full K, V in its HBM region. Only PE 0 of CUBE 0 computes (the + kernel early-returns on every other rank), so only one rank reads + from its HBM copy and writes the output. + """ + configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) + dp_repl = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_decode_cube_repl_pe_tp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + def _make_bench_fn(panel: str): kind, params = _PANEL_DISPATCH[panel] def _bench_fn(ctx): if kind == "decode": _run_decode_panel(ctx, panel=panel, **params) + elif kind == "decode_cube_repl_pe_tp": + _run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params) else: raise RuntimeError( f"milestone-gqa-decode-4cases panel {panel!r} has " - f"unsupported kind={kind!r}; only 'decode' is allowed." + f"unsupported kind={kind!r}." ) return _bench_fn diff --git a/tests/attention/test_milestone_gqa_decode_4cases.py b/tests/attention/test_milestone_gqa_decode_4cases.py index 2655531..cbc7d4f 100644 --- a/tests/attention/test_milestone_gqa_decode_4cases.py +++ b/tests/attention/test_milestone_gqa_decode_4cases.py @@ -39,6 +39,7 @@ from kernbench.topology.builder import resolve_topology TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml" _CASE4_PANEL = "single_kv_group_decode_gqa_cube_sp_pe_sp" +_CASE2_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_tp" _CUBE_RE = re.compile(r"\bcube(\d+)\b") @@ -161,6 +162,112 @@ def test_case4_root_at_center_cube_6(): # ── T4: 2-phase AR ipcq pattern matches the predicted Case-4 traffic ─ +def _run_case2_smoke(*, S_kv: int): + """Drive the Case 2 decode panel via the case-specific runner. + + Case 2 = Cube-Repl × PE-TP. K, V are replicated everywhere (the + slide-11 memory waste); for B=1 only one rank does the work; no + inter-rank comm. + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _run_decode_panel_cube_repl_pe_tp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_decode_panel_cube_repl_pe_tp( + ctx, panel=_CASE2_PANEL, + C=8, P=8, + T_q=1, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +# ── Case 2 — T1: panel registered ─────────────────────────────────── + + +def test_case2_panel_registered(): + """The Case 2 panel must be in the bench's ``_PANELS`` + + ``_PANEL_DISPATCH`` with the expected single-KV-group dims. + + Case 2: Cube-Repl × PE-TP. K, V replicated everywhere + (8 KB/tok/PE — slide-11 memory waste); no inter-rank comm. + For B=1 only one rank works (PEs 1-7 idle — slide-11 calls + out this PE-TP waste). + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE2_PANEL in _PANELS, ( + f"{_CASE2_PANEL!r} not in _PANELS; got {_PANELS}" + ) + assert _CASE2_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE2_PANEL] + assert kind == "decode_cube_repl_pe_tp" + assert params.get("C") == 8 + assert params.get("P") == 8 + assert params.get("T_q") == 1 + assert params.get("S_kv") == 131_072 + assert params.get("d_head") == 128 + assert params.get("h_q") == 8 + assert params.get("h_kv") == 1 + + +# ── Case 2 — T2: smoke runner completes ───────────────────────────── + + +def test_case2_runner_smoke(): + """Case 2 runner drives the new kernel to completion at smoke S_kv.""" + result = _run_case2_smoke(S_kv=8192) + assert result.completion.ok, ( + f"Case 2 decode smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +# ── Case 2 — T3: zero inter-rank comm by design ───────────────────── + + +def test_case2_zero_ipcq_copy_no_comm(): + """Case 2's defining property: full KV per rank ⇒ NO inter-rank + communication. Slide 11 lists comm cost as 'none'. + """ + result = _run_case2_smoke(S_kv=8192) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy == 0, ( + f"Case 2 must have zero inter-rank comm; got ipcq_copy={n_copy}" + ) + + +# ── Case 2 — T4: single dma_write from cube 0 (B=1 single-rank work) ─ + + +def test_case2_single_dma_write_at_cube_0(): + """For B=1, only PE 0 of CUBE 0 does the work (the inherent PE-TP + waste at B=1). Exactly 1 dma_write, from cube 0. + """ + result = _run_case2_smoke(S_kv=8192) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == {0}, ( + f"Case 2 B=1 single writer must be cube 0; " + f"got cubes={sorted(distinct)}" + ) + + +# ── Case 4 — T4 (existing, kept) ──────────────────────────────────── + + def test_case4_two_level_ar_ipcq_pattern(): """Total ipcq_copy for the Case 4 reduce at (C, P, sub_w) = (8, 8, 4):