From ae942f695988bd72bf5806a46096891da103c447 Mon Sep 17 00:00:00 2001 From: Mukesh Garg Date: Mon, 15 Jun 2026 15:18:41 -0700 Subject: [PATCH] =?UTF-8?q?gqa(decode-4cases):=20Case=203=20=E2=80=94=20Cu?= =?UTF-8?q?be-Repl=20=C3=97=20PE-SP=20(intra-CUBE=20AR=20only;=208=C3=97?= =?UTF-8?q?=20memory)=20(5C.C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the third 4-cases panel: KV replicated per cube (8× memory waste), PEs SP on S_kv within each cube, intra-CUBE 8-way reduce on (m, ℓ, O), and no inter-CUBE comm (every cube ends with full answer; designated writer = cube 0). Reuses the row-chain + col-bridge intra-CUBE pattern that anchors Case 4 (21 ipcq_copy per cube × 8 cubes = 168 total). 12 tests pass (4 Case 4 + 4 Case 2 + 4 Case 3). Co-Authored-By: Claude Opus 4.7 --- .../_gqa_attention_decode_cube_repl_pe_sp.py | 135 ++++++++++++++++++ .../benches/milestone_gqa_decode_4cases.py | 49 ++++++- .../test_milestone_gqa_decode_4cases.py | 119 +++++++++++++++ 3 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_sp.py diff --git a/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_sp.py b/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_sp.py new file mode 100644 index 0000000..5e1a427 --- /dev/null +++ b/src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_sp.py @@ -0,0 +1,135 @@ +"""GQA decode kernel — Case 3 (Cube-Repl × PE-SP). + +Per GQA_full_deck.pptx slide 11: + - K, V replicated across all 8 cubes (the 8× memory waste). + - PEs SP on S_kv inside each cube: each PE attends to its + ``S_local = S_kv / P`` slice. + - Intra-CUBE 8-way reduce on the partial ``(m, ℓ, O)`` triple + (row chain + col bridge over the 2×4 PE grid, same structural + pattern as Case 4's intra-CUBE phase). + - NO inter-CUBE communication — every cube ends with the full + answer (8× redundant compute is the inherent cost of Case 3). + - Designated writer: only PE 0 of CUBE 0 stores ``O`` to avoid + 8 redundant DMA writes. + +Tensor layout: + Q : (T_q, h_q · d_head) replicated on every rank. + K : (S_kv, h_kv · d_head) with cube=replicate, pe=row_wise — + each PE owns its (S_local, h_kv·d_head) shard within its cube. + V : same as K. + O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores. + +Topology / SFR: + - ``configure_sfr_intercube_multisip`` provides the intra_* / + E/W/N/S namespaces; only the intra_* lanes are exercised here. +""" +from __future__ import annotations + +from kernbench.benches._gqa_attention_decode_long import _merge_running + + +TILE_S_KV = 1024 # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width). + + +def gqa_attention_decode_cube_repl_pe_sp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-3 decode: PE-SP attention; replicated KV per cube; intra-CUBE AR only.""" + G = h_q // h_kv + S_local = S_kv // P # each PE owns S_kv/P (cube=replicate, pe=row_wise) + pe_id = tl.program_id(axis=0) + cube_id = tl.program_id(axis=1) + + # ── Local attention (S_kv-axis tile sweep, ADR-0063 §A.2) ── + Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16") + n_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV + KV_ROW_BYTES = d_head * 2 # f16 + + tile_s0 = min(TILE_S_KV, S_local) + K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16") + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + centered = scores - m_local + exp_scores = tl.exp(centered) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + for tile_idx in range(1, n_tiles): + tile_start = tile_idx * TILE_S_KV + tile_s = min(TILE_S_KV, S_local - tile_start) + with tl.scratch_scope(): + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + scores_t = tl.dot(Q, K_T_t) + m_tile = tl.max(scores_t, axis=-1) + centered_t = scores_t - m_tile + exp_scores_t = tl.exp(centered_t) + l_tile = tl.sum(exp_scores_t, axis=-1) + O_tile = tl.dot(exp_scores_t, V_t) + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_tile, l_tile, O_tile, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Intra-CUBE 8-way reduce-to-PE0 (row chain + col bridge) ── + PE_GRID_COLS = 4 + pe_col = pe_id % PE_GRID_COLS + pe_row = pe_id // PE_GRID_COLS + pe_cols_used = min(PE_GRID_COLS, P) + pe_rows_used = (P + PE_GRID_COLS - 1) // PE_GRID_COLS + + if pe_cols_used > 1: + if pe_col < pe_cols_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_E", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_E", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_E", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_col > 0: + tl.send(dir="intra_W", src=m_local) + tl.send(dir="intra_W", src=l_local) + tl.send(dir="intra_W", src=O_local) + + if pe_col == 0 and pe_rows_used > 1: + if pe_row < pe_rows_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_S", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_S", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_S", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_row > 0: + tl.send(dir="intra_N", src=m_local) + tl.send(dir="intra_N", src=l_local) + tl.send(dir="intra_N", src=O_local) + + # ── Final normalise + store (designated writer: cube 0, PE 0) ── + if pe_id == 0 and cube_id == 0: + O_final = O_local / l_local + tl.store(o_ptr, O_final) diff --git a/src/kernbench/benches/milestone_gqa_decode_4cases.py b/src/kernbench/benches/milestone_gqa_decode_4cases.py index 267fcb7..a8dad48 100644 --- a/src/kernbench/benches/milestone_gqa_decode_4cases.py +++ b/src/kernbench/benches/milestone_gqa_decode_4cases.py @@ -32,6 +32,9 @@ import json import os from pathlib import Path +from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import ( + gqa_attention_decode_cube_repl_pe_sp_kernel, +) from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import ( gqa_attention_decode_cube_repl_pe_tp_kernel, ) @@ -58,7 +61,8 @@ _SWEEP_JSON = _OUTPUT_DIR / "sweep.json" _PANELS = ( "single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 ★ optimal "single_kv_group_decode_gqa_cube_repl_pe_tp", # Case 2 (no comm; 8× memory) - # Cases 1, 3 to be added by 5C.A/C + "single_kv_group_decode_gqa_cube_repl_pe_sp", # Case 3 (intra-CUBE AR only; 8× memory) + # Case 1 to be added by 5C.A ) # Each entry: (kind, panel-specific params). @@ -82,6 +86,14 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = { "T_q": 1, "S_kv": 131_072, "d_head": 128, "h_q": 8, "h_kv": 1, }), + "single_kv_group_decode_gqa_cube_repl_pe_sp": ("decode_cube_repl_pe_sp", { + # Case 3: K, V replicated per cube (8× memory); PEs SP on S_kv + # within each cube. Intra-CUBE 8-way reduce; no inter-CUBE comm + # (every cube ends with full answer; designated writer = cube 0). + "C": 8, "P": 8, + "T_q": 1, "S_kv": 131_072, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), } @@ -119,6 +131,39 @@ def _run_decode_panel_cube_repl_pe_tp( ) +def _run_decode_panel_cube_repl_pe_sp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 3 runner: K, V replicated per cube; PEs SP on S_kv within cube. + + DPPolicy models the cluster-wide 8× memory waste — every cube + holds full K, V in its HBM region, then splits the S_kv axis + row_wise across its 8 PEs. The kernel does an intra-CUBE 8-way + reduce on (m, ℓ, O); only cube 0's PE 0 writes the output. + """ + configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) + dp_full = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="replicate", pe="row_wise", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_decode_cube_repl_pe_sp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + def _make_bench_fn(panel: str): kind, params = _PANEL_DISPATCH[panel] @@ -127,6 +172,8 @@ def _make_bench_fn(panel: str): _run_decode_panel(ctx, panel=panel, **params) elif kind == "decode_cube_repl_pe_tp": _run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params) + elif kind == "decode_cube_repl_pe_sp": + _run_decode_panel_cube_repl_pe_sp(ctx, panel=panel, **params) else: raise RuntimeError( f"milestone-gqa-decode-4cases panel {panel!r} has " diff --git a/tests/attention/test_milestone_gqa_decode_4cases.py b/tests/attention/test_milestone_gqa_decode_4cases.py index cbc7d4f..08e2ce1 100644 --- a/tests/attention/test_milestone_gqa_decode_4cases.py +++ b/tests/attention/test_milestone_gqa_decode_4cases.py @@ -39,6 +39,7 @@ from kernbench.topology.builder import resolve_topology TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml" _CASE4_PANEL = "single_kv_group_decode_gqa_cube_sp_pe_sp" +_CASE3_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_sp" _CASE2_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_tp" _CUBE_RE = re.compile(r"\bcube(\d+)\b") @@ -265,6 +266,124 @@ def test_case2_single_dma_write_at_cube_0(): ) +# ── Case 3 (Cube-Repl × PE-SP) ────────────────────────────────────── + + +def _run_case3_smoke(*, S_kv: int): + """Drive the Case 3 decode panel via the case-specific runner. + + Case 3 = Cube-Repl × PE-SP. K, V replicated per cube; S_kv split + 8-way across PEs within each cube. Intra-CUBE 8-way reduce on + (m, ℓ, O); no inter-CUBE comm (every cube ends with full answer). + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _run_decode_panel_cube_repl_pe_sp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_decode_panel_cube_repl_pe_sp( + ctx, panel=_CASE3_PANEL, + C=8, P=8, + T_q=1, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +# ── Case 3 — T1: panel registered ─────────────────────────────────── + + +def test_case3_panel_registered(): + """The Case 3 panel must be in the bench's ``_PANELS`` + + ``_PANEL_DISPATCH`` with the expected single-KV-group dims. + + Case 3: Cube-Repl × PE-SP. K, V replicated per cube; PEs SP on + S_kv. Intra-CUBE 8-way AR; no inter-CUBE comm. + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE3_PANEL in _PANELS, ( + f"{_CASE3_PANEL!r} not in _PANELS; got {_PANELS}" + ) + assert _CASE3_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE3_PANEL] + assert kind == "decode_cube_repl_pe_sp" + assert params.get("C") == 8 + assert params.get("P") == 8 + assert params.get("T_q") == 1 + assert params.get("S_kv") == 131_072 + assert params.get("d_head") == 128 + assert params.get("h_q") == 8 + assert params.get("h_kv") == 1 + + +# ── Case 3 — T2: smoke runner completes ───────────────────────────── + + +def test_case3_runner_smoke(): + """Case 3 runner drives the new kernel to completion at smoke S_kv.""" + result = _run_case3_smoke(S_kv=8192) + assert result.completion.ok, ( + f"Case 3 decode smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +# ── Case 3 — T3: intra-CUBE-only AR (168 ipcq, no inter-CUBE) ─────── + + +def test_case3_intra_cube_ar_only_ipcq(): + """Case 3 reduce pattern: per-CUBE 8-way PE-SP AR (same structural + cost as Case 4's intra-CUBE phase = 21 ipcq_copy per cube), and + NO inter-CUBE traffic (each cube has a full copy of KV). + + per-CUBE intra (2×4 PE grid): + row chain along intra_W: cols 1,2,3 each row × 2 rows × + 3 tensors (m, ℓ, O) = 18 + col bridge along intra_N: pe4 only × 3 tensors = 3 + per-CUBE intra total = 21 + × 8 CUBEs = 168 + + inter-CUBE: 0 (replicated KV ⇒ no AllReduce needed). + + Grand total: 168. + """ + result = _run_case3_smoke(S_kv=8192) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy == 168, ( + f"Case 3 expected 168 ipcq_copy (intra-CUBE only, no inter-CUBE); " + f"got {n_copy}" + ) + + +# ── Case 3 — T4: single dma_write from cube 0 (designated writer) ─── + + +def test_case3_single_dma_write_at_cube_0(): + """Every cube ends with the full answer after intra-CUBE AR; only + the designated writer (cube 0, PE 0) stores O to avoid 8 redundant + DMAs. + """ + result = _run_case3_smoke(S_kv=8192) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == {0}, ( + f"Case 3 designated writer must be cube 0; " + f"got cubes={sorted(distinct)}" + ) + + # ── Case 4 — T4 (existing, kept) ────────────────────────────────────