diff --git a/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_tp.py b/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_tp.py new file mode 100644 index 0000000..0899742 --- /dev/null +++ b/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_tp.py @@ -0,0 +1,238 @@ +"""GQA decode kernel — Case 1 (Cube-SP × PE-TP). + +Per GQA_full_deck.pptx slide 11: + - K, V split S_kv-wise across the 8 cubes (cube=row_wise); + replicated within each cube (pe=replicate). Each active rank + owns S_local = S_kv / C. + - PEs nominally split on the batch dim (PE-TP). At B=1 (the + slide-11 single-user default) only PE 0 of each cube has work; + PEs 1-7 of every cube idle (PE-TP-at-B=1 waste). + - NO intra-CUBE communication (only PE 0 holds a valid partial). + - Inter-CUBE 8-way reduce on (m, ℓ, O) via the lrab-adapted + center-root pattern (ADR-0060 §4.2): Phase 1 row reduce + converges at root_col; Phase 2 col reduce on root_col converges + at root_row. For sub_w=4, sub_h=2: root_col=2, root_row=1, + root_cube=6 (lrab geometric center). + +Tensor layout: + Q : (T_q, h_q · d_head) replicated on every rank. + K : (S_kv, h_kv · d_head) with cube=row_wise, pe=replicate — + each cube's PE 0 owns the full (S_local, h_kv·d_head) shard. + V : same as K. + O : (T_q, h_q · d_head) — only PE 0 of CUBE 6 stores. + +Topology / SFR: + - Requires ``configure_sfr_intercube_multisip`` for the E/W/N/S + inter-CUBE lanes used by the lrab reduce. +""" +from __future__ import annotations + +from kernbench.benches._gqa_attention_decode_long import _merge_running + + +TILE_S_KV = 1024 # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width). + + +def gqa_attention_decode_cube_sp_pe_tp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-1 decode: PE 0 per cube does attention; inter-CUBE lrab AR only.""" + # B=1 single-user decode + PE-TP: only PE 0 per cube has work. + pe_id = tl.program_id(axis=0) + cube_id = tl.program_id(axis=1) + if pe_id != 0: + return + + # Cube-SP geometry: 4×2 sub-mesh, lrab center-root cube = 6. + sub_w = 4 + sub_h = C // sub_w + if sub_w < 2 or sub_h < 2 or sub_w * sub_h != C: + raise ValueError( + f"Case 1 requires C decomposable as sub_w=4, sub_h>=2; got C={C}" + ) + root_col = sub_w // 2 + root_row = sub_h // 2 + root_cube = root_row * sub_w + root_col + + G = h_q // h_kv + S_local = S_kv // C # cube=row_wise, pe=replicate ⇒ S_local per cube + KV_ROW_BYTES = d_head * 2 # f16 + + # ── Local attention (S_kv-axis tile sweep, ADR-0063 §A.2) ── + Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16") + n_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV + + tile_s0 = min(TILE_S_KV, S_local) + K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16") + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + centered = scores - m_local + exp_scores = tl.exp(centered) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + for tile_idx in range(1, n_tiles): + tile_start = tile_idx * TILE_S_KV + tile_s = min(TILE_S_KV, S_local - tile_start) + with tl.scratch_scope(): + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + scores_t = tl.dot(Q, K_T_t) + m_tile = tl.max(scores_t, axis=-1) + centered_t = scores_t - m_tile + exp_scores_t = tl.exp(centered_t) + l_tile = tl.sum(exp_scores_t, axis=-1) + O_tile = tl.dot(exp_scores_t, V_t) + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_tile, l_tile, O_tile, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Inter-CUBE lrab-adapted center-root reduce (ADR-0060 §4.2) ── + row = cube_id // sub_w + col = cube_id % sub_w + + # Phase 1: row reduce — converge at col == root_col. + if col == 0 and root_col > 0: + tl.send(dir="E", src=m_local) + tl.send(dir="E", src=l_local) + tl.send(dir="E", src=O_local) + elif 0 < col < root_col: + with tl.scratch_scope(): + m_other = tl.recv(dir="W", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="W", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="W", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + tl.send(dir="E", src=m_local) + tl.send(dir="E", src=l_local) + tl.send(dir="E", src=O_local) + elif col == root_col: + if root_col > 0: + with tl.scratch_scope(): + m_other = tl.recv(dir="W", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="W", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="W", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if sub_w - 1 > root_col: + with tl.scratch_scope(): + m_other = tl.recv(dir="E", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="E", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="E", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + elif root_col < col < sub_w - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="E", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="E", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="E", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + tl.send(dir="W", src=m_local) + tl.send(dir="W", src=l_local) + tl.send(dir="W", src=O_local) + elif col == sub_w - 1 and sub_w - 1 > root_col: + tl.send(dir="W", src=m_local) + tl.send(dir="W", src=l_local) + tl.send(dir="W", src=O_local) + + # Phase 2: col reduce on col == root_col — converge at row == root_row. + if col == root_col: + if row == 0 and root_row > 0: + tl.send(dir="S", src=m_local) + tl.send(dir="S", src=l_local) + tl.send(dir="S", src=O_local) + elif 0 < row < root_row: + with tl.scratch_scope(): + m_other = tl.recv(dir="N", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="N", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="N", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + tl.send(dir="S", src=m_local) + tl.send(dir="S", src=l_local) + tl.send(dir="S", src=O_local) + elif row == root_row: + if root_row > 0: + with tl.scratch_scope(): + m_other = tl.recv(dir="N", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="N", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="N", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if sub_h - 1 > root_row: + with tl.scratch_scope(): + m_other = tl.recv(dir="S", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="S", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="S", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + elif root_row < row < sub_h - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="S", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="S", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="S", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + tl.send(dir="N", src=m_local) + tl.send(dir="N", src=l_local) + tl.send(dir="N", src=O_local) + elif row == sub_h - 1 and sub_h - 1 > root_row: + tl.send(dir="N", src=m_local) + tl.send(dir="N", src=l_local) + tl.send(dir="N", src=O_local) + + # ── Final normalise + store (root only: PE 0 of cube 6) ── + if cube_id == root_cube: + O_final = O_local / l_local + tl.store(o_ptr, O_final) diff --git a/src/kernbench/benches/milestone_gqa_decode_4cases.py b/src/kernbench/benches/milestone_gqa_decode_4cases.py index a8dad48..5ee5041 100644 --- a/src/kernbench/benches/milestone_gqa_decode_4cases.py +++ b/src/kernbench/benches/milestone_gqa_decode_4cases.py @@ -38,6 +38,9 @@ from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import ( from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import ( gqa_attention_decode_cube_repl_pe_tp_kernel, ) +from kernbench.benches._gqa_attention_decode_cube_sp_pe_tp import ( + gqa_attention_decode_cube_sp_pe_tp_kernel, +) from kernbench.benches.milestone_gqa_headline import ( _ccl_cfg, _run_decode_panel, @@ -62,7 +65,7 @@ _PANELS = ( "single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 ★ optimal "single_kv_group_decode_gqa_cube_repl_pe_tp", # Case 2 (no comm; 8× memory) "single_kv_group_decode_gqa_cube_repl_pe_sp", # Case 3 (intra-CUBE AR only; 8× memory) - # Case 1 to be added by 5C.A + "single_kv_group_decode_gqa_cube_sp_pe_tp", # Case 1 (inter-CUBE lrab only; PE-TP B=1 waste) ) # Each entry: (kind, panel-specific params). @@ -94,6 +97,14 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = { "T_q": 1, "S_kv": 131_072, "d_head": 128, "h_q": 8, "h_kv": 1, }), + "single_kv_group_decode_gqa_cube_sp_pe_tp": ("decode_cube_sp_pe_tp", { + # Case 1: K, V split across cubes (S_local = S_kv/C per cube); + # PEs TP on batch — at B=1 only PE 0 of each cube works (PE-TP + # waste). Inter-CUBE lrab AR (root = cube 6); no intra-CUBE comm. + "C": 8, "P": 8, + "T_q": 1, "S_kv": 131_072, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), } @@ -164,6 +175,40 @@ def _run_decode_panel_cube_repl_pe_sp( ) +def _run_decode_panel_cube_sp_pe_tp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 1 runner: K, V split across cubes; PE-TP single-rank work at B=1. + + DPPolicy: K, V are cube=row_wise (S_local = S_kv/C per cube), + pe=replicate (full S_local within each cube). At B=1, the kernel + runs only on PE 0 of every cube; PEs 1-7 early-return. PE 0 per + cube does local attention on S_local, then participates in the + inter-CUBE lrab AR; PE 0 of cube 6 (lrab center) writes O. + """ + configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) + dp_full = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="row_wise", pe="replicate", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_decode_cube_sp_pe_tp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + def _make_bench_fn(panel: str): kind, params = _PANEL_DISPATCH[panel] @@ -174,6 +219,8 @@ def _make_bench_fn(panel: str): _run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params) elif kind == "decode_cube_repl_pe_sp": _run_decode_panel_cube_repl_pe_sp(ctx, panel=panel, **params) + elif kind == "decode_cube_sp_pe_tp": + _run_decode_panel_cube_sp_pe_tp(ctx, panel=panel, **params) else: raise RuntimeError( f"milestone-gqa-decode-4cases panel {panel!r} has " diff --git a/tests/attention/test_milestone_gqa_decode_4cases.py b/tests/attention/test_milestone_gqa_decode_4cases.py index 08e2ce1..5dc2111 100644 --- a/tests/attention/test_milestone_gqa_decode_4cases.py +++ b/tests/attention/test_milestone_gqa_decode_4cases.py @@ -41,6 +41,7 @@ TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml" _CASE4_PANEL = "single_kv_group_decode_gqa_cube_sp_pe_sp" _CASE3_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_sp" _CASE2_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_tp" +_CASE1_PANEL = "single_kv_group_decode_gqa_cube_sp_pe_tp" _CUBE_RE = re.compile(r"\bcube(\d+)\b") @@ -384,6 +385,126 @@ def test_case3_single_dma_write_at_cube_0(): ) +# ── Case 1 (Cube-SP × PE-TP) ──────────────────────────────────────── + + +def _run_case1_smoke(*, S_kv: int): + """Drive the Case 1 decode panel via the case-specific runner. + + Case 1 = Cube-SP × PE-TP. K, V split S_kv-wise across the 8 cubes + (cube=row_wise), replicated within each cube (pe=replicate). PEs + nominally split on the batch dim (PE-TP); at B=1 only PE 0 of + each cube has work; PEs 1-7 idle. Inter-CUBE 8-way reduce via the + lrab-adapted center-root pattern (root cube 6); no intra-CUBE comm. + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _run_decode_panel_cube_sp_pe_tp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_decode_panel_cube_sp_pe_tp( + ctx, panel=_CASE1_PANEL, + C=8, P=8, + T_q=1, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +# ── Case 1 — T1: panel registered ─────────────────────────────────── + + +def test_case1_panel_registered(): + """The Case 1 panel must be in the bench's ``_PANELS`` + + ``_PANEL_DISPATCH`` with the expected single-KV-group dims. + + Case 1: Cube-SP × PE-TP. K, V split across cubes; PEs TP on + batch. At B=1 only PE 0 of each cube works (PE-TP waste). + Inter-CUBE lrab AR; no intra-CUBE comm. + """ + from kernbench.benches.milestone_gqa_decode_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE1_PANEL in _PANELS, ( + f"{_CASE1_PANEL!r} not in _PANELS; got {_PANELS}" + ) + assert _CASE1_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE1_PANEL] + assert kind == "decode_cube_sp_pe_tp" + assert params.get("C") == 8 + assert params.get("P") == 8 + assert params.get("T_q") == 1 + assert params.get("S_kv") == 131_072 + assert params.get("d_head") == 128 + assert params.get("h_q") == 8 + assert params.get("h_kv") == 1 + + +# ── Case 1 — T2: smoke runner completes ───────────────────────────── + + +def test_case1_runner_smoke(): + """Case 1 runner drives the new kernel to completion at smoke S_kv.""" + result = _run_case1_smoke(S_kv=8192) + assert result.completion.ok, ( + f"Case 1 decode smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +# ── Case 1 — T3: inter-CUBE lrab only (21 ipcq, no intra-CUBE) ────── + + +def test_case1_inter_cube_lrab_only_ipcq(): + """Case 1 reduce pattern: only PE 0 of each cube has work (PE-TP + at B=1), so NO intra-CUBE AR. Inter-CUBE 8-way reduce uses the + lrab-adapted center-root pattern (same structural cost as Case 4's + inter-CUBE phase). + + Inter-CUBE lrab (sub_w=4, sub_h=2): + Phase 1 row reduce — 3 sends/row × 3 tensors × 2 rows = 18 + Phase 2 col reduce — cube 2 → S × 3 tensors = 3 + inter-CUBE total = 21 + + Intra-CUBE: 0 (PEs 1-7 idle, no partials to merge). + + Grand total: 21. + """ + result = _run_case1_smoke(S_kv=8192) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy == 21, ( + f"Case 1 expected 21 ipcq_copy (inter-CUBE lrab only, no intra-CUBE); " + f"got {n_copy}" + ) + + +# ── Case 1 — T4: root at lrab center cube 6 ───────────────────────── + + +def test_case1_root_at_center_cube_6(): + """Case 1 uses the same lrab-adapted center-root reduce as Case 4's + inter-CUBE phase; the answer lands at the lrab center cube + (sub_w=4, sub_h=2 → root_col=2, root_row=1 → root_cube=6). + """ + result = _run_case1_smoke(S_kv=8192) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == {6}, ( + f"Case 1 root must be the lrab center cube 6; " + f"got cubes={sorted(distinct)}" + ) + + # ── Case 4 — T4 (existing, kept) ────────────────────────────────────