diff --git a/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_sp.py b/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_sp.py new file mode 100644 index 0000000..2f06c0c --- /dev/null +++ b/src/kernbench/benches/_gqa_attention_decode_cube_sp_pe_sp.py @@ -0,0 +1,46 @@ +"""GQA decode kernel — Case 4 (Cube-SP × PE-SP). ★ slide-11 optimal. + +This is a thin entry-point wrapper around the underlying decode_long +lrab-adapted center-root kernel with ``sub_w`` baked to 4 (the C=8 +single-KV-head-group geometry: 4×2 cube sub-mesh, root cube 6). The +math lives in ``_gqa_attention_decode_long.py`` (which still serves +non-4-cases panels via ``sub_w=0`` and the headline GQA bench); this +file exists so the 4-cases bench refers to all four cases via +parallel ``_gqa_attention_decode_.py`` kernel-file names. + +Per GQA_full_deck.pptx slide 11: + - K, V split 64-way (cube=row_wise, pe=row_wise); each rank owns + S_local = S_kv / (C·P). + - Two-level reduce on the partial (m, ℓ, O): intra-CUBE 8-way + (row chain + col bridge over the 2×4 PE grid) then inter-CUBE + 2-phase lrab over the 4×2 cube sub-mesh — converges at cube 6. + - PE 0 of CUBE 6 stores O. +""" +from __future__ import annotations + +from kernbench.benches._gqa_attention_decode_long import ( + gqa_attention_decode_long_kernel, +) + + +def gqa_attention_decode_cube_sp_pe_sp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-4 decode: Cube-SP × PE-SP, lrab center-root at cube 6.""" + gqa_attention_decode_long_kernel( + q_ptr, k_ptr, v_ptr, o_ptr, + T_q, S_kv, h_q, h_kv, d_head, C, P, 4, # sub_w=4 baked in + tl=tl, + ) diff --git a/src/kernbench/benches/milestone_gqa_decode_4cases.py b/src/kernbench/benches/milestone_gqa_decode_4cases.py index 5ee5041..29bb541 100644 --- a/src/kernbench/benches/milestone_gqa_decode_4cases.py +++ b/src/kernbench/benches/milestone_gqa_decode_4cases.py @@ -38,12 +38,14 @@ from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import ( from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import ( gqa_attention_decode_cube_repl_pe_tp_kernel, ) +from kernbench.benches._gqa_attention_decode_cube_sp_pe_sp import ( + gqa_attention_decode_cube_sp_pe_sp_kernel, +) from kernbench.benches._gqa_attention_decode_cube_sp_pe_tp import ( gqa_attention_decode_cube_sp_pe_tp_kernel, ) from kernbench.benches.milestone_gqa_headline import ( _ccl_cfg, - _run_decode_panel, _summarize_op_log, ) from kernbench.benches.registry import bench @@ -74,10 +76,11 @@ _PANELS = ( # 8 cubes (head-parallel group), 8 PEs/cube # S_kv = 128K (long-context decode), T_q = 1 (one new token per pass) _PANEL_DISPATCH: dict[str, tuple[str, dict]] = { - "single_kv_group_decode_gqa_cube_sp_pe_sp": ("decode", { + "single_kv_group_decode_gqa_cube_sp_pe_sp": ("decode_cube_sp_pe_sp", { # Case 4: KV split 64-way (Cube-SP × PE-SP), 2-level reduce. - # sub_w=4 ⇒ sub_h=2 ⇒ lrab center-root cube = (1,2) = cube 6. - "C": 8, "P": 8, "sub_w": 4, + # The sub_w=4/sub_h=2 lrab center-root geometry (root cube 6) is + # baked into the Case-4 wrapper kernel. + "C": 8, "P": 8, "T_q": 1, "S_kv": 131_072, "d_head": 128, "h_q": 8, "h_kv": 1, }), @@ -209,12 +212,44 @@ def _run_decode_panel_cube_sp_pe_tp( ) +def _run_decode_panel_cube_sp_pe_sp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 4 runner: K, V split 64-way (Cube-SP × PE-SP). + + DPPolicy: K, V are cube=row_wise, pe=row_wise — each rank owns + S_local = S_kv/(C·P). The wrapper kernel bakes in the lrab + sub_w=4 / sub_h=2 geometry (root cube 6, ADR-0060 §4.2). + """ + configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg()) + dp_full = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="row_wise", pe="row_wise", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_decode_cube_sp_pe_sp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + def _make_bench_fn(panel: str): kind, params = _PANEL_DISPATCH[panel] def _bench_fn(ctx): - if kind == "decode": - _run_decode_panel(ctx, panel=panel, **params) + if kind == "decode_cube_sp_pe_sp": + _run_decode_panel_cube_sp_pe_sp(ctx, panel=panel, **params) elif kind == "decode_cube_repl_pe_tp": _run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params) elif kind == "decode_cube_repl_pe_sp": diff --git a/tests/attention/test_milestone_gqa_decode_4cases.py b/tests/attention/test_milestone_gqa_decode_4cases.py index 5dc2111..efbf0e3 100644 --- a/tests/attention/test_milestone_gqa_decode_4cases.py +++ b/tests/attention/test_milestone_gqa_decode_4cases.py @@ -98,10 +98,13 @@ def test_case4_panel_registered(): Headline config: C = 8 (head-parallel CUBE Group) P = 8 (intra-CUBE PE-SP) - sub_w = 4 (lrab-adapted center-root reduce; root cube 6) T_q = 1 (decode: one new token per pass) S_kv = 131_072 (LLaMA long-context decode target) d_head = 128, h_q = 8, h_kv = 1 + + The lrab sub_w=4 / sub_h=2 geometry is baked into the Case 4 + wrapper kernel (``_gqa_attention_decode_cube_sp_pe_sp.py``); it + is not a panel parameter. """ from kernbench.benches.milestone_gqa_decode_4cases import ( _PANEL_DISPATCH, @@ -112,10 +115,11 @@ def test_case4_panel_registered(): ) assert _CASE4_PANEL in _PANEL_DISPATCH kind, params = _PANEL_DISPATCH[_CASE4_PANEL] - assert kind == "decode", f"kind={kind!r}, expected 'decode'" + assert kind == "decode_cube_sp_pe_sp", ( + f"kind={kind!r}, expected 'decode_cube_sp_pe_sp'" + ) assert params.get("C") == 8 assert params.get("P") == 8 - assert params.get("sub_w") == 4 assert params.get("T_q") == 1 assert params.get("S_kv") == 131_072 assert params.get("d_head") == 128