gqa(decode-4cases): give Case 4 its own thin kernel file for naming symmetry
Cases 1-3 each have a dedicated _gqa_attention_decode_<case>.py
kernel file; Case 4 previously reached into _gqa_attention_decode_long.py
via the headline bench's _run_decode_panel helper, breaking the
one-file-per-case convention. Adds _gqa_attention_decode_cube_sp_pe_sp.py
as a 20-line wrapper that bakes in sub_w=4 (the C=8 lrab geometry)
and gives Case 4 its own kind ("decode_cube_sp_pe_sp") and helper
(_run_decode_panel_cube_sp_pe_sp). decode_long.py is unchanged
(still serves the legacy decode_long tests). 16 tests pass.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,46 @@
|
|||||||
|
"""GQA decode kernel — Case 4 (Cube-SP × PE-SP). ★ slide-11 optimal.
|
||||||
|
|
||||||
|
This is a thin entry-point wrapper around the underlying decode_long
|
||||||
|
lrab-adapted center-root kernel with ``sub_w`` baked to 4 (the C=8
|
||||||
|
single-KV-head-group geometry: 4×2 cube sub-mesh, root cube 6). The
|
||||||
|
math lives in ``_gqa_attention_decode_long.py`` (which still serves
|
||||||
|
non-4-cases panels via ``sub_w=0`` and the headline GQA bench); this
|
||||||
|
file exists so the 4-cases bench refers to all four cases via
|
||||||
|
parallel ``_gqa_attention_decode_<case>.py`` kernel-file names.
|
||||||
|
|
||||||
|
Per GQA_full_deck.pptx slide 11:
|
||||||
|
- K, V split 64-way (cube=row_wise, pe=row_wise); each rank owns
|
||||||
|
S_local = S_kv / (C·P).
|
||||||
|
- Two-level reduce on the partial (m, ℓ, O): intra-CUBE 8-way
|
||||||
|
(row chain + col bridge over the 2×4 PE grid) then inter-CUBE
|
||||||
|
2-phase lrab over the 4×2 cube sub-mesh — converges at cube 6.
|
||||||
|
- PE 0 of CUBE 6 stores O.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from kernbench.benches._gqa_attention_decode_long import (
|
||||||
|
gqa_attention_decode_long_kernel,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def gqa_attention_decode_cube_sp_pe_sp_kernel(
|
||||||
|
q_ptr: int,
|
||||||
|
k_ptr: int,
|
||||||
|
v_ptr: int,
|
||||||
|
o_ptr: int,
|
||||||
|
T_q: int,
|
||||||
|
S_kv: int,
|
||||||
|
h_q: int,
|
||||||
|
h_kv: int,
|
||||||
|
d_head: int,
|
||||||
|
C: int,
|
||||||
|
P: int,
|
||||||
|
*,
|
||||||
|
tl,
|
||||||
|
) -> None:
|
||||||
|
"""Case-4 decode: Cube-SP × PE-SP, lrab center-root at cube 6."""
|
||||||
|
gqa_attention_decode_long_kernel(
|
||||||
|
q_ptr, k_ptr, v_ptr, o_ptr,
|
||||||
|
T_q, S_kv, h_q, h_kv, d_head, C, P, 4, # sub_w=4 baked in
|
||||||
|
tl=tl,
|
||||||
|
)
|
||||||
@@ -38,12 +38,14 @@ from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import (
|
|||||||
from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import (
|
from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import (
|
||||||
gqa_attention_decode_cube_repl_pe_tp_kernel,
|
gqa_attention_decode_cube_repl_pe_tp_kernel,
|
||||||
)
|
)
|
||||||
|
from kernbench.benches._gqa_attention_decode_cube_sp_pe_sp import (
|
||||||
|
gqa_attention_decode_cube_sp_pe_sp_kernel,
|
||||||
|
)
|
||||||
from kernbench.benches._gqa_attention_decode_cube_sp_pe_tp import (
|
from kernbench.benches._gqa_attention_decode_cube_sp_pe_tp import (
|
||||||
gqa_attention_decode_cube_sp_pe_tp_kernel,
|
gqa_attention_decode_cube_sp_pe_tp_kernel,
|
||||||
)
|
)
|
||||||
from kernbench.benches.milestone_gqa_headline import (
|
from kernbench.benches.milestone_gqa_headline import (
|
||||||
_ccl_cfg,
|
_ccl_cfg,
|
||||||
_run_decode_panel,
|
|
||||||
_summarize_op_log,
|
_summarize_op_log,
|
||||||
)
|
)
|
||||||
from kernbench.benches.registry import bench
|
from kernbench.benches.registry import bench
|
||||||
@@ -74,10 +76,11 @@ _PANELS = (
|
|||||||
# 8 cubes (head-parallel group), 8 PEs/cube
|
# 8 cubes (head-parallel group), 8 PEs/cube
|
||||||
# S_kv = 128K (long-context decode), T_q = 1 (one new token per pass)
|
# S_kv = 128K (long-context decode), T_q = 1 (one new token per pass)
|
||||||
_PANEL_DISPATCH: dict[str, tuple[str, dict]] = {
|
_PANEL_DISPATCH: dict[str, tuple[str, dict]] = {
|
||||||
"single_kv_group_decode_gqa_cube_sp_pe_sp": ("decode", {
|
"single_kv_group_decode_gqa_cube_sp_pe_sp": ("decode_cube_sp_pe_sp", {
|
||||||
# Case 4: KV split 64-way (Cube-SP × PE-SP), 2-level reduce.
|
# Case 4: KV split 64-way (Cube-SP × PE-SP), 2-level reduce.
|
||||||
# sub_w=4 ⇒ sub_h=2 ⇒ lrab center-root cube = (1,2) = cube 6.
|
# The sub_w=4/sub_h=2 lrab center-root geometry (root cube 6) is
|
||||||
"C": 8, "P": 8, "sub_w": 4,
|
# baked into the Case-4 wrapper kernel.
|
||||||
|
"C": 8, "P": 8,
|
||||||
"T_q": 1, "S_kv": 131_072,
|
"T_q": 1, "S_kv": 131_072,
|
||||||
"d_head": 128, "h_q": 8, "h_kv": 1,
|
"d_head": 128, "h_q": 8, "h_kv": 1,
|
||||||
}),
|
}),
|
||||||
@@ -209,12 +212,44 @@ def _run_decode_panel_cube_sp_pe_tp(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_decode_panel_cube_sp_pe_sp(
|
||||||
|
ctx, *, panel: str, C: int, P: int,
|
||||||
|
T_q: int, S_kv: int,
|
||||||
|
d_head: int, h_q: int, h_kv: int,
|
||||||
|
) -> None:
|
||||||
|
"""Case 4 runner: K, V split 64-way (Cube-SP × PE-SP).
|
||||||
|
|
||||||
|
DPPolicy: K, V are cube=row_wise, pe=row_wise — each rank owns
|
||||||
|
S_local = S_kv/(C·P). The wrapper kernel bakes in the lrab
|
||||||
|
sub_w=4 / sub_h=2 geometry (root cube 6, ADR-0060 §4.2).
|
||||||
|
"""
|
||||||
|
configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg())
|
||||||
|
dp_full = DPPolicy(cube="replicate", pe="replicate",
|
||||||
|
num_cubes=C, num_pes=P)
|
||||||
|
dp_kv = DPPolicy(cube="row_wise", pe="row_wise",
|
||||||
|
num_cubes=C, num_pes=P)
|
||||||
|
q = ctx.zeros((T_q, h_q * d_head),
|
||||||
|
dtype="f16", dp=dp_full, name=f"{panel}_q")
|
||||||
|
k = ctx.zeros((S_kv, h_kv * d_head),
|
||||||
|
dtype="f16", dp=dp_kv, name=f"{panel}_k")
|
||||||
|
v = ctx.zeros((S_kv, h_kv * d_head),
|
||||||
|
dtype="f16", dp=dp_kv, name=f"{panel}_v")
|
||||||
|
o = ctx.empty((T_q, h_q * d_head),
|
||||||
|
dtype="f16", dp=dp_full, name=f"{panel}_o")
|
||||||
|
ctx.launch(
|
||||||
|
panel, gqa_attention_decode_cube_sp_pe_sp_kernel,
|
||||||
|
q, k, v, o,
|
||||||
|
T_q, S_kv, h_q, h_kv, d_head, C, P,
|
||||||
|
_auto_dim_remap=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _make_bench_fn(panel: str):
|
def _make_bench_fn(panel: str):
|
||||||
kind, params = _PANEL_DISPATCH[panel]
|
kind, params = _PANEL_DISPATCH[panel]
|
||||||
|
|
||||||
def _bench_fn(ctx):
|
def _bench_fn(ctx):
|
||||||
if kind == "decode":
|
if kind == "decode_cube_sp_pe_sp":
|
||||||
_run_decode_panel(ctx, panel=panel, **params)
|
_run_decode_panel_cube_sp_pe_sp(ctx, panel=panel, **params)
|
||||||
elif kind == "decode_cube_repl_pe_tp":
|
elif kind == "decode_cube_repl_pe_tp":
|
||||||
_run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params)
|
_run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params)
|
||||||
elif kind == "decode_cube_repl_pe_sp":
|
elif kind == "decode_cube_repl_pe_sp":
|
||||||
|
|||||||
@@ -98,10 +98,13 @@ def test_case4_panel_registered():
|
|||||||
Headline config:
|
Headline config:
|
||||||
C = 8 (head-parallel CUBE Group)
|
C = 8 (head-parallel CUBE Group)
|
||||||
P = 8 (intra-CUBE PE-SP)
|
P = 8 (intra-CUBE PE-SP)
|
||||||
sub_w = 4 (lrab-adapted center-root reduce; root cube 6)
|
|
||||||
T_q = 1 (decode: one new token per pass)
|
T_q = 1 (decode: one new token per pass)
|
||||||
S_kv = 131_072 (LLaMA long-context decode target)
|
S_kv = 131_072 (LLaMA long-context decode target)
|
||||||
d_head = 128, h_q = 8, h_kv = 1
|
d_head = 128, h_q = 8, h_kv = 1
|
||||||
|
|
||||||
|
The lrab sub_w=4 / sub_h=2 geometry is baked into the Case 4
|
||||||
|
wrapper kernel (``_gqa_attention_decode_cube_sp_pe_sp.py``); it
|
||||||
|
is not a panel parameter.
|
||||||
"""
|
"""
|
||||||
from kernbench.benches.milestone_gqa_decode_4cases import (
|
from kernbench.benches.milestone_gqa_decode_4cases import (
|
||||||
_PANEL_DISPATCH,
|
_PANEL_DISPATCH,
|
||||||
@@ -112,10 +115,11 @@ def test_case4_panel_registered():
|
|||||||
)
|
)
|
||||||
assert _CASE4_PANEL in _PANEL_DISPATCH
|
assert _CASE4_PANEL in _PANEL_DISPATCH
|
||||||
kind, params = _PANEL_DISPATCH[_CASE4_PANEL]
|
kind, params = _PANEL_DISPATCH[_CASE4_PANEL]
|
||||||
assert kind == "decode", f"kind={kind!r}, expected 'decode'"
|
assert kind == "decode_cube_sp_pe_sp", (
|
||||||
|
f"kind={kind!r}, expected 'decode_cube_sp_pe_sp'"
|
||||||
|
)
|
||||||
assert params.get("C") == 8
|
assert params.get("C") == 8
|
||||||
assert params.get("P") == 8
|
assert params.get("P") == 8
|
||||||
assert params.get("sub_w") == 4
|
|
||||||
assert params.get("T_q") == 1
|
assert params.get("T_q") == 1
|
||||||
assert params.get("S_kv") == 131_072
|
assert params.get("S_kv") == 131_072
|
||||||
assert params.get("d_head") == 128
|
assert params.get("d_head") == 128
|
||||||
|
|||||||
Reference in New Issue
Block a user