gqa(decode-4cases): Case 3 — Cube-Repl × PE-SP (intra-CUBE AR only; 8× memory) (5C.C)
Adds the third 4-cases panel: KV replicated per cube (8× memory waste), PEs SP on S_kv within each cube, intra-CUBE 8-way reduce on (m, ℓ, O), and no inter-CUBE comm (every cube ends with full answer; designated writer = cube 0). Reuses the row-chain + col-bridge intra-CUBE pattern that anchors Case 4 (21 ipcq_copy per cube × 8 cubes = 168 total). 12 tests pass (4 Case 4 + 4 Case 2 + 4 Case 3). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
"""GQA decode kernel — Case 3 (Cube-Repl × PE-SP).
|
||||
|
||||
Per GQA_full_deck.pptx slide 11:
|
||||
- K, V replicated across all 8 cubes (the 8× memory waste).
|
||||
- PEs SP on S_kv inside each cube: each PE attends to its
|
||||
``S_local = S_kv / P`` slice.
|
||||
- Intra-CUBE 8-way reduce on the partial ``(m, ℓ, O)`` triple
|
||||
(row chain + col bridge over the 2×4 PE grid, same structural
|
||||
pattern as Case 4's intra-CUBE phase).
|
||||
- NO inter-CUBE communication — every cube ends with the full
|
||||
answer (8× redundant compute is the inherent cost of Case 3).
|
||||
- Designated writer: only PE 0 of CUBE 0 stores ``O`` to avoid
|
||||
8 redundant DMA writes.
|
||||
|
||||
Tensor layout:
|
||||
Q : (T_q, h_q · d_head) replicated on every rank.
|
||||
K : (S_kv, h_kv · d_head) with cube=replicate, pe=row_wise —
|
||||
each PE owns its (S_local, h_kv·d_head) shard within its cube.
|
||||
V : same as K.
|
||||
O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores.
|
||||
|
||||
Topology / SFR:
|
||||
- ``configure_sfr_intercube_multisip`` provides the intra_* /
|
||||
E/W/N/S namespaces; only the intra_* lanes are exercised here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from kernbench.benches._gqa_attention_decode_long import _merge_running
|
||||
|
||||
|
||||
TILE_S_KV = 1024 # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width).
|
||||
|
||||
|
||||
def gqa_attention_decode_cube_repl_pe_sp_kernel(
|
||||
q_ptr: int,
|
||||
k_ptr: int,
|
||||
v_ptr: int,
|
||||
o_ptr: int,
|
||||
T_q: int,
|
||||
S_kv: int,
|
||||
h_q: int,
|
||||
h_kv: int,
|
||||
d_head: int,
|
||||
C: int,
|
||||
P: int,
|
||||
*,
|
||||
tl,
|
||||
) -> None:
|
||||
"""Case-3 decode: PE-SP attention; replicated KV per cube; intra-CUBE AR only."""
|
||||
G = h_q // h_kv
|
||||
S_local = S_kv // P # each PE owns S_kv/P (cube=replicate, pe=row_wise)
|
||||
pe_id = tl.program_id(axis=0)
|
||||
cube_id = tl.program_id(axis=1)
|
||||
|
||||
# ── Local attention (S_kv-axis tile sweep, ADR-0063 §A.2) ──
|
||||
Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16")
|
||||
n_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV
|
||||
KV_ROW_BYTES = d_head * 2 # f16
|
||||
|
||||
tile_s0 = min(TILE_S_KV, S_local)
|
||||
K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16")
|
||||
V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16")
|
||||
scores = tl.dot(Q, K_T)
|
||||
m_local = tl.max(scores, axis=-1)
|
||||
centered = scores - m_local
|
||||
exp_scores = tl.exp(centered)
|
||||
l_local = tl.sum(exp_scores, axis=-1)
|
||||
O_local = tl.dot(exp_scores, V)
|
||||
|
||||
for tile_idx in range(1, n_tiles):
|
||||
tile_start = tile_idx * TILE_S_KV
|
||||
tile_s = min(TILE_S_KV, S_local - tile_start)
|
||||
with tl.scratch_scope():
|
||||
K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES,
|
||||
shape=(d_head, tile_s), dtype="f16")
|
||||
V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES,
|
||||
shape=(tile_s, d_head), dtype="f16")
|
||||
scores_t = tl.dot(Q, K_T_t)
|
||||
m_tile = tl.max(scores_t, axis=-1)
|
||||
centered_t = scores_t - m_tile
|
||||
exp_scores_t = tl.exp(centered_t)
|
||||
l_tile = tl.sum(exp_scores_t, axis=-1)
|
||||
O_tile = tl.dot(exp_scores_t, V_t)
|
||||
m_new, l_new, O_new = _merge_running(
|
||||
m_local, l_local, O_local, m_tile, l_tile, O_tile, tl=tl,
|
||||
)
|
||||
tl.copy_to(m_local, m_new)
|
||||
tl.copy_to(l_local, l_new)
|
||||
tl.copy_to(O_local, O_new)
|
||||
|
||||
# ── Intra-CUBE 8-way reduce-to-PE0 (row chain + col bridge) ──
|
||||
PE_GRID_COLS = 4
|
||||
pe_col = pe_id % PE_GRID_COLS
|
||||
pe_row = pe_id // PE_GRID_COLS
|
||||
pe_cols_used = min(PE_GRID_COLS, P)
|
||||
pe_rows_used = (P + PE_GRID_COLS - 1) // PE_GRID_COLS
|
||||
|
||||
if pe_cols_used > 1:
|
||||
if pe_col < pe_cols_used - 1:
|
||||
with tl.scratch_scope():
|
||||
m_other = tl.recv(dir="intra_E", shape=m_local.shape, dtype="f16")
|
||||
l_other = tl.recv(dir="intra_E", shape=l_local.shape, dtype="f16")
|
||||
O_other = tl.recv(dir="intra_E", shape=O_local.shape, dtype="f16")
|
||||
m_new, l_new, O_new = _merge_running(
|
||||
m_local, l_local, O_local, m_other, l_other, O_other, tl=tl,
|
||||
)
|
||||
tl.copy_to(m_local, m_new)
|
||||
tl.copy_to(l_local, l_new)
|
||||
tl.copy_to(O_local, O_new)
|
||||
if pe_col > 0:
|
||||
tl.send(dir="intra_W", src=m_local)
|
||||
tl.send(dir="intra_W", src=l_local)
|
||||
tl.send(dir="intra_W", src=O_local)
|
||||
|
||||
if pe_col == 0 and pe_rows_used > 1:
|
||||
if pe_row < pe_rows_used - 1:
|
||||
with tl.scratch_scope():
|
||||
m_other = tl.recv(dir="intra_S", shape=m_local.shape, dtype="f16")
|
||||
l_other = tl.recv(dir="intra_S", shape=l_local.shape, dtype="f16")
|
||||
O_other = tl.recv(dir="intra_S", shape=O_local.shape, dtype="f16")
|
||||
m_new, l_new, O_new = _merge_running(
|
||||
m_local, l_local, O_local, m_other, l_other, O_other, tl=tl,
|
||||
)
|
||||
tl.copy_to(m_local, m_new)
|
||||
tl.copy_to(l_local, l_new)
|
||||
tl.copy_to(O_local, O_new)
|
||||
if pe_row > 0:
|
||||
tl.send(dir="intra_N", src=m_local)
|
||||
tl.send(dir="intra_N", src=l_local)
|
||||
tl.send(dir="intra_N", src=O_local)
|
||||
|
||||
# ── Final normalise + store (designated writer: cube 0, PE 0) ──
|
||||
if pe_id == 0 and cube_id == 0:
|
||||
O_final = O_local / l_local
|
||||
tl.store(o_ptr, O_final)
|
||||
@@ -32,6 +32,9 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import (
|
||||
gqa_attention_decode_cube_repl_pe_sp_kernel,
|
||||
)
|
||||
from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import (
|
||||
gqa_attention_decode_cube_repl_pe_tp_kernel,
|
||||
)
|
||||
@@ -58,7 +61,8 @@ _SWEEP_JSON = _OUTPUT_DIR / "sweep.json"
|
||||
_PANELS = (
|
||||
"single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 ★ optimal
|
||||
"single_kv_group_decode_gqa_cube_repl_pe_tp", # Case 2 (no comm; 8× memory)
|
||||
# Cases 1, 3 to be added by 5C.A/C
|
||||
"single_kv_group_decode_gqa_cube_repl_pe_sp", # Case 3 (intra-CUBE AR only; 8× memory)
|
||||
# Case 1 to be added by 5C.A
|
||||
)
|
||||
|
||||
# Each entry: (kind, panel-specific params).
|
||||
@@ -82,6 +86,14 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = {
|
||||
"T_q": 1, "S_kv": 131_072,
|
||||
"d_head": 128, "h_q": 8, "h_kv": 1,
|
||||
}),
|
||||
"single_kv_group_decode_gqa_cube_repl_pe_sp": ("decode_cube_repl_pe_sp", {
|
||||
# Case 3: K, V replicated per cube (8× memory); PEs SP on S_kv
|
||||
# within each cube. Intra-CUBE 8-way reduce; no inter-CUBE comm
|
||||
# (every cube ends with full answer; designated writer = cube 0).
|
||||
"C": 8, "P": 8,
|
||||
"T_q": 1, "S_kv": 131_072,
|
||||
"d_head": 128, "h_q": 8, "h_kv": 1,
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@@ -119,6 +131,39 @@ def _run_decode_panel_cube_repl_pe_tp(
|
||||
)
|
||||
|
||||
|
||||
def _run_decode_panel_cube_repl_pe_sp(
|
||||
ctx, *, panel: str, C: int, P: int,
|
||||
T_q: int, S_kv: int,
|
||||
d_head: int, h_q: int, h_kv: int,
|
||||
) -> None:
|
||||
"""Case 3 runner: K, V replicated per cube; PEs SP on S_kv within cube.
|
||||
|
||||
DPPolicy models the cluster-wide 8× memory waste — every cube
|
||||
holds full K, V in its HBM region, then splits the S_kv axis
|
||||
row_wise across its 8 PEs. The kernel does an intra-CUBE 8-way
|
||||
reduce on (m, ℓ, O); only cube 0's PE 0 writes the output.
|
||||
"""
|
||||
configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg())
|
||||
dp_full = DPPolicy(cube="replicate", pe="replicate",
|
||||
num_cubes=C, num_pes=P)
|
||||
dp_kv = DPPolicy(cube="replicate", pe="row_wise",
|
||||
num_cubes=C, num_pes=P)
|
||||
q = ctx.zeros((T_q, h_q * d_head),
|
||||
dtype="f16", dp=dp_full, name=f"{panel}_q")
|
||||
k = ctx.zeros((S_kv, h_kv * d_head),
|
||||
dtype="f16", dp=dp_kv, name=f"{panel}_k")
|
||||
v = ctx.zeros((S_kv, h_kv * d_head),
|
||||
dtype="f16", dp=dp_kv, name=f"{panel}_v")
|
||||
o = ctx.empty((T_q, h_q * d_head),
|
||||
dtype="f16", dp=dp_full, name=f"{panel}_o")
|
||||
ctx.launch(
|
||||
panel, gqa_attention_decode_cube_repl_pe_sp_kernel,
|
||||
q, k, v, o,
|
||||
T_q, S_kv, h_q, h_kv, d_head, C, P,
|
||||
_auto_dim_remap=False,
|
||||
)
|
||||
|
||||
|
||||
def _make_bench_fn(panel: str):
|
||||
kind, params = _PANEL_DISPATCH[panel]
|
||||
|
||||
@@ -127,6 +172,8 @@ def _make_bench_fn(panel: str):
|
||||
_run_decode_panel(ctx, panel=panel, **params)
|
||||
elif kind == "decode_cube_repl_pe_tp":
|
||||
_run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params)
|
||||
elif kind == "decode_cube_repl_pe_sp":
|
||||
_run_decode_panel_cube_repl_pe_sp(ctx, panel=panel, **params)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"milestone-gqa-decode-4cases panel {panel!r} has "
|
||||
|
||||
Reference in New Issue
Block a user