ywkang/kernbench2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gqa(decode-4cases): Case 2 — Cube-Repl × PE-TP (no comm; 8× memory) (5C.B)

Browse Source 
Second case in the GQA decode 4-cases comparative study per
GQA_full_deck.pptx slide 11. Case 2 replicates K, V across all 8
cubes × 8 PEs (the slide-11 8 KB/tok/PE memory waste) and has zero
inter-rank comm. For B=1 (single-user decode default), only PE 0
of CUBE 0 has work — the inherent PE-TP waste slide 11 calls out.

Changes:
- New kernel: src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py
  Simplest of the 4 cases. Active rank loads full Q/K/V from HBM,
  computes attention via S_kv tile sweep with online-softmax merge,
  writes O. All non-(0,0) ranks early-return. No tl.send/recv.
- src/kernbench/benches/milestone_gqa_decode_4cases.py:
    - Add panel single_kv_group_decode_gqa_cube_repl_pe_tp (Case 2)
      to _PANELS and _PANEL_DISPATCH.
    - Add _run_decode_panel_cube_repl_pe_tp helper: DPPolicy K/V/Q/O
      = cube=replicate, pe=replicate (models 8× memory waste).
    - Extend _make_bench_fn to dispatch kind="decode_cube_repl_pe_tp"
      to the new runner.
- tests/attention/test_milestone_gqa_decode_4cases.py:
    4 new tests assert Case 2 contract: panel registered, smoke
    completion, zero ipcq_copy (no comm), single dma_write from cube 0.

Verification: 8/8 tests pass (4 Case 4 anchor + 4 new Case 2).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Mukesh Garg
2026-06-15 14:59:58 -07:00
parent 65c365f858
commit 5672c8f3ef
3 changed files with 259 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_tp.py
+101
View File
@@ -0,0 +1,101 @@
"""GQA decode kernel — Case 2 (Cube-Repl × PE-TP).
Per GQA_full_deck.pptx slide 11:
- K, V replicated across all 8 cubes × 8 PEs (the 8 KB/tok/PE
memory waste — this is the inherent cost Case 2 demonstrates).
- PEs nominally split on the batch dim (PE-TP). For B=1 (single-
user decode, the slide-11 default), only PE 0 of CUBE 0 has
work; the other 63 ranks idle.
- NO inter-rank communication (each rank has full KV — slide 11
lists comm cost as "none").
This kernel is the simplest of the 4 cases by design: one active
rank does the full attention locally; everyone else early-returns.
The DPPolicy at the call site models the cluster-wide 8× memory
waste even though only one rank reads from HBM.
Tensor layout (B=1):
Q : (T_q, h_q · d_head) replicated on every rank; loaded as
(G · T_q, d_head) on the active rank.
K : (S_kv, h_kv · d_head) replicated on every rank.
V : (S_kv, h_kv · d_head) replicated on every rank.
O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores.
Topology / SFR:
- configure_sfr_intercube_multisip is fine but not strictly required
(no inter-rank sends/recvs happen).
"""
from __future__ import annotations
TILE_S_KV = 1024 # match decode_long — per-tile S_kv width (ADR-0063 §A.2).
def gqa_attention_decode_cube_repl_pe_tp_kernel(
q_ptr: int,
k_ptr: int,
v_ptr: int,
o_ptr: int,
T_q: int,
S_kv: int,
h_q: int,
h_kv: int,
d_head: int,
C: int,
P: int,
*,
tl,
) -> None:
"""Case-2 decode: single-rank attention; full KV per rank; no comm."""
pe_id = tl.program_id(axis=0)
cube_id = tl.program_id(axis=1)
# B=1 single-user decode + PE-TP: only one rank has work.
# Slide-11 acknowledges this PE-TP waste at B=1.
if pe_id != 0 or cube_id != 0:
return
G = h_q // h_kv
n_tiles = (S_kv + TILE_S_KV - 1) // TILE_S_KV
KV_ROW_BYTES = d_head * 2 # f16
# ── Load Q (full; replicated; M-folded for GQA reuse) ──
Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16")
# ── Tile 0: bootstrap persistent (m, ℓ, O) ──
tile_s0 = min(TILE_S_KV, S_kv)
K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16")
V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16")
scores = tl.dot(Q, K_T)
m_local = tl.max(scores, axis=-1)
centered = scores - m_local
exp_scores = tl.exp(centered)
l_local = tl.sum(exp_scores, axis=-1)
O_local = tl.dot(exp_scores, V)
# ── Tiles 1..N: fold via online-softmax merge in scratch_scope ──
for tile_idx in range(1, n_tiles):
tile_start = tile_idx * TILE_S_KV
tile_s = min(TILE_S_KV, S_kv - tile_start)
with tl.scratch_scope():
K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES,
shape=(d_head, tile_s), dtype="f16")
V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES,
shape=(tile_s, d_head), dtype="f16")
scores_t = tl.dot(Q, K_T_t)
m_tile = tl.max(scores_t, axis=-1)
centered_t = scores_t - m_tile
exp_scores_t = tl.exp(centered_t)
l_tile = tl.sum(exp_scores_t, axis=-1)
O_tile = tl.dot(exp_scores_t, V_t)
m_new = tl.maximum(m_local, m_tile)
scale_old = tl.exp(m_local - m_new)
scale_new = tl.exp(m_tile - m_new)
l_new = l_local * scale_old + l_tile * scale_new
O_new = O_local * scale_old + O_tile * scale_new
tl.copy_to(m_local, m_new)
tl.copy_to(l_local, l_new)
tl.copy_to(O_local, O_new)
# ── Final normalise + store ──
O_final = O_local / l_local
tl.store(o_ptr, O_final)
src/kernbench/benches/milestone_gqa_decode_4cases.py
+51 -4
View File
@@ -32,12 +32,17 @@ import json
import os
from pathlib import Path
from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import (
gqa_attention_decode_cube_repl_pe_tp_kernel,
)
from kernbench.benches.milestone_gqa_headline import (
_ccl_cfg,
_run_decode_panel,
_summarize_op_log,
)
from kernbench.benches.registry import bench
from kernbench.ccl.sfr_config import configure_sfr_intercube_multisip
from kernbench.policy.placement.dp import DPPolicy
_OUTPUT_DIR = (
Path(__file__).resolve().parent
@@ -51,11 +56,12 @@ _SWEEP_JSON = _OUTPUT_DIR / "sweep.json"
_PANELS = (
"single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4
# Cases 1-3 to be added by 5C.A/B/C
"single_kv_group_decode_gqa_cube_sp_pe_sp", # Case 4 ★ optimal
"single_kv_group_decode_gqa_cube_repl_pe_tp", # Case 2 (no comm; 8× memory)
# Cases 1, 3 to be added by 5C.A/C
)
# Each entry: (kind, panel-specific params for _run_decode_panel).
# Each entry: (kind, panel-specific params).
# LLaMA-3.1-70B single-KV-head group target:
# 1 KV head, h_q = 8 (G = 8 group), d_head = 128
# 8 cubes (head-parallel group), 8 PEs/cube
@@ -68,22 +74,63 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = {
"T_q": 1, "S_kv": 131_072,
"d_head": 128, "h_q": 8, "h_kv": 1,
}),
"single_kv_group_decode_gqa_cube_repl_pe_tp": ("decode_cube_repl_pe_tp", {
# Case 2: K, V replicated everywhere (8× memory waste); PEs TP
# on batch. For B=1 only one rank works (slide-11 PE-TP waste).
# No inter-rank communication.
"C": 8, "P": 8,
"T_q": 1, "S_kv": 131_072,
"d_head": 128, "h_q": 8, "h_kv": 1,
}),
}
# ── Per-panel runner ─────────────────────────────────────────────────
def _run_decode_panel_cube_repl_pe_tp(
ctx, *, panel: str, C: int, P: int,
T_q: int, S_kv: int,
d_head: int, h_q: int, h_kv: int,
) -> None:
"""Case 2 runner: K, V replicated everywhere; B=1 single-rank work.
DPPolicy models the cluster-wide memory waste — every rank holds
full K, V in its HBM region. Only PE 0 of CUBE 0 computes (the
kernel early-returns on every other rank), so only one rank reads
from its HBM copy and writes the output.
"""
configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg())
dp_repl = DPPolicy(cube="replicate", pe="replicate",
num_cubes=C, num_pes=P)
q = ctx.zeros((T_q, h_q * d_head),
dtype="f16", dp=dp_repl, name=f"{panel}_q")
k = ctx.zeros((S_kv, h_kv * d_head),
dtype="f16", dp=dp_repl, name=f"{panel}_k")
v = ctx.zeros((S_kv, h_kv * d_head),
dtype="f16", dp=dp_repl, name=f"{panel}_v")
o = ctx.empty((T_q, h_q * d_head),
dtype="f16", dp=dp_repl, name=f"{panel}_o")
ctx.launch(
panel, gqa_attention_decode_cube_repl_pe_tp_kernel,
q, k, v, o,
T_q, S_kv, h_q, h_kv, d_head, C, P,
_auto_dim_remap=False,
)
def _make_bench_fn(panel: str):
kind, params = _PANEL_DISPATCH[panel]
def _bench_fn(ctx):
if kind == "decode":
_run_decode_panel(ctx, panel=panel, **params)
elif kind == "decode_cube_repl_pe_tp":
_run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params)
else:
raise RuntimeError(
f"milestone-gqa-decode-4cases panel {panel!r} has "
f"unsupported kind={kind!r}; only 'decode' is allowed."
f"unsupported kind={kind!r}."
)
return _bench_fn