gqa(decode-4cases): Case 3 — Cube-Repl × PE-SP (intra-CUBE AR only; 8× memory) (5C.C)

Adds the third 4-cases panel: KV replicated per cube (8× memory
waste), PEs SP on S_kv within each cube, intra-CUBE 8-way reduce on
(m, ℓ, O), and no inter-CUBE comm (every cube ends with full answer;
designated writer = cube 0). Reuses the row-chain + col-bridge
intra-CUBE pattern that anchors Case 4 (21 ipcq_copy per cube × 8
cubes = 168 total). 12 tests pass (4 Case 4 + 4 Case 2 + 4 Case 3).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/kernbench/benches/_gqa_attention_decode_cube_repl_pe_sp.py

@@ -0,0 +1,135 @@
+"""GQA decode kernel — Case 3 (Cube-Repl × PE-SP).
+
+Per GQA_full_deck.pptx slide 11:
+  - K, V replicated across all 8 cubes (the 8× memory waste).
+  - PEs SP on S_kv inside each cube: each PE attends to its
+    ``S_local = S_kv / P`` slice.
+  - Intra-CUBE 8-way reduce on the partial ``(m, ℓ, O)`` triple
+    (row chain + col bridge over the 2×4 PE grid, same structural
+    pattern as Case 4's intra-CUBE phase).
+  - NO inter-CUBE communication — every cube ends with the full
+    answer (8× redundant compute is the inherent cost of Case 3).
+  - Designated writer: only PE 0 of CUBE 0 stores ``O`` to avoid
+    8 redundant DMA writes.
+
+Tensor layout:
+  Q : (T_q, h_q · d_head) replicated on every rank.
+  K : (S_kv, h_kv · d_head) with cube=replicate, pe=row_wise —
+      each PE owns its (S_local, h_kv·d_head) shard within its cube.
+  V : same as K.
+  O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores.
+
+Topology / SFR:
+  - ``configure_sfr_intercube_multisip`` provides the intra_* /
+    E/W/N/S namespaces; only the intra_* lanes are exercised here.
+"""
+from __future__ import annotations
+
+from kernbench.benches._gqa_attention_decode_long import _merge_running
+
+
+TILE_S_KV = 1024  # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width).
+
+
+def gqa_attention_decode_cube_repl_pe_sp_kernel(
+    q_ptr: int,
+    k_ptr: int,
+    v_ptr: int,
+    o_ptr: int,
+    T_q: int,
+    S_kv: int,
+    h_q: int,
+    h_kv: int,
+    d_head: int,
+    C: int,
+    P: int,
+    *,
+    tl,
+) -> None:
+    """Case-3 decode: PE-SP attention; replicated KV per cube; intra-CUBE AR only."""
+    G = h_q // h_kv
+    S_local = S_kv // P  # each PE owns S_kv/P (cube=replicate, pe=row_wise)
+    pe_id = tl.program_id(axis=0)
+    cube_id = tl.program_id(axis=1)
+
+    # ── Local attention (S_kv-axis tile sweep, ADR-0063 §A.2) ──
+    Q = tl.load(q_ptr, shape=(G * T_q, d_head), dtype="f16")
+    n_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV
+    KV_ROW_BYTES = d_head * 2  # f16
+
+    tile_s0 = min(TILE_S_KV, S_local)
+    K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16")
+    V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16")
+    scores = tl.dot(Q, K_T)
+    m_local = tl.max(scores, axis=-1)
+    centered = scores - m_local
+    exp_scores = tl.exp(centered)
+    l_local = tl.sum(exp_scores, axis=-1)
+    O_local = tl.dot(exp_scores, V)
+
+    for tile_idx in range(1, n_tiles):
+        tile_start = tile_idx * TILE_S_KV
+        tile_s = min(TILE_S_KV, S_local - tile_start)
+        with tl.scratch_scope():
+            K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES,
+                            shape=(d_head, tile_s), dtype="f16")
+            V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES,
+                          shape=(tile_s, d_head), dtype="f16")
+            scores_t = tl.dot(Q, K_T_t)
+            m_tile = tl.max(scores_t, axis=-1)
+            centered_t = scores_t - m_tile
+            exp_scores_t = tl.exp(centered_t)
+            l_tile = tl.sum(exp_scores_t, axis=-1)
+            O_tile = tl.dot(exp_scores_t, V_t)
+            m_new, l_new, O_new = _merge_running(
+                m_local, l_local, O_local, m_tile, l_tile, O_tile, tl=tl,
+            )
+            tl.copy_to(m_local, m_new)
+            tl.copy_to(l_local, l_new)
+            tl.copy_to(O_local, O_new)
+
+    # ── Intra-CUBE 8-way reduce-to-PE0 (row chain + col bridge) ──
+    PE_GRID_COLS = 4
+    pe_col = pe_id % PE_GRID_COLS
+    pe_row = pe_id // PE_GRID_COLS
+    pe_cols_used = min(PE_GRID_COLS, P)
+    pe_rows_used = (P + PE_GRID_COLS - 1) // PE_GRID_COLS
+
+    if pe_cols_used > 1:
+        if pe_col < pe_cols_used - 1:
+            with tl.scratch_scope():
+                m_other = tl.recv(dir="intra_E", shape=m_local.shape, dtype="f16")
+                l_other = tl.recv(dir="intra_E", shape=l_local.shape, dtype="f16")
+                O_other = tl.recv(dir="intra_E", shape=O_local.shape, dtype="f16")
+                m_new, l_new, O_new = _merge_running(
+                    m_local, l_local, O_local, m_other, l_other, O_other, tl=tl,
+                )
+                tl.copy_to(m_local, m_new)
+                tl.copy_to(l_local, l_new)
+                tl.copy_to(O_local, O_new)
+        if pe_col > 0:
+            tl.send(dir="intra_W", src=m_local)
+            tl.send(dir="intra_W", src=l_local)
+            tl.send(dir="intra_W", src=O_local)
+
+    if pe_col == 0 and pe_rows_used > 1:
+        if pe_row < pe_rows_used - 1:
+            with tl.scratch_scope():
+                m_other = tl.recv(dir="intra_S", shape=m_local.shape, dtype="f16")
+                l_other = tl.recv(dir="intra_S", shape=l_local.shape, dtype="f16")
+                O_other = tl.recv(dir="intra_S", shape=O_local.shape, dtype="f16")
+                m_new, l_new, O_new = _merge_running(
+                    m_local, l_local, O_local, m_other, l_other, O_other, tl=tl,
+                )
+                tl.copy_to(m_local, m_new)
+                tl.copy_to(l_local, l_new)
+                tl.copy_to(O_local, O_new)
+        if pe_row > 0:
+            tl.send(dir="intra_N", src=m_local)
+            tl.send(dir="intra_N", src=l_local)
+            tl.send(dir="intra_N", src=O_local)
+
+    # ── Final normalise + store (designated writer: cube 0, PE 0) ──
+    if pe_id == 0 and cube_id == 0:
+        O_final = O_local / l_local
+        tl.store(o_ptr, O_final)

src/kernbench/benches/milestone_gqa_decode_4cases.py

@@ -32,6 +32,9 @@ import json
 import os
 from pathlib import Path
 
+from kernbench.benches._gqa_attention_decode_cube_repl_pe_sp import (
+    gqa_attention_decode_cube_repl_pe_sp_kernel,
+)
 from kernbench.benches._gqa_attention_decode_cube_repl_pe_tp import (
     gqa_attention_decode_cube_repl_pe_tp_kernel,
 )
@@ -58,7 +61,8 @@ _SWEEP_JSON = _OUTPUT_DIR / "sweep.json"
 _PANELS = (
     "single_kv_group_decode_gqa_cube_sp_pe_sp",      # Case 4 ★ optimal
     "single_kv_group_decode_gqa_cube_repl_pe_tp",    # Case 2 (no comm; 8× memory)
-    # Cases 1, 3 to be added by 5C.A/C
+    "single_kv_group_decode_gqa_cube_repl_pe_sp",    # Case 3 (intra-CUBE AR only; 8× memory)
+    # Case 1 to be added by 5C.A
 )
 
 # Each entry: (kind, panel-specific params).
@@ -82,6 +86,14 @@ _PANEL_DISPATCH: dict[str, tuple[str, dict]] = {
         "T_q": 1, "S_kv": 131_072,
         "d_head": 128, "h_q": 8, "h_kv": 1,
     }),
+    "single_kv_group_decode_gqa_cube_repl_pe_sp": ("decode_cube_repl_pe_sp", {
+        # Case 3: K, V replicated per cube (8× memory); PEs SP on S_kv
+        # within each cube. Intra-CUBE 8-way reduce; no inter-CUBE comm
+        # (every cube ends with full answer; designated writer = cube 0).
+        "C": 8, "P": 8,
+        "T_q": 1, "S_kv": 131_072,
+        "d_head": 128, "h_q": 8, "h_kv": 1,
+    }),
 }
 
 
@@ -119,6 +131,39 @@ def _run_decode_panel_cube_repl_pe_tp(
     )
 
 
+def _run_decode_panel_cube_repl_pe_sp(
+    ctx, *, panel: str, C: int, P: int,
+    T_q: int, S_kv: int,
+    d_head: int, h_q: int, h_kv: int,
+) -> None:
+    """Case 3 runner: K, V replicated per cube; PEs SP on S_kv within cube.
+
+    DPPolicy models the cluster-wide 8× memory waste — every cube
+    holds full K, V in its HBM region, then splits the S_kv axis
+    row_wise across its 8 PEs. The kernel does an intra-CUBE 8-way
+    reduce on (m, ℓ, O); only cube 0's PE 0 writes the output.
+    """
+    configure_sfr_intercube_multisip(ctx.engine, ctx.spec, _ccl_cfg())
+    dp_full = DPPolicy(cube="replicate", pe="replicate",
+                       num_cubes=C, num_pes=P)
+    dp_kv = DPPolicy(cube="replicate", pe="row_wise",
+                     num_cubes=C, num_pes=P)
+    q = ctx.zeros((T_q, h_q * d_head),
+                  dtype="f16", dp=dp_full, name=f"{panel}_q")
+    k = ctx.zeros((S_kv, h_kv * d_head),
+                  dtype="f16", dp=dp_kv, name=f"{panel}_k")
+    v = ctx.zeros((S_kv, h_kv * d_head),
+                  dtype="f16", dp=dp_kv, name=f"{panel}_v")
+    o = ctx.empty((T_q, h_q * d_head),
+                  dtype="f16", dp=dp_full, name=f"{panel}_o")
+    ctx.launch(
+        panel, gqa_attention_decode_cube_repl_pe_sp_kernel,
+        q, k, v, o,
+        T_q, S_kv, h_q, h_kv, d_head, C, P,
+        _auto_dim_remap=False,
+    )
+
+
 def _make_bench_fn(panel: str):
     kind, params = _PANEL_DISPATCH[panel]
 
@@ -127,6 +172,8 @@ def _make_bench_fn(panel: str):
             _run_decode_panel(ctx, panel=panel, **params)
         elif kind == "decode_cube_repl_pe_tp":
             _run_decode_panel_cube_repl_pe_tp(ctx, panel=panel, **params)
+        elif kind == "decode_cube_repl_pe_sp":
+            _run_decode_panel_cube_repl_pe_sp(ctx, panel=panel, **params)
         else:
             raise RuntimeError(
                 f"milestone-gqa-decode-4cases panel {panel!r} has "

tests/attention/test_milestone_gqa_decode_4cases.py

@@ -39,6 +39,7 @@ from kernbench.topology.builder import resolve_topology
 TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml"
 
 _CASE4_PANEL = "single_kv_group_decode_gqa_cube_sp_pe_sp"
+_CASE3_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_sp"
 _CASE2_PANEL = "single_kv_group_decode_gqa_cube_repl_pe_tp"
 _CUBE_RE = re.compile(r"\bcube(\d+)\b")
 
@@ -265,6 +266,124 @@ def test_case2_single_dma_write_at_cube_0():
     )
 
 
+# ── Case 3 (Cube-Repl × PE-SP) ──────────────────────────────────────
+
+
+def _run_case3_smoke(*, S_kv: int):
+    """Drive the Case 3 decode panel via the case-specific runner.
+
+    Case 3 = Cube-Repl × PE-SP. K, V replicated per cube; S_kv split
+    8-way across PEs within each cube. Intra-CUBE 8-way reduce on
+    (m, ℓ, O); no inter-CUBE comm (every cube ends with full answer).
+    """
+    from kernbench.benches.milestone_gqa_decode_4cases import (
+        _run_decode_panel_cube_repl_pe_sp,
+    )
+    topo = resolve_topology(str(TOPOLOGY_DEFAULT))
+
+    def _bench_fn(ctx):
+        _run_decode_panel_cube_repl_pe_sp(
+            ctx, panel=_CASE3_PANEL,
+            C=8, P=8,
+            T_q=1, S_kv=S_kv,
+            d_head=128, h_q=8, h_kv=1,
+        )
+
+    return run_bench(
+        topology=topo, bench_fn=_bench_fn,
+        device=resolve_device(None),
+        engine_factory=_engine_factory,
+    )
+
+
+# ── Case 3 — T1: panel registered ───────────────────────────────────
+
+
+def test_case3_panel_registered():
+    """The Case 3 panel must be in the bench's ``_PANELS`` +
+    ``_PANEL_DISPATCH`` with the expected single-KV-group dims.
+
+    Case 3: Cube-Repl × PE-SP. K, V replicated per cube; PEs SP on
+    S_kv. Intra-CUBE 8-way AR; no inter-CUBE comm.
+    """
+    from kernbench.benches.milestone_gqa_decode_4cases import (
+        _PANEL_DISPATCH,
+        _PANELS,
+    )
+    assert _CASE3_PANEL in _PANELS, (
+        f"{_CASE3_PANEL!r} not in _PANELS; got {_PANELS}"
+    )
+    assert _CASE3_PANEL in _PANEL_DISPATCH
+    kind, params = _PANEL_DISPATCH[_CASE3_PANEL]
+    assert kind == "decode_cube_repl_pe_sp"
+    assert params.get("C") == 8
+    assert params.get("P") == 8
+    assert params.get("T_q") == 1
+    assert params.get("S_kv") == 131_072
+    assert params.get("d_head") == 128
+    assert params.get("h_q") == 8
+    assert params.get("h_kv") == 1
+
+
+# ── Case 3 — T2: smoke runner completes ─────────────────────────────
+
+
+def test_case3_runner_smoke():
+    """Case 3 runner drives the new kernel to completion at smoke S_kv."""
+    result = _run_case3_smoke(S_kv=8192)
+    assert result.completion.ok, (
+        f"Case 3 decode smoke at C=8 P=8 must complete; "
+        f"got {result.completion}"
+    )
+
+
+# ── Case 3 — T3: intra-CUBE-only AR (168 ipcq, no inter-CUBE) ───────
+
+
+def test_case3_intra_cube_ar_only_ipcq():
+    """Case 3 reduce pattern: per-CUBE 8-way PE-SP AR (same structural
+    cost as Case 4's intra-CUBE phase = 21 ipcq_copy per cube), and
+    NO inter-CUBE traffic (each cube has a full copy of KV).
+
+      per-CUBE intra (2×4 PE grid):
+        row chain along intra_W: cols 1,2,3 each row × 2 rows ×
+          3 tensors (m, ℓ, O)                          = 18
+        col bridge along intra_N: pe4 only × 3 tensors =  3
+        per-CUBE intra total                           = 21
+        × 8 CUBEs                                      = 168
+
+      inter-CUBE: 0 (replicated KV ⇒ no AllReduce needed).
+
+      Grand total: 168.
+    """
+    result = _run_case3_smoke(S_kv=8192)
+    assert result.completion.ok
+    n_copy = _count(result.engine.op_log, "ipcq_copy")
+    assert n_copy == 168, (
+        f"Case 3 expected 168 ipcq_copy (intra-CUBE only, no inter-CUBE); "
+        f"got {n_copy}"
+    )
+
+
+# ── Case 3 — T4: single dma_write from cube 0 (designated writer) ───
+
+
+def test_case3_single_dma_write_at_cube_0():
+    """Every cube ends with the full answer after intra-CUBE AR; only
+    the designated writer (cube 0, PE 0) stores O to avoid 8 redundant
+    DMAs.
+    """
+    result = _run_case3_smoke(S_kv=8192)
+    assert result.completion.ok
+    cubes = _dma_write_cubes(result.engine.op_log)
+    assert cubes, "expected at least one dma_write for the final O store"
+    distinct = set(cubes)
+    assert distinct == {0}, (
+        f"Case 3 designated writer must be cube 0; "
+        f"got cubes={sorted(distinct)}"
+    )
+
+
 # ── Case 4 — T4 (existing, kept) ────────────────────────────────────