diff --git a/scripts/paper/paper_plot_gqa_prefill_long_ctx_4cases.py b/scripts/paper/paper_plot_gqa_prefill_long_ctx_4cases.py new file mode 100644 index 0000000..9e5366f --- /dev/null +++ b/scripts/paper/paper_plot_gqa_prefill_long_ctx_4cases.py @@ -0,0 +1,253 @@ +"""Comparative figures for milestone-gqa-prefill-long-ctx-4cases. + +Mirror of paper_plot_gqa_decode_long_ctx_4cases but for the prefill +variant. Reads sweep.json (emitted by ``kernbench run --bench +milestone-gqa-prefill-long-ctx-4cases``) and writes four PNGs into +``docs/report/1H-codesign-paper/figures/``: + + gqa_prefill_long_ctx_4cases_latency.png end-to-end latency per case + gqa_prefill_long_ctx_4cases_traffic.png ipcq/dma op-count breakdown + gqa_prefill_long_ctx_4cases_memory.png per-PE KV bytes per case + gqa_prefill_long_ctx_4cases_parallelism.png active-PE × S_local work load + +Run (after the bench): + GQA_PREFILL_LONG_CTX_4CASES_RUN=1 python -m kernbench.cli.main run \\ + --bench milestone-gqa-prefill-long-ctx-4cases --topology topology.yaml + python scripts/paper/paper_plot_gqa_prefill_long_ctx_4cases.py +""" +from __future__ import annotations + +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +_REPO_ROOT = Path(__file__).resolve().parents[2] +# Sweep JSON + PNGs live together under the bench output dir. +_FIG_DIR = ( + _REPO_ROOT / "src" / "kernbench" / "benches" + / "1H_milestone_output" / "gqa" / "long_ctx" +) +_SWEEP_JSON = _FIG_DIR / "sweep_prefill.json" + +# Panel name → (short label, case ordinal for left-to-right plot order). +_CASE_INFO = { + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_tp": ( + "Case 1\nCube-SP × PE-TP", 1), + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_tp": ( + "Case 2\nCube-Repl × PE-TP", 2), + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_sp": ( + "Case 3\nCube-Repl × PE-SP", 3), + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_sp": ( + "Case 4 ★\nCube-SP × PE-SP", 4), +} + + +def _load() -> list[dict]: + return json.loads(_SWEEP_JSON.read_text())["rows"] + + +def _sorted_by_case(rows: list[dict]) -> list[dict]: + return sorted(rows, key=lambda r: _CASE_INFO[r["panel"]][1]) + + +def _plot_latency(rows: list[dict]) -> Path: + rows = _sorted_by_case(rows) + labels = [_CASE_INFO[r["panel"]][0] for r in rows] + lat_us = [r["latency_ns"] / 1e3 for r in rows] + colors = ["#888", "#888", "#888", "#3b6ea5"] # Case 4 highlighted + fig, ax = plt.subplots(figsize=(8.0, 4.5)) + bars = ax.bar(labels, lat_us, color=colors, width=0.6) + ax.set_ylabel("end-to-end latency (µs)") + ax.set_title( + "Long-context prefill 4-cases — end-to-end latency per case\n" + "LLaMA-3.1-70B single-KV-head group (8 cubes × 8 PEs)" + ) + ax.bar_label(bars, fmt="%.1f", padding=3, fontsize=9) + ax.grid(axis="y", ls=":", alpha=0.5) + ax.set_ylim(0, max(lat_us) * 1.15) + fig.tight_layout() + out = _FIG_DIR / "gqa_prefill_long_ctx_4cases_latency.png" + fig.savefig(out, dpi=150) + plt.close(fig) + return out + + +def _plot_traffic(rows: list[dict]) -> Path: + rows = _sorted_by_case(rows) + labels = [_CASE_INFO[r["panel"]][0] for r in rows] + x = list(range(len(rows))) + keys = ["ipcq_copy_count", "dma_read_count", "dma_write_count"] + disp = ["IPCQ copy", "DMA read", "DMA write"] + colors = ["#c0504d", "#9bbb59", "#8064a2"] + w = 0.25 + fig, ax = plt.subplots(figsize=(9.0, 4.5)) + for i, (k, d, c) in enumerate(zip(keys, disp, colors)): + vals = [r["op_log_summary"][k] for r in rows] + ax.bar([xi + (i - 1) * w for xi in x], vals, width=w, label=d, color=c) + ax.set_xticks(list(x)) + ax.set_xticklabels(labels, fontsize=9) + ax.set_ylabel("op count") + ax.set_title("Long-context prefill 4-cases — op-count breakdown per case") + ax.legend(fontsize=9) + ax.grid(axis="y", ls=":", alpha=0.5) + fig.tight_layout() + out = _FIG_DIR / "gqa_prefill_long_ctx_4cases_traffic.png" + fig.savefig(out, dpi=150) + plt.close(fig) + return out + + +def _s_local_per_pe(panel: str, *, S_kv: int, C: int, P: int) -> int: + """S_local (token count) each PE attends over locally. + + Encodes the cube/pe sharding axes from the panel name: + cube_sp_pe_tp (Case 1): S_kv / C (pe=replicate within cube) + cube_repl_pe_tp (Case 2): S_kv (full S_kv per active PE) + cube_repl_pe_sp (Case 3): S_kv / P (pe=row_wise within cube) + cube_sp_pe_sp (Case 4): S_kv / (C·P) (★ 64-way split) + """ + S_per_cube = S_kv if "cube_repl" in panel else S_kv // C + return S_per_cube // P if "pe_sp" in panel else S_per_cube + + +def _active_pe_count(panel: str, *, C: int, P: int) -> int: + """Number of PEs doing non-idle attention work. + + Prefill T_q≫1 means PE-TP is *useful* (not wasted as in decode): + cube_sp_pe_tp (Case 1): C·P (T_q sharded across all 64 ranks) + cube_repl_pe_tp (Case 2): P (CUBE 0 only; T_q sharded across its P PEs) + cube_repl_pe_sp (Case 3): C·P (all PEs busy but cubes redundant) + cube_sp_pe_sp (Case 4): C·P (all 64 PEs doing unique work) + """ + if "cube_repl" in panel and "pe_tp" in panel: + return P + return C * P + + +def _kv_bytes_per_pe(panel: str, *, S_kv: int, h_kv: int, + d_head: int, C: int, P: int) -> int: + """KV bytes a single PE references (K + V, f16, 2 B/elem).""" + s_local = _s_local_per_pe(panel, S_kv=S_kv, C=C, P=P) + return 2 * s_local * h_kv * d_head * 2 + + +def _plot_memory(rows: list[dict]) -> Path: + """Per-PE KV bytes — Case 4 wins (64-way split).""" + rows = _sorted_by_case(rows) + labels = [_CASE_INFO[r["panel"]][0] for r in rows] + mib_per_pe = [ + _kv_bytes_per_pe( + r["panel"], S_kv=r["S_kv"], h_kv=r["h_kv"], + d_head=r["d_head"], C=r["C"], P=r["P"], + ) / (1024 * 1024) + for r in rows + ] + colors = ["#888", "#c0504d", "#888", "#3b6ea5"] + fig, ax = plt.subplots(figsize=(8.0, 4.5)) + bars = ax.bar(labels, mib_per_pe, color=colors, width=0.6) + ax.set_ylabel("KV bytes per PE (MiB, K + V, f16)") + ax.set_title( + "Long-context prefill 4-cases — KV memory per PE\n" + "(one KV-head group; per-layer, full S_kv state)" + ) + ax.bar_label(bars, fmt="%.3f", padding=3, fontsize=9) + ax.grid(axis="y", ls=":", alpha=0.5) + ax.set_ylim(0, max(mib_per_pe) * 1.15) + fig.tight_layout() + out = _FIG_DIR / "gqa_prefill_long_ctx_4cases_memory.png" + fig.savefig(out, dpi=150) + plt.close(fig) + return out + + +def _t_q_per_pe(panel: str, *, T_q: int, C: int, P: int) -> int: + """T_q row count each active PE computes attention for. + + cube_sp_pe_tp (Case 1): T_q / (C·P) (T_q sharded across all 64 ranks) + cube_repl_pe_tp (Case 2): T_q / P (T_q sharded across CUBE 0's P PEs) + cube_repl_pe_sp (Case 3): T_q (Q replicated on every PE) + cube_sp_pe_sp (Case 4): T_q / C (Q sharded by cube, replicated within) + """ + if "cube_sp" in panel and "pe_tp" in panel: + return T_q // (C * P) + if "cube_repl" in panel and "pe_tp" in panel: + return T_q // P + if "cube_repl" in panel and "pe_sp" in panel: + return T_q + return T_q // C # cube_sp_pe_sp + + +def _s_kv_processed_per_pe(panel: str, *, S_kv: int, C: int, P: int) -> int: + """S_kv tokens each PE actually processes attention over. + + Differs from ``_s_local_per_pe`` (OWNED KV bytes): for cases with + a Ring (Case 1, 4) each PE sees C ring steps so processes more + tokens than it locally owns. + + cube_sp_pe_tp (Case 1): S_kv (Ring + pe=replicate within cube) + cube_repl_pe_tp (Case 2): S_kv (full KV per PE) + cube_repl_pe_sp (Case 3): S_kv / P (pe=row_wise; no Ring) + cube_sp_pe_sp (Case 4): S_kv / P (Ring restores full S_kv/P per PE) + """ + if "cube_sp" in panel and "pe_tp" in panel: + return S_kv + if "cube_repl" in panel and "pe_tp" in panel: + return S_kv + return S_kv // P # both pe_sp cases + + +def _plot_parallelism(rows: list[dict]) -> Path: + """Total compute work (PE × T_q × S_kv token-pairs) — exposes Case 3's + redundancy. Cases 1, 2, 4 all do the same total work (correct + attention over T_q × S_kv); Case 3 does C× more (cubes redundantly + repeat the same compute because K/V is cube-replicated). + """ + rows = _sorted_by_case(rows) + labels = [_CASE_INFO[r["panel"]][0] for r in rows] + total_work = [ + _active_pe_count(r["panel"], C=r["C"], P=r["P"]) + * _t_q_per_pe(r["panel"], T_q=r["T_q"], C=r["C"], P=r["P"]) + * _s_kv_processed_per_pe( + r["panel"], S_kv=r["S_kv"], C=r["C"], P=r["P"], + ) + for r in rows + ] + colors = ["#888", "#888", "#c0504d", "#3b6ea5"] # Case 3 red, Case 4 highlighted + fig, ax = plt.subplots(figsize=(8.0, 4.5)) + bars = ax.bar(labels, total_work, color=colors, width=0.6) + ax.set_ylabel( + "total compute (PE × T_q × S_kv token-pairs; lower ⇒ less wasted work)" + ) + ax.set_title( + "Long-context prefill 4-cases — total compute work across active PEs\n" + "(Case 3 replicates K/V across 8 cubes ⇒ 8× redundant compute)" + ) + ax.bar_label(bars, fmt="%d", padding=3, fontsize=9) + ax.grid(axis="y", ls=":", alpha=0.5) + ax.set_ylim(0, max(total_work) * 1.15) + fig.tight_layout() + out = _FIG_DIR / "gqa_prefill_long_ctx_4cases_parallelism.png" + fig.savefig(out, dpi=150) + plt.close(fig) + return out + + +def main() -> None: + rows = _load() + _FIG_DIR.mkdir(parents=True, exist_ok=True) + p1 = _plot_latency(rows) + p2 = _plot_traffic(rows) + p3 = _plot_memory(rows) + p4 = _plot_parallelism(rows) + print(f"wrote {p1}") + print(f"wrote {p2}") + print(f"wrote {p3}") + print(f"wrote {p4}") + + +if __name__ == "__main__": + main() diff --git a/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_sp.py b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_sp.py new file mode 100644 index 0000000..c5c5d2f --- /dev/null +++ b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_sp.py @@ -0,0 +1,177 @@ +"""GQA prefill kernel — Case 3 (Cube-Repl × PE-SP) at single-KV-head group. + +Per GQA_full_deck.pptx slide 11 (prefill counterpart, T_q ≫ 1): + - K, V replicated across all 8 cubes (the 8× memory waste). + - PEs SP on S_kv inside each cube: each PE attends to its + ``S_local = S_kv / P`` slice. + - Q replicated on every rank (every cube does full T_q × S_local). + - Intra-CUBE 8-way reduce on the partial ``(m, ℓ, O)`` triple + per Q-tile (row chain along intra_W + col bridge along intra_N + over the 2×4 PE grid). + - NO inter-CUBE communication — every cube ends with the full + answer (8× redundant compute is the inherent cost of Case 3). + - Designated writer: only PE 0 of CUBE 0 stores ``O`` to avoid + 8 redundant DMA writes. + +Q-axis tiling (vs decode Case 3): an outer Q-tile loop wraps the +entire per-tile pipeline (Q load → bootstrap → S_kv-tile fold → +intra-CUBE AR → store) in ``tl.scratch_scope``. Peak scratch is +bounded by ``TILE_Q × TILE_S_KV`` instead of ``T_q × S_kv``, so the +kernel runs at large T_q without exceeding the ~1 MB per-PE budget. +Each Q-tile's (m, ℓ, O) is independent of other Q-tiles, so no +state leaks across iterations. + +Tensor layout: + Q : (T_q, h_q · d_head) replicated on every rank; loaded per + Q-tile as ``(TILE_Q, d_head)`` slices (G is folded into the + T_q row axis, total ``G·T_q`` rows). + K : (S_kv, h_kv · d_head) with cube=replicate, pe=row_wise — each + PE owns its ``(S_local, h_kv·d_head)`` shard within its cube. + V : same as K. + O : (T_q, h_q · d_head) — only PE 0 of CUBE 0 stores, one + ``(TILE_Q, d_head)`` slice per outer Q-tile iteration. + +Topology / SFR: + - Requires intra_* lanes; ``configure_sfr_intercube_ring`` (with the + snake submesh) or ``configure_sfr_intercube_multisip`` both install + them. +""" +from __future__ import annotations + + +TILE_S_KV = 64 # per-tile S_kv width (small to keep ``scores`` bounded). +TILE_Q = 64 # per-tile Q row count (outer-loop tile size). + + +def _merge_running(m_local, l_local, O_local, m_other, l_other, O_other, *, tl): + """Online-softmax merge of two partial ``(m, ℓ, O)`` triples.""" + m_new = tl.maximum(m_local, m_other) + scale_old = tl.exp(m_local - m_new) + scale_new = tl.exp(m_other - m_new) + l_new = l_local * scale_old + l_other * scale_new + O_new = O_local * scale_old + O_other * scale_new + return m_new, l_new, O_new + + +def gqa_attention_prefill_long_ctx_cube_repl_pe_sp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-3 prefill: PE-SP S_kv split + Q-tiled intra-CUBE AR.""" + G = h_q // h_kv + S_local = S_kv // P # each PE owns S_kv/P (cube=replicate, pe=row_wise) + pe_id = tl.program_id(axis=0) + cube_id = tl.program_id(axis=1) + + PE_GRID_COLS = 4 + pe_col = pe_id % PE_GRID_COLS + pe_row = pe_id // PE_GRID_COLS + pe_cols_used = min(PE_GRID_COLS, P) + pe_rows_used = (P + PE_GRID_COLS - 1) // PE_GRID_COLS + + KV_ROW_BYTES = d_head * 2 # f16 + Q_ROW_BYTES = d_head * 2 # G is folded into the row axis, not d_head + total_q_rows = G * T_q + n_q_tiles = (total_q_rows + TILE_Q - 1) // TILE_Q + n_kv_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV + + # ── Outer Q-tile loop ── + # Per Q-tile: load Q-slice, run full S_kv attention, intra-CUBE AR, + # store. All scratch is recycled at outer-scope exit. + for q_idx in range(n_q_tiles): + q_start = q_idx * TILE_Q + q_rows = min(TILE_Q, total_q_rows - q_start) + with tl.scratch_scope(): + Q = tl.load(q_ptr + q_start * Q_ROW_BYTES, + shape=(q_rows, d_head), dtype="f16") + + # ── Bootstrap (S_kv tile 0) ── + # K_T, V, scores, exp_scores live in the OUTER (Q-tile) + # scope. They survive the inner S_kv-tile loop (each + # iteration's allocations are recycled) and are freed + # together when the outer scope exits. + tile_s0 = min(TILE_S_KV, S_local) + K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16") + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + exp_scores = tl.exp(scores - m_local) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + # ── S_kv tiles 1..N: fold via online-softmax merge ── + for tile_idx in range(1, n_kv_tiles): + tile_start = tile_idx * TILE_S_KV + tile_s = min(TILE_S_KV, S_local - tile_start) + with tl.scratch_scope(): + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + scores_t = tl.dot(Q, K_T_t) + m_tile = tl.max(scores_t, axis=-1) + exp_scores_t = tl.exp(scores_t - m_tile) + l_tile = tl.sum(exp_scores_t, axis=-1) + O_tile = tl.dot(exp_scores_t, V_t) + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, + m_tile, l_tile, O_tile, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Intra-CUBE 8-way reduce-to-PE0 (row chain + col bridge) ── + # (m_local, l_local, O_local) are still alive in the outer + # scope; the send/recv operate on them directly. + if pe_cols_used > 1: + if pe_col < pe_cols_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_E", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_E", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_E", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, + m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_col > 0: + tl.send(dir="intra_W", src=m_local) + tl.send(dir="intra_W", src=l_local) + tl.send(dir="intra_W", src=O_local) + + if pe_col == 0 and pe_rows_used > 1: + if pe_row < pe_rows_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_S", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_S", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_S", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, + m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_row > 0: + tl.send(dir="intra_N", src=m_local) + tl.send(dir="intra_N", src=l_local) + tl.send(dir="intra_N", src=O_local) + + # ── Store this Q-tile (designated writer: cube 0, PE 0) ── + if pe_id == 0 and cube_id == 0: + O_final = O_local / l_local + tl.store(o_ptr + q_start * Q_ROW_BYTES, O_final) diff --git a/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_tp.py b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_tp.py new file mode 100644 index 0000000..263136f --- /dev/null +++ b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_repl_pe_tp.py @@ -0,0 +1,128 @@ +"""GQA prefill kernel — Case 2 (Cube-Repl × PE-TP) at single-KV-head group. + +Per GQA_full_deck.pptx slide 11 (prefill counterpart, T_q ≫ 1): + - K, V replicated across all 8 cubes (the 8× memory waste — this is + the inherent cost Case 2 demonstrates). PE-replicate within each + cube too: every PE on every cube holds an HBM copy. + - PE-TP on T_q: only CUBE 0 has work; within CUBE 0 the P PEs split + the T_q axis disjointly. CUBEs 1..7 idle (slide-11 "PE-TP only + uses one cube; replicate is pure waste here"). + - NO inter-rank communication (each active rank has full KV and + disjoint T_q rows — slide 11 lists comm cost as "none"). + +Distinction from decode Case 2: decode T_q=1 makes PE-TP wasteful +(only PE 0 of CUBE 0 works → 1 of 64 ranks active). Prefill T_q≫1 +makes PE-TP useful — all P PEs of CUBE 0 work on disjoint T_q rows +(P of 64 ranks active; CUBEs 1..7 are pure memory waste). + +Q-axis tiling: an outer Q-tile loop wraps the entire per-tile +pipeline (Q load → bootstrap → S_kv-tile fold → store) in +``tl.scratch_scope``. Peak scratch is bounded by +``TILE_Q × TILE_S_KV`` instead of ``T_q_local × S_kv``, so the +kernel runs at large T_q without exceeding the ~1 MB per-PE budget. +Each Q-tile is independent (disjoint output rows; no inter-tile state). + +Tensor layout: + Q : (T_q, h_q · d_head) cube=replicate, pe=row_wise on T_q. Only + CUBE 0's PEs load — each owns ``G · T_q_local`` total rows with + ``T_q_local = T_q / P``; loaded per Q-tile. + K : (S_kv, h_kv · d_head) replicated on every rank. + V : (S_kv, h_kv · d_head) replicated on every rank. + O : (T_q, h_q · d_head) cube=replicate, pe=row_wise on T_q — + only CUBE 0's PEs write their disjoint ``T_q_local`` rows, + one ``(TILE_Q, d_head)`` slice per outer iteration. + +Topology / SFR: + - SFR setup is benign (no inter-rank sends/recvs); any of the + ``configure_sfr_intercube_*`` helpers works. + +Requires ``T_q % P == 0``. +""" +from __future__ import annotations + + +TILE_S_KV = 64 # per-tile S_kv width (small to keep ``scores`` bounded). +TILE_Q = 64 # per-tile Q row count (outer-loop tile size). + + +def gqa_attention_prefill_long_ctx_cube_repl_pe_tp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-2 prefill: CUBE 0 only; Q-tiled PE-TP; full KV per rank; no comm.""" + if T_q % P != 0: + raise ValueError( + f"Case 2 prefill requires T_q={T_q} divisible by P={P}" + ) + + cube_id = tl.program_id(axis=1) + # CUBE 0 only — CUBEs 1..7 are pure memory-replication waste, no work. + if cube_id != 0: + return + + G = h_q // h_kv + T_q_local = T_q // P + n_kv_tiles = (S_kv + TILE_S_KV - 1) // TILE_S_KV + KV_ROW_BYTES = d_head * 2 # f16 + Q_ROW_BYTES = d_head * 2 + total_q_rows = G * T_q_local + n_q_tiles = (total_q_rows + TILE_Q - 1) // TILE_Q + + # ── Outer Q-tile loop ── + # Per Q-tile: load Q-slice, run full S_kv attention, store. All + # scratch is recycled at outer-scope exit, so peak usage is bounded + # by TILE_Q × TILE_S_KV. + for q_idx in range(n_q_tiles): + q_start = q_idx * TILE_Q + q_rows = min(TILE_Q, total_q_rows - q_start) + with tl.scratch_scope(): + Q = tl.load(q_ptr + q_start * Q_ROW_BYTES, + shape=(q_rows, d_head), dtype="f16") + + # ── Bootstrap (S_kv tile 0) ── + tile_s0 = min(TILE_S_KV, S_kv) + K_T = tl.load(k_ptr, shape=(d_head, tile_s0), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s0, d_head), dtype="f16") + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + exp_scores = tl.exp(scores - m_local) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + # ── S_kv tiles 1..N: fold via online-softmax merge ── + for tile_idx in range(1, n_kv_tiles): + tile_start = tile_idx * TILE_S_KV + tile_s = min(TILE_S_KV, S_kv - tile_start) + with tl.scratch_scope(): + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + scores_t = tl.dot(Q, K_T_t) + m_tile = tl.max(scores_t, axis=-1) + exp_scores_t = tl.exp(scores_t - m_tile) + l_tile = tl.sum(exp_scores_t, axis=-1) + O_tile = tl.dot(exp_scores_t, V_t) + m_new = tl.maximum(m_local, m_tile) + scale_old = tl.exp(m_local - m_new) + scale_new = tl.exp(m_tile - m_new) + l_new = l_local * scale_old + l_tile * scale_new + O_new = O_local * scale_old + O_tile * scale_new + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Store this Q-tile (each active PE writes its slice) ── + O_final = O_local / l_local + tl.store(o_ptr + q_start * Q_ROW_BYTES, O_final) diff --git a/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_sp.py b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_sp.py new file mode 100644 index 0000000..2d576c7 --- /dev/null +++ b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_sp.py @@ -0,0 +1,186 @@ +"""GQA prefill kernel — Case 4 (Cube-SP × PE-SP) at single-KV-head group. ★ + +Per GQA_full_deck.pptx slide 11 (prefill counterpart, T_q ≫ 1): + - K, V split 64-way (cube=row_wise, pe=row_wise on S_kv); each rank + owns ``S_local_pe = S_kv / (C·P)``. + - Q, O split T_q-wise across cubes (cube=row_wise on T_q), + pe=replicate within each cube — each cube owns + ``T_q_cube = T_q / C`` rows, all P PEs of the cube hold a copy. + - Inter-CUBE Ring KV (snake over 4×2 sub-mesh, ADR-0060 §5.5): + P parallel rings — each PE drives its own same-lane ring via + independent IPCQ channels. Over C ring steps each PE has + processed its strided lane of S_kv (= ``S_kv / P`` tokens) for + the cube's T_q_cube query rows. + - Intra-CUBE 8-way reduce on the partial ``(m, ℓ, O)`` triple + (row chain along intra_W + col bridge along intra_N over the + 2×4 PE grid) — combines the P PE-lanes per cube so cube's PE 0 + holds the full S_kv attention for its T_q_cube rows. + - NO inter-CUBE reduce — each cube writes its own disjoint + T_q_cube rows of O (PE 0 of each cube is the writer). + +Distinction from decode Case 4: decode T_q=1 forces an inter-CUBE +lrab reduce-to-root over (m,ℓ,O). Prefill T_q≫1 allows Q to be +T_q-sharded across cubes, so disjoint output rows avoid the +inter-cube reduce — the prefill-native pattern (per user's +"Ring KV" choice). + +Tensor layout: + Q : (T_q, h_q · d_head) sharded cube=row_wise on T_q, pe=replicate + within cube; each rank loads ``(G · T_q_cube, d_head)``. + K : (S_kv, h_kv · d_head) cube=row_wise + pe=row_wise on S_kv; + each rank owns ``(S_local_pe, h_kv·d_head)``. + V : same as K. + O : (T_q, h_q · d_head) sharded same as Q — PE 0 of each cube + writes its ``(G · T_q_cube, d_head)`` slice; PEs 1..P-1 idle + the write. + +Topology / SFR: + - Requires ``configure_sfr_intercube_ring(submesh_shape=(2, 4))`` — + that helper installs both the snake E/W ring across 8 cubes and + the intra_* lanes needed for the intra-CUBE reduce. + +Requires ``T_q % C == 0`` and ``S_kv % (C · P) == 0``. +""" +from __future__ import annotations + + +TILE_S_KV = 1024 # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width). + + +def _merge_running(m_local, l_local, O_local, m_other, l_other, O_other, *, tl): + """Online-softmax merge of two partial ``(m, ℓ, O)`` triples.""" + m_new = tl.maximum(m_local, m_other) + scale_old = tl.exp(m_local - m_new) + scale_new = tl.exp(m_other - m_new) + l_new = l_local * scale_old + l_other * scale_new + O_new = O_local * scale_old + O_other * scale_new + return m_new, l_new, O_new + + +def gqa_attention_prefill_long_ctx_cube_sp_pe_sp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-4 prefill: Ring KV across cubes + intra-CUBE AR; disjoint T_q/C rows per cube.""" + if T_q % C != 0: + raise ValueError( + f"Case 4 prefill requires T_q={T_q} divisible by C={C}" + ) + if S_kv % (C * P) != 0: + raise ValueError( + f"Case 4 prefill requires S_kv={S_kv} divisible by C·P={C * P}" + ) + + G = h_q // h_kv + T_q_cube = T_q // C + S_local_pe = S_kv // (C * P) + KV_ROW_BYTES = d_head * 2 # f16 + n_tiles = (S_local_pe + TILE_S_KV - 1) // TILE_S_KV + + pe_id = tl.program_id(axis=0) + + # Q is this cube's T_q slice (pe=replicate within cube ⇒ every PE + # loads the same G·T_q_cube rows). + Q = tl.load(q_ptr, shape=(G * T_q_cube, d_head), dtype="f16") + + # ── Bootstrap (t=0, k=0): load own rank's tile 0; send W for ring ── + # Persistent (m, ℓ, O) must live OUTSIDE tl.scratch_scope (kernbench + # scope teardown discards in-scope tensors). + tile_s = min(TILE_S_KV, S_local_pe) + K_T = tl.load(k_ptr, shape=(d_head, tile_s), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s, d_head), dtype="f16") + if C > 1: + tl.send(dir="W", src=K_T) + tl.send(dir="W", src=V) + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + exp_scores = tl.exp(scores - m_local) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + # ── Nested loop (outer tile, inner ring step) — P parallel rings ── + for t in range(n_tiles): + tile_start = t * TILE_S_KV + tile_s = min(TILE_S_KV, S_local_pe - tile_start) + k_start = 1 if t == 0 else 0 + for k in range(k_start, C): + with tl.scratch_scope(): + if k == 0: + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + else: + K_T_t = tl.recv(dir="E", shape=(d_head, tile_s), dtype="f16") + V_t = tl.recv(dir="E", shape=(tile_s, d_head), dtype="f16") + if k < C - 1: + tl.send(dir="W", src=K_T_t) + tl.send(dir="W", src=V_t) + scores_t = tl.dot(Q, K_T_t) + m_step = tl.max(scores_t, axis=-1) + exp_scores_t = tl.exp(scores_t - m_step) + l_step = tl.sum(exp_scores_t, axis=-1) + O_step = tl.dot(exp_scores_t, V_t) + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_step, l_step, O_step, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Intra-CUBE 8-way reduce-to-PE0 (row chain + col bridge) ── + PE_GRID_COLS = 4 + pe_col = pe_id % PE_GRID_COLS + pe_row = pe_id // PE_GRID_COLS + pe_cols_used = min(PE_GRID_COLS, P) + pe_rows_used = (P + PE_GRID_COLS - 1) // PE_GRID_COLS + + if pe_cols_used > 1: + if pe_col < pe_cols_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_E", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_E", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_E", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_col > 0: + tl.send(dir="intra_W", src=m_local) + tl.send(dir="intra_W", src=l_local) + tl.send(dir="intra_W", src=O_local) + + if pe_col == 0 and pe_rows_used > 1: + if pe_row < pe_rows_used - 1: + with tl.scratch_scope(): + m_other = tl.recv(dir="intra_S", shape=m_local.shape, dtype="f16") + l_other = tl.recv(dir="intra_S", shape=l_local.shape, dtype="f16") + O_other = tl.recv(dir="intra_S", shape=O_local.shape, dtype="f16") + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_other, l_other, O_other, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + if pe_row > 0: + tl.send(dir="intra_N", src=m_local) + tl.send(dir="intra_N", src=l_local) + tl.send(dir="intra_N", src=O_local) + + # ── Final normalise + store (PE 0 of each cube writes its T_q_cube rows) ── + if pe_id == 0: + O_final = O_local / l_local + tl.store(o_ptr, O_final) diff --git a/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_tp.py b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_tp.py new file mode 100644 index 0000000..fba4e81 --- /dev/null +++ b/src/kernbench/benches/gqa_helpers/long_ctx/_gqa_attention_prefill_long_ctx_cube_sp_pe_tp.py @@ -0,0 +1,136 @@ +"""GQA prefill kernel — Case 1 (Cube-SP × PE-TP) at single-KV-head group. + +Per GQA_full_deck.pptx slide 11 (prefill counterpart, T_q ≫ 1): + - K, V split S_kv-wise across the 8 cubes (cube=row_wise); + replicated within each cube (pe=replicate). Each cube owns + ``S_local = S_kv / C``. + - Q, O split T_q-wise across cubes AND across PEs intra-cube + (cube=row_wise, pe=row_wise on T_q). Each rank owns disjoint + ``T_q_local = T_q / (C · P)`` query rows. + - Inter-CUBE Ring KV (snake over 4×2 sub-mesh, ADR-0060 §5.5): + P parallel rings — each PE drives its own same-lane ring via + independent IPCQ channels. Over C ring steps the cube-owned + K/V blocks rotate through all C cubes, so every rank sees + every block. + - NO inter-CUBE reduce — each rank writes its own disjoint + T_q_local rows of O. + - NO intra-CUBE reduce — PEs are disjoint on T_q. + +Distinction from decode Case 1: decode T_q=1 makes PE-TP wasteful +(only PE 0 of each cube works). Prefill T_q≫1 makes PE-TP useful — +all 64 ranks work on disjoint Q rows. + +Tensor layout: + Q : (T_q, h_q · d_head) sharded (cube_row_wise, pe_row_wise) on T_q; + each rank loads ``(G · T_q_local, d_head)``. + K : (S_kv, h_kv · d_head) with cube=row_wise, pe=replicate — each + cube's PE 0..P-1 each hold an HBM copy of the cube's + ``(S_local, h_kv·d_head)`` shard. + V : same as K. + O : (T_q, h_q · d_head) sharded same as Q — each rank writes its + own ``(G · T_q_local, d_head)`` slice. + +Topology / SFR: + - Requires ``configure_sfr_intercube_ring(submesh_shape=(2, 4))`` + for the snake E/W ring across 8 cubes (wrap from cube 3 ↔ cube 7). + +Requires ``T_q % (C · P) == 0``. +""" +from __future__ import annotations + + +TILE_S_KV = 1024 # ADR-0063 §A.2 S_kv-axis tile sweep (per-tile width). + + +def _merge_running(m_local, l_local, O_local, m_other, l_other, O_other, *, tl): + """Online-softmax merge of two partial ``(m, ℓ, O)`` triples.""" + m_new = tl.maximum(m_local, m_other) + scale_old = tl.exp(m_local - m_new) + scale_new = tl.exp(m_other - m_new) + l_new = l_local * scale_old + l_other * scale_new + O_new = O_local * scale_old + O_other * scale_new + return m_new, l_new, O_new + + +def gqa_attention_prefill_long_ctx_cube_sp_pe_tp_kernel( + q_ptr: int, + k_ptr: int, + v_ptr: int, + o_ptr: int, + T_q: int, + S_kv: int, + h_q: int, + h_kv: int, + d_head: int, + C: int, + P: int, + *, + tl, +) -> None: + """Case-1 prefill: Ring KV across cubes; disjoint T_q rows per rank.""" + if T_q % (C * P) != 0: + raise ValueError( + f"Case 1 prefill requires T_q={T_q} divisible by C·P={C * P}" + ) + + G = h_q // h_kv + T_q_local = T_q // (C * P) + S_local = S_kv // C + KV_ROW_BYTES = d_head * 2 # f16 + n_tiles = (S_local + TILE_S_KV - 1) // TILE_S_KV + + # Q is this rank's own disjoint T_q slice (G·T_q_local rows for the + # G-grouped attention). + Q = tl.load(q_ptr, shape=(G * T_q_local, d_head), dtype="f16") + + # ── Bootstrap (t=0, k=0): load own cube's tile 0; send W for ring ── + # Persistent (m, ℓ, O) must live OUTSIDE tl.scratch_scope (kernbench + # scope teardown discards in-scope tensors). Mirrors decode Cases + # 1, 3, 4 and the existing prefill_long bootstrap pattern. + tile_s = min(TILE_S_KV, S_local) + K_T = tl.load(k_ptr, shape=(d_head, tile_s), dtype="f16") + V = tl.load(v_ptr, shape=(tile_s, d_head), dtype="f16") + if C > 1: + tl.send(dir="W", src=K_T) + tl.send(dir="W", src=V) + scores = tl.dot(Q, K_T) + m_local = tl.max(scores, axis=-1) + exp_scores = tl.exp(scores - m_local) + l_local = tl.sum(exp_scores, axis=-1) + O_local = tl.dot(exp_scores, V) + + # ── Nested loop (outer tile, inner ring step) ── + # Each tile propagates around the ring before the next tile starts; + # IPCQ in-flight depth stays at 1 per direction per PE-lane. + for t in range(n_tiles): + tile_start = t * TILE_S_KV + tile_s = min(TILE_S_KV, S_local - tile_start) + k_start = 1 if t == 0 else 0 + for k in range(k_start, C): + with tl.scratch_scope(): + if k == 0: + K_T_t = tl.load(k_ptr + tile_start * KV_ROW_BYTES, + shape=(d_head, tile_s), dtype="f16") + V_t = tl.load(v_ptr + tile_start * KV_ROW_BYTES, + shape=(tile_s, d_head), dtype="f16") + else: + K_T_t = tl.recv(dir="E", shape=(d_head, tile_s), dtype="f16") + V_t = tl.recv(dir="E", shape=(tile_s, d_head), dtype="f16") + if k < C - 1: + tl.send(dir="W", src=K_T_t) + tl.send(dir="W", src=V_t) + scores_t = tl.dot(Q, K_T_t) + m_step = tl.max(scores_t, axis=-1) + exp_scores_t = tl.exp(scores_t - m_step) + l_step = tl.sum(exp_scores_t, axis=-1) + O_step = tl.dot(exp_scores_t, V_t) + m_new, l_new, O_new = _merge_running( + m_local, l_local, O_local, m_step, l_step, O_step, tl=tl, + ) + tl.copy_to(m_local, m_new) + tl.copy_to(l_local, l_new) + tl.copy_to(O_local, O_new) + + # ── Final normalise + store (every rank writes its own T_q_local rows) ── + O_final = O_local / l_local + tl.store(o_ptr, O_final) diff --git a/src/kernbench/benches/gqa_helpers/long_ctx/gqa_prefill_long_ctx_4cases.py b/src/kernbench/benches/gqa_helpers/long_ctx/gqa_prefill_long_ctx_4cases.py new file mode 100644 index 0000000..db9a533 --- /dev/null +++ b/src/kernbench/benches/gqa_helpers/long_ctx/gqa_prefill_long_ctx_4cases.py @@ -0,0 +1,384 @@ +"""milestone-gqa-prefill-long-ctx-4cases: long-context prefill 4-cases study. + +Prefill counterpart to milestone-gqa-decode-long-ctx-4cases. Per +GQA_full_deck.pptx slides 11-17: 4 KV-cache sharding strategies on +the LLaMA-3.1-70B single-KV-head group (1 KV head, 8 Q heads, +d_head=128) on 8 cubes × 8 PEs. + +Bench size T_q=S_kv=512 (equal — prompt length matches K/V length +for prefill). Cases 2 and 3 use Q-axis tiling (``TILE_Q``) so the +per-PE scratch is bounded by ``TILE_Q × TILE_S_KV`` regardless of +T_q — much larger sizes are possible at the cost of bench +wall-clock (Case 2's single-cube serial path dominates). + + Case 1 Cube-SP / PE-TP → KV split S_kv across cubes (Ring), + Q/O split T_q across all 64 ranks + Case 2 Cube-Repl / PE-TP → full KV per cube; only CUBE 0's P PEs + split T_q; CUBEs 1..7 are pure memory waste + Case 3 Cube-Repl / PE-SP → full KV per cube; PEs SP on S_kv + (intra-cube AR); 8× redundant compute + Case 4 Cube-SP / PE-SP → KV split 64-way + Q split T_q across cubes; + Ring KV + intra-cube AR per cube ★ optimal + +Each case is a separate panel. The bench drives all panels in one +invocation and writes per-panel op_log_summary + latency to sweep.json +so the comparative analysis (latency, comm volume, parallelism, +memory) can be generated from a single sweep. + +Distinction from decode: decode T_q=1 makes PE-TP cases wasteful (1 +active PE). Prefill T_q≫1 makes PE-TP useful (full parallelism). The +narrative differs — the figures emphasise Case 2/3 memory waste and +Case 3 compute redundancy, not PE-TP-at-B=1 wasted ranks. + +Per-panel try/except in ``run()`` writes whichever cases succeed — +the bench tolerates per-panel scratch / SFR failures so sweep.json +always lands with at least the successful rows + a ``failures`` list. + +Gated by ``GQA_PREFILL_LONG_CTX_4CASES_RUN=1``. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +from kernbench.benches.gqa_helpers.long_ctx._gqa_attention_prefill_long_ctx_cube_repl_pe_sp import ( + gqa_attention_prefill_long_ctx_cube_repl_pe_sp_kernel, +) +from kernbench.benches.gqa_helpers.long_ctx._gqa_attention_prefill_long_ctx_cube_repl_pe_tp import ( + gqa_attention_prefill_long_ctx_cube_repl_pe_tp_kernel, +) +from kernbench.benches.gqa_helpers.long_ctx._gqa_attention_prefill_long_ctx_cube_sp_pe_sp import ( + gqa_attention_prefill_long_ctx_cube_sp_pe_sp_kernel, +) +from kernbench.benches.gqa_helpers.long_ctx._gqa_attention_prefill_long_ctx_cube_sp_pe_tp import ( + gqa_attention_prefill_long_ctx_cube_sp_pe_tp_kernel, +) +from kernbench.benches.gqa_helpers.shared._gqa_panel_helpers import ( + _ccl_cfg, + _summarize_op_log, +) +from kernbench.ccl.sfr_config import configure_sfr_intercube_ring +from kernbench.policy.placement.dp import DPPolicy + +# File is at benches/gqa_helpers/long_ctx/ — go up 2 parents to reach +# benches/, then into 1H_milestone_output/gqa/long_ctx/. +_OUTPUT_DIR = ( + Path(__file__).resolve().parents[2] + / "1H_milestone_output" / "gqa" / "long_ctx" +) +_SWEEP_JSON = _OUTPUT_DIR / "sweep_prefill.json" + + +# ── Panel registry ─────────────────────────────────────────────────── + + +_PANELS = ( + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_sp", # Case 4 ★ optimal + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_tp", # Case 2 + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_sp", # Case 3 + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_tp", # Case 1 +) + +# Each entry: (kind, panel-specific params). +# LLaMA-3.1-70B single-KV-head group target: +# 1 KV head, h_q = 8 (G = 8 group), d_head = 128 +# 8 cubes (head-parallel group), 8 PEs/cube +# T_q = S_kv = 4096 (long-context prefill) +_PANEL_DISPATCH: dict[str, tuple[str, dict]] = { + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_sp": ("prefill_long_ctx_cube_sp_pe_sp", { + # Case 4: KV split 64-way + Q split T_q across cubes; Ring KV + # + intra-CUBE AR per cube. PE 0 of each cube writes its + # T_q/C disjoint output rows. + "C": 8, "P": 8, + "T_q": 512, "S_kv": 512, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_tp": ("prefill_long_ctx_cube_repl_pe_tp", { + # Case 2: K, V replicated everywhere (8× memory waste); within + # CUBE 0 the P PEs split T_q disjointly. CUBEs 1..7 idle. + # No inter-rank communication. + "C": 8, "P": 8, + "T_q": 512, "S_kv": 512, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), + "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_sp": ("prefill_long_ctx_cube_repl_pe_sp", { + # Case 3: K, V replicated per cube (8× memory); PEs SP on S_kv + # within each cube. Intra-CUBE 8-way reduce; no inter-CUBE comm + # (every cube does redundant T_q × S_kv compute; designated + # writer = cube 0). + "C": 8, "P": 8, + "T_q": 512, "S_kv": 512, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), + "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_tp": ("prefill_long_ctx_cube_sp_pe_tp", { + # Case 1: K, V split S_kv across cubes (S_local = S_kv/C per + # cube); Q/O split T_q across all 64 ranks. Inter-CUBE snake + # Ring KV (P parallel rings); no intra-CUBE comm. + "C": 8, "P": 8, + "T_q": 512, "S_kv": 512, + "d_head": 128, "h_q": 8, "h_kv": 1, + }), +} + + +# ── Per-panel runner ───────────────────────────────────────────────── + + +def _ring_sfr(ctx): + """Install snake Ring SFR (also provides intra_* lanes for AR cases). + + Snake submesh of shape (2, 4) over the default 4×4 cube mesh + yields a Hamiltonian cycle through all 8 cubes + (0→1→2→3→7→6→5→4→0), with intra_* lanes available for the + Case 3/4 intra-CUBE reduce. + """ + configure_sfr_intercube_ring( + ctx.engine, ctx.spec, _ccl_cfg(), + submesh_shape=(2, 4), + ) + + +def _run_prefill_panel_long_ctx_cube_repl_pe_tp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 2 runner: K, V replicated everywhere; CUBE 0 PE-TP on T_q. + + DPPolicy models the cluster-wide 8× memory waste — every rank + holds full K, V in its HBM region. Within CUBE 0, P PEs split + T_q (so P of 64 ranks actually compute). CUBEs 1..7 idle. + """ + _ring_sfr(ctx) + dp_repl = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + dp_qo = DPPolicy(cube="replicate", pe="row_wise", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_repl, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_prefill_long_ctx_cube_repl_pe_tp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + +def _run_prefill_panel_long_ctx_cube_repl_pe_sp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 3 runner: K, V replicated per cube; PEs SP on S_kv intra-cube. + + DPPolicy models the cluster-wide 8× memory waste — every cube + holds full K, V; intra-cube splits the S_kv axis row_wise across + 8 PEs. Every rank holds full Q (cube=replicate, pe=replicate). + The kernel does an intra-CUBE 8-way reduce on (m, ℓ, O); only + cube 0's PE 0 writes the output. + """ + _ring_sfr(ctx) + dp_full = DPPolicy(cube="replicate", pe="replicate", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="replicate", pe="row_wise", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_full, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_prefill_long_ctx_cube_repl_pe_sp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + +def _run_prefill_panel_long_ctx_cube_sp_pe_tp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 1 runner: K, V split S_kv across cubes (Ring); Q/O split T_q 64-way. + + DPPolicy: K, V are cube=row_wise on S_kv (S_local = S_kv/C per + cube), pe=replicate within cube. Q, O are cube=row_wise + + pe=row_wise on T_q — every rank owns T_q/(C·P) disjoint Q rows. + The kernel runs P parallel Rings through the snake submesh + (each PE has its own IPCQ lane); no intra-CUBE reduce. + """ + _ring_sfr(ctx) + dp_qo = DPPolicy(cube="row_wise", pe="row_wise", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="row_wise", pe="replicate", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_prefill_long_ctx_cube_sp_pe_tp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + +def _run_prefill_panel_long_ctx_cube_sp_pe_sp( + ctx, *, panel: str, C: int, P: int, + T_q: int, S_kv: int, + d_head: int, h_q: int, h_kv: int, +) -> None: + """Case 4 runner: K, V split 64-way; Q split T_q across cubes. + + DPPolicy: K, V are cube=row_wise + pe=row_wise on S_kv — each + rank owns S_local_pe = S_kv/(C·P). Q, O are cube=row_wise on + T_q (T_q_cube = T_q/C per cube), pe=replicate within cube. Ring + KV through the snake submesh (P parallel lanes per cube); after + the ring, an intra-CUBE 8-way AR combines PE lanes so cube's + PE 0 has the full attention for its T_q_cube rows. PE 0 of each + cube writes its disjoint output rows (no inter-cube reduce). + """ + _ring_sfr(ctx) + dp_qo = DPPolicy(cube="row_wise", pe="replicate", + num_cubes=C, num_pes=P) + dp_kv = DPPolicy(cube="row_wise", pe="row_wise", + num_cubes=C, num_pes=P) + q = ctx.zeros((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_q") + k = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_k") + v = ctx.zeros((S_kv, h_kv * d_head), + dtype="f16", dp=dp_kv, name=f"{panel}_v") + o = ctx.empty((T_q, h_q * d_head), + dtype="f16", dp=dp_qo, name=f"{panel}_o") + ctx.launch( + panel, gqa_attention_prefill_long_ctx_cube_sp_pe_sp_kernel, + q, k, v, o, + T_q, S_kv, h_q, h_kv, d_head, C, P, + _auto_dim_remap=False, + ) + + +def _make_bench_fn(panel: str): + kind, params = _PANEL_DISPATCH[panel] + + def _bench_fn(ctx): + if kind == "prefill_long_ctx_cube_sp_pe_sp": + _run_prefill_panel_long_ctx_cube_sp_pe_sp(ctx, panel=panel, **params) + elif kind == "prefill_long_ctx_cube_repl_pe_tp": + _run_prefill_panel_long_ctx_cube_repl_pe_tp(ctx, panel=panel, **params) + elif kind == "prefill_long_ctx_cube_repl_pe_sp": + _run_prefill_panel_long_ctx_cube_repl_pe_sp(ctx, panel=panel, **params) + elif kind == "prefill_long_ctx_cube_sp_pe_tp": + _run_prefill_panel_long_ctx_cube_sp_pe_tp(ctx, panel=panel, **params) + else: + raise RuntimeError( + f"milestone-gqa-prefill-long-ctx-4cases panel {panel!r} has " + f"unsupported kind={kind!r}." + ) + return _bench_fn + + +# ── Panel metrics (sweep.json carries these for the comparative plot) ─ + + +_ENGINE_SUFFIXES = ( + "pe_gemm", "pe_math", "pe_dma", "pe_fetch_store", "pe_ipcq", "pe_cpu", +) + + +def _end_to_end_ns(op_log) -> float: + """End-to-end window: ``max(t_end) - min(t_start)`` over all records.""" + if not op_log: + return 0.0 + return max(r.t_end for r in op_log) - min(r.t_start for r in op_log) + + +def _engine_occupancy_ns(op_log) -> dict[str, float]: + """Per-engine summed occupancy (component_id suffix match).""" + return { + eng: sum( + r.t_end - r.t_start + for r in op_log + if r.component_id.endswith("." + eng) + ) + for eng in _ENGINE_SUFFIXES + } + + +def _run_panel(panel: str, topology: str) -> dict: + from kernbench.runtime_api.bench_runner import run_bench + from kernbench.runtime_api.types import resolve_device + from kernbench.sim_engine.engine import GraphEngine + from kernbench.topology.builder import resolve_topology + + topo = resolve_topology(topology) + result = run_bench( + topology=topo, bench_fn=_make_bench_fn(panel), + device=resolve_device(None), + engine_factory=lambda t, d: GraphEngine( + getattr(t, "topology_obj", t), enable_data=True, + ), + ) + if not result.completion.ok: + raise RuntimeError( + f"milestone-gqa-prefill-long-ctx-4cases panel {panel!r} failed: " + f"{result.completion}" + ) + kind, params = _PANEL_DISPATCH[panel] + op_log = result.engine.op_log + return { + "panel": panel, + "kind": kind, + **params, + "op_log_summary": _summarize_op_log(op_log), + "latency_ns": _end_to_end_ns(op_log), + "engine_occupancy_ns": _engine_occupancy_ns(op_log), + } + + +# ── Sweep entry (called by the umbrella ``milestone_1h_gqa``) ──────── + + +def run_sweep(topology: str = "topology.yaml") -> int: + """Drive all prefill case panels; write sweep.json. Returns row count. + + Per-panel try/except: even with Q-axis tiling, very large T_q or + unusual config combinations can hit the per-PE scratch budget. We + keep going so sweep.json carries whichever panels succeed; failures + land in a ``failures`` list for follow-up. + """ + _OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + rows: list[dict] = [] + failures: list[dict] = [] + for panel in _PANELS: + try: + rows.append(_run_panel(panel, topology)) + except Exception as e: + print(f" panel {panel!r} FAILED: {e}") + failures.append({"panel": panel, "error": str(e)}) + sweep = { + "version": 1, + "panels": list(_PANELS), + "rows": rows, + "failures": failures, + } + _SWEEP_JSON.write_text(json.dumps(sweep, indent=2)) + print(f" gqa-prefill-long-ctx-4cases: {len(rows)} rows -> {_SWEEP_JSON}") + return len(rows) diff --git a/tests/attention/test_milestone_gqa_prefill_long_ctx_4cases.py b/tests/attention/test_milestone_gqa_prefill_long_ctx_4cases.py new file mode 100644 index 0000000..f4a2132 --- /dev/null +++ b/tests/attention/test_milestone_gqa_prefill_long_ctx_4cases.py @@ -0,0 +1,471 @@ +"""Tests for the long-context prefill 4-cases comparative-study bench. + +Prefill counterpart to test_milestone_gqa_decode_long_ctx_4cases. The +4 cases differ in how KV cache is sharded across the 8 cubes and 8 +PEs of a single KV-head group on LLaMA-3.1-70B GQA: + + Case 1 Cube-SP / PE-TP → KV split by S_kv across cubes (Ring); + Q/O split T_q across all 64 ranks + Case 2 Cube-Repl / PE-TP → full KV per cube; only CUBE 0's P PEs + split T_q (CUBEs 1..7 = pure memory waste) + Case 3 Cube-Repl / PE-SP → full KV per cube; PEs SP on S_kv + (intra-cube AR); 8× redundant compute + Case 4 Cube-SP / PE-SP → KV split 64-way + Q split T_q across cubes; + Ring KV + intra-cube AR per cube ★ optimal + +Smoke sizes (T_q=256, S_kv=2048) keep the per-PE scratch comfortably +inside budget; the headline T_q=S_kv=4096 runs come from +``kernbench run --bench milestone-gqa-prefill-long-ctx-4cases``, not +pytest. +""" +from __future__ import annotations + +import re +from pathlib import Path + +from kernbench.runtime_api.bench_runner import run_bench +from kernbench.runtime_api.types import resolve_device +from kernbench.sim_engine.engine import GraphEngine +from kernbench.topology.builder import resolve_topology + +TOPOLOGY_DEFAULT = Path(__file__).resolve().parents[2] / "topology.yaml" + +_CASE4_PANEL = "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_sp" +_CASE3_PANEL = "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_sp" +_CASE2_PANEL = "single_kv_group_prefill_long_ctx_gqa_cube_repl_pe_tp" +_CASE1_PANEL = "single_kv_group_prefill_long_ctx_gqa_cube_sp_pe_tp" +_CUBE_RE = re.compile(r"\bcube(\d+)\b") + +# Smoke sizes: T_q=256 (divisible by C·P=64 for Case 1, by P=8 for +# Case 2, by C=8 for Case 4) and S_kv=2048 (≥ 2·TILE_S_KV so the +# per-tile fold loop runs; Case 1/4 have S_local sufficient to exercise +# the Ring). Assertions are size-independent. The headline T_q=S_kv=4096 +# bench runs come from the env-gated bench, not pytest. +# Smoke sizes — kept small so pytest stays fast: +# T_q=64 : minimum that satisfies Case 1's ``T_q % (C·P) == 0``. +# S_kv=512: enough to exercise multi-tile fold for Cases 2/3 +# (TILE_S_KV=64 → 8 tiles for Case 2; 1 tile/PE for Case 3 +# after intra-cube S_kv split) without burning minutes on +# Case 2's single-rank serial path. +# Cases 2/3 also use TILE_S_KV=64 in their kernels to keep the +# persistent ``scores`` tensor inside the 1 MB scratch budget at +# T_q=64. Headline T_q=S_kv=4096 runs come from the env-gated bench, +# not pytest. +_SMOKE_T_Q = 64 +_SMOKE_S_KV = 512 + +_HEADLINE_T_Q = 512 +_HEADLINE_S_KV = 512 + + +def _engine_factory(t, d): + return GraphEngine(getattr(t, "topology_obj", t), enable_data=True) + + +def _count(op_log, name: str) -> int: + return sum(1 for r in op_log if r.op_name == name) + + +def _dma_write_cubes(op_log) -> list[int]: + cubes: list[int] = [] + for r in op_log: + if r.op_name != "dma_write": + continue + m = _CUBE_RE.search(r.component_id) + if m is not None: + cubes.append(int(m.group(1))) + return cubes + + +# ── Case 4 (Cube-SP × PE-SP) — ★ optimal ──────────────────────────── + + +def _run_case4_smoke(*, T_q: int, S_kv: int): + """Drive the Case 4 prefill panel via the case-specific runner.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _run_prefill_panel_long_ctx_cube_sp_pe_sp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_prefill_panel_long_ctx_cube_sp_pe_sp( + ctx, panel=_CASE4_PANEL, + C=8, P=8, + T_q=T_q, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +def test_case4_panel_registered(): + """Case 4 panel must be in ``_PANELS`` + ``_PANEL_DISPATCH`` with + the LLaMA-3.1-70B single-KV-group prefill dims (T_q=S_kv=4096). + """ + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE4_PANEL in _PANELS, f"{_CASE4_PANEL!r} not in {_PANELS}" + assert _CASE4_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE4_PANEL] + assert kind == "prefill_long_ctx_cube_sp_pe_sp" + assert params == { + "C": 8, "P": 8, + "T_q": _HEADLINE_T_Q, "S_kv": _HEADLINE_S_KV, + "d_head": 128, "h_q": 8, "h_kv": 1, + } + + +def test_case4_runner_smoke(): + """Case 4 runner drives the new kernel to completion at smoke sizes.""" + result = _run_case4_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok, ( + f"Case 4 prefill smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +def test_case4_writers_one_per_cube(): + """Case 4 prefill: each cube writes its own disjoint T_q/C rows + (PE 0 of each cube). Output cubes must cover all 8. + """ + result = _run_case4_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == set(range(8)), ( + f"Case 4 expected PE 0 of every cube to write; got cubes={sorted(distinct)}" + ) + + +def test_case4_ipcq_copy_positive(): + """Case 4 prefill has both inter-CUBE Ring and intra-CUBE AR + traffic; total ipcq_copy must be strictly positive. + """ + result = _run_case4_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy > 0, f"Case 4 must have Ring + intra-CUBE comm; got 0 ipcq_copy" + + +# ── Case 2 (Cube-Repl × PE-TP) ────────────────────────────────────── + + +def _run_case2_smoke(*, T_q: int, S_kv: int): + """Drive the Case 2 prefill panel via the case-specific runner.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _run_prefill_panel_long_ctx_cube_repl_pe_tp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_prefill_panel_long_ctx_cube_repl_pe_tp( + ctx, panel=_CASE2_PANEL, + C=8, P=8, + T_q=T_q, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +def test_case2_panel_registered(): + """Case 2 panel registered with single-KV-group prefill dims.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE2_PANEL in _PANELS, f"{_CASE2_PANEL!r} not in {_PANELS}" + assert _CASE2_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE2_PANEL] + assert kind == "prefill_long_ctx_cube_repl_pe_tp" + assert params == { + "C": 8, "P": 8, + "T_q": _HEADLINE_T_Q, "S_kv": _HEADLINE_S_KV, + "d_head": 128, "h_q": 8, "h_kv": 1, + } + + +def test_case2_runner_smoke(): + """Case 2 runner drives the new kernel to completion at smoke sizes.""" + result = _run_case2_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok, ( + f"Case 2 prefill smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +def test_case2_zero_ipcq_copy_no_comm(): + """Case 2's defining property: full KV per rank ⇒ NO inter-rank + communication. Slide 11 lists comm cost as 'none'. + """ + result = _run_case2_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy == 0, ( + f"Case 2 must have zero inter-rank comm; got ipcq_copy={n_copy}" + ) + + +def test_case2_writers_only_cube_0(): + """Case 2: only CUBE 0 active (PEs 0..P-1 each writing their T_q/P + rows). All dma_writes come from cube 0. + """ + result = _run_case2_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == {0}, ( + f"Case 2 writers must be cube 0 only; got cubes={sorted(distinct)}" + ) + + +# ── Case 3 (Cube-Repl × PE-SP) ────────────────────────────────────── + + +def _run_case3_smoke(*, T_q: int, S_kv: int): + """Drive the Case 3 prefill panel via the case-specific runner.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _run_prefill_panel_long_ctx_cube_repl_pe_sp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_prefill_panel_long_ctx_cube_repl_pe_sp( + ctx, panel=_CASE3_PANEL, + C=8, P=8, + T_q=T_q, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +def test_case3_panel_registered(): + """Case 3 panel registered with single-KV-group prefill dims.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE3_PANEL in _PANELS, f"{_CASE3_PANEL!r} not in {_PANELS}" + assert _CASE3_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE3_PANEL] + assert kind == "prefill_long_ctx_cube_repl_pe_sp" + assert params == { + "C": 8, "P": 8, + "T_q": _HEADLINE_T_Q, "S_kv": _HEADLINE_S_KV, + "d_head": 128, "h_q": 8, "h_kv": 1, + } + + +def test_case3_runner_smoke(): + """Case 3 runner drives the new kernel to completion at smoke sizes.""" + result = _run_case3_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok, ( + f"Case 3 prefill smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +def test_case3_intra_cube_ar_only_ipcq(): + """Case 3 reduce pattern: per-CUBE 8-way PE-SP AR (same structural + cost as decode Case 3's intra-CUBE phase = 21 ipcq_copy per cube), + and NO inter-CUBE traffic (each cube has a full copy of KV). + + Q-axis tiling tradeoff: prefill Case 3 wraps the intra-CUBE AR + INSIDE the outer Q-tile loop (the kernel can't hold full + (m, ℓ, O) for the whole T_q in scratch). So the AR runs once + per Q-tile → total ipcq = n_q_tiles × per-tile cost. + + per-CUBE intra (2×4 PE grid), PER Q-TILE: + row chain along intra_W: cols 1,2,3 each row × 2 rows × + 3 tensors (m, ℓ, O) = 18 + col bridge along intra_N: pe4 only × 3 tensors = 3 + per-CUBE intra total = 21 + × 8 CUBEs = 168 + × n_q_tiles (= G·T_q / TILE_Q = 8·64/64 = 8) = 1344 + + inter-CUBE: 0 (replicated KV ⇒ no AllReduce needed). + """ + from kernbench.benches.gqa_helpers.long_ctx._gqa_attention_prefill_long_ctx_cube_repl_pe_sp import ( + TILE_Q, + ) + + result = _run_case3_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + G = 8 # h_q / h_kv + n_q_tiles = (G * _SMOKE_T_Q + TILE_Q - 1) // TILE_Q + expected = n_q_tiles * 168 + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy == expected, ( + f"Case 3 expected {expected} ipcq_copy " + f"({n_q_tiles} Q-tiles × 168 per Q-tile intra-CUBE AR); " + f"got {n_copy}" + ) + + +def test_case3_single_dma_write_at_cube_0(): + """Every cube ends with the full answer after intra-CUBE AR; only + the designated writer (cube 0, PE 0) stores O. + """ + result = _run_case3_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == {0}, ( + f"Case 3 designated writer must be cube 0; got cubes={sorted(distinct)}" + ) + + +# ── Case 1 (Cube-SP × PE-TP) ──────────────────────────────────────── + + +def _run_case1_smoke(*, T_q: int, S_kv: int): + """Drive the Case 1 prefill panel via the case-specific runner.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _run_prefill_panel_long_ctx_cube_sp_pe_tp, + ) + topo = resolve_topology(str(TOPOLOGY_DEFAULT)) + + def _bench_fn(ctx): + _run_prefill_panel_long_ctx_cube_sp_pe_tp( + ctx, panel=_CASE1_PANEL, + C=8, P=8, + T_q=T_q, S_kv=S_kv, + d_head=128, h_q=8, h_kv=1, + ) + + return run_bench( + topology=topo, bench_fn=_bench_fn, + device=resolve_device(None), + engine_factory=_engine_factory, + ) + + +def test_case1_panel_registered(): + """Case 1 panel registered with single-KV-group prefill dims.""" + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _PANEL_DISPATCH, + _PANELS, + ) + assert _CASE1_PANEL in _PANELS, f"{_CASE1_PANEL!r} not in {_PANELS}" + assert _CASE1_PANEL in _PANEL_DISPATCH + kind, params = _PANEL_DISPATCH[_CASE1_PANEL] + assert kind == "prefill_long_ctx_cube_sp_pe_tp" + assert params == { + "C": 8, "P": 8, + "T_q": _HEADLINE_T_Q, "S_kv": _HEADLINE_S_KV, + "d_head": 128, "h_q": 8, "h_kv": 1, + } + + +def test_case1_runner_smoke(): + """Case 1 runner drives the new kernel to completion at smoke sizes.""" + result = _run_case1_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok, ( + f"Case 1 prefill smoke at C=8 P=8 must complete; " + f"got {result.completion}" + ) + + +def test_case1_ipcq_copy_positive_ring_only(): + """Case 1 has inter-CUBE Ring (P parallel lanes) only; total ipcq + must be positive. No intra-CUBE comm (PEs are disjoint on T_q). + """ + result = _run_case1_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + n_copy = _count(result.engine.op_log, "ipcq_copy") + assert n_copy > 0, f"Case 1 must have Ring KV comm; got 0 ipcq_copy" + + +def test_case1_writers_all_64_ranks(): + """Case 1 prefill: every rank owns disjoint T_q/(C·P) rows and + writes them. All 8 cubes appear in dma_write component ids. + """ + result = _run_case1_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + cubes = _dma_write_cubes(result.engine.op_log) + assert cubes, "expected at least one dma_write for the final O store" + distinct = set(cubes) + assert distinct == set(range(8)), ( + f"Case 1 expected every cube to write; got cubes={sorted(distinct)}" + ) + + +# ── Panel-metrics helpers + _run_panel wiring ─────────────────────── + + +_EXPECTED_ENGINES = { + "pe_gemm", "pe_math", "pe_dma", "pe_fetch_store", "pe_ipcq", "pe_cpu", +} + + +def test_panel_metrics_helpers_present_and_correct(): + """Bench file must expose ``_end_to_end_ns`` and + ``_engine_occupancy_ns`` for the comparative plot script. + Exercised via the Case 2 smoke runner's op_log (cheapest case). + """ + from kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases import ( + _end_to_end_ns, + _engine_occupancy_ns, + ) + result = _run_case2_smoke(T_q=_SMOKE_T_Q, S_kv=_SMOKE_S_KV) + assert result.completion.ok + op_log = result.engine.op_log + + lat = _end_to_end_ns(op_log) + assert lat > 0, f"expected positive end-to-end latency; got {lat}" + + occ = _engine_occupancy_ns(op_log) + assert isinstance(occ, dict) + assert set(occ.keys()) >= _EXPECTED_ENGINES, ( + f"engine_occupancy_ns missing required engines; " + f"got {set(occ.keys())}" + ) + assert occ["pe_gemm"] > 0, ( + f"expected pe_gemm occupancy > 0; got {occ['pe_gemm']}" + ) + + +def test_run_panel_returns_latency_and_engine_occupancy(monkeypatch): + """``_run_panel`` must include ``latency_ns`` + ``engine_occupancy_ns`` + alongside ``op_log_summary`` so sweep.json carries them for the + comparative plot script. Uses monkeypatch to swap the Case 2 panel's + T_q/S_kv to smoke sizes for speed. + """ + import kernbench.benches.gqa_helpers.long_ctx.gqa_prefill_long_ctx_4cases as mod + + orig_kind, orig_params = mod._PANEL_DISPATCH[_CASE2_PANEL] + fast_params = {**orig_params, "T_q": _SMOKE_T_Q, "S_kv": _SMOKE_S_KV} + monkeypatch.setitem( + mod._PANEL_DISPATCH, _CASE2_PANEL, (orig_kind, fast_params), + ) + + row = mod._run_panel(_CASE2_PANEL, str(TOPOLOGY_DEFAULT)) + assert "op_log_summary" in row + assert "latency_ns" in row, ( + f"_run_panel row missing latency_ns; keys={sorted(row.keys())}" + ) + assert row["latency_ns"] > 0 + assert "engine_occupancy_ns" in row, ( + f"_run_panel row missing engine_occupancy_ns; " + f"keys={sorted(row.keys())}" + ) + assert isinstance(row["engine_occupancy_ns"], dict) + assert set(row["engine_occupancy_ns"].keys()) >= _EXPECTED_ENGINES