gqa(adr-0065): N3 — GEMM-epilogue accumulate dataflow (O = O.corr + P.V)

Two fixes make the recipe's P.V GEMM fold into the rescaled accumulator: (1) op_log marks the composite gemm record accumulate=True when an 'add' epilogue targets the GEMM's own output; data_executor._execute_gemm then does O = O_existing + a@b instead of overwriting. (2) PE_SCHEDULER records the composite's numeric op at COMPLETION (in _retire_on_done) instead of at dispatch, so the GEMM's t_start sorts AFTER its prologue MATH ops -> it reads the computed P and the rescaled O. Verified end-to-end (DataExecutor): the full recipe produces O = O_old*corr + P@V matching a numpy flash-attention reference. Suite 816 pass / 3 pre-existing fail. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 23:06:01 -07:00
parent 3d4a3d43c4
commit 5bbe293bea
4 changed files with 100 additions and 13 deletions
@@ -93,3 +93,66 @@ def test_softmax_merge_computes_m_l_O_rescale():
    assert np.allclose(m_got, m_new, atol=2e-2), f"m: {m_got.ravel()[:3]} vs {m_new.ravel()[:3]}"
    assert np.allclose(l_got, l_new, rtol=5e-2, atol=5e-2), "l mismatch"
    assert np.allclose(O_got, O_resc, rtol=5e-2, atol=5e-2), "O rescale mismatch"
+
+
+def test_softmax_merge_full_accumulates_O():
+    """The full recipe (prologue MATH + GEMM P·V + accumulate) in data mode:
+    O = O_old·corr + P·V (ADR-0065 N3). The GEMM record is recorded at
+    completion (sorts after the prologue) and accumulates into the rescaled O."""
+    from kernbench.sim_engine.op_log import OpRecord, _extract_op_info
+
+    rng = np.random.default_rng(1)
+    s = rng.standard_normal((G, TILE)).astype(np.float16)
+    m0 = rng.standard_normal((G, 1)).astype(np.float16)
+    l0 = np.abs(rng.standard_normal((G, 1))).astype(np.float16)
+    O0 = rng.standard_normal((G, D)).astype(np.float16)
+    Vd = rng.standard_normal((TILE, D)).astype(np.float16)
+
+    sh = _tcm(0x1000, (G, TILE))
+    mh = _tcm(0x2000, (G, 1))
+    lh = _tcm(0x3000, (G, 1))
+    Oh = _tcm(0x4000, (G, D))
+    V = TensorHandle(id="V", addr=0x5000, shape=(TILE, D), dtype="f16",
+                     nbytes=2 * TILE * D, space="hbm")
+
+    tl = TLContext(pe_id=0, num_programs=1, scratch_base=0x100000,
+                   scratch_size=1 << 20)
+    tl.composite(
+        prologue=[{"op": "softmax_merge", "s": sh, "m": mh, "l": lh, "O": Oh}],
+        op="gemm", b=V, out=Oh,
+        epilogue=[{"op": "add", "other": Oh}],
+    )
+    cmd = [c for c in tl.commands if isinstance(c, CompositeCmd)][-1]
+    plan = generate_plan_from_ops(cmd.ops, tile_m=128, tile_k=128, tile_n=128,
+                                  bytes_per_element=2,
+                                  pe_prefix="sip0.cube0.pe0")
+    _kind, _name, gparams = _extract_op_info(cmd)
+    assert gparams.get("accumulate") is True
+
+    store = MemoryStore()
+    store.write("tcm", sh.addr, s)
+    store.write("tcm", mh.addr, m0)
+    store.write("tcm", lh.addr, l0)
+    store.write("tcm", Oh.addr, O0)
+    store.write("hbm", V.addr, Vd)
+
+    records = [
+        OpRecord(t_start=float(i), t_end=float(i), component_id="pe_math",
+                 op_kind="math", op_name=st.params["op"], params=dict(st.params))
+        for i, st in enumerate(plan.prologue_stages)
+    ]
+    # GEMM recorded at completion → sorts after the prologue MATH ops.
+    records.append(OpRecord(t_start=100.0, t_end=100.0, component_id="pe_gemm",
+                            op_kind="gemm", op_name="composite_gemm",
+                            params=gparams))
+    DataExecutor(records, store).run()
+
+    sf = s.astype(np.float32)
+    m_new = np.maximum(m0.astype(np.float32), sf.max(-1, keepdims=True))
+    corr = np.exp(m0.astype(np.float32) - m_new)
+    P = np.exp(sf - m_new)
+    O_expected = O0.astype(np.float32) * corr + P @ Vd.astype(np.float32)
+
+    O_got = store.read("tcm", Oh.addr, shape=(G, D), dtype="f16").astype(np.float32)
+    assert np.allclose(O_got, O_expected, rtol=8e-2, atol=8e-2), \
+        f"O accumulate mismatch: {O_got[0, :3]} vs {O_expected[0, :3]}"