"""4-way comparative study — IPCQ vs HW alternatives. Architectures compared (the chosen design + 3 HW alternatives): Case 1) Doorbell + Polling (traditional MMIO doorbell) 2 DMA transactions (data + doorbell), peer polls or IRQ. Case 2) HMQ (Hardware Message Queue, NVLink-style) separate HMQ engine pushes descriptors, large tensors still need DMA → duplicated datapath. Case 3) RDMA-CQ (Completion Queue, InfiniBand / RoCE) DMA write → CQE auto-posted at peer → CQ poll / IRQ. Case 4) IPCQ ★ chosen (HW Ring + Credit Return) ring buffer + credit return, control plane in PE_IPCQ HW, data plane in PE_DMA, direction-addressed (E/W/N/S NoC + UCIe). Outputs PNGs into src/kernbench/benches/1H_milestone_output/IPCQ/: ipcq_alternatives_architecture.png 2×2 block diagram ipcq_alternatives_latency.png per-send timing stack (4 bars) ipcq_alternatives_opcounts.png control-plane op count ipcq_alternatives_sequence.png 2×2 swimlane flow ipcq_alternatives_decision_matrix.png 5-criterion "why IPCQ won" matrix Cycle / op counts are step-counts; numbers labelled "illustrative" are order-of-magnitude comparators (no benchmark target). """ from __future__ import annotations import textwrap from pathlib import Path import matplotlib.patches as mpatches import matplotlib.pyplot as plt _OUT_DIR = ( Path(__file__).resolve().parents[2] / "src" / "kernbench" / "benches" / "1H_milestone_output" / "IPCQ" ) _COLOR = { "ipcq": "#3b6ea5", # blue — chosen "doorbell": "#c0504d", # red "hmq": "#e07a3f", # orange "rdma": "#8064a2", # purple "neutral": "#888888", } _VERDICT = { "doorbell": "", "hmq": "", "rdma": "", "ipcq": "", } _TITLE = { "doorbell": "Case 1: Doorbell + Polling", "hmq": "Case 2: HMQ (HW Message Queue)", "rdma": "Case 3: RDMA-CQ (Completion Queue)", "ipcq": "Case 4 ★: IPCQ (HW Ring + Credit Return)", } _SHORT = { "doorbell": "Case 1:\nDoorbell + Polling", "hmq": "Case 2: HMQ\n(HW Message Queue)", "rdma": "Case 3: RDMA-CQ\n(Completion Queue)", "ipcq": "Case 4 ★: IPCQ\n(HW Ring + Credit Return)", } _ORDER = ["doorbell", "hmq", "rdma", "ipcq"] # ─── Diagram 1 : 2×2 architecture blocks ──────────────────────────── def _plot_architecture() -> Path: fig, axes = plt.subplots(2, 2, figsize=(17.0, 12.0)) def _block(ax, x, y, w, h, text, color="#dde3ec", border="#333", fontsize=9.5, bold=True): ax.add_patch(mpatches.FancyBboxPatch( (x, y), w, h, boxstyle="round,pad=0.02,rounding_size=0.04", facecolor=color, edgecolor=border, linewidth=1.4)) ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=fontsize, fontweight="bold" if bold else "normal") def _arrow(ax, x1, y1, x2, y2, color="#333", lw=1.7, style="-"): ax.annotate("", xy=(x2, y2), xytext=(x1, y1), arrowprops=dict(arrowstyle="->", color=color, lw=lw, ls=style, shrinkA=2, shrinkB=2)) def _verdict_box(ax, kind): # Name plate at the bottom of every panel. IPCQ (chosen) gets a # green background; the others get an architecture-tinted bg. is_chosen = kind == "ipcq" ax.text(5.0, 0.75, _TITLE[kind], ha="center", va="center", fontsize=11.5, fontweight="bold", color=("#1B5E20" if is_chosen else "white"), bbox=dict( facecolor=("#E8F5E9" if is_chosen else _COLOR[kind]), edgecolor=("#2E7D32" if is_chosen else _COLOR[kind]), boxstyle="round,pad=0.50", linewidth=1.8)) def _set_panel(ax, kind): ax.set_xlim(0, 10); ax.set_ylim(0, 10); ax.axis("off") # Visible frame around the panel, tinted by case color. ax.add_patch(mpatches.FancyBboxPatch( (0.15, 0.15), 9.7, 9.7, boxstyle="round,pad=0.05,rounding_size=0.10", facecolor="white", edgecolor=_COLOR[kind], linewidth=2.2, zorder=0)) # (D) IPCQ — Case 4, chosen ax = axes[1, 1]; _set_panel(ax, "ipcq") _block(ax, 0.5, 7.0, 2.4, 1.4, "Sender PE\nkernel\n(tl.send)", color="#cfe2ff", border=_COLOR["ipcq"]) _block(ax, 0.5, 4.6, 2.4, 1.5, "PE_IPCQ (HW)\nhead/tail/credit\n→ in-line ring push", color="#dee9f5", border=_COLOR["ipcq"], fontsize=8) _block(ax, 0.5, 2.2, 2.4, 1.5, "PE_DMA\ndirect push\n(no host CPU)", color="#dee9f5", border=_COLOR["ipcq"], fontsize=8) _block(ax, 4.0, 3.5, 2.0, 2.0, "NoC / UCIe\n(direction-addr)", color="#f5f5f5", border=_COLOR["neutral"], fontsize=8.5) _block(ax, 7.2, 2.2, 2.4, 1.5, "PE_DMA\nlanding\nTCM", color="#dee9f5", border=_COLOR["ipcq"], fontsize=8) _block(ax, 7.2, 4.6, 2.4, 1.5, "PE_IPCQ (HW)\ntail advance +\ncredit piggyback", color="#dee9f5", border=_COLOR["ipcq"], fontsize=8) _block(ax, 7.2, 7.0, 2.4, 1.4, "Receiver PE\nkernel\n(tl.recv)", color="#cfe2ff", border=_COLOR["ipcq"]) _arrow(ax, 1.7, 7.0, 1.7, 6.1, color=_COLOR["ipcq"]) _arrow(ax, 1.7, 4.6, 1.7, 3.7, color=_COLOR["ipcq"]) _arrow(ax, 2.9, 2.95, 4.0, 4.0, color=_COLOR["ipcq"]) _arrow(ax, 6.0, 4.0, 7.2, 2.95, color=_COLOR["ipcq"]) _arrow(ax, 8.4, 3.7, 8.4, 4.6, color=_COLOR["ipcq"]) _arrow(ax, 8.4, 6.1, 8.4, 7.0, color=_COLOR["ipcq"]) ax.annotate("", xy=(2.9, 5.35), xytext=(7.2, 5.35), arrowprops=dict(arrowstyle="->", color=_COLOR["ipcq"], ls=":", lw=1.0)) ax.text(5.0, 5.6, "credit return (piggyback)", ha="center", fontsize=7.5, style="italic", color=_COLOR["ipcq"]) _verdict_box(ax, "ipcq") # (A) Doorbell + polling — Case 1 ax = axes[0, 0]; _set_panel(ax, "doorbell") _block(ax, 0.5, 7.0, 2.4, 1.4, "Sender\nkernel", color="#fde2e0", border=_COLOR["doorbell"]) _block(ax, 0.5, 4.6, 2.4, 1.5, "DMA write\n(data)", color="#f5e0dd", border=_COLOR["doorbell"], fontsize=8.5) _block(ax, 0.5, 2.2, 2.4, 1.5, "DMA write\n(doorbell)", color="#f5e0dd", border=_COLOR["doorbell"], fontsize=8.5) _block(ax, 4.0, 3.5, 2.0, 2.0, "NoC / UCIe", color="#f5f5f5", border=_COLOR["neutral"], fontsize=8.5) _block(ax, 7.2, 4.6, 2.4, 1.5, "Doorbell\nregister", color="#f5e0dd", border=_COLOR["doorbell"], fontsize=8.5) _block(ax, 7.2, 2.2, 2.4, 1.5, "Receiver\nlanding TCM", color="#f5e0dd", border=_COLOR["doorbell"], fontsize=8.5) _block(ax, 7.2, 7.0, 2.4, 1.4, "Receiver\nkernel\n(poll / IRQ)", color="#fde2e0", border=_COLOR["doorbell"]) _arrow(ax, 1.7, 7.0, 1.7, 6.1, color=_COLOR["doorbell"]) _arrow(ax, 1.7, 4.6, 1.7, 3.7, color=_COLOR["doorbell"]) _arrow(ax, 2.9, 5.35, 4.0, 4.5, color=_COLOR["doorbell"]) _arrow(ax, 2.9, 2.95, 4.0, 3.7, color=_COLOR["doorbell"]) _arrow(ax, 6.0, 4.5, 7.2, 5.35, color=_COLOR["doorbell"]) _arrow(ax, 6.0, 3.7, 7.2, 2.95, color=_COLOR["doorbell"]) _arrow(ax, 8.4, 6.1, 8.4, 7.0, color=_COLOR["doorbell"]) _verdict_box(ax, "doorbell") # (B) HMQ — Case 2 ax = axes[0, 1]; _set_panel(ax, "hmq") _block(ax, 0.5, 7.0, 2.4, 1.4, "Sender CPU\nbuild desc.", color="#ffe0c4", border=_COLOR["hmq"]) _block(ax, 0.5, 4.6, 2.4, 1.5, "HMQ engine\n(push desc.)", color="#f5e0c8", border=_COLOR["hmq"], fontsize=8.5) _block(ax, 0.5, 2.2, 2.4, 1.5, "PE_DMA\n(large tensor)", color="#f5e0c8", border=_COLOR["hmq"], fontsize=8.5) _block(ax, 4.0, 3.5, 2.0, 2.0, "NoC / UCIe\n(both paths)", color="#f5f5f5", border=_COLOR["neutral"], fontsize=8.5) _block(ax, 7.2, 4.6, 2.4, 1.5, "Peer HMQ\n(pop desc.)", color="#f5e0c8", border=_COLOR["hmq"], fontsize=8.5) _block(ax, 7.2, 2.2, 2.4, 1.5, "PE_DMA\nlanding TCM", color="#f5e0c8", border=_COLOR["hmq"], fontsize=8.5) _block(ax, 7.2, 7.0, 2.4, 1.4, "Receiver CPU\nuse data ptr", color="#ffe0c4", border=_COLOR["hmq"]) _arrow(ax, 1.7, 7.0, 1.7, 6.1, color=_COLOR["hmq"]) _arrow(ax, 1.7, 4.6, 1.7, 3.7, color=_COLOR["hmq"]) _arrow(ax, 2.9, 5.35, 4.0, 4.5, color=_COLOR["hmq"], style="--") _arrow(ax, 2.9, 2.95, 4.0, 3.7, color=_COLOR["hmq"]) _arrow(ax, 6.0, 4.5, 7.2, 5.35, color=_COLOR["hmq"], style="--") _arrow(ax, 6.0, 3.7, 7.2, 2.95, color=_COLOR["hmq"]) _arrow(ax, 8.4, 4.6, 8.4, 3.7, color=_COLOR["hmq"], style=":") _arrow(ax, 8.4, 6.1, 8.4, 7.0, color=_COLOR["hmq"]) ax.text(5.0, 6.6, "desc path (dashed) + DMA path (solid)\n" "= duplicated datapath", ha="center", fontsize=7.5, style="italic", color=_COLOR["hmq"]) _verdict_box(ax, "hmq") # (C) RDMA-CQ — Case 3 ax = axes[1, 0]; _set_panel(ax, "rdma") _block(ax, 0.5, 7.0, 2.4, 1.4, "Sender\nkernel", color="#e6d8f0", border=_COLOR["rdma"]) _block(ax, 0.5, 4.6, 2.4, 1.5, "DMA write\n(carries CQE\ntag)", color="#ddd2eb", border=_COLOR["rdma"], fontsize=8.5) _block(ax, 0.5, 2.2, 2.4, 1.5, "WQE\n(work-queue entry)", color="#ddd2eb", border=_COLOR["rdma"], fontsize=8.5) _block(ax, 4.0, 3.5, 2.0, 2.0, "NoC / UCIe", color="#f5f5f5", border=_COLOR["neutral"], fontsize=8.5) _block(ax, 7.2, 4.6, 2.4, 1.5, "CQ\n(auto-posted\nCQE)", color="#ddd2eb", border=_COLOR["rdma"], fontsize=8.5) _block(ax, 7.2, 2.2, 2.4, 1.5, "Receiver\nlanding TCM", color="#ddd2eb", border=_COLOR["rdma"], fontsize=8.5) _block(ax, 7.2, 7.0, 2.4, 1.4, "Receiver\nkernel\n(poll CQ / IRQ)", color="#e6d8f0", border=_COLOR["rdma"]) _arrow(ax, 1.7, 7.0, 1.7, 6.1, color=_COLOR["rdma"]) _arrow(ax, 1.7, 4.6, 1.7, 3.7, color=_COLOR["rdma"]) _arrow(ax, 2.9, 5.35, 4.0, 4.5, color=_COLOR["rdma"]) _arrow(ax, 2.9, 2.95, 4.0, 3.7, color=_COLOR["rdma"], style="--") _arrow(ax, 6.0, 4.5, 7.2, 5.35, color=_COLOR["rdma"]) _arrow(ax, 6.0, 3.7, 7.2, 2.95, color=_COLOR["rdma"], style="--") _arrow(ax, 8.4, 6.1, 8.4, 7.0, color=_COLOR["rdma"]) _verdict_box(ax, "rdma") fig.suptitle( "IPCQ HW Alternatives — Architecture Comparison", fontsize=14.5, fontweight="bold", y=0.995) fig.tight_layout(rect=(0, 0, 1, 0.97)) out = _OUT_DIR / "ipcq_alternatives_architecture.png" fig.savefig(out, dpi=150, bbox_inches="tight") plt.close(fig) return out # ─── Diagram 2 : Per-send latency breakdown (vertical stacks) ─────── def _plot_latency() -> Path: # (label, short, illustrative cycles, fill) steps = { "ipcq": [ ("kernel tl.send", "tl.send", 5, "#7CA7D9"), ("PE_IPCQ ring push", "ring push", 4, "#9CBCDE"), ("PE_DMA push (NoC hop)", "DMA push", 12, "#BCD2E5"), ("PE_IPCQ tail + credit", "tail+credit", 4, "#DCE5ED"), ("kernel tl.recv unblock", "tl.recv", 3, "#EEF2F6"), ], "doorbell": [ ("DMA write data", "DMA(data)", 12, "#D58680"), ("Fence (data before doorbell)", "fence", 6, "#DC9C95"), ("DMA write doorbell", "DMA(db)", 10, "#E1AFA8"), ("Receiver poll loop / IRQ", "poll / IRQ", 25, "#E8C4BE"), ("Kernel use data", "use data", 3, "#EFD8D4"), ], "hmq": [ ("CPU build descriptor", "build desc", 8, "#E29465"), ("HMQ push (control path)", "HMQ push", 7, "#E5A57F"), ("DMA push (data, parallel)", "DMA push", 12, "#E8B69A"), ("Peer HMQ pop", "HMQ pop", 7, "#ECC9B6"), ("CPU read data ptr", "read ptr", 4, "#F1DBCE"), ], "rdma": [ ("Build WQE", "WQE", 8, "#A68FB8"), ("DMA write + CQE post", "DMA+CQE", 14, "#B6A4C4"), ("CQE arrives in peer CQ", "CQE arrive", 6, "#C5B9D0"), ("CQ poll / IRQ", "poll / IRQ", 22, "#D5CEDD"), ("Kernel use data", "use data", 3, "#E5E2EA"), ], } totals = {k: sum(c for _, _, c, _ in steps[k]) for k in _ORDER} base = totals["ipcq"] fig, ax = plt.subplots(figsize=(15.0, 9.0)) x = list(range(len(_ORDER))) for i, k in enumerate(_ORDER): bottom = 0 for name, short, cy, color in steps[k]: ax.bar(x[i], cy, bottom=bottom, width=0.62, color=color, edgecolor="black", linewidth=0.7) # Wrap label so it fits within bar width. if cy >= 8: wrapped = textwrap.fill(name, width=18) label = f"{wrapped}\n({cy} cy)" fontsize = 8.5 else: wrapped = textwrap.fill(short, width=12) label = f"{wrapped}\n({cy})" fontsize = 7.5 ax.text(x[i], bottom + cy / 2, label, ha="center", va="center", fontsize=fontsize, fontweight="bold", color="#111") bottom += cy # Total above bar. ax.text(x[i], bottom + max(totals.values()) * 0.012, f"{bottom} cy ({bottom/base:.1f}× IPCQ)", ha="center", va="bottom", fontsize=10.5, fontweight="bold", color=_COLOR[k]) # X-tick labels: case names (★ already baked into _SHORT for IPCQ). ax.set_xticks(x) ax.set_xticklabels([_SHORT[k] for k in _ORDER], fontsize=10.5, fontweight="bold") for tick, k in zip(ax.get_xticklabels(), _ORDER): tick.set_color(_COLOR[k]) ax.set_ylabel("illustrative cycles per single PE-to-PE send") ax.set_ylim(0, max(totals.values()) * 1.18) ax.grid(axis="y", ls=":", alpha=0.5) ax.set_title( "Per-send timing — control + data path " "(illustrative cycle stack)", fontsize=12.5, fontweight="bold", pad=12) fig.tight_layout() out = _OUT_DIR / "ipcq_alternatives_latency.png" fig.savefig(out, dpi=150) plt.close(fig) return out # ─── Diagram 3 : Op-count / control-plane traffic ─────────────────── def _plot_opcounts() -> Path: # Each row = one ops-category, columns = the 4 alternatives. categories = [ "DMA transactions\n(data path)", "Doorbell\nregister writes", "HMQ descriptor\npush/pop pairs", "CQE\n(completion entry)", "Polling /\nIRQ wake-ups", "Ring slot\nadvance (HW)", "Credit return\npacket", ] # Per-architecture op count (in declared _ORDER: door,hmq,rdma,ipcq). counts_by_kind = { # cat: door hmq rdma ipcq "doorbell": [2, 1, 0, 0, 1, 0, 0], "hmq": [1, 0, 1, 0, 0, 0, 0], "rdma": [1, 0, 0, 1, 1, 0, 0], "ipcq": [1, 0, 0, 0, 0, 1, 1], } totals = {k: sum(counts_by_kind[k]) for k in _ORDER} x = list(range(len(categories))) w = 0.18 fig, ax = plt.subplots(figsize=(14.0, 6.0)) offsets = [-1.5 * w, -0.5 * w, 0.5 * w, 1.5 * w] for c, kind in enumerate(_ORDER): vals = counts_by_kind[kind] ax.bar([xi + offsets[c] for xi in x], vals, width=w, color=_COLOR[kind], edgecolor="black", label=f"{_SHORT[kind]} (total {totals[kind]})") ax.set_xticks(x) ax.set_xticklabels(categories, fontsize=9) ax.set_ylabel("ops per single PE-to-PE data transfer") ax.set_yticks([0, 1, 2]) ax.set_title( "Control-plane op count per data transfer\n" "Case 1: 2 DMA + poll · Case 2: desc push/pop + DMA · " "Case 3: DMA + CQE + poll · " "Case 4 (IPCQ): 1 DMA + in-line ring + credit (piggybacked v2)", fontsize=11.5, fontweight="bold") ax.grid(axis="y", ls=":", alpha=0.5) ax.legend(loc="upper right", fontsize=9.5, framealpha=0.95, ncol=2) ax.set_ylim(0, 2.6) fig.tight_layout() out = _OUT_DIR / "ipcq_alternatives_opcounts.png" fig.savefig(out, dpi=150) plt.close(fig) return out # ─── Diagram 4 : 2×2 swimlane sequence per implementation ─────────── def _plot_sequence() -> Path: fig, axes = plt.subplots(2, 2, figsize=(16.0, 11.0)) def _lane(ax, x, label, color): ax.axvline(x, ymin=0.04, ymax=0.96, color=color, lw=2.0, alpha=0.7) ax.text(x, 9.65, label, ha="center", va="bottom", fontsize=10, fontweight="bold", color=color) def _msg(ax, x1, x2, y, text, color="#333", style="-"): ax.annotate("", xy=(x2, y), xytext=(x1, y), arrowprops=dict(arrowstyle="->", color=color, lw=1.3, ls=style)) ax.text((x1 + x2) / 2, y + 0.18, text, ha="center", fontsize=8, color="#222", fontweight="bold", bbox=dict(facecolor="white", edgecolor="none", pad=1.0)) def _act(ax, x, y_top, y_bot, color="#9bbb59", w=0.30): ax.add_patch(mpatches.Rectangle( (x - w / 2, y_bot), w, y_top - y_bot, facecolor=color, edgecolor="black", linewidth=0.4, alpha=0.5)) def _setup(ax, kind, subtitle): ax.set_xlim(0, 10); ax.set_ylim(0, 10); ax.axis("off") ax.set_title(f"{_TITLE[kind]}\n{subtitle}", fontsize=11, fontweight="bold", color=_COLOR[kind], pad=8) def _verdict(ax, kind): if not _VERDICT[kind]: return ax.text(5.0, 0.5, _VERDICT[kind], ha="center", va="center", fontsize=10, fontweight="bold", bbox=dict(facecolor="#E8F5E9", edgecolor="#2E7D32", boxstyle="round,pad=0.4", linewidth=1.5)) # (D) IPCQ — Case 4, chosen ax = axes[1, 1] _setup(ax, "ipcq", "tl.send → ring push → DMA → tail advance + credit piggyback → tl.recv") _lane(ax, 1.5, "Sender\nkernel", _COLOR["ipcq"]) _lane(ax, 5.0, "PE_IPCQ +\nPE_DMA (HW)", "#666") _lane(ax, 8.5, "Receiver\nkernel", _COLOR["ipcq"]) _act(ax, 1.5, 9.0, 7.0) _msg(ax, 1.5, 5.0, 8.4, "tl.send(payload)", color=_COLOR["ipcq"]) _act(ax, 5.0, 7.0, 5.0, color="#88aacc") _msg(ax, 5.0, 8.5, 6.0, "DMA push → TCM", color=_COLOR["ipcq"]) _msg(ax, 5.0, 1.5, 5.2, "tail advance (HW)", color=_COLOR["ipcq"], style=":") _act(ax, 8.5, 5.5, 3.0) _msg(ax, 8.5, 1.5, 3.7, "credit (piggyback)", color=_COLOR["ipcq"], style=":") _msg(ax, 8.5, 8.5, 2.5, "tl.recv → unblock", color=_COLOR["ipcq"]) _verdict(ax, "ipcq") # (A) Doorbell + polling — Case 1 ax = axes[0, 0] _setup(ax, "doorbell", "DMA(data) → fence → DMA(doorbell) → peer polls") _lane(ax, 1.5, "Sender\nkernel", _COLOR["doorbell"]) _lane(ax, 4.3, "PE_DMA", "#666") _lane(ax, 6.7, "Doorbell\nreg + TCM", "#666") _lane(ax, 9.0, "Receiver\nkernel", _COLOR["doorbell"]) _act(ax, 1.5, 9.0, 6.5) _msg(ax, 1.5, 4.3, 8.4, "issue DMA(data)", color=_COLOR["doorbell"]) _msg(ax, 4.3, 6.7, 7.5, "data → TCM", color=_COLOR["doorbell"]) _msg(ax, 1.5, 4.3, 6.8, "fence + DMA(doorbell)", color=_COLOR["doorbell"]) _msg(ax, 4.3, 6.7, 6.1, "doorbell write", color=_COLOR["doorbell"]) _act(ax, 9.0, 5.5, 1.5) _msg(ax, 9.0, 6.7, 4.5, "poll doorbell ...", color=_COLOR["doorbell"], style=":") _msg(ax, 9.0, 6.7, 3.5, " poll doorbell ...", color=_COLOR["doorbell"], style=":") _msg(ax, 9.0, 6.7, 2.5, " poll → set", color=_COLOR["doorbell"]) _verdict(ax, "doorbell") # (B) HMQ — Case 2 ax = axes[0, 1] _setup(ax, "hmq", "CPU build desc → HMQ push (control) + DMA (data) → peer pop + DMA recv") _lane(ax, 1.0, "Sender\nCPU", _COLOR["hmq"]) _lane(ax, 3.4, "HMQ\nengine", "#666") _lane(ax, 5.6, "PE_DMA", "#666") _lane(ax, 7.8, "Peer HMQ +\nTCM", "#666") _lane(ax, 9.5, "Receiver\nCPU", _COLOR["hmq"]) _act(ax, 1.0, 9.0, 6.5) _msg(ax, 1.0, 3.4, 8.4, "build desc", color=_COLOR["hmq"]) _msg(ax, 1.0, 5.6, 7.5, "DMA push (data)", color=_COLOR["hmq"]) _msg(ax, 3.4, 7.8, 6.8, "desc relay", color=_COLOR["hmq"], style="--") _msg(ax, 5.6, 7.8, 5.8, "data → TCM", color=_COLOR["hmq"]) _act(ax, 9.5, 5.0, 2.0) _msg(ax, 9.5, 7.8, 4.0, "pop desc", color=_COLOR["hmq"], style="--") _msg(ax, 9.5, 9.5, 3.0, "use ptr", color=_COLOR["hmq"]) _verdict(ax, "hmq") # (C) RDMA-CQ — Case 3 ax = axes[1, 0] _setup(ax, "rdma", "WQE post → DMA + auto-CQE → peer CQ poll / IRQ") _lane(ax, 1.0, "Sender\nkernel", _COLOR["rdma"]) _lane(ax, 3.4, "WQE +\nDMA", "#666") _lane(ax, 5.8, "NoC", "#666") _lane(ax, 7.6, "Peer CQ +\nTCM", "#666") _lane(ax, 9.5, "Receiver\nkernel", _COLOR["rdma"]) _act(ax, 1.0, 9.0, 6.5) _msg(ax, 1.0, 3.4, 8.4, "post WQE", color=_COLOR["rdma"]) _msg(ax, 3.4, 5.8, 7.5, "DMA + CQE tag", color=_COLOR["rdma"]) _msg(ax, 5.8, 7.6, 6.5, "data + CQE", color=_COLOR["rdma"]) _act(ax, 9.5, 5.5, 1.5) _msg(ax, 9.5, 7.6, 4.5, "poll CQ ...", color=_COLOR["rdma"], style=":") _msg(ax, 9.5, 7.6, 3.5, " poll CQ ...", color=_COLOR["rdma"], style=":") _msg(ax, 9.5, 7.6, 2.5, " CQE arrived", color=_COLOR["rdma"]) _verdict(ax, "rdma") fig.suptitle( "PE-to-PE communication flow — IPCQ vs alternatives " "(swimlane sequence)", fontsize=14, fontweight="bold", y=0.995) fig.tight_layout(rect=(0, 0, 1, 0.97)) out = _OUT_DIR / "ipcq_alternatives_sequence.png" fig.savefig(out, dpi=150, bbox_inches="tight") plt.close(fig) return out # ─── Diagram 5 : Decision matrix — why IPCQ won ──────────────────── def _plot_decision_matrix() -> Path: """Side-by-side comparison across the 5 criteria. Rows = criteria, columns = the 4 architectures in case order. Green ✓ highlights the cells where IPCQ wins; everything else is neutral. """ # Rows in column order: doorbell, hmq, rdma, ipcq. criteria = [ ("Latency (single send)", "High (2 DMAs +\n fence + poll/IRQ)", "Mid (desc relay\n + DMA parallel)", "High (DMA + CQE\n + poll/IRQ)", "Low (5 events,\n ~28 cy in-line)"), ("Host CPU on critical path?", "Yes (issue both\n DMAs + poll)", "Yes (CPU builds\n descriptor)", "Yes (post WQE\n + poll CQ)", "No (kernel-side\n HW push)"), ("Polling / IRQ wake-up?", "Yes (poll or IRQ\n on doorbell)", "No (CPU just\n reads desc ptr)", "Yes (poll or IRQ\n on CQ)", "No (tail advance\n + piggyback credit)"), ("Duplicated control + data\n datapaths?", "No (1 datapath,\n but 2 DMAs)", "Yes (HMQ engine\n + DMA in parallel)", "Partial (CQE rides\n with DMA)", "No (PE_IPCQ ctrl,\n PE_DMA data)"), ("Right-sized for single-owner\n PE-to-PE?", "Yes", "Yes (but extra HW)", "Multi-tenant focus\n (extra isolation)", "Yes"), ] cols = [_SHORT[k] for k in _ORDER] # Green ✓ only where IPCQ has a clear win. good = {(r, "ipcq") for r in range(len(criteria))} fig, ax = plt.subplots(figsize=(14.0, 8.0)) ax.axis("off") n_rows = len(criteria) + 1 n_cols = len(cols) + 1 col_w = [3.0, 2.2, 2.2, 2.2, 2.2] # narrower cells row_h = 1.30 # Header row headers = ["Criterion"] + cols x_left = 0.0 y_top = (n_rows - 1) * row_h x_acc = [x_left] for cw in col_w: x_acc.append(x_acc[-1] + cw) fig_w = x_acc[-1] fig_h = n_rows * row_h for ci, h in enumerate(headers): cw = col_w[ci] cx = x_acc[ci] if ci == 0: fill = "#1F4E79"; text_color = "white" text = h else: kind = _ORDER[ci - 1] fill = _COLOR[kind]; text_color = "white" text = h + ("\n" + _VERDICT[kind] if _VERDICT[kind] else "") ax.add_patch(mpatches.Rectangle( (cx, y_top), cw, row_h, facecolor=fill, edgecolor="black", linewidth=0.8)) ax.text(cx + cw / 2, y_top + row_h / 2, text, ha="center", va="center", fontsize=11, fontweight="bold", color=text_color) # Body rows for ri, row in enumerate(criteria): rname, *cells = row # Bottom-most rows render first → reverse so first criterion # sits just below the header. y = (n_rows - 2 - ri) * row_h # Criterion cell. ax.add_patch(mpatches.Rectangle( (x_acc[0], y), col_w[0], row_h, facecolor="#F0F4F8", edgecolor="black", linewidth=0.5)) ax.text(x_acc[0] + 0.15, y + row_h / 2, rname, ha="left", va="center", fontsize=11.5, fontweight="bold", color="#1F4E79") # Architecture cells. for ci, kind in enumerate(_ORDER): cx = x_acc[ci + 1]; cw = col_w[ci + 1] # Verdict word (Yes/No/High/Mid/Low/Partial/etc.) is the # part before the first "(" — bolded. Bracketed qualifier # stays regular weight. raw = " ".join(cells[ci].split()) if "(" in raw: pre, _, rest = raw.partition("(") verdict = pre.strip() bracket = textwrap.fill("(" + rest.strip(), width=22) else: verdict = raw bracket = "" if (ri, kind) in good: fill = "#E8F5E9"; mark = "✓ " else: fill = "#F5F7FA"; mark = "" ax.add_patch(mpatches.Rectangle( (cx, y), cw, row_h, facecolor=fill, edgecolor="black", linewidth=0.5)) cy = y + row_h / 2 if bracket: ax.text(cx + cw / 2, cy + 0.30, mark + verdict, ha="center", va="center", fontsize=11.5, fontweight="bold", color="#111") ax.text(cx + cw / 2, cy - 0.30, bracket, ha="center", va="center", fontsize=10, color="#444") else: ax.text(cx + cw / 2, cy, mark + verdict, ha="center", va="center", fontsize=11.5, fontweight="bold", color="#111") ax.set_xlim(-0.1, fig_w + 0.1) ax.set_ylim(-0.1, fig_h + 0.1) ax.set_title( "Why IPCQ (HW Ring + Credit Return)? — decision matrix", fontsize=14, fontweight="bold", pad=14) fig.tight_layout() out = _OUT_DIR / "ipcq_alternatives_decision_matrix.png" fig.savefig(out, dpi=150, bbox_inches="tight") plt.close(fig) return out # ─── Diagram 7 : Control vs Data plane overlay (4 panels) ─────────── def _plot_control_data_overlay() -> Path: """2×2 panels (one per case). Each panel uses solid arrows for data path and dashed for control path so the reader sees where each design moves the control burden.""" fig, axes = plt.subplots(2, 2, figsize=(16.0, 11.0)) DATA = "#1b4f8a" CTRL = "#c0504d" def _box(ax, x, y, w, h, text, fontsize=9): ax.add_patch(mpatches.FancyBboxPatch( (x, y), w, h, boxstyle="round,pad=0.02,rounding_size=0.06", facecolor="#f7f7f7", edgecolor="#333", linewidth=1.2)) ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=fontsize, fontweight="bold", color="#111") def _arrow(ax, x1, y1, x2, y2, kind, lw=2.0): color, style = (DATA, "-") if kind == "data" else (CTRL, (0, (5, 3))) ax.annotate("", xy=(x2, y2), xytext=(x1, y1), arrowprops=dict(arrowstyle="->", color=color, lw=lw, ls=style, shrinkA=3, shrinkB=3)) def _setup(ax, kind): ax.set_xlim(0, 10); ax.set_ylim(0, 10); ax.axis("off") ax.add_patch(mpatches.FancyBboxPatch( (0.15, 0.15), 9.7, 9.7, boxstyle="round,pad=0.05,rounding_size=0.10", facecolor="white", edgecolor=_COLOR[kind], linewidth=2.2, zorder=0)) ax.text(5.0, 0.65, _TITLE[kind], ha="center", va="center", fontsize=11.5, fontweight="bold", color=("#1B5E20" if kind == "ipcq" else "white"), bbox=dict( facecolor=("#E8F5E9" if kind == "ipcq" else _COLOR[kind]), edgecolor=("#2E7D32" if kind == "ipcq" else _COLOR[kind]), boxstyle="round,pad=0.40", linewidth=1.5)) # Helper to draw sender/NoC/receiver row of 5 nodes. def _row(ax, labels): xs = [1.0, 3.0, 5.0, 7.0, 9.0] positions = [] for x, lbl in zip(xs, labels): _box(ax, x - 0.9, 5.0, 1.8, 1.6, lbl, fontsize=8.5) positions.append((x, 5.8)) return positions # ── Case 1: Doorbell + Polling ── ax = axes[0, 0]; _setup(ax, "doorbell") pos = _row(ax, ["Sender\nkernel", "PE_DMA", "NoC / UCIe", "Doorbell\nreg + TCM", "Receiver\nkernel (poll)"]) # data path: kernel→DMA→NoC→TCM (top of node row visually) _arrow(ax, pos[0][0] + 0.9, 5.8, pos[1][0] - 0.9, 5.8, "data") _arrow(ax, pos[1][0] + 0.9, 6.0, pos[2][0] - 0.9, 6.0, "data") _arrow(ax, pos[2][0] + 0.9, 6.0, pos[3][0] - 0.9, 6.0, "data") # control path: sender→DMA (doorbell submit)→NoC→doorbell reg→poll _arrow(ax, pos[0][0] + 0.9, 5.4, pos[1][0] - 0.9, 5.4, "ctrl") _arrow(ax, pos[1][0] + 0.9, 5.2, pos[2][0] - 0.9, 5.2, "ctrl") _arrow(ax, pos[2][0] + 0.9, 5.2, pos[3][0] - 0.9, 5.2, "ctrl") _arrow(ax, pos[4][0] - 0.9, 5.2, pos[3][0] + 0.9, 5.2, "ctrl") # poll ax.text(5.0, 7.4, "2 DMA transactions + receiver polling loop", ha="center", fontsize=10.5, color="#333", style="italic") # ── Case 2: HMQ ── ax = axes[0, 1]; _setup(ax, "hmq") pos = _row(ax, ["Sender\nCPU", "HMQ\nengine", "NoC / UCIe", "Peer\nHMQ + TCM", "Receiver\nCPU"]) # control: CPU → HMQ → NoC → Peer HMQ _arrow(ax, pos[0][0] + 0.9, 6.0, pos[1][0] - 0.9, 6.0, "ctrl") _arrow(ax, pos[1][0] + 0.9, 6.0, pos[2][0] - 0.9, 6.0, "ctrl") _arrow(ax, pos[2][0] + 0.9, 6.0, pos[3][0] - 0.9, 6.0, "ctrl") _arrow(ax, pos[3][0] + 0.9, 6.0, pos[4][0] - 0.9, 6.0, "ctrl") # data (parallel path via PE_DMA): sender CPU → NoC → TCM _arrow(ax, pos[0][0] + 0.9, 5.2, pos[2][0] - 0.9, 5.2, "data") _arrow(ax, pos[2][0] + 0.9, 5.2, pos[3][0] - 0.9, 5.2, "data") ax.text(5.0, 7.4, "control + data on TWO parallel paths " "(duplicated)", ha="center", fontsize=10.5, color="#333", style="italic") # ── Case 3: RDMA-CQ ── ax = axes[1, 0]; _setup(ax, "rdma") pos = _row(ax, ["Sender\nkernel", "WQE + DMA", "NoC / UCIe", "Peer CQ\n+ TCM", "Receiver\nkernel (poll)"]) # data: kernel → DMA → NoC → TCM (with CQE tag) _arrow(ax, pos[0][0] + 0.9, 5.8, pos[1][0] - 0.9, 5.8, "data") _arrow(ax, pos[1][0] + 0.9, 5.8, pos[2][0] - 0.9, 5.8, "data") _arrow(ax, pos[2][0] + 0.9, 5.8, pos[3][0] - 0.9, 5.8, "data") # control: post WQE + auto-CQE arrives + poll/IRQ _arrow(ax, pos[0][0] + 0.9, 5.2, pos[1][0] - 0.9, 5.2, "ctrl") _arrow(ax, pos[2][0] + 0.9, 5.0, pos[3][0] - 0.9, 5.0, "ctrl") _arrow(ax, pos[4][0] - 0.9, 5.0, pos[3][0] + 0.9, 5.0, "ctrl") ax.text(5.0, 7.4, "CQE rides with DMA + CQ poll/IRQ at receiver", ha="center", fontsize=10.5, color="#333", style="italic") # ── Case 4: IPCQ ── ax = axes[1, 1]; _setup(ax, "ipcq") pos = _row(ax, ["Sender PE\n(tl.send)", "PE_IPCQ\n+ PE_DMA", "NoC / UCIe", "PE_IPCQ\n+ PE_DMA", "Receiver PE\n(tl.recv)"]) # data: kernel → PE_DMA → NoC → TCM _arrow(ax, pos[0][0] + 0.9, 5.8, pos[1][0] - 0.9, 5.8, "data") _arrow(ax, pos[1][0] + 0.9, 5.8, pos[2][0] - 0.9, 5.8, "data") _arrow(ax, pos[2][0] + 0.9, 5.8, pos[3][0] - 0.9, 5.8, "data") _arrow(ax, pos[3][0] + 0.9, 5.8, pos[4][0] - 0.9, 5.8, "data") # control: PE_IPCQ ring push + meta + credit piggyback _arrow(ax, pos[1][0] + 0.9, 5.2, pos[2][0] - 0.9, 5.2, "ctrl") _arrow(ax, pos[2][0] + 0.9, 5.2, pos[3][0] - 0.9, 5.2, "ctrl") # credit return arrow (dashed back) ax.annotate("", xy=(pos[1][0] - 0.4, 6.8), xytext=(pos[3][0] + 0.4, 6.8), arrowprops=dict(arrowstyle="->", color="#2E7D32", lw=1.6, ls=(0, (4, 3)))) ax.text(5.0, 7.05, "credit piggyback (no separate ACK)", ha="center", fontsize=9, color="#2E7D32", fontweight="bold") ax.text(5.0, 7.6, "single in-line control token + single DMA " "(no host CPU, no polling)", ha="center", fontsize=10.5, color="#333", style="italic") # Global figure legend. fig.text(0.5, 0.025, "— Data path - - - Control path " "- - - Credit return", ha="center", va="center", fontsize=11.5, fontweight="bold") fig.suptitle( "Control vs Data Plane Overlay — where each design moves " "the control burden", fontsize=14.5, fontweight="bold", y=0.995) fig.tight_layout(rect=(0, 0.04, 1, 0.97)) out = _OUT_DIR / "ipcq_alternatives_control_data_overlay.png" fig.savefig(out, dpi=150, bbox_inches="tight") plt.close(fig) return out def main() -> None: _OUT_DIR.mkdir(parents=True, exist_ok=True) for plot in (_plot_architecture, _plot_latency, _plot_opcounts, _plot_sequence, _plot_decision_matrix, _plot_control_data_overlay): print(f"wrote {plot()}") if __name__ == "__main__": main()