diff --git a/docs/report/1H-codesign-paper/build/main.pdf b/docs/report/1H-codesign-paper/build/main.pdf index 7917a43..7f7e0a4 100644 Binary files a/docs/report/1H-codesign-paper/build/main.pdf and b/docs/report/1H-codesign-paper/build/main.pdf differ diff --git a/docs/report/1H-codesign-paper/figures/gemm_hbm_bw_util.png b/docs/report/1H-codesign-paper/figures/gemm_hbm_bw_util.png new file mode 100644 index 0000000..daba8bd Binary files /dev/null and b/docs/report/1H-codesign-paper/figures/gemm_hbm_bw_util.png differ diff --git a/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_measured.png b/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_measured.png index 7226df2..72cda5d 100644 Binary files a/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_measured.png and b/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_measured.png differ diff --git a/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_theoretical_vs_measured.png b/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_theoretical_vs_measured.png index fdbcb5b..87c0a7d 100644 Binary files a/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_theoretical_vs_measured.png and b/docs/report/1H-codesign-paper/figures/gemm_mac_utilization_theoretical_vs_measured.png differ diff --git a/docs/report/1H-codesign-paper/figures/gemm_per_pe_tflops.png b/docs/report/1H-codesign-paper/figures/gemm_per_pe_tflops.png new file mode 100644 index 0000000..c09da36 Binary files /dev/null and b/docs/report/1H-codesign-paper/figures/gemm_per_pe_tflops.png differ diff --git a/docs/report/1H-codesign-paper/figures/gemm_stage_breakdown.png b/docs/report/1H-codesign-paper/figures/gemm_stage_breakdown.png index 1058993..6cbb77e 100644 Binary files a/docs/report/1H-codesign-paper/figures/gemm_stage_breakdown.png and b/docs/report/1H-codesign-paper/figures/gemm_stage_breakdown.png differ diff --git a/docs/report/1H-codesign-paper/sections/02-platform.tex b/docs/report/1H-codesign-paper/sections/02-platform.tex index 5519c32..4eb0875 100644 --- a/docs/report/1H-codesign-paper/sections/02-platform.tex +++ b/docs/report/1H-codesign-paper/sections/02-platform.tex @@ -311,9 +311,9 @@ translates into physically reasonable kernel latencies. First, in the GEMM study (\S\ref{sec:gemm}), simulator-measured MAC efficiency tracks an analytic ideal-pipeline model within -\SI{2.2}{ppt} across every swept shape---from a single-tile -$M{=}K{=}N{=}32$ at $\sim\SI{7.5}{\percent}$ up to the deep-$K$ -$K{=}3072$ case at $\sim\SI{88}{\percent}$. The residual gap is +\SI{1.4}{ppt} across every swept shape---from a single-tile +$M{=}K{=}N{=}32$ at $\sim\SI{7.7}{\percent}$ up to the deep-$K$ +$K{=}3072$ case at $\sim\SI{90}{\percent}$. The residual gap is fill/tail overhead the closed-form pipeline model omits. Second, in the all-reduce study (\S\ref{sec:allreduce}, diff --git a/docs/report/1H-codesign-paper/sections/03-gemm.tex b/docs/report/1H-codesign-paper/sections/03-gemm.tex index d8ecad7..d49c6d3 100644 --- a/docs/report/1H-codesign-paper/sections/03-gemm.tex +++ b/docs/report/1H-codesign-paper/sections/03-gemm.tex @@ -43,12 +43,84 @@ kernel to fuse its softmax work into the GEMM pipeline \subsection{Results} We sweep eight GEMM shapes spanning square, tall, wide, and deep-$K$ -geometries, under three operand-staging variants -(\textsf{ref\_ref}, both operands streamed from HBM; \textsf{load\_ref}, -one operand resident in TCM; \textsf{load\_load}, both resident). -Figure~\ref{fig:gemm-util} reports MAC utilization and efficiency, and -Figure~\ref{fig:gemm-stages} breaks the kernel into per-stage engine -busy time. +geometries under two operand-staging variants that bracket the +realistic LLM cases. In \textsf{load\_ref}, the activation $A$ is +pre-staged on chip and only the weight $W$ streams from HBM during +the composite (the ``activation in TCM, weights from HBM'' case +typical of decoding with a small batch). In \textsf{ref\_ref}, both +operands stream from HBM during the composite (the ``cold both +sides'' case, where the activation does not fit on-chip or is itself +the output of a producer composite that wrote back to HBM). Both +variants run the same composite command; only the up-front staging +of $A$ differs. + +We evaluate the composite GEMM along two axes that together describe +how well it lands on the hardware: \emph{HBM bandwidth utilization} +(does the workload saturate the per-PE HBM bandwidth, i.e.\ is it +memory-bound on this configuration?) and \emph{achieved GEMM +throughput} (what fraction of the \SI{8}{\tera\flop\per\second} +per-PE peak does the kernel actually deliver). Both metrics use the +composite window as the denominator, so the up-front \textsf{tl.load} +of $A$ in \textsf{load\_ref} is excluded — only HBM traffic and +compute that happen \emph{inside} the composite count. + +\begin{figure}[t] +\centering +\includegraphics[width=\linewidth]{gemm_hbm_bw_util.png} +\caption{Per-PE HBM bandwidth utilization during the composite window, +for the two staging variants. The ceiling is the per-PE HBM bandwidth +(\SI{256}{\giga\byte\per\second}; dashed line at \SI{100}{\percent}). +\textsf{ref\_ref} streams both $A$ and $W$ and so always sits at a +higher BW-utilization than \textsf{load\_ref} on the same shape; +both variants saturate ($>\SI{85}{\percent}$) once the kernel +has enough total bytes to keep the link busy (deep-$K$ $K{=}3072$, +or low-reuse output-dominated $M{=}128,K{=}8,N{=}128$). Small or +single-tile shapes do not have enough total traffic to saturate the +link and are bottlenecked by pipeline fill rather than HBM.} +\label{fig:gemm-bw} +\end{figure} + +\begin{figure}[t] +\centering +\includegraphics[width=\linewidth]{gemm_per_pe_tflops.png} +\caption{Per-PE GEMM throughput delivered during the composite window +(reference line at the \SI{8}{\tera\flop\per\second} per-PE engine +peak). At the deep-$K$ corner $M{=}32,K{=}3072,N{=}32$ the +activation-pre-staged kernel reaches +\SI{7.18}{\tera\flop\per\second} ($\sim\SI{90}{\percent}$ of peak); +the both-from-HBM variant drops to +\SI{3.83}{\tera\flop\per\second} ($\sim\SI{48}{\percent}$) at the +same shape because the second operand doubles HBM pressure and the +kernel hits the BW ceiling shown in Fig.~\ref{fig:gemm-bw}. +Under-tile shapes ($M{=}K{=}N{=}32$, $M{=}8$, $K{=}8$) are +hard-capped by tile-fill at \SI{50}{\percent}, \SI{25}{\percent}, +\SI{12.5}{\percent} of peak regardless of staging.} +\label{fig:gemm-tflops} +\end{figure} + +The composite reaches the hardware roofline in two distinct regimes. +\emph{Memory-bound}: $K{=}3072$ deep-$K$ saturates HBM (\SI{88}{\percent} +load\_ref, \SI{94}{\percent} ref\_ref) and the low-reuse +$M{=}128,K{=}8,N{=}128$ corner saturates even harder +($>\SI{96}{\percent}$ in both), because back-to-back output +writes dominate. \emph{Compute-rich}: at the same deep-$K$ shape, +\textsf{load\_ref} delivers \SI{7.18}{\tera\flop\per\second} +($\sim\SI{90}{\percent}$ of the per-PE peak) — the configuration's +clean win. The \emph{distance between the two variants} at the same +shape is the cost of \emph{not} pre-staging $A$: at $K{=}3072$ the +throughput halves (\SI{7.18}{\tera\flop\per\second} $\to$ +\SI{3.83}{\tera\flop\per\second}) because the second operand doubles +HBM pressure +and the kernel hits the BW ceiling. This is the operational +take-away — when the activation can live on-chip the composite +delivers near-peak GEMM; when it cannot the BW ceiling dominates and +throughput halves. + +Figure~\ref{fig:gemm-util} validates that simulator-measured +efficiency tracks an analytic ideal-pipeline model (within +\SI{1.4}{ppt} across every swept shape), and +Figure~\ref{fig:gemm-stages} resolves the per-stage engine wall-clock +that backs the above interpretation. \begin{figure}[t] \centering @@ -60,9 +132,9 @@ cannot fill the MAC tile and cap at \SI{50}{\percent}, \SI{25}{\percent}, \SI{12.5}{\percent}. For tile-filling shapes, efficiency climbs with tile count---from \textasciitilde\SI{15}{\percent} at one tile to \textasciitilde% -\SI{88}{\percent} measured at 48 tiles ($K{=}3072$)---and the +\SI{90}{\percent} measured at 48 tiles ($K{=}3072$)---and the measured bars track the analytic ideal-pipeline prediction within -\SI{2.2}{ppt} across every shape.} +\SI{1.4}{ppt} across every shape.} \label{fig:gemm-util} \end{figure} @@ -81,31 +153,24 @@ where the output write becomes the largest stage.} \label{fig:gemm-stages} \end{figure} -Two regularities stand out. (i) Utilization is governed by how -completely the problem fills the MAC tile: the three under-tile shapes -are hard-capped well below \SI{100}{\percent}, independent of how the -kernel is issued. (ii) Among tile-filling shapes, efficiency is -governed by tile count---more tiles amortize the one-time pipeline -fill, so the deep-$K$ shape reaches \textasciitilde\SI{88}{\percent} -of peak while a single-tile shape reaches only -\textasciitilde\SI{15}{\percent}. The stage breakdown explains why: -with 48 tiles all three pipelined stages (DMA in, Fetch, GEMM) run -concurrently at the per-tile bandwidth limit, whereas the low-reuse -shape spends most of its time moving data in and out. - \subsection{Analysis and meaning} -The composite command does not manufacture bandwidth or MAC throughput; it -removes the two software-shaped obstacles between a GEMM and its hardware -roofline. By paying issue cost once per GEMM and chaining tile-stages -through token self-routing, it lets the scheduler keep the pipeline full, -so compute-rich shapes actually reach the efficiency their arithmetic -intensity allows ($\sim$\SI{88}{\percent} measured at 48 tiles), and -data-bound shapes actually reach their DMA bound instead of stalling on -command overhead. The close agreement between measured and theoretical -efficiency (Figure~\ref{fig:gemm-util}) is also the report's primary -validation that KernBench's latency model is faithful in the regime that -matters. The hardware implication is concrete: a single-command, -self-routing tile pipeline is the issue mechanism that makes the MAC array -usable, and it is a prerequisite---not a luxury---for the fused attention -kernel of \S\ref{sec:gqa}. +The composite command does not manufacture bandwidth or MAC +throughput; it removes the two software-shaped obstacles between a +GEMM and its hardware roofline. By paying the issue cost once per +GEMM rather than once per tile-stage, and by chaining tile-stages +through token self-routing, it lets the scheduler keep every stage +busy on a different tile, so compute-rich shapes actually reach the +efficiency their arithmetic intensity allows +($\sim$\SI{90}{\percent} of GEMM peak at the deep-$K$ +\textsf{load\_ref} corner) and data-bound shapes actually reach +their HBM-BW ceiling instead of stalling on command overhead. + +The split between \textsf{load\_ref} and \textsf{ref\_ref} also makes +the hardware-software boundary explicit: when the activation fits in +on-chip storage the composite is BW-headroom-rich and saturates the +GEMM engine; when it does not, both operands compete for the same +HBM port and the kernel is BW-bound. This is the lever the GQA +kernel of \S\ref{sec:gqa} reaches for next — keeping the right +working set on-chip so the composite pipeline lands in the +compute-rich regime rather than the BW-bound one. diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_hbm_bw_util.png b/src/kernbench/benches/1H_milestone_output/gemm/gemm_hbm_bw_util.png new file mode 100644 index 0000000..daba8bd Binary files /dev/null and b/src/kernbench/benches/1H_milestone_output/gemm/gemm_hbm_bw_util.png differ diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_measured.png b/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_measured.png index 438d161..72cda5d 100644 Binary files a/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_measured.png and b/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_measured.png differ diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_theoretical_vs_measured.png b/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_theoretical_vs_measured.png index de6e96b..87c0a7d 100644 Binary files a/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_theoretical_vs_measured.png and b/src/kernbench/benches/1H_milestone_output/gemm/gemm_mac_utilization_theoretical_vs_measured.png differ diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_per_pe_tflops.png b/src/kernbench/benches/1H_milestone_output/gemm/gemm_per_pe_tflops.png new file mode 100644 index 0000000..c09da36 Binary files /dev/null and b/src/kernbench/benches/1H_milestone_output/gemm/gemm_per_pe_tflops.png differ diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_stage_breakdown.png b/src/kernbench/benches/1H_milestone_output/gemm/gemm_stage_breakdown.png index c1201a2..6cbb77e 100644 Binary files a/src/kernbench/benches/1H_milestone_output/gemm/gemm_stage_breakdown.png and b/src/kernbench/benches/1H_milestone_output/gemm/gemm_stage_breakdown.png differ diff --git a/src/kernbench/benches/1H_milestone_output/gemm/gemm_sweep.json b/src/kernbench/benches/1H_milestone_output/gemm/gemm_sweep.json index 69eb131..6816a0b 100644 --- a/src/kernbench/benches/1H_milestone_output/gemm/gemm_sweep.json +++ b/src/kernbench/benches/1H_milestone_output/gemm/gemm_sweep.json @@ -33,22 +33,22 @@ "bytes_hbm": 6144, "arith_intensity": 10.666666666666666, "tile_count_expected": 1, - "sim_wall_clock_s": 0.297, + "sim_wall_clock_s": 0.112, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 79.0, + "wall_ns": 79.0, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "pe_math": { "occupancy_ns": 0, @@ -58,29 +58,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 58.0, + "wall_ns": 58.0, + "record_count": 2 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "MATH": { "occupancy_ns": 0, @@ -88,8 +88,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 135.38400000000001, + "composite_window_ns": 135.38400000000001 }, { "M": 32, @@ -100,22 +100,22 @@ "bytes_hbm": 6144, "arith_intensity": 10.666666666666666, "tile_count_expected": 1, - "sim_wall_clock_s": 0.239, + "sim_wall_clock_s": 0.108, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 71.0, + "wall_ns": 71.0, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "pe_math": { "occupancy_ns": 0, @@ -125,29 +125,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 29.0, + "wall_ns": 29.0, + "record_count": 1 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "MATH": { "occupancy_ns": 0, @@ -155,8 +155,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 171.394, + "composite_window_ns": 106.38400000000001 }, { "M": 32, @@ -167,22 +167,22 @@ "bytes_hbm": 6144, "arith_intensity": 10.666666666666666, "tile_count_expected": 1, - "sim_wall_clock_s": 0.243, + "sim_wall_clock_s": 0.105, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 63.0, + "wall_ns": 63.0, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "pe_math": { "occupancy_ns": 0, @@ -192,29 +192,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "MATH": { "occupancy_ns": 0, @@ -222,8 +222,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 183.394, + "composite_window_ns": 77.38400000000001 }, { "M": 32, @@ -234,22 +234,22 @@ "bytes_hbm": 10240, "arith_intensity": 12.8, "tile_count_expected": 1, - "sim_wall_clock_s": 0.427, + "sim_wall_clock_s": 0.169, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 79.0, + "wall_ns": 79.0, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "pe_math": { "occupancy_ns": 0, @@ -259,29 +259,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 58.0, + "wall_ns": 58.0, + "record_count": 2 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "MATH": { "occupancy_ns": 0, @@ -289,8 +289,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 135.38400000000001, + "composite_window_ns": 135.38400000000001 }, { "M": 32, @@ -301,22 +301,22 @@ "bytes_hbm": 10240, "arith_intensity": 12.8, "tile_count_expected": 1, - "sim_wall_clock_s": 0.336, + "sim_wall_clock_s": 0.163, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 79.0, + "wall_ns": 79.0, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "pe_math": { "occupancy_ns": 0, @@ -326,29 +326,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 29.0, + "wall_ns": 29.0, + "record_count": 1 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 }, "MATH": { "occupancy_ns": 0, @@ -356,8 +356,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 179.394, + "composite_window_ns": 106.38400000000001 }, { "M": 32, @@ -368,22 +368,558 @@ "bytes_hbm": 10240, "arith_intensity": 12.8, "tile_count_expected": 1, + "sim_wall_clock_s": 0.104, + "engines": { + "pe_dma": { + "occupancy_ns": 79.0, + "wall_ns": 79.0, + "record_count": 3 + }, + "pe_fetch_store": { + "occupancy_ns": 40.0, + "wall_ns": 40.0, + "record_count": 2 + }, + "pe_gemm": { + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + }, + "DMA_WRITE": { + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 + }, + "FETCH": { + "occupancy_ns": 32.0, + "wall_ns": 32.0, + "record_count": 1 + }, + "STORE": { + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 + }, + "GEMM": { + "occupancy_ns": 16.384000000000015, + "wall_ns": 16.384000000000015, + "record_count": 1 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 191.394, + "composite_window_ns": 77.38400000000001 + }, + { + "M": 32, + "K": 128, + "N": 32, + "variant": "ref_ref", + "flops": 262144, + "bytes_hbm": 18432, + "arith_intensity": 14.222222222222221, + "tile_count_expected": 2, + "sim_wall_clock_s": 0.162, + "engines": { + "pe_dma": { + "occupancy_ns": 158.995, + "wall_ns": 98.0, + "record_count": 5 + }, + "pe_fetch_store": { + "occupancy_ns": 72.0, + "wall_ns": 56.0, + "record_count": 3 + }, + "pe_gemm": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 137.995, + "wall_ns": 77.0, + "record_count": 4 + }, + "DMA_WRITE": { + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 + }, + "FETCH": { + "occupancy_ns": 64.0, + "wall_ns": 48.0, + "record_count": 2 + }, + "STORE": { + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 + }, + "GEMM": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 154.76800000000003, + "composite_window_ns": 154.76800000000003 + }, + { + "M": 32, + "K": 128, + "N": 32, + "variant": "load_ref", + "flops": 262144, + "bytes_hbm": 18432, + "arith_intensity": 14.222222222222221, + "tile_count_expected": 2, + "sim_wall_clock_s": 0.183, + "engines": { + "pe_dma": { + "occupancy_ns": 139.995, + "wall_ns": 111.0, + "record_count": 4 + }, + "pe_fetch_store": { + "occupancy_ns": 72.0, + "wall_ns": 56.0, + "record_count": 3 + }, + "pe_gemm": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 73.995, + "wall_ns": 45.0, + "record_count": 2 + }, + "DMA_WRITE": { + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 + }, + "FETCH": { + "occupancy_ns": 64.0, + "wall_ns": 48.0, + "record_count": 2 + }, + "STORE": { + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 + }, + "GEMM": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 211.77800000000002, + "composite_window_ns": 122.76800000000003 + }, + { + "M": 32, + "K": 128, + "N": 32, + "variant": "load_load", + "flops": 262144, + "bytes_hbm": 18432, + "arith_intensity": 14.222222222222221, + "tile_count_expected": 2, + "sim_wall_clock_s": 0.106, + "engines": { + "pe_dma": { + "occupancy_ns": 112.0, + "wall_ns": 107.0, + "record_count": 3 + }, + "pe_fetch_store": { + "occupancy_ns": 87.995, + "wall_ns": 56.0, + "record_count": 3 + }, + "pe_gemm": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + }, + "DMA_WRITE": { + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 + }, + "FETCH": { + "occupancy_ns": 79.995, + "wall_ns": 48.0, + "record_count": 2 + }, + "STORE": { + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 + }, + "GEMM": { + "occupancy_ns": 33.152000000000044, + "wall_ns": 32.76800000000003, + "record_count": 2 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 223.77800000000002, + "composite_window_ns": 93.76800000000003 + }, + { + "M": 32, + "K": 128, + "N": 128, + "variant": "ref_ref", + "flops": 1048576, + "bytes_hbm": 49152, + "arith_intensity": 21.333333333333332, + "tile_count_expected": 8, + "sim_wall_clock_s": 0.177, + "engines": { + "pe_dma": { + "occupancy_ns": 1798.8600000000001, + "wall_ns": 339.0, + "record_count": 20 + }, + "pe_fetch_store": { + "occupancy_ns": 341.3119999999999, + "wall_ns": 160.0, + "record_count": 12 + }, + "pe_gemm": { + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 1703.8600000000001, + "wall_ns": 269.0, + "record_count": 16 + }, + "DMA_WRITE": { + "occupancy_ns": 95.0, + "wall_ns": 94.0, + "record_count": 4 + }, + "FETCH": { + "occupancy_ns": 272.0, + "wall_ns": 152.0, + "record_count": 8 + }, + "STORE": { + "occupancy_ns": 69.3119999999999, + "wall_ns": 69.3119999999999, + "record_count": 4 + }, + "GEMM": { + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 354.384, + "composite_window_ns": 354.384 + }, + { + "M": 32, + "K": 128, + "N": 128, + "variant": "load_ref", + "flops": 1048576, + "bytes_hbm": 49152, + "arith_intensity": 21.333333333333332, + "tile_count_expected": 8, + "sim_wall_clock_s": 0.22, + "engines": { + "pe_dma": { + "occupancy_ns": 819.8600000000001, + "wall_ns": 256.0, + "record_count": 13 + }, + "pe_fetch_store": { + "occupancy_ns": 341.3119999999999, + "wall_ns": 160.0, + "record_count": 12 + }, + "pe_gemm": { + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 679.8600000000001, + "wall_ns": 141.0, + "record_count": 8 + }, + "DMA_WRITE": { + "occupancy_ns": 95.0, + "wall_ns": 94.0, + "record_count": 4 + }, + "FETCH": { + "occupancy_ns": 272.0, + "wall_ns": 152.0, + "record_count": 8 + }, + "STORE": { + "occupancy_ns": 69.3119999999999, + "wall_ns": 69.3119999999999, + "record_count": 4 + }, + "GEMM": { + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 315.394, + "composite_window_ns": 226.38400000000001 + }, + { + "M": 32, + "K": 128, + "N": 128, + "variant": "load_load", + "flops": 1048576, + "bytes_hbm": 49152, + "arith_intensity": 21.333333333333332, + "tile_count_expected": 8, + "sim_wall_clock_s": 0.112, + "engines": { + "pe_dma": { + "occupancy_ns": 283.0, + "wall_ns": 238.07200000000012, + "record_count": 6 + }, + "pe_fetch_store": { + "occupancy_ns": 839.252, + "wall_ns": 152.0, + "record_count": 12 + }, + "pe_gemm": { + "occupancy_ns": 141.82400000000052, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + }, + "DMA_WRITE": { + "occupancy_ns": 96.0, + "wall_ns": 56.072000000000116, + "record_count": 4 + }, + "FETCH": { + "occupancy_ns": 703.8600000000001, + "wall_ns": 144.0, + "record_count": 8 + }, + "STORE": { + "occupancy_ns": 135.39199999999983, + "wall_ns": 87.23199999999997, + "record_count": 4 + }, + "GEMM": { + "occupancy_ns": 141.82400000000052, + "wall_ns": 131.07200000000012, + "record_count": 8 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 418.0820000000001, + "composite_window_ns": 192.07200000000012 + }, + { + "M": 32, + "K": 3072, + "N": 32, + "variant": "ref_ref", + "flops": 6291456, + "bytes_hbm": 395264, + "arith_intensity": 15.917098445595855, + "tile_count_expected": 48, + "sim_wall_clock_s": 0.2, + "engines": { + "pe_dma": { + "occupancy_ns": 56319.35999999987, + "wall_ns": 1569.9999999999998, + "record_count": 97 + }, + "pe_fetch_store": { + "occupancy_ns": 1543.9999999999995, + "wall_ns": 791.9999999999998, + "record_count": 49 + }, + "pe_gemm": { + "occupancy_ns": 1219.5840000000169, + "wall_ns": 786.4320000000005, + "record_count": 48 + }, + "pe_math": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "stages": { + "DMA_READ": { + "occupancy_ns": 56298.35999999987, + "wall_ns": 1548.9999999999998, + "record_count": 96 + }, + "DMA_WRITE": { + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 + }, + "FETCH": { + "occupancy_ns": 1535.9999999999995, + "wall_ns": 783.9999999999998, + "record_count": 48 + }, + "STORE": { + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 + }, + "GEMM": { + "occupancy_ns": 1219.5840000000169, + "wall_ns": 786.4320000000005, + "record_count": 48 + }, + "MATH": { + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 + } + }, + "pe_window_ns": 1644.4320000000005, + "composite_window_ns": 1644.4320000000005 + }, + { + "M": 32, + "K": 3072, + "N": 32, + "variant": "load_ref", + "flops": 6291456, + "bytes_hbm": 395264, + "arith_intensity": 15.917098445595855, + "tile_count_expected": 48, "sim_wall_clock_s": 0.237, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 20236.359999999866, + "wall_ns": 1582.9999999999998, + "record_count": 50 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 1543.9999999999998, + "wall_ns": 791.9999999999998, + "record_count": 49 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 1219.584000000017, + "wall_ns": 786.4320000000007, + "record_count": 48 }, "pe_math": { "occupancy_ns": 0, @@ -393,29 +929,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 19434.359999999866, + "wall_ns": 780.9999999999998, + "record_count": 48 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 1535.9999999999998, + "wall_ns": 783.9999999999998, + "record_count": 48 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 1219.584000000017, + "wall_ns": 786.4320000000007, + "record_count": 48 }, "MATH": { "occupancy_ns": 0, @@ -423,544 +959,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 32, - "variant": "ref_ref", - "flops": 262144, - "bytes_hbm": 18432, - "arith_intensity": 14.222222222222221, - "tile_count_expected": 2, - "sim_wall_clock_s": 0.322, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 32, - "variant": "load_ref", - "flops": 262144, - "bytes_hbm": 18432, - "arith_intensity": 14.222222222222221, - "tile_count_expected": 2, - "sim_wall_clock_s": 0.247, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 32, - "variant": "load_load", - "flops": 262144, - "bytes_hbm": 18432, - "arith_intensity": 14.222222222222221, - "tile_count_expected": 2, - "sim_wall_clock_s": 0.247, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 128, - "variant": "ref_ref", - "flops": 1048576, - "bytes_hbm": 49152, - "arith_intensity": 21.333333333333332, - "tile_count_expected": 8, - "sim_wall_clock_s": 0.359, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 128, - "variant": "load_ref", - "flops": 1048576, - "bytes_hbm": 49152, - "arith_intensity": 21.333333333333332, - "tile_count_expected": 8, - "sim_wall_clock_s": 0.368, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 128, - "N": 128, - "variant": "load_load", - "flops": 1048576, - "bytes_hbm": 49152, - "arith_intensity": 21.333333333333332, - "tile_count_expected": 8, - "sim_wall_clock_s": 0.241, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 3072, - "N": 32, - "variant": "ref_ref", - "flops": 6291456, - "bytes_hbm": 395264, - "arith_intensity": 15.917098445595855, - "tile_count_expected": 48, - "sim_wall_clock_s": 0.318, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 - }, - { - "M": 32, - "K": 3072, - "N": 32, - "variant": "load_ref", - "flops": 6291456, - "bytes_hbm": 395264, - "arith_intensity": 15.917098445595855, - "tile_count_expected": 48, - "sim_wall_clock_s": 0.295, - "engines": { - "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 - }, - "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 - }, - "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "pe_math": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "stages": { - "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 - }, - "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 - }, - "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 - }, - "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 - }, - "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 - }, - "MATH": { - "occupancy_ns": 0, - "wall_ns": 0.0, - "record_count": 0 - } - }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 1701.4420000000007, + "composite_window_ns": 876.4320000000005 }, { "M": 32, @@ -971,22 +971,22 @@ "bytes_hbm": 395264, "arith_intensity": 15.917098445595855, "tile_count_expected": 48, - "sim_wall_clock_s": 0.321, + "sim_wall_clock_s": 0.208, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 2310.0, + "wall_ns": 1569.9999999999998, + "record_count": 3 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 19586.359999999877, + "wall_ns": 792.0, + "record_count": 49 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 1219.584000000017, + "wall_ns": 786.4320000000007, + "record_count": 48 }, "pe_math": { "occupancy_ns": 0, @@ -996,29 +996,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 21.0, + "wall_ns": 21.0, + "record_count": 1 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 19578.359999999877, + "wall_ns": 784.0, + "record_count": 48 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 8.0, + "wall_ns": 8.0, + "record_count": 1 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 1219.584000000017, + "wall_ns": 786.4320000000007, + "record_count": 48 }, "MATH": { "occupancy_ns": 0, @@ -1026,8 +1026,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 2440.442000000001, + "composite_window_ns": 847.4320000000007 }, { "M": 8, @@ -1038,22 +1038,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 8, - "sim_wall_clock_s": 0.219, + "sim_wall_clock_s": 0.089, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 1798.8600000000001, + "wall_ns": 339.0, + "record_count": 20 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 341.3119999999999, + "wall_ns": 160.0, + "record_count": 12 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "pe_math": { "occupancy_ns": 0, @@ -1063,29 +1063,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 1703.8600000000001, + "wall_ns": 269.0, + "record_count": 16 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 95.0, + "wall_ns": 94.0, + "record_count": 4 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 272.0, + "wall_ns": 152.0, + "record_count": 8 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 69.3119999999999, + "wall_ns": 69.3119999999999, + "record_count": 4 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "MATH": { "occupancy_ns": 0, @@ -1093,8 +1093,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 354.384, + "composite_window_ns": 354.384 }, { "M": 8, @@ -1105,22 +1105,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 8, - "sim_wall_clock_s": 0.337, + "sim_wall_clock_s": 0.155, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 795.8600000000001, + "wall_ns": 232.0, + "record_count": 13 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 341.3119999999999, + "wall_ns": 160.0, + "record_count": 12 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "pe_math": { "occupancy_ns": 0, @@ -1130,29 +1130,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 679.8600000000001, + "wall_ns": 141.0, + "record_count": 8 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 95.0, + "wall_ns": 94.0, + "record_count": 4 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 272.0, + "wall_ns": 152.0, + "record_count": 8 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 69.3119999999999, + "wall_ns": 69.3119999999999, + "record_count": 4 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 135.29600000000028, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "MATH": { "occupancy_ns": 0, @@ -1160,8 +1160,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 291.394, + "composite_window_ns": 226.38400000000001 }, { "M": 8, @@ -1172,22 +1172,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 8, - "sim_wall_clock_s": 0.356, + "sim_wall_clock_s": 0.234, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 258.0, + "wall_ns": 218.07200000000012, + "record_count": 6 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 839.252, + "wall_ns": 152.0, + "record_count": 12 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 141.82400000000052, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "pe_math": { "occupancy_ns": 0, @@ -1197,29 +1197,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 96.0, + "wall_ns": 56.072000000000116, + "record_count": 4 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 703.8600000000001, + "wall_ns": 144.0, + "record_count": 8 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 135.39199999999983, + "wall_ns": 87.23199999999997, + "record_count": 4 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 141.82400000000052, + "wall_ns": 131.07200000000012, + "record_count": 8 }, "MATH": { "occupancy_ns": 0, @@ -1227,8 +1227,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 418.0820000000001, + "composite_window_ns": 192.07200000000012 }, { "M": 128, @@ -1239,22 +1239,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 16, - "sim_wall_clock_s": 0.372, + "sim_wall_clock_s": 0.119, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 7854.6320000000005, + "wall_ns": 653.0, + "record_count": 48 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 1367.9359999999997, + "wall_ns": 328.384, + "record_count": 32 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 264.83200000000033, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "pe_math": { "occupancy_ns": 0, @@ -1264,29 +1264,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 6479.400000000001, + "wall_ns": 525.0, + "record_count": 32 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 1375.232, + "wall_ns": 296.0, + "record_count": 16 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 804.0, + "wall_ns": 316.0, + "record_count": 16 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 563.9359999999997, + "wall_ns": 280.0, + "record_count": 16 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 264.83200000000033, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "MATH": { "occupancy_ns": 0, @@ -1294,8 +1294,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 653.0, + "composite_window_ns": 653.0 }, { "M": 128, @@ -1306,22 +1306,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 16, - "sim_wall_clock_s": 0.245, + "sim_wall_clock_s": 0.183, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 3779.6320000000005, + "wall_ns": 418.0, + "record_count": 33 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 1367.9359999999997, + "wall_ns": 328.384, + "record_count": 32 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 264.83200000000033, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "pe_math": { "occupancy_ns": 0, @@ -1331,29 +1331,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 2383.4000000000005, + "wall_ns": 269.0, + "record_count": 16 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 1375.232, + "wall_ns": 296.0, + "record_count": 16 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 804.0, + "wall_ns": 316.0, + "record_count": 16 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 563.9359999999997, + "wall_ns": 280.0, + "record_count": 16 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 264.83200000000033, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "MATH": { "occupancy_ns": 0, @@ -1361,8 +1361,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 462.01, + "composite_window_ns": 397.0 }, { "M": 128, @@ -1373,22 +1373,22 @@ "bytes_hbm": 36864, "arith_intensity": 7.111111111111111, "tile_count_expected": 16, - "sim_wall_clock_s": 0.243, + "sim_wall_clock_s": 0.179, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 858.0, + "wall_ns": 183.0, + "record_count": 18 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 4395.175999999999, + "wall_ns": 324.0, + "record_count": 32 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 308.224000000002, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "pe_math": { "occupancy_ns": 0, @@ -1398,29 +1398,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 816.0, + "wall_ns": 141.0, + "record_count": 16 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 2431.4000000000005, + "wall_ns": 272.0, + "record_count": 16 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 1963.775999999998, + "wall_ns": 275.616, + "record_count": 16 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 308.224000000002, + "wall_ns": 262.14400000000023, + "record_count": 16 }, "MATH": { "occupancy_ns": 0, @@ -1428,8 +1428,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 511.01, + "composite_window_ns": 405.0 }, { "M": 512, @@ -1440,22 +1440,22 @@ "bytes_hbm": 1572864, "arith_intensity": 170.66666666666666, "tile_count_expected": 2048, - "sim_wall_clock_s": 0.357, + "sim_wall_clock_s": 1.241, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 104867178.97599977, + "wall_ns": 67597.00000000001, + "record_count": 4352 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 1206911.5840000017, + "wall_ns": 33800.00000000001, + "record_count": 2304 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 36417.40799998986, + "wall_ns": 33554.43199999787, + "record_count": 2048 }, "pe_math": { "occupancy_ns": 0, @@ -1465,29 +1465,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 100695823.35999978, + "wall_ns": 65549.00000000001, + "record_count": 4096 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 4171355.616000002, + "wall_ns": 34616.00000000001, + "record_count": 256 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 1076760.0000000005, + "wall_ns": 33776.00000000001, + "record_count": 2048 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 130151.58400000134, + "wall_ns": 31914.784000000276, + "record_count": 256 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 36417.40799998986, + "wall_ns": 33554.43199999787, + "record_count": 2048 }, "MATH": { "occupancy_ns": 0, @@ -1495,8 +1495,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 67597.00000000001, + "composite_window_ns": 67597.00000000001 }, { "M": 512, @@ -1507,22 +1507,22 @@ "bytes_hbm": 1572864, "arith_intensity": 170.66666666666666, "tile_count_expected": 2048, - "sim_wall_clock_s": 0.364, + "sim_wall_clock_s": 0.742, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 37760375.97599977, + "wall_ns": 36890.0, + "record_count": 2305 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 1206911.5840000017, + "wall_ns": 33800.0, + "record_count": 2304 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 36417.40799998891, + "wall_ns": 33554.43199999761, + "record_count": 2048 }, "pe_math": { "occupancy_ns": 0, @@ -1532,29 +1532,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 33586959.359999776, + "wall_ns": 32781.0, + "record_count": 2048 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 4171355.616000001, + "wall_ns": 34616.0, + "record_count": 256 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 1076760.0000000002, + "wall_ns": 33776.0, + "record_count": 2048 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 130151.58400000146, + "wall_ns": 31914.784000000003, + "record_count": 256 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 36417.40799998891, + "wall_ns": 33554.43199999761, + "record_count": 2048 }, "MATH": { "occupancy_ns": 0, @@ -1562,8 +1562,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 36934.01, + "composite_window_ns": 34829.0 }, { "M": 512, @@ -1574,22 +1574,22 @@ "bytes_hbm": 1572864, "arith_intensity": 170.66666666666666, "tile_count_expected": 2048, - "sim_wall_clock_s": 0.239, + "sim_wall_clock_s": 0.406, "engines": { "pe_dma": { - "occupancy_ns": 16594.192, - "wall_ns": 15271.192000000001, - "record_count": 576 + "occupancy_ns": 142065.0, + "wall_ns": 6170.0, + "record_count": 258 }, "pe_fetch_store": { - "occupancy_ns": 8704.0, - "wall_ns": 7624.0, - "record_count": 320 + "occupancy_ns": 37794382.84800001, + "wall_ns": 33796.0, + "record_count": 2304 }, "pe_gemm": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 838467.5839981049, + "wall_ns": 33554.43199999738, + "record_count": 2048 }, "pe_math": { "occupancy_ns": 0, @@ -1599,29 +1599,29 @@ }, "stages": { "DMA_READ": { - "occupancy_ns": 15250.192000000001, - "wall_ns": 15250.192000000001, - "record_count": 512 + "occupancy_ns": 0, + "wall_ns": 0.0, + "record_count": 0 }, "DMA_WRITE": { - "occupancy_ns": 1344.0, - "wall_ns": 1344.0, - "record_count": 64 + "occupancy_ns": 135936.0, + "wall_ns": 2061.0, + "record_count": 256 }, "FETCH": { - "occupancy_ns": 8192.0, - "wall_ns": 7616.0, - "record_count": 256 + "occupancy_ns": 33593103.359999776, + "wall_ns": 32784.0, + "record_count": 2048 }, "STORE": { - "occupancy_ns": 512.0, - "wall_ns": 512.0, - "record_count": 64 + "occupancy_ns": 4201279.488000238, + "wall_ns": 33632.928, + "record_count": 256 }, "GEMM": { - "occupancy_ns": 4194.304000000004, - "wall_ns": 4194.304000000004, - "record_count": 256 + "occupancy_ns": 838467.5839981049, + "wall_ns": 33554.43199999738, + "record_count": 2048 }, "MATH": { "occupancy_ns": 0, @@ -1629,8 +1629,8 @@ "record_count": 0 } }, - "pe_window_ns": 15327.576000000001, - "composite_window_ns": 15327.576000000001 + "pe_window_ns": 38990.01, + "composite_window_ns": 34837.0 } ] } \ No newline at end of file diff --git a/src/kernbench/benches/milestone_1h_gemm.py b/src/kernbench/benches/milestone_1h_gemm.py index 611437a..16ce468 100644 --- a/src/kernbench/benches/milestone_1h_gemm.py +++ b/src/kernbench/benches/milestone_1h_gemm.py @@ -277,12 +277,35 @@ STAGE_COLORS = { # MAC-utilization analytic model constants. # Pipeline assumes back-to-back small-tile DMAs amortize their per-request # head latency through the HBM_CTRL queue (ADR-0014 D4 issue-only channel -# hold). Steady-state cycle is BW-bound at one tile / T_STAGE. -_T_STAGE = 16.0 # steady-state cycle (ns/tile) +# hold). Steady-state cycle is BW-bound at one tile / T_STAGE, where +# T_STAGE is derived from the hardware peak so that analytic and measured +# share the same per-PE GEMM-throughput denominator +# (\\SI{8}{\\tera\\flop\\per\\second} = 8000 flops/ns) — see +# `_t_stage_ns_from_tile()` below. _T_PIPELINE_FILL = 60.0 # first-tile sequential fill: DMA head # + DMA + FETCH + GEMM stages _T_PIPELINE_TAIL = 30.0 # tail: last STORE + DMA_WRITE + drain +# Per-PE GEMM peak. Single source of truth for both the analytic +# `compute_total = tiles * T_STAGE` term and the measured-eff +# denominator. Matches topology.yaml pe_gemm.peak_tflops_f16. +_GEMM_PEAK_TFLOPS = 8.0 +_GEMM_PEAK_FLOPS_PER_NS = _GEMM_PEAK_TFLOPS * 1000.0 # = 8000 flops/ns + + +def _t_stage_ns_from_tile(tile_flops: int) -> float: + """Per-tile bottleneck cycle (ns) implied by the hardware peak. + + The composite pipeline's steady-state stage cycle equals the time the + GEMM engine spends on one tile at peak throughput. Deriving T_STAGE + from tile_flops / peak (rather than hardcoding 16 ns) keeps the + analytic model self-consistent with the measured-eff denominator: a + tile with tile_flops = 131072 and an \\SI{8}{\\tera\\flop\\per\\second} + engine takes exactly 131072 / 8000 = 16.384 ns of GEMM time. + """ + return tile_flops / _GEMM_PEAK_FLOPS_PER_NS + + _PLOT_VARIANT = "load_ref" @@ -342,10 +365,15 @@ def _grouped_bar_png( _xtick_labels(shape_labels, tile_counts, flagged), fontsize=8, ) ax.set_ylabel(y_label) - ax.set_title(title, fontsize=13, fontweight="bold") + # Title with optional subtitle stacked above (subtitle in lighter ink, + # title in bold below it). Coordinates are in axes-fraction so they + # survive tight_layout without colliding with the data area. if subtitle: - ax.text(0.5, 1.01, subtitle, transform=ax.transAxes, ha="center", + ax.text(0.5, 1.10, subtitle, transform=ax.transAxes, ha="center", va="bottom", fontsize=8, color="#475569") + ax.set_title(title, fontsize=13, fontweight="bold", pad=22) + else: + ax.set_title(title, fontsize=13, fontweight="bold") if threshold is not None: ax.axhline(threshold, ls="--", color="gray", lw=1.0) ax.legend(fontsize=8, loc="upper right") @@ -420,6 +448,7 @@ def emit_mac_utilization_measured( flagged = [_under_tile(*k, TILE_M, TILE_K, TILE_N) for k in shapes] tile_counts = [by_shape[k]["tile_count_expected"] for k in shapes] + t_stage_ns = _t_stage_ns_from_tile(tile_flops) gemm_util, useful_eff = [], [] for k in shapes: r = by_shape[k] @@ -428,7 +457,7 @@ def emit_mac_utilization_measured( tiles = r["tile_count_expected"] gu = useful / (tile_flops * tiles) * 100 gemm_util.append(gu) - compute_total = tiles * _T_STAGE + compute_total = tiles * t_stage_ns wall = _T_PIPELINE_FILL + compute_total + _T_PIPELINE_TAIL ueff = (compute_total * (gu / 100.0) / wall) * 100 if wall > 0 else 0.0 useful_eff.append(ueff) @@ -458,7 +487,7 @@ def emit_mac_utilization_theoretical_vs_measured( tile = data["tile_sizes"] TILE_M, TILE_K, TILE_N = tile["M"], tile["K"], tile["N"] tile_flops = 2 * TILE_M * TILE_K * TILE_N - peak_per_ns = tile_flops / _T_STAGE + t_stage_ns = _t_stage_ns_from_tile(tile_flops) by_shape = {(r["M"], r["K"], r["N"]): r for r in rows if r["variant"] == _PLOT_VARIANT} @@ -479,11 +508,11 @@ def emit_mac_utilization_theoretical_vs_measured( gu_t.append(gut * 100) rec = r.get("stages", {}).get("GEMM", {}).get("record_count", 0) or tiles gu_m.append((useful / (tile_flops * rec) * 100) if rec else 0.0) - compute_total = tiles * _T_STAGE + compute_total = tiles * t_stage_ns wall_t = _T_PIPELINE_FILL + compute_total + _T_PIPELINE_TAIL eff_t.append((compute_total * gut / wall_t * 100) if wall_t > 0 else 0.0) cw = r.get("composite_window_ns", 0.0) or 0.0 - eff_m.append((useful / cw / peak_per_ns * 100) if cw > 0 else 0.0) + eff_m.append((useful / cw / _GEMM_PEAK_FLOPS_PER_NS * 100) if cw > 0 else 0.0) series = { "GEMM util % (theoretical)": gu_t, @@ -507,6 +536,202 @@ def emit_mac_utilization_theoretical_vs_measured( ) +# ── per-PE composite-pipeline plots: HBM BW util & GEMM TFLOPS ───────── +# +# These two plots evaluate the composite GEMM kernel against the +# per-PE HBM bandwidth (256 GB/s) and per-PE GEMM peak (8 TFLOP/s). +# Both metrics use composite_window_ns as the denominator — the +# composite-internal pipeline window, with any pre-composite tl.load +# of operand A excluded by construction. The two variants on each plot +# answer different staging questions: +# +# - load_ref: activation A pre-staged on chip, only weight W streamed +# from HBM during the composite. ("Weight-only HBM" case.) +# - ref_ref: both A and W streamed from HBM during the composite. +# ("Both A and W from HBM" case.) +# +# load_load is omitted from these plots — with both operands pre-staged +# the composite carries no HBM traffic and the BW metric collapses to 0, +# which is uninformative against the saturation question. + +# Per-tile DMA payload sizes (one TILE_M x TILE_K or TILE_K x TILE_N +# read; one TILE_M x TILE_N write per output tile). +_PIPELINE_VARIANTS = ("load_ref", "ref_ref") +_VARIANT_LABEL = { + "load_ref": "Activation pre-staged", + "ref_ref": "A and W both from HBM", +} +_VARIANT_COLOR = { + "load_ref": "#3B82F6", # blue + "ref_ref": "#F59E0B", # orange +} + +# Per-PE HBM peak BW (GB/s); matches topology.yaml hbm_to_router_bw_gbs +# and (8 PCs / PE) x (32 GB/s / PC). +_HBM_BW_PER_PE_GBS = 256.0 +# Per-PE GEMM peak (TFLOP/s); matches topology.yaml pe_gemm.peak_tflops_f16. +_GEMM_PEAK_TFLOPS = 8.0 + + +def _shape_summary(rows: list[dict], tile: dict): + """Common shape-axis state for the BW / TFLOPS plots. + + Returns (shape_labels, tile_counts, flagged, shape_keys) — only + shapes that actually appear in `rows` for any of the pipeline + variants, in their original order. + """ + seen: list[tuple[int, int, int]] = [] + by_key: dict[tuple[int, int, int], dict] = {} + for r in rows: + key = (r["M"], r["K"], r["N"]) + if key not in by_key: + by_key[key] = r + seen.append(key) + shape_labels = [_shape_label(by_key[k]) for k in seen] + flagged = [_under_tile(*k, tile["M"], tile["K"], tile["N"]) for k in seen] + tile_counts = [by_key[k]["tile_count_expected"] for k in seen] + return shape_labels, tile_counts, flagged, seen + + +def _composite_bytes_through_hbm(r: dict, tile: dict) -> float: + """Total bytes transferred via HBM during the composite window. + + Derived from the simulator's per-stage record counts so it + automatically reflects the variant's pre-staging choices: + load_ref skips A reads, ref_ref reads both A and B, load_load + reads neither. Each DMA record carries a full hardware tile + (TILE_M x TILE_K read for A, TILE_K x TILE_N read for B, + TILE_M x TILE_N write for the output), regardless of how much + of that tile is "useful" — which is the actual byte traffic the + HBM controller sees. + """ + bpe = 2 # f16; matches matmul_composite.py default + a_tile_bytes = tile["M"] * tile["K"] * bpe + b_tile_bytes = tile["K"] * tile["N"] * bpe + out_tile_bytes = tile["M"] * tile["N"] * bpe + # The schedule emits one DMA_READ per (A,B) per non-pinned operand. + # Without per-record operand tags in the aggregated JSON, treat all + # DMA_READ records as B-shaped reads on load_ref (B is the only + # streamed operand) and average the two tile shapes on ref_ref. + # In the current topology both tile shapes are 4 KiB so this + # distinction is moot — kept symbolic for future tile-size sweeps. + variant = r.get("variant") + n_read = r["stages"]["DMA_READ"]["record_count"] + n_write = r["stages"]["DMA_WRITE"]["record_count"] + if variant == "ref_ref": + # A reads + B reads, roughly half each. + read_bytes = (n_read // 2) * a_tile_bytes \ + + (n_read - n_read // 2) * b_tile_bytes + else: + read_bytes = n_read * b_tile_bytes + write_bytes = n_write * out_tile_bytes + return float(read_bytes + write_bytes) + + +def _by_variant(rows: list[dict], variant: str) -> dict: + return {(r["M"], r["K"], r["N"]): r + for r in rows if r.get("variant") == variant} + + +def emit_hbm_bw_utilization( + sweep_json: Path | str = DEFAULT_SWEEP_JSON, + out_dir: Path | str = DEFAULT_PLOTS_DIR, +) -> str | None: + """Per-PE HBM bandwidth utilization during the composite window. + + Two grouped bars per shape: load_ref ("activation pre-staged") + vs ref_ref ("both A and W from HBM"). Y-axis is BW + util % against the per-PE HBM ceiling (\\SI{256}{\\giga\\byte\\per + \\second}). + """ + data = _load_sweep_data(sweep_json) + rows = [r for r in data["rows"] if r.get("variant") in _PIPELINE_VARIANTS] + if not rows: + return None + tile = data["tile_sizes"] + shape_labels, tile_counts, flagged, shape_keys = _shape_summary(rows, tile) + + series: dict = {} + for variant in _PIPELINE_VARIANTS: + by = _by_variant(rows, variant) + vals = [] + for k in shape_keys: + r = by.get(k) + if r is None: + vals.append(0.0) + continue + cw = r.get("composite_window_ns", 0.0) or 0.0 + if cw <= 0: + vals.append(0.0) + continue + bytes_hbm = _composite_bytes_through_hbm(r, tile) + # bytes_hbm [B] / cw [ns] = GB/s (since 1 B/ns = 1 GB/s). + bw_gbs = bytes_hbm / cw + vals.append(bw_gbs / _HBM_BW_PER_PE_GBS * 100.0) + series[_VARIANT_LABEL[variant]] = vals + colors = {_VARIANT_LABEL[v]: _VARIANT_COLOR[v] for v in _PIPELINE_VARIANTS} + + return _grouped_bar_png( + "gemm_hbm_bw_util.png", out_dir=Path(out_dir), + title="GEMM HBM bandwidth utilization (per PE, composite window)", + subtitle=(f"BW util = bytes through HBM / composite window " + f"/ {_HBM_BW_PER_PE_GBS:.0f} GB/s. " + f"Saturation → memory-bound regime."), + shape_labels=shape_labels, tile_counts=tile_counts, flagged=flagged, + series=series, colors=colors, y_label="HBM BW utilization (%)", + threshold=100.0, + ) + + +def emit_per_pe_tflops( + sweep_json: Path | str = DEFAULT_SWEEP_JSON, + out_dir: Path | str = DEFAULT_PLOTS_DIR, +) -> str | None: + """Per-PE GEMM throughput in TFLOP/s during the composite window. + + Two grouped bars per shape (load_ref vs ref_ref). Reference line + at the per-PE GEMM engine peak (\\SI{8}{\\tera\\flop\\per\\second}). + """ + data = _load_sweep_data(sweep_json) + rows = [r for r in data["rows"] if r.get("variant") in _PIPELINE_VARIANTS] + if not rows: + return None + tile = data["tile_sizes"] + shape_labels, tile_counts, flagged, shape_keys = _shape_summary(rows, tile) + + series: dict = {} + for variant in _PIPELINE_VARIANTS: + by = _by_variant(rows, variant) + vals = [] + for k in shape_keys: + r = by.get(k) + if r is None: + vals.append(0.0) + continue + cw = r.get("composite_window_ns", 0.0) or 0.0 + if cw <= 0: + vals.append(0.0) + continue + M, K, N = r["M"], r["K"], r["N"] + useful = 2 * M * K * N + # flops / ns -> GFLOP/s; / 1000 -> TFLOP/s. + tflops = useful / cw / 1000.0 + vals.append(tflops) + series[_VARIANT_LABEL[variant]] = vals + colors = {_VARIANT_LABEL[v]: _VARIANT_COLOR[v] for v in _PIPELINE_VARIANTS} + + return _grouped_bar_png( + "gemm_per_pe_tflops.png", out_dir=Path(out_dir), + title="GEMM achieved throughput (per PE, composite window)", + subtitle=(f"TFLOP/s = 2 · M · K · N / composite " + f"window. Peak per PE = " + f"{_GEMM_PEAK_TFLOPS:.0f} TFLOP/s (dashed line)."), + shape_labels=shape_labels, tile_counts=tile_counts, flagged=flagged, + series=series, colors=colors, y_label="Per-PE TFLOP/s", + threshold=_GEMM_PEAK_TFLOPS, + ) + + def emit_all_gemm_plots( sweep_json: Path | str = DEFAULT_SWEEP_JSON, out_dir: Path | str = DEFAULT_PLOTS_DIR, @@ -515,7 +740,9 @@ def emit_all_gemm_plots( paths = [] for fn in (emit_stage_breakdown, emit_mac_utilization_measured, - emit_mac_utilization_theoretical_vs_measured): + emit_mac_utilization_theoretical_vs_measured, + emit_hbm_bw_utilization, + emit_per_pe_tflops): p = fn(sweep_json, out_dir) if p: paths.append(p)