diff --git a/docs/report/1H-codesign-paper/build/main.pdf b/docs/report/1H-codesign-paper/build/main.pdf index 10352fe..7cdc3ce 100644 Binary files a/docs/report/1H-codesign-paper/build/main.pdf and b/docs/report/1H-codesign-paper/build/main.pdf differ diff --git a/docs/report/1H-codesign-paper/figures/cube_architecture.pdf b/docs/report/1H-codesign-paper/figures/cube_architecture.pdf index b9b91d3..fbbc7a6 100644 Binary files a/docs/report/1H-codesign-paper/figures/cube_architecture.pdf and b/docs/report/1H-codesign-paper/figures/cube_architecture.pdf differ diff --git a/docs/report/1H-codesign-paper/figures/cube_architecture.svg b/docs/report/1H-codesign-paper/figures/cube_architecture.svg new file mode 100644 index 0000000..29e9b59 --- /dev/null +++ b/docs/report/1H-codesign-paper/figures/cube_architecture.svg @@ -0,0 +1,312 @@ + + cube + + CUBE TOPOLOGY — 17.0×14.0mm | 6×6 Router Mesh | n_to_one mode | 64 pseudo-ch + Per-PE: 8 ch × 32.0 GB/s = 256.0 GB/s | Cube total: 64 × 32.0 = 2048.0 GB/s + + + HBM_CTRL | 64 pseudo channels + Total BW: 2048 GB/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PE0×8ch + PE1×8ch + PE2×8ch + PE3×8ch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PE4×8ch + PE5×8ch + PE6×8ch + PE7×8ch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r0c0 + + PE0 + + + r0c1 + + PE1 + + + r0c2 + + r0c3 + + r0c4 + + r0c5 + + r1c0 + + r1c1 + + r1c2 + + M_CPU + + + r1c3 + + r1c4 + + PE2 + + + r1c5 + + PE3 + + + r2c0 + + r2c1 + + r2c4 + + r2c5 + + r3c0 + + SRAM + + + r3c1 + + r3c4 + + r3c5 + + r4c0 + + PE4 + + + r4c1 + + PE5 + + + r4c2 + + r4c3 + + r4c4 + + r4c5 + + r5c0 + + r5c1 + + r5c2 + + r5c3 + + r5c4 + + PE6 + + + r5c5 + + PE7 + + + 256GB/s + + 256GB/s + + 256GB/s + + 256GB/s + + 256GB/s + + 256GB/s + + 256GB/s + + 256GB/s + + UCIe-W + + c0 + + + c1 + + + c2 + + + c3 + + + UCIe-N + + c0 + + + c1 + + + c2 + + + c3 + + + UCIe-E + + c0 + + + c1 + + + c2 + + + c3 + + + UCIe-S + + c0 + + + c1 + + + c2 + + + c3 + + + PE Router + + M_CPU / SRAM + + UCIe + + Relay + + HBM Link + + Mesh Link + \ No newline at end of file diff --git a/docs/report/1H-codesign-paper/figures/sip_architecture.pdf b/docs/report/1H-codesign-paper/figures/sip_architecture.pdf index 86e878a..f5fde6d 100644 Binary files a/docs/report/1H-codesign-paper/figures/sip_architecture.pdf and b/docs/report/1H-codesign-paper/figures/sip_architecture.pdf differ diff --git a/docs/report/1H-codesign-paper/figures/sip_architecture.svg b/docs/report/1H-codesign-paper/figures/sip_architecture.svg new file mode 100644 index 0000000..9104792 --- /dev/null +++ b/docs/report/1H-codesign-paper/figures/sip_architecture.svg @@ -0,0 +1,94 @@ + + sip + + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 1.0mm 512GB/s + + 2.5mm 512GB/s + + 2.5mm 512GB/s + + 2.5mm 512GB/s + + 2.5mm 512GB/s + + CUBE (0,0) + + CUBE (1,0) + + CUBE (2,0) + + CUBE (3,0) + + CUBE (0,1) + + CUBE (1,1) + + CUBE (2,1) + + CUBE (3,1) + + CUBE (0,2) + + CUBE (1,2) + + CUBE (2,2) + + CUBE (3,2) + + CUBE (0,3) + + CUBE (1,3) + + CUBE (2,3) + + CUBE (3,3) + + IO io0 + \ No newline at end of file diff --git a/docs/report/1H-codesign-paper/main.pdf b/docs/report/1H-codesign-paper/main.pdf new file mode 100644 index 0000000..da09f8d Binary files /dev/null and b/docs/report/1H-codesign-paper/main.pdf differ diff --git a/docs/report/1H-codesign-paper/sections/01-introduction.tex b/docs/report/1H-codesign-paper/sections/01-introduction.tex index b1e140b..e6715be 100644 --- a/docs/report/1H-codesign-paper/sections/01-introduction.tex +++ b/docs/report/1H-codesign-paper/sections/01-introduction.tex @@ -25,9 +25,12 @@ serves as the common evaluation platform for all mechanisms and kernels discussed in this study. This report focuses on Grouped-Query Attention (GQA), one of the most -performance- and bandwidth-critical components of LLM inference. Modern -decoder-only models such as Llama~3 and Mistral have largely transitioned -from GPT-3-style multi-head attention (MHA) to GQA, in which multiple +performance- and bandwidth-critical components of LLM inference. GQA +dominates inference-time memory traffic and KV-cache capacity in modern +LLM serving, making it the primary bandwidth bottleneck on memory-centric +architectures such as AHBM. Modern decoder-only models such as Llama~3 +and Mistral have largely transitioned from GPT-3-style multi-head +attention (MHA) to GQA, in which multiple query heads share a single KV head to reduce KV cache capacity and memory-bandwidth requirements. While GQA improves system efficiency at the model level, mapping it efficiently onto AHBM introduces three diff --git a/docs/report/1H-codesign-paper/sections/01-introduction_ko.tex b/docs/report/1H-codesign-paper/sections/01-introduction_ko.tex new file mode 100644 index 0000000..5207d96 --- /dev/null +++ b/docs/report/1H-codesign-paper/sections/01-introduction_ko.tex @@ -0,0 +1,73 @@ +\section{서론} +\label{sec:intro} + +AHBM은 연산 유닛을 HBM 스택 내부에 직접 통합한 메모리 중심 가속기 +아키텍처이다. 각 처리 요소(PE)는 전용 HBM 슬라이스와 짝지어지며, +TCM과 SRAM을 거쳐 메모리와 MAC 어레이 사이의 데이터를 단계적으로 +이동시킨다. 이러한 메모리 중심 구조에서 커널 성능은 연산 처리량만이 +아니라, 메모리 계층과 PE 사이에 데이터를 얼마나 효율적으로 배치하고, +이동시키며, 공유하는가에 의해 결정된다. 따라서 AHBM에서의 AI 커널 +최적화는 커널 알고리즘과 아키텍처 메커니즘을 함께 발전시키는 +하드웨어--소프트웨어 코디자인을 필요로 한다. + +상세한 성능 분석과 신속한 설계 탐색을 위해, 우리는 +\textbf{KernBench}---AHBM을 위한 소스 수준 이산 사건 시뮬레이션 +플랫폼---을 개발하였다. KernBench는 메모리 시스템 지연, PE 실행 +모델, PE 간 통신, 호스트 측 오케스트레이션을 포함한 AHBM 실행 모델을 +구현하면서, 커널과 호스트 소프트웨어를 소스 코드로부터 직접 실행한다. +이를 통해 실행 거동을 세밀하게 관찰할 수 있을 뿐만 아니라, 컴파일러나 +런타임과 같은 상위 소프트웨어 스택과 독립적으로 하드웨어--소프트웨어 +코디자인 선택을 체계적으로 평가할 수 있다. 본 보고서에 제시된 모든 +결과는 KernBench를 사용하여 얻은 것이며, KernBench는 본 연구에서 +다루는 모든 메커니즘과 커널의 공통 평가 플랫폼 역할을 한다. + +본 보고서는 LLM 추론에서 가장 성능 및 대역폭 임계 구성요소 중 하나인 +Grouped-Query Attention(GQA)에 초점을 맞춘다. Llama~3, Mistral과 +같은 최신 디코더 전용 모델은 GPT-3 형태의 다중-헤드 어텐션(MHA)으로 +부터 GQA---여러 질의 헤드가 하나의 KV 헤드를 공유함으로써 KV 캐시 +용량과 메모리 대역폭 요구량을 줄이는 방식---로 대거 전환되었다. GQA는 +모델 수준에서 시스템 효율을 개선하지만, 이를 AHBM에 효율적으로 +매핑하기 위해서는 세 가지 아키텍처 요구사항이 발생한다: PE 간 통신을 +최소화하기 위한 KV 캐시와 가중치의 최적 배치, 불가피한 PE 간 트래픽에 +대한 저오버헤드 지원, 그리고 각 PE 내에서 메모리 접근과 연산의 +효율적인 파이프라이닝이다. + +이러한 요구사항을 충족하기 위해, 본 보고서는 세 가지 하드웨어-- +소프트웨어 코디자인 메커니즘을 제안한다. 첫째, GQA-aware 데이터 +배치는 TCM/SRAM/HBM 계층에 KV 캐시와 가중치를 분산 배치하여 통신 +오버헤드를 줄이고 데이터 지역성을 향상시킨다. 둘째, PE\_IPCQ는 리덕션 +등 통신 집약적 연산을 위한 효율적인 온디바이스 집합 통신 프리미티브를 +제공한다. 셋째, composite-command GEMM 파이프라인은 PE\_SCHEDULER의 +제어 하에 각 PE 내부에서 메모리 이동과 연산을 긴밀하게 파이프라이닝 +하여, 명령 오버헤드를 줄이면서도 MAC 어레이의 가동률을 효율적으로 +유지한다. + +연산 측 인에이블러(composite-command GEMM 파이프라인, +\S\ref{sec:gemm})와 통신 측 인에이블러(PE\_IPCQ, +\S\ref{sec:allreduce})는 우선 독립적으로 개발 및 평가된다. 이후 융합 +GQA 커널(\S\ref{sec:gqa})은 이들을 GQA-aware 데이터 배치와 결합하여 +AHBM 상에서 종단간 어텐션 구현을 시연한다. 대응 관계는 직접적이다: +어텐션의 $QK^{\top}$와 $PV$ 곱은 정확히 composite-command 파이프라인이 +이득을 주는 GEMM이며, KV 리덕션은 정확히 PE\_IPCQ가 이득을 주는 집합 +연산이다. 이러한 메커니즘들이 결합되어, 융합 GQA 커널은 AHBM의 HBM +대역폭을 효율적으로 활용할 수 있게 된다. + +본 연구에서 GQA가 주된 동기 부여 워크로드 역할을 하지만, 도출된 +메커니즘들은 훨씬 광범위한 AI 커널 분류군에서 재사용 가능한 구성 +요소로 의도되었다. PE\_IPCQ는 분산 및 통신 집약적 워크로드 전반에 +걸쳐 집합 통신을 지원할 수 있으며, composite-command 실행은 GEMM +기반 커널, 피드포워드 네트워크(FFN), 정규화, 그 외 융합 연산자 +파이프라인에 적용 가능하다. 계층적 데이터 배치 프레임워크와 함께 이 +메커니즘들은 AHBM에서의 향후 AI 커널과 통신 라이브러리의 기반을 +형성한다. 2026년 하반기에는 이 기반을 FFN 및 MoE 주도 워크로드와 +완전한 LLM 실행의 종단간 최적화로 확장할 예정이다. + +본 보고서의 나머지 부분은 다음과 같이 구성된다. +\S\ref{sec:platform}에서는 KernBench 플랫폼과 본 연구 전반에 걸쳐 +사용된 AHBM 구성을 기술한다. \S\ref{sec:gemm}, +\S\ref{sec:allreduce}, \S\ref{sec:gqa}에서는 각각 composite-command +GEMM 파이프라인, PE\_IPCQ 집합 통신, 융합 GQA 커널을 다룬다. +\S\ref{sec:discussion}에서는 이러한 결과의 광범위한 아키텍처적 함의를 +논의한다. 마지막으로 \S\ref{sec:conclusion}과 \S\ref{sec:future}에서는 +주요 결과를 요약하고 FFN, MoE, 전체 모델 최적화에 관한 향후 과제를 +제시한다. diff --git a/docs/report/1H-codesign-paper/sections/02-platform.tex b/docs/report/1H-codesign-paper/sections/02-platform.tex index 3c9cd69..3ef56c4 100644 --- a/docs/report/1H-codesign-paper/sections/02-platform.tex +++ b/docs/report/1H-codesign-paper/sections/02-platform.tex @@ -1,16 +1,15 @@ \begin{figure*}[t] \centering -\begin{subfigure}[b]{0.42\textwidth} +\begin{subfigure}[b]{0.495\textwidth} \centering - \includegraphics[width=\linewidth,height=0.6\linewidth,keepaspectratio]{sip_architecture.pdf} - \caption{SIP level: $4{\times}4$ CUBE mesh + IO chiplet.} + \includegraphics[width=\linewidth,height=0.7\linewidth,keepaspectratio]{sip_architecture.pdf} + \caption{SIP Architecture} \label{fig:sip-arch} \end{subfigure}\hfill \begin{subfigure}[b]{0.42\textwidth} \centering - \includegraphics[width=\linewidth,height=0.6\linewidth,keepaspectratio]{cube_architecture.pdf} - \caption{CUBE level (zoom-in of one CUBE in (a)): 8 PEs, HBM - channels, M\_CPU, SRAM, NoC router mesh, UCIe links.} + \includegraphics[width=\linewidth,height=0.84\linewidth]{cube_architecture.pdf} + \caption{CUBE Architecture} \label{fig:cube-arch} \end{subfigure} @@ -26,9 +25,12 @@ on-device collective control plane.} \label{fig:pe-arch} \end{subfigure} -\caption{Modeled hardware graph at the SIP, CUBE, and PE levels -(one example configuration; specific parameters in \S\ref{sec:hw} / -Table~\ref{tab:hw}). KernBench is not tied to this particular +\caption{Modeled hardware graph at the SIP, CUBE, and PE levels. +This figure is an \emph{illustrative topology} chosen for +readability; the \emph{experimental configuration} used throughout +this report (port counts, slice counts, and link parameters) is +specified in \S\ref{sec:hw} / Table~\ref{tab:hw}, and numbers in the +two may differ. KernBench is not tied to this particular arrangement: each box is a modeled component node, each line a directed link with bandwidth and propagation attributes, and any topology that respects those attributes is supported.} @@ -144,8 +146,20 @@ KernBench is layered along the flow of a request: controllers, and the inter-chiplet links. \end{itemize} -Data and timing are handled in two passes, so that a kernel's numeric -results and its latency are computed consistently but independently. +Data and timing are handled in two passes, so that a kernel's +numeric results and its latency are computed consistently but +independently. \textbf{Pass~1 (timing)} runs the kernel under +the discrete-event engine: memory ops (\textsf{tl.load}, +\textsf{tl.store}) execute against a host-side \textsf{MemoryStore} +and return real tensor data, while compute ops (GEMM, vector-math) +only emit records into an op-log carrying their operands, shapes, +dtypes, and scheduled time. \textbf{Pass~2 (data)} replays that +op-log offline in numpy, producing the actual numeric outputs and +comparing them against a reference computation under per-dtype +tolerances (e.g.\ \texttt{rtol}/\texttt{atol} $=10^{-3}$ for f16). +Pass~2 is optional---runs that need only latency skip it---but when +enabled it guarantees that every reported timing number corresponds +to a kernel whose numeric output has been verified end-to-end. \subsection{Latency model: graph traversal and contention} \label{sec:latency} @@ -185,7 +199,7 @@ fires them one at a time, with ties broken under a deterministic policy so that the same kernel on the same topology always yields the same trace. Per-request correlation IDs are stamped at injection and carried through every hop, so the path from injection to completion -is fully traceable. Every nanosecond in a reported latency +is fully traceable. Every modeled latency contribution corresponds to exactly one of these events on exactly one node or edge---there is no slack in the budget. @@ -202,6 +216,65 @@ collective engines hold the request for their service time before releasing it downstream. Each of these is attached to a specific node or edge in the graph; together they make up the entire latency budget. +Beyond data movement and execution latency, KernBench +also models the control-plane cost required to issue work +to accelerator engines. Since every DMA, GEMM, vector- +math, and IPCQ operation is initiated through the +\textsf{PE\_CPU} $\rightarrow$ \textsf{PE\_SCHED} path, +command dispatch latency contributes to the end-to-end +execution time of all kernels. + +\paragraph{Command dispatch overhead model.} The cost incurred by +the \textsf{PE\_CPU} when it dispatches a command to one of the +accelerator engines is modelled structurally rather than with a +per-operation calibration table. The \textsf{PE\_CPU} charges, per +command, +\[ + d_{\text{cmd}} = \textsf{FIXED} + b_{\text{logical}} \cdot R, +\] +This linear-in-size form reflects the serialization cost of writing a +command descriptor into the scheduler queue: a fixed per-command +bookkeeping cost plus a byte-wise descriptor-transfer cost. +Concretely, $b_{\text{logical}}$ is the command's hardware-logical byte +size, \textsf{FIXED} captures the fixed per-command cost (queue-tail +update, completion registration) and $R$ captures the per-byte cost of +serializing the command descriptor into the scheduler queue. This +\textsf{FIXED} is distinct from Table~\ref{tab:hw}'s +\SI{2}{\nano\second} / \SI{1}{\nano\second} \textsf{PE\_CPU} / +\textsf{PE\_SCHED} fixed costs: the latter are per-component +traversal overheads each command pays when it transits those nodes, +whereas the 40-cycle \textsf{FIXED} term is the command-descriptor +issue cost. The +default anchoring (\textsf{FIXED} $= 40$ cycles, $R = 0.0625$ +cycles/byte, i.e.\ \SI{16}{\byte\per\cycle}, at \SI{1}{\giga\hertz}) +places a typical composite at roughly \SI{43}{\nano\second}, and a +hard cap on a composite's descriptor size prevents the model from +rewarding arbitrarily large fused commands beyond what real descriptor +queues accept. In the configurations measured here, command issue is +not the bottleneck---data movement is---so this term stays small +relative to DMA and collective time. + +\begin{table*}[t] +\centering +\caption{PE\,$\to$\,HBM DMA latency probe at varying hop distances +(\SI{32}{\kibi\byte} transfer; output captured from \texttt{kernbench +probe}). \emph{Util\%} is the achieved bandwidth as a fraction of the +path's bottleneck-edge bandwidth.} +\label{tab:probe-pe-dma} +\small +\begin{tabular}{@{}lrrr@{}} +\toprule +Case & Latency~(\si{\nano\second}) & Util\% (\,32\,KiB) & Util\% (\,1\,MiB) \\ +\midrule +PE\,$\to$\,local HBM & 141.0 & 90.8 & 100.0 \\ +PE\,$\to$\,same-half HBM & 147.9 & 86.6 & ~99.9 \\ +PE\,$\to$\,cross-half HBM & 161.2 & 79.4 & ~99.8 \\ +PE\,$\to$\,cross-CUBE (best) & 330.5 & 77.5 & ~99.6 \\ +PE\,$\to$\,cross-CUBE (worst) & 677.1 & 37.8 & ~97.8 \\ +\bottomrule +\end{tabular} +\end{table*} + \subsubsection{Congestion and contention modeling} \label{sec:congestion} @@ -225,26 +298,6 @@ it reveals where the real bottlenecks form and which hardware levers actually relieve them---which is exactly the question the codesign work in this report turns on. -\paragraph{Command dispatch overhead model.} The cost incurred by -the \textsf{PE\_CPU} when it dispatches a command to one of the -accelerator engines is modelled structurally rather than with a -per-operation calibration table. The \textsf{PE\_CPU} charges, per -command, -\[ - d_{\text{cmd}} = \textsf{FIXED} + b_{\text{logical}} \cdot R, -\] -where $b_{\text{logical}}$ is the command's hardware-logical byte size, -\textsf{FIXED} captures the fixed per-command cost (queue-tail update, -completion registration) and $R$ captures the per-byte cost of -serializing the command descriptor into the scheduler queue. The -default anchoring (\textsf{FIXED} $= 40$ cycles, $R = 0.0625$ -cycles/byte, i.e.\ \SI{16}{\byte\per\cycle}, at \SI{1}{\giga\hertz}) -places a typical composite at roughly \SI{43}{\nano\second}, and a -hard cap on a composite's descriptor size prevents the model from -rewarding arbitrarily large fused commands beyond what real descriptor -queues accept. In the configurations measured here, command issue is -not the bottleneck---data movement is---so this term stays small -relative to DMA and collective time. \subsection{Accuracy} \label{sec:accuracy} @@ -254,41 +307,29 @@ dominate kernel latency on this class of hardware: per-edge bandwidth occupancy and flit-level serialization, HBM pseudo-channel parallelism, and per-component switching overhead. Two independent cross-checks drawn from the experiments in this report confirm that this precision -translates into physically reasonable kernel latencies. First, in the -GEMM study (\S\ref{sec:gemm}), simulator-measured MAC efficiency -tracks an analytic ideal-pipeline model within roughly +translates into physically reasonable kernel latencies. + +First, in the GEMM study (\S\ref{sec:gemm}), simulator-measured MAC +efficiency tracks an analytic ideal-pipeline model within roughly \SIrange{10}{20}{\percent} across a wide range of tile counts; the residual gap is attributable to pipeline-fill and DMA effects the -analytic model omits. Second, in the all-reduce study -(\S\ref{sec:allreduce}, Fig.~\ref{fig:allreduce-cmp}), simulator -latency for a 2D-torus over six devices follows the expected -startup-plus-per-packet shape across the entire payload sweep---tight -at small payloads where startup dominates, and within a single-digit -multiplicative factor at the largest payloads, where the residual gap -is explained by per-router switching the analytic shape elides. A -single-device point from an external full-system simulator (FSIM) at -the largest payload sits an order of magnitude above the KernBench -multi-device torus, illustrating the well-known gap between an -achievable-kernel number and a full end-to-end-stack number rather -than a model error. The known simplifications---FIFO router -arbitration (instead of round-robin), HBM scheduler without -write-buffer reordering, no bank conflict, no refresh or thermal -effects, and no upstream backpressure---are the price of a -deterministic, inspectable model. These simplifications bound the -absolute accuracy, but the agreement with both analytic models and the -external full-system simulator data above indicates that KernBench is -sufficiently accurate for evaluating the \emph{relative} -hardware--software design trade-offs (tiling A vs.\ B, topology X -vs.\ Y, with vs.\ without composite command, mesh vs.\ torus) that -are the primary objective of this work. +analytic model omits. -A third source of confidence comes from directly probing the -simulator's per-traversal behaviour. Running \texttt{kernbench probe} -on the modelled topology issues a sequence of PE-to-HBM DMA reads at -progressively greater hop distances and reports the per-component -overhead, per-edge serialization, and per-PC drain that the model -charges (Table~\ref{tab:probe-pe-dma}). Three properties stand out. -First, the reported latency increases \emph{monotonically} with hop +Second, in the all-reduce study (\S\ref{sec:allreduce}, +Fig.~\ref{fig:allreduce-cmp}), simulator latency for a 2D-torus over +six devices follows the expected startup-plus-per-packet shape across +the entire payload sweep---tight at small payloads where startup +dominates, and within a single-digit multiplicative factor at the +largest payloads, where the residual gap is explained by per-router +switching the analytic shape elides. + +Third, the simulator's per-traversal behaviour can be probed +directly. Running \texttt{kernbench probe} on the modelled topology +issues a sequence of PE-to-HBM DMA reads at progressively greater +hop distances and reports the per-component overhead, per-edge +serialization, and per-PC drain that the model charges +(Table~\ref{tab:probe-pe-dma}). Three properties stand out. First, +the reported latency increases \emph{monotonically} with hop count---from \SI{141}{\nano\second} at the local HBM slice to \SI{677}{\nano\second} at the worst-case remote-CUBE slice---with the increment per added hop matching the per-router overhead and the @@ -308,26 +349,19 @@ self-consistent---a model error in any of them would surface as a non-monotonic or under-utilising curve here long before it polluted a kernel-level measurement. -\begin{table*}[t] -\centering -\caption{PE\,$\to$\,HBM DMA latency probe at varying hop distances -(\SI{32}{\kibi\byte} transfer; output captured from \texttt{kernbench -probe}). \emph{Util\%} is the achieved bandwidth as a fraction of the -path's bottleneck-edge bandwidth.} -\label{tab:probe-pe-dma} -\small -\begin{tabular}{@{}lrrr@{}} -\toprule -Case & Latency~(\si{\nano\second}) & Util\% (\,32\,KiB) & Util\% (\,1\,MiB) \\ -\midrule -PE\,$\to$\,local HBM & 141.0 & 90.8 & 100.0 \\ -PE\,$\to$\,same-half HBM & 147.9 & 86.6 & ~99.9 \\ -PE\,$\to$\,cross-half HBM & 161.2 & 79.4 & ~99.8 \\ -PE\,$\to$\,cross-CUBE (best) & 330.5 & 77.5 & ~99.6 \\ -PE\,$\to$\,cross-CUBE (worst) & 677.1 & 37.8 & ~97.8 \\ -\bottomrule -\end{tabular} -\end{table*} +The known simplifications---FIFO router arbitration (instead of +round-robin), HBM scheduler without write-buffer reordering, no bank +conflict, no refresh or thermal effects, and no upstream +backpressure---are the price of a deterministic, inspectable model. +These simplifications bound the absolute accuracy, but the agreement +with both analytic models and the probe's internal-consistency +checks above indicates that KernBench is sufficiently accurate for +evaluating the \emph{relative} hardware--software design trade-offs +(tiling A vs.\ B, topology X vs.\ Y, with vs.\ without composite +command, mesh vs.\ torus) that are the primary objective of this +work. + + \subsection{Modeled hardware configuration} \label{sec:hw} @@ -344,10 +378,22 @@ such that inference workloads can effectively saturate the available HBM bandwidth. The aggregate compute throughput is therefore balanced against memory-system bandwidth rather than being intentionally over- or -under-provisioned. HBM bandwidth is distributed evenly -across the PEs within a CUBE, with each PE responsible -for servicing approximately one-eighth of the CUBE's -memory bandwidth through its dedicated HBM channels. +under-provisioned. Concretely, 8 PEs $\times$ +\SI{8}{\tera\flop\per\second} give roughly +\SI{64}{\tera\flop\per\second} of f16 compute per CUBE +against \SI{2048}{\giga\byte\per\second} of HBM bandwidth, +which places the balanced point at about +$\sim$31~FLOP/byte; inference decode kernels typically +operate well below that, so HBM bandwidth---not compute---is +the natural ceiling on this configuration. For reference, the +GQA decode kernels evaluated in \S\ref{sec:gqa} operate at +arithmetic intensity well below this balance point, so their +ceiling is set by HBM and inter-PE traffic rather than by GEMM +throughput. HBM bandwidth +is distributed evenly across the PEs within a CUBE, with +each PE responsible for servicing approximately one-eighth +of the CUBE's memory bandwidth through its dedicated HBM +channels. The on-chip interconnect is configured using bandwidth and latency parameters representative of commercially @@ -371,7 +417,7 @@ payload sizes are introduced in their respective sections. \begin{table}[t] \centering -\caption{Modeled hardware configuration (shared by all experiments).} +\caption{Modeled hardware configuration} \label{tab:hw} \small \begin{tabular}{@{}ll@{}} @@ -389,11 +435,12 @@ GEMM engine peak & \SI{8}{\tera\flop\per\second} (f16) \\ TCM (on-PE) & \SI{16}{\mega\byte}, \SI{512}{\giga\byte\per\second} R/W \\ \quad kernel scratch & \SI{1}{\mega\byte} \\ DMA engines & 1 read + 1 write \\ -CPU / scheduler overhead & \SI{2}{\nano\second} / \SI{1}{\nano\second} \\ +\textsf{PE\_CPU} fixed cost & \SI{2}{\nano\second} \\ +\textsf{PE\_SCHED} fixed cost & \SI{1}{\nano\second} \\ \midrule \multicolumn{2}{@{}l}{\emph{Memory (per CUBE)}} \\ HBM capacity & \SI{48}{\giga\byte} (8 slices) \\ -HBM aggregate BW & \SI{1024}{\giga\byte\per\second} \\ +HBM aggregate BW & \SI{2048}{\giga\byte\per\second} \\ HBM pseudo-channels & 64 (8 per PE), \SI{32}{\giga\byte\per\second} each \\ SRAM (shared) & \SI{32}{\mega\byte}, \SI{128}{\giga\byte\per\second} link \\ HBM burst & \SI{256}{\byte} \\ diff --git a/scripts/paper/paper_cube_architecture.py b/scripts/paper/paper_cube_architecture.py new file mode 100644 index 0000000..57e5a12 --- /dev/null +++ b/scripts/paper/paper_cube_architecture.py @@ -0,0 +1,155 @@ +"""Re-emit cube_view.svg in an academic (white-background, large-font) +palette and convert it to PDF for the 1H-codesign-paper Figure 2. + +Source of truth: docs/diagrams/cube_view.svg (generated by +src/kernbench/topology/visualizer.py:_render_cube_view_svg, dark theme). + +This script does a targeted color/font/size remap on the dark palette so +the resulting figure prints well on a white paper page. If the upstream +palette or geometry in visualizer.py changes, the maps below must be +reviewed. +""" +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SRC_SVG = REPO / "docs" / "diagrams" / "cube_view.svg" +OUT_DIR = REPO / "docs" / "report" / "1H-codesign-paper" / "figures" +OUT_SVG = OUT_DIR / "cube_architecture.svg" +OUT_PDF = OUT_DIR / "cube_architecture.pdf" + +# ── 1. Legend-icon safety: legend rects whose general color rule below +# would turn them white-on-white. Apply BEFORE the wholesale fill rules. +LEGEND_FIXUP: list[tuple[str, str]] = [ + # "Relay" legend icon (slate-700 router) + ('fill="#334155" stroke="#475569" stroke-width="0.5"', + 'fill="#94a3b8" stroke="#475569" stroke-width="0.5"'), + # "Mesh Link" legend icon (slate-600) + ('fill="#475569" stroke="#475569" stroke-width="0.5"', + 'fill="#94a3b8" stroke="#475569" stroke-width="0.5"'), +] + +# ── 2. Color remap: dark theme -> academic (white) theme +COLOR_MAP: list[tuple[str, str]] = [ + # page background slate-900 -> white + ('fill="#0f172a"', 'fill="#ffffff"'), + # title text slate-400 -> slate-800 + ('fill="#94a3b8"', 'fill="#1f2937"'), + # subtitle text slate-500 -> slate-600 + ('fill="#64748b"', 'fill="#475569"'), + # router has-attach stroke slate-500 -> slate-600 + ('stroke="#64748b"', 'stroke="#475569"'), + # router has-attach fill slate-600 -> white + ('fill="#475569"', 'fill="#ffffff"'), + # mesh lines + cube boundary stroke slate-600 -> slate-400 + ('stroke="#475569"', 'stroke="#94a3b8"'), + # router no-attach fill slate-700 -> white + ('fill="#334155"', 'fill="#ffffff"'), + # component block dark fills -> white (PE / M_CPU / SRAM / UCIe) + ('fill="#2d1f3d"', 'fill="#ffffff"'), + ('fill="#451a03"', 'fill="#ffffff"'), + ('fill="#1c1917"', 'fill="#ffffff"'), + ('fill="#1e1b4b"', 'fill="#ffffff"'), + # HBM zone background emerald-950 -> emerald-50 + ('fill="#052e16"', 'fill="#ecfdf5"'), + # router label text white -> slate-800 + ('fill="white"', 'fill="#1f2937"'), + # boost low-alpha emerald annotations so PE/HBM BW labels read on light bg + ('fill="#10b98188"', 'fill="#047857"'), + ('fill="#05966988"', 'fill="#059669"'), +] + +# ── 3. UCIe palette desaturation: bright violet/indigo/fuchsia is too +# attention-grabbing on a white page; remap to a slate gradient. +# NOTE: side-effect on PE2/PE3 HBM port-bar colors (which share +# #8b5cf6/#a78bfa with UCIe) — PE labels still convey identity. +# Applied AFTER the academic color map so the new slate values are +# not picked up by the rules above. +UCIE_DESATURATE: list[tuple[str, str]] = [ + # UCIe block stroke/text (violet-500) -> slate-600 + ('"#8b5cf6"', '"#475569"'), + # UCIe cell 1 / PE3 port bar (violet-400) -> slate-400 + ('"#a78bfa"', '"#94a3b8"'), + # UCIe cell 0 (indigo-400) -> slate-300 + ('"#818cf8"', '"#cbd5e1"'), + # UCIe cell 2 (purple-400) -> slate-500 + ('"#c084fc"', '"#64748b"'), + # UCIe cell 3 (fuchsia-400) -> gray-700 + ('"#e879f9"', '"#374151"'), +] + +# ── 4. Font-size bumps. The CUBE figure is rendered at half-text-width +# (~250pt) inside the side-by-side subfigure in 02-platform.tex, so +# native fonts get crushed ~3x by \linewidth scaling. We push the +# bumps to the legibility limit of the layout (router-label text +# stays inside a slightly enlarged circle; legend items may touch). +FONT_MAP: dict[str, str] = { + "5": "10", + "6": "12", + "7": "14", + "8": "13", # legend rect text — capped by upstream layout spacing + # (advance = 7*len(label)+24 was sized for ~font 8); + # font 13 keeps each item's text inside its slot. + "9": "14", + "10": "15", + "11": "16", + "14": "18", # title — kept moderate so it does not overflow canvas +} + +# ── 5. Router circle radius bump (only circles use r="8"). Enlarged so +# the bumped router labels stay inside the circle. +RADIUS_MAP: list[tuple[str, str]] = [ + (' r="8"', ' r="17"'), +] + +# ── 6. Tighten whitespace: move legend just below dashed box and crop +# the unused canvas margins so LaTeX's \linewidth scaling does not +# shrink the figure text any more than necessary. +LAYOUT_FIXUP: list[tuple[str, str]] = [ + # Move legend rects up (dashed box bottom is at y=760) + (' y="865"', ' y="775"'), + # Move legend text baselines up to match (offset = 9 px) + (' y="874"', ' y="784"'), + # Tight crop: 5 px margin around dashed box + cut top/bottom whitespace + ('', + ''), +] + + +def _bump_font(m: re.Match) -> str: + return f'font-size="{FONT_MAP.get(m.group(1), m.group(1))}"' + + +def main() -> None: + if not SRC_SVG.exists(): + raise SystemExit(f"source SVG missing: {SRC_SVG}") + rsvg = shutil.which("rsvg-convert") + if rsvg is None: + raise SystemExit("rsvg-convert not found (brew install librsvg)") + + svg = SRC_SVG.read_text(encoding="utf-8") + for old, new in (LEGEND_FIXUP + COLOR_MAP + UCIE_DESATURATE + + RADIUS_MAP + LAYOUT_FIXUP): + if old not in svg: + print(f"warn: pattern not present in source SVG: {old}") + svg = svg.replace(old, new) + svg = re.sub(r'font-size="(\d+)"', _bump_font, svg) + + OUT_DIR.mkdir(parents=True, exist_ok=True) + OUT_SVG.write_text(svg, encoding="utf-8") + subprocess.run( + [rsvg, "-f", "pdf", "-o", str(OUT_PDF), str(OUT_SVG)], + check=True, + ) + print(f"wrote {OUT_SVG.relative_to(REPO)}") + print(f"wrote {OUT_PDF.relative_to(REPO)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/paper/paper_sip_architecture.py b/scripts/paper/paper_sip_architecture.py new file mode 100644 index 0000000..8b12e98 --- /dev/null +++ b/scripts/paper/paper_sip_architecture.py @@ -0,0 +1,91 @@ +"""Re-emit sip_view.svg in an academic palette (white background, black +strokes) and convert it to PDF for the 1H-codesign-paper Figure 1. + +Source of truth: docs/diagrams/sip_view.svg (generated by +src/kernbench/topology/visualizer.py, dark-ish theme). + +Treatment per user request: keep the layout intact, just force a white +canvas and turn every stroke black; promote faint label text to black so +all annotations stay legible on white. +""" +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SRC_SVG = REPO / "docs" / "diagrams" / "sip_view.svg" +OUT_DIR = REPO / "docs" / "report" / "1H-codesign-paper" / "figures" +OUT_SVG = OUT_DIR / "sip_architecture.svg" +OUT_PDF = OUT_DIR / "sip_architecture.pdf" + +# Drop the "SIP VIEW" title; the figure caption already names the level. +TITLE_REMOVE: list[tuple[str, str]] = [ + (' SIP VIEW\n', + ''), +] + +COLOR_MAP: list[tuple[str, str]] = [ + # canvas background slate-50 -> pure white + ('fill="#f8fafc"', 'fill="#ffffff"'), + # all strokes -> black + ('stroke="#3b82f6"', 'stroke="#000000"'), # UCIe mesh blue lines + ('stroke="#475569"', 'stroke="#000000"'), # cube block borders + ('stroke="#0ea5e9"', 'stroke="#000000"'), # I/O sky-blue lines + # link annotation text slate-500 -> black for readability + ('fill="#64748b"', 'fill="#000000"'), +] + +# Font-size bumps so labels survive LaTeX \linewidth scaling at the +# half-text-width subfigure. CUBE block labels overflow the 48px block +# rects, which is acceptable here. +FONT_MAP: dict[str, str] = { + "7": "10", + "14": "17", +} + +# Tighten whitespace: cube grid occupies x=[84,564], y=[128,520]; IO +# chiplet sits around y~50. With the title removed, crop top to y=40 so +# the IO chiplet keeps a small headroom. Crop 70px on each side and 113 +# px from bottom. +LAYOUT_FIXUP: list[tuple[str, str]] = [ + ('', + ''), +] + + +def _bump_font(m: re.Match) -> str: + return f'font-size="{FONT_MAP.get(m.group(1), m.group(1))}"' + + +def main() -> None: + if not SRC_SVG.exists(): + raise SystemExit(f"source SVG missing: {SRC_SVG}") + rsvg = shutil.which("rsvg-convert") + if rsvg is None: + raise SystemExit("rsvg-convert not found (brew install librsvg)") + + svg = SRC_SVG.read_text(encoding="utf-8") + for old, new in TITLE_REMOVE + COLOR_MAP + LAYOUT_FIXUP: + if old not in svg: + print(f"warn: pattern not present in source SVG: {old}") + svg = svg.replace(old, new) + svg = re.sub(r'font-size="(\d+)"', _bump_font, svg) + + OUT_DIR.mkdir(parents=True, exist_ok=True) + OUT_SVG.write_text(svg, encoding="utf-8") + subprocess.run( + [rsvg, "-f", "pdf", "-o", str(OUT_PDF), str(OUT_SVG)], + check=True, + ) + print(f"wrote {OUT_SVG.relative_to(REPO)}") + print(f"wrote {OUT_PDF.relative_to(REPO)}") + + +if __name__ == "__main__": + main() diff --git a/topology.yaml b/topology.yaml index 3f83a46..e1238d4 100644 --- a/topology.yaml +++ b/topology.yaml @@ -98,7 +98,6 @@ cube: memory_map: hbm_total_gb_per_cube: 48 hbm_slices_per_cube: 8 - hbm_total_bw_gbs: 1024.0 hbm_mapping_mode: n_to_one # one_to_one | n_to_one (ADR-0017 D8) hbm_pseudo_channels: 64 # total pseudo channels per cube hbm_channels_per_pe: 8 # = pseudo_channels / pes_per_cube