diff --git a/tests/analytical_visualization/app.py b/tests/analytical_visualization/app.py index be8581a..c84702a 100644 --- a/tests/analytical_visualization/app.py +++ b/tests/analytical_visualization/app.py @@ -555,12 +555,13 @@ if _warnings: # Auto Suggest is renamed "Parallelism" since it only varies parallelism # knobs (hardware is held fixed at the sidebar values). (tab_layout, tab_auto, tab_memory, tab_stages, tab_compare, tab_hw, - tab_roofline) = st.tabs([ + tab_roofline, tab_planning) = st.tabs([ "Physical layout", "Auto Suggest Parallelism", "Memory breakdown", "Per-stage latency", "Save & compare", "Auto Hardware", "Chip roofline & B*", + "Capacity planning", ]) @@ -2642,15 +2643,44 @@ with tab_roofline: st.dataframe(pd.DataFrame(_formula_rows), width='stretch', hide_index=True) - st.divider() - # ── Capacity planning: three-axis GPU count ──────────────────── - st.subheader("How hyperscalers pick GPU count") - st.markdown( - "**N_GPUs = max(capacity_floor, KV_headroom, throughput_SLO) × N_replicas.** " - "Three independent axes gate the minimum deployment size." +# ── TAB 8: Capacity planning ───────────────────────────────────── +with tab_planning: + st.subheader("Capacity planning & standards") + st.caption( + "How hyperscalers decide GPU count from a workload, what SLO " + "targets they aim for, and what deployment patterns they use. " + "All numbers below use the sidebar model + the Chip Roofline " + "tab's effective chip (from its FLOPs/BW knobs, or sidebar " + "defaults if you haven't opened that tab)." ) + # ── Section 1: three-axis sizing formula + calculator ───────── + st.markdown("### 1. GPU count = max(A, B, C) × N_replicas") + + _axes_rows = [ + {"Axis": "A. Capacity floor", + "Formula": "⌈ N·b / HBM_per_PE ⌉", + "Meaning": ("Min PEs to hold ONE replica's weights alone. " + "The bare floor — below this weights don't fit."), + "Grows with": "Model size (N)"}, + {"Axis": "B. KV headroom", + "Formula": "⌈ (N·b + users_per_replica · S_kv · kv_bpt) / HBM_per_PE ⌉", + "Meaning": ("Min PEs to hold weights + all KV of the users " + "assigned to this replica."), + "Grows with": "Users × context"}, + {"Axis": "C. Throughput SLO", + "Formula": ("N_replicas = ⌈ n_users / B_at_SLO ⌉, where " + "B_at_SLO = largest B s.t. step_latency ≤ TPOT SLO"), + "Meaning": ("Enough replicas so each carries ≤ B_at_SLO users " + "and meets per-token latency SLO."), + "Grows with": "Users, or tighter SLO"}, + ] + st.dataframe(pd.DataFrame(_axes_rows), width='stretch', + hide_index=True) + + st.markdown("#### Live calculator") + _cp_a, _cp_b, _cp_c = st.columns(3) with _cp_a: _cp_users = st.number_input( @@ -2661,7 +2691,7 @@ with tab_roofline: with _cp_b: _cp_ctx = st.number_input( "Avg context / user (tokens)", min_value=1, - max_value=2_000_000, value=int(_rf_skv), step=128, + max_value=2_000_000, value=int(s_kv), step=128, key="_cp_avg_ctx", help="Mean S_kv across the active users.", ) @@ -2673,115 +2703,155 @@ with tab_roofline: "Tighter = smaller B per replica, more replicas."), ) + # Use the base sidebar chip for the calculator so it's stable. + _cp_machine = _default_machine _sizing = size_deployment( - _scaled_machine, model, + _cp_machine, model, n_users=int(_cp_users), avg_ctx=int(_cp_ctx), tpot_slo_s=_cp_slo_ms / 1000.0, ) _s1, _s2, _s3, _s4 = st.columns(4) - _s1.metric("Axis A: Capacity", + _s1.metric("A. Capacity", f"{_sizing.pes_axis_a_capacity} PEs", - help="Min PEs to hold one replica's weights alone. " - "Below this, weights don't fit at any sharding.") - _s2.metric("Axis B: KV headroom", + help="Min PEs to hold weights alone.") + _s2.metric("B. KV headroom", f"{_sizing.pes_axis_b_kv} PEs", - help="Min PEs to hold weights + all KV of this replica's users. " - "Grows with users × context.") - _s3.metric("Axis C: Throughput", + help="Min PEs to hold weights + all replica KV.") + _s3.metric("C. Throughput", (f"{_sizing.n_replicas} replicas" if _sizing.b_at_slo > 0 else "SLO INFEASIBLE"), - help=(f"B_at_SLO = {_sizing.b_at_slo} per replica; " - f"need ceil({_cp_users}/{_sizing.users_per_replica}) " + help=(f"B_at_SLO = {_sizing.b_at_slo}; ceil(" + f"{_cp_users}/{_sizing.users_per_replica}) " f"= {_sizing.n_replicas} replicas.")) - _s4.metric("Total GPUs (recommended)", + _s4.metric("Total GPUs", f"{_sizing.total_pes}", delta=f"binds: {_sizing.binding_axis}", help="max(A, B) × N_replicas.") if _sizing.b_at_slo == 0: st.error( - f"SLO infeasible on this chip: even B=1 step latency exceeds " - f"{_cp_slo_ms} ms at S_kv={_cp_ctx:,}. Increase the SLO, " - f"shorten the context, or move to a faster chip (raise the " - f"FLOPs/BW knobs above)." + f"SLO infeasible: even B=1 step latency exceeds " + f"{_cp_slo_ms} ms at S_kv = {_cp_ctx:,}. Loosen the SLO, " + f"shorten context, or use a faster chip." ) else: _binding_hint = { - "capacity": ("Weights dominate — the model is big relative to " - "per-PE HBM. Adding replicas or bigger HBM chips " - "buys the most headroom."), - "kv": ("KV cache dominates — users × context is filling HBM " - "after weights. Shorter context / smaller B / more " - "aggressive CP sharding help most."), - "throughput": ("Latency SLO forces you into more replicas — " - "one replica can only serve ~" - f"{_sizing.b_at_slo} users at this SLO. " - "Loosening TPOT or picking a faster chip cuts " - "the replica count."), + "capacity": ("Weights dominate. Adding replicas or bigger-HBM " + "chips buys the most headroom."), + "kv": ("KV cache dominates. Shorter context / smaller B / " + "more aggressive CP sharding help most."), + "throughput": (f"Latency SLO forces more replicas — one " + f"replica serves ~{_sizing.b_at_slo} users. " + "Loosening TPOT or faster chip cuts count."), }[_sizing.binding_axis] st.caption( f"**Binding axis: {_sizing.binding_axis}.** {_binding_hint} " - f"At this config: B_at_SLO = **{_sizing.b_at_slo}** per " - f"replica → **{_sizing.users_per_replica}** users/replica × " - f"**{_sizing.n_replicas}** replicas = **{_sizing.total_pes} " - f"GPUs** total." + f"B_at_SLO = **{_sizing.b_at_slo}** / replica × " + f"**{_sizing.n_replicas}** replicas = " + f"**{_sizing.total_pes} GPUs**." ) st.divider() - # ── High-level info: same-setup vs different-setup ──────────── - st.subheader("Same setup or different for short vs long context?") - st.markdown( - "**In practice: hybrid — one elastic pool with length-tier " - "routing on top, plus prefill/decode disaggregation at the " - "frontier.**" + # ── Section 2: SLO targets by workload ───────────────────────── + st.markdown("### 2. SLO standards — TTFT & TPOT by workload") + st.caption( + "TTFT = Time To First Token (dominated by prefill of the prompt). " + "TPOT = Time Per Output Token (one decode step per token). " + "Real workload targets vary widely; the table is a rough guide." ) + _slo_rows = [ + {"Use case": "Voice / realtime", + "TTFT target": "200–300 ms (whole pipeline)", + "TPOT / ITL target": "10–25 ms", + "What binds": "TPOT — smooth speech needs tight per-token cadence"}, + {"Use case": "Code completion", + "TTFT target": "< 100 ms", + "TPOT / ITL target": "mostly n/a (short response)", + "What binds": "TTFT — user cursor is waiting"}, + {"Use case": "Interactive chat", + "TTFT target": "300 ms – 1 s", + "TPOT / ITL target": "20–50 ms", + "What binds": "Balanced; both matter to perceived latency"}, + {"Use case": "Agentic / multi-step", + "TTFT target": "as low as possible — compounds across steps", + "TPOT / ITL target": "maximize throughput", + "What binds": "Throughput — many sequential LLM calls"}, + {"Use case": "Batch / offline", + "TTFT target": "seconds to minutes", + "TPOT / ITL target": "irrelevant", + "What binds": "Cost/token — pack B to the ceiling"}, + ] + st.dataframe(pd.DataFrame(_slo_rows), width='stretch', + hide_index=True) - with st.expander("Layer 1 — One base replica config serving mixed traffic", - expanded=False): - st.markdown( - "- **PagedAttention** (vLLM) manages KV as fixed-size pages, " - "so different users' contexts pack efficiently into the same HBM.\n" - "- The scheduler admits requests based on remaining page pool: " - "*'does this new user's expected max_tokens fit?'*\n" - "- **Continuous batching** mixes different-length contexts in " - "the same forward pass.\n" - "- Works well for ~90% of traffic: short-to-medium context " - "(chat, RAG, code snippets)." - ) + st.divider() - with st.expander("Layer 2 — Length-tier routing for the tail", - expanded=False): - st.markdown( - "Frontier providers (OpenAI, Anthropic, Google) run **separate " - "GPU pools per context tier**:\n" - "- **Standard tier** (up to 32k or 128k): normal replica " - "config, high B, high utilization.\n" - "- **Long-context tier** (128k–1M+): bigger replicas with " - "more chips per instance, more CP sharding to spread KV, " - "lower B per replica.\n\n" - "A router at the API gateway inspects the request's max " - "context and dispatches accordingly. This is why Gemini's " - "1M-context tier is more expensive per token than standard — " - "those requests can't share efficient replicas with short-" - "context traffic." - ) + # ── Section 3: Hybrid deployment layers (table form) ────────── + st.markdown("### 3. Same setup or different for short vs long context?") + st.caption( + "**Hybrid: one elastic pool + length-tier routing on top + " + "prefill/decode disaggregation at the frontier.**" + ) + _layers_rows = [ + {"Layer": "1. Elastic pool", + "Serves": "Mixed traffic (~90% of requests, chat/RAG/code)", + "Key mechanism": "PagedAttention (KV in fixed-size pages) + " + "continuous batching", + "Admission rule": "Scheduler checks remaining KV page pool per " + "iteration; admits if user's expected " + "max_tokens fits", + "Typical config": "Standard replica (e.g. TP=8, CP=1..4)"}, + {"Layer": "2. Length-tier routing", + "Serves": "The long-context tail (128k–1M+)", + "Key mechanism": "API gateway inspects max context, dispatches " + "to a dedicated pool", + "Admission rule": "Standard tier for < 32-128k; long-context " + "tier otherwise", + "Typical config": ("Long-context replicas have more chips per " + "instance + heavier CP sharding, lower B " + "(e.g. TP=8, CP=32)")}, + {"Layer": "3. Disaggregated prefill/decode", + "Serves": "Frontier deployments (DistServe, Splitwise pattern)", + "Key mechanism": "Prefill on FLOPs-heavy pool; decode on HBM-BW-" + "heavy pool; KV moves over NVLink/RDMA", + "Admission rule": "Prefill goes to a prefill GPU; KV then " + "handed off to decode pool", + "Typical config": ("Two GPU pools with different chip mixes " + "optimized for prefill vs decode " + "rooflines")}, + ] + st.dataframe(pd.DataFrame(_layers_rows), width='stretch', + hide_index=True) - with st.expander("Layer 3 — Disaggregated prefill / decode " - "(DistServe, Splitwise)", expanded=False): - st.markdown( - "- **Prefill** (compute-heavy, high FLOPs/byte) runs on one " - "GPU pool.\n" - "- **Decode** (memory-heavy, HBM BW dominated) runs on " - "another.\n" - "- **KV cache is transferred over NVLink/RDMA** between the " - "pools.\n" - "- Long-context prefill goes to specialized 'prefill GPUs' " - "that then hand off KV to a decode-optimized pool.\n\n" - "Rationale: prefill and decode have opposite roofline " - "signatures. Running both on one homogeneous cluster wastes " - "either FLOPs (when decode dominates) or HBM BW (when " - "prefill dominates)." - ) + st.divider() + + # ── Section 4: Binding-axis playbook ────────────────────────── + st.markdown("### 4. What to do when each axis binds") + _playbook_rows = [ + {"Binding axis": "A. Capacity (weights)", + "Symptom": "Weights alone are close to per-PE HBM budget", + "First lever": "Increase TP or PP to shard weights across more PEs", + "Second lever": "Move to a bigger-HBM chip; use lower-precision " + "weights (FP8/INT4)", + "Cost impact": "Sub-linear — you pay for more chips, but " + "utilization stays high"}, + {"Binding axis": "B. KV headroom", + "Symptom": "Weights fit fine but users × context blows HBM", + "First lever": "Increase CP (shard sequence dim), or reduce " + "batch (accept fewer users)", + "Second lever": "GQA / MQA / MLA to shrink kv_bpt; sparse " + "attention; KV compression (INT4 KV)", + "Cost impact": "Direct — long-context requests fundamentally " + "cost more"}, + {"Binding axis": "C. Throughput SLO", + "Symptom": "Latency budget forces small B per replica", + "First lever": "Add more replicas (linear scale)", + "Second lever": "Loosen SLO if product allows; faster chip; " + "reduce model size; speculative decoding", + "Cost impact": "Linear in replica count"}, + ] + st.dataframe(pd.DataFrame(_playbook_rows), width='stretch', + hide_index=True)