ywkang
049e3d8bb3
Move benches/ -> src/kernbench/benches/ and src/kernbench/cli/probe.py -> src/kernbench/probes/probe.py. Each bench self-registers via @bench(name=..., description=...); kernbench list enumerates benches with auto-assigned indices, --bench accepts kebab-case name or numeric index. Audit at package-import time fails if any non-underscore module forgets the decorator. ADR-0010 (EN + KO) updated to reflect the new resolver path, list subcommand, and probes package separation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
96 lines
2.7 KiB
Python
96 lines
2.7 KiB
Python
"""Tests for kernbench.benches.registry — @bench decorator + resolve/list."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from kernbench.benches import registry
|
|
|
|
|
|
EXPECTED_NAMES = [
|
|
"ccl-allreduce",
|
|
"gemm-single-pe",
|
|
"gpt3-qkv",
|
|
"ipcq-allreduce",
|
|
"matmul-composite",
|
|
"qkv-gemm",
|
|
"qkv-gemm-multi-pe",
|
|
"va-offset-verify",
|
|
]
|
|
|
|
|
|
def test_registry_lists_all_benches():
|
|
specs = registry.list_all()
|
|
names = [s.name for s in specs]
|
|
assert names == EXPECTED_NAMES
|
|
|
|
|
|
def test_registry_indices_are_1_based_sorted_by_name():
|
|
specs = registry.list_all()
|
|
assert [s.index for s in specs] == list(range(1, len(EXPECTED_NAMES) + 1))
|
|
assert sorted(s.name for s in specs) == [s.name for s in specs]
|
|
|
|
|
|
def test_resolve_by_name_returns_spec():
|
|
spec = registry.resolve("gemm-single-pe")
|
|
assert spec.name == "gemm-single-pe"
|
|
assert callable(spec.run)
|
|
assert spec.description.strip()
|
|
|
|
|
|
def test_resolve_by_index_string_matches_list_order():
|
|
specs = registry.list_all()
|
|
third = specs[2]
|
|
resolved = registry.resolve(str(third.index))
|
|
assert resolved is third
|
|
|
|
|
|
def test_resolve_unknown_name_raises():
|
|
with pytest.raises(ValueError, match="kernbench list"):
|
|
registry.resolve("does-not-exist")
|
|
|
|
|
|
def test_resolve_unknown_index_raises():
|
|
with pytest.raises(ValueError, match="kernbench list"):
|
|
registry.resolve("99")
|
|
|
|
|
|
def test_resolve_empty_identifier_raises():
|
|
with pytest.raises(ValueError):
|
|
registry.resolve("")
|
|
|
|
|
|
def test_bench_decorator_rejects_invalid_name():
|
|
with pytest.raises(ValueError, match="kebab-case"):
|
|
registry.bench(name="Invalid_Name", description="x")
|
|
|
|
|
|
def test_bench_decorator_rejects_empty_description():
|
|
with pytest.raises(ValueError, match="non-empty"):
|
|
registry.bench(name="ok-name", description=" ")
|
|
|
|
|
|
def test_audit_raises_on_missing_decorator():
|
|
with pytest.raises(RuntimeError, match="missing @bench decorator"):
|
|
registry._audit_modules(
|
|
imported=["kernbench.benches.fake_no_dec", "kernbench.benches.real"],
|
|
registered={"kernbench.benches.real"},
|
|
)
|
|
|
|
|
|
def test_audit_passes_when_all_registered():
|
|
registry._audit_modules(
|
|
imported=["kernbench.benches.a", "kernbench.benches.b"],
|
|
registered={"kernbench.benches.a", "kernbench.benches.b"},
|
|
)
|
|
|
|
|
|
def test_duplicate_name_at_finalize_fails(monkeypatch):
|
|
"""_finalize() rejects two pending entries with the same name."""
|
|
monkeypatch.setattr(registry, "_PENDING", [
|
|
("dup", "d1", lambda: None),
|
|
("dup", "d2", lambda: None),
|
|
])
|
|
monkeypatch.setattr(registry, "_REGISTRY", {})
|
|
with pytest.raises(RuntimeError, match="duplicate bench name"):
|
|
registry._finalize()
|