benches: package as kernbench.benches, add @bench registry + list subcommand
Move benches/ -> src/kernbench/benches/ and src/kernbench/cli/probe.py -> src/kernbench/probes/probe.py. Each bench self-registers via @bench(name=..., description=...); kernbench list enumerates benches with auto-assigned indices, --bench accepts kebab-case name or numeric index. Audit at package-import time fails if any non-underscore module forgets the decorator. ADR-0010 (EN + KO) updated to reflect the new resolver path, list subcommand, and probes package separation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
"""Tests for kernbench.benches.registry — @bench decorator + resolve/list."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from kernbench.benches import registry
|
||||
|
||||
|
||||
EXPECTED_NAMES = [
|
||||
"ccl-allreduce",
|
||||
"gemm-single-pe",
|
||||
"gpt3-qkv",
|
||||
"ipcq-allreduce",
|
||||
"matmul-composite",
|
||||
"qkv-gemm",
|
||||
"qkv-gemm-multi-pe",
|
||||
"va-offset-verify",
|
||||
]
|
||||
|
||||
|
||||
def test_registry_lists_all_benches():
|
||||
specs = registry.list_all()
|
||||
names = [s.name for s in specs]
|
||||
assert names == EXPECTED_NAMES
|
||||
|
||||
|
||||
def test_registry_indices_are_1_based_sorted_by_name():
|
||||
specs = registry.list_all()
|
||||
assert [s.index for s in specs] == list(range(1, len(EXPECTED_NAMES) + 1))
|
||||
assert sorted(s.name for s in specs) == [s.name for s in specs]
|
||||
|
||||
|
||||
def test_resolve_by_name_returns_spec():
|
||||
spec = registry.resolve("gemm-single-pe")
|
||||
assert spec.name == "gemm-single-pe"
|
||||
assert callable(spec.run)
|
||||
assert spec.description.strip()
|
||||
|
||||
|
||||
def test_resolve_by_index_string_matches_list_order():
|
||||
specs = registry.list_all()
|
||||
third = specs[2]
|
||||
resolved = registry.resolve(str(third.index))
|
||||
assert resolved is third
|
||||
|
||||
|
||||
def test_resolve_unknown_name_raises():
|
||||
with pytest.raises(ValueError, match="kernbench list"):
|
||||
registry.resolve("does-not-exist")
|
||||
|
||||
|
||||
def test_resolve_unknown_index_raises():
|
||||
with pytest.raises(ValueError, match="kernbench list"):
|
||||
registry.resolve("99")
|
||||
|
||||
|
||||
def test_resolve_empty_identifier_raises():
|
||||
with pytest.raises(ValueError):
|
||||
registry.resolve("")
|
||||
|
||||
|
||||
def test_bench_decorator_rejects_invalid_name():
|
||||
with pytest.raises(ValueError, match="kebab-case"):
|
||||
registry.bench(name="Invalid_Name", description="x")
|
||||
|
||||
|
||||
def test_bench_decorator_rejects_empty_description():
|
||||
with pytest.raises(ValueError, match="non-empty"):
|
||||
registry.bench(name="ok-name", description=" ")
|
||||
|
||||
|
||||
def test_audit_raises_on_missing_decorator():
|
||||
with pytest.raises(RuntimeError, match="missing @bench decorator"):
|
||||
registry._audit_modules(
|
||||
imported=["kernbench.benches.fake_no_dec", "kernbench.benches.real"],
|
||||
registered={"kernbench.benches.real"},
|
||||
)
|
||||
|
||||
|
||||
def test_audit_passes_when_all_registered():
|
||||
registry._audit_modules(
|
||||
imported=["kernbench.benches.a", "kernbench.benches.b"],
|
||||
registered={"kernbench.benches.a", "kernbench.benches.b"},
|
||||
)
|
||||
|
||||
|
||||
def test_duplicate_name_at_finalize_fails(monkeypatch):
|
||||
"""_finalize() rejects two pending entries with the same name."""
|
||||
monkeypatch.setattr(registry, "_PENDING", [
|
||||
("dup", "d1", lambda: None),
|
||||
("dup", "d2", lambda: None),
|
||||
])
|
||||
monkeypatch.setattr(registry, "_REGISTRY", {})
|
||||
with pytest.raises(RuntimeError, match="duplicate bench name"):
|
||||
registry._finalize()
|
||||
Reference in New Issue
Block a user