rsp2k
f1d0f8df70
--access-cost C makes a route pay C to pass where it would close another pin's way out (its end cells, the via it may need there, and the clearance around both) while that pin still has connections to route. A pin inside a wire keepout can only leave by a via; its stub to the cell centre was track inside the keepout, which the exact check refused. The via now goes on the pin itself. Pin stubs are also checked exactly against wire keepouts, as the exact engine will check them. bench/diagnose.py reports what each refused connection collided with.
207 lines
8.9 KiB
Python
207 lines
8.9 KiB
Python
"""Route boards under several connection orders and compare variants fairly.
|
|
|
|
One routing run says little: which connection routes first moves a result by
|
|
ten connections either way on a crowded board. This runs every board with
|
|
every variant under the same seeds (``--seed``), so variant A and variant B
|
|
face the same order on seed 3, and compares them seed by seed.
|
|
|
|
uv run python bench/bench.py BOARDS -v base= -v hist1="--history-cost 1" \\
|
|
--seeds 6 --jobs 4 --out bench/results/hist.jsonl
|
|
|
|
BOARDS is a text file, one board per line: a name, a DSN path and any flags
|
|
that board always needs, e.g.
|
|
|
|
sense /path/to/charge-sense.dsn --edge-clearance 0.5
|
|
# comments and blank lines are ignored
|
|
|
|
A variant is ``NAME=FLAGS``, or ``NAME@SRC=FLAGS`` to run the code under SRC
|
|
(a worktree's ``src`` directory) rather than the installed package. The first
|
|
variant is the baseline. Results are appended to --out as JSON lines, and runs
|
|
already in it are skipped, so an interrupted benchmark picks up where it left
|
|
off. Exits non-zero if any run crashed or timed out.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
|
|
NOT_ROUTED = re.compile(r"freeroute: (\d+) connection\(s\) not routed")
|
|
TO_ZONE = re.compile(r"freeroute: (\d+) more connection\(s\) left to a zone fill")
|
|
|
|
|
|
def parse_boards(path: Path) -> list[tuple[str, Path, list[str]]]:
|
|
boards = []
|
|
for line in path.read_text().splitlines():
|
|
words = shlex.split(line, comments=True)
|
|
if not words:
|
|
continue
|
|
if len(words) < 2:
|
|
raise SystemExit(f"{path}: want 'name path [flags]', got {line!r}")
|
|
dsn = Path(words[1]).expanduser()
|
|
if not dsn.is_file():
|
|
raise SystemExit(f"{path}: no such DSN {dsn}")
|
|
boards.append((words[0], dsn, words[2:]))
|
|
return boards
|
|
|
|
|
|
def parse_variant(spec: str) -> tuple[str, str | None, list[str]]:
|
|
head, sep, flags = spec.partition("=")
|
|
if not sep or not head:
|
|
raise SystemExit(f"variant {spec!r}: want NAME=FLAGS or NAME@SRC=FLAGS")
|
|
name, _, src = head.partition("@")
|
|
if src and not (Path(src) / "freeroute").is_dir():
|
|
raise SystemExit(f"variant {name}: {src} has no freeroute package")
|
|
return name, src or None, shlex.split(flags)
|
|
|
|
|
|
def run_one(board, variant, seed: int, timeout: float) -> dict:
|
|
name, dsn, board_flags = board
|
|
vname, src, flags = variant
|
|
env = dict(os.environ)
|
|
if src:
|
|
env["PYTHONPATH"] = src + os.pathsep + env.get("PYTHONPATH", "")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
cmd = [
|
|
sys.executable, "-m", "freeroute.cli", str(dsn), "-do", f"{tmp}/out.ses",
|
|
*board_flags, *flags, "--seed", str(seed),
|
|
]
|
|
start = time.monotonic()
|
|
proc = subprocess.Popen(
|
|
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
|
|
)
|
|
expired = threading.Event()
|
|
|
|
def expire() -> None:
|
|
expired.set()
|
|
proc.kill()
|
|
|
|
timer = threading.Timer(timeout, expire)
|
|
timer.start()
|
|
err = proc.stderr.read()
|
|
# reap it here rather than through Popen, to keep its peak memory
|
|
_, wstatus, usage = os.wait4(proc.pid, 0)
|
|
timer.cancel()
|
|
proc.returncode = os.waitstatus_to_exitcode(wstatus)
|
|
proc.stderr.close()
|
|
status = "timeout" if expired.is_set() else (
|
|
"ok" if proc.returncode == 0 else f"exit {proc.returncode}"
|
|
)
|
|
rss_mb = round(usage.ru_maxrss / 1024)
|
|
secs = round(time.monotonic() - start, 1)
|
|
missing = NOT_ROUTED.search(err)
|
|
zone = TO_ZONE.search(err)
|
|
return {
|
|
"board": name, "variant": vname, "seed": seed, "status": status,
|
|
"unrouted": int(missing.group(1)) if missing else 0,
|
|
"to_zone": int(zone.group(1)) if zone else 0,
|
|
"secs": secs, "rss_mb": rss_mb,
|
|
"tail": err.strip().splitlines()[-1] if status != "ok" and err.strip() else "",
|
|
}
|
|
|
|
|
|
def mean_sd(xs: list[float]) -> tuple[float, float]:
|
|
m = sum(xs) / len(xs)
|
|
sd = math.sqrt(sum((x - m) ** 2 for x in xs) / (len(xs) - 1)) if len(xs) > 1 else 0.0
|
|
return m, sd
|
|
|
|
|
|
def report(rows: list[dict], boards, variants) -> None:
|
|
ok = {(r["board"], r["variant"], r["seed"]): r for r in rows if r["status"] == "ok"}
|
|
base = variants[0][0]
|
|
print(f"\nunrouted per board (lower is better), baseline {base!r}\n")
|
|
print(f"{'board':14} {'variant':14} {'n':>3} {'mean':>6} {'sd':>5} {'min':>4} {'max':>4}"
|
|
f" {'secs':>6} vs baseline, seed by seed")
|
|
for bname, _, _ in boards:
|
|
for vname, _, _ in variants:
|
|
runs = sorted((k[2], r) for k, r in ok.items() if k[:2] == (bname, vname))
|
|
if not runs:
|
|
print(f"{bname:14} {vname:14} no finished runs")
|
|
continue
|
|
xs = [r["unrouted"] for _, r in runs]
|
|
m, sd = mean_sd(xs)
|
|
t, _ = mean_sd([r["secs"] for _, r in runs])
|
|
line = (f"{bname:14} {vname:14} {len(xs):>3} {m:>6.1f} {sd:>5.1f} {min(xs):>4}"
|
|
f" {max(xs):>4} {t:>6.0f}")
|
|
if vname != base:
|
|
diffs = [r["unrouted"] - ok[(bname, base, s)]["unrouted"]
|
|
for s, r in runs if (bname, base, s) in ok]
|
|
if diffs:
|
|
dm, dsd = mean_sd(diffs)
|
|
se = dsd / math.sqrt(len(diffs)) if len(diffs) > 1 else float("inf")
|
|
better = sum(d < 0 for d in diffs)
|
|
worse = sum(d > 0 for d in diffs)
|
|
# a difference inside two standard errors is not a finding
|
|
verdict = ("better" if dm < 0 else "worse") if abs(dm) > 2 * se else "no clear change"
|
|
line += (f" {dm:+.1f} ± {se:.1f} ({better} better, {worse} worse,"
|
|
f" {len(diffs) - better - worse} same) {verdict}")
|
|
print(line)
|
|
failed = [r for r in rows if r["status"] != "ok"]
|
|
for r in failed:
|
|
print(f"FAILED {r['board']} {r['variant']} seed {r['seed']}: {r['status']} {r['tail']}")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("boards", type=Path, help="board list: name, DSN path, flags")
|
|
ap.add_argument("-v", "--variant", action="append", required=True,
|
|
help="NAME=FLAGS or NAME@SRC=FLAGS; the first is the baseline")
|
|
ap.add_argument("--seeds", type=int, default=6, help="orders per board and variant")
|
|
ap.add_argument("--first-seed", type=int, default=0)
|
|
ap.add_argument("--jobs", type=int, default=4, help="runs at once")
|
|
ap.add_argument("--timeout", type=float, default=1800, help="seconds per run")
|
|
ap.add_argument("--out", type=Path, required=True, help="JSON lines, appended")
|
|
ap.add_argument("--report-only", action="store_true", help="summarise --out, run nothing")
|
|
args = ap.parse_args(argv)
|
|
|
|
boards = parse_boards(args.boards)
|
|
variants = [parse_variant(v) for v in args.variant]
|
|
if len({v[0] for v in variants}) != len(variants):
|
|
raise SystemExit("variant names must differ")
|
|
seeds = range(args.first_seed, args.first_seed + args.seeds)
|
|
|
|
rows: list[dict] = []
|
|
if args.out.exists():
|
|
rows = [json.loads(line) for line in args.out.read_text().splitlines() if line.strip()]
|
|
done = {(r["board"], r["variant"], r["seed"]) for r in rows if r["status"] == "ok"}
|
|
todo = [(b, v, s) for b in boards for v in variants for s in seeds
|
|
if (b[0], v[0], s) not in done]
|
|
|
|
if not args.report_only and todo:
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
print(f"{len(todo)} runs, {args.jobs} at a time", file=sys.stderr)
|
|
# retried runs replace their earlier failure
|
|
rows = [r for r in rows if (r["board"], r["variant"], r["seed"]) in done]
|
|
with ThreadPoolExecutor(args.jobs) as pool, args.out.open("w") as out:
|
|
for r in rows:
|
|
out.write(json.dumps(r) + "\n")
|
|
futures = [pool.submit(run_one, b, v, s, args.timeout) for b, v, s in todo]
|
|
for n, fut in enumerate(as_completed(futures), 1):
|
|
r = fut.result()
|
|
rows.append(r)
|
|
out.write(json.dumps(r) + "\n")
|
|
out.flush()
|
|
print(f"[{n}/{len(todo)}] {r['board']} {r['variant']} seed {r['seed']}:"
|
|
f" {r['status']} unrouted {r['unrouted']} ({r['secs']}s,"
|
|
f" {r['rss_mb']} MB)", file=sys.stderr)
|
|
|
|
wanted = {(b[0], v[0], s) for b in boards for v in variants for s in seeds}
|
|
rows = [r for r in rows if (r["board"], r["variant"], r["seed"]) in wanted]
|
|
report(rows, boards, variants)
|
|
return 1 if any(r["status"] != "ok" for r in rows) or len(rows) < len(wanted) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|