PythagorasGoal/scripts/research/blind/score_all.py

"""score_all — the ORCHESTRATOR's authoritative, single-scorer leaderboard.

After the fleet writes its modules into agents/, this script is the judge. For every
agent_*.py it:
  1. runs the CAUSALITY guard (a leaky signal is disqualified, no matter its PnL),
  2. evaluates on the HELD-OUT TEST tail (true out-of-sample) for Series A and B,
  3. evaluates on FULL for context,
and prints a leaderboard sorted by out-of-sample risk-adjusted quality, always showing
PnL and max drawdown side by side, against the buy&hold benchmark.

  uv run python scripts/research/blind/score_all.py [--split test|full]
Writes results to scripts/research/blind/leaderboard.json
"""
from __future__ import annotations

import argparse
import importlib.util
import json
import sys
import traceback
from pathlib import Path

import numpy as np

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import blindlib as bl  # noqa: E402

AGENTS = HERE / "agents"


def _load_signal(path: Path):
    spec = importlib.util.spec_from_file_location(path.stem, path)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod.signal


def _benchmark(split: str) -> dict:
    bh = lambda df: np.ones(len(df))
    out = {}
    for s in ("A", "B"):
        out[s] = bl.evaluate(bh, s, split)
    out["combined"] = {
        "pnl_mean": round(float(np.mean([out[s]["pnl"] for s in ("A", "B")])), 4),
        "maxdd_worst": round(float(np.max([out[s]["maxdd"] for s in ("A", "B")])), 4),
        "sharpe_mean": round(float(np.mean([out[s]["sharpe"] for s in ("A", "B")])), 3),
    }
    return out


def score_one(path: Path, split: str) -> dict:
    rec = {"name": path.stem, "path": str(path)}
    try:
        signal = _load_signal(path)
    except Exception as e:
        rec.update(error=f"import: {e}", causal=False)
        return rec
    try:
        caus = bl.causality_ok(signal)
        rec["causal"] = bool(caus.get("ok"))
        rec["causality"] = caus
    except Exception as e:
        rec.update(error=f"causality: {e}", causal=False)
        return rec
    per = {}
    try:
        for s in ("A", "B"):
            per[s] = bl.evaluate(signal, s, split)
        rec["A"], rec["B"] = per["A"], per["B"]
        rec["pnl_mean"] = round(float(np.mean([per[s]["pnl"] for s in ("A", "B")])), 4)
        rec["pnl_min"] = round(float(np.min([per[s]["pnl"] for s in ("A", "B")])), 4)
        rec["maxdd_worst"] = round(float(np.max([per[s]["maxdd"] for s in ("A", "B")])), 4)
        rec["maxdd_mean"] = round(float(np.mean([per[s]["maxdd"] for s in ("A", "B")])), 4)
        rec["sharpe_mean"] = round(float(np.mean([per[s]["sharpe"] for s in ("A", "B")])), 3)
        rec["sharpe_min"] = round(float(np.min([per[s]["sharpe"] for s in ("A", "B")])), 3)
        # return-per-unit-drawdown (robust to the buy&hold "huge PnL, huge DD" trap)
        dd = max(rec["maxdd_worst"], 1e-6)
        rec["calmar"] = round(rec["pnl_mean"] / dd, 3)
    except Exception as e:
        rec.update(error=f"eval: {e}\n{traceback.format_exc()[-400:]}")
    return rec


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--split", default="test", choices=["test", "full"])
    args = ap.parse_args()

    mods = sorted(p for p in AGENTS.glob("agent_*.py"))
    bench = _benchmark(args.split)
    rows = [score_one(p, args.split) for p in mods]

    valid = [r for r in rows if r.get("causal") and "sharpe_mean" in r]
    leaks = [r for r in rows if r.get("causal") is False]
    broke = [r for r in rows if "error" in r and r.get("causal") is not False]

    valid.sort(key=lambda r: r["sharpe_min"], reverse=True)

    bh = bench["combined"]
    print(f"\n{'='*100}")
    print(f"  BLIND-SIGNAL LEADERBOARD — split={args.split.upper()}  "
          f"({len(mods)} modules: {len(valid)} valid, {len(leaks)} leak-flagged, {len(broke)} broken)")
    print(f"  BENCHMARK buy&hold:  PnL {bh['pnl_mean']*100:+.0f}%   maxDD {bh['maxdd_worst']*100:.0f}%   "
          f"Sharpe {bh['sharpe_mean']:.2f}")
    print(f"{'='*100}")
    print(f"  {'#':>2} {'strategy':<34} {'PnL_A':>7} {'PnL_B':>7} {'PnLmin':>7} "
          f"{'DDworst':>7} {'Sh_min':>6} {'Calmar':>6}")
    print(f"  {'-'*92}")
    for i, r in enumerate(valid[:30], 1):
        print(f"  {i:>2} {r['name'][:34]:<34} {r['A']['pnl']*100:>+6.0f}% {r['B']['pnl']*100:>+6.0f}% "
              f"{r['pnl_min']*100:>+6.0f}% {r['maxdd_worst']*100:>6.0f}% "
              f"{r['sharpe_min']:>6.2f} {r['calmar']:>6.2f}")
    if leaks:
        print(f"\n  LEAK-FLAGGED (disqualified): {', '.join(r['name'] for r in leaks[:20])}")
    if broke:
        print(f"  BROKEN: {', '.join(r['name'] for r in broke[:20])}")

    out = {"split": args.split, "benchmark": bench, "valid": valid,
           "leaks": leaks, "broken": broke, "n_modules": len(mods)}
    (HERE / "leaderboard.json").write_text(json.dumps(out, indent=2, default=str))
    print(f"\n  -> {HERE/'leaderboard.json'}")


if __name__ == "__main__":
    main()