"""blind_eval — the single command agents and the orchestrator use to score a signal. Loads a module that defines `signal(df) -> position[]`, runs the leak-free evaluator, and prints ONE json line with PnL + maxDD (+ context). Also runs the causality guard. # agent, tuning on the visible training curves: uv run python scripts/research/blind/blind_eval.py --module --split train # orchestrator, the honest out-of-sample verdict on the held-out tail: uv run python scripts/research/blind/blind_eval.py --module --split test Series: by default both A and B are scored and a COMBINED row (equal-weight average of the two PnL/DD, plus the min) is added — "anticipate the overlaid curves", not one asset. """ from __future__ import annotations import argparse import importlib.util import json import sys from pathlib import Path import numpy as np sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/blind") import blindlib as bl # noqa: E402 def _load_signal(module_path: str): path = Path(module_path).resolve() spec = importlib.util.spec_from_file_location(path.stem, path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) if not hasattr(mod, "signal"): raise AttributeError(f"{path} has no `signal(df)` function") return mod.signal def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--module", required=True) ap.add_argument("--split", default="train", choices=["train", "test", "full"]) ap.add_argument("--series", default="both", choices=["A", "B", "both"]) ap.add_argument("--no-causality", action="store_true") args = ap.parse_args() try: signal = _load_signal(args.module) except Exception as e: print(json.dumps({"error": f"load failed: {e}"})) sys.exit(0) series = ("A", "B") if args.series == "both" else (args.series,) out = {"module": args.module, "split": args.split, "series": {}} # causality guard once (on Series A, full) — a leaky signal is invalid everywhere. if not args.no_causality: try: out["causality"] = bl.causality_ok(signal) except Exception as e: out["causality"] = {"ok": False, "reason": f"causality check raised: {e}"} pnls, dds, sharpes = [], [], [] for s in series: try: rep = bl.evaluate(signal, s, args.split) out["series"][s] = rep pnls.append(rep["pnl"]); dds.append(rep["maxdd"]); sharpes.append(rep["sharpe"]) except Exception as e: out["series"][s] = {"error": str(e)} if pnls: out["combined"] = { "pnl_mean": round(float(np.mean(pnls)), 4), "pnl_min": round(float(np.min(pnls)), 4), "maxdd_mean": round(float(np.mean(dds)), 4), "maxdd_worst": round(float(np.max(dds)), 4), "sharpe_mean": round(float(np.mean(sharpes)), 3), "sharpe_min": round(float(np.min(sharpes)), 3), } print(json.dumps(out)) if __name__ == "__main__": main()