research(blind): 52 agenti ciechi su curve anonime BTC/ETH — orchestratore valuta PnL/maxDD, niente di nuovo regge
Flotta di 52 subagenti "esperti di segnali" su storico BTC/ETH ANONIMIZZATO (Series A/B rebased a 100, calendario sintetico, split 70/30) — non sanno cosa siano. Ognuno scrive un signal(df)->position causale (script o ML), tunato solo sul train. Orchestratore valuta su PnL e maxDD nel test held-out. Harness cieco leak-free (riusabile): - make_blind.py: export anonimo + overlay; blindlib.py: evaluator con shift della posizione + GUARDIA DI CAUSALITA' online (squalifica ogni look-ahead, ML incluso); blind_eval.py CLI; score_all.py giudice OOS; verify_top.py (corr-al-trend, fee-stress, jackknife). - 52/52 passano la guardia (zero leak su tutta la flotta). Esito OOS (benchmark buy&hold: -7% PnL, 68% DD): - top = macd (+21%, DD 11%, Sh 0.84), accel, vol_of_vol, regime_switch, rf, obv — tutti trend/vol-regime. Sharpe OOS ~0.84 decade dal train ~1.4. Mean-rev e ML in fondo. - 3 scettici indipendenti: REFUTED. regime-luck (top-5 bar = 67-102% del PnL); trend-redundancy (HAC alpha t=+0.9..+1.5, nessuno >1.96 — TSMOM travestito); overfit (accel/vov knife-edge). Verdetto: ri-conferma CIECA e indipendente del soffitto direzionale ~1.3. macd = classe-TP01, forward-monitor non deploy. Diario 2026-06-21-blind-signal-fleet.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
"""blind_eval — the single command agents and the orchestrator use to score a signal.
|
||||
|
||||
Loads a module that defines `signal(df) -> position[]`, runs the leak-free evaluator,
|
||||
and prints ONE json line with PnL + maxDD (+ context). Also runs the causality guard.
|
||||
|
||||
# agent, tuning on the visible training curves:
|
||||
uv run python scripts/research/blind/blind_eval.py --module <path.py> --split train
|
||||
|
||||
# orchestrator, the honest out-of-sample verdict on the held-out tail:
|
||||
uv run python scripts/research/blind/blind_eval.py --module <path.py> --split test
|
||||
|
||||
Series: by default both A and B are scored and a COMBINED row (equal-weight average of
|
||||
the two PnL/DD, plus the min) is added — "anticipate the overlaid curves", not one asset.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/blind")
|
||||
import blindlib as bl # noqa: E402
|
||||
|
||||
|
||||
def _load_signal(module_path: str):
|
||||
path = Path(module_path).resolve()
|
||||
spec = importlib.util.spec_from_file_location(path.stem, path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
if not hasattr(mod, "signal"):
|
||||
raise AttributeError(f"{path} has no `signal(df)` function")
|
||||
return mod.signal
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--module", required=True)
|
||||
ap.add_argument("--split", default="train", choices=["train", "test", "full"])
|
||||
ap.add_argument("--series", default="both", choices=["A", "B", "both"])
|
||||
ap.add_argument("--no-causality", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
try:
|
||||
signal = _load_signal(args.module)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"load failed: {e}"}))
|
||||
sys.exit(0)
|
||||
|
||||
series = ("A", "B") if args.series == "both" else (args.series,)
|
||||
out = {"module": args.module, "split": args.split, "series": {}}
|
||||
|
||||
# causality guard once (on Series A, full) — a leaky signal is invalid everywhere.
|
||||
if not args.no_causality:
|
||||
try:
|
||||
out["causality"] = bl.causality_ok(signal)
|
||||
except Exception as e:
|
||||
out["causality"] = {"ok": False, "reason": f"causality check raised: {e}"}
|
||||
|
||||
pnls, dds, sharpes = [], [], []
|
||||
for s in series:
|
||||
try:
|
||||
rep = bl.evaluate(signal, s, args.split)
|
||||
out["series"][s] = rep
|
||||
pnls.append(rep["pnl"]); dds.append(rep["maxdd"]); sharpes.append(rep["sharpe"])
|
||||
except Exception as e:
|
||||
out["series"][s] = {"error": str(e)}
|
||||
|
||||
if pnls:
|
||||
out["combined"] = {
|
||||
"pnl_mean": round(float(np.mean(pnls)), 4),
|
||||
"pnl_min": round(float(np.min(pnls)), 4),
|
||||
"maxdd_mean": round(float(np.mean(dds)), 4),
|
||||
"maxdd_worst": round(float(np.max(dds)), 4),
|
||||
"sharpe_mean": round(float(np.mean(sharpes)), 3),
|
||||
"sharpe_min": round(float(np.min(sharpes)), 3),
|
||||
}
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user