perf(backtest): vectorize engine + parallel LLM propose + multi-fold validation tool
- backtest/engine.py: state machine numpy invece di pd.iterrows()
- 16.8x speedup su 2y (470ms -> 28ms), 11.3x su 7y (1744ms -> 154ms)
- 7 parity test parametrici vs reference iterrows assicurano equivalenza
- orchestrator/run.py + run_phase1.py: --llm-concurrency N
- ThreadPoolExecutor parallelizza hypothesis_agent.propose() per generazione
- 5-8x speedup wall time GA loop (OpenRouter qwen-2.5 regge 6-10 concorrenti)
- default 1 = backward-compat sequenziale
- scripts/validate_run.py: validation multi-fold standalone
- prende run_id + top-K + N-folds expanding-window su dataset esteso (7y)
- rivela overfitter mascherati da fitness IS alta (vedi
phase1-extended-001: elite IS Sharpe 1.93 collassa OOS a -1.00)
- ranking per robust_score = min(fitness_oos) su tutti i fold
Test 250/250 pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
@@ -0,0 +1,271 @@
|
||||
"""Multi-fold validation di un run esistente.
|
||||
|
||||
Prende un ``run_id`` da ``state/runs.db``, seleziona i top-K genomi per fitness IS,
|
||||
e li rivaluta su N fold expanding-window di un dataset OHLCV (tipicamente piu'
|
||||
lungo del train del GA). Output: per-fold + aggregati (mean / min / std) della
|
||||
fitness OOS.
|
||||
|
||||
Use case: il GA puo' selezionare un "lucky-shot" overfit a uno specifico regime.
|
||||
Validare i top-K su finestre temporali diverse rivela quali strategie sono
|
||||
robuste vs overfitter.
|
||||
|
||||
Esempio::
|
||||
|
||||
python scripts/validate_run.py \\
|
||||
--run-id e263651598894da688d95fda90a34a96 \\
|
||||
--top-k 10 --n-folds 4 \\
|
||||
--symbol BTC-PERPETUAL --timeframe 1h \\
|
||||
--start 2018-09-01 --end 2026-01-01
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd # type: ignore[import-untyped]
|
||||
|
||||
from multi_swarm_core.agents.adversarial import AdversarialAgent
|
||||
from multi_swarm_core.agents.falsification import FalsificationAgent
|
||||
from multi_swarm_core.agents.hypothesis import _try_parse
|
||||
from multi_swarm_core.cerbero.client import CerberoClient
|
||||
from multi_swarm_core.config import load_settings
|
||||
from multi_swarm_core.data.cerbero_ohlcv import CerberoOHLCVLoader, OHLCVRequest
|
||||
from multi_swarm_core.data.splits import expanding_walk_forward
|
||||
from multi_swarm_core.ga.fitness import compute_fitness
|
||||
from multi_swarm_core.persistence.repository import Repository
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Multi-fold validation di top-K genomi")
|
||||
p.add_argument("--run-id", required=True, help="run_id da validare")
|
||||
p.add_argument("--top-k", type=int, default=10, help="quanti genomi top valutare")
|
||||
p.add_argument("--n-folds", type=int, default=4, help="numero fold expanding-window")
|
||||
p.add_argument(
|
||||
"--train-ratio",
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="frazione iniziale per il train iniziale (folds testano la coda)",
|
||||
)
|
||||
p.add_argument("--symbol", default="BTC-PERPETUAL")
|
||||
p.add_argument("--timeframe", default="1h")
|
||||
p.add_argument("--exchange", default="deribit", choices=["deribit", "bybit", "hyperliquid"])
|
||||
p.add_argument("--start", default="2018-09-01T00:00:00+00:00")
|
||||
p.add_argument("--end", default="2026-01-01T00:00:00+00:00")
|
||||
p.add_argument("--fees-bp", type=float, default=5.0)
|
||||
p.add_argument("--n-trials-dsr", type=int, default=50)
|
||||
p.add_argument(
|
||||
"--fees-eat-alpha-threshold", type=float, default=0.5,
|
||||
)
|
||||
p.add_argument(
|
||||
"--flat-too-long-threshold", type=float, default=0.95,
|
||||
)
|
||||
p.add_argument(
|
||||
"--undertrading-threshold", type=int, default=10,
|
||||
)
|
||||
p.add_argument(
|
||||
"--fitness-v2", action="store_true",
|
||||
help="Coerente con --fitness-v2 del run originale",
|
||||
)
|
||||
p.add_argument(
|
||||
"--fitness-soft-penalty", type=float, default=0.4,
|
||||
)
|
||||
p.add_argument(
|
||||
"--output-json",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Path JSON dove salvare i risultati (default: stdout solo)",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
settings = load_settings()
|
||||
|
||||
# Repository: top-K genomi per fitness IS, con raw_text parsable.
|
||||
repo = Repository(settings.ga_db_path)
|
||||
repo.init_schema()
|
||||
run = repo.get_run(args.run_id)
|
||||
if run is None:
|
||||
raise SystemExit(f"run_id non trovato: {args.run_id}")
|
||||
print(f"Validating run: {run['name']} ({args.run_id})")
|
||||
print(f" status: {run['status']}, cost: ${run['total_cost_usd']:.4f}")
|
||||
|
||||
all_evals = repo.list_evaluations(args.run_id)
|
||||
parseable = [
|
||||
e for e in all_evals
|
||||
if e.get("raw_text") and not e.get("parse_error") and e["fitness"] > 0
|
||||
]
|
||||
parseable.sort(key=lambda e: e["fitness"], reverse=True)
|
||||
|
||||
# Dedup by genome_id (gli elite vengono salvati una sola volta ma possono apparire
|
||||
# in evaluations multiple se rivalutati con eval_oos_during_loop).
|
||||
seen_ids: set[str] = set()
|
||||
top_genomes: list[dict] = []
|
||||
for e in parseable:
|
||||
if e["genome_id"] in seen_ids:
|
||||
continue
|
||||
seen_ids.add(e["genome_id"])
|
||||
top_genomes.append(e)
|
||||
if len(top_genomes) >= args.top_k:
|
||||
break
|
||||
print(f" selected top-{len(top_genomes)} genomes for validation")
|
||||
|
||||
# OHLCV: carica il dataset esteso.
|
||||
token = (
|
||||
settings.cerbero_mainnet_token.get_secret_value()
|
||||
if settings.cerbero_mainnet_token
|
||||
else settings.cerbero_testnet_token.get_secret_value()
|
||||
)
|
||||
cerbero = CerberoClient(
|
||||
base_url=settings.cerbero_base_url,
|
||||
token=token,
|
||||
bot_tag=settings.cerbero_bot_tag,
|
||||
)
|
||||
loader = CerberoOHLCVLoader(client=cerbero, cache_dir=settings.series_dir)
|
||||
req = OHLCVRequest(
|
||||
symbol=args.symbol,
|
||||
timeframe=args.timeframe,
|
||||
start=datetime.fromisoformat(args.start),
|
||||
end=datetime.fromisoformat(args.end),
|
||||
exchange=args.exchange,
|
||||
)
|
||||
ohlcv = loader.load(req)
|
||||
print(f" OHLCV: {len(ohlcv)} bars from {ohlcv.index[0]} to {ohlcv.index[-1]}")
|
||||
|
||||
splits = expanding_walk_forward(
|
||||
ohlcv.index, train_ratio=args.train_ratio, n_folds=args.n_folds,
|
||||
)
|
||||
print(f" generated {len(splits)} folds")
|
||||
for s in splits:
|
||||
print(f" fold {s.fold}: test [{s.test_idx[0]} -> {s.test_idx[-1]}] ({len(s.test_idx)} bars)")
|
||||
|
||||
fals_agent = FalsificationAgent(fees_bp=args.fees_bp, n_trials_dsr=args.n_trials_dsr)
|
||||
adv_agent = AdversarialAgent(
|
||||
fees_bp=args.fees_bp,
|
||||
fees_eat_alpha_threshold=args.fees_eat_alpha_threshold,
|
||||
flat_too_long_threshold=args.flat_too_long_threshold,
|
||||
undertrading_threshold=args.undertrading_threshold,
|
||||
)
|
||||
hard_kill = (
|
||||
("no_trades", "degenerate", "undertrading") if args.fitness_v2 else None
|
||||
)
|
||||
|
||||
# Itera per ogni genome + fold.
|
||||
results: list[dict] = []
|
||||
for gi, ev in enumerate(top_genomes):
|
||||
strategy, parse_err = _try_parse(ev["raw_text"] or "")
|
||||
if strategy is None:
|
||||
print(f" [{gi}] {ev['genome_id'][:12]} skip (parse error: {parse_err})")
|
||||
continue
|
||||
|
||||
per_fold: list[dict] = []
|
||||
for s in splits:
|
||||
test_df = ohlcv.loc[s.test_idx]
|
||||
try:
|
||||
fals = fals_agent.evaluate(strategy, test_df)
|
||||
adv = adv_agent.review(strategy, test_df)
|
||||
fit = compute_fitness(
|
||||
fals, adv,
|
||||
hard_kill_findings=hard_kill,
|
||||
adversarial_soft_penalty=args.fitness_soft_penalty,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f" fold {s.fold} eval failed: {e}")
|
||||
continue
|
||||
per_fold.append({
|
||||
"fold": s.fold,
|
||||
"fitness": float(fit),
|
||||
"sharpe": float(fals.sharpe),
|
||||
"dsr": float(fals.dsr),
|
||||
"dsr_pvalue": float(fals.dsr_pvalue),
|
||||
"return": float(fals.total_return),
|
||||
"max_dd": float(fals.max_drawdown),
|
||||
"n_trades": int(fals.n_trades),
|
||||
"test_start": str(s.test_idx[0]),
|
||||
"test_end": str(s.test_idx[-1]),
|
||||
})
|
||||
|
||||
if not per_fold:
|
||||
continue
|
||||
|
||||
fits = [pf["fitness"] for pf in per_fold]
|
||||
sharps = [pf["sharpe"] for pf in per_fold]
|
||||
results.append({
|
||||
"genome_id": ev["genome_id"],
|
||||
"fitness_is": float(ev["fitness"]),
|
||||
"sharpe_is": float(ev["sharpe"]),
|
||||
"folds": per_fold,
|
||||
"fitness_oos_mean": statistics.mean(fits),
|
||||
"fitness_oos_min": min(fits),
|
||||
"fitness_oos_max": max(fits),
|
||||
"fitness_oos_std": statistics.pstdev(fits) if len(fits) > 1 else 0.0,
|
||||
"sharpe_oos_mean": statistics.mean(sharps),
|
||||
"sharpe_oos_min": min(sharps),
|
||||
"robust_score": min(fits), # min across folds = pessimismo
|
||||
})
|
||||
|
||||
# Ranking finale: per robust_score (min fitness) decrescente.
|
||||
results.sort(key=lambda r: r["robust_score"], reverse=True)
|
||||
|
||||
print()
|
||||
print(f"{'='*120}")
|
||||
print(f"VALIDATION RESULTS ({len(results)} genomes, {len(splits)} folds)")
|
||||
print(f"{'='*120}")
|
||||
print(
|
||||
f"{'rank':>4} {'genome':12} {'fit_is':>8} {'sh_is':>7} "
|
||||
f"{'fit_mean':>9} {'fit_min':>8} {'fit_max':>8} {'fit_std':>8} "
|
||||
f"{'sh_mean':>8} {'sh_min':>8} {'robust':>7}"
|
||||
)
|
||||
print("-" * 120)
|
||||
for rank, r in enumerate(results, 1):
|
||||
print(
|
||||
f"{rank:>4} {r['genome_id'][:12]:12} "
|
||||
f"{r['fitness_is']:>8.4f} {r['sharpe_is']:>7.3f} "
|
||||
f"{r['fitness_oos_mean']:>9.4f} {r['fitness_oos_min']:>8.4f} "
|
||||
f"{r['fitness_oos_max']:>8.4f} {r['fitness_oos_std']:>8.4f} "
|
||||
f"{r['sharpe_oos_mean']:>8.3f} {r['sharpe_oos_min']:>8.3f} "
|
||||
f"{r['robust_score']:>7.4f}"
|
||||
)
|
||||
|
||||
if results:
|
||||
winner = results[0]
|
||||
print()
|
||||
print(f"ROBUST WINNER: {winner['genome_id']}")
|
||||
print(f" fitness_is={winner['fitness_is']:.4f}, "
|
||||
f"fitness_oos_min={winner['fitness_oos_min']:.4f}, "
|
||||
f"fitness_oos_mean={winner['fitness_oos_mean']:.4f}")
|
||||
print(f" sharpe_is={winner['sharpe_is']:.3f}, "
|
||||
f"sharpe_oos_min={winner['sharpe_oos_min']:.3f}")
|
||||
print(f" per-fold breakdown:")
|
||||
for pf in winner["folds"]:
|
||||
print(
|
||||
f" fold {pf['fold']} [{pf['test_start'][:10]} .. {pf['test_end'][:10]}]: "
|
||||
f"fit={pf['fitness']:.4f} sharpe={pf['sharpe']:.3f} "
|
||||
f"ret={pf['return']:.3f} n_trades={pf['n_trades']}"
|
||||
)
|
||||
|
||||
if args.output_json:
|
||||
payload = {
|
||||
"run_id": args.run_id,
|
||||
"run_name": run["name"],
|
||||
"n_folds": len(splits),
|
||||
"top_k_requested": args.top_k,
|
||||
"top_k_evaluated": len(results),
|
||||
"symbol": args.symbol,
|
||||
"timeframe": args.timeframe,
|
||||
"start": args.start,
|
||||
"end": args.end,
|
||||
"ohlcv_bars": len(ohlcv),
|
||||
"results": results,
|
||||
}
|
||||
args.output_json.write_text(json.dumps(payload, indent=2, default=str))
|
||||
print(f"\nResults saved to: {args.output_json}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user