From 242724ba05bbf0a2094be14602b08f9ff6e11a0b Mon Sep 17 00:00:00 2001 From: AdrianoDev Date: Tue, 12 May 2026 17:31:22 +0200 Subject: [PATCH] feat(phase-2.6): Walk-Forward Validation + min-trades filter parametrico MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due fondamenta scientifiche per filtrare overfit e lucky-shot: 1) undertrading_threshold parametrico (era hardcoded 10): - AdversarialAgent.__init__(undertrading_threshold=10) - CLI flag --undertrading-threshold - Aggiunto a hard_kill_findings v2 default {"no_trades", "degenerate", "undertrading"}: ora un genome con 1 trade fortunato (es. genome 80be6bcc-1trade-fit-0.21 di fitness-v2-combo) viene killato anche sotto fitness v2 soft-kill. - Test parametric: undertrading_threshold=25 → 15 trade triggerano HIGH. 2) Walk-Forward Validation (WFA): - RunConfig.wfa_train_split (None=off, 00) vengono rivalutati sul test_ohlcv via falsification+adversarial+compute_fitness. - Schema migration: evaluations + fitness_oos, sharpe_oos, return_oos, max_dd_oos, n_trades_oos (ALTER TABLE con try/except per DB pre-2.6). - Repository.update_evaluation_oos helper per popolare colonne OOS. - CLI flags --wfa-train-split, --wfa-top-k. - Test integration: train_split=0.7 → fitness_oos popolato per top_k. Motivazione: la fase 2.5 ha generato 17 run con fitness fino a 0.36 + DSR positivo, ma OOS test su 7 anni mostra che flat-ablation top crolla -37% mentre fitness-v2 top regge (+143%). WFA in-run permette ora di vedere direttamente il degradation train→test senza eseguire backtest separati, rendendo possibile filtrare overfit early durante l'ottimizzazione. Tests (+2 → 193 totale): - test_undertrading_threshold_parametric - test_e2e_wfa_populates_fitness_oos Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/run_phase1.py | 25 +++++++++- src/multi_swarm/agents/adversarial.py | 9 +++- src/multi_swarm/orchestrator/run.py | 56 +++++++++++++++++++++-- src/multi_swarm/persistence/repository.py | 35 ++++++++++++++ src/multi_swarm/persistence/schema.py | 5 ++ tests/integration/test_e2e_minimal_run.py | 32 +++++++++++++ tests/unit/test_adversarial.py | 43 +++++++++++++++++ 7 files changed, 198 insertions(+), 7 deletions(-) diff --git a/scripts/run_phase1.py b/scripts/run_phase1.py index f454815..fa2a763 100644 --- a/scripts/run_phase1.py +++ b/scripts/run_phase1.py @@ -49,11 +49,17 @@ def parse_args() -> argparse.Namespace: default=0.95, help="Adversarial gate: kill se signal flat > soglia delle bar (default 0.95, ablation 0.98)", ) + p.add_argument( + "--undertrading-threshold", + type=int, + default=10, + help="Adversarial: kill se n_trades < soglia (default 10, bump per filtrare lucky-shot)", + ) p.add_argument( "--fitness-v2", action="store_true", help=( - "Attiva fitness v2: solo {no_trades, degenerate} azzerano; " + "Attiva fitness v2: solo {no_trades, degenerate, undertrading} azzerano; " "gli altri HIGH applicano soft penalty multiplicativa" ), ) @@ -63,6 +69,18 @@ def parse_args() -> argparse.Namespace: default=0.4, help="v2: fattore soft penalty per HIGH non-hard (default 0.4 → 1 HIGH → 0.71x)", ) + p.add_argument( + "--wfa-train-split", + type=float, + default=None, + help="Walk-forward: frazione bar usate per training (es. 0.7 = primi 70%% in-sample, ultimi 30%% OOS)", + ) + p.add_argument( + "--wfa-top-k", + type=int, + default=5, + help="Walk-forward: quanti top genomi rivalutare OOS (default 5)", + ) return p.parse_args() @@ -119,10 +137,13 @@ def main() -> None: prompt_mutation_weight=args.prompt_mutation_weight, fees_eat_alpha_threshold=args.fees_eat_alpha_threshold, flat_too_long_threshold=args.flat_too_long_threshold, + undertrading_threshold=args.undertrading_threshold, fitness_hard_kill_findings=( - ("no_trades", "degenerate") if args.fitness_v2 else None + ("no_trades", "degenerate", "undertrading") if args.fitness_v2 else None ), fitness_adversarial_soft_penalty=args.fitness_soft_penalty, + wfa_train_split=args.wfa_train_split, + wfa_top_k=args.wfa_top_k, ) run_id = run_phase1(cfg, ohlcv=ohlcv, llm=llm) diff --git a/src/multi_swarm/agents/adversarial.py b/src/multi_swarm/agents/adversarial.py index a585b99..6576a81 100644 --- a/src/multi_swarm/agents/adversarial.py +++ b/src/multi_swarm/agents/adversarial.py @@ -64,10 +64,12 @@ class AdversarialAgent: fees_bp: float = 5.0, fees_eat_alpha_threshold: float = 0.5, flat_too_long_threshold: float = 0.95, + undertrading_threshold: int = 10, ) -> None: self._engine = BacktestEngine(fees_bp=fees_bp) self._fees_eat_alpha_threshold = fees_eat_alpha_threshold self._flat_too_long_threshold = flat_too_long_threshold + self._undertrading_threshold = undertrading_threshold def review(self, strategy: Strategy, ohlcv: pd.DataFrame) -> AdversarialReport: signal_fn = compile_strategy(strategy) @@ -118,12 +120,15 @@ class AdversarialAgent: # Undertrading: < 10 trade -> HIGH (Phase 1.5: era < 5 MEDIUM). # Sample size troppo piccolo per distinguere edge da rumore: e' # un "lucky shot" non riproducibile out-of-sample. - if n_trades < 10: + if n_trades < self._undertrading_threshold: report.findings.append( Finding( name="undertrading", severity=Severity.HIGH, - detail=f"only {n_trades} trades — likely lucky shot (<10 over training)", + detail=( + f"only {n_trades} trades — likely lucky shot " + f"(<{self._undertrading_threshold} over training)" + ), ) ) diff --git a/src/multi_swarm/orchestrator/run.py b/src/multi_swarm/orchestrator/run.py index b9aed56..2d403e8 100644 --- a/src/multi_swarm/orchestrator/run.py +++ b/src/multi_swarm/orchestrator/run.py @@ -52,10 +52,15 @@ class RunConfig: prompt_mutation_weight: float = 0.0 # Phase 2.5: opt-in LLM mutator fees_eat_alpha_threshold: float = 0.5 # adversarial gate, allenta verso 0.7-0.8 flat_too_long_threshold: float = 0.95 # adversarial gate, allenta verso 0.98-0.99 + undertrading_threshold: int = 10 # min trades, sotto = "lucky shot" HIGH # Fitness v2: tuple non vuota → soft-kill (solo findings listate azzerano). # None/empty → v1 (tutti HIGH azzerano, backward compat). fitness_hard_kill_findings: tuple[str, ...] | None = None fitness_adversarial_soft_penalty: float = 0.4 + # Walk-Forward Validation: train sui primi train_split% delle bar, OOS re-eval + # dei top genomi sui restanti. None/0 = no WFA (eval full ohlcv). + wfa_train_split: float | None = None + wfa_top_k: int = 5 # quanti top genomi rivalutare OOS def run_phase1( @@ -78,7 +83,16 @@ def run_phase1( } run_id = repo.create_run(name=cfg.run_name, config=config_dict) - market = build_market_summary(ohlcv, symbol=cfg.symbol, timeframe=cfg.timeframe) + # WFA split: se attivo, GA usa solo train_ohlcv; OOS re-eval su test_ohlcv a fine run. + if cfg.wfa_train_split is not None and 0.0 < cfg.wfa_train_split < 1.0: + split_idx = int(len(ohlcv) * cfg.wfa_train_split) + train_ohlcv = ohlcv.iloc[:split_idx] + test_ohlcv = ohlcv.iloc[split_idx:] + else: + train_ohlcv = ohlcv + test_ohlcv = None + + market = build_market_summary(train_ohlcv, symbol=cfg.symbol, timeframe=cfg.timeframe) hypothesis_agent = HypothesisAgent(llm=llm) falsification_agent = FalsificationAgent( @@ -88,6 +102,7 @@ def run_phase1( fees_bp=cfg.fees_bp, fees_eat_alpha_threshold=cfg.fees_eat_alpha_threshold, flat_too_long_threshold=cfg.flat_too_long_threshold, + undertrading_threshold=cfg.undertrading_threshold, ) cost_tracker = CostTracker() @@ -146,8 +161,8 @@ def run_phase1( fitnesses[genome.id] = 0.0 continue - fals = falsification_agent.evaluate(proposal.strategy, ohlcv) - adv = adversarial_agent.review(proposal.strategy, ohlcv) + fals = falsification_agent.evaluate(proposal.strategy, train_ohlcv) + adv = adversarial_agent.review(proposal.strategy, train_ohlcv) for finding in adv.findings: repo.save_adversarial_finding( run_id=run_id, @@ -197,6 +212,41 @@ def run_phase1( run_id=run_id if cfg.prompt_mutation_weight > 0 else None, ) + # WFA re-eval: i top_k genomi (by fitness in-sample > 0) vengono rivalutati + # sul test_ohlcv. Le metriche OOS finiscono in evaluations.fitness_oos etc. + if test_ohlcv is not None and len(test_ohlcv) >= 100: + from ..agents.hypothesis import _try_parse # noqa: PLC0415 + + all_evals = repo.list_evaluations(run_id) + top_evals = sorted( + (e for e in all_evals if e["fitness"] > 0 and not e.get("parse_error")), + key=lambda x: x["fitness"], + reverse=True, + )[: cfg.wfa_top_k] + for ev in top_evals: + strategy, parse_err = _try_parse(ev["raw_text"] or "") + if strategy is None: + continue + try: + fals_oos = falsification_agent.evaluate(strategy, test_ohlcv) + adv_oos = adversarial_agent.review(strategy, test_ohlcv) + except Exception: # noqa: BLE001 + continue + fit_oos = compute_fitness( + fals_oos, adv_oos, + hard_kill_findings=cfg.fitness_hard_kill_findings, + adversarial_soft_penalty=cfg.fitness_adversarial_soft_penalty, + ) + repo.update_evaluation_oos( + run_id=run_id, + genome_id=ev["genome_id"], + fitness_oos=fit_oos, + sharpe_oos=float(fals_oos.sharpe), + return_oos=float(fals_oos.total_return), + max_dd_oos=float(fals_oos.max_drawdown), + n_trades_oos=int(fals_oos.n_trades), + ) + repo.complete_run( run_id, total_cost=repo.total_cost(run_id), status="completed" ) diff --git a/src/multi_swarm/persistence/repository.py b/src/multi_swarm/persistence/repository.py index 09f4b39..3d97657 100644 --- a/src/multi_swarm/persistence/repository.py +++ b/src/multi_swarm/persistence/repository.py @@ -34,6 +34,18 @@ class Repository: ) except sqlite3.OperationalError: pass # colonna già presente + # Migration WFA: colonne fitness_oos e altre OOS su evaluations. + for col_def in ( + "fitness_oos REAL", + "sharpe_oos REAL", + "return_oos REAL", + "max_dd_oos REAL", + "n_trades_oos INTEGER", + ): + try: + conn.execute(f"ALTER TABLE evaluations ADD COLUMN {col_def}") + except sqlite3.OperationalError: + pass @staticmethod def _now() -> str: @@ -175,6 +187,29 @@ class Repository: ), ) + def update_evaluation_oos( + self, + run_id: str, + genome_id: str, + fitness_oos: float, + sharpe_oos: float, + return_oos: float, + max_dd_oos: float, + n_trades_oos: int, + ) -> None: + """Aggiorna le metriche OOS per un genome (WFA re-eval).""" + with self._conn() as conn: + conn.execute( + """UPDATE evaluations SET + fitness_oos=?, sharpe_oos=?, return_oos=?, + max_dd_oos=?, n_trades_oos=? + WHERE run_id=? AND genome_id=?""", + ( + fitness_oos, sharpe_oos, return_oos, + max_dd_oos, n_trades_oos, run_id, genome_id, + ), + ) + def list_evaluations(self, run_id: str) -> list[dict[str, Any]]: with self._conn() as conn: rows = conn.execute( diff --git a/src/multi_swarm/persistence/schema.py b/src/multi_swarm/persistence/schema.py index c66456f..3bcec0c 100644 --- a/src/multi_swarm/persistence/schema.py +++ b/src/multi_swarm/persistence/schema.py @@ -45,6 +45,11 @@ CREATE TABLE IF NOT EXISTS evaluations ( parse_error TEXT, raw_text TEXT, eval_ts TEXT NOT NULL, + fitness_oos REAL, + sharpe_oos REAL, + return_oos REAL, + max_dd_oos REAL, + n_trades_oos INTEGER, PRIMARY KEY (run_id, genome_id), FOREIGN KEY (run_id) REFERENCES runs(id) ); diff --git a/tests/integration/test_e2e_minimal_run.py b/tests/integration/test_e2e_minimal_run.py index bca4765..32cec27 100644 --- a/tests/integration/test_e2e_minimal_run.py +++ b/tests/integration/test_e2e_minimal_run.py @@ -100,3 +100,35 @@ def test_e2e_minimal_run_completes( assert len(gens) == 2 evals = repo.list_evaluations(run_id) assert len(evals) >= 5 # almeno una popolazione + + +def test_e2e_wfa_populates_fitness_oos( + tmp_path: Path, + synthetic_ohlcv, + fake_llm, + mocker, +): + """WFA: train_split=0.7 → top genomi devono avere fitness_oos popolato.""" + cfg = RunConfig( + run_name="e2e-wfa-test", + population_size=5, + n_generations=2, + elite_k=1, + tournament_k=2, + p_crossover=0.5, + seed=42, + model_tier=ModelTier.C, + symbol="BTC/USDT", + timeframe="1h", + fees_bp=5.0, + n_trials_dsr=10, + db_path=tmp_path / "runs.db", + wfa_train_split=0.7, + wfa_top_k=3, + ) + run_id = run_phase1(cfg, ohlcv=synthetic_ohlcv, llm=fake_llm) + repo = Repository(db_path=tmp_path / "runs.db") + evals = repo.list_evaluations(run_id) + # Almeno 1 genome con fitness > 0 deve avere fitness_oos popolato. + oos_evals = [e for e in evals if e.get("fitness_oos") is not None] + assert len(oos_evals) >= 1, f"Nessun OOS popolato; evals={evals}" diff --git a/tests/unit/test_adversarial.py b/tests/unit/test_adversarial.py index 9de9997..3c0b663 100644 --- a/tests/unit/test_adversarial.py +++ b/tests/unit/test_adversarial.py @@ -194,6 +194,49 @@ def test_undertrading_under_10_is_high(monkeypatch: pytest.MonkeyPatch, ) +def test_undertrading_threshold_parametric(monkeypatch: pytest.MonkeyPatch, + ohlcv: pd.DataFrame) -> None: + """undertrading_threshold=25 → 15 trade vengono killati come HIGH.""" + fake_trades = [ + _make_trade( + ohlcv.index[i * 30], + ohlcv.index[i * 30 + 10], + entry_price=100.0, + exit_price=101.0, + ) + for i in range(15) + ] + fake_signals = pd.Series( + [Side.LONG] * 250 + [Side.FLAT] * 250, index=ohlcv.index, dtype=object + ) + + def fake_run(self, ohlcv: pd.DataFrame, signals: pd.Series) -> BacktestResult: # type: ignore[no-untyped-def] + return BacktestResult( + equity_curve=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="equity"), + returns=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="returns"), + trades=fake_trades, + ) + + def fake_compile(strategy): # type: ignore[no-untyped-def] + return lambda df: fake_signals + + monkeypatch.setattr("multi_swarm.agents.adversarial.BacktestEngine.run", fake_run) + monkeypatch.setattr("multi_swarm.agents.adversarial.compile_strategy", fake_compile) + + ast = parse_strategy(_MINIMAL_STRATEGY_SRC) + # Default threshold 10: 15 trade NON killato + agent_default = AdversarialAgent() + rep_default = agent_default.review(ast, ohlcv) + assert not any(f.name == "undertrading" for f in rep_default.findings) + # Threshold 25: 15 trade killato + agent_strict = AdversarialAgent(undertrading_threshold=25) + rep_strict = agent_strict.review(ast, ohlcv) + assert any( + f.name == "undertrading" and f.severity == Severity.HIGH + for f in rep_strict.findings + ) + + def test_overtrading_with_tighter_threshold(monkeypatch: pytest.MonkeyPatch, ohlcv: pd.DataFrame) -> None: """n_trades > n_bars/20 -> MEDIUM overtrading (Phase 1.5: era /5)."""