From 9742df3a1fb901dcd3f2d0748c666a2a3e695980 Mon Sep 17 00:00:00 2001 From: Adriano Dal Pastro Date: Sat, 16 May 2026 11:07:40 +0000 Subject: [PATCH] =?UTF-8?q?fix(fitness):=20hardening=20anti-overfit=20post?= =?UTF-8?q?-7y=20incident=20=E2=80=94=203=20correzioni?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident: extended run elite (Sharpe IS 1.93) net-negativo su 7y continuous (fees=101% del gross). Multi-fold validation NON sufficiente: ogni fold restarta equity, mascherando accumulo fees compound. Correzioni: 1) Default --start esteso a 2018-09-01 (7.3 anni) - Copre bear 2018-19, halving 2020, COVID, ATH 2021, winter 2022, ETF rally 2024, regime corrente. - Una finestra corta (2y) lasciava il GA libero di overfit single-regime. 2) fees_eat_alpha promosso a hard-kill in fitness v2 - Da soft-penalty 0.4x a hard-kill 0 fitness. - Una strategia con fees > 50% del gross non e' recuperabile via selection: il prodotto del GA non puo' deployare con quel cost burden. 3) Nuovo finding negative_net_pnl (HIGH, hard-kill) - Fires quando sum(trade.net_pnl) < 0 sul training window. - Cattura: gross negativo (no edge direzionale) E gross positivo ma fees > gross (edge insufficiente). - Sintesi del net-after-fees su finestra continua come "deal-breaker" non negoziabile via soft penalty. CLI: - --fitness-hard-kill per override esplicito. - Default v2: no_trades,degenerate,undertrading,fees_eat_alpha,negative_net_pnl. Test: - 252 pass (251 + 1 nuovo: test_negative_net_pnl_fires_on_negative_gross). - Test e2e WFA aggiornato: passa fitness_hard_kill_findings=('no_trades',) perche' il fixture sintetico non produce strategie profittevoli. - test_no_findings_on_reasonable_strategy rinominato: test_rsi_mean_reversion_loses_money_on_synthetic_data (riflette semantica reale: RSI mean-rev su synthetic ohlcv ha net negativo). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 9 +- scripts/run_phase1.py | 49 +++++++- .../multi_swarm_core/agents/adversarial.py | 23 +++- .../tests/integration/test_e2e_minimal_run.py | 9 +- .../tests/unit/test_adversarial.py | 108 +++++++++++++++++- 5 files changed, 182 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e39b8d7..b64c877 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Stack: Python 3.13, uv workspace, hatchling, pytest+pytest-mock+responses, opena ```bash uv sync # installa entrambi i workspace member come editable cp .env.example .env # compila CERBERO_*_TOKEN e OPENROUTER_API_KEY -uv run pytest # 250 test attesi (246 core + 4 smoke strategy_crypto) +uv run pytest # 252 test attesi (248 core + 4 smoke strategy_crypto) ``` ### Variabili .env richieste @@ -151,15 +151,16 @@ uv run mypy src/ scripts/ uv run python scripts/smoke_run.py # Run reale Phase 1/2 (Cerbero + OpenRouter, ~$0.15-0.25 per run K=20 10gen, -# ~$0.40-0.55 per run esteso K=40 20gen con WFA OOS) +# ~$0.40-0.55 per run esteso K=40 20gen con WFA OOS). +# Default --start ora 2018-09-01 (7.3y, copre bear+halving+covid+ATH+winter+ETF). uv run python scripts/run_phase1.py \ --name run-XXX \ --exchange deribit --symbol BTC-PERPETUAL --timeframe 1h \ - --start 2024-01-01T00:00:00+00:00 \ - --end 2026-01-01T00:00:00+00:00 \ --population-size 20 --n-generations 10 \ --prompt-mutation-weight 0.30 --fitness-v2 \ --llm-concurrency 8 # 5-8x speedup wall time (default 1) +# fitness-v2 hardened: hard-kill su {no_trades, degenerate, undertrading, +# fees_eat_alpha, negative_net_pnl}. Override via --fitness-hard-kill CSV. # Default --prompt-library: importlib.resources del package strategy_crypto/prompts.json # Multi-fold validation di un run esistente (anti-overfit, 7y expanding-window) diff --git a/scripts/run_phase1.py b/scripts/run_phase1.py index 0f1d299..7a760d7 100644 --- a/scripts/run_phase1.py +++ b/scripts/run_phase1.py @@ -19,6 +19,30 @@ def _default_prompt_library_path() -> Path: return Path(str(importlib.resources.files("strategy_crypto") / "prompts.json")) +# Default v2 hard-kill list: oltre ai degenerate originali, fees_eat_alpha e +# negative_net_pnl sono deal-breaker non recuperabili via soft penalty (vedi +# 7y-overfit incident 2026-05-16: elite IS Sharpe 1.93 -> net -5% su 7y per fees). +_DEFAULT_V2_HARD_KILL: tuple[str, ...] = ( + "no_trades", + "degenerate", + "undertrading", + "fees_eat_alpha", + "negative_net_pnl", +) + + +def _resolve_hard_kill(args) -> tuple[str, ...] | None: + """Resolve la lista hard-kill da CLI args. + + Priority: ``--fitness-hard-kill`` esplicito > default v2 > ``None`` (v1). + """ + if args.fitness_hard_kill: + return tuple(s.strip() for s in args.fitness_hard_kill.split(",") if s.strip()) + if args.fitness_v2: + return _DEFAULT_V2_HARD_KILL + return None + + def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Multi-Swarm Phase 1 runner") p.add_argument("--name", default="phase1-spike-001") @@ -35,7 +59,10 @@ def parse_args() -> argparse.Namespace: ) p.add_argument("--symbol", default="BTC-PERPETUAL") p.add_argument("--timeframe", default="1h") - p.add_argument("--start", default="2024-01-01T00:00:00+00:00") + # Default esteso a 7.3 anni: copre bear 2018-19, halving 2020, COVID, + # ATH 2021, winter 2022, ETF rally 2024, regime corrente. Una finestra + # corta lascia il GA libero di overfit a un singolo regime. + p.add_argument("--start", default="2018-09-01T00:00:00+00:00") p.add_argument("--end", default="2026-01-01T00:00:00+00:00") p.add_argument("--fees-bp", type=float, default=5.0) p.add_argument("--n-trials-dsr", type=int, default=50) @@ -67,8 +94,10 @@ def parse_args() -> argparse.Namespace: "--fitness-v2", action="store_true", help=( - "Attiva fitness v2: solo {no_trades, degenerate, undertrading} azzerano; " - "gli altri HIGH applicano soft penalty multiplicativa" + "Attiva fitness v2: hard-kill su {no_trades, degenerate, undertrading, " + "fees_eat_alpha, negative_net_pnl}; gli altri HIGH applicano soft penalty " + "multiplicativa. Versione hardened post 7y-overfit incident: fees + " + "net negativo non sono recuperabili." ), ) p.add_argument( @@ -77,6 +106,16 @@ def parse_args() -> argparse.Namespace: default=0.4, help="v2: fattore soft penalty per HIGH non-hard (default 0.4 → 1 HIGH → 0.71x)", ) + p.add_argument( + "--fitness-hard-kill", + type=str, + default=None, + help=( + "Override comma-separated della lista di finding name che azzerano la " + "fitness in modalita' v2. Es: 'no_trades,degenerate'. Default v2: " + "no_trades,degenerate,undertrading,fees_eat_alpha,negative_net_pnl." + ), + ) p.add_argument( "--wfa-train-split", type=float, @@ -188,9 +227,7 @@ def main() -> None: fees_eat_alpha_threshold=args.fees_eat_alpha_threshold, flat_too_long_threshold=args.flat_too_long_threshold, undertrading_threshold=args.undertrading_threshold, - fitness_hard_kill_findings=( - ("no_trades", "degenerate", "undertrading") if args.fitness_v2 else None - ), + fitness_hard_kill_findings=_resolve_hard_kill(args), fitness_adversarial_soft_penalty=args.fitness_soft_penalty, wfa_train_split=args.wfa_train_split, wfa_top_k=args.wfa_top_k, diff --git a/src/multi_swarm_core/multi_swarm_core/agents/adversarial.py b/src/multi_swarm_core/multi_swarm_core/agents/adversarial.py index 6576a81..7cfaf9c 100644 --- a/src/multi_swarm_core/multi_swarm_core/agents/adversarial.py +++ b/src/multi_swarm_core/multi_swarm_core/agents/adversarial.py @@ -172,10 +172,11 @@ class AdversarialAgent: ) ) - # Fees-eat-alpha: gross_pnl > 0 ma fees > 50% del lordo. + # Fees-eat-alpha: gross_pnl > 0 ma fees > soglia del lordo. # La strategia ha edge teorico ma il margine viene mangiato dai # costi di transazione: non sostenibile in produzione. - # Se gross_pnl <= 0 il check non si applica (gia' perdente). + # Se gross_pnl <= 0 il check non si applica (la condizione e' coperta + # da ``negative_net_pnl`` sotto). gross_pnl = sum(t.gross_pnl for t in result.trades) total_fees = sum(t.fees for t in result.trades) if gross_pnl > 0 and total_fees / gross_pnl > self._fees_eat_alpha_threshold: @@ -190,4 +191,22 @@ class AdversarialAgent: ) ) + # Negative-net-pnl: somma di ``trade.net_pnl`` < 0 sul training. + # Cattura sia il caso "gross negativo" (no edge direzionale) sia il + # caso "gross positivo ma fees superiori a gross" (edge insufficiente). + # Sintesi del net-after-fees su finestra continua: deal-breaker, non + # negoziabile via soft penalty. + net_pnl = gross_pnl - total_fees + if net_pnl < 0: + report.findings.append( + Finding( + name="negative_net_pnl", + severity=Severity.HIGH, + detail=( + f"Net PnL ${net_pnl:.2f} < 0 after fees over {n_bars} bars; " + f"gross ${gross_pnl:.2f}, fees ${total_fees:.2f}" + ), + ) + ) + return report diff --git a/src/multi_swarm_core/tests/integration/test_e2e_minimal_run.py b/src/multi_swarm_core/tests/integration/test_e2e_minimal_run.py index b9b7067..5498a61 100644 --- a/src/multi_swarm_core/tests/integration/test_e2e_minimal_run.py +++ b/src/multi_swarm_core/tests/integration/test_e2e_minimal_run.py @@ -108,7 +108,13 @@ def test_e2e_wfa_populates_fitness_oos( fake_llm, mocker, ): - """WFA: train_split=0.7 → top genomi devono avere fitness_oos popolato.""" + """WFA: train_split=0.7 → top genomi devono avere fitness_oos popolato. + + Usa fitness v2 con hard-kill minimale (solo no_trades): il fixture sintetico + non produce strategie profittevoli, quindi i check aggressivi + fees_eat_alpha/negative_net_pnl azzererebbero tutti i genomi rendendo + inverificabile il wiring WFA. + """ cfg = RunConfig( run_name="e2e-wfa-test", population_size=5, @@ -125,6 +131,7 @@ def test_e2e_wfa_populates_fitness_oos( db_path=tmp_path / "runs.db", wfa_train_split=0.7, wfa_top_k=3, + fitness_hard_kill_findings=("no_trades",), ) run_id = run_phase1(cfg, ohlcv=synthetic_ohlcv, llm=fake_llm) repo = Repository(db_path=tmp_path / "runs.db") diff --git a/src/multi_swarm_core/tests/unit/test_adversarial.py b/src/multi_swarm_core/tests/unit/test_adversarial.py index f9b381b..ee6c0f0 100644 --- a/src/multi_swarm_core/tests/unit/test_adversarial.py +++ b/src/multi_swarm_core/tests/unit/test_adversarial.py @@ -3,7 +3,6 @@ import json import numpy as np import pandas as pd import pytest - from multi_swarm_core.agents.adversarial import ( AdversarialAgent, AdversarialReport, @@ -54,7 +53,10 @@ def test_degenerate_always_long_flagged(ohlcv: pd.DataFrame) -> None: assert any(f.name == "degenerate" and f.severity == Severity.HIGH for f in report.findings) -def test_no_findings_on_reasonable_strategy(ohlcv: pd.DataFrame) -> None: +def test_rsi_mean_reversion_loses_money_on_synthetic_data(ohlcv: pd.DataFrame) -> None: + """RSI mean-reversion sul fixture sintetico ha net negativo: deve firare + negative_net_pnl (deal-breaker). Conferma che il check cattura strategie + che perdono sul training, indipendentemente dal motivo (no edge / fees).""" src = json.dumps( { "rules": [ @@ -84,8 +86,59 @@ def test_no_findings_on_reasonable_strategy(ohlcv: pd.DataFrame) -> None: ast = parse_strategy(src) agent = AdversarialAgent() report = agent.review(ast, ohlcv) + assert any( + f.name == "negative_net_pnl" and f.severity == Severity.HIGH + for f in report.findings + ) + + +def test_profitable_strategy_no_high_findings( + monkeypatch: pytest.MonkeyPatch, ohlcv: pd.DataFrame +) -> None: + """Sanity test: una strategia con gross > 0 e fees << gross + n_trades ragionevole + + signal misto non deve produrre nessun finding HIGH.""" + n = 15 + # entry=100 exit=110 gross=10 per trade, fees a 5bp -> 0.105 per trade + # totali: gross=150, fees=1.575 -> net=+148.4 + fake_trades = [ + _make_trade( + ohlcv.index[i * 30], + ohlcv.index[i * 30 + 1], + entry_price=100.0, + exit_price=110.0, + ) + for i in range(n) + ] + # 50/50 LONG/FLAT per evitare degenerate/flat_too_long/time_in_market. + fake_signals = pd.Series( + [Side.LONG if i % 2 == 0 else Side.FLAT for i in range(len(ohlcv))], + index=ohlcv.index, + dtype=object, + ) + + def fake_run(self, ohlcv: pd.DataFrame, signals: pd.Series) -> BacktestResult: # type: ignore[no-untyped-def] + return BacktestResult( + equity_curve=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="equity"), + returns=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="returns"), + trades=fake_trades, + ) + + def fake_compile(strategy): # type: ignore[no-untyped-def] + return lambda df: fake_signals + + monkeypatch.setattr( + "multi_swarm_core.agents.adversarial.BacktestEngine.run", fake_run + ) + monkeypatch.setattr( + "multi_swarm_core.agents.adversarial.compile_strategy", fake_compile + ) + + ast = parse_strategy(_MINIMAL_STRATEGY_SRC) + report = AdversarialAgent().review(ast, ohlcv) high_findings = [f for f in report.findings if f.severity == Severity.HIGH] - assert len(high_findings) == 0 + assert high_findings == [], ( + f"expected no HIGH findings, got: {[f.name for f in high_findings]}" + ) def test_zero_trade_strategy_flagged(ohlcv: pd.DataFrame) -> None: @@ -383,6 +436,55 @@ def test_fees_eat_alpha_flagged(monkeypatch: pytest.MonkeyPatch, ) +def test_negative_net_pnl_fires_on_negative_gross( + monkeypatch: pytest.MonkeyPatch, ohlcv: pd.DataFrame +) -> None: + """gross_pnl < 0 (perdente direzionale) -> HIGH negative_net_pnl. + fees_eat_alpha NON deve firare perche' la sua condizione richiede gross > 0. + """ + n = 15 + # entry=100 exit=95 gross=-5 per trade (LONG perdente) + fake_trades = [ + _make_trade( + ohlcv.index[i * 30], + ohlcv.index[i * 30 + 1], + entry_price=100.0, + exit_price=95.0, + ) + for i in range(n) + ] + fake_signals = pd.Series( + [Side.LONG if i % 2 == 0 else Side.FLAT for i in range(len(ohlcv))], + index=ohlcv.index, + dtype=object, + ) + + def fake_run(self, ohlcv, signals): # type: ignore[no-untyped-def] + return BacktestResult( + equity_curve=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="equity"), + returns=pd.Series([0.0] * len(ohlcv), index=ohlcv.index, name="returns"), + trades=fake_trades, + ) + + def fake_compile(strategy): # type: ignore[no-untyped-def] + return lambda df: fake_signals + + monkeypatch.setattr( + "multi_swarm_core.agents.adversarial.BacktestEngine.run", fake_run + ) + monkeypatch.setattr( + "multi_swarm_core.agents.adversarial.compile_strategy", fake_compile + ) + + ast = parse_strategy(_MINIMAL_STRATEGY_SRC) + report = AdversarialAgent().review(ast, ohlcv) + assert any( + f.name == "negative_net_pnl" and f.severity == Severity.HIGH + for f in report.findings + ) + assert not any(f.name == "fees_eat_alpha" for f in report.findings) + + def test_time_in_market_too_high_flagged(monkeypatch: pytest.MonkeyPatch, ohlcv: pd.DataFrame) -> None: """Signal LONG per >80% delle bar -> HIGH time_in_market_too_high."""