feat(phase-2.6): Walk-Forward Validation + min-trades filter parametrico
Due fondamenta scientifiche per filtrare overfit e lucky-shot:
1) undertrading_threshold parametrico (era hardcoded 10):
- AdversarialAgent.__init__(undertrading_threshold=10)
- CLI flag --undertrading-threshold
- Aggiunto a hard_kill_findings v2 default
{"no_trades", "degenerate", "undertrading"}: ora un genome con 1 trade
fortunato (es. genome 80be6bcc-1trade-fit-0.21 di fitness-v2-combo) viene
killato anche sotto fitness v2 soft-kill.
- Test parametric: undertrading_threshold=25 → 15 trade triggerano HIGH.
2) Walk-Forward Validation (WFA):
- RunConfig.wfa_train_split (None=off, 0<x<1=on) + wfa_top_k=5
- run_phase1: split ohlcv in train/test; GA usa solo train; a fine GA
i top_k genomi (by fitness in-sample, fitness>0) vengono rivalutati
sul test_ohlcv via falsification+adversarial+compute_fitness.
- Schema migration: evaluations + fitness_oos, sharpe_oos, return_oos,
max_dd_oos, n_trades_oos (ALTER TABLE con try/except per DB pre-2.6).
- Repository.update_evaluation_oos helper per popolare colonne OOS.
- CLI flags --wfa-train-split, --wfa-top-k.
- Test integration: train_split=0.7 → fitness_oos popolato per top_k.
Motivazione: la fase 2.5 ha generato 17 run con fitness fino a 0.36 + DSR
positivo, ma OOS test su 7 anni mostra che flat-ablation top crolla -37%
mentre fitness-v2 top regge (+143%). WFA in-run permette ora di vedere
direttamente il degradation train→test senza eseguire backtest separati,
rendendo possibile filtrare overfit early durante l'ottimizzazione.
Tests (+2 → 193 totale):
- test_undertrading_threshold_parametric
- test_e2e_wfa_populates_fitness_oos
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -64,10 +64,12 @@ class AdversarialAgent:
|
||||
fees_bp: float = 5.0,
|
||||
fees_eat_alpha_threshold: float = 0.5,
|
||||
flat_too_long_threshold: float = 0.95,
|
||||
undertrading_threshold: int = 10,
|
||||
) -> None:
|
||||
self._engine = BacktestEngine(fees_bp=fees_bp)
|
||||
self._fees_eat_alpha_threshold = fees_eat_alpha_threshold
|
||||
self._flat_too_long_threshold = flat_too_long_threshold
|
||||
self._undertrading_threshold = undertrading_threshold
|
||||
|
||||
def review(self, strategy: Strategy, ohlcv: pd.DataFrame) -> AdversarialReport:
|
||||
signal_fn = compile_strategy(strategy)
|
||||
@@ -118,12 +120,15 @@ class AdversarialAgent:
|
||||
# Undertrading: < 10 trade -> HIGH (Phase 1.5: era < 5 MEDIUM).
|
||||
# Sample size troppo piccolo per distinguere edge da rumore: e'
|
||||
# un "lucky shot" non riproducibile out-of-sample.
|
||||
if n_trades < 10:
|
||||
if n_trades < self._undertrading_threshold:
|
||||
report.findings.append(
|
||||
Finding(
|
||||
name="undertrading",
|
||||
severity=Severity.HIGH,
|
||||
detail=f"only {n_trades} trades — likely lucky shot (<10 over training)",
|
||||
detail=(
|
||||
f"only {n_trades} trades — likely lucky shot "
|
||||
f"(<{self._undertrading_threshold} over training)"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -52,10 +52,15 @@ class RunConfig:
|
||||
prompt_mutation_weight: float = 0.0 # Phase 2.5: opt-in LLM mutator
|
||||
fees_eat_alpha_threshold: float = 0.5 # adversarial gate, allenta verso 0.7-0.8
|
||||
flat_too_long_threshold: float = 0.95 # adversarial gate, allenta verso 0.98-0.99
|
||||
undertrading_threshold: int = 10 # min trades, sotto = "lucky shot" HIGH
|
||||
# Fitness v2: tuple non vuota → soft-kill (solo findings listate azzerano).
|
||||
# None/empty → v1 (tutti HIGH azzerano, backward compat).
|
||||
fitness_hard_kill_findings: tuple[str, ...] | None = None
|
||||
fitness_adversarial_soft_penalty: float = 0.4
|
||||
# Walk-Forward Validation: train sui primi train_split% delle bar, OOS re-eval
|
||||
# dei top genomi sui restanti. None/0 = no WFA (eval full ohlcv).
|
||||
wfa_train_split: float | None = None
|
||||
wfa_top_k: int = 5 # quanti top genomi rivalutare OOS
|
||||
|
||||
|
||||
def run_phase1(
|
||||
@@ -78,7 +83,16 @@ def run_phase1(
|
||||
}
|
||||
run_id = repo.create_run(name=cfg.run_name, config=config_dict)
|
||||
|
||||
market = build_market_summary(ohlcv, symbol=cfg.symbol, timeframe=cfg.timeframe)
|
||||
# WFA split: se attivo, GA usa solo train_ohlcv; OOS re-eval su test_ohlcv a fine run.
|
||||
if cfg.wfa_train_split is not None and 0.0 < cfg.wfa_train_split < 1.0:
|
||||
split_idx = int(len(ohlcv) * cfg.wfa_train_split)
|
||||
train_ohlcv = ohlcv.iloc[:split_idx]
|
||||
test_ohlcv = ohlcv.iloc[split_idx:]
|
||||
else:
|
||||
train_ohlcv = ohlcv
|
||||
test_ohlcv = None
|
||||
|
||||
market = build_market_summary(train_ohlcv, symbol=cfg.symbol, timeframe=cfg.timeframe)
|
||||
|
||||
hypothesis_agent = HypothesisAgent(llm=llm)
|
||||
falsification_agent = FalsificationAgent(
|
||||
@@ -88,6 +102,7 @@ def run_phase1(
|
||||
fees_bp=cfg.fees_bp,
|
||||
fees_eat_alpha_threshold=cfg.fees_eat_alpha_threshold,
|
||||
flat_too_long_threshold=cfg.flat_too_long_threshold,
|
||||
undertrading_threshold=cfg.undertrading_threshold,
|
||||
)
|
||||
cost_tracker = CostTracker()
|
||||
|
||||
@@ -146,8 +161,8 @@ def run_phase1(
|
||||
fitnesses[genome.id] = 0.0
|
||||
continue
|
||||
|
||||
fals = falsification_agent.evaluate(proposal.strategy, ohlcv)
|
||||
adv = adversarial_agent.review(proposal.strategy, ohlcv)
|
||||
fals = falsification_agent.evaluate(proposal.strategy, train_ohlcv)
|
||||
adv = adversarial_agent.review(proposal.strategy, train_ohlcv)
|
||||
for finding in adv.findings:
|
||||
repo.save_adversarial_finding(
|
||||
run_id=run_id,
|
||||
@@ -197,6 +212,41 @@ def run_phase1(
|
||||
run_id=run_id if cfg.prompt_mutation_weight > 0 else None,
|
||||
)
|
||||
|
||||
# WFA re-eval: i top_k genomi (by fitness in-sample > 0) vengono rivalutati
|
||||
# sul test_ohlcv. Le metriche OOS finiscono in evaluations.fitness_oos etc.
|
||||
if test_ohlcv is not None and len(test_ohlcv) >= 100:
|
||||
from ..agents.hypothesis import _try_parse # noqa: PLC0415
|
||||
|
||||
all_evals = repo.list_evaluations(run_id)
|
||||
top_evals = sorted(
|
||||
(e for e in all_evals if e["fitness"] > 0 and not e.get("parse_error")),
|
||||
key=lambda x: x["fitness"],
|
||||
reverse=True,
|
||||
)[: cfg.wfa_top_k]
|
||||
for ev in top_evals:
|
||||
strategy, parse_err = _try_parse(ev["raw_text"] or "")
|
||||
if strategy is None:
|
||||
continue
|
||||
try:
|
||||
fals_oos = falsification_agent.evaluate(strategy, test_ohlcv)
|
||||
adv_oos = adversarial_agent.review(strategy, test_ohlcv)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
fit_oos = compute_fitness(
|
||||
fals_oos, adv_oos,
|
||||
hard_kill_findings=cfg.fitness_hard_kill_findings,
|
||||
adversarial_soft_penalty=cfg.fitness_adversarial_soft_penalty,
|
||||
)
|
||||
repo.update_evaluation_oos(
|
||||
run_id=run_id,
|
||||
genome_id=ev["genome_id"],
|
||||
fitness_oos=fit_oos,
|
||||
sharpe_oos=float(fals_oos.sharpe),
|
||||
return_oos=float(fals_oos.total_return),
|
||||
max_dd_oos=float(fals_oos.max_drawdown),
|
||||
n_trades_oos=int(fals_oos.n_trades),
|
||||
)
|
||||
|
||||
repo.complete_run(
|
||||
run_id, total_cost=repo.total_cost(run_id), status="completed"
|
||||
)
|
||||
|
||||
@@ -34,6 +34,18 @@ class Repository:
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
pass # colonna già presente
|
||||
# Migration WFA: colonne fitness_oos e altre OOS su evaluations.
|
||||
for col_def in (
|
||||
"fitness_oos REAL",
|
||||
"sharpe_oos REAL",
|
||||
"return_oos REAL",
|
||||
"max_dd_oos REAL",
|
||||
"n_trades_oos INTEGER",
|
||||
):
|
||||
try:
|
||||
conn.execute(f"ALTER TABLE evaluations ADD COLUMN {col_def}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _now() -> str:
|
||||
@@ -175,6 +187,29 @@ class Repository:
|
||||
),
|
||||
)
|
||||
|
||||
def update_evaluation_oos(
|
||||
self,
|
||||
run_id: str,
|
||||
genome_id: str,
|
||||
fitness_oos: float,
|
||||
sharpe_oos: float,
|
||||
return_oos: float,
|
||||
max_dd_oos: float,
|
||||
n_trades_oos: int,
|
||||
) -> None:
|
||||
"""Aggiorna le metriche OOS per un genome (WFA re-eval)."""
|
||||
with self._conn() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE evaluations SET
|
||||
fitness_oos=?, sharpe_oos=?, return_oos=?,
|
||||
max_dd_oos=?, n_trades_oos=?
|
||||
WHERE run_id=? AND genome_id=?""",
|
||||
(
|
||||
fitness_oos, sharpe_oos, return_oos,
|
||||
max_dd_oos, n_trades_oos, run_id, genome_id,
|
||||
),
|
||||
)
|
||||
|
||||
def list_evaluations(self, run_id: str) -> list[dict[str, Any]]:
|
||||
with self._conn() as conn:
|
||||
rows = conn.execute(
|
||||
|
||||
@@ -45,6 +45,11 @@ CREATE TABLE IF NOT EXISTS evaluations (
|
||||
parse_error TEXT,
|
||||
raw_text TEXT,
|
||||
eval_ts TEXT NOT NULL,
|
||||
fitness_oos REAL,
|
||||
sharpe_oos REAL,
|
||||
return_oos REAL,
|
||||
max_dd_oos REAL,
|
||||
n_trades_oos INTEGER,
|
||||
PRIMARY KEY (run_id, genome_id),
|
||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user