Adriano
242724ba05
Due fondamenta scientifiche per filtrare overfit e lucky-shot:
1) undertrading_threshold parametrico (era hardcoded 10):
- AdversarialAgent.__init__(undertrading_threshold=10)
- CLI flag --undertrading-threshold
- Aggiunto a hard_kill_findings v2 default
{"no_trades", "degenerate", "undertrading"}: ora un genome con 1 trade
fortunato (es. genome 80be6bcc-1trade-fit-0.21 di fitness-v2-combo) viene
killato anche sotto fitness v2 soft-kill.
- Test parametric: undertrading_threshold=25 → 15 trade triggerano HIGH.
2) Walk-Forward Validation (WFA):
- RunConfig.wfa_train_split (None=off, 0<x<1=on) + wfa_top_k=5
- run_phase1: split ohlcv in train/test; GA usa solo train; a fine GA
i top_k genomi (by fitness in-sample, fitness>0) vengono rivalutati
sul test_ohlcv via falsification+adversarial+compute_fitness.
- Schema migration: evaluations + fitness_oos, sharpe_oos, return_oos,
max_dd_oos, n_trades_oos (ALTER TABLE con try/except per DB pre-2.6).
- Repository.update_evaluation_oos helper per popolare colonne OOS.
- CLI flags --wfa-train-split, --wfa-top-k.
- Test integration: train_split=0.7 → fitness_oos popolato per top_k.
Motivazione: la fase 2.5 ha generato 17 run con fitness fino a 0.36 + DSR
positivo, ma OOS test su 7 anni mostra che flat-ablation top crolla -37%
mentre fitness-v2 top regge (+143%). WFA in-run permette ora di vedere
direttamente il degradation train→test senza eseguire backtest separati,
rendendo possibile filtrare overfit early durante l'ottimizzazione.
Tests (+2 → 193 totale):
- test_undertrading_threshold_parametric
- test_e2e_wfa_populates_fitness_oos
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
135 lines
3.6 KiB
Python
135 lines
3.6 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from multi_swarm.genome.hypothesis import ModelTier
|
|
from multi_swarm.llm.client import CompletionResult
|
|
from multi_swarm.orchestrator.run import RunConfig, run_phase1
|
|
from multi_swarm.persistence.repository import Repository
|
|
|
|
|
|
@pytest.fixture
|
|
def synthetic_ohlcv():
|
|
idx = pd.date_range("2024-01-01", periods=500, freq="1h", tz="UTC")
|
|
close = 100 + np.cumsum(np.random.RandomState(0).normal(0.01, 1.0, 500))
|
|
return pd.DataFrame(
|
|
{
|
|
"open": close,
|
|
"high": close + 0.5,
|
|
"low": close - 0.5,
|
|
"close": close,
|
|
"volume": 1.0,
|
|
},
|
|
index=idx,
|
|
)
|
|
|
|
|
|
_STRATEGY_PAYLOAD = json.dumps(
|
|
{
|
|
"rules": [
|
|
{
|
|
"condition": {
|
|
"op": "gt",
|
|
"args": [
|
|
{"kind": "indicator", "name": "rsi", "params": [14]},
|
|
{"kind": "literal", "value": 70.0},
|
|
],
|
|
},
|
|
"action": "entry-short",
|
|
},
|
|
{
|
|
"condition": {
|
|
"op": "lt",
|
|
"args": [
|
|
{"kind": "indicator", "name": "rsi", "params": [14]},
|
|
{"kind": "literal", "value": 30.0},
|
|
],
|
|
},
|
|
"action": "entry-long",
|
|
},
|
|
]
|
|
}
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_llm(mocker):
|
|
"""LLM mock che ritorna sempre una strategia JSON valida."""
|
|
fake = mocker.MagicMock()
|
|
fake.complete.return_value = CompletionResult(
|
|
text="```json\n" + _STRATEGY_PAYLOAD + "\n```",
|
|
input_tokens=200,
|
|
output_tokens=80,
|
|
tier=ModelTier.C,
|
|
model="qwen",
|
|
)
|
|
return fake
|
|
|
|
|
|
def test_e2e_minimal_run_completes(
|
|
tmp_path: Path,
|
|
synthetic_ohlcv,
|
|
fake_llm,
|
|
mocker,
|
|
):
|
|
cfg = RunConfig(
|
|
run_name="e2e-test",
|
|
population_size=5,
|
|
n_generations=2,
|
|
elite_k=1,
|
|
tournament_k=2,
|
|
p_crossover=0.5,
|
|
seed=42,
|
|
model_tier=ModelTier.C,
|
|
symbol="BTC/USDT",
|
|
timeframe="1h",
|
|
fees_bp=5.0,
|
|
n_trials_dsr=10,
|
|
db_path=tmp_path / "runs.db",
|
|
)
|
|
|
|
run_id = run_phase1(cfg, ohlcv=synthetic_ohlcv, llm=fake_llm)
|
|
|
|
repo = Repository(db_path=tmp_path / "runs.db")
|
|
run = repo.get_run(run_id)
|
|
assert run["status"] == "completed"
|
|
gens = repo.list_generations(run_id)
|
|
assert len(gens) == 2
|
|
evals = repo.list_evaluations(run_id)
|
|
assert len(evals) >= 5 # almeno una popolazione
|
|
|
|
|
|
def test_e2e_wfa_populates_fitness_oos(
|
|
tmp_path: Path,
|
|
synthetic_ohlcv,
|
|
fake_llm,
|
|
mocker,
|
|
):
|
|
"""WFA: train_split=0.7 → top genomi devono avere fitness_oos popolato."""
|
|
cfg = RunConfig(
|
|
run_name="e2e-wfa-test",
|
|
population_size=5,
|
|
n_generations=2,
|
|
elite_k=1,
|
|
tournament_k=2,
|
|
p_crossover=0.5,
|
|
seed=42,
|
|
model_tier=ModelTier.C,
|
|
symbol="BTC/USDT",
|
|
timeframe="1h",
|
|
fees_bp=5.0,
|
|
n_trials_dsr=10,
|
|
db_path=tmp_path / "runs.db",
|
|
wfa_train_split=0.7,
|
|
wfa_top_k=3,
|
|
)
|
|
run_id = run_phase1(cfg, ohlcv=synthetic_ohlcv, llm=fake_llm)
|
|
repo = Repository(db_path=tmp_path / "runs.db")
|
|
evals = repo.list_evaluations(run_id)
|
|
# Almeno 1 genome con fitness > 0 deve avere fitness_oos popolato.
|
|
oos_evals = [e for e in evals if e.get("fitness_oos") is not None]
|
|
assert len(oos_evals) >= 1, f"Nessun OOS popolato; evals={evals}"
|