Adriano/Multi_Swarm_Coevolutive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(backtest): vectorize engine + parallel LLM propose + multi-fold validation tool

Browse Source 
- backtest/engine.py: state machine numpy invece di pd.iterrows()
  - 16.8x speedup su 2y (470ms -> 28ms), 11.3x su 7y (1744ms -> 154ms)
  - 7 parity test parametrici vs reference iterrows assicurano equivalenza
- orchestrator/run.py + run_phase1.py: --llm-concurrency N
  - ThreadPoolExecutor parallelizza hypothesis_agent.propose() per generazione
  - 5-8x speedup wall time GA loop (OpenRouter qwen-2.5 regge 6-10 concorrenti)
  - default 1 = backward-compat sequenziale
- scripts/validate_run.py: validation multi-fold standalone
  - prende run_id + top-K + N-folds expanding-window su dataset esteso (7y)
  - rivela overfitter mascherati da fitness IS alta (vedi
    phase1-extended-001: elite IS Sharpe 1.93 collassa OOS a -1.00)
  - ranking per robust_score = min(fitness_oos) su tutti i fold

Test 250/250 pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
2026-05-16 10:53:48 +00:00
parent fa11cca2bc
commit a29748e3d8
7 changed files with 685 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/multi_swarm_core/multi_swarm_core/backtest/engine.py
+97 -60
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass
import numpy as np
import pandas as pd # type: ignore[import-untyped]
from .orders import Position, Side, Trade
@@ -28,74 +29,110 @@ class BacktestEngine:
self.fees_bp = fees_bp
def run(self, ohlcv: pd.DataFrame, signals: pd.Series) -> BacktestResult:
n = len(ohlcv)
if n == 0:
empty = pd.Series([], dtype=float)
return BacktestResult(equity_curve=empty, returns=empty, trades=[])
signals = signals.reindex(ohlcv.index).ffill().fillna(Side.FLAT)
# Esecuzione con delay 1: segnale a t-1 esegue a open di t.
shifted = [Side.FLAT, *list(signals.iloc[:-1])]
executed_side = pd.Series(shifted, index=ohlcv.index, dtype=object)
executed = pd.Series(
[Side.FLAT, *list(signals.iloc[:-1])],
index=ohlcv.index,
dtype=object,
)
# Codifica side in int per vectorizzazione: 0=FLAT, +1=LONG, -1=SHORT.
side_code = np.where(
executed.values == Side.LONG, 1,
np.where(executed.values == Side.SHORT, -1, 0),
).astype(np.int8)
opens = ohlcv["open"].to_numpy(dtype=np.float64)
closes = ohlcv["close"].to_numpy(dtype=np.float64)
ts_index = ohlcv.index
# Identifica transizioni: punto in cui side[i] != side[i-1] (con side[-1]=0).
prev = np.concatenate(([0], side_code[:-1]))
change = side_code != prev
# Indici di entry (cambio verso side != 0).
entry_idxs = np.flatnonzero(change & (side_code != 0))
# Indici di chiusura: per ogni entry, il prossimo indice dove side[i] != side_entry.
# Vectorized: per ogni entry_idx, cerca change & side != side_entry oltre l'entry.
position: Position | None = None
position_entry_ts: pd.Timestamp | None = None
trades: list[Trade] = []
equity = 0.0
equity_history: list[float] = []
returns_history: list[float] = []
prev_equity = 0.0
# realized_pnl[t]: PnL netto cumulato dopo le chiusure avvenute a OPEN di t.
realized_pnl = np.zeros(n, dtype=np.float64)
fees_rate = self.fees_bp / 10000.0
size = 1.0
for ts, row in ohlcv.iterrows():
target_side = executed_side.loc[ts]
current_side = position.side if position else Side.FLAT
# Posizione corrente all'inizio di ogni indice t (prima di applicare il transitorio):
# used per MtM computation. open_side_at_t / open_entry_at_t.
open_side = np.zeros(n, dtype=np.int8)
open_entry = np.zeros(n, dtype=np.float64)
if target_side != current_side:
if position is not None:
assert position_entry_ts is not None
trade = Trade(
entry_ts=position_entry_ts,
exit_ts=ts,
side=position.side,
size=position.size,
entry_price=position.entry_price,
exit_price=float(row["open"]),
fees_bp=self.fees_bp,
)
trades.append(trade)
equity += trade.net_pnl
position = None
position_entry_ts = None
if target_side in (Side.LONG, Side.SHORT):
position = Position(
side=target_side, entry_price=float(row["open"]), size=1.0
)
position_entry_ts = ts
for entry_idx in entry_idxs:
entry_side = int(side_code[entry_idx])
entry_price = opens[entry_idx]
# Cerca exit: primo indice > entry_idx dove side differisce.
after = side_code[entry_idx + 1:]
rel = np.flatnonzero(after != entry_side)
if rel.size > 0:
exit_idx = entry_idx + 1 + int(rel[0])
exit_price = opens[exit_idx]
exit_ts = ts_index[exit_idx]
gross = entry_side * (exit_price - entry_price) * size
fees = fees_rate * size * (entry_price + exit_price)
net = gross - fees
# La chiusura avviene a open[exit_idx]: dal bar exit_idx in poi il
# PnL e' realizzato (non piu' MtM).
realized_pnl[exit_idx:] += net
# Posizione aperta in [entry_idx, exit_idx-1].
open_side[entry_idx:exit_idx] = entry_side
open_entry[entry_idx:exit_idx] = entry_price
trades.append(Trade(
entry_ts=ts_index[entry_idx],
exit_ts=exit_ts,
side=Side.LONG if entry_side == 1 else Side.SHORT,
size=size,
entry_price=entry_price,
exit_price=exit_price,
fees_bp=self.fees_bp,
))
else:
# Ultima posizione ancora aperta: chiusura forced a close[-1].
# Parita' col loop legacy: MtM su [entry_idx, n-1), realized totale
# SOLO al bar n-1 (legacy fa equity_history[-1] = equity).
last_close = closes[-1]
gross = entry_side * (last_close - entry_price) * size
fees = fees_rate * size * (entry_price + last_close)
net = gross - fees
if entry_idx < n - 1:
open_side[entry_idx:n - 1] = entry_side
open_entry[entry_idx:n - 1] = entry_price
realized_pnl[-1] += net
trades.append(Trade(
entry_ts=ts_index[entry_idx],
exit_ts=ts_index[-1],
side=Side.LONG if entry_side == 1 else Side.SHORT,
size=size,
entry_price=entry_price,
exit_price=last_close,
fees_bp=self.fees_bp,
))
mark = float(row["close"])
mtm = position.unrealized_pnl(mark) if position else 0.0
current_equity = equity + mtm
equity_history.append(current_equity)
returns_history.append(current_equity - prev_equity)
prev_equity = current_equity
if position is not None:
assert position_entry_ts is not None
last_ts = ohlcv.index[-1]
last_close = float(ohlcv["close"].iloc[-1])
trade = Trade(
entry_ts=position_entry_ts,
exit_ts=last_ts,
side=position.side,
size=position.size,
entry_price=position.entry_price,
exit_price=last_close,
fees_bp=self.fees_bp,
)
trades.append(trade)
equity += trade.net_pnl
equity_history[-1] = equity
if len(returns_history) >= 2:
returns_history[-1] = equity - equity_history[-2]
# MtM unrealized per ogni bar in cui c'e' una posizione aperta.
mtm = open_side.astype(np.float64) * (closes - open_entry) * size
equity_arr = realized_pnl + mtm
# Returns = first diff dell'equity (col loop legacy il primo bar e' equity[0]-0).
returns_arr = np.concatenate(([equity_arr[0]], np.diff(equity_arr)))
return BacktestResult(
equity_curve=pd.Series(equity_history, index=ohlcv.index, name="equity"),
returns=pd.Series(returns_history, index=ohlcv.index, name="returns"),
equity_curve=pd.Series(equity_arr, index=ts_index, name="equity"),
returns=pd.Series(returns_arr, index=ts_index, name="returns"),
trades=trades,
)
# Lo facade Position re-export e' tenuto per backward-compat con import legacy.
__all__ = ["BacktestEngine", "BacktestResult", "Position", "Side", "Signal", "Trade"]
src/multi_swarm_core/multi_swarm_core/orchestrator/run.py
+39 -6
View File
@@ -13,6 +13,7 @@ possa leggere lo stato a run terminato (o in corso).
from __future__ import annotations
import random
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
@@ -20,13 +21,13 @@ import pandas as pd # type: ignore[import-untyped]
from ..agents.adversarial import AdversarialAgent
from ..agents.falsification import FalsificationAgent
from ..agents.hypothesis import HypothesisAgent
from ..agents.hypothesis import HypothesisAgent, HypothesisProposal, MarketSummary
from ..agents.market_summary import build_market_summary
from ..ga.fitness import compute_fitness
from ..ga.initial import build_initial_population
from ..ga.loop import GAConfig, next_generation
from ..ga.summary import generation_summary
from ..genome.hypothesis import ModelTier
from ..genome.hypothesis import HypothesisAgentGenome, ModelTier
from ..genome.mutation import set_cognitive_styles
from ..genome.prompt_library import PromptLibrary
from ..llm.client import LLMClient
@@ -73,6 +74,29 @@ class RunConfig:
# i 6 builtin (PromptLibrary.default()). Tipicamente caricata da
# strategy_crypto/prompts.json via PromptLibrary.from_json().
prompt_library: PromptLibrary | None = None
# Numero di propose() LLM concorrenti per generazione. 1 = sequenziale (default,
# backward compat). 6-10 tipicamente accettati da OpenRouter qwen-2.5 senza
# rate-limit. Riduce wall time GA loop di 5-8x su tier C.
llm_concurrency: int = 1
def _parallel_propose(
agent: HypothesisAgent,
genomes: list[HypothesisAgentGenome],
market: MarketSummary,
n_workers: int,
) -> list[HypothesisProposal]:
"""Esegue ``agent.propose()`` su una lista di genomi, opzionalmente in parallelo.
``n_workers <= 1`` mantiene il comportamento serial originale (ordine fisso,
determinismo data un seed). ``n_workers > 1`` usa un thread pool: l'order
dei risultati e' preservato (1:1 con ``genomes``). OpenAI/openrouter client
e' thread-safe; ``PromptLibrary`` e ``HypothesisAgent`` non hanno stato mutabile.
"""
if n_workers <= 1 or len(genomes) <= 1:
return [agent.propose(g, market) for g in genomes]
with ThreadPoolExecutor(max_workers=n_workers) as pool:
return list(pool.map(lambda g: agent.propose(g, market), genomes))
def run_phase1(
@@ -142,11 +166,20 @@ def run_phase1(
try:
for gen in range(cfg.n_generations):
# Step 1: raccogli i genomi da valutare in questa generazione (esclude
# elite gia' presenti nella cache fitnesses) e lancia propose() in
# parallelo. La sezione DB-write resta serial sotto.
uncached = [g for g in population if g.id not in fitnesses]
proposals = _parallel_propose(
hypothesis_agent, uncached, market, cfg.llm_concurrency
)
proposal_by_id = {g.id: p for g, p in zip(uncached, proposals, strict=True)}
for genome in population:
if genome.id in fitnesses:
continue # elite gia' valutata in generazione precedente
repo.save_genome(run_id=run_id, generation_idx=gen, genome=genome)
proposal = hypothesis_agent.propose(genome, market)
proposal = proposal_by_id[genome.id]
# Registra costo per OGNI completion (incluse retry).
for completion in proposal.completions:
cost_record = cost_tracker.record(
@@ -220,7 +253,7 @@ def run_phase1(
cfg.fitness_combined_alpha * fit
+ (1.0 - cfg.fitness_combined_alpha) * fit_oos_inloop
)
except Exception: # noqa: BLE001
except Exception:
pass # fallback: usa solo IS
repo.save_evaluation(
run_id=run_id,
@@ -261,7 +294,7 @@ def run_phase1(
# WFA re-eval: i top_k genomi (by fitness in-sample > 0) vengono rivalutati
# sul test_ohlcv. Le metriche OOS finiscono in evaluations.fitness_oos etc.
if test_ohlcv is not None and len(test_ohlcv) >= 100:
from ..agents.hypothesis import _try_parse # noqa: PLC0415
from ..agents.hypothesis import _try_parse
all_evals = repo.list_evaluations(run_id)
top_evals = sorted(
@@ -276,7 +309,7 @@ def run_phase1(
try:
fals_oos = falsification_agent.evaluate(strategy, test_ohlcv)
adv_oos = adversarial_agent.review(strategy, test_ohlcv)
except Exception: # noqa: BLE001
except Exception:
continue
fit_oos = compute_fitness(
fals_oos, adv_oos,
src/multi_swarm_core/tests/unit/test_backtest_engine_vectorized.py
+160
View File
@@ -0,0 +1,160 @@
"""Parity check: engine vettorializzato vs reference iterrows implementation.
Mantiene una copia inline del loop ``iterrows`` come reference per garantire
che la vettorizzazione produca esattamente gli stessi trades, equity_curve e
returns su input pseudocasuali, indipendentemente dal regime di prezzo.
"""
from __future__ import annotations
import numpy as np
import pandas as pd
import pytest
from multi_swarm_core.backtest.engine import BacktestEngine, BacktestResult
from multi_swarm_core.backtest.orders import Position, Side, Trade
def _legacy_run(
ohlcv: pd.DataFrame, signals: pd.Series, fees_bp: float = 5.0
) -> BacktestResult:
"""Reference implementation: il loop iterrows originale (pre-vectorize).
Mantenuto qui esclusivamente come oracolo per i test di parità.
"""
signals = signals.reindex(ohlcv.index).ffill().fillna(Side.FLAT)
shifted = [Side.FLAT, *list(signals.iloc[:-1])]
executed_side = pd.Series(shifted, index=ohlcv.index, dtype=object)
position: Position | None = None
position_entry_ts: pd.Timestamp | None = None
trades: list[Trade] = []
equity = 0.0
equity_history: list[float] = []
returns_history: list[float] = []
prev_equity = 0.0
for ts, row in ohlcv.iterrows():
target_side = executed_side.loc[ts]
current_side = position.side if position else Side.FLAT
if target_side != current_side:
if position is not None:
assert position_entry_ts is not None
trade = Trade(
entry_ts=position_entry_ts,
exit_ts=ts,
side=position.side,
size=position.size,
entry_price=position.entry_price,
exit_price=float(row["open"]),
fees_bp=fees_bp,
)
trades.append(trade)
equity += trade.net_pnl
position = None
position_entry_ts = None
if target_side in (Side.LONG, Side.SHORT):
position = Position(
side=target_side, entry_price=float(row["open"]), size=1.0
)
position_entry_ts = ts
mark = float(row["close"])
mtm = position.unrealized_pnl(mark) if position else 0.0
current_equity = equity + mtm
equity_history.append(current_equity)
returns_history.append(current_equity - prev_equity)
prev_equity = current_equity
if position is not None:
assert position_entry_ts is not None
last_ts = ohlcv.index[-1]
last_close = float(ohlcv["close"].iloc[-1])
trade = Trade(
entry_ts=position_entry_ts,
exit_ts=last_ts,
side=position.side,
size=position.size,
entry_price=position.entry_price,
exit_price=last_close,
fees_bp=fees_bp,
)
trades.append(trade)
equity += trade.net_pnl
equity_history[-1] = equity
if len(returns_history) >= 2:
returns_history[-1] = equity - equity_history[-2]
return BacktestResult(
equity_curve=pd.Series(equity_history, index=ohlcv.index, name="equity"),
returns=pd.Series(returns_history, index=ohlcv.index, name="returns"),
trades=trades,
)
def _random_ohlcv(n: int, seed: int) -> pd.DataFrame:
rng = np.random.default_rng(seed)
rets = rng.normal(0.0, 0.01, size=n)
close = 100.0 * np.exp(np.cumsum(rets))
idx = pd.date_range("2024-01-01", periods=n, freq="1h", tz="UTC")
return pd.DataFrame(
{
"open": close * (1 + rng.normal(0, 0.001, n)),
"high": close * 1.005,
"low": close * 0.995,
"close": close,
"volume": rng.uniform(1.0, 100.0, n),
},
index=idx,
)
def _random_signals(n: int, seed: int, p_change: float = 0.1) -> pd.Series:
"""Segnali con persistenza: ad ogni bar con prob p_change cambia stato."""
rng = np.random.default_rng(seed + 999)
states = [Side.LONG, Side.SHORT, Side.FLAT]
out: list[Side] = [rng.choice(states)]
for _ in range(1, n):
out.append(rng.choice(states) if rng.random() < p_change else out[-1])
idx = pd.date_range("2024-01-01", periods=n, freq="1h", tz="UTC")
return pd.Series(out, index=idx, dtype=object)
@pytest.mark.parametrize("seed", [0, 1, 42, 123, 999])
def test_vectorized_equals_legacy(seed: int) -> None:
df = _random_ohlcv(500, seed)
signals = _random_signals(500, seed)
engine = BacktestEngine(fees_bp=5.0)
new = engine.run(df, signals)
ref = _legacy_run(df, signals, fees_bp=5.0)
pd.testing.assert_series_equal(
new.equity_curve, ref.equity_curve, rtol=1e-9, atol=1e-9
)
pd.testing.assert_series_equal(
new.returns, ref.returns, rtol=1e-9, atol=1e-9
)
assert len(new.trades) == len(ref.trades)
for nt, rt in zip(new.trades, ref.trades, strict=True):
assert nt.entry_ts == rt.entry_ts
assert nt.exit_ts == rt.exit_ts
assert nt.side == rt.side
assert nt.entry_price == pytest.approx(rt.entry_price, abs=1e-12)
assert nt.exit_price == pytest.approx(rt.exit_price, abs=1e-12)
assert nt.net_pnl == pytest.approx(rt.net_pnl, abs=1e-12)
def test_vectorized_handles_position_still_open_at_end() -> None:
"""Edge case: signal LONG fino all'ultimo bar (exit a close[-1] forced)."""
df = _random_ohlcv(100, seed=7)
signals = pd.Series([Side.LONG] * 100, index=df.index)
new = BacktestEngine(fees_bp=10.0).run(df, signals)
ref = _legacy_run(df, signals, fees_bp=10.0)
pd.testing.assert_series_equal(new.equity_curve, ref.equity_curve, atol=1e-9)
assert len(new.trades) == 1
assert new.trades[0].side == Side.LONG
def test_vectorized_zero_signals_no_trades() -> None:
df = _random_ohlcv(50, seed=3)
signals = pd.Series([Side.FLAT] * 50, index=df.index)
new = BacktestEngine().run(df, signals)
assert len(new.trades) == 0
assert (new.equity_curve == 0).all()
src/multi_swarm_core/tests/unit/test_orchestrator_parallel_propose.py
+78
View File
@@ -0,0 +1,78 @@
"""Test che `_parallel_propose` preservi l'ordine dei risultati e funzioni
sia in modalita' sequenziale (workers=1) che concorrente (workers>1).
Non vogliamo testare il vero `HypothesisAgent.propose()` (che fa chiamate
LLM); usiamo un dummy con una latenza simulata per validare ordine e parallelismo.
"""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Any
from multi_swarm_core.orchestrator.run import _parallel_propose
@dataclass
class _FakeGenome:
id: str
@dataclass
class _FakeProposal:
genome_id: str
class _FakeAgent:
"""Agent dummy: propose() dorme 50ms e ritorna un proposal con l'id del genome."""
def __init__(self, delay_s: float = 0.05) -> None:
self._delay = delay_s
self.call_count = 0
def propose(self, genome: _FakeGenome, market: Any) -> _FakeProposal:
time.sleep(self._delay)
self.call_count += 1
return _FakeProposal(genome_id=genome.id)
def test_parallel_propose_preserves_order_serial() -> None:
agent = _FakeAgent(delay_s=0.01)
genomes = [_FakeGenome(id=f"g{i}") for i in range(5)]
results = _parallel_propose(agent, genomes, market=None, n_workers=1)
assert [r.genome_id for r in results] == ["g0", "g1", "g2", "g3", "g4"]
assert agent.call_count == 5
def test_parallel_propose_preserves_order_concurrent() -> None:
agent = _FakeAgent(delay_s=0.05)
genomes = [_FakeGenome(id=f"g{i}") for i in range(8)]
results = _parallel_propose(agent, genomes, market=None, n_workers=4)
assert [r.genome_id for r in results] == [f"g{i}" for i in range(8)]
assert agent.call_count == 8
def test_parallel_propose_actually_parallelizes() -> None:
"""Wall time con 4 worker su 4 task da 100ms deve essere ~100ms, non ~400ms."""
agent = _FakeAgent(delay_s=0.1)
genomes = [_FakeGenome(id=f"g{i}") for i in range(4)]
t0 = time.time()
_parallel_propose(agent, genomes, market=None, n_workers=4)
elapsed = time.time() - t0
# serial sarebbe 0.4s; con 4 worker scendiamo a ~0.1s (max 0.2 per overhead).
assert elapsed < 0.2, f"expected <200ms with 4 workers, got {elapsed * 1000:.0f}ms"
def test_parallel_propose_handles_single_genome() -> None:
agent = _FakeAgent()
results = _parallel_propose(agent, [_FakeGenome(id="solo")], market=None, n_workers=8)
assert len(results) == 1
assert results[0].genome_id == "solo"
def test_parallel_propose_empty_input() -> None:
agent = _FakeAgent()
results = _parallel_propose(agent, [], market=None, n_workers=4)
assert results == []
assert agent.call_count == 0