diff --git a/.gitignore b/.gitignore index 0c4da25..3509a89 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ data/regime/dispersion_features.parquet # storico catena opzioni importato da cerbero-bite (rigenerabile: options_fetcher.py) data/options/ data/_reset_backup/ + +# game artifacts (log/json di scripts/games e gate) +data/games/ diff --git a/docs/diary/2026-06-09-blind-traders-game.md b/docs/diary/2026-06-09-blind-traders-game.md new file mode 100644 index 0000000..2e5cfda --- /dev/null +++ b/docs/diary/2026-06-09-blind-traders-game.md @@ -0,0 +1,56 @@ +# 2026-06-09 — Gioco "Blind Traders": 100 agenti ciechi + +## Setup +100 agenti LLM (haiku) ricevono due serie anonime **X** e **Y** — in realta' +**BTC** e **ETH** 1h/15m/5m, mai etichettate — e devono proporre UNA regola che +"anticipi" i movimenti per un PnL netto positivo (fee 0.10% RT) con **>=10 +trade/mese**. Non sanno cosa siano i dati. L'orchestratore (engine deterministico) +valuta ogni strategia, assegna un punteggio su **PNL + %win**, da' **90 epoche di +elaborazione** (hill-climb dei parametri) e **ogni 10 epoche blocca il 10% meno +profittevole** -> restano i **10 piu' profittevoli**. + +Infrastruttura in `scripts/games/`: +- `engine.py` — dati anonimizzati, 6 famiglie segnale (zscore/breakout/ma_cross/ + rsi/momentum/pairs), backtester causale fee-aware, scoring (>=10 tpm o squalifica). +- `agent_brief.py` — digest ANONIMO (stat aggregate + finestra normalizzata) + menu. +- `arena.py` — torneo a **3 finestre**: TRAIN (hill-climb), VALID (cull+rank + dell'orchestratore), TEST (OOS puro, mai ottimizzato). Anti-overfit. +- `run_game.py` — carica le 100 spec degli agenti e lancia il torneo. + +## Risultato emergente +I 100 agenti ciechi, leggendo SOLO le statistiche anonime (autocorrelazione +negativa, "after_big_move_continues_pct" ~30-40% => le mosse estreme rientrano), +hanno **riscoperto da soli che il mercato e' mean-reverting**: 100/100 reversion, +67 hanno scelto il detector pairs, 30 zscore. Esattamente la lezione storica del +progetto (edge = reversione; pairs ETH/BTC il piu' robusto) — senza sapere che +fosse crypto. + +## Classifica finale (top 10) — tutti PAIRS su 15m +Vincitore **agente #91** (15m, pairs market-neutral sul log-ratio X/Y): +- TEST/OOS puro: **PnL +3126%**, **win 77%**, **108.9 trade/mese**, **Sharpe 20.3** +- Full-period: PnL +8052%, win 70%, 94 tpm, Sharpe 12.2 (9604 trade) +- params: lookback 66, entry 1.67σ, exit 1.0σ, max_bars 35 +- ipotesi (cieca): "Y altamente reversivo, X/Y log-ratio strong mean-reversion + (-0.43 autocorr), bassa correlazione cross-asset -> pairs market-neutral". + +Tutti i 10 finalisti: pairs 15m, TEST Sharpe medio 19.9, tpm 66-109 (>>10). + +## Caveat onesti +- Numeri OOS ottimistici: PnL additivo a notional fisso, **niente slippage sulle 2 + gambe**, finestra OOS calma, 15m molti trade. Coerente col caveat PR01 del + progetto (Sharpe reale atteso ~4-5, non 20). Il valore del gioco e' il **metodo** + (scoperta cieca + selezione anti-overfit), non il livello assoluto di Sharpe. +- La convergenza su pairs conferma robustezza ma riduce la diversita': i 10 finalisti + sono varianti della stessa idea (ETH/BTC spread). Per un portafoglio servirebbe + diversificare (gia' fatto altrove: fade + honest + shape). + +## Re-run "sobrio" con slippage (0.05%/lato) +`GAME_SLIP=0.0005` -> i pairs pagano +0.20% RT extra (4 lati). Lo slippage spinge +l'ottimizzatore verso **meno churn**: tpm dei finalisti 66-109 -> **40-47**, Sharpe +top-10 ~20 -> ~13.5. Vincitore **#43** (15m pairs): TEST PnL **+2091%**, win 77%, +**46.9 tpm**, Sharpe **15.6**. La gerarchia (pairs 15m domina) e la robustezza +reggono lo stress; lo Sharpe reale atteso resta ~4-5 (OOS calmo + PnL additivo). +Log: `data/games/game_slip.log`. + +Artefatti: `data/games/tournament_result.json`, `data/games/specs/agent_*.json`, +`engine.set_slippage()` (env `GAME_SLIP`). diff --git a/docs/diary/2026-06-09-pairs15m-live-path.md b/docs/diary/2026-06-09-pairs15m-live-path.md new file mode 100644 index 0000000..3aee169 --- /dev/null +++ b/docs/diary/2026-06-09-pairs15m-live-path.md @@ -0,0 +1,82 @@ +# 2026-06-09 — Percorso live 15m per ETH/BTC pairs: COSTRUITO e VALIDATO + +Seguito di `2026-06-09-pairs15m-port06-gate.md` (il gate passa, edge reale e non +artefatto flat). Qui si costruisce e VALIDA l'infrastruttura per eseguire il pairs +ETH/BTC a 15m con flat-skip, alla pari del backtest (disciplina validate_worker_pairs). + +## 1. Engine canonico (regression-locked) +`scripts/analysis/pairs_research.py`: aggiunti `aligned_ohlc`, `is_flat_ohlc`, +`pairs_sim_flat(..., flat_skip, scan_buffer)`. Regola di uscita **LIVE-REALIZABLE**: +la condizione (|z|<=z_exit O bars>=max_bars) ARMA `exit_ready`; si esce al CLOSE della +PRIMA barra PULITA successiva (mai a un prezzo passato come faceva il prototipo push-back). +- **Regression-lock**: `pairs_sim_flat(flat_skip=False)` == `pairs_sim` ESATTO + (ETH/BTC 1h 1756 trade, 15m 9388 trade, ret/dd/sharpe identici al bit). + +## 2. PairsWorker esteso (retrocompatibile) +`src/live/pairs_worker.py`: param `flat_skip`, stato `exit_ready` (persistito), tick +ora fa merge OHLC e rileva le candele flat (O=H=L=C in UNA gamba). Entry saltato su barra +stale; uscita con la stessa regola exit_ready dell'engine. **Default off = comportamento +1h storico invariato** (se mancano le colonne OHLC, flat=False). + +## 3. Runner: fetch sub-orario (inerte finche' non c'e' uno sleeve 15m) +`src/portfolio/runner.py`: `_SUBHOURLY={5m,15m,30m}`, `_LOOKBACK_DAYS` esteso; il loop +fetcha DIRETTO da Cerbero i timeframe sub-orari per (asset,tf) (non resamplabili dal 1h) e +un router `_series_for` instrada la serie giusta a ogni worker. Zero impatto sul live +attuale: nessuno sleeve e' 15m → `subhourly_needs` vuoto → ramo morto. + +## 4. VALIDAZIONE (validate_worker_pairs.py) — TUTTO OK +Replay bar-per-bar del worker == backtest: +| caso | worker | backtest | match | +|---|---|---|---| +| ETH/BTC 1h | 1756 trd, cap 2.886.616 | 1756, 2.886.616 | **OK esatto** | +| BTC/LTC 1h | 599 trd, cap 16.861 | 599, 16.861 | **OK esatto** | +| **ETH/BTC 15m-flat** | **8452 trd** | **8453 trd** (cap entro 0.15%) | **OK** | +(1 trade di differenza = posizione finale aperta non chiusa nel replay, atteso.) + +## 5. Gate finale (engine == worker) — PROMOSSO +`pairs15m_gate_final.py` (corr 1h vs 15m = 0.372, 3201 ingressi flat saltati): +| variante ETH/BTC | FULL Sh | FULL DD | OOS Sh | OOS DD | +|---|---|---|---|---| +| baseline 1h | 6.43 | 3.96 | 8.58 | 1.36 | +| **SWAP 15m-flat** | 7.31 | 3.55 | **9.95** | **1.26** | +| **BLEND 1h+15m** | 7.03 | 3.66 | 9.57 | 1.24 | +Entrambi PROMOSSI (a fee backtest). Caveat slippage del gate precedente invariato → il +BLEND e' la forma raccomandata (meta' allocazione sul 1h pulito, slippage-robusto). + +## Stato e attivazione (NON fatta — decisione di deploy) +Tutto il PERCORSO e' pronto e validato, ma il 15m **non e' attivo nel portafoglio live**: +attivarlo cambia il trading reale e va deciso esplicitamente. Per accenderlo: +1. `_defs.py`: aggiungere SleeveSpec pairs ETH/BTC a 15m (tf="15m", + params={n:66,z_in:1.674,z_exit:1.0,max_bars:35,flat_skip:True}) — come SWAP della 1h o + come 2a sleeve (BLEND) sotto il cap PAIRS. +2. `report_families.build_everything` / `sleeves`: l'equity del nuovo sleeve dal + `pairs_sim_flat(tf=15m, flat_skip=True)` (per parita' backtest==report). +3. Shadow smoke su testnet (come `live_smoke_pairs.py`) prima del paper reale. +4. `deploy.sh` (bump+rebuild) — il runner gia' fetcha 15m e passa flat_skip via spec.params. + +Test suite: nessuna regressione (1h byte-exact). Artefatti: pairs_research.py, +pairs_worker.py, runner.py, validate_worker_pairs.py, pairs15m_gate_final.py. + +## ATTIVAZIONE IN REALE (2026-06-09) — BLEND, mezza size +Deciso: BLEND (sleeve 15m ACCANTO al 1h, non swap). Implementato: +- `_defs.py`: SleeveSpec `PR_ETHBTC_15M` (tf=15m, flat_skip, params.position_size=0.10 + = meta' del family PAIRS 0.20) in PAIRS -> entra in PORT04/05/06. +- `report_families.build_everything`: equity da `pairs_sim_flat(tf=15m, flat_skip=True, pos=0.075)` + (mezza size, == intento live) con sid PR_ETHBTC_15M. +- `runner.pos_for_spec`: override PER-SLEEVE (params.position_size) > famiglia > globale. +- **Mezza size perche'** a peso pieno il 15m pesava il 25.8% del rischio PORT06 (vs 9.5% del + 1h): dimezzato -> 11.5% vs 10.6%, bilanciato. Disciplina come la cap SHAPE; rispetta il + caveat slippage (il 15m non domina il book). + +**PORT06 col BLEND (mezza size)**: FULL Sharpe **6.43->7.20** DD **3.96->3.68**, +OOS Sharpe **8.58->9.66** DD **1.36->1.31**. Migliora tutto. + +**Smoke live 15m** (`pairs15m_live_smoke.py`): Cerbero serve candele 15m FRESCHE per +ETH e BTC (ultima barra 0 min fa, flat live 2-3%), worker flat-skip ticca OK. Esecuzione +reale a 2 gambe gia' coperta da `live_pairs_smoke.py` (livello strumento, tf-indipendente). + +**Regression-lock aggiornati** (miglioria attesa, non regressione): test_definitions +(17->18 sleeve), test_backtest_parity_cap (FULL 6.47->7.20, OOS 8.82->9.66). Suite verde. + +Live: il runner fetcha 15m diretto, costruisce il PairsWorker(flat_skip) col pos 0.10, +e lo esegue reale a 2 gambe (pairs_enabled). Attivazione via deploy (bump+rebuild). diff --git a/docs/diary/2026-06-09-pairs15m-port06-gate.md b/docs/diary/2026-06-09-pairs15m-port06-gate.md new file mode 100644 index 0000000..2dd195a --- /dev/null +++ b/docs/diary/2026-06-09-pairs15m-port06-gate.md @@ -0,0 +1,89 @@ +# 2026-06-09 — ETH/BTC pairs a 15m: gate PORT06 (dal gioco Blind Traders) + +## Origine +Il gioco "Blind Traders" (100 agenti ciechi) ha eletto come vincitore una variante +ETH/BTC pairs su **15m** (config #43: n=66 z_in=1.67 z_exit=1.0 max_bars=35). Domanda: +e' un vero miglioramento o un duplicato piu' veloce della sleeve PR01 ETH/BTC gia' +deployata a 1h? Testato sul serio con l'engine di PRODUZIONE `pairs_sim` + gate PORT06. +Script: `scripts/analysis/pairs15m_port06_gate.py`. + +## Risultati +- **Parita' OK** (corr 1.00000): l'harness riproduce esattamente il sleeve canonico + PR_ETHBTC → gate affidabile. +- **CORRELAZIONE 1h vs 15m = 0.349** (rendimenti giornalieri). **SMENTISCE la mia + ipotesi iniziale "duplicato ridondante"**: a 15m cattura eventi di reversione DIVERSI + → e' un diversificatore reale, non una doppia scommessa sullo stesso spread. +- **Robustezza 15m**: griglia n×z_in → **16/16 celle Sharpe>1** (9-12), plateau non picco. + Non e' un punto overfit del gioco. +- **Standalone**: 15m fa 9388 trade (vs 1756 a 1h), Sharpe 11.7 (vs 4.36), DD 54% (vs 48%), + 8/9 anni+ . (Le % FULL sono esplose dal compounding pos0.15·lev3 su 9k trade → guardare + Sharpe/DD/anni, non il livello %.) + +## Gate PORT06 (pos0.15 lev3 canonico, OOS da 2024-10-12) +| variante ETH/BTC | FULL Sh | FULL DD | OOS Sh | OOS DD | +|---|---|---|---|---| +| **baseline 1h** | 6.43 | 3.96 | 8.58 | 1.36 | +| **SWAP 15m** | 7.64 | 3.49 | **10.39** | **1.26** | +| **BLEND 1h+15m** | 7.30 | 3.63 | 9.95 | 1.24 | + +A fee di backtest (0.20% RT/coppia) **entrambe PROMOSSE**: Sharpe su e DD giu' ovunque. + +## Stress slippage a livello PORT06 (il vero rischio: 15m = 5× i trade) +| fee_rt | RT/coppia | PORT06 FULL Sh | FULL DD | OOS Sh | OOS DD | std Sh | std oDD | +|---|---|---|---|---|---|---|---| +| baseline 1h | 0.20% | 6.43 | 3.96 | 8.58 | 1.36 | 4.36 | 16% | +| 15m | 0.20% | 7.64 | 3.49 | 10.39 | 1.26 | 11.7 | 13% | +| 15m | 0.40% | 7.04 | 4.08 | 9.78 | 1.45 | 8.5 | 27% | +| 15m | 0.60% | 6.43 | 4.67 | 9.15 | 1.66 | 5.3 | 47% | + +**Degradazione graziosa ma reale**: il vantaggio di **Sharpe** sopravvive fino a slippage +pessimista (OOS 9.15 > 8.58 anche a 0.60%), ma il vantaggio di **DD si perde gia' a 0.40%** +(FULL DD 4.08 > 3.96 baseline; standalone oDD esplode 13→27→47%). La regola del progetto +("ri-gateare ogni filtro quando cambiano i costi") qui taglia: la frequenza 5× rende la +sleeve slippage-sensitive. + +## Verdetto +- **NON un duplicato** (corr 0.35) e **NON overfit** (16/16 robusto) → la mia liquidazione + iniziale era SBAGLIATA, lo dico chiaro. +- **Passa il gate a fee di backtest, marginale sotto slippage**: migliora Sharpe sempre, ma + sotto slippage realistico (≥0.40% RT) peggiora leggermente il DD di portafoglio. +- **Due rischi di produzione NON ancora quantificati**: (a) qualita' dati ETH 15m (14-30%/anno + candele flat O=H=L=C → fill non eseguibili che gonfierebbero il backtest), (b) fill/liquidita' + reale a 2 gambe a 15m (5× ordini). Il worker pairs e' validato a 1h, non a 15m. + +**Raccomandazione**: NON swap diretto in live. Candidato promettente → percorso forward: +preferire il **BLEND 1h+15m** (tiene il DD pulito del 1h e raccoglie il rendimento +decorrelato del 15m) **dopo** un check sull'impatto delle candele flat 15m sui pairs. +Allineato a come il progetto tratta FR01 (robusto ma non deployato finche' non domina pulito). +Resta come record di ricerca; deploy solo se il check flat-candle e' pulito. + +## CHECK FLAT-CANDLE (pairs15m_flatcheck.py) — PULITO +Rischio: ETH 15m ha molte candele flat (O=H=L=C) → close stale che gonfia z-score → +reversione FINTA non eseguibile. Test: +- **Prevalenza**: ETH 15m **16.4% medio** (fino 30% nel 2022); BTC 15m solo 3.5%. Reale. +- **Fill toccati**: 12.9% degli entry e 15.2% degli exit cadono su una barra flat. +- **Test decisivo** (entry/exit SOLO su barre pulite, non-flat in entrambe le gambe): + rimuove 11.2% dei trade, **Sharpe trattenuto all'83%** (11.74→9.70; OOS Sharpe 18.4). + Se l'edge fosse un artefatto flat, filtrando crollerebbe → **NON crolla. NON e' artefatto.** +- **Gate PORT06 col 15m FLAT-FILTRATO** (corr 1h vs 15m-flat = 0.366, ancora decorrelato): + - SWAP 15m-flat: FULL 7.32/3.55, OOS **9.99/1.26** → PROMOSSO + - BLEND 1h+15m-flat: FULL 7.05/3.66, OOS **9.60/1.24** → PROMOSSO + +## Conclusione (3 box su 4 puliti) +✅ NON duplicato (corr 0.35-0.37) ✅ robusto (16/16) ✅ NON artefatto flat (83% Sharpe) +⚠️ slippage-sensitive: a fee backtest passa pulito; a slippage ≥0.40% RT il vantaggio di +Sharpe regge ma il DD-edge si assottiglia. Il **BLEND** mitiga (meta' allocazione resta sul +1h pulito e slippage-robusto) → e' la forma deployabile. + +## Realta' del deploy (perche' NON tocco ancora il live) +Il gate passa a livello BACKTEST. Ma il live NON puo' eseguire un sleeve 15m oggi: +- la live pairs gira SOLO a 1h (`PairsWorker`, validato da `validate_worker_pairs` a 1h); + il runner risampla a 1h/4h/1d, non gestisce un leg pairs a 15m. +- un BLEND richiede DUE sotto-sleeve ETH/BTC (1h + 15m) dentro il cap PAIRS, e il + **flat-skip va replicato nel worker live** (altrimenti il live tradera' le barre stale che + il backtest esclude → divergenza backtest-vs-live, la classe di bug che il progetto teme). +Editare `_defs.py` cambierebbe solo il backtest/report, NON il live → sarebbe ingannevole. +**Percorso deploy corretto** (da confermare): (1) estendere `PairsWorker`/runner al 15m + +flat-skip; (2) `validate_worker_pairs` a 15m (replay == backtest esatto); (3) aggiungere lo +sleeve 15m sotto il cap PAIRS; (4) shadow su testnet prima del paper. Finche' (1)-(2) non +sono fatti e validati, resta **record di ricerca PROMOSSO ma non live**. diff --git a/scripts/analysis/pairs15m_flatcheck.py b/scripts/analysis/pairs15m_flatcheck.py new file mode 100644 index 0000000..06575ad --- /dev/null +++ b/scripts/analysis/pairs15m_flatcheck.py @@ -0,0 +1,203 @@ +"""Check candele FLAT (O=H=L=C, liquidita' zero) sui pairs ETH/BTC a 15m. + +Rischio noto (CLAUDE.md): ETH 15m ha 14-30%/anno di candele flat per bassa liquidita' +del perpetuo. Su un pairs, un close stale gonfia lo z-score (l'altra gamba si muove, +questa e' ferma) -> segnale di "reversione" FINTO che rientra solo quando la gamba +stale si sblocca: profitto NON eseguibile dal vivo. Questo gonfierebbe il backtest 15m. + +Test: + [1] prevalenza candele flat per anno (ETH 15m, BTC 15m). + [2] quanti trade del pairs 15m hanno ENTRY/EXIT su una candela flat (gamba stale). + [3] re-sim flat-aware: entry/exit SOLO su barre pulite (non-flat in ENTRAMBE le gambe) + -> quanto sopravvive l'edge? (parita': senza flat-skip == pairs_sim). + [4] gate PORT06 col 15m flat-filtrato vs baseline 1h. + + uv run python scripts/analysis/pairs15m_flatcheck.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT)) + +from src.data.downloader import load_data +from scripts.analysis.pairs_research import pairs_sim, OOS_FRAC, FEE_RT, LEV, POS, BARS_YEAR +from scripts.analysis.report_families import daily_from +from scripts.analysis.combine_portfolio import metrics, SPLIT, OOS_DATE +from scripts.analysis.pairs15m_port06_gate import port_metrics, eth_btc_daily, UNIV_1H, GAME_15M +from scripts.portfolios._defs import PORTFOLIOS +from src.portfolio.sleeves import all_sleeve_equities + + +def aligned2(a, b, tf="15m"): + """Merge con OHLC di ENTRAMBE le gambe (serve per rilevare i flat su entrambe).""" + da = load_data(a, tf)[["timestamp", "open", "high", "low", "close"]].rename( + columns=lambda x: x + "_a" if x != "timestamp" else x) + db = load_data(b, tf)[["timestamp", "open", "high", "low", "close"]].rename( + columns=lambda x: x + "_b" if x != "timestamp" else x) + m = da.merge(db, on="timestamp", how="inner").reset_index(drop=True) + m["dt"] = pd.to_datetime(m["timestamp"], unit="ms", utc=True) + return m + + +def is_flat(o, h, l, c): + return (o == h) & (h == l) & (l == c) + + +def flat_prevalence(asset, tf="15m"): + d = load_data(asset, tf) + d = d.copy() + d["dt"] = pd.to_datetime(d["timestamp"], unit="ms", utc=True) + fl = is_flat(d["open"].values, d["high"].values, d["low"].values, d["close"].values) + d["flat"] = fl + by = d.groupby(d["dt"].dt.year)["flat"].mean() * 100 + return by, fl.mean() * 100 + + +def pairs_sim_flataware(a, b, tf="15m", n=66, z_in=1.674, z_exit=1.0, max_bars=35, + jump_max=0.08, fee_rt=FEE_RT, lev=LEV, pos=POS, + split_frac=0.0, skip_flat=True): + """Come pairs_sim ma: entry/exit consentiti SOLO su barre pulite (se skip_flat). + Ritorna anche n_entry_flat / n_exit_flat (diagnostica, calcolata sempre).""" + m = aligned2(a, b, tf) + ca, cb = m["close_a"].values, m["close_b"].values + flat_a = is_flat(m["open_a"].values, m["high_a"].values, m["low_a"].values, ca) + flat_b = is_flat(m["open_b"].values, m["high_b"].values, m["low_b"].values, cb) + flat = flat_a | flat_b # barra "sporca" se una delle due gambe e' flat + r = np.log(ca / cb) + dr = np.abs(np.diff(r, prepend=r[0])) + ma = pd.Series(r).rolling(n).mean().values + sd = pd.Series(r).rolling(n).std().values + z = (r - ma) / np.where(sd == 0, np.nan, sd) + ts = m["dt"]; N = len(r) + split = int(N * split_frac) + fee = 2 * fee_rt * lev + cap = peak = 1000.0; dd = 0.0; last = -1 + trades = wins = 0; rets = []; yearly = {} + eq_ts, eq_v = [], [] + n_entry_flat = n_exit_flat = 0 + for i in range(n + 1, N - 1): + if i < split or np.isnan(z[i]) or dr[i] > jump_max or i <= last: + continue + if z[i] <= -z_in: + d = 1 + elif z[i] >= z_in: + d = -1 + else: + continue + if flat[i]: + n_entry_flat += 1 + if skip_flat: + continue # non si entra su una gamba stale + # exit: |z|<=z_exit o max_bars; se skip_flat, salta le barre flat come uscita + j = min(i + max_bars, N - 1) + for k in range(1, max_bars + 1): + jj = i + k + if jj >= N: + j = N - 1; break + if skip_flat and flat[jj]: + j = jj # avanza, non esce su barra stale + continue + if abs(z[jj]) <= z_exit: + j = jj; break + j = jj + if flat[j]: + n_exit_flat += 1 + if skip_flat: + # spingi all'ultima barra pulita entro l'orizzonte + back = j + while back > i and flat[back]: + back -= 1 + j = back if back > i else j + retA = (ca[j] - ca[i]) / ca[i] + retB = (cb[j] - cb[i]) / cb[i] + ret = (retA - retB) * d * lev - fee + cap = max(cap + cap * pos * ret, 10.0) + peak = max(peak, cap); dd = max(dd, (peak - cap) / peak) + trades += 1; wins += ret > 0; rets.append(ret * pos); last = j + eq_ts.append(ts.iloc[j]); eq_v.append(cap) + yearly[ts.iloc[i].year] = yearly.get(ts.iloc[i].year, 0.0) + ret * 100 + yrs_span = (ts.iloc[-1] - ts.iloc[max(split, 0)]).days / 365.25 or 1 + sharpe = 0.0 + if len(rets) > 1 and np.std(rets) > 0: + sharpe = float(np.mean(rets) / np.std(rets) * np.sqrt(trades / yrs_span)) + ret_tot = (cap / 1000 - 1) * 100 + return dict(trades=trades, win=wins / trades * 100 if trades else 0, ret=ret_tot, + dd=dd * 100, sharpe=sharpe, yearly=yearly, eq_ts=eq_ts, eq_v=eq_v, + n_entry_flat=n_entry_flat, n_exit_flat=n_exit_flat) + + +def main(): + print("=" * 100) + print(" CHECK FLAT-CANDLE — ETH/BTC pairs 15m (gate condizionato)") + print("=" * 100) + + # [1] prevalenza + print("\n[1] Prevalenza candele flat (O=H=L=C) per anno, 15m:") + for asset in ("ETH", "BTC"): + by, tot = flat_prevalence(asset, "15m") + print(f" {asset}: media {tot:.1f}% | " + + " ".join(f"{y}:{v:.0f}%" for y, v in by.items())) + + # [2] quanti trade toccano un flat (sim SENZA skip per diagnostica) + diag = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=False) + tr = diag["trades"] + print(f"\n[2] Trade 15m totali: {tr} | entry su barra flat: {diag['n_entry_flat']} " + f"({diag['n_entry_flat']/tr*100:.1f}%) | exit su barra flat: {diag['n_exit_flat']} " + f"({diag['n_exit_flat']/tr*100:.1f}%)") + + # [3] parita' + edge filtrato + print("\n[3] Edge 15m: NO-skip (== pairs_sim) vs FLAT-AWARE (entry/exit solo barre pulite):") + # parita': flataware skip_flat=False deve ~== pairs_sim + base_ps = pairs_sim("ETH", "BTC", **GAME_15M, pos=POS, lev=LEV) + print(f" parita' pairs_sim : trd {base_ps['trades']:>5d} Sh {base_ps['sharpe']:.2f} " + f"DD {base_ps['dd']:.0f}% ret {base_ps['ret']:+.0f}%") + print(f" flataware (no-skip) : trd {diag['trades']:>5d} Sh {diag['sharpe']:.2f} " + f"DD {diag['dd']:.0f}% ret {diag['ret']:+.0f}%") + filt = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=True) + filt_o = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=True, split_frac=1 - OOS_FRAC) + print(f" FLAT-AWARE (skip) : trd {filt['trades']:>5d} Sh {filt['sharpe']:.2f} " + f"DD {filt['dd']:.0f}% ret {filt['ret']:+.0f}% | OOS Sh {filt_o['sharpe']:.2f} DD {filt_o['dd']:.0f}%") + drop = (1 - filt['trades'] / diag['trades']) * 100 + sh_keep = filt['sharpe'] / diag['sharpe'] * 100 if diag['sharpe'] else 0 + verdict = "EDGE NON artefatto flat" if sh_keep > 70 else "EDGE in larga parte ARTEFATTO flat" + print(f" -> rimossi {drop:.1f}% dei trade; Sharpe trattenuto {sh_keep:.0f}% ({verdict})") + + # [4] gate PORT06 col 15m flat-filtrato + print("\n[4] GATE PORT06 — ETH/BTC: baseline 1h vs SWAP 15m-FLATAWARE vs BLEND:") + p = PORTFOLIOS["PORT06"] + pair_ids = [s.sid for s in p.sleeves if s.sid.startswith("PR_")] + eq_base = dict(all_sleeve_equities()) + e1h, _ = eth_btc_daily(UNIV_1H) + e15f = daily_from(filt["eq_ts"], filt["eq_v"]) + # blend 1h + 15m-flataware (50/50 daily-rebalanced) + from scripts.analysis.pairs15m_port06_gate import blend + eblend = blend(e1h, e15f, 0.5) + corr = e1h.pct_change().fillna(0).corr(e15f.pct_change().fillna(0)) + print(f" corr 1h vs 15m-flataware: {corr:.3f}") + print(f" {'variante':<18s} | {'FULL Sh':>8s}{'FULL DD%':>9s}{'CAGR':>6s} | {'OOS Sh':>7s}{'OOS DD%':>8s}") + print(" " + "-" * 70) + res = {} + for tag, eth in [("baseline 1h", e1h), ("SWAP 15m-flat", e15f), ("BLEND 1h+15m-flat", eblend)]: + members = dict(eq_base); members["PR_ETHBTC"] = eth + f, o = port_metrics(members, p) + res[tag] = (f, o) + print(f" {tag:<18s} | {f['sharpe']:>8.2f}{f['dd']:>9.2f}{f['cagr']:>5.0f}%" + f" | {o['sharpe']:>7.2f}{o['dd']:>8.2f}") + + fb, ob = res["baseline 1h"] + print("\n VERDETTO (vs baseline 1h, fee backtest): Sharpe non peggiora E DD <= baseline") + for tag in ("SWAP 15m-flat", "BLEND 1h+15m-flat"): + f, o = res[tag] + ok = o["sharpe"] >= ob["sharpe"] - 0.02 and o["dd"] <= ob["dd"] + 1e-9 and f["sharpe"] >= fb["sharpe"] - 0.02 and f["dd"] <= fb["dd"] + 1e-9 + print(f" {tag:<18s}: OOS {ob['sharpe']:.2f}->{o['sharpe']:.2f} DD {ob['dd']:.2f}->{o['dd']:.2f}" + f" | FULL {fb['sharpe']:.2f}->{f['sharpe']:.2f} DD {fb['dd']:.2f}->{f['dd']:.2f} => {'PROMOSSO' if ok else 'bocciato'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/pairs15m_gate_final.py b/scripts/analysis/pairs15m_gate_final.py new file mode 100644 index 0000000..3c17d5b --- /dev/null +++ b/scripts/analysis/pairs15m_gate_final.py @@ -0,0 +1,62 @@ +"""GATE PORT06 FINALE — ETH/BTC 15m flat-skip, engine canonico pairs_sim_flat. + +Usa pairs_sim_flat(flat_skip=True), cioe' la STESSA semantica live-realizable del +PairsWorker (uscita alla prima barra pulita), validata da validate_worker_pairs. +Conferma i numeri deployabili: baseline 1h vs SWAP 15m vs BLEND 1h+15m. + + uv run python scripts/analysis/pairs15m_gate_final.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT)) + +from scripts.analysis.pairs_research import pairs_sim_flat +from scripts.analysis.report_families import daily_from +from scripts.analysis.pairs15m_port06_gate import (port_metrics, eth_btc_daily, blend, + UNIV_1H, POS, LEV) +from scripts.portfolios._defs import PORTFOLIOS +from src.portfolio.sleeves import all_sleeve_equities + +CFG_15M = dict(n=66, z_in=1.674, z_exit=1.0, max_bars=35) + + +def main(): + p = PORTFOLIOS["PORT06"] + eq_base = dict(all_sleeve_equities()) + e1h, _ = eth_btc_daily(UNIV_1H) + r15 = pairs_sim_flat("ETH", "BTC", tf="15m", **CFG_15M, flat_skip=True, pos=POS, lev=LEV) + e15 = daily_from(r15["eq_ts"], r15["eq_v"]) + eblend = blend(e1h, e15, 0.5) + corr = e1h.pct_change().fillna(0).corr(e15.pct_change().fillna(0)) + + print("=" * 92) + print(" GATE PORT06 FINALE — ETH/BTC 15m flat-skip (pairs_sim_flat == worker live)") + print(f" 15m: {r15['trades']} trade, {r15['n_skip_entry']} ingressi flat saltati | " + f"corr 1h vs 15m = {corr:.3f}") + print("=" * 92) + print(f" {'variante':<18s} | {'FULL Sh':>8s}{'FULL DD%':>9s}{'CAGR':>6s} | {'OOS Sh':>7s}{'OOS DD%':>8s}") + print(" " + "-" * 70) + res = {} + for tag, eth in [("baseline 1h", e1h), ("SWAP 15m-flat", e15), ("BLEND 1h+15m", eblend)]: + members = dict(eq_base); members["PR_ETHBTC"] = eth + f, o = port_metrics(members, p) + res[tag] = (f, o) + print(f" {tag:<18s} | {f['sharpe']:>8.2f}{f['dd']:>9.2f}{f['cagr']:>5.0f}%" + f" | {o['sharpe']:>7.2f}{o['dd']:>8.2f}") + fb, ob = res["baseline 1h"] + print("\n Promosso se OOS Sharpe non peggiora E DD<=baseline (PORT06):") + for tag in ("SWAP 15m-flat", "BLEND 1h+15m"): + f, o = res[tag] + ok = o["sharpe"] >= ob["sharpe"] - 0.02 and o["dd"] <= ob["dd"] + 1e-9 \ + and f["sharpe"] >= fb["sharpe"] - 0.02 and f["dd"] <= fb["dd"] + 1e-9 + print(f" {tag:<18s}: OOS {ob['sharpe']:.2f}->{o['sharpe']:.2f} DD {ob['dd']:.2f}->{o['dd']:.2f}" + f" | FULL {fb['sharpe']:.2f}->{f['sharpe']:.2f} DD {fb['dd']:.2f}->{f['dd']:.2f}" + f" => {'PROMOSSO' if ok else 'bocciato'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/pairs15m_live_smoke.py b/scripts/analysis/pairs15m_live_smoke.py new file mode 100644 index 0000000..be5c3a5 --- /dev/null +++ b/scripts/analysis/pairs15m_live_smoke.py @@ -0,0 +1,81 @@ +"""Smoke LIVE del nuovo percorso 15m: fetch DIRETTO 15m da Cerbero per ETH/BTC + +freschezza + flat-fraction + un tick reale del PairsWorker(flat_skip). + +Verifica cio' che il backtest non vede: che Cerbero serva candele 15m fresche per +entrambe le gambe (il runner ora le fetcha dirette, non resamplate dal 1h) e che il +worker 15m le processi senza errori. NON apre ordini reali (l'esecuzione a 2 gambe e' +gia' coperta da live_pairs_smoke.py, indipendente dal timeframe). + + uv run python scripts/analysis/pairs15m_live_smoke.py +""" +from __future__ import annotations + +import sys +import shutil +import tempfile +from datetime import datetime, timezone, timedelta +from pathlib import Path + +import pandas as pd + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT)) + +from src.live.cerbero_client import CerberoClient +from src.live.multi_runner import INSTRUMENT_MAP +from src.live.pairs_worker import PairsWorker + +CFG = {"n": 66, "z_in": 1.674, "z_exit": 1.0, "max_bars": 35, "flat_skip": True} + + +def fetch15(cli, asset, days=14): + inst = INSTRUMENT_MAP.get(asset, f"{asset}-PERPETUAL") + end = datetime.now(timezone.utc) + start = end - timedelta(days=days) + candles = cli.get_historical_v2(inst, start.strftime("%Y-%m-%d"), + end.strftime("%Y-%m-%d"), "15m") + if not candles: + return inst, None + df = pd.DataFrame(candles) + df["timestamp"] = df["timestamp"].astype("int64") + return inst, df.sort_values("timestamp").reset_index(drop=True) + + +def main(): + print("=" * 84) + print(" SMOKE LIVE — ETH/BTC pairs 15m (fetch diretto Cerbero + tick worker flat-skip)") + print("=" * 84) + cli = CerberoClient() + inst_a, da = fetch15(cli, "ETH") + inst_b, db = fetch15(cli, "BTC") + ok = True + for asset, inst, df in [("ETH", inst_a, da), ("BTC", inst_b, db)]: + if df is None or df.empty: + print(f" {asset} ({inst}): NESSUNA candela 15m -> FAIL"); ok = False; continue + last = pd.to_datetime(df["timestamp"].iloc[-1], unit="ms", utc=True) + age_min = (datetime.now(timezone.utc) - last).total_seconds() / 60 + flat = ((df["open"] == df["high"]) & (df["high"] == df["low"]) & + (df["low"] == df["close"])).mean() * 100 + fresh = age_min < 60 + print(f" {asset} ({inst}): {len(df)} barre 15m | ultima {last:%Y-%m-%d %H:%M} " + f"({age_min:.0f} min fa, {'FRESCO' if fresh else 'STALE'}) | flat {flat:.1f}%") + ok &= fresh + if da is None or db is None: + print("\n ESITO: FAIL (feed 15m assente)."); return + # tick reale del worker 15m + tmp = Path(tempfile.mkdtemp(prefix="smoke15m_")) + try: + w = PairsWorker("ETH", "BTC", "15m", params=CFG, fee_rt=0.001, data_dir=tmp) + df_a = pd.DataFrame({"timestamp": da["timestamp"], "open": da["open"], "high": da["high"], + "low": da["low"], "close": da["close"]}) + df_b = pd.DataFrame({"timestamp": db["timestamp"], "open": db["open"], "high": db["high"], + "low": db["low"], "close": db["close"]}) + w.tick(df_a, df_b) + print(f"\n Worker 15m flat_skip={w.flat_skip} -> tick OK | {w.status_summary}") + print(f" ESITO: {'OK — feed 15m fresco e worker ticca' if ok else 'ATTENZIONE: feed 15m stale/parziale'}") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/pairs15m_port06_gate.py b/scripts/analysis/pairs15m_port06_gate.py new file mode 100644 index 0000000..ef60ec2 --- /dev/null +++ b/scripts/analysis/pairs15m_port06_gate.py @@ -0,0 +1,169 @@ +"""GATE PORT06 — ETH/BTC pairs a 15m (origine: gioco "Blind Traders", vincitore #43). + +Domanda onesta sollevata dal gioco: la coppia ETH/BTC (gia' deployata in PR01 a 1h, +config UNIV n=50 z_in=2.0 z_exit=0.75 max_bars=72) MIGLIORA se girata a 15m con la +config trovata dal gioco (n=66 z_in=1.67 z_exit=1.0 max_bars=35), oppure e' solo una +variante piu' veloce, correlata, dello STESSO spread? + +Metodo (engine di PRODUZIONE pairs_sim, NON il motore-giocattolo del gioco): + [1] PARITA': pairs_sim ETH/BTC 1h UNIV (pos0.15 lev3) == sleeve canonico PR_ETHBTC. + [2] CORRELAZIONE 1h vs 15m (rendimenti giornalieri): se ~1 e' ridondante. + [3] STANDALONE 1h vs 15m (+ griglia robustezza n x z_in su 15m, + stress fee 2x). + [4] GATE PORT06: baseline(1h) vs SWAP(15m) vs BLEND(0.5*1h+0.5*15m) per la sleeve + ETH/BTC; promosso se vs baseline l'OOS Sharpe non peggiora E il DD scende + (PORT06 e famiglia), come gli altri gate del progetto. + + uv run python scripts/analysis/pairs15m_port06_gate.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT)) + +from scripts.analysis.combine_portfolio import port_returns, metrics, SPLIT, OOS_DATE, IDX +from scripts.analysis.pairs_research import pairs_sim, OOS_FRAC +from scripts.analysis.report_families import daily_from +from scripts.portfolios._defs import PORTFOLIOS +from src.portfolio import weighting as W + +POS, LEV = 0.15, 3.0 # config CANONICA (== build_everything) +UNIV_1H = dict(tf="1h", n=50, z_in=2.0, z_exit=0.75, max_bars=72) +GAME_15M = dict(tf="15m", n=66, z_in=1.674, z_exit=1.0, max_bars=35) # vincitore gioco + + +def eth_btc_daily(cfg): + r = pairs_sim("ETH", "BTC", **{**cfg, "pos": POS, "lev": LEV}) + return daily_from(r["eq_ts"], r["eq_v"]), r + + +def std_metrics(cfg, fee_rt=0.001): + f = pairs_sim("ETH", "BTC", **{**cfg, "pos": POS, "lev": LEV, "fee_rt": fee_rt}) + o = pairs_sim("ETH", "BTC", **{**cfg, "pos": POS, "lev": LEV, "fee_rt": fee_rt, + "split_frac": 1 - OOS_FRAC}) + yrs = f["yearly"]; pos_y = sum(1 for v in yrs.values() if v > 0) + return f, o, pos_y, len(yrs) + + +def port_metrics(members, p): + ids = p.sleeve_ids + dr = pd.DataFrame({i: members[i].pct_change().fillna(0.0) for i in ids}) + w = W.weight_vector(p.weighting, ids, dr, weights=p.weights, + caps=p.caps, clusters=p.clusters, lookback=p.vol_lookback) + drp = port_returns({i: members[i] for i in ids}, w) + return metrics(drp), metrics(drp, lo=SPLIT) + + +def fam_metrics(eqs): + dr = port_returns(eqs) + return metrics(dr), metrics(dr, lo=SPLIT) + + +def blend(e1, e2, w1=0.5): + """Sleeve combinata: media pesata dei rendimenti giornalieri (ribilancio 1D).""" + r1 = e1.reindex(IDX).ffill().bfill().pct_change().fillna(0.0) + r2 = e2.reindex(IDX).ffill().bfill().pct_change().fillna(0.0) + rb = w1 * r1 + (1 - w1) * r2 + eq = (1 + rb).cumprod() + return eq / eq.iloc[0] + + +def main(): + p = PORTFOLIOS["PORT06"] + pair_ids = [s.sid for s in p.sleeves if s.sid.startswith("PR_")] + print("=" * 100) + print(" GATE PORT06 — ETH/BTC pairs 15m (vincitore gioco) vs 1h deployato") + print(f" pos={POS} lev={LEV} (canonico) | OOS da {OOS_DATE} | coppie PORT06: {pair_ids}") + print("=" * 100) + + from src.portfolio.sleeves import all_sleeve_equities + eq_base = dict(all_sleeve_equities()) + + # [1] PARITA' + print("\n[1] PARITA' pairs_sim ETH/BTC 1h UNIV (pos0.15 lev3) == sleeve canonico PR_ETHBTC:") + e1h, r1h = eth_btc_daily(UNIV_1H) + base = eq_base["PR_ETHBTC"] + corr = base.pct_change().fillna(0).corr(e1h.pct_change().fillna(0)) + rb = (base.iloc[-1] / base.iloc[0] - 1) * 100 + rr = (e1h.iloc[-1] / e1h.iloc[0] - 1) * 100 + par_ok = corr > 0.999 and abs(rr - rb) <= max(1.0, abs(rb) * 0.01) + print(f" corr={corr:.5f} ret canon {rb:+.0f}% vs replay {rr:+.0f}% " + f"{'OK' if par_ok else '<-- MISMATCH (STOP)'}") + if not par_ok: + return + + # [2] CORRELAZIONE 1h vs 15m + e15, r15 = eth_btc_daily(GAME_15M) + c = e1h.pct_change().fillna(0).corr(e15.pct_change().fillna(0)) + print(f"\n[2] CORRELAZIONE rendimenti giornalieri ETH/BTC 1h vs 15m: {c:.3f}") + print(f" {'(quasi-duplicato se >0.8; diversificatore se <0.5)':<60s}") + + # [3] STANDALONE 1h vs 15m + print("\n[3] STANDALONE ETH/BTC (netto fee 0.20% RT/coppia, leva 3x):") + print(f" {'cfg':<10s}{'trd':>6s}{'win%':>6s}{'FULL%':>9s}{'OOS%':>9s}{'CAGR%':>7s}" + f"{'DD%':>6s}{'oDD%':>7s}{'Shrp':>6s}{'anni+':>7s}{'fee2x FULL%':>12s}") + for tag, cfg in [("1h UNIV", UNIV_1H), ("15m gioco", GAME_15M)]: + f, o, py, ny = std_metrics(cfg) + f2, _, _, _ = std_metrics(cfg, fee_rt=0.002) + print(f" {tag:<10s}{f['trades']:>6d}{f['win']:>6.1f}{f['ret']:>+9.0f}{o['ret']:>+9.0f}" + f"{f['cagr']:>7.0f}{f['dd']:>6.0f}{o['dd']:>7.0f}{f['sharpe']:>6.2f}" + f"{f'{py}/{ny}':>7s}{f2['ret']:>+12.0f}") + + # robustezza: plateau n x z_in su 15m (Sharpe>1?) + print("\n Robustezza 15m (Sharpe full, griglia n x z_in, z_exit=1.0 max_bars=35):") + ns = [40, 50, 66, 80]; zs = [1.5, 1.7, 2.0, 2.5] + cells = 0; tot = 0 + hdr = " n\\z_in " + "".join(f"{z:>7.1f}" for z in zs) + print(hdr) + for n in ns: + row = f" {n:>6d} " + for z in zs: + s = pairs_sim("ETH", "BTC", tf="15m", n=n, z_in=z, z_exit=1.0, + max_bars=35, pos=POS, lev=LEV)["sharpe"] + tot += 1; cells += s > 1 + row += f"{s:>7.2f}" + print(row) + print(f" -> {cells}/{tot} celle Sharpe>1 (plateau se ~tutte; picco se poche)") + + # [4] GATE PORT06 + print("\n[4] GATE PORT06 — sleeve ETH/BTC: baseline(1h) vs SWAP(15m) vs BLEND(50/50):") + variants = { + "baseline 1h": e1h, + "SWAP 15m": e15, + "BLEND 1h+15m": blend(e1h, e15, 0.5), + } + print(f" {'variante':<14s} | {'FULL Sh':>8s}{'FULL DD%':>9s}{'CAGR':>6s}" + f" | {'OOS Sh':>7s}{'OOS DD%':>8s} | {'famSh':>6s}{'famDD%':>7s}") + print(" " + "-" * 78) + res = {} + for tag, eth in variants.items(): + members = dict(eq_base) + members["PR_ETHBTC"] = eth + f, o = port_metrics(members, p) + fam_eqs = {sid: (eth if sid == "PR_ETHBTC" else eq_base[sid]) for sid in pair_ids} + ff, _ = fam_metrics(fam_eqs) + res[tag] = (f, o, ff) + print(f" {tag:<14s} | {f['sharpe']:>8.2f}{f['dd']:>9.2f}{f['cagr']:>5.0f}%" + f" | {o['sharpe']:>7.2f}{o['dd']:>8.2f} | {ff['sharpe']:>6.2f}{ff['dd']:>7.1f}") + + # VERDETTO + fb, ob, _ = res["baseline 1h"] + print("\n" + "=" * 100) + print(" VERDETTO vs baseline 1h: promosso se OOS Sharpe non peggiora E DD scende (PORT06 e famiglia)") + print("=" * 100) + for tag in ("SWAP 15m", "BLEND 1h+15m"): + f, o, ff = res[tag] + ok = (o["sharpe"] >= ob["sharpe"] - 0.02 and o["dd"] <= ob["dd"] + 1e-9 + and f["sharpe"] >= fb["sharpe"] - 0.02) + print(f" {tag:<14s}: OOS Sh {ob['sharpe']:.2f}->{o['sharpe']:.2f} " + f"DD {ob['dd']:.2f}->{o['dd']:.2f} | FULL Sh {fb['sharpe']:.2f}->{f['sharpe']:.2f} " + f"DD {fb['dd']:.2f}->{f['dd']:.2f} => {'PROMOSSO' if ok else 'bocciato'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/pairs_research.py b/scripts/analysis/pairs_research.py index 883c8ed..df8e8e6 100644 --- a/scripts/analysis/pairs_research.py +++ b/scripts/analysis/pairs_research.py @@ -95,6 +95,98 @@ def pairs_sim(a, b, tf="1h", n=50, z_in=2.0, z_exit=0.5, max_bars=72, eq_ts=eq_ts, eq_v=eq_v) +def aligned_ohlc(a: str, b: str, tf: str = "1h"): + """Come aligned ma con OHLC di ENTRAMBE le gambe (serve a rilevare candele flat).""" + da = load_data(a, tf)[["timestamp", "open", "high", "low", "close"]].rename( + columns=lambda x: x + "_a" if x != "timestamp" else x) + db = load_data(b, tf)[["timestamp", "open", "high", "low", "close"]].rename( + columns=lambda x: x + "_b" if x != "timestamp" else x) + m = da.merge(db, on="timestamp", how="inner").reset_index(drop=True) + m["dt"] = pd.to_datetime(m["timestamp"], unit="ms", utc=True) + return m + + +def is_flat_ohlc(o, h, l, c): + """Candela flat (O=H=L=C): prezzo fermo / liquidita' zero -> fill non eseguibile.""" + return (o == h) & (h == l) & (l == c) + + +def pairs_sim_flat(a, b, tf="1h", n=50, z_in=2.0, z_exit=0.5, max_bars=72, + jump_max=0.08, fee_rt=FEE_RT, lev=LEV, pos=POS, split_frac=0.0, + flat_skip=False, scan_buffer=192): + """Engine pairs GENERALIZZATO con opzione flat-skip LIVE-REALIZABLE. + + Identico a pairs_sim quando flat_skip=False (regression-lock verificato). + Con flat_skip=True: + - ENTRY: saltata se la barra d'ingresso e' flat in UNA delle due gambe (prezzo stale). + - EXIT: la condizione di uscita (|z|<=z_exit O bars>=max_bars) arma 'exit_ready'; + si esce al CLOSE della PRIMA barra PULITA successiva (mai a un prezzo passato). + scan_buffer = barre extra oltre max_bars concesse per trovare la barra pulita. + Questa e' la stessa regola implementata nel PairsWorker live (flat_skip) -> parita'. + """ + m = aligned_ohlc(a, b, tf) + ca, cb = m["close_a"].values, m["close_b"].values + N = len(ca) + if flat_skip: + flat = (is_flat_ohlc(m["open_a"].values, m["high_a"].values, m["low_a"].values, ca) + | is_flat_ohlc(m["open_b"].values, m["high_b"].values, m["low_b"].values, cb)) + else: + flat = np.zeros(N, dtype=bool) + r = np.log(ca / cb) + dr = np.abs(np.diff(r, prepend=r[0])) + ma = pd.Series(r).rolling(n).mean().values + sd = pd.Series(r).rolling(n).std().values + z = (r - ma) / np.where(sd == 0, np.nan, sd) + ts = m["dt"] + split = int(N * split_frac) + fee = 2 * fee_rt * lev + cap = peak = 1000.0; dd = 0.0; last = -1 + trades = wins = 0; rets = []; yearly = {} + eq_ts, eq_v = [], [] + n_skip_entry = 0 + kmax = max_bars + (scan_buffer if flat_skip else 0) + for i in range(n + 1, N - 1): + if i < split or np.isnan(z[i]) or dr[i] > jump_max or i <= last: + continue + if z[i] <= -z_in: + d = 1 + elif z[i] >= z_in: + d = -1 + else: + continue + if flat[i]: + n_skip_entry += 1 + continue # niente ingresso su barra stale + # uscita live-realizable: arma a |z|<=z_exit o max_bars, esci alla prima barra pulita + exit_ready = False; j = i + for k in range(1, kmax + 1): + jj = i + k + if jj >= N: + j = N - 1; break + if not exit_ready and (abs(z[jj]) <= z_exit or k >= max_bars): + exit_ready = True + if exit_ready and not flat[jj]: + j = jj; break + j = jj + retA = (ca[j] - ca[i]) / ca[i] + retB = (cb[j] - cb[i]) / cb[i] + ret = (retA - retB) * d * lev - fee + cap = max(cap + cap * pos * ret, 10.0) + peak = max(peak, cap); dd = max(dd, (peak - cap) / peak) + trades += 1; wins += ret > 0; rets.append(ret * pos); last = j + eq_ts.append(ts.iloc[j]); eq_v.append(cap) + yearly[ts.iloc[i].year] = yearly.get(ts.iloc[i].year, 0.0) + ret * 100 + yrs_span = (ts.iloc[-1] - ts.iloc[max(split, 0)]).days / 365.25 or 1 + sharpe = 0.0 + if len(rets) > 1 and np.std(rets) > 0: + sharpe = float(np.mean(rets) / np.std(rets) * np.sqrt(trades / yrs_span)) + ret_tot = (cap / 1000 - 1) * 100 + cagr = ((cap / 1000) ** (1 / yrs_span) - 1) * 100 if cap > 0 else -100 + return dict(trades=trades, win=wins / trades * 100 if trades else 0, ret=ret_tot, + cagr=cagr, dd=dd * 100, sharpe=sharpe, yearly=yearly, + eq_ts=eq_ts, eq_v=eq_v, n_skip_entry=n_skip_entry) + + def check_no_lookahead(): """Perturba il FUTURO del ratio e verifica che z[i] non cambi (causalita').""" m = aligned("ETH", "BTC") diff --git a/scripts/analysis/report_families.py b/scripts/analysis/report_families.py index 3c619b8..6117510 100644 --- a/scripts/analysis/report_families.py +++ b/scripts/analysis/report_families.py @@ -28,7 +28,7 @@ from scripts.analysis.combine_portfolio import ( build_all_sleeves, port_returns, metrics, yearly_returns, SPLIT, OOS_DATE, IDX, ) from scripts.analysis.honest_improve2 import _daily_equity, _norm -from scripts.analysis.pairs_research import pairs_sim +from scripts.analysis.pairs_research import pairs_sim, pairs_sim_flat from scripts.analysis.tsmom_research import tsmom_sim from scripts.strategies.PR01_pairs_reversion import PAIRS from scripts.analysis.shape_ml_validate import shape_daily_equity @@ -46,6 +46,16 @@ def build_everything(): for a, b, p in PAIRS: r = pairs_sim(a, b, **p) pairs[f"PR_{a}{b}"] = daily_from(r["eq_ts"], r["eq_v"]) + # BLEND ETH/BTC 15m flat-skip (gioco Blind Traders -> gate PORT06, decorrelato 0.37 + # dal 1h, edge non-artefatto-flat, worker validato). Engine LIVE-REALIZABLE identico + # al PairsWorker (pairs_sim_flat). Diari 2026-06-09-pairs15m-*.md. + # MEZZA size (pos 0.075 = meta' della canonica 0.15): a peso uguale il 15m, piu' + # volatile, contribuirebbe ~26% del rischio PORT06 (vs ~9% del 1h). Dimezzarlo lo + # riporta in linea col 1h -> blend-tilt, non scommessa dominante (col caveat slippage). + # Coerente col live (params.position_size=0.10 = meta' del family PAIRS 0.20). + r15 = pairs_sim_flat("ETH", "BTC", tf="15m", n=66, z_in=1.674, z_exit=1.0, + max_bars=35, flat_skip=True, pos=0.075) + pairs["PR_ETHBTC_15M"] = daily_from(r15["eq_ts"], r15["eq_v"]) t = tsmom_sim() tsm = {"TSM01": daily_from(t["eq_ts"], t["eq_v"])} shape = {f"SH_{a}": _norm(shape_daily_equity(a, IDX)) for a in ("BTC", "ETH")} diff --git a/scripts/analysis/validate_worker_pairs.py b/scripts/analysis/validate_worker_pairs.py index 84dcf1d..6c1b842 100644 --- a/scripts/analysis/validate_worker_pairs.py +++ b/scripts/analysis/validate_worker_pairs.py @@ -18,56 +18,70 @@ PROJECT_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(PROJECT_ROOT)) from src.live.pairs_worker import PairsWorker -from scripts.analysis.pairs_research import aligned, pairs_sim +from scripts.analysis.pairs_research import aligned, aligned_ohlc, pairs_sim, pairs_sim_flat from scripts.strategies.PR01_pairs_reversion import PAIRS -WINDOW = 60 # finestra trailing minima (>= n+2): z[i] corretto, replay veloce +# Config 15m promossa dal gate (gioco Blind Traders + flat-skip): vedi +# docs/diary/2026-06-09-pairs15m-port06-gate.md +CFG_15M = dict(n=66, z_in=1.674, z_exit=1.0, max_bars=35, flat_skip=True) -def replay(a: str, b: str, params: dict, data_dir: Path) -> PairsWorker: - m = aligned(a, b) - df_a = m[["timestamp"]].copy(); df_a["close"] = m["close_a"].values - df_b = m[["timestamp"]].copy(); df_b["close"] = m["close_b"].values - w = PairsWorker(a, b, "1h", params=params, fee_rt=0.001, data_dir=data_dir) - # replay veloce: niente I/O su file / log / notifiche ad ogni tick (servono solo le metriche finali) +def replay(a, b, params, data_dir, tf="1h", ohlc=False) -> PairsWorker: + if ohlc: + m = aligned_ohlc(a, b, tf) + df_a = pd.DataFrame({"timestamp": m["timestamp"], "open": m["open_a"], + "high": m["high_a"], "low": m["low_a"], "close": m["close_a"]}) + df_b = pd.DataFrame({"timestamp": m["timestamp"], "open": m["open_b"], + "high": m["high_b"], "low": m["low_b"], "close": m["close_b"]}) + else: + m = aligned(a, b, tf) + df_a = m[["timestamp"]].copy(); df_a["close"] = m["close_a"].values + df_b = m[["timestamp"]].copy(); df_b["close"] = m["close_b"].values + w = PairsWorker(a, b, tf, params=params, fee_rt=0.001, data_dir=data_dir) w._save_state = lambda: None w._log = lambda *a, **k: None w._notify = lambda *a, **k: None - n = w.n - for k in range(n + 2, len(m) + 1): - lo = max(0, k - WINDOW) + window = max(60, w.n + 6) # finestra trailing >= n+? : z[i] corretto + for k in range(w.n + 2, len(m) + 1): + lo = max(0, k - window) w.tick(df_a.iloc[lo:k], df_b.iloc[lo:k]) - # chiudi eventuale posizione aperta a fine serie (come fa il backtest col troncamento) return w +def _row(label, w, bt): + bt_cap = 1000.0 * (1 + bt["ret"] / 100) + cap_match = abs(w.capital - bt_cap) / bt_cap < 0.02 if bt_cap else False + trd_match = abs(w.total_trades - bt["trades"]) <= max(2, bt["trades"] * 0.02) + ok = "OK" if (cap_match and trd_match) else "DIFF" + ww = w.total_wins / w.total_trades * 100 if w.total_trades else 0 + print(f" {label:<16s}{w.capital:>13.0f}{w.total_trades:>6d}{ww:>6.1f} | " + f"{bt_cap:>14.0f}{bt['trades']:>6d}{bt['win']:>6.1f} {ok}") + return ok == "OK" + + def main(): - print("=" * 96) - print(" VALIDAZIONE PairsWorker — replay live vs backtest pairs_sim (fee 0.20% RT/coppia)") - print("=" * 96) - print(f" {'coppia':<10s}{'WORKER cap':>12s}{'trd':>5s}{'win%':>6s} | {'BACKTEST cap':>13s}{'trd':>5s}{'win%':>6s} match?") - print(" " + "-" * 88) - # Sottoinsieme rappresentativo: il codice del worker e' identico per ogni coppia, - # quindi 2 coppie con strutture diverse (alt/major e major/alt) bastano a provare - # l'equivalenza. ~135s/coppia su 73k barre orarie. Per validarle tutte: usa PAIRS. - subset = [pp for pp in PAIRS if (pp[0], pp[1]) in {("ETH", "BTC"), ("BTC", "LTC")}] + print("=" * 100) + print(" VALIDAZIONE PairsWorker — replay live == backtest (fee 0.20% RT/coppia)") + print("=" * 100) + print(f" {'caso':<16s}{'WORKER cap':>13s}{'trd':>6s}{'win%':>6s} | " + f"{'BACKTEST cap':>14s}{'trd':>6s}{'win%':>6s} match?") + print(" " + "-" * 92) tmp = Path(tempfile.mkdtemp(prefix="pairs_validate_")) + allok = True try: - for a, b, p in subset: - w = replay(a, b, p, tmp) - bt = pairs_sim(a, b, **p) - bt_cap = 1000.0 * (1 + bt["ret"] / 100) - cap_match = abs(w.capital - bt_cap) / bt_cap < 0.02 if bt_cap else False - trd_match = abs(w.total_trades - bt["trades"]) <= max(2, bt["trades"] * 0.02) - ok = "OK" if (cap_match and trd_match) else "DIFF" - ww = w.total_wins / w.total_trades * 100 if w.total_trades else 0 - print(f" {a+'/'+b:<10s}{w.capital:>12.0f}{w.total_trades:>5d}{ww:>6.1f} | " - f"{bt_cap:>13.0f}{bt['trades']:>5d}{bt['win']:>6.1f} {ok}") + # [A] REGRESSIONE 1h (flat_skip=False, close-only) vs pairs_sim + for a, b, p in [pp for pp in PAIRS if (pp[0], pp[1]) in {("ETH", "BTC"), ("BTC", "LTC")}]: + w = replay(a, b, p, tmp, tf="1h", ohlc=False) + allok &= _row(f"{a}/{b} 1h", w, pairs_sim(a, b, **p)) + # [B] NUOVO: 15m flat-skip (OHLC) vs pairs_sim_flat + w = replay("ETH", "BTC", CFG_15M, tmp, tf="15m", ohlc=True) + bt = pairs_sim_flat("ETH", "BTC", tf="15m", **CFG_15M) + allok &= _row("ETH/BTC 15m-flat", w, bt) finally: shutil.rmtree(tmp, ignore_errors=True) - print(" " + "-" * 88) - print(" match = capitale entro 2% e trade entro 2% del backtest. Differenze minime sono") - print(" attese (gestione bar finale/troncamento), ma la semantica deve coincidere.") + print(" " + "-" * 92) + print(" match = capitale e trade entro 2% del backtest (diff minime = bar finale aperta).") + print(f" ESITO COMPLESSIVO: {'TUTTO OK' if allok else 'DIFFERENZE -> INDAGARE'}") if __name__ == "__main__": diff --git a/scripts/games/agent_brief.py b/scripts/games/agent_brief.py new file mode 100644 index 0000000..c936ef0 --- /dev/null +++ b/scripts/games/agent_brief.py @@ -0,0 +1,99 @@ +""" +agent_brief — genera il "digest" ANONIMO che ogni agente cieco riceve. + +L'agente non sa che sono BTC/ETH ne' che e' crypto: vede solo due serie X e Y +(rinominate dal motore A/B), una finestra normalizzata (base 100) e statistiche +aggregate. Da queste deve proporre una regola che "anticipi" i movimenti. + +Genera anche il MENU dei blocchi (famiglie + range parametri) che l'agente puo' +comporre, in modo che l'output sia una spec backtestabile. +""" +from __future__ import annotations + +import json + +import numpy as np + +from scripts.games.engine import load_anon + + +def _stats(close, high, low): + r = np.diff(np.log(close)) + r = r[np.isfinite(r)] + out = { + "n_bars": int(len(close)), + "ret_vol_pct": round(float(np.std(r) * 100), 4), + "ret_autocorr_lag1": round(float(np.corrcoef(r[:-1], r[1:])[0, 1]), 4), + "ret_autocorr_lag5": round(float(np.corrcoef(r[:-5], r[5:])[0, 1]), 4), + "pct_up_bars": round(float(np.mean(r > 0) * 100), 2), + "skew": round(float(((r - r.mean()) ** 3).mean() / (r.std() ** 3 + 1e-12)), 3), + "kurtosis": round(float(((r - r.mean()) ** 4).mean() / (r.std() ** 4 + 1e-12)), 2), + } + # tendenza a rientrare dopo grandi mosse (|z|>2): segno del rendimento successivo + z = (r - r.mean()) / (r.std() + 1e-12) + big = np.where(np.abs(z[:-1]) > 2)[0] + if len(big) > 20: + nxt = r[big + 1] + same = np.sign(r[big]) == np.sign(nxt) + out["after_big_move_continues_pct"] = round(float(np.mean(same) * 100), 1) + return out + + +def make_digest(tf: str, window: int = 60, seed: int = 0): + data = load_anon(tf) + n = data["n"] + # finestra recente normalizzata (base 100) per "vedere" la forma + s = max(0, n - window) + dig = {"timeframe_id": {"1h": "T1", "15m": "T2", "5m": "T3"}.get(tf, "T?"), + "n_bars_total": n, "series": {}} + for name in ("A", "B"): + o = data[name] + c = o["close"] + norm = (c[s:] / c[s] * 100.0) + dig["series"][{"A": "X", "B": "Y"}[name]] = { + "stats": _stats(c, o["high"], o["low"]), + "recent_window_norm": [round(float(v), 2) for v in norm], + } + # relazione fra le due serie + ra = np.diff(np.log(data["A"]["close"])) + rb = np.diff(np.log(data["B"]["close"])) + m = min(len(ra), len(rb)) + dig["XY_return_correlation"] = round(float(np.corrcoef(ra[:m], rb[:m])[0, 1]), 4) + lr = np.log(data["A"]["close"][:m + 1] / data["B"]["close"][:m + 1]) + dig["XY_logratio_ret_autocorr"] = round( + float(np.corrcoef(np.diff(lr)[:-1], np.diff(lr)[1:])[0, 1]), 4) + return dig + + +MENU = { + "obiettivo": ("Proponi UNA regola che anticipi i movimenti futuri per un PnL " + "netto positivo dopo costi (0.10% andata+ritorno per trade). " + "Servono >=10 operazioni al mese. Non sai cosa siano X e Y."), + "famiglie": { + "zscore": "fade/segui lo z-score del prezzo su 'lookback' barre (entry_thr in sigma)", + "breakout": "rottura del canale max/min su 'lookback' barre (reversion=fade la rottura)", + "ma_cross": "incrocio EMA veloce(lookback)/lenta(lookback*slow_mult)", + "rsi": "RSI(lookback); entry_thr scala le bande attorno a 50", + "momentum": "rendimento su 'lookback' barre vs soglia entry_thr (%)", + "pairs": "market-neutral sullo z del log-rapporto X/Y (long una/short l'altra)", + }, + "direzione": ["reversion (vai contro la mossa)", "trend (segui la mossa)"], + "serie": ["X", "Y (solo per single-family)", "pairs usa entrambe"], + "exit": "tp_atr / sl_atr (in unita' ATR), max_bars (durata massima)", + "range": { + "lookback": "5-120", "entry_thr": "1.0-3.5", "tp_atr": "0.5-4.0", + "sl_atr": "1.0-5.0", "max_bars": "6-120", "slow_mult": "2-6", + "exit_thr (pairs)": "0.2-1.0", + }, + "output_schema": { + "family": "una di [zscore,breakout,ma_cross,rsi,momentum,pairs]", + "series": "X|Y|AB(pairs)", "direction": "reversion|trend", + "params": "dict coi parametri scelti", "hypothesis": "1-2 frasi: cosa hai notato", + }, +} + + +if __name__ == "__main__": + import sys + tf = sys.argv[1] if len(sys.argv) > 1 else "1h" + print(json.dumps(make_digest(tf), indent=2)[:2000]) diff --git a/scripts/games/arena.py b/scripts/games/arena.py new file mode 100644 index 0000000..486de70 --- /dev/null +++ b/scripts/games/arena.py @@ -0,0 +1,231 @@ +""" +Arena — tournament orchestrator per il gioco "Blind Traders". + +100 agenti partono da una spec di strategia (creata alla cieca: vedi +agent_brief.py / workflow). L'orchestratore valuta ogni spec con il backtest +deterministico (engine.evaluate) su TRAIN, da' epoche di elaborazione (ogni +agente affina la propria strategia via hill-climb sui parametri) e OGNI 10 +EPOCHE blocca il 10% meno profittevole. Restano i 10 piu' profittevoli. + +Punteggio = fitness su PNL + %win, con vincolo >=10 trade/mese (engine). +""" +from __future__ import annotations + +import json +import random +from pathlib import Path + +import numpy as np + +from scripts.games.engine import load_anon, splits3, evaluate + +OUT = Path("data/games") +OUT.mkdir(parents=True, exist_ok=True) + +# Spazio parametri per famiglia (min, max, tipo) +SPACE = { + "zscore": dict(lookback=(10, 100, "i"), entry_thr=(1.0, 3.5, "f"), + tp_atr=(0.5, 4.0, "f"), sl_atr=(1.0, 5.0, "f"), + max_bars=(6, 72, "i")), + "breakout": dict(lookback=(12, 120, "i"), entry_thr=(0.0, 0.0, "f"), + tp_atr=(0.5, 4.0, "f"), sl_atr=(1.0, 5.0, "f"), + max_bars=(6, 72, "i")), + "ma_cross": dict(lookback=(5, 50, "i"), slow_mult=(2.0, 6.0, "f"), + entry_thr=(0.0, 0.0, "f"), tp_atr=(0.5, 4.0, "f"), + sl_atr=(1.0, 5.0, "f"), max_bars=(6, 72, "i")), + "rsi": dict(lookback=(7, 30, "i"), entry_thr=(1.0, 4.0, "f"), + tp_atr=(0.5, 4.0, "f"), sl_atr=(1.0, 5.0, "f"), + max_bars=(6, 72, "i")), + "momentum": dict(lookback=(6, 72, "i"), entry_thr=(1.0, 6.0, "f"), + tp_atr=(0.5, 4.0, "f"), sl_atr=(1.0, 5.0, "f"), + max_bars=(6, 72, "i")), + "pairs": dict(lookback=(20, 120, "i"), entry_thr=(1.5, 3.0, "f"), + exit_thr=(0.2, 1.0, "f"), max_bars=(24, 120, "i")), +} +SINGLE_FAMILIES = ["zscore", "breakout", "ma_cross", "rsi", "momentum"] +DIRECTIONS = ["reversion", "trend"] +TIMEFRAMES = ["1h", "15m", "5m"] # timing diversi su cui competono gli agenti + + +def _rand_param(rng, lo, hi, typ): + if typ == "i": + return int(rng.randint(int(lo), int(hi))) + return round(rng.uniform(lo, hi), 3) + + +def random_spec(rng): + if rng.random() < 0.25: + fam = "pairs" + else: + fam = rng.choice(SINGLE_FAMILIES) + params = {} + for k, (lo, hi, typ) in SPACE[fam].items(): + params[k] = _rand_param(rng, lo, hi, typ) + spec = {"family": fam, "params": params, "tf": rng.choice(TIMEFRAMES)} + if fam == "pairs": + spec["series"] = "AB" + else: + spec["series"] = rng.choice(["A", "B"]) + spec["params"]["direction"] = rng.choice(DIRECTIONS) + return spec + + +def mutate(spec, rng, strength=0.25): + """Perturba la spec (hill-climb). Per lo piu' numerica; raramente + cambia direzione/serie. La famiglia resta fissa (identita' dell'agente).""" + s = json.loads(json.dumps(spec)) + fam = s["family"] + # perturba 1-2 parametri numerici + keys = [k for k in SPACE[fam] if SPACE[fam][k][0] != SPACE[fam][k][1]] + for k in rng.sample(keys, k=min(len(keys), rng.randint(1, 2))): + lo, hi, typ = SPACE[fam][k] + cur = s["params"][k] + span = (hi - lo) * strength + nv = cur + rng.uniform(-span, span) + nv = max(lo, min(hi, nv)) + s["params"][k] = int(round(nv)) if typ == "i" else round(nv, 3) + if fam != "pairs": + if rng.random() < 0.10: + s["params"]["direction"] = rng.choice(DIRECTIONS) + if rng.random() < 0.05: + s["series"] = rng.choice(["A", "B"]) + # il timeframe resta l'identita' dell'agente (timing fisso) -> non muta + return s + + +def _normalize(spec): + """Completa/ripulisce una spec proposta da un agente (robustezza).""" + fam = spec.get("family") + if fam not in SPACE: + fam = "zscore" + out = {"family": fam, "params": {}} + for k, (lo, hi, typ) in SPACE[fam].items(): + v = spec.get("params", {}).get(k, (lo + hi) / 2) + try: + v = float(v) + except Exception: + v = (lo + hi) / 2 + v = max(lo, min(hi, v)) + out["params"][k] = int(round(v)) if typ == "i" else round(v, 3) + out["tf"] = spec.get("tf") if spec.get("tf") in TIMEFRAMES else "1h" + if fam == "pairs": + out["series"] = "AB" + else: + out["series"] = spec.get("series", "A") if spec.get("series") in ("A", "B") else "A" + d = spec.get("params", {}).get("direction") or spec.get("direction") + out["params"]["direction"] = d if d in DIRECTIONS else "reversion" + return out + + +class Agent: + def __init__(self, aid, spec, brief=""): + self.id = aid + self.spec = _normalize(spec) + self.brief = brief # cosa "dice" l'agente (ipotesi NL) + self.train_fit = -1e9 # criterio di hill-climb (l'agente ottimizza qui) + self.valid_fit = -1e9 # criterio dell'orchestratore (cull + rank) + self.metrics = {} # metriche TRAIN + self.vmetrics = {} # metriche VALID + self.alive = True + self.culled_epoch = None + + @property + def tf(self): + return self.spec.get("tf", "1h") + + def score(self, datasets, splits_map): + data = datasets[self.tf] + tr, va, _ = splits_map[self.tf] + self.metrics = evaluate(data, self.spec, tr) + self.vmetrics = evaluate(data, self.spec, va) + self.train_fit = self.metrics["fitness"] + self.valid_fit = self.vmetrics["fitness"] + + +def run_tournament(specs, briefs=None, seed=7, + epochs=90, cull_every=10, cull_n=10, log=print): + rng = random.Random(seed) + # carica solo i timeframe effettivamente usati dagli agenti + used_tfs = sorted({_normalize(s).get("tf", "1h") for s in specs}) + datasets = {tf: load_anon(tf) for tf in used_tfs} + splits_map = {tf: splits3(datasets[tf], 0.60, 0.20) for tf in used_tfs} + briefs = briefs or [""] * len(specs) + + agents = [Agent(i, s, briefs[i] if i < len(briefs) else "") + for i, s in enumerate(specs)] + for a in agents: + a.score(datasets, splits_map) + + alive = lambda: [a for a in agents if a.alive] + log(f"[epoch 0] {len(alive())} agenti | best VALID fit " + f"{max(a.valid_fit for a in agents):.1f}") + + history = [] + for ep in range(1, epochs + 1): + # elaborazione: l'agente affina sul TRAIN (cio' che vede); ricalcola VALID + for a in alive(): + cand = mutate(a.spec, rng) + data = datasets[a.tf] + tr, va, _ = splits_map[a.tf] + m = evaluate(data, cand, tr) + if m["fitness"] > a.train_fit: + a.spec = _normalize(cand) + a.metrics, a.train_fit = m, m["fitness"] + a.vmetrics = evaluate(data, a.spec, va) + a.valid_fit = a.vmetrics["fitness"] + # cull ogni N epoche: l'ORCHESTRATORE blocca il 10% meno profittevole + # in VALIDATION (generalizzazione, non overfit sul train) + if ep % cull_every == 0: + av = sorted(alive(), key=lambda a: a.valid_fit) + k = cull_n if len(av) - cull_n >= 10 else max(0, len(av) - 10) + for a in av[:k]: + a.alive = False + a.culled_epoch = ep + log(f"[epoch {ep:2d}] cull {k:2d} -> {len(alive()):3d} vivi | " + f"best VALID {max(a.valid_fit for a in alive()):.1f} | " + f"worst-alive {min(a.valid_fit for a in alive()):.1f}") + history.append({"epoch": ep, "alive": len(alive()), + "best_valid": max(a.valid_fit for a in alive())}) + + survivors = sorted(alive(), key=lambda a: a.valid_fit, reverse=True) + # report finale: TEST = OOS puro mai toccato dall'ottimizzazione + results = [] + for rank, a in enumerate(survivors, 1): + data = datasets[a.tf] + _, _, te = splits_map[a.tf] + test = evaluate(data, a.spec, te) + full = evaluate(data, a.spec, None) + results.append({ + "rank": rank, "agent": a.id, "spec": a.spec, "brief": a.brief, + "tf": a.tf, "train": a.metrics, "valid": a.vmetrics, + "test": test, "full": full, + }) + payload = {"n_agents": len(specs), "epochs": epochs, + "survivors": len(survivors), "results": results, + "history": history, + "reveal": {"A": "BTC", "B": "ETH", "tf": "1h"}} + (OUT / "tournament_result.json").write_text(json.dumps(payload, indent=2)) + return payload + + +def leaderboard(payload, top=10, log=print): + log("\n================ CLASSIFICA FINALE (top %d) ================" % top) + log("VALID = finestra su cui l'orchestratore giudica | TEST = OOS puro (mai ottimizzato)") + log(f"{'#':>2} {'ag':>4} {'tf':>3} {'famiglia':>9} {'ser':>3} {'dir':>9} " + f"{'TEpnl%':>8} {'TEwin':>5} {'TEtpm':>6} {'TEsh':>5} {'VApnl%':>8} {'VAwin':>5}") + for r in payload["results"][:top]: + sp = r["spec"]; te = r["test"]; va = r["valid"] + d = sp["params"].get("direction", "-") + log(f"{r['rank']:>2} {r['agent']:>4} {sp.get('tf','1h'):>3} {sp['family']:>9} " + f"{sp['series']:>3} {d:>9} {te['pnl_pct']:>8.0f} {te['win_rate']*100:>4.0f}% " + f"{te['tpm']:>6.1f} {te['sharpe']:>5.1f} {va['pnl_pct']:>8.0f} " + f"{va['win_rate']*100:>4.0f}%") + + +if __name__ == "__main__": + import sys + # modalita' test: 100 agenti random + rng = random.Random(42) + specs = [random_spec(rng) for _ in range(100)] + payload = run_tournament(specs, seed=42) + leaderboard(payload) diff --git a/scripts/games/engine.py b/scripts/games/engine.py new file mode 100644 index 0000000..1196672 --- /dev/null +++ b/scripts/games/engine.py @@ -0,0 +1,323 @@ +""" +Game engine — "Blind Traders" tournament. + +100 agenti ricevono due serie anonime (A, B) — in realta' BTC e ETH 1h — e +propongono strategie senza sapere cosa sono. L'orchestratore (questo motore) +valuta ogni strategia con un backtest deterministico, causale e fee-aware, e +assegna un punteggio su %win + PNL con vincolo >=10 trade/mese. + +Tutto causale (nessun look-ahead): i segnali alla barra i usano solo dati +fino a close[i]; l'ingresso e' a close[i], le uscite TP/SL/max_bars intrabar +dalle barre successive. +""" +from __future__ import annotations + +import numpy as np +import pandas as pd + +from src.data.downloader import load_data + +FEE_RT = 0.001 # 0.10% round-trip (taker Deribit, baseline progetto) +TF_BPM = {"5m": 12 * 24 * 30, "15m": 4 * 24 * 30, "1h": 24 * 30} # barre/mese per tf +MIN_TRADES_PER_MONTH = 10.0 + +# Slippage per LATO (oltre alle fee). 0 = come prima. Single-leg paga 2 lati +# (ingresso+uscita), i pairs ne pagano 4 (2 gambe x 2 lati). +_SLIP = 0.0 + + +def set_slippage(slip_per_side: float): + global _SLIP + _SLIP = float(slip_per_side) + + +# -------------------------------------------------------------------------- +# Dati anonimizzati +# -------------------------------------------------------------------------- +def load_anon(tf: str = "1h"): + """Carica BTC->A, ETH->B allineati sull'intersezione temporale. + + Ritorna un dict con array OHLC per A e B + datetime. I nomi reali NON + compaiono: gli agenti vedono solo 'A' e 'B'. + """ + btc = load_data("BTC", tf).copy() + eth = load_data("ETH", tf).copy() + for d in (btc, eth): + d["dt"] = pd.to_datetime(d["datetime"]) + btc = btc.set_index("dt") + eth = eth.set_index("dt") + idx = btc.index.intersection(eth.index) + btc = btc.loc[idx].sort_index() + eth = eth.loc[idx].sort_index() + out = {"dt": idx.to_numpy()} + for name, d in (("A", btc), ("B", eth)): + out[name] = { + "open": d["open"].to_numpy(float), + "high": d["high"].to_numpy(float), + "low": d["low"].to_numpy(float), + "close": d["close"].to_numpy(float), + "volume": d["volume"].to_numpy(float), + } + out["n"] = len(idx) + out["tf"] = tf + out["bpm"] = TF_BPM[tf] + return out + + +# -------------------------------------------------------------------------- +# Indicatori causali (vettorizzati) +# -------------------------------------------------------------------------- +def _roll_mean(x, w): + return pd.Series(x).rolling(w).mean().to_numpy() + + +def _roll_std(x, w): + return pd.Series(x).rolling(w).std(ddof=0).to_numpy() + + +def _ema(x, w): + return pd.Series(x).ewm(span=w, adjust=False).mean().to_numpy() + + +def _atr(high, low, close, w=14): + pc = np.roll(close, 1) + pc[0] = close[0] + tr = np.maximum(high - low, np.maximum(np.abs(high - pc), np.abs(low - pc))) + return pd.Series(tr).rolling(w).mean().to_numpy() + + +def _rsi(close, w=14): + d = np.diff(close, prepend=close[0]) + up = np.where(d > 0, d, 0.0) + dn = np.where(d < 0, -d, 0.0) + ru = pd.Series(up).ewm(alpha=1 / w, adjust=False).mean().to_numpy() + rd = pd.Series(dn).ewm(alpha=1 / w, adjust=False).mean().to_numpy() + rs = ru / (rd + 1e-12) + return 100 - 100 / (1 + rs) + + +# -------------------------------------------------------------------------- +# Famiglie di segnale -> array di posizione desiderata {-1,0,+1} alla barra i +# (causale: usa solo dati fino a close[i]). +1 = long, -1 = short. +# -------------------------------------------------------------------------- +def _signal_single(o, family, p): + """Segnale per una singola serie. Ritorna (pos_target, atr).""" + close = o["close"] + high, low = o["high"], o["low"] + n = len(close) + atr = _atr(high, low, close, 14) + pos = np.zeros(n) + lb = max(2, int(p["lookback"])) + thr = float(p["entry_thr"]) + sign = 1 if p.get("direction", "reversion") == "trend" else -1 + + if family == "zscore": + ma = _roll_mean(close, lb) + sd = _roll_std(close, lb) + z = (close - ma) / (sd + 1e-12) + pos = np.where(z > thr, sign * -1.0, np.where(z < -thr, sign * 1.0, 0.0)) + elif family == "breakout": + hh = pd.Series(high).rolling(lb).max().shift(1).to_numpy() + ll = pd.Series(low).rolling(lb).min().shift(1).to_numpy() + up = close > hh + dn = close < ll + # trend: break-up=long ; reversion: break-up=short + pos = np.where(up, sign * 1.0, np.where(dn, sign * -1.0, 0.0)) + elif family == "ma_cross": + fast = _ema(close, lb) + slow = _ema(close, max(lb + 2, int(lb * p.get("slow_mult", 3)))) + pos = np.where(fast > slow, sign * 1.0, sign * -1.0) + elif family == "rsi": + r = _rsi(close, lb) + hi = 50 + thr * 10 + lo = 50 - thr * 10 + pos = np.where(r > hi, sign * -1.0, np.where(r < lo, sign * 1.0, 0.0)) + elif family == "momentum": + ret = close / np.roll(close, lb) - 1 + ret[:lb] = 0 + pos = np.where(ret > thr / 100, sign * 1.0, + np.where(ret < -thr / 100, sign * -1.0, 0.0)) + else: + raise ValueError(f"unknown family {family}") + pos = np.nan_to_num(pos) + return pos, atr + + +# -------------------------------------------------------------------------- +# Backtest single-series (long/short con TP/SL/max_bars intrabar) +# -------------------------------------------------------------------------- +def _backtest_single(o, pos, atr, p, fee=FEE_RT): + close, high, low = o["close"], o["high"], o["low"] + n = len(close) + tp_atr = float(p.get("tp_atr", 2.0)) + sl_atr = float(p.get("sl_atr", 2.0)) + max_bars = int(p.get("max_bars", 24)) + rets = [] # net return per trade + # warmup + start = max(int(p["lookback"]) + 15, 20) + # indici candidati: solo barre con segnale != 0 (salta le barre flat) + cand = np.flatnonzero(pos[start:n - 1]) + start + ci = 0 + nc = len(cand) + while ci < nc: + i = int(cand[ci]) + d = pos[i] + if d == 0 or np.isnan(atr[i]) or atr[i] <= 0: + ci += 1 + continue + entry = close[i] + a = atr[i] + if d > 0: + tp = entry + tp_atr * a + sl = entry - sl_atr * a + else: + tp = entry - tp_atr * a + sl = entry + sl_atr * a + exit_px = None + j = i + 1 + end = min(n - 1, i + max_bars) + while j <= end: + hi, lo = high[j], low[j] + if d > 0: + if lo <= sl: # SL prioritario + exit_px = sl + break + if hi >= tp: + exit_px = tp + break + else: + if hi >= sl: + exit_px = sl + break + if lo <= tp: + exit_px = tp + break + j += 1 + if exit_px is None: + exit_px = close[end] + j = end + gross = d * (exit_px - entry) / entry + net = gross - fee - 2 * _SLIP # 2 lati di slippage + rets.append(net) + # salta al primo ingresso candidato OLTRE l'uscita (no overlap) + ci = int(np.searchsorted(cand, j + 1, side="left")) + return np.array(rets) + + +# -------------------------------------------------------------------------- +# Backtest cross-series (pairs market-neutral sullo z del log-ratio) +# -------------------------------------------------------------------------- +def _backtest_pairs(A, B, p, fee=FEE_RT): + a, b = A["close"], B["close"] + n = len(a) + lb = max(5, int(p["lookback"])) + z_in = float(p["entry_thr"]) + z_exit = float(p.get("exit_thr", 0.5)) + max_bars = int(p.get("max_bars", 72)) + lr = np.log(a / b) + ma = _roll_mean(lr, lb) + sd = _roll_std(lr, lb) + z = (lr - ma) / (sd + 1e-12) + rets = [] + start = max(lb + 5, 20) + zabs = np.abs(z) + zabs[:start] = 0.0 + zabs[np.isnan(zabs)] = 0.0 + cand = np.flatnonzero(zabs[:n - 1] > z_in) + ci = 0 + nc = len(cand) + while ci < nc: + i = int(cand[ci]) + d = -1 if z[i] > z_in else 1 # spread alto -> short A/long B ; basso -> long A/short B + ea, eb = a[i], b[i] + j = i + 1 + end = min(n - 1, i + max_bars) + while j <= end: + if abs(z[j]) <= z_exit: + break + j += 1 + j = min(j, end) + # PnL = gamba A (dir d) + gamba B (dir -d), fee su 2 gambe + ra = d * (a[j] - ea) / ea + rb = -d * (b[j] - eb) / eb + net = ra + rb - 2 * fee - 4 * _SLIP # 2 gambe x 2 lati di slippage + rets.append(net) + ci = int(np.searchsorted(cand, j + 1, side="left")) + return np.array(rets) + + +# -------------------------------------------------------------------------- +# Valutazione + scoring +# -------------------------------------------------------------------------- +def evaluate(data, spec, sl=None, fee=FEE_RT): + """Valuta una spec di strategia su uno slice [start,end) (sl=slice di indici). + + spec = {family, series, params{...}}. Ritorna dict metriche. + """ + family = spec["family"] + series = spec.get("series", "A") + p = spec["params"] + + def _slice(o): + if sl is None: + return o + s, e = sl + return {k: v[s:e] for k, v in o.items()} + + if family == "pairs": + A = _slice(data["A"]) + B = _slice(data["B"]) + rets = _backtest_pairs(A, B, p, fee) + nbars = len(A["close"]) + else: + o = _slice(data[series]) + pos, atr = _signal_single(o, family, p) + rets = _backtest_single(o, pos, atr, p, fee) + nbars = len(o["close"]) + + n_tr = len(rets) + months = nbars / data.get("bpm", TF_BPM["1h"]) + tpm = n_tr / months if months > 0 else 0.0 + if n_tr == 0: + return dict(n_trades=0, win_rate=0.0, pnl_pct=0.0, tpm=0.0, + sharpe=0.0, avg_ret=0.0, qualified=False, fitness=-1e6) + win_rate = float(np.mean(rets > 0)) + pnl = float(np.sum(rets)) * 100 # PnL additivo (notional fisso), % + equity = float(np.prod(1 + rets) - 1) * 100 # equity compounding, % + avg = float(np.mean(rets)) * 100 + sharpe = float(np.mean(rets) / (np.std(rets) + 1e-12) * np.sqrt(tpm * 12)) \ + if np.std(rets) > 0 else 0.0 + qualified = tpm >= MIN_TRADES_PER_MONTH + # fitness: PNL domina, win% come spinta secondaria; squalifica se pochi trade + fitness = pnl + 50.0 * win_rate + if not qualified: + fitness = -1e6 + pnl # ordinati ma fuori gioco + return dict(n_trades=n_tr, win_rate=win_rate, pnl_pct=pnl, equity_pct=equity, + tpm=tpm, sharpe=sharpe, avg_ret=avg, qualified=qualified, + fitness=fitness) + + +# Split a 3: TRAIN (hill-climb) / VALID (cull+rank dell'orchestratore) / TEST (OOS puro) +def splits3(data, train_frac=0.60, valid_frac=0.20): + n = data["n"] + c1 = int(n * train_frac) + c2 = int(n * (train_frac + valid_frac)) + return (0, c1), (c1, c2), (c2, n) + + +# compat: split a 2 (train/oos) +def splits(data, train_frac=0.70): + n = data["n"] + cut = int(n * train_frac) + return (0, cut), (cut, n) + + +if __name__ == "__main__": + data = load_anon("1h") + print("loaded", data["n"], "bars,", data["dt"][0], "->", data["dt"][-1]) + tr, oos = splits(data) + demo = {"family": "zscore", "series": "B", + "params": {"lookback": 20, "entry_thr": 2.0, "direction": "reversion", + "tp_atr": 1.5, "sl_atr": 2.0, "max_bars": 24}} + print("TRAIN", evaluate(data, demo, tr)) + print("OOS ", evaluate(data, demo, oos)) diff --git a/scripts/games/run_game.py b/scripts/games/run_game.py new file mode 100644 index 0000000..68e1bfa --- /dev/null +++ b/scripts/games/run_game.py @@ -0,0 +1,83 @@ +""" +run_game — carica le 100 strategie proposte dagli agenti ciechi (file in +data/games/specs/agent_*.json), lancia il torneo (epoche + cull) e stampa la +classifica finale, poi RIVELA cosa erano X e Y. + +Se mancano agenti (file assenti o malformati) riempie con spec casuali, cosi' +il gioco gira sempre a 100 concorrenti. +""" +from __future__ import annotations + +import json +import os +import random +from pathlib import Path + +from scripts.games import engine +from scripts.games.arena import random_spec, run_tournament, leaderboard, _normalize + +SPECS_DIR = Path("data/games/specs") +N = 100 + + +def load_specs(): + rng = random.Random(123) + specs, briefs, sources = [], [], [] + for i in range(N): + f = SPECS_DIR / f"agent_{i}.json" + spec = None + if f.exists(): + try: + raw = json.loads(f.read_text()) + fam = raw.get("family") + params = dict(raw.get("params", {})) + if "direction" in raw and "direction" not in params: + params["direction"] = raw["direction"] + spec = {"family": fam, "series": raw.get("series", "A"), + "tf": raw.get("tf", "1h"), "params": params} + # X->A, Y->B mapping (gli agenti vedono X/Y) + s = spec["series"] + spec["series"] = {"X": "A", "Y": "B", "AB": "AB", + "A": "A", "B": "B"}.get(s, "A") + spec = _normalize(spec) + briefs.append(str(raw.get("hypothesis", ""))[:300]) + sources.append("agent") + except Exception as e: + spec = None + if spec is None: + spec = random_spec(rng) + briefs.append("(spec mancante -> sostituto casuale)") + sources.append("random") + specs.append(spec) + n_agent = sources.count("agent") + print(f"caricati {n_agent}/{N} spec da agenti reali, " + f"{N - n_agent} sostituiti casuali") + return specs, briefs + + +def main(): + slip = float(os.environ.get("GAME_SLIP", "0.0")) + engine.set_slippage(slip) + if slip > 0: + print(f"SLIPPAGE attivo: {slip*100:.3f}%/lato " + f"(single-leg {2*slip*100:.2f}% RT extra, pairs {4*slip*100:.2f}% extra)") + specs, briefs = load_specs() + payload = run_tournament(specs, briefs=briefs, seed=2026, + epochs=90, cull_every=10, cull_n=10) + leaderboard(payload, top=10) + rev = payload["reveal"] + print(f"\n>>> RIVELAZIONE: Serie X = {rev['A']}, Serie Y = {rev['B']} " + f"(timeframe base {rev['tf']}). Gli agenti non lo sapevano. <<<") + # vincitore + w = payload["results"][0] + sp = w["spec"] + print(f"\nVINCITORE: agente #{w['agent']} su {w['tf']} | {sp['family']} " + f"{sp['series']} {sp['params'].get('direction','')}") + print(f" ipotesi dell'agente: {w['brief']}") + print(f" TEST(OOS): PnL {w['test']['pnl_pct']:.0f}% | win " + f"{w['test']['win_rate']*100:.0f}% | {w['test']['tpm']:.1f} trade/mese " + f"| Sharpe {w['test']['sharpe']:.1f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/portfolios/_defs.py b/scripts/portfolios/_defs.py index df76c41..54282d0 100644 --- a/scripts/portfolios/_defs.py +++ b/scripts/portfolios/_defs.py @@ -79,6 +79,13 @@ HONEST = [ ] PAIRS = [ SleeveSpec(kind="pairs", name="PR01", sid="PR_ETHBTC", a="ETH", b="BTC", cluster="ETH-rev"), + # BLEND timeframe: ETH/BTC anche a 15m (flat-skip), accanto al 1h. Decorrelato 0.37 dal + # 1h -> diversificatore intra-pairs. Worker validato (validate_worker_pairs 15m, replay + # == pairs_sim_flat). Gate PORT06: docs/diary/2026-06-09-pairs15m-live-path.md. + SleeveSpec(kind="pairs", name="PR01", sid="PR_ETHBTC_15M", a="ETH", b="BTC", tf="15m", + params={"n": 66, "z_in": 1.674, "z_exit": 1.0, "max_bars": 35, "flat_skip": True, + "position_size": 0.10}, # meta' del family PAIRS (0.20): blend-tilt + cluster="ETH-rev"), SleeveSpec(kind="pairs", name="PR01", sid="PR_LTCETH", a="LTC", b="ETH", cluster="ETH-rev"), SleeveSpec(kind="pairs", name="PR01", sid="PR_ADAETH", a="ADA", b="ETH", cluster="ETH-rev"), SleeveSpec(kind="pairs", name="PR01", sid="PR_BTCLTC", a="BTC", b="LTC", cluster="BTC-rev"), diff --git a/src/live/pairs_worker.py b/src/live/pairs_worker.py index ea68409..70fbafa 100644 --- a/src/live/pairs_worker.py +++ b/src/live/pairs_worker.py @@ -51,6 +51,11 @@ class PairsWorker: self.z_exit = float(p.get("z_exit", 0.75)) self.max_bars = int(p.get("max_bars", 72)) self.jump_max = float(p.get("jump_max", 0.08)) + # flat-skip (timeframe sub-orari, es. 15m): non entrare/uscire su candele flat + # (O=H=L=C, prezzo stale/liquidita' zero -> fill non eseguibile). LIVE-REALIZABLE: + # l'uscita arma exit_ready e si esegue alla prima barra PULITA. Parita' col backtest + # pairs_research.pairs_sim_flat(flat_skip=True). Default off = comportamento 1h storico. + self.flat_skip = bool(p.get("flat_skip", False)) self.initial_capital = capital self.position_size = position_size @@ -71,6 +76,7 @@ class PairsWorker: self.entry_z = 0.0 self.entry_time = "" self.bars_held = 0 + self.exit_ready = False # flat-skip: condizione di uscita armata, attende barra pulita self.total_trades = 0 self.total_wins = 0 self.last_bar_ts = 0 @@ -117,6 +123,7 @@ class PairsWorker: self.entry_z = s.get("entry_z", 0.0) self.entry_time = s.get("entry_time", "") self.bars_held = s.get("bars_held", 0) + self.exit_ready = s.get("exit_ready", False) self.total_trades = s.get("total_trades", 0) self.total_wins = s.get("total_wins", 0) self.last_bar_ts = s.get("last_bar_ts", 0) @@ -145,7 +152,8 @@ class PairsWorker: "capital": round(self.capital, 2), "in_position": self.in_position, "direction": self.direction, "entry_a": self.entry_a, "entry_b": self.entry_b, "entry_z": round(self.entry_z, 4), "entry_time": self.entry_time, - "bars_held": self.bars_held, "total_trades": self.total_trades, + "bars_held": self.bars_held, "exit_ready": self.exit_ready, + "total_trades": self.total_trades, "total_wins": self.total_wins, "last_bar_ts": self.last_bar_ts, "started_at": self.started_at, "last_update": datetime.now(timezone.utc).isoformat(), "real_capital": round(self.real_capital, 4), "real_in_position": self.real_in_position, @@ -185,6 +193,7 @@ class PairsWorker: self.entry_a, self.entry_b, self.entry_z = ca, cb, z self.entry_time = datetime.now(timezone.utc).isoformat() self.bars_held = 0 + self.exit_ready = False data = {"direction": "long_ratio" if d == 1 else "short_ratio", "long_leg": self.asset_a if d == 1 else self.asset_b, "short_leg": self.asset_b if d == 1 else self.asset_a, @@ -287,9 +296,13 @@ class PairsWorker: """Chiamato ad ogni poll con gli OHLCV aggiornati delle due gambe.""" if df_a is None or df_b is None or df_a.empty or df_b.empty: return - m = df_a[["timestamp", "close"]].rename(columns={"close": "ca"}).merge( - df_b[["timestamp", "close"]].rename(columns={"close": "cb"}), on="timestamp", how="inner" - ).sort_values("timestamp").reset_index(drop=True) + # merge OHLC quando disponibile (serve a rilevare le candele flat per il flat-skip); + # se le colonne OHLC mancano, flat resta False -> comportamento close-only invariato. + ohlc = ["open", "high", "low", "close"] + keep_a = ["timestamp"] + [c for c in ohlc if c in df_a.columns] + keep_b = ["timestamp"] + [c for c in ohlc if c in df_b.columns] + m = df_a[keep_a].merge(df_b[keep_b], on="timestamp", how="inner", + suffixes=("_a", "_b")).sort_values("timestamp").reset_index(drop=True) # Scarta la barra IN FORMAZIONE: entry ED exit valutati SOLO sul close di # barra COMPLETA, come il backtest (pairs_research: close settled) — # lezione EXIT-16. Detection condivisa: src.live.bars. @@ -298,7 +311,7 @@ class PairsWorker: m = m.iloc[:-1] if len(m) < self.n + 2: return - ca, cb = m["ca"].values, m["cb"].values + ca, cb = m["close_a"].values, m["close_b"].values z, dr = self._zscore(ca, cb) i = len(m) - 1 cur_ts = int(m["timestamp"].iloc[i]) @@ -306,19 +319,29 @@ class PairsWorker: if np.isnan(zi): self._save_state(); return + # flat della barra corrente (entrambe le gambe): O=H=L=C in una delle due + flat_i = False + if self.flat_skip and {"open_a", "high_a", "low_a"}.issubset(m.columns) \ + and {"open_b", "high_b", "low_b"}.issubset(m.columns): + fa = (m["open_a"].iloc[i] == m["high_a"].iloc[i] == m["low_a"].iloc[i] == ca[i]) + fb = (m["open_b"].iloc[i] == m["high_b"].iloc[i] == m["low_b"].iloc[i] == cb[i]) + flat_i = bool(fa or fb) + if self.in_position: if cur_ts > self.last_bar_ts: self.bars_held += 1 self.last_bar_ts = cur_ts - if abs(zi) <= self.z_exit: - self._close(float(ca[i]), float(cb[i]), float(zi), "mean_revert") - elif self.bars_held >= self.max_bars: - self._close(float(ca[i]), float(cb[i]), float(zi), "time_limit") + # arma l'uscita: |z|<=z_exit (rientro) o time-limit; poi esegui alla 1a barra pulita + if not self.exit_ready and (abs(zi) <= self.z_exit or self.bars_held >= self.max_bars): + self.exit_ready = True + if self.exit_ready and not flat_i: + reason = "mean_revert" if abs(zi) <= self.z_exit else "time_limit" + self._close(float(ca[i]), float(cb[i]), float(zi), reason) self._save_state() return - # flat: cerca ingresso (no look-ahead: z[i] usa solo dati <= i) - if dr[i] <= self.jump_max: + # cerca ingresso (no look-ahead: z[i] usa solo dati <= i); mai su barra stale + if dr[i] <= self.jump_max and not flat_i: if zi <= -self.z_in: self._open(1, float(ca[i]), float(cb[i]), float(zi)); self.last_bar_ts = cur_ts elif zi >= self.z_in: diff --git a/src/portfolio/runner.py b/src/portfolio/runner.py index 0d8ded9..3a6f72c 100644 --- a/src/portfolio/runner.py +++ b/src/portfolio/runner.py @@ -34,16 +34,22 @@ _MULTI_KINDS = ("basket", "rotation", "tsmom") DATA_DIR = Path("data/portfolio_paper") # giorni di storia da fetchare per timeframe (TSM01 1d usa 252 barre -> ~440 giorni col buffer) -_LOOKBACK_DAYS = {"1h": 90, "4h": 220, "1d": 440} +_LOOKBACK_DAYS = {"5m": 7, "15m": 14, "30m": 21, "1h": 90, "4h": 220, "1d": 440} +# timeframe SUB-orari: si fetchano DIRETTI da Cerbero (non resamplabili dal 1h). +_SUBHOURLY = {"5m", "15m", "30m"} # SH01 (ml) richiede >=4000 barre 1h (train_min di ml_wf_entries); 365g (~8760 barre) danno # margine ampio per il walk-forward. Difensivo: non dipende dal fetch 440g di TSM01/ROT02. _ML_LOOKBACK_DAYS = 365 -def pos_for_spec(sid: str, global_ps: float, family_overrides: dict[str, float]) -> float: - """position_size effettivo di uno sleeve: override per-famiglia (chiave = - weighting.family_of: PAIRS/FADE/HONEST/SHAPE/TSM) o globale.""" +def pos_for_spec(sid: str, global_ps: float, family_overrides: dict[str, float], + sleeve_ps: float | None = None) -> float: + """position_size effettivo di uno sleeve. Precedenza: override PER-SLEEVE + (spec.params['position_size'], es. il 15m a 0.10) > override per-FAMIGLIA + (weighting.family_of: PAIRS/FADE/...) > globale.""" from src.portfolio.weighting import family_of + if sleeve_ps is not None: + return float(sleeve_ps) return family_overrides.get(family_of(sid), global_ps) @@ -301,7 +307,7 @@ def run(config_path: str = "portfolios.yml"): ex, inst = _exec_for(s) pex, pinst = _pairs_exec_for(s) workers[s.sid] = build_worker_for(s, alloc[s.sid], p.leverage, - position_size=pos_for_spec(s.sid, position_size, ps_family), + position_size=pos_for_spec(s.sid, position_size, ps_family, s.params.get("position_size")), executor=ex, exec_instrument=inst, pairs_executor=pex, exec_instruments=pinst) if ps_family: @@ -312,7 +318,7 @@ def run(config_path: str = "portfolios.yml"): paper_dir = DATA_DIR.parent / "portfolio_paper_stats" paper_workers = {s.sid: build_worker_for(s, paper_notional, p.leverage, data_dir=paper_dir, - position_size=pos_for_spec(s.sid, position_size, ps_family)) + position_size=pos_for_spec(s.sid, position_size, ps_family, s.params.get("position_size"))) for s in paper_specs} # bootstrap storia full per gli sleeve ML (SH01): parquet locale + feed live. @@ -340,6 +346,18 @@ def run(config_path: str = "portfolios.yml"): for a in assets: asset_days[a] = max(asset_days.get(a, 0), days) + # timeframe SUB-orari (es. pairs 15m, flat-skip): non resamplabili dal 1h -> + # fetch DIRETTO da Cerbero per (asset, tf). Inerte se nessuno sleeve e' sub-orario. + subhourly_needs: dict[tuple[str, str], int] = {} + for s in supported: # live + paper + assets, tf = _spec_assets_tf(s) + if tf in _SUBHOURLY: + for a in assets: + subhourly_needs[(a, tf)] = max(subhourly_needs.get((a, tf), 0), + _LOOKBACK_DAYS.get(tf, 14)) + if subhourly_needs: + print(f"[runner] timeframe sub-orari (fetch diretto Cerbero): {sorted(subhourly_needs)}") + inst_map = dict(INSTRUMENT_MAP) last_day = "" stale_alerted: set[str] = set() # asset con alert STALE_FEED attivo (dedup per episodio) @@ -394,12 +412,30 @@ def run(config_path: str = "portfolios.yml"): raw1h[asset] = df.sort_values("timestamp").reset_index(drop=True) _check_stale_feed(asset, raw1h[asset], stale_alerted) - # tick di ogni worker col suo timeframe (resample dal 1h) + # fetch DIRETTO dei timeframe sub-orari (15m...) per (asset, tf) + raw_sub: dict[tuple[str, str], pd.DataFrame] = {} + for (asset, tf), days in subhourly_needs.items(): + inst = inst_map.get(asset, f"{asset}-PERPETUAL") + start = end - timedelta(days=days) + candles = client.get_historical_v2(inst, start.strftime("%Y-%m-%d"), + end.strftime("%Y-%m-%d"), tf) + if candles: + df = pd.DataFrame(candles) + df["timestamp"] = df["timestamp"].astype("int64") + raw_sub[(asset, tf)] = df.sort_values("timestamp").reset_index(drop=True) + + def _series_for(a, tf): + """Serie OHLC per (asset, tf): diretta se sub-oraria, altrimenti resample dal 1h.""" + if tf in _SUBHOURLY: + return raw_sub.get((a, tf)) + return _resample(raw1h[a], tf) if a in raw1h else None + + # tick di ogni worker col suo timeframe def _tick(s, w): assets, tf = _spec_assets_tf(s) - if any(a not in raw1h for a in assets): + res = {a: _series_for(a, tf) for a in assets} + if any(res[a] is None or len(res[a]) == 0 for a in assets): return - res = {a: _resample(raw1h[a], tf) for a in assets} if s.kind == "pairs": w.tick(res[s.a], res[s.b]) elif s.kind in _MULTI_KINDS: diff --git a/tests/portfolio/test_backtest_parity_cap.py b/tests/portfolio/test_backtest_parity_cap.py index 337e763..56f7039 100644 --- a/tests/portfolio/test_backtest_parity_cap.py +++ b/tests/portfolio/test_backtest_parity_cap.py @@ -8,6 +8,9 @@ def test_port06_cap_backtest_numbers_locked(): # Aggiornato 2026-05-31: il recupero dati BNB/DOGE/XRP (29 mag) ha ampliato la # copertura storica -> metriche migliorate (Sharpe 6.07->6.47, OOS 8.19->8.82, # DD 4.9%->4.1%). Nuovo baseline atteso, non una regressione. - assert r.full["sharpe"] == pytest.approx(6.47, abs=0.15) - assert r.oos["sharpe"] == pytest.approx(8.82, abs=0.25) - assert r.full["dd"] == pytest.approx(4.1, abs=0.5) + # Aggiornato 2026-06-09: aggiunto lo sleeve BLEND PR_ETHBTC_15M (ETH/BTC pairs 15m + # flat-skip, mezza size) -> miglioria attesa: FULL 6.47->7.20, OOS 8.82->9.66, + # DD 4.1%->3.7%. Vedi docs/diary/2026-06-09-pairs15m-live-path.md. + assert r.full["sharpe"] == pytest.approx(7.20, abs=0.15) + assert r.oos["sharpe"] == pytest.approx(9.66, abs=0.25) + assert r.full["dd"] == pytest.approx(3.68, abs=0.5) diff --git a/tests/portfolio/test_definitions.py b/tests/portfolio/test_definitions.py index 1ebb9ef..39021bc 100644 --- a/tests/portfolio/test_definitions.py +++ b/tests/portfolio/test_definitions.py @@ -8,8 +8,9 @@ def test_six_portfolios_defined(): def test_port06_is_master_shape_cap(): p = PORTFOLIOS["PORT06"] sids = set(p.sleeve_ids) - assert {"SH_BTC", "SH_ETH", "TSM01", "PR_ETHBTC"} <= sids - assert len(sids) == 17 + assert {"SH_BTC", "SH_ETH", "TSM01", "PR_ETHBTC", "PR_ETHBTC_15M"} <= sids + # 18 dal 2026-06-09: aggiunto lo sleeve BLEND PR_ETHBTC_15M (ETH/BTC pairs 15m flat-skip) + assert len(sids) == 18 # SHAPE cappata a 0.0588 (2026-06-05): SH01 senza SL by-design, esposizione dimezzata # (ricerca sh01_exit_lab: 11 famiglie di stop, 0 sopravvissute) assert p.weighting == "cap" and p.caps == {"PAIRS": 0.33, "SHAPE": 0.0588}