diff --git a/portfolios.yml b/portfolios.yml index 9cabc6b..5cfb431 100644 --- a/portfolios.yml +++ b/portfolios.yml @@ -35,6 +35,23 @@ overrides: # Override per-famiglia: irrilevante per il conto reale (i pairs sono PAPER), tenuto # solo perche' i worker pairs in sola-statistica dimensionino come da gate storico. position_size_family: {PAIRS: 0.13} + # PAPER_EXTRA (2026-06-18): sleeve paper definiti SOLO qui (NON in _defs.py/PORT06) -> + # NON entrano nel backtest canonico/regression-lock. Shadow STAGE 1 del Price Ladder: + # GridWorker SIM-only su feed Deribit BTC 1h (NESSUN ordine reale; kind=grid non e' mai + # eseguito reale per costruzione). Config = re-gate su dati puliti (branch + # price_ladder_research): regime-gate range trend_max 1.5, rd0.20/ru0.06, 6 livelli, + # sl0.10/tp0.03. position_size 0.15 PINNATO (canonico validato; senza, erediterebbe il + # 0.5 globale del micro-test). Gira in data/portfolio_paper_stats/GRID_BTC/. + paper_extra: + - sid: GRID_BTC + kind: grid + name: GRID + asset: BTC + tf: "1h" + cluster: BTC-rev + params: {tf: "1h", range_down: 0.20, range_up: 0.06, levels: 6, + sl_buf: 0.10, tp_buf: 0.03, max_bars: 720, + regime: range, trend_max: 1.5, position_size: 0.15} # Esecuzione REALE su Deribit MAINNET. Solo i 7 single-leg con TP/SL in metadata: # 6 fade (MR01/MR02/MR07 x BTC/ETH 15m) + DIP01 (BTC 1h). Ordini sui LINEARI USDC # (payoff lineare = matematica del backtest; fee/PnL in USDC). diff --git a/scripts/analysis/grid_game_gate.py b/scripts/analysis/grid_game_gate.py index cd0191d..26855b4 100644 --- a/scripts/analysis/grid_game_gate.py +++ b/scripts/analysis/grid_game_gate.py @@ -85,7 +85,8 @@ def grid_mtm(asset="ETH", *, tf, range_down, range_up, levels, sl_buf, tp_buf, hi = df["high"].to_numpy(float) lo = df["low"].to_numpy(float) cl = df["close"].to_numpy(float) - dt = pd.to_datetime(df["datetime"]).to_numpy() + dt = (pd.to_datetime(df["datetime"]) if "datetime" in df.columns + else pd.to_datetime(df["timestamp"], unit="ms", utc=True)).to_numpy() n = len(cl) ratio = ((1 + range_up) / (1 - range_down)) ** (1.0 / levels) if ratio - 1 <= 1.5 * 2 * fee_side: # vincolo break-even §4 diff --git a/scripts/analysis/validate_grid_worker.py b/scripts/analysis/validate_grid_worker.py index 2e90215..7c7fec9 100644 --- a/scripts/analysis/validate_grid_worker.py +++ b/scripts/analysis/validate_grid_worker.py @@ -1,9 +1,11 @@ -"""VALIDA GridWorker — replay live == backtest grid_mtm (gate obbligatorio del progetto). +"""VALIDA GridWorker — forward-tracking == backtest grid_mtm (gate obbligatorio del progetto). -Confronta il GridWorker (sim/paper, src/live/grid_worker.py) col motore canonico grid_mtm: - [A] un tick con tutta la storia -> capital == initial * grid_mtm(full).equity[-1] (parita'); - [B] replay INCREMENTALE (tick su finestra che cresce) -> converge allo stesso capitale finale; - [C] resume: reistanzia il worker (rilegge status.json) -> capitale persistito. +Il GridWorker (sim/paper) bootstrappa la storia FULL (start fisso) e mappa il capitale forward +dal deploy: capital = initial * eq[-1]/base_norm. Proprieta' da validare: + [A] LOGICA: la griglia sul feed full == backtest (stesso n_trades/win). + [B] FORWARD: deployato a T0 (bootstrap up to T0), dopo aver ticcato fino a T1 il capitale + e' initial * eq_full(T1)/eq_full(T0) — cioe' il ritorno della griglia DA T0 (causale). + [C] RESUME: reistanziato (rilegge base_norm da status.json) -> stesso capitale. Config = la finale del re-gate pulito: BTC 1h range1.5 rd0.20 ru0.06 L6 sl0.10 tp0.03. uv run python scripts/analysis/validate_grid_worker.py @@ -28,50 +30,58 @@ CFG = dict(tf="1h", range_down=0.20, range_up=0.06, levels=6, ASSET, INIT, POS, LEV, FEE = "BTC", 1000.0, 0.15, 3.0, 0.0005 +def _eq_last(df, mask): + cfg_bt = {k: v for k, v in CFG.items() if k not in ("regime", "trend_max")} + eqd, st = grid_mtm(ASSET, **cfg_bt, pos=POS, lev=LEV, fee_side=FEE, deploy_mask=mask, df=df) + return float(eqd.iloc[-1]), st + + def main(): df = load_data(ASSET, "1h") - # backtest canonico - mask = regime_mask(ASSET, "1h", trend_max=CFG["trend_max"]) - cfg_bt = {k: v for k, v in CFG.items() if k not in ("regime", "trend_max")} - eqd_bt, st_bt = grid_mtm(ASSET, **cfg_bt, pos=POS, lev=LEV, fee_side=FEE, deploy_mask=mask) - target_cap = INIT * float(eqd_bt.iloc[-1]) - print(f"[backtest] grid_mtm equity[-1]={eqd_bt.iloc[-1]:.6f} -> capital {target_cap:,.2f} " - f"(trades {st_bt['trades']}, win {st_bt['win']:.1f}%)") + n = len(df) + nboot = n - 400 # deploy a T0 = nboot, poi 400 barre forward + mask_full = regime_mask(ASSET, "1h", trend_max=CFG["trend_max"]) + + F, st_full = _eq_last(df, mask_full) # eq full @ T1 (== backtest) + B, _ = _eq_last(df.iloc[:nboot].reset_index(drop=True), mask_full[:nboot]) # eq @ T0 (causale) + expected = INIT * F / B + print(f"[backtest] eq@T0={B:.4f} eq@T1={F:.4f} -> capitale forward atteso {expected:,.2f} " + f"(trades full {st_full['trades']}, win {st_full['win']:.1f}%)") wd = Path(tempfile.mkdtemp(prefix="gridval_")) try: - # [A] un tick con tutta la storia + boot = df.iloc[:nboot].reset_index(drop=True) w = GridWorker("GRID_BTC", ASSET, CFG, INIT, wd, leverage=LEV, - position_size=POS, fee_side=FEE) + position_size=POS, fee_side=FEE, hist=boot) + # [A] logica: tick col feed full -> n_trades come backtest w.tick(df) - dA = abs(w.capital - target_cap) - print(f"[A] full-tick capital {w.capital:,.2f} delta {dA:.6f} " - f"{'OK' if dA < 1e-6 else 'MISMATCH'}") + dA = abs(w.n_trades - st_full["trades"]) + print(f"[A] logica n_trades worker {w.n_trades} vs backtest {st_full['trades']} " + f"{'OK' if dA == 0 else 'MISMATCH'}") - # [B] replay incrementale (ultimi 3 tick su finestra crescente) - n = len(df) - caps = [] - for end in (n - 200, n - 50, n): - wd2 = Path(tempfile.mkdtemp(prefix="gridval_inc_")) - wi = GridWorker("GRID_BTC", ASSET, CFG, INIT, wd2, leverage=LEV, - position_size=POS, fee_side=FEE) - wi.tick(df.iloc[:end].reset_index(drop=True)) - caps.append(wi.capital) - shutil.rmtree(wd2, ignore_errors=True) - dB = abs(caps[-1] - target_cap) - print(f"[B] incrementale capitali {[round(c,2) for c in caps]} " - f"finale delta {dB:.6f} {'OK' if dB < 1e-6 else 'MISMATCH'}") + # [B] forward: deploy a T0 (base), poi fino a T1 + wd2 = Path(tempfile.mkdtemp(prefix="gridval_fwd_")) + wf = GridWorker("GRID_BTC", ASSET, CFG, INIT, wd2, leverage=LEV, + position_size=POS, fee_side=FEE, hist=boot) + wf.tick(boot) # deploy: base_norm=B, capital=initial + cap0 = wf.capital + wf.tick(df.iloc[:n - 200].reset_index(drop=True)) + wf.tick(df) # fino a T1 + dB = abs(wf.capital - expected) + print(f"[B] forward: cap@deploy {cap0:,.2f} (atteso {INIT:,.0f}) cap@T1 {wf.capital:,.2f} " + f"(atteso {expected:,.2f}) delta {dB:.4f} {'OK' if dB < 0.05 else 'MISMATCH'}") # [C] resume - w2 = GridWorker("GRID_BTC", ASSET, CFG, INIT, wd, leverage=LEV, - position_size=POS, fee_side=FEE) - dC = abs(w2.capital - w.capital) - # status.json persiste capital a 4 decimali -> tolleranza = precisione di persistenza - print(f"[C] resume capital {w2.capital:,.2f} delta {dC:.6f} " - f"{'OK' if dC < 1e-3 else 'MISMATCH'} (persistenza 4 dec.)") + wr = GridWorker("GRID_BTC", ASSET, CFG, INIT, wd2, leverage=LEV, + position_size=POS, fee_side=FEE, hist=boot) + wr.tick(df) + dC = abs(wr.capital - wf.capital) + print(f"[C] resume cap {wr.capital:,.2f} delta {dC:.4f} " + f"{'OK' if dC < 0.05 else 'MISMATCH'} (base_norm persistito)") - ok = dA < 1e-6 and dB < 1e-6 and dC < 1e-3 - print(f"\n{'VALIDAZIONE OK: GridWorker replay == backtest' if ok else 'VALIDAZIONE FALLITA'}") + ok = dA == 0 and dB < 0.05 and dC < 0.05 + print(f"\n{'VALIDAZIONE OK: GridWorker forward-tracking == backtest' if ok else 'VALIDAZIONE FALLITA'}") + shutil.rmtree(wd2, ignore_errors=True) finally: shutil.rmtree(wd, ignore_errors=True) diff --git a/src/live/grid_worker.py b/src/live/grid_worker.py index d5972ec..6ae3fc7 100644 --- a/src/live/grid_worker.py +++ b/src/live/grid_worker.py @@ -45,7 +45,7 @@ class GridWorker: def __init__(self, sid: str, asset: str, params: dict, capital: float, work_dir: Path, leverage: float = 3.0, position_size: float = 0.15, - fee_side: float = 0.0005, notifier=None): + fee_side: float = 0.0005, notifier=None, hist: pd.DataFrame | None = None): self.sid = sid self.asset = asset self.p = dict(params) # tf,range_down,range_up,levels,sl_buf,tp_buf,max_bars,regime,trend_max @@ -59,6 +59,19 @@ class GridWorker: self.max_dd = 0.0 self.n_trades = 0 self.last_ts = "" + # base_norm = valore dell'equity-norm (cumulata da inizio storia) al DEPLOY: la + # capital forward = initial * eq[-1]/base_norm -> parte da `initial` e segue il + # ritorno della griglia DA QUEL MOMENTO (start FISSO: niente salti da finestra mobile). + self.base_norm = None + # bootstrap STORIA FULL (start fisso, come SH01): il feed live e' una finestra mobile, + # ma normalizzando su una serie a start fisso l'equity forward e' stabile. + if hist is None: + try: + from src.data.downloader import load_data + hist = load_data(asset, self.p.get("tf", "1h")) + except Exception: + hist = None + self.hist = hist self.work_dir = Path(work_dir) self.work_dir.mkdir(parents=True, exist_ok=True) self.status_path = self.work_dir / "status.json" @@ -66,6 +79,17 @@ class GridWorker: self.in_position = False # compat dashboard (la griglia non ha una posizione singola) self._load_state() + def _merge(self, live_df: pd.DataFrame) -> pd.DataFrame: + """Storia bootstrap + feed live, dedup su timestamp (il live prevale), start FISSO.""" + if self.hist is None or len(self.hist) == 0: + return live_df + cols = ["timestamp", "open", "high", "low", "close", "volume"] + h = self.hist[[c for c in cols if c in self.hist.columns]] + l = live_df[[c for c in cols if c in live_df.columns]] + m = pd.concat([h, l], ignore_index=True) + m = m.drop_duplicates(subset="timestamp", keep="last").sort_values("timestamp") + return m.reset_index(drop=True) + def _load_state(self): if not self.status_path.exists(): self._log("INIT", {"capital": round(self.capital, 2), "sid": self.sid}) @@ -76,14 +100,16 @@ class GridWorker: self.max_dd = s.get("max_dd", 0.0) self.n_trades = s.get("n_trades", 0) self.last_ts = s.get("last_ts", "") - self._log("RESUME", {"capital": round(self.capital, 2), "n_trades": self.n_trades}) + self.base_norm = s.get("base_norm") + self._log("RESUME", {"capital": round(self.capital, 2), "n_trades": self.n_trades, + "base_norm": self.base_norm}) def _save_state(self): self.status_path.write_text(json.dumps({ "sid": self.sid, "kind": self.KIND, "asset": self.asset, "capital": round(self.capital, 4), "peak": round(self.peak, 4), "max_dd": round(self.max_dd, 4), "n_trades": self.n_trades, - "in_position": self.in_position, "params": self.p, + "base_norm": self.base_norm, "in_position": self.in_position, "params": self.p, "last_ts": self.last_ts, "ts": datetime.now(timezone.utc).isoformat(), }, indent=2)) @@ -97,28 +123,33 @@ class GridWorker: pass def tick(self, df: pd.DataFrame): - """df = OHLCV live (open/high/low/close[/datetime]) fino ad ora. Ricomputa la griglia - col motore canonico e aggiorna capital = initial * equity_norm. SIM only (no ordini).""" + """df = OHLCV live (finestra mobile) fino ad ora. Merge con la storia bootstrap + (start FISSO), ricomputa la griglia col motore canonico, e mappa il capitale forward: + capital = initial * eq[-1]/base_norm (parte da `initial` al deploy, segue la griglia + da li' in poi). SIM only (nessun ordine reale).""" if df is None or len(df) < 40: return + full = self._merge(df) p = self.p regime = p.get("regime", "none") - mask = (_regime_mask(df, p.get("ema_n", 200), p.get("trend_max", 2.0)) + mask = (_regime_mask(full, p.get("ema_n", 200), p.get("trend_max", 2.0)) if regime == "range" else None) eqd, st = grid_mtm( self.asset, tf=p["tf"], range_down=p["range_down"], range_up=p["range_up"], levels=p["levels"], sl_buf=p["sl_buf"], tp_buf=p["tp_buf"], max_bars=p["max_bars"], pos=self.position_size, lev=self.leverage, fee_side=self.fee_side, - flat_skip=True, deploy_mask=mask, df=df) + flat_skip=True, deploy_mask=mask, df=full) if eqd is None or len(eqd) == 0: return - new_cap = self.initial_capital * float(eqd.iloc[-1]) - self.capital = max(new_cap, 0.0) + cur = float(eqd.iloc[-1]) + if self.base_norm is None or self.base_norm <= 0: + self.base_norm = cur # baseline al primo tick (deploy) + self.capital = max(self.initial_capital * cur / self.base_norm, 0.0) self.peak = max(self.peak, self.capital) if self.peak > 0: self.max_dd = max(self.max_dd, (self.peak - self.capital) / self.peak) self.n_trades = int(st.get("trades", self.n_trades)) - self.last_ts = str(df.iloc[-1].get("datetime", df.iloc[-1].get("timestamp", ""))) + self.last_ts = str(full.iloc[-1].get("timestamp", "")) self._save_state() self._log("GRID_MTM", {"capital": round(self.capital, 2), "n_trades": self.n_trades, "win": st.get("win"), "stops": st.get("stops"), diff --git a/src/portfolio/runner.py b/src/portfolio/runner.py index 36f4bba..5d53abf 100644 --- a/src/portfolio/runner.py +++ b/src/portfolio/runner.py @@ -23,6 +23,7 @@ from src.live.basket_trend_worker import BasketTrendWorker from src.live.rotation_worker import RotationWorker from src.live.tsmom_worker import TsmomWorker from src.live.xsec_worker import CrossSectionalWorker +from src.live.grid_worker import GridWorker from src.live.strategy_loader import load_strategy # Codice-breve sleeve -> nome modulo Strategy in scripts/strategies/ (worker single/ml) @@ -102,6 +103,13 @@ def build_worker_for(spec: SleeveSpec, alloc_capital: float, leverage: float, capital=alloc_capital, position_size=position_size, leverage=leverage, data_dir=data_dir, ) + if spec.kind == "grid": + # Price Ladder (griglia) — SIM/PAPER (shadow stage 1): nessun ordine reale. + return GridWorker( + sid=spec.sid, asset=spec.asset, params=spec.params, capital=alloc_capital, + work_dir=Path(data_dir) / spec.sid, leverage=leverage, + position_size=position_size, fee_side=0.0005, + ) module = _STRAT_MODULE.get(spec.name) if module is None: raise ValueError(f"sleeve live non supportato: {spec.name} (kind={spec.kind})") @@ -191,6 +199,8 @@ def _spec_assets_tf(spec: SleeveSpec): return [spec.a, spec.b], spec.tf if spec.kind in _MULTI_KINDS: return list(spec.params["universe"]), spec.params.get("tf", "1d" if spec.kind != "basket" else "4h") + if spec.kind == "grid": + return [spec.asset], spec.params.get("tf", spec.tf) return [spec.asset], spec.tf @@ -375,6 +385,20 @@ def run(config_path: str = "portfolios.yml"): paper_codes = {str(c).upper() for c in (_ov.get("paper_sleeves") or [])} live_specs = [s for s in supported if s.name.upper() not in paper_codes] paper_specs = [s for s in supported if s.name.upper() in paper_codes] + # PAPER_EXTRA: sleeve paper definiti SOLO in config (NON in p.sleeves) -> NON entrano + # nel backtest canonico/regression-lock (all_sleeve_equities non sa costruirli). Nato + # per il Price Ladder (kind=grid, shadow stage 1 sim). Parsing DIFENSIVO: un errore qui + # non deve crashare il runner mainnet. + for _ex in (_ov.get("paper_extra") or []): + try: + paper_specs.append(SleeveSpec( + kind=str(_ex["kind"]), name=str(_ex.get("name", _ex["sid"])), + sid=str(_ex["sid"]), asset=_ex.get("asset"), + tf=str(_ex.get("tf", "1h")), params=dict(_ex.get("params", {})), + cluster=str(_ex.get("cluster", "")), + )) + except Exception as e: + print(f"[runner] WARN paper_extra ignorato ({_ex}): {e}") if paper_specs: print(f"[runner] sleeve PAPER (solo statistica, fuori dal pool): " f"{[s.sid for s in paper_specs]}") @@ -467,7 +491,7 @@ def run(config_path: str = "portfolios.yml"): # lookback (giorni) richiesto per ogni asset = max sui worker che lo usano asset_days: dict[str, int] = {} - for s in supported: # live + PAPER (anche XS01/TR01/ROT02/TSM01) + for s in live_specs + paper_specs: # live + PAPER (incl. paper_extra grid) assets, tf = _spec_assets_tf(s) days = _LOOKBACK_DAYS.get(tf, 90) if s.kind == "ml": # SH01 ha bisogno di molta storia 1h @@ -478,7 +502,7 @@ def run(config_path: str = "portfolios.yml"): # timeframe SUB-orari (es. pairs 15m, flat-skip): non resamplabili dal 1h -> # fetch DIRETTO da Cerbero per (asset, tf). Inerte se nessuno sleeve e' sub-orario. subhourly_needs: dict[tuple[str, str], int] = {} - for s in supported: # live + paper + for s in live_specs + paper_specs: # live + paper (incl. paper_extra grid) assets, tf = _spec_assets_tf(s) if tf in _SUBHOURLY: for a in assets: @@ -603,6 +627,9 @@ def run(config_path: str = "portfolios.yml"): # interno fitta solo l'ultimo blocco (last_block_only nei params). w.tick(_with_history(ml_hist.get(s.asset), res[s.asset], ml_gap_warned, s.asset)) + elif s.kind == "grid": + # Price Ladder SIM/PAPER: ricomputa la griglia sul feed live (BTC 1h). + w.tick(res[s.asset]) else: # single (fade/dip): StrategyWorker su feed live. w.tick(res[s.asset])