Adriano Dal Pastro
14522262e6
Reset del progetto su fondamenta verificate dopo la scoperta che l'intera libreria "validata OOS" era artefatto di feed contaminato (print fantasma del feed Cerbero TESTNET + storico Binance/USDT). - Storico ricostruito da Deribit MAINNET (ccxt pubblico, tokenless) e CERTIFICATO (certify_feed.py): BTC/ETH puliti su TUTTA la storia (mediana 2-6 bps vs Coinbase USD), integrita' OHLC + coerenza resample (maxΔ 0.00) + cross-venue OK. Alt esclusi (illiquidi/divergenti: LTC/DOGE 50-82% barre flat; XRP/BNB non certificabili). - Verdetto sul feed pulito: FADE / PAIRS / XS01 / TSM01 morti (ogni portafoglio Sharpe -2.3..-3.0, DD ~40%); solo SH01 e frammenti HONEST con segnale residuo, da ri-validare in isolamento. - Cleanup "restart pulito": strategie, stack live (src/live, src/portfolio, runner/executor, yml, docker), ~100 script ricerca/gate, waste/games/ portfolios, dati non certificati + cache e 60+ diari -> archiviati in Old/ (preservati, non cancellati). Diario consolidato in un unico documento. - Skeleton ricerca tenuto: Strategy ABC + indicatori + src/fractal + src/backtest/engine + load_data; tool dati certificati (rebuild_history, certify_feed, audit_feed, multi_source_check). - Universo dati ATTIVO: solo BTC/ETH (5m/15m/1h); guardrail fisico (load_data su alt -> FileNotFoundError). Esecuzione DISABILITATA, conto flat. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
274 lines
13 KiB
Python
274 lines
13 KiB
Python
"""TIMING SWEEP — famiglie PAIRS & HONEST su 5/10/15/30m (vs live), goal 2026-06-14.
|
|
|
|
Domanda utente: i pairs e le honest beneficiano di timeframe piu' veloci, come hanno
|
|
fatto le fade (swap 1h->15m, v1.1.30)?
|
|
|
|
VINCOLO DATI (hard): solo BTC/ETH hanno 5m/15m/30m in locale (10m = resample da 5m, qui
|
|
in data/raw/{a}_10m.parquet temporanei). TUTTI gli alt (ADA/BNB/DOGE/LTC/SOL/XRP) sono
|
|
SOLO 1h. Conseguenze:
|
|
- PAIRS: solo ETH/BTC e' sweepabile sub-orario. Gli altri 4 pair (gambe alt:
|
|
LTC/ETH, ADA/ETH, BTC/LTC, ETH/SOL) restano 1h per mancanza di dati alt sub-orari.
|
|
- HONEST: solo DIP01 (BTC, mean-reversion) ha senso + dati. TR01 (trend EMA 4h su
|
|
basket alt) e ROT02 (rotazione dual-momentum 1d su universo alt) sono lente
|
|
(orizzonte multi-giorno/mese) E multi-asset-su-alt -> sub-orario infattibile (dati)
|
|
e insensato (momentum a 60g su barre 5min).
|
|
|
|
Tutto NETTO (fee 0.10% RT single / 0.20% RT per coppia a 2 gambe), leva 3x, OOS = held-out.
|
|
Engine CANONICI riusati (pairs_sim_flat, replica dip intrabar == dip_market_gated, gate
|
|
PORT06 == pairs30m_gate/dip01). Niente re-tuning dei parametri al cambio TF (anti-overfit,
|
|
come lo swap fade).
|
|
|
|
uv run python scripts/analysis/timing_sweep_pairs_honest.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from src.data.downloader import load_data
|
|
from scripts.analysis.pairs_research import pairs_sim_flat
|
|
from scripts.analysis.report_families import daily_from
|
|
from scripts.analysis.combine_portfolio import port_returns, metrics, SPLIT, OOS_DATE, IDX, _norm
|
|
from scripts.portfolios._defs import PORTFOLIOS
|
|
from src.portfolio.sleeves import all_sleeve_equities
|
|
from src.portfolio import weighting as W
|
|
|
|
TFS = ["5m", "10m", "15m", "30m", "1h"]
|
|
BARS_PER_DAY = {"5m": 288, "10m": 144, "15m": 96, "30m": 48, "1h": 24}
|
|
LEV, POS = 3.0, 0.15
|
|
EPOCH = pd.Timestamp(0, tz="UTC")
|
|
|
|
|
|
def _ensure_10m():
|
|
"""10m non e' nativo (download_all fa 5m/15m/30m/1h): lo resampla da 5m se manca.
|
|
Aggregazione OHLCV causale (first/max/min/last/sum). File gitignored, rigenerabile."""
|
|
for a in ("btc", "eth"):
|
|
dst = PROJECT_ROOT / f"data/raw/{a}_10m.parquet"
|
|
if dst.exists():
|
|
continue
|
|
df = load_data(a.upper(), "5m").sort_values("timestamp").reset_index(drop=True)
|
|
ts = pd.to_datetime(df["timestamp"], unit="ms", utc=True)
|
|
g = df.set_index(ts).resample("10min")
|
|
out = pd.DataFrame({"open": g["open"].first(), "high": g["high"].max(),
|
|
"low": g["low"].min(), "close": g["close"].last(),
|
|
"volume": g["volume"].sum()}).dropna()
|
|
out["timestamp"] = (out.index - EPOCH) // pd.Timedelta(milliseconds=1)
|
|
out.reset_index(drop=True).to_parquet(dst, index=False)
|
|
print(f" [bootstrap] generato {dst.name} ({len(out)} righe da 5m)")
|
|
|
|
# config UNIVERSALE pairs (1h, CLAUDE.md) — NON ri-tunata al cambio TF (anti-overfit)
|
|
PAIR_CFG = dict(n=50, z_in=2.0, z_exit=0.75, max_bars=72)
|
|
# config DIP01 canonica (dip_market_gated default, market_n=0 = live)
|
|
DIP_CFG = dict(n=50, z_in=2.5, sl_atr=2.5, max_bars=24)
|
|
|
|
|
|
# ============================ helper dati ============================
|
|
def flat_share(asset, tf):
|
|
df = load_data(asset, tf)
|
|
o, h, l, c = df["open"], df["high"], df["low"], df["close"]
|
|
return ((o == h) & (h == l) & (l == c)).mean() * 100
|
|
|
|
|
|
# ============================ DIP01 engine TF-aware ============================
|
|
def _atr(df, n=14):
|
|
h, l, c = df["high"].values, df["low"].values, df["close"].values
|
|
pc = np.roll(c, 1); pc[0] = c[0]
|
|
tr = np.maximum(h - l, np.maximum(np.abs(h - pc), np.abs(l - pc)))
|
|
return pd.Series(tr).rolling(n).mean().values
|
|
|
|
|
|
def dip_sim(asset, tf, n=50, z_in=2.5, sl_atr=2.5, max_bars=24,
|
|
fee_rt=0.001, oos_frac=0.0):
|
|
"""Replica TF-aware di honest_improve2.dip_market_gated(market_n=0): dip-buy z-score,
|
|
TP=SMA intrabar, SL=close-sl_atr*ATR intrabar (SL prioritario), max_bars. Engine canonico."""
|
|
df = load_data(asset, tf)
|
|
h, l, c = df["high"].values, df["low"].values, df["close"].values
|
|
N = len(c); ts = pd.to_datetime(df["timestamp"], unit="ms", utc=True)
|
|
ma = pd.Series(c).rolling(n).mean().values
|
|
sd = pd.Series(c).rolling(n).std().values
|
|
a = _atr(df, 14)
|
|
z = (c - ma) / np.where(sd == 0, np.nan, sd)
|
|
fee = fee_rt * LEV
|
|
cap = peak = 1000.0; dd = 0.0; last_exit = -1
|
|
eq_ts, eq_v = [], []; rets = []; trades = wins = 0; yearly = {}
|
|
split = int(N * (1 - oos_frac)) if oos_frac else 0
|
|
for i in range(n + 14, N):
|
|
if i < split or np.isnan(z[i]) or np.isnan(a[i]):
|
|
continue
|
|
if not (z[i] <= -z_in and z[i - 1] > -z_in):
|
|
continue
|
|
if i <= last_exit or i + 1 >= N:
|
|
continue
|
|
entry = c[i]; tp, sl, mb = ma[i], c[i] - sl_atr * a[i], max_bars
|
|
exit_p = c[min(i + mb, N - 1)]; j = min(i + mb, N - 1)
|
|
for k in range(1, mb + 1):
|
|
j = i + k
|
|
if j >= N:
|
|
j = N - 1; exit_p = c[j]; break
|
|
if l[j] <= sl:
|
|
exit_p = sl; break
|
|
if h[j] >= tp:
|
|
exit_p = tp; break
|
|
if k == mb:
|
|
exit_p = c[j]
|
|
ret = (exit_p - entry) / entry * LEV - fee
|
|
cap = max(cap + cap * POS * ret, 10.0)
|
|
peak = max(peak, cap); dd = max(dd, (peak - cap) / peak)
|
|
last_exit = j; trades += 1; wins += ret > 0; rets.append(ret * POS)
|
|
yearly[ts.iloc[i].year] = yearly.get(ts.iloc[i].year, 0.0) + ret * 100
|
|
eq_ts.append(ts.iloc[j]); eq_v.append(cap)
|
|
yrs = (ts.iloc[-1] - ts.iloc[max(split, 0)]).days / 365.25 or 1
|
|
sh = float(np.mean(rets) / np.std(rets) * np.sqrt(trades / yrs)) if len(rets) > 1 and np.std(rets) > 0 else 0.0
|
|
return dict(trades=trades, win=wins / trades * 100 if trades else 0, ret=(cap / 1000 - 1) * 100,
|
|
dd=dd * 100, sharpe=sh, yearly=yearly, eq_ts=eq_ts, eq_v=eq_v)
|
|
|
|
|
|
# ============================ gate PORT06 ============================
|
|
def port_metrics(members, ids, clusters, caps):
|
|
dr = pd.DataFrame({i: members[i].pct_change().fillna(0.0) for i in ids})
|
|
w = W.weight_vector("cap", ids, dr, caps=caps, clusters=clusters)
|
|
drp = port_returns({i: members[i] for i in ids}, w)
|
|
return metrics(drp), metrics(drp, lo=SPLIT), w
|
|
|
|
|
|
def daily_norm(eq_ts, eq_v):
|
|
return _norm(daily_from(eq_ts, eq_v))
|
|
|
|
|
|
# ============================ PART 1: dati ============================
|
|
def part1_data():
|
|
print("=" * 96)
|
|
print(" PART 1 — REALTA' DATI: flat-share (O=H=L=C, = print stale, rischio fill) per TF")
|
|
print("=" * 96)
|
|
print(f" {'asset':<6}" + "".join(f"{tf:>8}" for tf in TFS))
|
|
for a in ("BTC", "ETH"):
|
|
print(f" {a:<6}" + "".join(f"{flat_share(a, tf):>7.1f}%" for tf in TFS))
|
|
print("\n ALT (ADA/BNB/DOGE/LTC/SOL/XRP): SOLO 1h disponibile -> NON sweepabili sub-orario.")
|
|
print(" => pairs con gamba alt (4/5) e honest multi-asset (TR01/ROT02) bloccati a 1h dai dati.")
|
|
|
|
|
|
# ============================ PART 2: pairs ETH/BTC standalone ============================
|
|
def part2_pairs_standalone():
|
|
print("\n" + "=" * 96)
|
|
print(" PART 2 — PAIRS ETH/BTC standalone, config UNIVERSALE n=50 z_in=2.0 z_exit=0.75 mb=72")
|
|
print(f" (flat_skip=True, live-realizable; OOS held-out; fee 0.20% RT/coppia; f2x = OOS Sh a fee 2x)")
|
|
print("=" * 96)
|
|
print(f" {'tf':<5}{'trd':>7}{'FULL%':>9}{'DD%':>7}{'Sh':>7} | {'OOS%':>9}{'oDD%':>7}{'oSh':>7} | {'f2x_oSh':>8}{'mb_h':>6}")
|
|
eqs = {}
|
|
for tf in TFS:
|
|
f = pairs_sim_flat("ETH", "BTC", tf=tf, **PAIR_CFG, flat_skip=True)
|
|
o = pairs_sim_flat("ETH", "BTC", tf=tf, **PAIR_CFG, flat_skip=True, split_frac=1 - 0.30)
|
|
o2 = pairs_sim_flat("ETH", "BTC", tf=tf, **PAIR_CFG, flat_skip=True, split_frac=1 - 0.30, fee_rt=0.002)
|
|
eqs[tf] = daily_norm(f["eq_ts"], f["eq_v"])
|
|
mb_h = PAIR_CFG["max_bars"] / BARS_PER_DAY[tf] * 24
|
|
print(f" {tf:<5}{f['trades']:>7}{f['ret']:>9.0f}{f['dd']:>7.1f}{f['sharpe']:>7.2f}"
|
|
f" | {o['ret']:>9.0f}{o['dd']:>7.1f}{o['sharpe']:>7.2f} | {o2['sharpe']:>8.2f}{mb_h:>6.1f}")
|
|
# correlazioni daily fra TF
|
|
print("\n CORR rendimenti daily fra TF (alta = ridondante):")
|
|
print(f" {'':<6}" + "".join(f"{tf:>7}" for tf in TFS))
|
|
for t1 in TFS:
|
|
row = []
|
|
for t2 in TFS:
|
|
c = eqs[t1].pct_change().fillna(0).corr(eqs[t2].pct_change().fillna(0))
|
|
row.append(f"{c:>7.2f}")
|
|
print(f" {t1:<6}" + "".join(row))
|
|
return eqs
|
|
|
|
|
|
# ============================ PART 3: pairs gate PORT06 ============================
|
|
def part3_pairs_gate(eqs):
|
|
print("\n" + "=" * 96)
|
|
print(" PART 3 — GATE PORT06: aggiungere ETH/BTC 5m e/o 10m al BLEND live (1h+15m), mezza size")
|
|
print(f" (baseline = sleeve canonici live; OOS da {OOS_DATE})")
|
|
print("=" * 96)
|
|
p = PORTFOLIOS["PORT06"]
|
|
base = dict(all_sleeve_equities()) # include PR_ETHBTC (1h) + PR_ETHBTC_15M
|
|
ids0 = list(p.sleeve_ids); cl0 = p.clusters; caps = p.caps
|
|
f0, o0, _ = port_metrics(base, ids0, cl0, caps)
|
|
print(f" {'config':<26}{'FULL Sh':>9}{'FULL DD%':>10}{'OOS Sh':>9}{'OOS DD%':>9}")
|
|
print(f" {'ATTUALE (1h+15m)':<26}{f0['sharpe']:>9.2f}{f0['dd']:>10.2f}{o0['sharpe']:>9.2f}{o0['dd']:>9.2f}")
|
|
# half-size pairs equity (pos 0.075 come 15m live): ricalcolo eq a pos dimezzato
|
|
for tf in ("10m", "5m"):
|
|
fr = pairs_sim_flat("ETH", "BTC", tf=tf, **PAIR_CFG, flat_skip=True, pos=0.075)
|
|
cand = daily_norm(fr["eq_ts"], fr["eq_v"])
|
|
mem = dict(base); sid = f"PR_ETHBTC_{tf.upper()}"
|
|
mem[sid] = cand
|
|
ids = ids0 + [sid]; cl = dict(cl0); cl[sid] = "ETH-rev"
|
|
f1, o1, w1 = port_metrics(mem, ids, cl, caps)
|
|
ok = (o1["sharpe"] >= o0["sharpe"] - 0.02 and o1["dd"] <= o0["dd"] + 1e-9
|
|
and f1["sharpe"] >= f0["sharpe"] - 0.02 and f1["dd"] <= f0["dd"] + 1e-9)
|
|
verdict = "MIGLIORA (promosso)" if ok else "non domina (vedi numeri)"
|
|
print(f" {'+' + tf + ' (half size)':<26}{f1['sharpe']:>9.2f}{f1['dd']:>10.2f}{o1['sharpe']:>9.2f}{o1['dd']:>9.2f} {verdict}")
|
|
|
|
|
|
# ============================ PART 4: DIP01 ============================
|
|
def part4_dip():
|
|
print("\n" + "=" * 96)
|
|
print(" PART 4 — HONEST/DIP01 (BTC) standalone, config canonica n=50 z_in=2.5 sl_atr=2.5 mb=24")
|
|
print(" (engine == dip_market_gated market_n=0; OOS held-out; fee 0.10% RT; f2x = OOS Sh a fee 2x)")
|
|
print("=" * 96)
|
|
print(f" {'tf':<5}{'asset':<5}{'trd':>7}{'WR%':>6}{'FULL%':>9}{'DD%':>7}{'Sh':>7} | {'OOS%':>9}{'oDD%':>7}{'oSh':>7} | {'f2x_oSh':>8}{'mb_h':>6}")
|
|
eqs = {}
|
|
for asset in ("BTC", "ETH"):
|
|
for tf in TFS:
|
|
f = dip_sim(asset, tf, **DIP_CFG)
|
|
o = dip_sim(asset, tf, **DIP_CFG, oos_frac=0.30)
|
|
o2 = dip_sim(asset, tf, **DIP_CFG, oos_frac=0.30, fee_rt=0.002)
|
|
if asset == "BTC":
|
|
eqs[tf] = daily_norm(f["eq_ts"], f["eq_v"]) if f["eq_v"] else None
|
|
mb_h = DIP_CFG["max_bars"] / BARS_PER_DAY[tf] * 24
|
|
print(f" {tf:<5}{asset:<5}{f['trades']:>7}{f['win']:>6.1f}{f['ret']:>9.0f}{f['dd']:>7.1f}{f['sharpe']:>7.2f}"
|
|
f" | {o['ret']:>9.0f}{o['dd']:>7.1f}{o['sharpe']:>7.2f} | {o2['sharpe']:>8.2f}{mb_h:>6.1f}")
|
|
print()
|
|
# corr daily BTC fra TF
|
|
print(" CORR rendimenti daily DIP01 BTC fra TF:")
|
|
print(f" {'':<6}" + "".join(f"{tf:>7}" for tf in TFS))
|
|
for t1 in TFS:
|
|
row = []
|
|
for t2 in TFS:
|
|
if eqs.get(t1) is None or eqs.get(t2) is None:
|
|
row.append(f"{'-':>7}"); continue
|
|
c = eqs[t1].pct_change().fillna(0).corr(eqs[t2].pct_change().fillna(0))
|
|
row.append(f"{c:>7.2f}")
|
|
print(f" {t1:<6}" + "".join(row))
|
|
return eqs
|
|
|
|
|
|
# ============================ PART 5: DIP01 gate PORT06 ============================
|
|
def part5_dip_gate(eqs):
|
|
print("\n" + "=" * 96)
|
|
print(" PART 5 — GATE PORT06: SWAP DIP01_BTC 1h -> TF piu' veloce (sostituisce lo sleeve)")
|
|
print("=" * 96)
|
|
p = PORTFOLIOS["PORT06"]
|
|
base = dict(all_sleeve_equities())
|
|
ids0 = list(p.sleeve_ids); cl0 = p.clusters; caps = p.caps
|
|
f0, o0, _ = port_metrics(base, ids0, cl0, caps)
|
|
print(f" {'config':<26}{'FULL Sh':>9}{'FULL DD%':>10}{'OOS Sh':>9}{'OOS DD%':>9}")
|
|
print(f" {'ATTUALE (DIP01 1h)':<26}{f0['sharpe']:>9.2f}{f0['dd']:>10.2f}{o0['sharpe']:>9.2f}{o0['dd']:>9.2f}")
|
|
for tf in ("30m", "15m", "10m", "5m"):
|
|
if eqs.get(tf) is None:
|
|
continue
|
|
mem = dict(base); mem["DIP01_BTC"] = eqs[tf]
|
|
f1, o1, _ = port_metrics(mem, ids0, cl0, caps)
|
|
ok = (o1["sharpe"] >= o0["sharpe"] - 0.02 and o1["dd"] <= o0["dd"] + 1e-9
|
|
and f1["sharpe"] >= f0["sharpe"] - 0.02 and f1["dd"] <= f0["dd"] + 1e-9)
|
|
verdict = "MIGLIORA" if ok else "non domina"
|
|
print(f" {'DIP01 ' + tf:<26}{f1['sharpe']:>9.2f}{f1['dd']:>10.2f}{o1['sharpe']:>9.2f}{o1['dd']:>9.2f} {verdict}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_ensure_10m()
|
|
part1_data()
|
|
pe = part2_pairs_standalone()
|
|
part3_pairs_gate(pe)
|
|
de = part4_dip()
|
|
part5_dip_gate(de)
|
|
print("\n NB TR01/ROT02: nessuno sweep — dati alt solo 1h + orizzonte multi-giorno/mese")
|
|
print(" (trend EMA20/100 4h, rotazione momentum 60g 1d) rendono il sub-orario infattibile e insensato.")
|