chore(reset): v2.0.0 — storico certificato Deribit mainnet, ripartenza pulita
Reset del progetto su fondamenta verificate dopo la scoperta che l'intera libreria "validata OOS" era artefatto di feed contaminato (print fantasma del feed Cerbero TESTNET + storico Binance/USDT). - Storico ricostruito da Deribit MAINNET (ccxt pubblico, tokenless) e CERTIFICATO (certify_feed.py): BTC/ETH puliti su TUTTA la storia (mediana 2-6 bps vs Coinbase USD), integrita' OHLC + coerenza resample (maxΔ 0.00) + cross-venue OK. Alt esclusi (illiquidi/divergenti: LTC/DOGE 50-82% barre flat; XRP/BNB non certificabili). - Verdetto sul feed pulito: FADE / PAIRS / XS01 / TSM01 morti (ogni portafoglio Sharpe -2.3..-3.0, DD ~40%); solo SH01 e frammenti HONEST con segnale residuo, da ri-validare in isolamento. - Cleanup "restart pulito": strategie, stack live (src/live, src/portfolio, runner/executor, yml, docker), ~100 script ricerca/gate, waste/games/ portfolios, dati non certificati + cache e 60+ diari -> archiviati in Old/ (preservati, non cancellati). Diario consolidato in un unico documento. - Skeleton ricerca tenuto: Strategy ABC + indicatori + src/fractal + src/backtest/engine + load_data; tool dati certificati (rebuild_history, certify_feed, audit_feed, multi_source_check). - Universo dati ATTIVO: solo BTC/ETH (5m/15m/1h); guardrail fisico (load_data su alt -> FileNotFoundError). Esecuzione DISABILITATA, conto flat. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal Pastro
@@ -0,0 +1,203 @@
|
||||
"""Check candele FLAT (O=H=L=C, liquidita' zero) sui pairs ETH/BTC a 15m.
|
||||
|
||||
Rischio noto (CLAUDE.md): ETH 15m ha 14-30%/anno di candele flat per bassa liquidita'
|
||||
del perpetuo. Su un pairs, un close stale gonfia lo z-score (l'altra gamba si muove,
|
||||
questa e' ferma) -> segnale di "reversione" FINTO che rientra solo quando la gamba
|
||||
stale si sblocca: profitto NON eseguibile dal vivo. Questo gonfierebbe il backtest 15m.
|
||||
|
||||
Test:
|
||||
[1] prevalenza candele flat per anno (ETH 15m, BTC 15m).
|
||||
[2] quanti trade del pairs 15m hanno ENTRY/EXIT su una candela flat (gamba stale).
|
||||
[3] re-sim flat-aware: entry/exit SOLO su barre pulite (non-flat in ENTRAMBE le gambe)
|
||||
-> quanto sopravvive l'edge? (parita': senza flat-skip == pairs_sim).
|
||||
[4] gate PORT06 col 15m flat-filtrato vs baseline 1h.
|
||||
|
||||
uv run python scripts/analysis/pairs15m_flatcheck.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from src.data.downloader import load_data
|
||||
from scripts.analysis.pairs_research import pairs_sim, OOS_FRAC, FEE_RT, LEV, POS, BARS_YEAR
|
||||
from scripts.analysis.report_families import daily_from
|
||||
from scripts.analysis.combine_portfolio import metrics, SPLIT, OOS_DATE
|
||||
from scripts.analysis.pairs15m_port06_gate import port_metrics, eth_btc_daily, UNIV_1H, GAME_15M
|
||||
from scripts.portfolios._defs import PORTFOLIOS
|
||||
from src.portfolio.sleeves import all_sleeve_equities
|
||||
|
||||
|
||||
def aligned2(a, b, tf="15m"):
|
||||
"""Merge con OHLC di ENTRAMBE le gambe (serve per rilevare i flat su entrambe)."""
|
||||
da = load_data(a, tf)[["timestamp", "open", "high", "low", "close"]].rename(
|
||||
columns=lambda x: x + "_a" if x != "timestamp" else x)
|
||||
db = load_data(b, tf)[["timestamp", "open", "high", "low", "close"]].rename(
|
||||
columns=lambda x: x + "_b" if x != "timestamp" else x)
|
||||
m = da.merge(db, on="timestamp", how="inner").reset_index(drop=True)
|
||||
m["dt"] = pd.to_datetime(m["timestamp"], unit="ms", utc=True)
|
||||
return m
|
||||
|
||||
|
||||
def is_flat(o, h, l, c):
|
||||
return (o == h) & (h == l) & (l == c)
|
||||
|
||||
|
||||
def flat_prevalence(asset, tf="15m"):
|
||||
d = load_data(asset, tf)
|
||||
d = d.copy()
|
||||
d["dt"] = pd.to_datetime(d["timestamp"], unit="ms", utc=True)
|
||||
fl = is_flat(d["open"].values, d["high"].values, d["low"].values, d["close"].values)
|
||||
d["flat"] = fl
|
||||
by = d.groupby(d["dt"].dt.year)["flat"].mean() * 100
|
||||
return by, fl.mean() * 100
|
||||
|
||||
|
||||
def pairs_sim_flataware(a, b, tf="15m", n=66, z_in=1.674, z_exit=1.0, max_bars=35,
|
||||
jump_max=0.08, fee_rt=FEE_RT, lev=LEV, pos=POS,
|
||||
split_frac=0.0, skip_flat=True):
|
||||
"""Come pairs_sim ma: entry/exit consentiti SOLO su barre pulite (se skip_flat).
|
||||
Ritorna anche n_entry_flat / n_exit_flat (diagnostica, calcolata sempre)."""
|
||||
m = aligned2(a, b, tf)
|
||||
ca, cb = m["close_a"].values, m["close_b"].values
|
||||
flat_a = is_flat(m["open_a"].values, m["high_a"].values, m["low_a"].values, ca)
|
||||
flat_b = is_flat(m["open_b"].values, m["high_b"].values, m["low_b"].values, cb)
|
||||
flat = flat_a | flat_b # barra "sporca" se una delle due gambe e' flat
|
||||
r = np.log(ca / cb)
|
||||
dr = np.abs(np.diff(r, prepend=r[0]))
|
||||
ma = pd.Series(r).rolling(n).mean().values
|
||||
sd = pd.Series(r).rolling(n).std().values
|
||||
z = (r - ma) / np.where(sd == 0, np.nan, sd)
|
||||
ts = m["dt"]; N = len(r)
|
||||
split = int(N * split_frac)
|
||||
fee = 2 * fee_rt * lev
|
||||
cap = peak = 1000.0; dd = 0.0; last = -1
|
||||
trades = wins = 0; rets = []; yearly = {}
|
||||
eq_ts, eq_v = [], []
|
||||
n_entry_flat = n_exit_flat = 0
|
||||
for i in range(n + 1, N - 1):
|
||||
if i < split or np.isnan(z[i]) or dr[i] > jump_max or i <= last:
|
||||
continue
|
||||
if z[i] <= -z_in:
|
||||
d = 1
|
||||
elif z[i] >= z_in:
|
||||
d = -1
|
||||
else:
|
||||
continue
|
||||
if flat[i]:
|
||||
n_entry_flat += 1
|
||||
if skip_flat:
|
||||
continue # non si entra su una gamba stale
|
||||
# exit: |z|<=z_exit o max_bars; se skip_flat, salta le barre flat come uscita
|
||||
j = min(i + max_bars, N - 1)
|
||||
for k in range(1, max_bars + 1):
|
||||
jj = i + k
|
||||
if jj >= N:
|
||||
j = N - 1; break
|
||||
if skip_flat and flat[jj]:
|
||||
j = jj # avanza, non esce su barra stale
|
||||
continue
|
||||
if abs(z[jj]) <= z_exit:
|
||||
j = jj; break
|
||||
j = jj
|
||||
if flat[j]:
|
||||
n_exit_flat += 1
|
||||
if skip_flat:
|
||||
# spingi all'ultima barra pulita entro l'orizzonte
|
||||
back = j
|
||||
while back > i and flat[back]:
|
||||
back -= 1
|
||||
j = back if back > i else j
|
||||
retA = (ca[j] - ca[i]) / ca[i]
|
||||
retB = (cb[j] - cb[i]) / cb[i]
|
||||
ret = (retA - retB) * d * lev - fee
|
||||
cap = max(cap + cap * pos * ret, 10.0)
|
||||
peak = max(peak, cap); dd = max(dd, (peak - cap) / peak)
|
||||
trades += 1; wins += ret > 0; rets.append(ret * pos); last = j
|
||||
eq_ts.append(ts.iloc[j]); eq_v.append(cap)
|
||||
yearly[ts.iloc[i].year] = yearly.get(ts.iloc[i].year, 0.0) + ret * 100
|
||||
yrs_span = (ts.iloc[-1] - ts.iloc[max(split, 0)]).days / 365.25 or 1
|
||||
sharpe = 0.0
|
||||
if len(rets) > 1 and np.std(rets) > 0:
|
||||
sharpe = float(np.mean(rets) / np.std(rets) * np.sqrt(trades / yrs_span))
|
||||
ret_tot = (cap / 1000 - 1) * 100
|
||||
return dict(trades=trades, win=wins / trades * 100 if trades else 0, ret=ret_tot,
|
||||
dd=dd * 100, sharpe=sharpe, yearly=yearly, eq_ts=eq_ts, eq_v=eq_v,
|
||||
n_entry_flat=n_entry_flat, n_exit_flat=n_exit_flat)
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 100)
|
||||
print(" CHECK FLAT-CANDLE — ETH/BTC pairs 15m (gate condizionato)")
|
||||
print("=" * 100)
|
||||
|
||||
# [1] prevalenza
|
||||
print("\n[1] Prevalenza candele flat (O=H=L=C) per anno, 15m:")
|
||||
for asset in ("ETH", "BTC"):
|
||||
by, tot = flat_prevalence(asset, "15m")
|
||||
print(f" {asset}: media {tot:.1f}% | " +
|
||||
" ".join(f"{y}:{v:.0f}%" for y, v in by.items()))
|
||||
|
||||
# [2] quanti trade toccano un flat (sim SENZA skip per diagnostica)
|
||||
diag = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=False)
|
||||
tr = diag["trades"]
|
||||
print(f"\n[2] Trade 15m totali: {tr} | entry su barra flat: {diag['n_entry_flat']} "
|
||||
f"({diag['n_entry_flat']/tr*100:.1f}%) | exit su barra flat: {diag['n_exit_flat']} "
|
||||
f"({diag['n_exit_flat']/tr*100:.1f}%)")
|
||||
|
||||
# [3] parita' + edge filtrato
|
||||
print("\n[3] Edge 15m: NO-skip (== pairs_sim) vs FLAT-AWARE (entry/exit solo barre pulite):")
|
||||
# parita': flataware skip_flat=False deve ~== pairs_sim
|
||||
base_ps = pairs_sim("ETH", "BTC", **GAME_15M, pos=POS, lev=LEV)
|
||||
print(f" parita' pairs_sim : trd {base_ps['trades']:>5d} Sh {base_ps['sharpe']:.2f} "
|
||||
f"DD {base_ps['dd']:.0f}% ret {base_ps['ret']:+.0f}%")
|
||||
print(f" flataware (no-skip) : trd {diag['trades']:>5d} Sh {diag['sharpe']:.2f} "
|
||||
f"DD {diag['dd']:.0f}% ret {diag['ret']:+.0f}%")
|
||||
filt = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=True)
|
||||
filt_o = pairs_sim_flataware("ETH", "BTC", **GAME_15M, skip_flat=True, split_frac=1 - OOS_FRAC)
|
||||
print(f" FLAT-AWARE (skip) : trd {filt['trades']:>5d} Sh {filt['sharpe']:.2f} "
|
||||
f"DD {filt['dd']:.0f}% ret {filt['ret']:+.0f}% | OOS Sh {filt_o['sharpe']:.2f} DD {filt_o['dd']:.0f}%")
|
||||
drop = (1 - filt['trades'] / diag['trades']) * 100
|
||||
sh_keep = filt['sharpe'] / diag['sharpe'] * 100 if diag['sharpe'] else 0
|
||||
verdict = "EDGE NON artefatto flat" if sh_keep > 70 else "EDGE in larga parte ARTEFATTO flat"
|
||||
print(f" -> rimossi {drop:.1f}% dei trade; Sharpe trattenuto {sh_keep:.0f}% ({verdict})")
|
||||
|
||||
# [4] gate PORT06 col 15m flat-filtrato
|
||||
print("\n[4] GATE PORT06 — ETH/BTC: baseline 1h vs SWAP 15m-FLATAWARE vs BLEND:")
|
||||
p = PORTFOLIOS["PORT06"]
|
||||
pair_ids = [s.sid for s in p.sleeves if s.sid.startswith("PR_")]
|
||||
eq_base = dict(all_sleeve_equities())
|
||||
e1h, _ = eth_btc_daily(UNIV_1H)
|
||||
e15f = daily_from(filt["eq_ts"], filt["eq_v"])
|
||||
# blend 1h + 15m-flataware (50/50 daily-rebalanced)
|
||||
from scripts.analysis.pairs15m_port06_gate import blend
|
||||
eblend = blend(e1h, e15f, 0.5)
|
||||
corr = e1h.pct_change().fillna(0).corr(e15f.pct_change().fillna(0))
|
||||
print(f" corr 1h vs 15m-flataware: {corr:.3f}")
|
||||
print(f" {'variante':<18s} | {'FULL Sh':>8s}{'FULL DD%':>9s}{'CAGR':>6s} | {'OOS Sh':>7s}{'OOS DD%':>8s}")
|
||||
print(" " + "-" * 70)
|
||||
res = {}
|
||||
for tag, eth in [("baseline 1h", e1h), ("SWAP 15m-flat", e15f), ("BLEND 1h+15m-flat", eblend)]:
|
||||
members = dict(eq_base); members["PR_ETHBTC"] = eth
|
||||
f, o = port_metrics(members, p)
|
||||
res[tag] = (f, o)
|
||||
print(f" {tag:<18s} | {f['sharpe']:>8.2f}{f['dd']:>9.2f}{f['cagr']:>5.0f}%"
|
||||
f" | {o['sharpe']:>7.2f}{o['dd']:>8.2f}")
|
||||
|
||||
fb, ob = res["baseline 1h"]
|
||||
print("\n VERDETTO (vs baseline 1h, fee backtest): Sharpe non peggiora E DD <= baseline")
|
||||
for tag in ("SWAP 15m-flat", "BLEND 1h+15m-flat"):
|
||||
f, o = res[tag]
|
||||
ok = o["sharpe"] >= ob["sharpe"] - 0.02 and o["dd"] <= ob["dd"] + 1e-9 and f["sharpe"] >= fb["sharpe"] - 0.02 and f["dd"] <= fb["dd"] + 1e-9
|
||||
print(f" {tag:<18s}: OOS {ob['sharpe']:.2f}->{o['sharpe']:.2f} DD {ob['dd']:.2f}->{o['dd']:.2f}"
|
||||
f" | FULL {fb['sharpe']:.2f}->{f['sharpe']:.2f} DD {fb['dd']:.2f}->{f['dd']:.2f} => {'PROMOSSO' if ok else 'bocciato'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user