"""r0701_breadth_internals.py — BREADTH / MARKET-INTERNALS del mercato ALT come segnale su BTC/ETH. TESI (filone 2026-07-01) ------------------------ Gli "internals" del mercato crypto — la partecipazione degli alt — come segnale direzionale o gate di de-risk su BTC/ETH perp (2 gambe, eseguibile a ~$600): FAM-MA : % di alt sopra la propria SMA(N) (breadth classica) FAM-AD : advance/decline — frazione di advancers, SMA(N) (partecipazione giornaliera) FAM-RS : % di alt che BATTONO BTC sul ritorno a N giorni (risk-appetite relativo, ~mkt-neutral) FAM-TH : breadth-THRUST — delta della breadth MA20 su N giorni (thrust/collapse) Forme: LS (long/short), LF (long/flat), GATE (TP01 * gate binario). Tutte vol-target 20% cap 2x (LS/LF) o ereditano il sizing TP01 (GATE). RISCHI NOTI IN PARTENZA (CLAUDE.md, prior art) ---------------------------------------------- 1. MACRO regime-gate (2026-06-29) = SCARTATO: corr->TP01 0.989, il gate lavorava nel 2-3% dei giorni (TP01 e' gia' flat nei crash). Un breadth-gate rischia di essere LO STESSO artefatto: TP01 travestito. Qui DOBBIAMO riportare corr->TP01 + verdetto marginale + "giorni in cui il gate lavora" (gate off E TP01 non gia' flat). 2. trend-multiasset = ridondante (corr 0.74): la breadth degli alt e' correlata alla direzione del mercato -> il rischio che breadth>soglia == "BTC sopra trend" e' concreto. 3. is_hedge: un segnale che paga solo quando TP01 soffre e' un hedge, non alpha. 4. STORIA: l'universo HL parte dal 2024-01 -> ~2.2 anni utili post-warmup. In-sample (pre-HOLDOUT 2025-01-01) = SOLO ~8 mesi del 2024. Limite strutturale DICHIARATO: qualunque esito e' al massimo un LEAD, la selezione in-sample poggia su una finestra corta. METODO (obbligatorio, CLAUDE.md) -------------------------------- - Dati: 51 parquet certificati data/raw/hl_*_1d.parquet; PANEL = 49 alt (esclusi hl_btc/hl_eth dalla breadth; hl_btc usato solo come riferimento per FAM-RS). Barre a volume<=0 = sintetiche -> close mascherato NaN. Breadth definita solo con >= MIN_VALID(20) asset validi alla data. - Causalita': barre HL 1d e barre BTC/ETH 1d (altlib.get, resample Deribit) sono entrambe open-labeled 00:00 UTC -> il close del giorno D e' noto allo stesso istante su entrambe. Allineamento merge_asof backward (allow_exact) sul timestamp; eval_weights shifta la posizione (decisa a close[i], tenuta in i+1). Verifica al.causality_ok sul target end-to-end. - Selezione ONESTA (lezione SELECTION-ON-HOLDOUT 2026-06-29): la cella si sceglie con il SOLO Sharpe in-sample (pre-2025) sul candidato 50/50, MAI sull'hold-out; deflated Sharpe (Bailey & Lopez de Prado) su TUTTE le celle cercate; poi al.marginal_vs_tp01 (multi-cut, has_insample_edge, is_hedge) sulla cella scelta. NB: non si usa al.study_family_honest stock perche' la breadth non esiste pre-2024 e il padding (LS/LF=flat, GATE=TP01 pieno) contaminerebbe il ranking in-sample full-history (le celle GATE erediterebbero lo Sharpe 2019-2024 di TP01); la procedura qui sotto e' il MIRROR esatto di select_cell_insample + deflated_sharpe + study_marginal sulla FINESTRA COMUNE 2024-05+ (stessa libreria, stessi gate). - Fee 0.10% RT + sweep 0-0.30% RT; eval_weights_smallcap a $600. USO: uv run python scripts/research/r0701_breadth_internals.py """ from __future__ import annotations import glob import sys from pathlib import Path import numpy as np import pandas as pd _ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(_ROOT)) sys.path.insert(0, str(_ROOT / "scripts" / "research" / "alt")) import altlib as al # noqa: E402 from src.strategies.trend_portfolio import CANONICAL, TrendPortfolio # noqa: E402 RAW = _ROOT / "data" / "raw" HOLDOUT = al.HOLDOUT ASSETS = ("BTC", "ETH") MIN_VALID = 20 # asset validi minimi perche' la breadth esista START = pd.Timestamp("2026-01-01", tz="UTC") # placeholder, ridefinito sotto # finestra comune: HL parte 2024-01-01; warmup max = FAM-TH N=100 (20g MA + 100g delta) ~ 120g START = pd.Timestamp("2024-05-05", tz="UTC") MA_GRID = (20, 50, 100) THR_GRID = (0.3, 0.5, 0.7) FORMS = ("ls", "lf", "gate") FAMS = ("ma", "ad", "rs", "th") # =========================================================================== # PANEL ALT (49 asset, vol=0 mascherato) + riferimento BTC (hl_btc) # =========================================================================== def load_panel(): px, vol = {}, {} for f in sorted(glob.glob(str(RAW / "hl_*_1d.parquet"))): sym = Path(f).stem.replace("hl_", "").replace("_1d", "").upper() d = pd.read_parquet(f) idx = pd.to_datetime(d["timestamp"], unit="ms", utc=True) px[sym] = pd.Series(d["close"].values.astype(float), index=idx) vol[sym] = pd.Series(d["volume"].values.astype(float), index=idx) PX = pd.concat(px, axis=1).sort_index() VOL = pd.concat(vol, axis=1).sort_index().reindex_like(PX) PX = PX.mask(VOL <= 0) # barre sintetiche (vol=0) -> NaN (lezione 2026-06-20) btc_ref = PX["BTC"].copy() # riferimento FAM-RS (stessa venue/stesso close time) ALTS = PX.drop(columns=["BTC", "ETH"]) # breadth = SOLO alt return ALTS, btc_ref def _mask_min_valid(score: pd.Series, n_valid: pd.Series) -> pd.Series: s = score.copy() s[n_valid < MIN_VALID] = np.nan return s def breadth_ma(ALTS: pd.DataFrame, N: int) -> pd.Series: """% di alt validi con close > SMA(N). Causale (SMA su dati <= i).""" sma = ALTS.rolling(N, min_periods=N).mean() valid = ALTS.notna() & sma.notna() above = (ALTS > sma) & valid n_valid = valid.sum(axis=1) return _mask_min_valid(above.sum(axis=1) / n_valid.replace(0, np.nan), n_valid) def breadth_ad(ALTS: pd.DataFrame, N: int) -> pd.Series: """Advance/decline: frazione di advancers (ret 1g > 0) tra i validi, SMA(N).""" dr = ALTS.pct_change(fill_method=None) valid = dr.notna() adv = ((dr > 0) & valid).sum(axis=1) n_valid = valid.sum(axis=1) frac = adv / n_valid.replace(0, np.nan) frac[n_valid < MIN_VALID] = np.nan return frac.rolling(N, min_periods=N).mean() def breadth_rs(ALTS: pd.DataFrame, btc_ref: pd.Series, N: int) -> pd.Series: """% di alt che battono BTC sul ritorno a N giorni (risk-appetite relativo).""" altret = ALTS / ALTS.shift(N) - 1.0 btcret = (btc_ref / btc_ref.shift(N) - 1.0) valid = altret.notna() & btcret.notna().values[:, None] beat = altret.gt(btcret, axis=0) & valid n_valid = valid.sum(axis=1) return _mask_min_valid(beat.sum(axis=1) / n_valid.replace(0, np.nan), n_valid) def breadth_th(ALTS: pd.DataFrame, N: int) -> pd.Series: """Breadth-THRUST: 0.5 + delta a N giorni della breadth MA20 (thrust>0.5, collapse<0.5).""" b20 = breadth_ma(ALTS, 20) return 0.5 + (b20 - b20.shift(N)) # =========================================================================== # FACTORY — target_fn(df, asset) per una cella (fam, N, thr, form) # =========================================================================== _ALTS, _BTC_REF = load_panel() _BREADTH: dict[tuple, pd.Series] = {} for _N in MA_GRID: _BREADTH[("ma", _N)] = breadth_ma(_ALTS, _N) _BREADTH[("ad", _N)] = breadth_ad(_ALTS, _N) _BREADTH[("rs", _N)] = breadth_rs(_ALTS, _BTC_REF, _N) _BREADTH[("th", _N)] = breadth_th(_ALTS, _N) _TP01_POS: dict[str, np.ndarray] = {} def tp01_pos(df: pd.DataFrame, asset: str) -> np.ndarray: if asset not in _TP01_POS: _TP01_POS[asset] = TrendPortfolio(**CANONICAL).target_series(df) return _TP01_POS[asset] _EPOCH = pd.Timestamp("1970-01-01", tz="UTC") def _align(b: pd.Series, df: pd.DataFrame) -> np.ndarray: """Breadth (calendario HL) -> barre BTC/ETH. merge_asof backward, exact ok (stesso istante di close 00:00 UTC). NaN dove la breadth non esiste. NB timestamp via epoca esplicita: .view('int64') su DatetimeIndex tz-aware a risoluzione non-ns (pandas 2.x) da' la SCALA SBAGLIATA -> merge_asof matchava tutto all'ULTIMO valore (broadcast del futuro su tutta la storia = look-ahead che causality_ok non vede, perche' la serie breadth e' un input esterno fisso). Bug trovato e corretto in questa ricerca.""" ts_ms = ((b.index - _EPOCH) // pd.Timedelta(milliseconds=1)).astype("int64") g = pd.DataFrame({"timestamp": ts_ms, "b": b.values}) g = g.dropna(subset=["b"]).sort_values("timestamp") left = pd.DataFrame({"timestamp": df["timestamp"].astype("int64").values}) m = pd.merge_asof(left, g, on="timestamp", direction="backward") return m["b"].values.astype(float) def factory(tf: str = "1d", fam: str = "ma", N: int = 50, thr: float = 0.5, form: str = "ls"): b_series = _BREADTH[(fam, N)] def target_fn(df: pd.DataFrame, asset: str = "") -> np.ndarray: b = _align(b_series, df) if form == "gate": g = np.where(np.isfinite(b), np.where(b >= thr, 1.0, 0.0), 1.0) # no info -> no de-risk return tp01_pos(df, asset) * g if form == "ls": d = np.where(np.isfinite(b), np.where(b >= thr, 1.0, -1.0), 0.0) else: # lf d = np.where(np.isfinite(b), np.where(b >= thr, 1.0, 0.0), 0.0) return al.vol_target(d, df, 0.20, 30, 2.0) return target_fn GRID = [dict(fam=f, N=n, thr=t, form=fo) for f in FAMS for n in MA_GRID for t in THR_GRID for fo in FORMS] # =========================================================================== # DRIVER ONESTO sulla finestra comune (mirror di study_family_honest, stessi gate altlib) # =========================================================================== def cand_trim(fn) -> pd.Series: return al.candidate_daily(fn, tf="1d")[lambda s: s.index >= START] def abs_verdict_trimmed(fn) -> dict: """study_weights-equivalente sulla finestra comune 2024-05+ (fee sweep incluso).""" per_asset = {} fee_ok_all = True for a in ASSETS: df = al.get(a, "1d") tgt = np.asarray(fn(df, a), float) mask = (pd.to_datetime(df["datetime"], utc=True) >= START).values dft = df[mask].reset_index(drop=True) tgtt = tgt[mask] base = al.eval_weights(dft, tgtt, fee_side=al.FEE_SIDE) sweep = {f"{2*f*100:.2f}%RT": al.eval_weights(dft, tgtt, fee_side=f)["full"]["sharpe"] for f in al.FEE_SWEEP} fee_ok_all = fee_ok_all and sweep.get("0.20%RT", -9) > 0 per_asset[a] = dict(full=base["full"], holdout=base["holdout"], tim=base["time_in_market"], turnover=base["turnover_per_year"], fee_sweep=sweep, yearly=base["yearly"]) cell = dict(tf="1d", per_asset=per_asset, min_asset_full_sharpe=round(min(per_asset[a]["full"]["sharpe"] for a in ASSETS), 3), min_asset_holdout_sharpe=round(min(per_asset[a]["holdout"].get("sharpe", 0.0) for a in ASSETS), 3), full_sharpe=round(float(np.mean([per_asset[a]["full"]["sharpe"] for a in ASSETS])), 3), fee_survives=fee_ok_all) return dict(cells=[cell], verdict=al._verdict([cell])) def gate_work_diag(fn, thr_flat: float = 1e-6) -> dict: """Deep-dive ridondanza (lezione macro-gate): nei giorni in cui il segnale vorrebbe stare fuori/short, TP01 e' gia' flat da solo? 'lavora' = segnale off/short E TP01 non flat.""" out = {} for a in ASSETS: df = al.get(a, "1d") mask = (pd.to_datetime(df["datetime"], utc=True) >= START).values tgt = np.asarray(fn(df, a), float)[mask] tp = tp01_pos(df, a)[mask] off = tgt <= thr_flat # segnale fuori (o short per LS: qui solo "non long") works = off & (tp > thr_flat) out[a] = dict(days_off=round(float(off.mean()), 3), days_gate_works=round(float(works.mean()), 3), tp01_flat_days=round(float((tp <= thr_flat).mean()), 3), corr_pos=round(float(np.corrcoef(tgt, tp)[0, 1]), 3) if np.std(tgt) > 0 and np.std(tp) > 0 else None) return out def cell_activity(p: dict) -> float: """Frazione di giorni nello STATO DI MINORANZA del segnale (criterio STRUTTURALE, non di performance: una cella sempre-on e' buy&hold/TP01 travestito, non un segnale di breadth). ls: min(on, off); lf: frazione on... comunque = minoranza; gate: frazione off.""" b = _BREADTH[(p["fam"], p["N"])] bb = b[(b.index >= START)].dropna() on = float((bb >= p["thr"]).mean()) return round(min(on, 1.0 - on) if p["form"] == "ls" else (on if p["form"] == "lf" else 1.0 - on), 3) def full_report(label: str, chosen: dict, all_full: list, n_trials: int) -> dict: p = chosen["params"] fn = factory(**p) daily = cand_trim(fn) dsr, sr0 = al.deflated_sharpe(al._sh(daily), all_full, daily) print(f"\n==== {label}: {p} (attivita' {chosen['act']}) ====") print(f" standalone (finestra comune): IS {chosen['insample_sharpe']:+.2f} " f"FULL {chosen['full_sharpe']:+.2f} HOLD {chosen['hold_sharpe']:+.2f}") print(f" deflated Sharpe (su TUTTI i {n_trials} trial cercati): DSR={dsr:.3f} " f"(null-max atteso {sr0:.2f}) PASS>=0.95: {dsr >= 0.95}") marg = al.marginal_vs_tp01(daily) absr = abs_verdict_trimmed(fn) abs_grade = absr["verdict"]["grade"] earns_slot = (abs_grade != "FAIL" and marg.get("marginal_verdict") == "ADDS" and marg.get("robust_oos", False) and marg.get("beats_noise_null", False) and not marg.get("is_hedge", False)) rep = dict(name=f"BREADTH {p}", marginal=marg, abs_grade=abs_grade, marginal_verdict=marg.get("marginal_verdict"), earns_slot=earns_slot) print("\n" + al.fmt_marginal(rep)) honest = earns_slot and dsr >= 0.95 print(f" earns_slot_honest = earns_slot({earns_slot}) AND DSR>=0.95({dsr >= 0.95}) " f"=> {honest}") # assoluto trimmed + fee sweep c = absr["cells"][0] print(f"\n---- ASSOLUTO (finestra comune, verdetto {abs_grade}): " f"minFull {c['min_asset_full_sharpe']:+.2f} minHold {c['min_asset_holdout_sharpe']:+.2f} " f"feeOK={c['fee_survives']}") for a in ASSETS: pa = c["per_asset"][a] yr = " ".join(f"{y}:{d['ret']*100:+.0f}%" for y, d in pa["yearly"].items()) print(f" {a}: full Sh {pa['full']['sharpe']:+.2f} DD {pa['full']['maxdd']*100:.0f}% " f"hold Sh {pa['holdout'].get('sharpe', 0):+.2f} tim {pa['tim']} " f"turn/y {pa['turnover']} | {yr}") print(f" fee sweep: {pa['fee_sweep']}") # causalita' + smallcap $600 ca = al.causality_ok(fn, tf="1d") print(f"\n---- causality_ok: {ca['ok']} (max_tail_diff {ca['max_tail_diff']}, checked {ca['checked']})") for a in ASSETS: df = al.get(a, "1d") mask = (pd.to_datetime(df["datetime"], utc=True) >= START).values dft = df[mask].reset_index(drop=True) tgtt = np.asarray(fn(df, a), float)[mask] sc = al.eval_weights_smallcap(dft, tgtt, capital=600.0, min_order=5.0) print(f" smallcap $600 {a}: modeled Sh {sc['modeled']['sharpe']:+.2f} -> " f"real {sc['realistic']['sharpe']:+.2f} (haircut {sc['sharpe_haircut']:+.2f}, " f"{sc['n_executed_trades']} trade eseguiti)") # deep-dive ridondanza col trend (lezione macro-gate) print("---- RIDONDANZA COL TREND (il rischio n.1):") for a, d in gate_work_diag(fn).items(): print(f" {a}: corr(pos, TP01pos) {d['corr_pos']} giorni segnale-off {d['days_off']} " f"TP01-gia'-flat {d['tp01_flat_days']} GIORNI IN CUI LAVORA {d['days_gate_works']}") return dict(params=p, dsr=round(float(dsr), 3), earns_slot=earns_slot, earns_slot_honest=honest, marginal_verdict=marg.get("marginal_verdict"), abs_grade=abs_grade, corr_tp01=marg.get("corr_full"), is_hedge=marg.get("is_hedge")) def main(): print(f"=== r0701 BREADTH/INTERNALS — panel: {_ALTS.shape[1]} alt, " f"{_ALTS.index[0].date()} -> {_ALTS.index[-1].date()} | finestra analisi {START.date()}+ " f"(in-sample {START.date()} -> {HOLDOUT.date()} = ~8 mesi; storia ~2.2y: LIMITE DICHIARATO)") nv = _ALTS.notna().sum(axis=1) print(f" asset validi/D: min {int(nv.min())} med {int(nv.median())} max {int(nv.max())} " f"(MIN_VALID={MIN_VALID})") # ---- 1. tutte le celle: Sharpe in-sample (selezione) + full (DSR) sulla finestra comune rows = [] for p in GRID: fn = factory(**p) daily = cand_trim(fn) ins = daily[daily.index < HOLDOUT] if len(ins) < 60 or daily.std() == 0: continue rows.append(dict(params=p, act=cell_activity(p), insample_sharpe=round(al._sh(ins), 3), full_sharpe=round(al._sh(daily), 3), hold_sharpe=round(al._sh(daily[daily.index >= HOLDOUT]), 3))) rows.sort(key=lambda r: r["insample_sharpe"], reverse=True) all_full = [r["full_sharpe"] for r in rows] print(f"\n---- GRIGLIA: {len(GRID)} celle, {len(rows)} valutabili; " f"full>0: {sum(1 for s in all_full if s > 0)}/{len(all_full)}") print("---- TOP-15 per Sharpe IN-SAMPLE (selezione onesta: MAI sull'hold-out) " "[hold mostrato solo per trasparenza; act = frazione stato di minoranza]") for r in rows[:15]: p = r["params"] print(f" {p['fam']:>2s} N={p['N']:>3d} thr={p['thr']} {p['form']:>4s} act={r['act']:.2f} | " f"IS {r['insample_sharpe']:+.2f} full {r['full_sharpe']:+.2f} hold {r['hold_sharpe']:+.2f}") # ---- 2. PRIMARIO: cella scelta in-sample su TUTTA la griglia (procedura onesta pura) out1 = full_report("CELLA SCELTA IN-SAMPLE (tutta la griglia)", rows[0], all_full, len(rows)) # ---- 3. SECONDARIO (dichiarato): sole celle ATTIVE (minoranza >=10% — criterio strutturale # deciso a priori, NON di performance; DSR sempre deflazionato su TUTTI i trial). active = [r for r in rows if r["act"] >= 0.10] print(f"\n---- CELLE ATTIVE (act>=0.10): {len(active)}/{len(rows)}") out2 = None if active and active[0]["params"] != rows[0]["params"]: out2 = full_report("SECONDARIO: best cella ATTIVA in-sample", active[0], all_full, len(rows)) # ---- 4. marginal sui best per-forma tra le ATTIVE (trasparenza: il verdetto per forma) print("\n---- VERDETTO MARGINALE dei best-IN-SAMPLE ATTIVI per forma (contesto):") for fo in FORMS: sub = [r for r in active if r["params"]["form"] == fo] if not sub: continue rp = sub[0]["params"] m = al.marginal_vs_tp01(cand_trim(factory(**rp))) uh = m["blends"]["w25"]["uplift_hold"] print(f" {fo:>4s} {rp} act={sub[0]['act']:.2f}: {m.get('marginal_verdict')} " f"corr {m.get('corr_full')} IS-edge {m.get('cand_insample_sharpe')} " f"is_hedge {m.get('is_hedge')} uplift w25 full {m['blends']['w25']['uplift_full']:+.3f} " f"hold {uh if uh is None else format(uh, '+.3f')} multicut {m.get('multicut_uplift')}") print("\n==== SINTESI ====") print(f" primario: {out1}") print(f" secondario (attive): {out2}") print("==== FINE r0701 ====") if __name__ == "__main__": main()