PythagorasGoal/Old/scripts/analysis/xs01_dispersion_gate.py

"""XS01 dispersion-gate — la reversione cross-sectional va accesa solo in certi regimi?

Motivazione: l'edge XS01 e' concentrato (2025 domina, 2023 debole). Ipotesi da
testare: il fattore reversione cross-sezionale paga quando c'e' DISPERSIONE da
far rientrare (spread cross-section largo) e/o correlazione media alta (mosse
idiosincratiche = rumore che rientra), e perde nei regime-break (dispersione da
trend divergente, es. melt-up di un singolo asset).

Metodo (anti-multiple-testing):
  [1] DIAGNOSTICA: engine XS01 canonico SENZA gate, registrando per ogni trade
      il valore di 3 feature di regime alla barra di ENTRY (tutte causali,
      calcolate dallo stesso panel closes <= i):
        g_disp  = std cross-section del segnale stesso (logC[i]-logC[i-lb])
        g_corr  = correlazione media pairwise 72h (identita' var dell'indice)
        g_vol   = vol realizzata BTC 168h
      Bucket per quintili (quintili dal TRAIN) -> mean net per bucket,
      TRAIN e OOS SEPARATI. Si prosegue solo se la relazione e' monotona
      e con lo stesso segno in entrambe le finestre.
  [2] GATE: per la feature promossa, sweep soglie (percentili TRAIN
      30/40/50/60/70) -> TRAIN/OOS Sharpe/PnL/DD vs base. Serve PLATEAU.
  [3] Solo se [2] regge: gate PORT06 (swap equity sleeve XS01).

    uv run python scripts/analysis/xs01_dispersion_gate.py
"""
from __future__ import annotations

import sys
from pathlib import Path

import numpy as np
import pandas as pd

PROJECT_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(PROJECT_ROOT))

from scripts.strategies.XS01_cross_sectional import (
    aligned_panel, UNIVERSE, FEE_RT, LEV, POS, OOS_FRAC, LB, HOLD)

N_A = len(UNIVERSE)


def build_features(M, lb=LB):
    """Feature di regime causali dal panel closes (nessun feed esterno)."""
    logC = np.log(M.values)
    r = np.diff(logC, axis=0, prepend=logC[:1])          # ret orari (r[0]=0)
    R = pd.DataFrame(r, index=M.index)
    # g_disp: std cross-section del momentum lb (il segnale che fadiamo)
    D = pd.DataFrame(logC).diff(lb).to_numpy()
    g_disp = np.nanstd(D, axis=1)
    # g_corr 72h: avg pairwise corr via identita' della varianza dell'indice
    w = 72
    idx_var = R.mean(axis=1).rolling(w).var().to_numpy()
    mean_var = R.rolling(w).var().mean(axis=1).to_numpy()
    with np.errstate(divide="ignore", invalid="ignore"):
        g_corr = (N_A * idx_var / mean_var - 1) / (N_A - 1)
    # g_vol: vol BTC 168h annualizzata
    b = UNIVERSE.index("BTC")
    g_vol = R[b].rolling(168).std().to_numpy() * np.sqrt(24 * 365)
    return dict(g_disp=g_disp, g_corr=g_corr, g_vol=g_vol)


def sim_with_trace(M, feats, gate=None, lb=LB, hold=HOLD, fee_rt=FEE_RT,
                   lev=LEV, pos=POS):
    """Engine XS01 canonico (stessa logica/ordine di XS01_cross_sectional.xsec_sim)
    + trace per-trade (entry index, net, feature) + gate opzionale bool[i]."""
    C = M.values
    ts = pd.to_datetime(M.index, unit="ms", utc=True)
    n = len(C)
    logC = np.log(C)
    cap = peak = 1000.0
    dd = 0.0
    rows = []
    eq_ts, eq_v = [], []
    last = -1
    i = lb
    fee = 2 * fee_rt
    while i < n - hold:
        if i <= last:
            i += 1
            continue
        if gate is not None and not gate[i]:
            i += 1
            continue
        dm = (logC[i] - logC[i - lb])
        dm = dm - dm.mean()
        w = -dm
        gw = np.sum(np.abs(w))
        if gw < 1e-9:
            i += 1
            continue
        w = w / gw
        book = float(np.sum(w * (logC[i + hold] - logC[i])))
        net = book - fee
        cap = max(cap + cap * pos * lev * net, 10.0)
        peak = max(peak, cap)
        dd = max(dd, (peak - cap) / peak)
        rows.append((i, int(ts[i].year), net,
                     feats["g_disp"][i], feats["g_corr"][i], feats["g_vol"][i]))
        eq_ts.append(ts[i + hold])
        eq_v.append(cap)
        last = i + hold
        i += 1
    tr = pd.DataFrame(rows, columns=["i", "year", "net", "g_disp", "g_corr", "g_vol"])
    yrs_span = (ts[-1] - ts[0]).days / 365.25 or 1
    out = dict(trades=len(tr), cap=cap, dd=dd * 100, eq_ts=eq_ts, eq_v=eq_v, tr=tr)
    if len(tr) > 1 and tr["net"].std() > 0:
        out["sharpe"] = float(tr["net"].mean() / tr["net"].std()
                              * np.sqrt(len(tr) / yrs_span))
    else:
        out["sharpe"] = 0.0
    out["pnl_add"] = float(tr["net"].sum() * 100) if len(tr) else 0.0
    out["win"] = float((tr["net"] > 0).mean() * 100) if len(tr) else 0.0
    out["tpm"] = len(tr) / (yrs_span * 12)
    return out


def metrics_window(tr, lo, hi, yrs_span):
    t = tr[(tr["i"] >= lo) & (tr["i"] < hi)]
    if len(t) < 2 or t["net"].std() == 0:
        return dict(n=len(t), pnl=0.0, sh=0.0, win=0.0)
    sh = float(t["net"].mean() / t["net"].std() * np.sqrt(len(t) / yrs_span))
    return dict(n=len(t), pnl=float(t["net"].sum() * 100), sh=sh,
                win=float((t["net"] > 0).mean() * 100))


def main():
    M = aligned_panel()
    n = len(M)
    cut = int(n * (1 - OOS_FRAC))
    ts = pd.to_datetime(M.index, unit="ms", utc=True)
    feats = build_features(M)
    print("=" * 96)
    print(f"  XS01 dispersion-gate | panel {ts[0].date()} -> {ts[-1].date()} "
          f"({n} ore, 8 asset) | TRAIN 70% (-> {ts[cut].date()}) / OOS 30%")
    print("=" * 96)

    base = sim_with_trace(M, feats)
    tr = base["tr"]
    yrs_tr = (ts[cut] - ts[0]).days / 365.25
    yrs_oo = (ts[-1] - ts[cut]).days / 365.25

    # [1] DIAGNOSTICA per quintili (quintili dal TRAIN)
    print("\n[1] DIAGNOSTICA — mean net per trade (bps) per quintile feature @entry")
    ttr = tr[tr["i"] < cut]
    too = tr[tr["i"] >= cut]
    for g in ("g_disp", "g_corr", "g_vol"):
        qs = ttr[g].quantile([0.2, 0.4, 0.6, 0.8]).to_numpy()
        def bucket(x):
            return int(np.searchsorted(qs, x))
        print(f"    {g:<7s} | " + " | ".join(
            f"Q{q+1} TR {ttr[ttr[g].apply(bucket) == q]['net'].mean()*1e4:+6.1f} "
            f"OOS {too[too[g].apply(bucket) == q]['net'].mean()*1e4:+6.1f}"
            for q in range(5)) +
            f"  (n TR {len(ttr)}, OOS {len(too)})")

    # [2] GATE sweep — per ogni feature, tieni SOPRA o SOTTO il percentile
    print("\n[2] GATE — TRAIN/OOS vs base (soglie = percentili del TRAIN; "
          "side scelto dal segno della diagnostica TRAIN)")
    b_tr = metrics_window(tr, 0, cut, yrs_tr)
    b_oo = metrics_window(tr, cut, n, yrs_oo)
    print(f"    {'BASE':<24s} TRAIN n {b_tr['n']:>4} pnl {b_tr['pnl']:>+7.1f}% "
          f"Sh {b_tr['sh']:>5.2f} | OOS n {b_oo['n']:>4} pnl {b_oo['pnl']:>+7.1f}% "
          f"Sh {b_oo['sh']:>5.2f}")
    for g in ("g_disp", "g_corr", "g_vol"):
        # segno dal TRAIN: correlazione quintile->ret
        qs5 = ttr[g].quantile([0.2, 0.4, 0.6, 0.8]).to_numpy()
        means = [ttr[ttr[g].apply(lambda x: int(np.searchsorted(qs5, x))) == q]["net"].mean()
                 for q in range(5)]
        side = "above" if means[-1] > means[0] else "below"
        for pct in (30, 40, 50, 60, 70):
            thr = float(np.nanpercentile(feats[g][:cut], pct))
            gv = feats[g]
            gate = (gv >= thr) if side == "above" else (gv <= thr)
            gate = np.nan_to_num(gate, nan=False).astype(bool)
            r = sim_with_trace(M, feats, gate=gate)
            g_tr = metrics_window(r["tr"], 0, cut, yrs_tr)
            g_oo = metrics_window(r["tr"], cut, n, yrs_oo)
            print(f"    {g} {side} p{pct:<3d}{'':<6s} TRAIN n {g_tr['n']:>4} "
                  f"pnl {g_tr['pnl']:>+7.1f}% Sh {g_tr['sh']:>5.2f} | "
                  f"OOS n {g_oo['n']:>4} pnl {g_oo['pnl']:>+7.1f}% Sh {g_oo['sh']:>5.2f}")

    # breakdown annuale base (riferimento concentrazione)
    print("\n    base, net additivo per anno (%):",
          {int(y): round(float(v * 100), 1)
           for y, v in tr.groupby("year")["net"].sum().items()})


if __name__ == "__main__":
    main()