PythagorasGoal/Old/scripts/analysis/regime_lab.py

"""regime_lab — API condivisa per la ricerca strategie FRATTALI x REGIME (ARGO-proxy).

Allinea prezzo (OHLCV) + DVOL + funding in modo CAUSALE (no look-ahead: il valore di
regime alla barra i usa solo dati <= timestamp[i]) ed espone:
  - feature REGIME (ARGO-proxy backtestabili): dvol, dvol_pct (percentile rolling),
    rv (realized vol), vrp = dvol - rv, funding, funding_z, dvol_chg (proxy term-structure).
  - feature FRATTALI (src/fractal): rolling_hurst, higuchi, self_similarity, volatility_ratio,
    williams fractals (pivot), candle encoding.
  - validazione: report(name, entries, df) -> full/oos netto-fee + robustezza griglia/fee,
    riusando l'engine onesto di explore_lab (simulate/evaluate).

Convenzione entries (come explore_lab): lista di dict {i, d (+1/-1), tp, sl, max_bars}.
Ingresso ESEGUIBILE: i, d, tp, sl decisi con dati <= close[i].

Uso tipico in un agente:
    from scripts.analysis.regime_lab import load, report, regime_features, frac_features
    df = load("BTC", "1h")                 # OHLCV + colonne regime allineate
    R = regime_features(df); F = frac_features(df)
    entries = [...]                         # la tua logica
    print(report("MIA_STRATEGIA", entries, df))
"""
from __future__ import annotations

import sys
from pathlib import Path

import numpy as np
import pandas as pd

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))

from scripts.analysis.explore_lab import get_df, simulate, evaluate, atr, ema, rsi  # noqa: E402
from src.fractal.indicators import (  # noqa: E402
    rolling_hurst, fractal_dimension_higuchi, self_similarity_score, volatility_ratio,
)

# dati regime (DVOL/funding/feature) in data/regime/ — NON in data/raw/ (che e' solo OHLCV: i file
# estranei in data/raw inquinano la discovery asset del backtest). Vedi diary 2026-06-02-fade-lossguard.
RAW = ROOT / "data" / "regime"
RAW.mkdir(parents=True, exist_ok=True)


# --------------------------------------------------------------------------- dati
def _load_regime_series(asset: str) -> tuple[pd.DataFrame, pd.DataFrame]:
    a = asset.lower()
    dvol = pd.read_parquet(RAW / f"{a}_dvol.parquet") if (RAW / f"{a}_dvol.parquet").exists() else pd.DataFrame()
    fund = pd.read_parquet(RAW / f"{a}_funding.parquet") if (RAW / f"{a}_funding.parquet").exists() else pd.DataFrame()
    return dvol, fund


def load(asset: str, tf: str) -> pd.DataFrame:
    """OHLCV (explore_lab.get_df) + colonne regime allineate CAUSALMENTE (merge_asof backward).
    Ogni barra prezzo riceve l'ultimo DVOL/funding con timestamp <= timestamp barra."""
    df = get_df(asset, tf).copy()
    df["timestamp"] = df["timestamp"].astype("int64")
    dvol, fund = _load_regime_series(asset)
    if not dvol.empty:
        d = dvol[["timestamp", "dvol"]].astype({"timestamp": "int64"}).sort_values("timestamp")
        df = pd.merge_asof(df.sort_values("timestamp"), d, on="timestamp", direction="backward")
    else:
        df["dvol"] = np.nan
    if not fund.empty:
        col = "interest_1h" if "interest_1h" in fund.columns else fund.columns[1]
        f = fund[["timestamp", col]].astype({"timestamp": "int64"}).rename(columns={col: "funding"}).sort_values("timestamp")
        df = pd.merge_asof(df.sort_values("timestamp"), f, on="timestamp", direction="backward")
    else:
        df["funding"] = np.nan
    return df.reset_index(drop=True)


# ---------------------------------------------------------------- feature REGIME
def _rolling_pct(x: np.ndarray, win: int) -> np.ndarray:
    """Percentile rolling CAUSALE: rank di x[i] nella finestra [i-win, i] (solo passato)."""
    s = pd.Series(x)
    return s.rolling(win, min_periods=max(20, win // 4)).apply(
        lambda w: (w.iloc[-1] >= w).mean(), raw=False).values


_BARS_PER_YEAR = {"1h": 24 * 365, "4h": 6 * 365, "1d": 365}


def regime_features(df: pd.DataFrame, tf: str = "1h", pct_win: int = 252, rv_win: int = 24,
                    fund_win: int = 168) -> dict:
    """Tutte causali. dvol_pct/funding_z usano solo finestra passata. vrp = dvol - rv annualizz.
    tf serve ad annualizzare correttamente la realized vol (sqrt barre/anno per timeframe)."""
    c = df["close"].values.astype(float)
    dvol = df["dvol"].values.astype(float)
    fund = df["funding"].values.astype(float)
    ret = np.zeros_like(c); ret[1:] = np.diff(np.log(c))
    # realized vol annualizzata (punti %, scala come DVOL): std rolling * sqrt(barre/anno del tf)
    bpy = _BARS_PER_YEAR.get(tf, 24 * 365)
    rv = pd.Series(ret).rolling(rv_win).std().values * np.sqrt(bpy) * 100
    dvol_pct = _rolling_pct(dvol, pct_win)
    fmean = pd.Series(fund).rolling(fund_win).mean().values
    fstd = pd.Series(fund).rolling(fund_win).std().values
    funding_z = (fund - fmean) / np.where(fstd == 0, np.nan, fstd)
    dvol_chg = pd.Series(dvol).diff(rv_win).values     # proxy term-structure (DVOL in salita/discesa)
    return {
        "dvol": dvol, "dvol_pct": dvol_pct, "rv": rv, "vrp": dvol - rv,
        "funding": fund, "funding_z": funding_z, "dvol_chg": dvol_chg,
    }


# --------------------------------------------------------------- feature FRATTALI
def williams_fractals(df: pd.DataFrame, k: int = 2) -> tuple[np.ndarray, np.ndarray]:
    """Pivot di Bill Williams: frac_up[i]=high[i] e' il max delle 2k+1 barre centrate (causale a i+k).
    Ritorna due array bool (up=swing high confermato, dn=swing low). Confermati con ritardo k."""
    h, l = df["high"].values, df["low"].values
    n = len(h)
    up = np.zeros(n, bool); dn = np.zeros(n, bool)
    for i in range(k, n - k):
        if h[i] == max(h[i - k:i + k + 1]):
            up[i] = True
        if l[i] == min(l[i - k:i + k + 1]):
            dn[i] = True
    return up, dn


def frac_features(df: pd.DataFrame, hurst_win: int = 100, higuchi_win: int = 64,
                  step: int = 1) -> dict:
    """Feature frattali rolling, CAUSALI (finestra passata che termina a i). step>1: calcola
    ogni `step` barre e fa forward-fill (i frattali variano lentamente) -> molto piu' veloce."""
    c = df["close"].values.astype(float)
    n = len(c)
    hurst = rolling_hurst(c, window=hurst_win, step=step)  # gia' causale + stepped (src/fractal)
    vratio = np.full(n, np.nan)
    higuchi = np.full(n, np.nan)
    last_hi = last_vr = np.nan
    for i in range(higuchi_win, n):
        if (i - higuchi_win) % step == 0:
            last_hi = fractal_dimension_higuchi(c[i - higuchi_win:i])
            last_vr = volatility_ratio(c[max(0, i - 60):i])
        higuchi[i] = last_hi
        vratio[i] = last_vr
    up, dn = williams_fractals(df)
    return {"hurst": hurst, "higuchi": higuchi, "vratio": vratio,
            "frac_up": up, "frac_dn": dn}


# ------------------------------------------------------------------------- cache
_FEATCOLS_R = ("dvol", "dvol_pct", "rv", "vrp", "funding", "funding_z", "dvol_chg")
_FEATCOLS_F = ("hurst", "higuchi", "vratio", "frac_up", "frac_dn")


def _cache_path(asset: str, tf: str) -> Path:
    return RAW / f"features_{asset.lower()}_{tf}.parquet"


def build_cache(asset: str, tf: str, frac_step: int = 6) -> pd.DataFrame:
    """Precompute OHLCV + regime + frattali -> parquet condiviso (per i 100 agenti)."""
    df = load(asset, tf)
    R = regime_features(df, tf=tf)
    F = frac_features(df, step=frac_step)
    for k in _FEATCOLS_R:
        df[k] = R[k]
    for k in _FEATCOLS_F:
        df[k] = F[k]
    p = _cache_path(asset, tf)
    df.to_parquet(p)
    return df


def load_features(asset: str, tf: str) -> pd.DataFrame:
    """Carica la cache feature (la costruisce se manca). OHLCV + tutte le colonne regime+frattali."""
    p = _cache_path(asset, tf)
    if p.exists():
        return pd.read_parquet(p)
    return build_cache(asset, tf)


# ------------------------------------------------------------------- validazione
def report(name: str, entries: list[dict], df: pd.DataFrame, asset: str = "", tf: str = "") -> dict:
    """Netto-fee full + OOS (ultimo 30%) + sweep fee, via engine onesto di explore_lab.
    Ritorna dict compatto: trades, full/oos (ret%, sharpe, dd, acc), robust (OK su tutte le fee)."""
    if not entries:
        # struttura compatibile con robust() (tutti zero) -> robust()=False pulito, niente crash
        z = {"ret": 0.0, "sharpe": 0.0, "dd": 0.0, "trades": 0, "win": 0.0, "exposure": 0.0, "yearly": {}}
        print(f"  {name:<24s} NO ENTRIES")
        return {"full": dict(z), "oos": dict(z), "sweep": {0.0: 0.0, 0.0005: 0.0, 0.001: 0.0, 0.002: 0.0},
                "sweep_oos": {0.0: 0.0, 0.0005: 0.0, 0.001: 0.0, 0.002: 0.0}, "pos_yrs": 0, "n_yrs": 0}
    return evaluate(name, entries, df)                      # full + oos + fee sweep


if __name__ == "__main__":
    # smoke: una fade Bollinger gateata dal regime (DVOL alto) come esempio d'uso
    df = load("BTC", "1h")
    R = regime_features(df); F = frac_features(df)
    c = df["close"].values
    ma = pd.Series(c).rolling(50).mean().values
    sd = pd.Series(c).rolling(50).std().values
    a = atr(df, 14)
    ent = []
    for i in range(300, len(c) - 1):
        if np.isnan(sd[i]) or np.isnan(R["dvol_pct"][i]):
            continue
        if R["dvol_pct"][i] < 0.6:                          # gate: solo regime DVOL alto
            continue
        if c[i] < ma[i] - 2.5 * sd[i]:                      # fade banda bassa
            ent.append({"i": i, "d": 1, "tp": ma[i], "sl": c[i] - 2 * a[i], "max_bars": 24})
    print(f"smoke BTC 1h fade|DVOL>p60: {len(ent)} entries")
    print(report("SMOKE", ent, df))