PythagorasGoal/scripts/research/skyhook/runs/SKH2_VOLTGT.py

"""SKH2_VOLTGT — CAUSAL vol-target overlay on the V2 winner's daily return series.

Family: vol-target overlay [VOLTGT]. Wave goal: cut standalone maxDD < 30% while keeping
min-asset hold-out Sharpe >= ~0.70 and earns_slot True.

Method:
  * Build the winner's daily return series per asset (from the honest intrabar equity).
  * Scale each day t by  lev_t = min(cap, target_vol / rv_{t-1})  where rv_{t-1} is the
    trailing realized vol KNOWN AT t-1 (rolling window of past daily returns, .shift(1)).
    -> strictly causal: the scaler at day t uses returns up to and including day t-1 only.
  * scaled_ret_t = lev_t * ret_t.  Rebuild scaled equity, measure DD per asset + combined.
  * Run altlib.marginal_vs_tp01 on the 50/50 scaled-combined daily series.

We sweep target_vol in {15%,20%,25%}, cap in {1.5,2.0}, and a couple of vol windows.
We prove causality of the scaler two ways:
  (1) construction (shift(1) -> rv known at t-1),
  (2) an explicit truncated-prefix recompute: lev_t computed on the full history must equal
      lev_t recomputed from only the returns up to t-1.
The underlying winner entries are param-only -> their causality is sk.causality (0 mismatches).
"""
from __future__ import annotations
import sys
sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook")
import numpy as np
import pandas as pd
import skyhooklib as sk
import altlib as al
from src.strategies.skyhook import SkyhookParams

HOLDOUT = sk.HOLDOUT
FEE = sk.FEE_RT
ANN = np.sqrt(365.25)

# ---- the verified V2 winner (baseline to beat) ----------------------------------------
WINNER = SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16,
                       vola_lo=35.0, vola_hi=95.0, vol_lo=0.0)


def _sh(r: np.ndarray) -> float:
    r = np.asarray(r, float)
    r = r[np.isfinite(r)]
    if len(r) < 2 or np.std(r) == 0:
        return 0.0
    return float(np.mean(r) / np.std(r) * ANN)


def _maxdd_from_returns(r: pd.Series) -> float:
    eq = (1.0 + r.fillna(0.0)).cumprod().values
    pk = np.maximum.accumulate(eq)
    return float(np.max((pk - eq) / pk))


def _split_sharpe(r: pd.Series, mask) -> float:
    return _sh(r[mask].values)


def _trailing_rv(daily: pd.Series, win: int, mode: str) -> pd.Series:
    """Annualized trailing realized vol KNOWN AT t-1 (shift(1) -> strictly past data)."""
    if mode == "ewma":
        # EWMA std of past returns, reacts faster to a vol spike than a flat window
        v = daily.ewm(span=win, min_periods=max(10, win // 2)).std().shift(1)
    else:
        v = daily.rolling(win, min_periods=max(10, win // 2)).std().shift(1)
    return v * ANN


def vol_target_lev(daily: pd.Series, target_vol: float, cap: float, win: int,
                   floor: float = 0.0, mode: str = "roll") -> pd.Series:
    """CAUSAL leverage series. rv_{t-1} = annualized trailing realized vol KNOWN at t-1.
    lev_t = clip(target/rv_{t-1}, floor, cap).  cap<=1.0 => de-risk only (never lever up)."""
    rv = _trailing_rv(daily, win, mode)
    lev = (target_vol / rv).clip(lower=floor, upper=cap)
    # before we have enough history -> stay at min(1.0, cap) (no scaling, no look-ahead)
    lev = lev.where(rv.notna(), min(1.0, cap)).fillna(min(1.0, cap))
    return lev


def prove_scaler_causal(daily: pd.Series, target_vol: float, cap: float, win: int,
                        mode: str = "roll", n_checks: int = 60) -> dict:
    """Truncated-prefix recompute: lev_t built on the FULL series must equal lev_t rebuilt
    from only returns up to t-1. Any leak (un-shifted vol) would break this."""
    full = vol_target_lev(daily, target_vol, cap, win, mode=mode)
    n = len(daily)
    bad = 0
    checked = 0
    mp = max(10, win // 2)
    idxs = np.linspace(int(n * 0.5), n - 1, n_checks).astype(int)
    for t in sorted(set(idxs)):
        if t < 1:
            continue
        prefix = daily.iloc[:t]              # returns up to and including day t-1 ONLY
        if mode == "ewma":
            rv_prev = prefix.ewm(span=win, min_periods=mp).std().iloc[-1] * ANN
        else:
            rv_prev = prefix.rolling(win, min_periods=mp).std().iloc[-1] * ANN
        if (not np.isfinite(rv_prev)) or rv_prev == 0:
            lev_t = min(1.0, cap)
        else:
            lev_t = float(np.clip(target_vol / rv_prev, 0.0, cap))
        checked += 1
        if abs(float(full.iloc[t]) - lev_t) > 1e-9:
            bad += 1
    return dict(ok=bool(bad == 0), mismatches=int(bad), checked=int(checked))


def winner_daily(asset: str) -> pd.Series:
    """Winner's RAW daily return series for one asset (from honest intrabar equity)."""
    return sk.daily_returns(asset, WINNER, FEE)


def run_overlay(target_vol: float, cap: float, win: int, floor: float = 0.0,
                mode: str = "roll") -> dict:
    """Apply the causal vol-target overlay per asset, combine 50/50, report DD + marginal."""
    raw = {a: winner_daily(a) for a in ("BTC", "ETH")}
    scaled = {}
    lev_stats = {}
    scaler_caus = {}
    per_asset_dd_raw = {}
    per_asset_dd_scaled = {}
    per_asset_full_sh = {}
    per_asset_hold_sh = {}
    for a in ("BTC", "ETH"):
        d = raw[a]
        lev = vol_target_lev(d, target_vol, cap, win, floor, mode)
        s = (lev * d)
        scaled[a] = s
        lev_stats[a] = (round(float(lev.mean()), 3), round(float(lev.median()), 3),
                        round(float((lev >= cap - 1e-9).mean()), 3))
        scaler_caus[a] = prove_scaler_causal(d, target_vol, cap, win, mode)
        per_asset_dd_raw[a] = _maxdd_from_returns(d)
        per_asset_dd_scaled[a] = _maxdd_from_returns(s)
        hmask = s.index >= HOLDOUT
        per_asset_full_sh[a] = _sh(s.values)
        per_asset_hold_sh[a] = _split_sharpe(s, hmask)

    # combined 50/50 scaled series (same convention as sk.skyhook_daily_5050)
    J = pd.concat(scaled, axis=1, join="inner").fillna(0.0)
    comb = 0.5 * J["BTC"] + 0.5 * J["ETH"]
    comb_dd = _maxdd_from_returns(comb)
    comb_full_sh = _sh(comb.values)
    comb_hold_sh = _split_sharpe(comb, comb.index >= HOLDOUT)

    mg = al.marginal_vs_tp01(comb)

    max_dd = max(per_asset_dd_scaled.values())   # max over BTC & ETH (per-asset scaled DD)
    min_full = min(per_asset_full_sh.values())
    min_hold = min(per_asset_hold_sh.values())
    return dict(target_vol=target_vol, cap=cap, win=win, floor=floor, mode=mode,
                lev_stats=lev_stats, scaler_caus=scaler_caus,
                dd_raw=per_asset_dd_raw, dd_scaled=per_asset_dd_scaled,
                full_sh=per_asset_full_sh, hold_sh=per_asset_hold_sh,
                comb_dd=comb_dd, comb_full=comb_full_sh, comb_hold=comb_hold_sh,
                max_dd=max_dd, min_full=min_full, min_hold=min_hold, marginal=mg)


def fee_survives_winner() -> bool:
    """The vol-target overlay does NOT change trade count/turnover materially (it scales an
    already-net daily series), so fee survival is the WINNER's fee survival. Report it."""
    rep = sk.study("WINNER-fee", WINNER)
    ok = True
    for a, pa in rep["per_asset"].items():
        ok = ok and (pa["fee_sweep"].get("0.30%RT", -9) > 0)
    return ok, rep


def winner_min_trades() -> int:
    rep = sk.study("WINNER-tr", WINNER)
    return min(pa["full"]["n_trades"] for pa in rep["per_asset"].values())


if __name__ == "__main__":
    print("=" * 90)
    print("SKH2_VOLTGT — causal vol-target overlay on the V2 winner")
    print("Winner:", WINNER)
    print("=" * 90)

    # baseline reference: winner raw (no overlay) for context
    raw = {a: winner_daily(a) for a in ("BTC", "ETH")}
    Jr = pd.concat(raw, axis=1, join="inner").fillna(0.0)
    raw_comb = 0.5 * Jr["BTC"] + 0.5 * Jr["ETH"]
    print("\n--- WINNER RAW (no overlay), daily-return view ---")
    for a in ("BTC", "ETH"):
        print(f"  {a}: dailyFullSh={_sh(raw[a].values):+.2f} "
              f"holdSh={_split_sharpe(raw[a], raw[a].index>=HOLDOUT):+.2f} "
              f"DD(daily)={_maxdd_from_returns(raw[a])*100:.0f}%")
    print(f"  COMB: fullSh={_sh(raw_comb.values):+.2f} "
          f"holdSh={_split_sharpe(raw_comb, raw_comb.index>=HOLDOUT):+.2f} "
          f"DD={_maxdd_from_returns(raw_comb)*100:.0f}%")
    print("  NB daily-view Sharpe != intrabar headline Sharpe (winner minFull +0.83/minHold +0.81 "
          "are the intrabar numbers). The overlay's job is the DD; we judge marginal+DD on the daily series.")

    # KEY LESSON from v1 grid: cap>1.0 levers UP in low-vol regimes that precede crashes ->
    # per-asset BTC DD got WORSE (34%->43-55%). To CUT standalone per-asset DD<30% the cap must
    # be <=1.0 (DE-RISK ONLY: never amplify). We also test EWMA vol (reacts faster to spikes).
    GRID = []
    for mode in ("roll", "ewma"):
        for tv in (0.15, 0.20, 0.25):
            for cap in (0.8, 1.0):           # de-risk only
                for win in (20, 30):
                    GRID.append((tv, cap, win, mode))
    # FRONTIER scan: how much lever-up (cap) can we allow at tv=25 before BTC DD breaks 30%?
    # (rolling vol drove the highest uplift; we want max w25 uplift_hold subject to DD<0.30)
    for cap in (1.1, 1.2, 1.3):
        for win in (20, 30):
            GRID.append((0.25, cap, win, "roll"))
            GRID.append((0.25, cap, win, "ewma"))
    # plus a couple of cap=1.5 references to show the lever-up failure explicitly
    for tv in (0.20, 0.25):
        GRID.append((tv, 1.5, 20, "roll"))

    results = []
    for tv, cap, win, mode in GRID:
        r = run_overlay(tv, cap, win, mode=mode)
        results.append(r)
        mg = r["marginal"]
        w25 = mg.get("blends", {}).get("w25", {})
        print(f"\n[{mode} tv={tv:.0%} cap={cap} win={win}] "
              f"minFull(daily)={r['min_full']:+.2f} minHold={r['min_hold']:+.2f} "
              f"max_dd={r['max_dd']*100:.0f}% (BTC {r['dd_scaled']['BTC']*100:.0f}%/"
              f"ETH {r['dd_scaled']['ETH']*100:.0f}%  raw BTC {r['dd_raw']['BTC']*100:.0f}%/"
              f"ETH {r['dd_raw']['ETH']*100:.0f}%) combDD={r['comb_dd']*100:.0f}%")
        print(f"      lev BTC mean/med/atcap={r['lev_stats']['BTC']}  ETH={r['lev_stats']['ETH']}  "
              f"scalerCausal BTC={r['scaler_caus']['BTC']['ok']} ETH={r['scaler_caus']['ETH']['ok']}")
        print(f"      marginal: corr_full={mg.get('corr_full')} verdict={mg.get('marginal_verdict')} "
              f"insample_edge={mg.get('has_insample_edge')} hedge={mg.get('is_hedge')} "
              f"robust_oos={mg.get('robust_oos')} multicut={mg.get('multicut_persistent')} "
              f"cleanYr={mg.get('clean_year_uplift')} w25upHold={w25.get('uplift_hold')}")

    # ---- pick the best config: prioritize (1) max_dd<0.30, then (2) min_hold, then (3) w25 uplift_hold
    def beats(r):
        mg = r["marginal"]
        w25 = mg.get("blends", {}).get("w25", {})
        es = (mg.get("marginal_verdict") == "ADDS" and mg.get("robust_oos") is True
              and mg.get("is_hedge") is False)
        return (es and r["max_dd"] < 0.30
                and (w25.get("uplift_hold") or -9) >= 0.55 and r["min_hold"] >= 0.65)

    def _earns(r):
        mg = r["marginal"]
        return (mg.get("marginal_verdict") == "ADDS" and mg.get("robust_oos") is True
                and mg.get("is_hedge") is False)

    def score(r):
        mg = r["marginal"]
        w25 = mg.get("blends", {}).get("w25", {})
        uh = w25.get("uplift_hold") or -9
        # priority: (1) full BEATS, (2) DD<0.30 AND earns_slot AND hold>=0.65 (deployable DD-cut),
        # (3) max w25 uplift_hold, (4) max min_hold
        deployable = 1 if (r["max_dd"] < 0.30 and _earns(r) and r["min_hold"] >= 0.65) else 0
        return (1 if beats(r) else 0,
                deployable,
                round(uh, 3),
                round(r["min_hold"], 3))

    best = max(results, key=score)
    mg = best["marginal"]
    w25 = mg.get("blends", {}).get("w25", {})
    w50 = mg.get("blends", {}).get("w50", {})

    fee_ok, _ = fee_survives_winner()
    caus = sk.causality(WINNER, "BTC")
    caus_e = sk.causality(WINNER, "ETH")
    min_tr = winner_min_trades()
    scaler_ok = all(best["scaler_caus"][a]["ok"] for a in ("BTC", "ETH"))
    causality_ok = bool(caus["ok"] and caus_e["ok"] and scaler_ok)

    es = (mg.get("marginal_verdict") == "ADDS" and mg.get("robust_oos") is True
          and mg.get("is_hedge") is False)
    earns_slot = bool(es)   # study grade for winner is PASS (it's the verified winner)
    beats_winner = bool(earns_slot and best["max_dd"] < 0.30
                        and (w25.get("uplift_hold") or -9) >= 0.55 and best["min_hold"] >= 0.65)

    print("\n" + "=" * 90)
    print("FINAL — BEST VOL-TARGET OVERLAY CONFIG")
    print("=" * 90)
    print(f"  config: mode={best['mode']} target_vol={best['target_vol']:.0%} cap={best['cap']} win={best['win']}")
    print(f"  minFull(daily) = {best['min_full']:+.3f}")
    print(f"  minHold(daily) = {best['min_hold']:+.3f}  (BTC {best['hold_sh']['BTC']:+.2f} / ETH {best['hold_sh']['ETH']:+.2f})")
    print(f"  standalone max_dd (max BTC&ETH scaled) = {best['max_dd']:.4f} "
          f"(BTC {best['dd_scaled']['BTC']:.3f} / ETH {best['dd_scaled']['ETH']:.3f})")
    print(f"  RAW winner daily DD (no overlay)        = BTC {best['dd_raw']['BTC']:.3f} / ETH {best['dd_raw']['ETH']:.3f}")
    print(f"  combined scaled equity max_dd           = {best['comb_dd']:.4f}")
    print(f"  n_trades_min (winner) = {min_tr}")
    print(f"  fee@0.30%RT survives (winner)           = {fee_ok}")
    print(f"  causality_ok (winner entries + scaler)  = {causality_ok}  "
          f"[winner BTC {caus} ETH {caus_e}; scaler BTC {best['scaler_caus']['BTC']} ETH {best['scaler_caus']['ETH']}]")
    print(f"\n  MARGINAL vs TP01 (on scaled 50/50 daily):")
    print(f"    corr_full           = {mg.get('corr_full')}")
    print(f"    corr_hold           = {mg.get('corr_hold')}")
    print(f"    verdict             = {mg.get('marginal_verdict')}")
    print(f"    has_insample_edge   = {mg.get('has_insample_edge')}  cand_insample_sharpe={mg.get('cand_insample_sharpe')}")
    print(f"    is_hedge            = {mg.get('is_hedge')}")
    print(f"    robust_oos          = {mg.get('robust_oos')}")
    print(f"    multicut_persistent = {mg.get('multicut_persistent')}  multicut={mg.get('multicut_uplift')}")
    print(f"    clean_year_uplift   = {mg.get('clean_year_uplift')}  jackknife_min={mg.get('jackknife_min_uplift')}")
    print(f"    blend w25: uplift_hold={w25.get('uplift_hold')} uplift_full={w25.get('uplift_full')} "
          f"hold={w25.get('hold')} full={w25.get('full')} dd={w25.get('dd')}")
    print(f"    blend w50: full={w50.get('full')} hold={w50.get('hold')} "
          f"uplift_hold={w50.get('uplift_hold')} dd={w50.get('dd')}")
    print(f"\n  earns_slot   = {earns_slot}")
    print(f"  BEATS_WINNER = {beats_winner}")
    print("=" * 90)

    # machine-readable tail for the harness
    import json
    out = dict(
        family="voltgt",
        best_config=dict(strategy="winner+voltgt", mode=best["mode"],
                         target_vol=best["target_vol"], cap=best["cap"], win=best["win"],
                         winner=dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24,
                                     uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0)),
        min_full=round(best["min_full"], 3), min_hold=round(best["min_hold"], 3),
        max_dd=round(best["max_dd"], 4), comb_dd=round(best["comb_dd"], 4),
        n_trades_min=min_tr, fee_ok=fee_ok, causality_ok=causality_ok,
        corr_full=mg.get("corr_full"), verdict=mg.get("marginal_verdict"),
        has_insample_edge=mg.get("has_insample_edge"), is_hedge=mg.get("is_hedge"),
        robust_oos=mg.get("robust_oos"), multicut=mg.get("multicut_persistent"),
        clean_year_uplift=mg.get("clean_year_uplift"),
        w25_uplift_hold=w25.get("uplift_hold"),
        earns_slot=earns_slot, beats_winner=beats_winner,
    )
    print("JSON " + json.dumps(out, default=str))