PythagorasGoal/scripts/research/skyhook/runs/SKH2_PATTERN_CONF.py

"""SKH2_PATTERN_CONF — breakout CONFIRMATION filter family (DD-reduction wave).

GOAL of the wave: cut standalone maxDD < 30% (max over BTC&ETH) while keeping
min-asset HOLD-OUT Sharpe >= ~0.70 and earns_slot == True.

FAMILY = breakout confirmation. The main DD source is FALSE breakouts (whipsaws).
We require CONFIRMATION before allowing the composer to fire, via a STRUCTURAL
htf_features patch (causal, shift-safe). Confirmation modes (all use data <= close[i]):

  persist2  : the breakout must PERSIST -> the *previous* HTF close also broke the
              donchian level that was active one bar earlier (2 consecutive breakouts).
  close_loc : the breakout close must sit in the upper/lower `loc_thr` of the HTF
              bar range (close near the high for a long, near the low for a short)
              -> rejects exhaustion wicks that close back inside the bar.
  roc_agree : HTF ROC (close/close[-roc_n]-1) sign must agree with the breakout dir.
  combos    : AND-combinations of the above.

We monkeypatch S.htf_features INSIDE skyhooklib's namespace for the duration of each
study (same safe pattern as SKH_R_EXPAND_study.py): only the feature/composer builder
changes; pattern donchian, regime bands, entry/exit and ALL eval code are unchanged.

Baseline regime/exit params = the verified V2 WINNER:
  SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16,
                vola_lo=35.0, vola_hi=95.0, vol_lo=0.0)

CAUSALITY: every confirmation feature is built from HTF columns and SHIFTED so that
only data up to (and including) the breakout-bar close is used; donchian itself is
shift(1) (strictly prior bars). We verify with sk.causality (truncated-prefix) which
re-runs skyhook_entries on a prefix of BOTH frames -> our patched htf_features is
exercised on the prefix and must match the full run.
"""
from __future__ import annotations

import sys
sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook")

import numpy as np
import pandas as pd

import skyhooklib as sk
from src.strategies import skyhook as S
from src.strategies.skyhook import SkyhookParams, HTF_MIN
from src.strategies.skyhook import atr as _atr, chande01 as _chande01

ORIG_FEAT = S.htf_features


# ---------------------------------------------------------------------------
# Confirmed htf_features builder (STRUCTURAL). Same shape/columns as the engine's
# htf_features, but the composer's pattern leg is gated by a CONFIRMATION mask.
# ---------------------------------------------------------------------------
def conf_htf_features(htf: pd.DataFrame, p: SkyhookParams, *,
                      modes=("persist2",), loc_thr: float = 0.34, roc_n: int = 1):
    """Causal regime+CONFIRMED-pattern features indexed by HTF close.

    modes: subset of {"persist2","close_loc","roc_agree"} ANDed together as the
    confirmation requirement on top of the raw donchian breakout.
    """
    h = htf["high"].values.astype(float)
    l = htf["low"].values.astype(float)
    c = htf["close"].values.astype(float)
    n = len(c)

    # --- regime (unchanged) ---
    buz_vola = _chande01(_atr(htf, p.atr_win), p.n_vola)
    buz_volume = _chande01(htf["volume"].values, p.n_volume)
    regime_ok = ((buz_vola >= p.vola_lo) & (buz_vola <= p.vola_hi)
                 & (buz_volume >= p.vol_lo) & (buz_volume <= p.vol_hi))

    # --- raw donchian breakout (leak-free: close[i] vs max/min of n PRIOR bars) ---
    hi_n = pd.Series(h).rolling(p.ptn_n, min_periods=p.ptn_n).max().shift(1).values
    lo_n = pd.Series(l).rolling(p.ptn_n, min_periods=p.ptn_n).min().shift(1).values
    ptn_long = c > hi_n
    ptn_short = c < lo_n

    # --- CONFIRMATION masks (all causal: built from data <= close[i]) ---
    conf_long = np.ones(n, dtype=bool)
    conf_short = np.ones(n, dtype=bool)

    if "persist2" in modes:
        # previous bar's close also broke the donchian level active ONE bar earlier.
        # ptn_long shifted by 1 == "did the prior bar break out?" (its own causal level).
        prev_long = np.concatenate(([False], ptn_long[:-1]))
        prev_short = np.concatenate(([False], ptn_short[:-1]))
        conf_long &= prev_long
        conf_short &= prev_short

    if "close_loc" in modes:
        rng = h - l
        with np.errstate(divide="ignore", invalid="ignore"):
            pos = np.where(rng > 0, (c - l) / rng, 0.5)   # 0=at low, 1=at high; current bar only
        conf_long &= (pos >= (1.0 - loc_thr))
        conf_short &= (pos <= loc_thr)

    if "roc_agree" in modes:
        cprev = pd.Series(c).shift(roc_n).values            # close roc_n bars ago (causal)
        with np.errstate(divide="ignore", invalid="ignore"):
            roc = np.where(np.isfinite(cprev) & (cprev != 0), c / cprev - 1.0, 0.0)
        conf_long &= (roc > 0.0)
        conf_short &= (roc < 0.0)

    comp_long = regime_ok & ptn_long & conf_long
    comp_short = regime_ok & ptn_short & conf_short
    if p.long_only:
        comp_short = np.zeros_like(comp_short, dtype=bool)

    close_ts = htf["timestamp"].astype("int64").values + HTF_MIN * 60 * 1000
    return pd.DataFrame({
        "close_ts": close_ts,
        "buz_vola": buz_vola, "buz_volume": buz_volume,
        "comp_long": comp_long.astype(bool), "comp_short": comp_short.astype(bool),
    })


def make_patched(**cfg):
    def _feat(htf, p):
        return conf_htf_features(htf, p, **cfg)
    return _feat


def study_conf(name, p, cfg, do_marginal=True):
    """Run sk.study/causality/marginal with htf_features patched to the confirmed builder."""
    S.htf_features = make_patched(**cfg)
    try:
        rep = sk.study(name, p)
        caus_btc = sk.causality(p, "BTC")
        caus_eth = sk.causality(p, "ETH")
        marg = sk.marginal(p) if do_marginal else None
    finally:
        S.htf_features = ORIG_FEAT
    return rep, (caus_btc, caus_eth), marg


def _earns_slot(rep, marg):
    grade_ok = rep["verdict"]["grade"] != "FAIL"
    adds = marg.get("marginal_verdict") == "ADDS"
    robust = bool(marg.get("robust_oos"))
    hedge = bool(marg.get("is_hedge"))
    return bool(grade_ok and adds and robust and (not hedge))


def _beats_winner(rep, marg, max_dd):
    es = _earns_slot(rep, marg)
    w25 = marg.get("blends", {}).get("w25", {}).get("uplift_hold")
    mh = rep["verdict"]["min_asset_holdout_sharpe"]
    return bool(es and max_dd < 0.30 and (w25 is not None and w25 >= 0.55) and mh >= 0.65)


# Baseline regime/exit = verified V2 winner
WIN = dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16,
           vola_lo=35.0, vola_hi=95.0, vol_lo=0.0)


def win_params():
    return SkyhookParams(**WIN)


def win_params_ex(**over):
    d = dict(WIN); d.update(over); return SkyhookParams(**d)


if __name__ == "__main__":
    p = win_params()

    # ---- PHASE 1: scan confirmation modes (quick, no marginal yet) ----
    # winner's exit = sl_atr 2.5 / tp_atr 7.0. close_loc+roc was the clear leader (DD 31.4%,
    # minHold +1.13). Now drive DD<30% via STRONGER close-location confirmation (tighter
    # loc_thr) and TIGHTER stops (sl_atr down), and an upper-vola cap to skip blow-offs.
    CL = ("close_loc",)   # roc_agree is ~collinear with close_loc (no-op); drop it -> cleaner
    C = dict(modes=CL, loc_thr=0.40)
    scan = {
        "RAW (no conf, =winner)": (p, dict(modes=())),
        "cl 0.40 vH90": (win_params_ex(vola_hi=90.0), C),
        "cl 0.40 vH88": (win_params_ex(vola_hi=88.0), C),
        "cl 0.40 vH85": (win_params_ex(vola_hi=85.0), C),
        # volume floor: skip thin (low-volume-cycle) breakouts -> fewer false ETH whipsaws
        "cl 0.40 vH90 volLo20": (win_params_ex(vola_hi=90.0, vol_lo=20.0), C),
        "cl 0.40 vH90 volLo35": (win_params_ex(vola_hi=90.0, vol_lo=35.0), C),
        # raise vola_lo floor (skip dead-vol regimes) + cap top
        "cl 0.40 vL45 vH90": (win_params_ex(vola_lo=45.0, vola_hi=90.0), C),
        # tighten long hold to cut give-back on the trend reversals
        "cl 0.40 vH90 uL20": (win_params_ex(vola_hi=90.0, uscitalong=20), C),
        "cl 0.40 vH90 uL20 uS14": (win_params_ex(vola_hi=90.0, uscitalong=20, uscitashort=14), C),
        # tp tighter to bank wins sooner (less round-trip give-back) — keeps DD lower
        "cl 0.40 vH90 tp6": (win_params_ex(vola_hi=90.0, tp_atr=6.0), C),
        "cl 0.40 vH90 tp6 uL20": (win_params_ex(vola_hi=90.0, tp_atr=6.0, uscitalong=20), C),
        "cl 0.40 vH90 volLo20 uL20": (win_params_ex(vola_hi=90.0, vol_lo=20.0, uscitalong=20), C),
    }
    print("=" * 78)
    print("PHASE 1 SCAN — confirmation modes on the V2 winner (DD-focus)")
    print("=" * 78)
    rows = []
    for name, (pp, cfg) in scan.items():
        rep, caus, _ = study_conf(name, pp, cfg, do_marginal=False)
        v = rep["verdict"]
        mdd = max(rep["per_asset"][a]["full"]["maxdd"] for a in rep["per_asset"])
        cb = caus[0]["ok"] and caus[1]["ok"]
        nmin = min(rep["per_asset"][a]["full"]["n_trades"] for a in rep["per_asset"])
        rows.append((name, (pp, cfg), v["grade"], v["min_asset_full_sharpe"],
                     v["min_asset_holdout_sharpe"], mdd, nmin, v["fee_survives"], cb))
        print(f"  {name:30s} grade={v['grade']:4s} minFull={v['min_asset_full_sharpe']:+.2f} "
              f"minHold={v['min_asset_holdout_sharpe']:+.2f} maxDD={mdd*100:4.0f}% "
              f"nMin={nmin:4d} feeOK={v['fee_survives']} caus={cb}")

    # pick candidates: grade != FAIL, maxDD lowest, holdout decent. Take all with DD<32% & minHold>0.4.
    cands = [r for r in rows if r[2] != "FAIL" and r[7] and r[8]
             and r[5] < 0.33 and r[4] >= 0.40 and r[0] != "RAW (no conf, =winner)"]
    # sort: sub-30% DD first (the wave goal), then highest hold
    cands.sort(key=lambda r: (r[5] >= 0.30, r[5], -r[4]))
    print("\nPHASE 1 candidates (DD<35%, minHold>=0.40, feeOK, causal), best DD first:")
    for r in cands:
        print(f"  {r[0]:24s} DD={r[5]*100:.0f}% minHold={r[4]:+.2f} minFull={r[3]:+.2f}")

    # ---- PHASE 2: full marginal on the top few ----
    top = cands[:5] if cands else []
    if not top:
        # fall back to the lowest-DD non-FAIL configs regardless of hold threshold
        fb = [r for r in rows if r[2] != "FAIL" and r[7] and r[8]]
        fb.sort(key=lambda r: r[5])
        top = fb[:3]

    print("\n" + "=" * 78)
    print("PHASE 2 — full marginal vs TP01 on top confirmation candidates")
    print("=" * 78)

    best = None  # (beats, earns, max_dd, minHold, name, cfg, rep, marg)
    for r in top:
        name, (pp, cfg) = r[0], r[1]
        rep, caus, marg = study_conf(name, pp, cfg, do_marginal=True)
        v = rep["verdict"]
        mdd = max(rep["per_asset"][a]["full"]["maxdd"] for a in rep["per_asset"])
        cb = caus[0]["ok"] and caus[1]["ok"]
        es = _earns_slot(rep, marg)
        bw = _beats_winner(rep, marg, mdd)
        w25 = marg.get("blends", {}).get("w25", {})
        w50 = marg.get("blends", {}).get("w50", {})
        print(f"\n----- {name}  cfg={cfg} -----")
        print(sk.fmt(rep))
        print(f"causality: BTC={caus[0]} ETH={caus[1]}  -> ok={cb}")
        print(f"marginal: verdict={marg.get('marginal_verdict')} corr_full={marg.get('corr_full')} "
              f"corr_hold={marg.get('corr_hold')}")
        print(f"          has_insample_edge={marg.get('has_insample_edge')} "
              f"cand_insample_sharpe={marg.get('cand_insample_sharpe')} "
              f"is_hedge={marg.get('is_hedge')} robust_oos={marg.get('robust_oos')} "
              f"multicut_persistent={marg.get('multicut_persistent')}")
        print(f"          clean_year_uplift={marg.get('clean_year_uplift')} "
              f"jackknife_min_uplift={marg.get('jackknife_min_uplift')} "
              f"multicut_uplift={marg.get('multicut_uplift')}")
        print(f"          blend w25: uplift_hold={w25.get('uplift_hold')} uplift_full={w25.get('uplift_full')} "
              f"hold={w25.get('hold')} full={w25.get('full')}")
        print(f"          blend w50: full={w50.get('full')} hold={w50.get('hold')} "
              f"uplift_hold={w50.get('uplift_hold')} dd={w50.get('dd')}")
        print(f"  EARNS_SLOT={es}  BEATS_WINNER={bw}")
        key = (bw, es, -mdd, v["min_asset_holdout_sharpe"])
        cur = dict(beats=bw, earns=es, max_dd=mdd, minHold=v["min_asset_holdout_sharpe"],
                   minFull=v["min_asset_full_sharpe"], name=name, cfg=cfg, rep=rep, marg=marg,
                   base=sk._params_dict(pp),
                   caus=cb, nmin=min(rep["per_asset"][a]["full"]["n_trades"] for a in rep["per_asset"]),
                   feeOK=v["fee_survives"], grade=v["grade"])
        if best is None or key > best[0]:
            best = (key, cur)

    # ---- FINAL BLOCK ----
    print("\n" + "#" * 78)
    print("FINAL — BEST PATTERN_CONF CONFIG")
    print("#" * 78)
    if best is None:
        print("NO non-FAIL candidate found.")
    else:
        b = best[1]
        m = b["marg"]
        w25 = m.get("blends", {}).get("w25", {})
        w50 = m.get("blends", {}).get("w50", {})
        print(f"name        : {b['name']}")
        print(f"cfg         : {b['cfg']}")
        print(f"base params : {b['base']}")
        print(f"grade       : {b['grade']}")
        print(f"minFull     : {b['minFull']:+.3f}")
        print(f"minHold     : {b['minHold']:+.3f}")
        print(f"max_dd      : {b['max_dd']:.4f}  ({b['max_dd']*100:.1f}%)")
        print(f"n_trades_min: {b['nmin']}")
        print(f"fee@0.30%   : survives={b['feeOK']}")
        print(f"causality_ok: {b['caus']}")
        print(f"--- marginal dict ---")
        print(f"  corr_full          : {m.get('corr_full')}")
        print(f"  corr_hold          : {m.get('corr_hold')}")
        print(f"  marginal_verdict   : {m.get('marginal_verdict')}")
        print(f"  has_insample_edge  : {m.get('has_insample_edge')}")
        print(f"  cand_insample_sh   : {m.get('cand_insample_sharpe')}")
        print(f"  is_hedge           : {m.get('is_hedge')}")
        print(f"  robust_oos         : {m.get('robust_oos')}")
        print(f"  multicut_persistent: {m.get('multicut_persistent')}")
        print(f"  clean_year_uplift  : {m.get('clean_year_uplift')}")
        print(f"  blend w25 uplift_hold: {w25.get('uplift_hold')}")
        print(f"  blend w25 uplift_full: {w25.get('uplift_full')}")
        print(f"  blend w50          : full={w50.get('full')} hold={w50.get('hold')} dd={w50.get('dd')}")
        print(f"EARNS_SLOT  : {b['earns']}")
        print(f"BEATS_WINNER: {b['beats']}")