"""SKH2_PATTERN_CONF — breakout CONFIRMATION filter family (DD-reduction wave). GOAL of the wave: cut standalone maxDD < 30% (max over BTCÐ) while keeping min-asset HOLD-OUT Sharpe >= ~0.70 and earns_slot == True. FAMILY = breakout confirmation. The main DD source is FALSE breakouts (whipsaws). We require CONFIRMATION before allowing the composer to fire, via a STRUCTURAL htf_features patch (causal, shift-safe). Confirmation modes (all use data <= close[i]): persist2 : the breakout must PERSIST -> the *previous* HTF close also broke the donchian level that was active one bar earlier (2 consecutive breakouts). close_loc : the breakout close must sit in the upper/lower `loc_thr` of the HTF bar range (close near the high for a long, near the low for a short) -> rejects exhaustion wicks that close back inside the bar. roc_agree : HTF ROC (close/close[-roc_n]-1) sign must agree with the breakout dir. combos : AND-combinations of the above. We monkeypatch S.htf_features INSIDE skyhooklib's namespace for the duration of each study (same safe pattern as SKH_R_EXPAND_study.py): only the feature/composer builder changes; pattern donchian, regime bands, entry/exit and ALL eval code are unchanged. Baseline regime/exit params = the verified V2 WINNER: SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0) CAUSALITY: every confirmation feature is built from HTF columns and SHIFTED so that only data up to (and including) the breakout-bar close is used; donchian itself is shift(1) (strictly prior bars). We verify with sk.causality (truncated-prefix) which re-runs skyhook_entries on a prefix of BOTH frames -> our patched htf_features is exercised on the prefix and must match the full run. """ from __future__ import annotations import sys sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook") import numpy as np import pandas as pd import skyhooklib as sk from src.strategies import skyhook as S from src.strategies.skyhook import SkyhookParams, HTF_MIN from src.strategies.skyhook import atr as _atr, chande01 as _chande01 ORIG_FEAT = S.htf_features # --------------------------------------------------------------------------- # Confirmed htf_features builder (STRUCTURAL). Same shape/columns as the engine's # htf_features, but the composer's pattern leg is gated by a CONFIRMATION mask. # --------------------------------------------------------------------------- def conf_htf_features(htf: pd.DataFrame, p: SkyhookParams, *, modes=("persist2",), loc_thr: float = 0.34, roc_n: int = 1): """Causal regime+CONFIRMED-pattern features indexed by HTF close. modes: subset of {"persist2","close_loc","roc_agree"} ANDed together as the confirmation requirement on top of the raw donchian breakout. """ h = htf["high"].values.astype(float) l = htf["low"].values.astype(float) c = htf["close"].values.astype(float) n = len(c) # --- regime (unchanged) --- buz_vola = _chande01(_atr(htf, p.atr_win), p.n_vola) buz_volume = _chande01(htf["volume"].values, p.n_volume) regime_ok = ((buz_vola >= p.vola_lo) & (buz_vola <= p.vola_hi) & (buz_volume >= p.vol_lo) & (buz_volume <= p.vol_hi)) # --- raw donchian breakout (leak-free: close[i] vs max/min of n PRIOR bars) --- hi_n = pd.Series(h).rolling(p.ptn_n, min_periods=p.ptn_n).max().shift(1).values lo_n = pd.Series(l).rolling(p.ptn_n, min_periods=p.ptn_n).min().shift(1).values ptn_long = c > hi_n ptn_short = c < lo_n # --- CONFIRMATION masks (all causal: built from data <= close[i]) --- conf_long = np.ones(n, dtype=bool) conf_short = np.ones(n, dtype=bool) if "persist2" in modes: # previous bar's close also broke the donchian level active ONE bar earlier. # ptn_long shifted by 1 == "did the prior bar break out?" (its own causal level). prev_long = np.concatenate(([False], ptn_long[:-1])) prev_short = np.concatenate(([False], ptn_short[:-1])) conf_long &= prev_long conf_short &= prev_short if "close_loc" in modes: rng = h - l with np.errstate(divide="ignore", invalid="ignore"): pos = np.where(rng > 0, (c - l) / rng, 0.5) # 0=at low, 1=at high; current bar only conf_long &= (pos >= (1.0 - loc_thr)) conf_short &= (pos <= loc_thr) if "roc_agree" in modes: cprev = pd.Series(c).shift(roc_n).values # close roc_n bars ago (causal) with np.errstate(divide="ignore", invalid="ignore"): roc = np.where(np.isfinite(cprev) & (cprev != 0), c / cprev - 1.0, 0.0) conf_long &= (roc > 0.0) conf_short &= (roc < 0.0) comp_long = regime_ok & ptn_long & conf_long comp_short = regime_ok & ptn_short & conf_short if p.long_only: comp_short = np.zeros_like(comp_short, dtype=bool) close_ts = htf["timestamp"].astype("int64").values + HTF_MIN * 60 * 1000 return pd.DataFrame({ "close_ts": close_ts, "buz_vola": buz_vola, "buz_volume": buz_volume, "comp_long": comp_long.astype(bool), "comp_short": comp_short.astype(bool), }) def make_patched(**cfg): def _feat(htf, p): return conf_htf_features(htf, p, **cfg) return _feat def study_conf(name, p, cfg, do_marginal=True): """Run sk.study/causality/marginal with htf_features patched to the confirmed builder.""" S.htf_features = make_patched(**cfg) try: rep = sk.study(name, p) caus_btc = sk.causality(p, "BTC") caus_eth = sk.causality(p, "ETH") marg = sk.marginal(p) if do_marginal else None finally: S.htf_features = ORIG_FEAT return rep, (caus_btc, caus_eth), marg def _earns_slot(rep, marg): grade_ok = rep["verdict"]["grade"] != "FAIL" adds = marg.get("marginal_verdict") == "ADDS" robust = bool(marg.get("robust_oos")) hedge = bool(marg.get("is_hedge")) return bool(grade_ok and adds and robust and (not hedge)) def _beats_winner(rep, marg, max_dd): es = _earns_slot(rep, marg) w25 = marg.get("blends", {}).get("w25", {}).get("uplift_hold") mh = rep["verdict"]["min_asset_holdout_sharpe"] return bool(es and max_dd < 0.30 and (w25 is not None and w25 >= 0.55) and mh >= 0.65) # Baseline regime/exit = verified V2 winner WIN = dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0) def win_params(): return SkyhookParams(**WIN) def win_params_ex(**over): d = dict(WIN); d.update(over); return SkyhookParams(**d) if __name__ == "__main__": p = win_params() # ---- PHASE 1: scan confirmation modes (quick, no marginal yet) ---- # winner's exit = sl_atr 2.5 / tp_atr 7.0. close_loc+roc was the clear leader (DD 31.4%, # minHold +1.13). Now drive DD<30% via STRONGER close-location confirmation (tighter # loc_thr) and TIGHTER stops (sl_atr down), and an upper-vola cap to skip blow-offs. CL = ("close_loc",) # roc_agree is ~collinear with close_loc (no-op); drop it -> cleaner C = dict(modes=CL, loc_thr=0.40) scan = { "RAW (no conf, =winner)": (p, dict(modes=())), "cl 0.40 vH90": (win_params_ex(vola_hi=90.0), C), "cl 0.40 vH88": (win_params_ex(vola_hi=88.0), C), "cl 0.40 vH85": (win_params_ex(vola_hi=85.0), C), # volume floor: skip thin (low-volume-cycle) breakouts -> fewer false ETH whipsaws "cl 0.40 vH90 volLo20": (win_params_ex(vola_hi=90.0, vol_lo=20.0), C), "cl 0.40 vH90 volLo35": (win_params_ex(vola_hi=90.0, vol_lo=35.0), C), # raise vola_lo floor (skip dead-vol regimes) + cap top "cl 0.40 vL45 vH90": (win_params_ex(vola_lo=45.0, vola_hi=90.0), C), # tighten long hold to cut give-back on the trend reversals "cl 0.40 vH90 uL20": (win_params_ex(vola_hi=90.0, uscitalong=20), C), "cl 0.40 vH90 uL20 uS14": (win_params_ex(vola_hi=90.0, uscitalong=20, uscitashort=14), C), # tp tighter to bank wins sooner (less round-trip give-back) — keeps DD lower "cl 0.40 vH90 tp6": (win_params_ex(vola_hi=90.0, tp_atr=6.0), C), "cl 0.40 vH90 tp6 uL20": (win_params_ex(vola_hi=90.0, tp_atr=6.0, uscitalong=20), C), "cl 0.40 vH90 volLo20 uL20": (win_params_ex(vola_hi=90.0, vol_lo=20.0, uscitalong=20), C), } print("=" * 78) print("PHASE 1 SCAN — confirmation modes on the V2 winner (DD-focus)") print("=" * 78) rows = [] for name, (pp, cfg) in scan.items(): rep, caus, _ = study_conf(name, pp, cfg, do_marginal=False) v = rep["verdict"] mdd = max(rep["per_asset"][a]["full"]["maxdd"] for a in rep["per_asset"]) cb = caus[0]["ok"] and caus[1]["ok"] nmin = min(rep["per_asset"][a]["full"]["n_trades"] for a in rep["per_asset"]) rows.append((name, (pp, cfg), v["grade"], v["min_asset_full_sharpe"], v["min_asset_holdout_sharpe"], mdd, nmin, v["fee_survives"], cb)) print(f" {name:30s} grade={v['grade']:4s} minFull={v['min_asset_full_sharpe']:+.2f} " f"minHold={v['min_asset_holdout_sharpe']:+.2f} maxDD={mdd*100:4.0f}% " f"nMin={nmin:4d} feeOK={v['fee_survives']} caus={cb}") # pick candidates: grade != FAIL, maxDD lowest, holdout decent. Take all with DD<32% & minHold>0.4. cands = [r for r in rows if r[2] != "FAIL" and r[7] and r[8] and r[5] < 0.33 and r[4] >= 0.40 and r[0] != "RAW (no conf, =winner)"] # sort: sub-30% DD first (the wave goal), then highest hold cands.sort(key=lambda r: (r[5] >= 0.30, r[5], -r[4])) print("\nPHASE 1 candidates (DD<35%, minHold>=0.40, feeOK, causal), best DD first:") for r in cands: print(f" {r[0]:24s} DD={r[5]*100:.0f}% minHold={r[4]:+.2f} minFull={r[3]:+.2f}") # ---- PHASE 2: full marginal on the top few ---- top = cands[:5] if cands else [] if not top: # fall back to the lowest-DD non-FAIL configs regardless of hold threshold fb = [r for r in rows if r[2] != "FAIL" and r[7] and r[8]] fb.sort(key=lambda r: r[5]) top = fb[:3] print("\n" + "=" * 78) print("PHASE 2 — full marginal vs TP01 on top confirmation candidates") print("=" * 78) best = None # (beats, earns, max_dd, minHold, name, cfg, rep, marg) for r in top: name, (pp, cfg) = r[0], r[1] rep, caus, marg = study_conf(name, pp, cfg, do_marginal=True) v = rep["verdict"] mdd = max(rep["per_asset"][a]["full"]["maxdd"] for a in rep["per_asset"]) cb = caus[0]["ok"] and caus[1]["ok"] es = _earns_slot(rep, marg) bw = _beats_winner(rep, marg, mdd) w25 = marg.get("blends", {}).get("w25", {}) w50 = marg.get("blends", {}).get("w50", {}) print(f"\n----- {name} cfg={cfg} -----") print(sk.fmt(rep)) print(f"causality: BTC={caus[0]} ETH={caus[1]} -> ok={cb}") print(f"marginal: verdict={marg.get('marginal_verdict')} corr_full={marg.get('corr_full')} " f"corr_hold={marg.get('corr_hold')}") print(f" has_insample_edge={marg.get('has_insample_edge')} " f"cand_insample_sharpe={marg.get('cand_insample_sharpe')} " f"is_hedge={marg.get('is_hedge')} robust_oos={marg.get('robust_oos')} " f"multicut_persistent={marg.get('multicut_persistent')}") print(f" clean_year_uplift={marg.get('clean_year_uplift')} " f"jackknife_min_uplift={marg.get('jackknife_min_uplift')} " f"multicut_uplift={marg.get('multicut_uplift')}") print(f" blend w25: uplift_hold={w25.get('uplift_hold')} uplift_full={w25.get('uplift_full')} " f"hold={w25.get('hold')} full={w25.get('full')}") print(f" blend w50: full={w50.get('full')} hold={w50.get('hold')} " f"uplift_hold={w50.get('uplift_hold')} dd={w50.get('dd')}") print(f" EARNS_SLOT={es} BEATS_WINNER={bw}") key = (bw, es, -mdd, v["min_asset_holdout_sharpe"]) cur = dict(beats=bw, earns=es, max_dd=mdd, minHold=v["min_asset_holdout_sharpe"], minFull=v["min_asset_full_sharpe"], name=name, cfg=cfg, rep=rep, marg=marg, base=sk._params_dict(pp), caus=cb, nmin=min(rep["per_asset"][a]["full"]["n_trades"] for a in rep["per_asset"]), feeOK=v["fee_survives"], grade=v["grade"]) if best is None or key > best[0]: best = (key, cur) # ---- FINAL BLOCK ---- print("\n" + "#" * 78) print("FINAL — BEST PATTERN_CONF CONFIG") print("#" * 78) if best is None: print("NO non-FAIL candidate found.") else: b = best[1] m = b["marg"] w25 = m.get("blends", {}).get("w25", {}) w50 = m.get("blends", {}).get("w50", {}) print(f"name : {b['name']}") print(f"cfg : {b['cfg']}") print(f"base params : {b['base']}") print(f"grade : {b['grade']}") print(f"minFull : {b['minFull']:+.3f}") print(f"minHold : {b['minHold']:+.3f}") print(f"max_dd : {b['max_dd']:.4f} ({b['max_dd']*100:.1f}%)") print(f"n_trades_min: {b['nmin']}") print(f"fee@0.30% : survives={b['feeOK']}") print(f"causality_ok: {b['caus']}") print(f"--- marginal dict ---") print(f" corr_full : {m.get('corr_full')}") print(f" corr_hold : {m.get('corr_hold')}") print(f" marginal_verdict : {m.get('marginal_verdict')}") print(f" has_insample_edge : {m.get('has_insample_edge')}") print(f" cand_insample_sh : {m.get('cand_insample_sharpe')}") print(f" is_hedge : {m.get('is_hedge')}") print(f" robust_oos : {m.get('robust_oos')}") print(f" multicut_persistent: {m.get('multicut_persistent')}") print(f" clean_year_uplift : {m.get('clean_year_uplift')}") print(f" blend w25 uplift_hold: {w25.get('uplift_hold')}") print(f" blend w25 uplift_full: {w25.get('uplift_full')}") print(f" blend w50 : full={w50.get('full')} hold={w50.get('hold')} dd={w50.get('dd')}") print(f"EARNS_SLOT : {b['earns']}") print(f"BEATS_WINNER: {b['beats']}")