"""SKH2_ENS_STRUCT — cross-definition ENSEMBLE [ENS_STRUCT]. WAVE GOAL: cut the V2 winner's STANDALONE max-DD below 30% (BTC 34% / ETH 31% is the only unmet goal) while keeping min-asset HOLD-OUT >= ~0.70 and earns_slot True. IDEA: the V2 winner uses ONE regime definition (Chande01 cycle band on ATR + Donchian breakout). Its drawdowns come from that one signal source firing into the wrong tape. If we ensemble it with STRUCTURALLY DIFFERENT regime definitions — (B) causal PERCENTILE-RANK regime (SKH_R_PCTL) and (C) VOLATILITY-EXPANSION regime (SKH_R_EXPAND) — that disagree about WHEN to trade, their drawdowns are imperfectly correlated. Equal-weighting the three daily-return streams (per asset, then 50/50 across BTC+ETH) should reduce the COMBINED-equity DD below any single member's DD, at a modest cost to full Sharpe. The ensemble is EQUAL-WEIGHT on DAILY RETURNS (a constant-weight rebalanced book of three sub-sleeves on the SAME asset). Standalone DD = max-DD of the COMBINED equity curve (per asset, max over BTC & ETH). Marginal vs TP01 uses the 50/50 BTC+ETH combined daily series. All three members are causal/leak-free (winner via sk.causality; PCTL/EXPAND via their own truncated-prefix guards, re-run here). Equal-weighting causal streams stays causal. """ from __future__ import annotations import sys sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook") import importlib.util import numpy as np import pandas as pd import skyhooklib as sk import altlib as al from src.backtest.harness import backtest_signals from src.strategies import skyhook as S from src.strategies.skyhook import SkyhookParams HOLDOUT = sk.HOLDOUT FEE = sk.FEE_RT # --- import the two structural entry builders without running their __main__ ---------------- def _load(modname, path): spec = importlib.util.spec_from_file_location(modname, path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod PCTL = _load("skr_pctl", "/opt/docker/PythagorasGoal/scripts/research/skyhook/runs/SKH_R_PCTL.py") EXPD = _load("skr_expd", "/opt/docker/PythagorasGoal/scripts/research/skyhook/runs/SKH_R_EXPAND.py") # --- the V2 winner from the prior wave (Chande/Donchian regime) ----------------------------- WINNER = SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0) # pattern/exit shared knobs for the structural members. Match the WINNER's exit profile # (sl2.5/tp7.0, asym time exits) so the difference is the REGIME, not the exit. def member_params(**kw): base = dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16) base.update(kw) return SkyhookParams(**base) # ============================================================================================ # Per-asset equity helpers. We need the FULL bar-level equity curve of each member so we can # build the COMBINED equity (for an honest combined max-DD), AND the daily-return series (for # Sharpe split + marginal). All on the SAME 230m execution index. # ============================================================================================ def member_equity(asset, kind, p, cfg=None): """Return (eq, idx) bar-level equity for a member. kind in {'winner','pctl','expand'}.""" ltf, htf = sk.frames(asset) if kind == "winner": ent = S.skyhook_entries(ltf, htf, p) elif kind == "pctl": ent = PCTL.pctl_entries(ltf, htf, p, **cfg) elif kind == "expand": ent = EXPD.expand_entries(ltf, htf, p, **cfg) else: raise ValueError(kind) m = backtest_signals(ltf, ent, fee_rt=FEE, leverage=1.0, asset=asset, tf="230m") idx = pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True)) return np.asarray(m.equity, float), idx, int(m.n_trades) def eq_to_daily_ret(eq, idx): """bar equity -> daily simple returns (last-of-day, ffill, pct_change).""" s = pd.Series(eq, index=idx) return s.resample("1D").last().ffill().pct_change() def combined_daily_ret(asset, members): """Equal-weight DAILY returns of the members on one asset -> combined daily return series. Each member contributes its own daily return; the equal-weight portfolio return is the mean. Members are aligned on the union of daily timestamps; a member that has not started yet (NaN) contributes 0 that day and is dropped from the active-weight denominator (outer-join with renormalized equal weights), so early days where only some members trade are handled.""" drs = {} ntr = {} for name, kind, p, cfg in members: eq, idx, nt = member_equity(asset, kind, p, cfg) drs[name] = eq_to_daily_ret(eq, idx) ntr[name] = nt D = pd.concat(drs, axis=1, join="outer") # renormalized equal weight across the members that are ACTIVE (non-NaN) each day active = D.notna() w = active.div(active.sum(axis=1).replace(0, np.nan), axis=0) comb = (D.fillna(0.0) * w.fillna(0.0)).sum(axis=1) comb = comb.dropna() return comb, ntr def combined_metrics(comb): """Sharpe (full + hold-out) and max-DD from a DAILY combined return series.""" comb = comb[np.isfinite(comb.values)] eq = (1.0 + comb).cumprod().values idx = comb.index def _sh(r): r = r[np.isfinite(r)] return float(np.mean(r) / np.std(r) * np.sqrt(365.25)) if len(r) and np.std(r) > 0 else 0.0 pk = np.maximum.accumulate(eq) dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0 full_sh = _sh(comb.values) hmask = idx >= HOLDOUT hold_sh = _sh(comb.values[hmask]) if hmask.sum() > 5 else 0.0 # hold-out DD on the hold-out slice of the combined equity eqh = eq[hmask] pkh = np.maximum.accumulate(eqh) if len(eqh) else np.array([1.0]) ddh = float(np.max((pkh - eqh) / pkh)) if len(eqh) else 0.0 yrs = {} for y in sorted(set(idx.year)): rr = comb.values[idx.year == y] yrs[int(y)] = round(float((1 + pd.Series(rr)).prod() - 1), 4) return dict(full_sharpe=round(full_sh, 3), hold_sharpe=round(hold_sh, 3), maxdd=round(dd, 4), hold_maxdd=round(ddh, 4), yearly=yrs) # --- per-member standalone DD (for the correlation/decorrelation diagnostic) ---------------- def member_standalone_dd(asset, kind, p, cfg=None): eq, idx, nt = member_equity(asset, kind, p, cfg) pk = np.maximum.accumulate(eq) dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0 return dd, nt # ============================================================================================ # Fee robustness of the ENSEMBLE: re-run every member at fee f, recombine, check Sharpe>0. # ============================================================================================ def ensemble_fee_sweep(members, fees=(0.0, 0.001, 0.002, 0.003)): rows = {} for f in fees: ok = True for asset in ("BTC", "ETH"): drs = {} for name, kind, p, cfg in members: ltf, htf = sk.frames(asset) if kind == "winner": ent = S.skyhook_entries(ltf, htf, p) elif kind == "pctl": ent = PCTL.pctl_entries(ltf, htf, p, **cfg) else: ent = EXPD.expand_entries(ltf, htf, p, **cfg) m = backtest_signals(ltf, ent, fee_rt=f, leverage=1.0, asset=asset, tf="230m") drs[name] = eq_to_daily_ret(np.asarray(m.equity, float), pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True))) D = pd.concat(drs, axis=1, join="outer") active = D.notna() w = active.div(active.sum(axis=1).replace(0, np.nan), axis=0) comb = (D.fillna(0.0) * w.fillna(0.0)).sum(axis=1).dropna() sh = combined_metrics(comb)["full_sharpe"] rows[(f, asset)] = sh ok = ok and (sh > 0) rows[(f, "ok")] = ok return rows def ensemble_daily_5050(members): """50/50 BTC+ETH combined daily series for marginal_vs_tp01.""" cb, _ = combined_daily_ret("BTC", members) ce, _ = combined_daily_ret("ETH", members) J = pd.concat({"BTC": cb, "ETH": ce}, axis=1, join="inner").fillna(0.0) return 0.5 * J["BTC"] + 0.5 * J["ETH"] # ============================================================================================ def run_ensemble(tag, members): print(f"\n========== ENSEMBLE: {tag} ==========") print(f" members: {[m[0] for m in members]}") per = {} for asset in ("BTC", "ETH"): comb, ntr = combined_daily_ret(asset, members) met = combined_metrics(comb) per[asset] = dict(met=met, ntr=ntr) print(f" {asset}: FULL Sh={met['full_sharpe']:+.2f} HOLD Sh={met['hold_sharpe']:+.2f}" f" maxDD={met['maxdd']*100:.0f}% holdDD={met['hold_maxdd']*100:.0f}% ntrades={ntr}") print(f" yearly: " + " ".join(f"{y}:{v*100:+.0f}%" for y, v in met['yearly'].items())) min_full = min(per[a]["met"]["full_sharpe"] for a in per) min_hold = min(per[a]["met"]["hold_sharpe"] for a in per) max_dd = max(per[a]["met"]["maxdd"] for a in per) n_trades_min = min(sum(per[a]["ntr"].values()) for a in per) print(f" -> minFull={min_full:+.2f} minHold={min_hold:+.2f} maxDD={max_dd*100:.0f}%" f" nTrades(min,sum-of-members)={n_trades_min}") return dict(per=per, min_full=min_full, min_hold=min_hold, max_dd=max_dd, n_trades_min=n_trades_min) if __name__ == "__main__": print("=== SKH2_ENS_STRUCT: cross-definition regime ensemble (DD reduction) ===") # --- 0) Baseline: the V2 winner ALONE (reference) ----------------------------------- print("\n--- V2 WINNER standalone (Chande/Donchian) ---") w_per = {} for a in ("BTC", "ETH"): r = sk.run_asset(a, WINNER, FEE) w_per[a] = r print(f" {a}: FULL Sh={r['full']['sharpe']:+.2f} DD={r['full']['maxdd']*100:.0f}%" f" n={r['full']['n_trades']} | HOLD Sh={r['holdout']['sharpe']:+.2f}") w_minfull = min(w_per[a]["full"]["sharpe"] for a in w_per) w_minhold = min(w_per[a]["holdout"]["sharpe"] for a in w_per) w_maxdd = max(w_per[a]["full"]["maxdd"] for a in w_per) print(f" WINNER: minFull={w_minfull:+.2f} minHold={w_minhold:+.2f} maxDD={w_maxdd*100:.0f}%") # --- 1) Structural member configs (different regime definitions) -------------------- pP = member_params() pE = member_params() # PCTL: expanding percentile-rank regime. From SKH_R_PCTL_final, the lower-DD / robust # band was the low-vola band; we use a mid band with a modest volume floor so it disagrees # with the winner's HIGH-vola Chande band (different regime -> decorrelated DD). PCTL_CFG = dict(vola_win=None, vol_win=None, vola_lo=0.0, vola_hi=0.70, vol_lo=0.0, vol_hi=1.0) PCTL_CFG2 = dict(vola_win=None, vol_win=None, vola_lo=0.10, vola_hi=0.80, vol_lo=0.30, vol_hi=1.0) # EXPAND: volatility-expansion regime (ATR above its MA + volume elevated). EXP_CFG = dict(w_atr=20, k_atr=1.00, w_vol=20, k_vol=1.00) EXP_CFG2 = dict(w_atr=20, k_atr=1.10, w_vol=20, k_vol=1.20) # --- standalone DD + pairwise correlation of the three regime streams (BTC) --------- print("\n--- standalone member DD + pairwise daily-return corr (decorrelation check) ---") for a in ("BTC", "ETH"): dW, nW = member_standalone_dd(a, "winner", WINNER) dP, nP = member_standalone_dd(a, "pctl", pP, PCTL_CFG) dE, nE = member_standalone_dd(a, "expand", pE, EXP_CFG) print(f" {a} standalone DD: winner={dW*100:.0f}%(n{nW}) pctl={dP*100:.0f}%(n{nP}) expand={dE*100:.0f}%(n{nE})") # corr of daily returns rw = eq_to_daily_ret(*member_equity(a, "winner", WINNER)[:2]) rp = eq_to_daily_ret(*member_equity(a, "pctl", pP, PCTL_CFG)[:2]) re_ = eq_to_daily_ret(*member_equity(a, "expand", pE, EXP_CFG)[:2]) DD = pd.concat({"W": rw, "P": rp, "E": re_}, axis=1, join="inner").fillna(0.0) cc = DD.corr() print(f" corr W-P={cc.loc['W','P']:.2f} W-E={cc.loc['W','E']:.2f} P-E={cc.loc['P','E']:.2f}") # --- 2) Candidate ensembles --------------------------------------------------------- candidates = { "WPE (winner+pctlLo+expand)": [ ("winner", "winner", WINNER, None), ("pctl", "pctl", pP, PCTL_CFG), ("expand", "expand", pE, EXP_CFG), ], "WP (winner+pctlLo)": [ ("winner", "winner", WINNER, None), ("pctl", "pctl", pP, PCTL_CFG), ], "WE (winner+expand)": [ ("winner", "winner", WINNER, None), ("expand", "expand", pE, EXP_CFG), ], "WPE2 (winner+pctlMid+expandStrong)": [ ("winner", "winner", WINNER, None), ("pctl", "pctl", pP, PCTL_CFG2), ("expand", "expand", pE, EXP_CFG2), ], } results = {} for tag, members in candidates.items(): results[tag] = (members, run_ensemble(tag, members)) # --- 3) Pick best by: max_dd<0.30, then maximize min_hold --------------------------- def score(v): # prefer DD<0.30 AND min_hold>=0.65; among those, maximize min_hold then -DD ok = v["max_dd"] < 0.30 and v["min_hold"] >= 0.65 return (1 if ok else 0, v["min_hold"], -v["max_dd"]) best_tag = max(results, key=lambda t: score(results[t][1])) best_members, best_v = results[best_tag] print(f"\n*** BEST ENSEMBLE = {best_tag} ***") print(f" minFull={best_v['min_full']:+.2f} minHold={best_v['min_hold']:+.2f}" f" maxDD={best_v['max_dd']*100:.0f}% nTradesMin={best_v['n_trades_min']}") # --- 4) Causality: winner via sk; structural members via their own guards ----------- print("\n--- causality (all active members of best ensemble) ---") caus_ok = True kinds = {m[0]: (m[1], m[2], m[3]) for m in best_members} if "winner" in kinds: cb = sk.causality(WINNER, "BTC"); ce = sk.causality(WINNER, "ETH") print(f" winner: BTC {cb} ETH {ce}") caus_ok = caus_ok and cb["ok"] and ce["ok"] if "pctl" in kinds: _, p, cfg = kinds["pctl"] cb = PCTL.check_causality(cfg, p, "BTC"); ce = PCTL.check_causality(cfg, p, "ETH") print(f" pctl: BTC {cb} ETH {ce}") caus_ok = caus_ok and cb["ok"] and ce["ok"] if "expand" in kinds: _, p, cfg = kinds["expand"] cb = EXPD.check_causality(cfg, p, "BTC"); ce = EXPD.check_causality(cfg, p, "ETH") print(f" expand: BTC {cb} ETH {ce}") caus_ok = caus_ok and cb["ok"] and ce["ok"] print(f" causality_ok (all members) = {caus_ok}") # --- 5) Fee sweep on the ensemble --------------------------------------------------- print("\n--- ensemble fee sweep (FULL Sharpe per asset) ---") fsw = ensemble_fee_sweep(best_members) for f in (0.0, 0.001, 0.002, 0.003): print(f" {f*100:.2f}%RT: BTC={fsw[(f,'BTC')]:+.2f} ETH={fsw[(f,'ETH')]:+.2f} ok={fsw[(f,'ok')]}") fee_survives = fsw[(0.003, "ok")] print(f" fee_survives 0.30%RT (both): {fee_survives}") # --- 6) Marginal vs TP01 on the ensemble 50/50 series ------------------------------- print("\n--- marginal vs TP01 (best ensemble, 50/50 BTC+ETH) ---") cand = ensemble_daily_5050(best_members) marg = al.marginal_vs_tp01(cand) corr_full = marg.get("corr_full") verdict = marg.get("marginal_verdict") has_edge = marg.get("has_insample_edge") is_hedge = marg.get("is_hedge") robust_oos = marg.get("robust_oos") multicut = marg.get("multicut_persistent") clean_year = marg.get("clean_year_uplift") w25 = marg.get("blends", {}).get("w25", {}) w50 = marg.get("blends", {}).get("w50", {}) up25_hold = w25.get("uplift_hold") print(f" corr_full={corr_full} corr_hold={marg.get('corr_hold')}") print(f" marginal_verdict={verdict} robust_oos={robust_oos} multicut_persistent={multicut}") print(f" has_insample_edge={has_edge} (cand_insample_sharpe={marg.get('cand_insample_sharpe')}) is_hedge={is_hedge}") print(f" clean_year_uplift={clean_year} jackknife_min_uplift={marg.get('jackknife_min_uplift')}") print(f" multicut_uplift={marg.get('multicut_uplift')}") print(f" blend w25: uplift_hold={up25_hold} uplift_full={w25.get('uplift_full')} dd={w25.get('dd')}") print(f" blend w50: full={w50.get('full')} hold={w50.get('hold')} uplift_hold={w50.get('uplift_hold')} dd={w50.get('dd')}") # --- 7) Grade + earns_slot + beats_winner ------------------------------------------ n_trades_min = best_v["n_trades_min"] grade = "PASS" if (n_trades_min >= 20 and best_v["min_full"] >= 0.5 and best_v["min_hold"] >= 0.2 and fee_survives) else \ ("WEAK" if (n_trades_min >= 20 and best_v["min_full"] >= 0.3 and best_v["min_hold"] >= 0.0) else "FAIL") earns_slot = (grade != "FAIL") and verdict == "ADDS" and robust_oos and (not is_hedge) beats = (earns_slot and best_v["max_dd"] < 0.30 and (up25_hold is not None and up25_hold >= 0.55) and best_v["min_hold"] >= 0.65) print("\n=========== FINAL ===========") print(f"BEST CONFIG = {best_tag}") print(f" members:") for m in best_members: print(f" {m[0]}: kind={m[1]} cfg={m[3]}") print(f" minFull={best_v['min_full']:+.2f} minHold={best_v['min_hold']:+.2f}" f" max_dd={best_v['max_dd']*100:.1f}% n_trades_min={n_trades_min}") print(f" fee@0.30%RT survives={fee_survives} causality_ok={caus_ok} grade={grade}") print(f" marginal: corr_full={corr_full} verdict={verdict} insample_edge={has_edge}" f" is_hedge={is_hedge} robust_oos={robust_oos} multicut={multicut}") print(f" clean_year_uplift={clean_year} blend_w25_uplift_hold={up25_hold}") print(f" blend_w50: full={w50.get('full')} hold={w50.get('hold')} dd={w50.get('dd')}") print(f" earns_slot={earns_slot} beats_winner={beats}") print(f"\n (WINNER ref: minFull={w_minfull:+.2f} minHold={w_minhold:+.2f} maxDD={w_maxdd*100:.0f}%)") # machine-readable tail for the agent import json print("\nRESULT_JSON=" + json.dumps({ "best_tag": best_tag, "best_config": {"members": [{"name": m[0], "kind": m[1], "cfg": m[3]} for m in best_members]}, "min_full": best_v["min_full"], "min_hold": best_v["min_hold"], "max_dd": best_v["max_dd"], "n_trades_min": n_trades_min, "fee_survives": bool(fee_survives), "causality_ok": bool(caus_ok), "grade": grade, "corr_full": corr_full, "marginal_verdict": verdict, "has_insample_edge": bool(has_edge), "is_hedge": bool(is_hedge), "robust_oos": bool(robust_oos), "multicut_persistent": bool(multicut), "clean_year_uplift": clean_year, "blend_w25_uplift_hold": up25_hold, "earns_slot": bool(earns_slot), "beats_winner": bool(beats), }, default=str))