"""SKH2_DUALTF_PTN — LTF (230m) CONFIRMATION of the HTF (690m) breakout at entry. FAMILY: DUALTF_PTN. Hypothesis (DD-cut): the V2 winner enters on a fresh HTF Donchian breakout regardless of where the LTF exec-frame is. If we ALSO require the LTF to confirm the breakout at the entry bar (LTF close[i] above its own EMA(n) for longs / below for shorts, or LTF short-term momentum agrees), we avoid entering against a freshly-turned LTF. Fewer "fight the exec-frame" fills -> fewer immediate stop-outs -> lower standalone maxDD, ideally without gutting the hold-out edge. BASELINE (V2 winner): SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0). minFull +0.83, minHold +0.81, maxDD BTC 34% / ETH 31% (THE PROBLEM), marginal ADDS. WHAT THIS SCRIPT DOES (all leak-free): * Reuse S.htf_features (V2 composer, Chande regime) -> comp_long/comp_short on HTF close. * merge backward to LTF (S.merge_htf_to_ltf) -> causal HTF signal at each LTF bar. * Compute LTF confirmation features at close[i]: EMA(n) of LTF close, and an LTF momentum (close[i] vs close[i-mom]). All strictly causal (no shift into the future). * AND the HTF composer with the LTF confirmation: long only if comp_long & ltf_up; short only if comp_short & ltf_dn. (ltf_up/ltf_dn defined by chosen confirm mode.) * Same entry/exit machinery as V2 (sl/tp ATR multiples, asymmetric max_bars, 1/day). CAUSALITY: every LTF feature uses ltf data with index <= i. EMA via ewm(adjust=False) is a pure causal recursion; momentum uses close[i] and close[i-mom]. We prove it with a truncated-prefix recompute (same protocol as sk.causality) on our custom entries. """ from __future__ import annotations import sys sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook") import numpy as np import pandas as pd import skyhooklib as sk from src.backtest.harness import backtest_signals from src.strategies import skyhook as S from src.strategies.skyhook import SkyhookParams HOLDOUT = sk.HOLDOUT FEE = sk.FEE_RT # V2 winner params (the baseline to beat). LTF confirmation rides on top of these. WINNER = dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16, vola_lo=35.0, vola_hi=95.0, vol_lo=0.0) def winner_params(**kw): base = dict(WINNER) base.update(kw) return SkyhookParams(**base) # --------------------------------------------------------------------------- # Causal LTF confirmation features (computed on the 230m exec frame, at close[i]). # --------------------------------------------------------------------------- def ltf_confirm(ltf_close: np.ndarray, *, ema_n: int, mom_n: int) -> tuple[np.ndarray, np.ndarray]: """Return (ltf_up, ltf_dn) boolean masks per LTF bar, strictly causal. ltf_up := close[i] > EMA_n(close)[i] AND close[i] > close[i-mom_n] (momentum agrees) ltf_dn := close[i] < EMA_n(close)[i] AND close[i] < close[i-mom_n] EMA via ewm(adjust=False): a causal recursion (uses only data <= i).""" c = pd.Series(np.asarray(ltf_close, float)) ema = c.ewm(span=ema_n, adjust=False).mean().values cc = c.values mom_up = np.zeros(len(cc), dtype=bool) mom_dn = np.zeros(len(cc), dtype=bool) if mom_n > 0: mom_up[mom_n:] = cc[mom_n:] > cc[:-mom_n] mom_dn[mom_n:] = cc[mom_n:] < cc[:-mom_n] else: mom_up[:] = True mom_dn[:] = True up = (cc > ema) & mom_up dn = (cc < ema) & mom_dn return up, dn def ltf_confirm_modes(ltf_close: np.ndarray, *, ema_n: int, mom_n: int, mode: str, slope_n: int = 0): """Causal LTF confirmation masks (ltf_up, ltf_dn). All features use data <= i. Components: ema_up := close[i] > EMA_n(close)[i] mom_up := close[i] > close[i-mom_n] (sustained move over mom_n bars) slope_up:= EMA_n(close)[i] > EMA_n(close)[i-slope_n] (LTF trend is rising) if slope_n>0 Modes: 'ema' -> ema_up 'mom' -> mom_up 'both' -> ema_up & mom_up 'or' -> ema_up | mom_up 'slope' -> slope_up only (EMA itself rising/falling) 'ema_slope' -> ema_up & slope_up (above a rising EMA = real LTF uptrend, strict) 'all' -> ema_up & mom_up & slope_up (strictest) """ c = pd.Series(np.asarray(ltf_close, float)) ema = c.ewm(span=ema_n, adjust=False).mean().values cc = c.values n = len(cc) ema_up = cc > ema ema_dn = cc < ema mom_up = np.zeros(n, dtype=bool) mom_dn = np.zeros(n, dtype=bool) if mom_n > 0: mom_up[mom_n:] = cc[mom_n:] > cc[:-mom_n] mom_dn[mom_n:] = cc[mom_n:] < cc[:-mom_n] else: mom_up[:] = True mom_dn[:] = True slope_up = np.zeros(n, dtype=bool) slope_dn = np.zeros(n, dtype=bool) if slope_n > 0: slope_up[slope_n:] = ema[slope_n:] > ema[:-slope_n] slope_dn[slope_n:] = ema[slope_n:] < ema[:-slope_n] else: slope_up[:] = True slope_dn[:] = True if mode == "ema": return ema_up, ema_dn if mode == "mom": return mom_up, mom_dn if mode == "or": return (ema_up | mom_up), (ema_dn | mom_dn) if mode == "slope": return slope_up, slope_dn if mode == "ema_slope": return (ema_up & slope_up), (ema_dn & slope_dn) if mode == "all": return (ema_up & mom_up & slope_up), (ema_dn & mom_dn & slope_dn) # default 'both' return (ema_up & mom_up), (ema_dn & mom_dn) def ltf_not_overextended(ltf_close: np.ndarray, ltf_atr: np.ndarray, *, ema_n: int, max_ext_atr: float): """REJECT (return False) when the LTF is already overextended from its EMA at entry: a long-breakout fired when close[i] is already > ema + max_ext_atr*ATR_LTF[i] is a LATE fill (mean-reversion-prone, big-stop risk). Confirmation = NOT overextended. ltf_up := (close - ema) <= max_ext_atr*ATR (still room to run, not blown off) ltf_dn := (ema - close) <= max_ext_atr*ATR All causal: ema, ATR, close all at i.""" c = pd.Series(np.asarray(ltf_close, float)) ema = c.ewm(span=ema_n, adjust=False).mean().values cc = c.values a = np.asarray(ltf_atr, float) a = np.where(np.isfinite(a) & (a > 0), a, np.nan) ext = (cc - ema) / a # long: not too far ABOVE ema ; short: not too far BELOW ema up = np.where(np.isfinite(ext), ext <= max_ext_atr, False) dn = np.where(np.isfinite(ext), (-ext) <= max_ext_atr, False) return up, dn # --------------------------------------------------------------------------- # Custom entries: V2 HTF composer AND LTF confirmation. # --------------------------------------------------------------------------- def dualtf_entries(ltf: pd.DataFrame, htf: pd.DataFrame, p: SkyhookParams, *, ema_n: int, mom_n: int, mode: str, slope_n: int = 0, max_ext_atr: float = 0.0) -> list: feat = S.htf_features(htf, p) # V2 composer (Chande regime + Donchian) m = S.merge_htf_to_ltf(ltf, feat) # causal backward merge c = m["close"].values.astype(float) a = S.atr(m, p.ltf_atr_win) comp_long = np.nan_to_num(m["comp_long"].values).astype(bool) comp_short = np.nan_to_num(m["comp_short"].values).astype(bool) if mode == "notext": ltf_up, ltf_dn = ltf_not_overextended(c, a, ema_n=ema_n, max_ext_atr=max_ext_atr) else: ltf_up, ltf_dn = ltf_confirm_modes(c, ema_n=ema_n, mom_n=mom_n, mode=mode, slope_n=slope_n) comp_long = comp_long & ltf_up comp_short = comp_short & ltf_dn days = pd.to_datetime(m["datetime"]).dt.floor("D").values entries: list = [None] * len(m) count_today: dict = {} for i in range(len(m)): if not np.isfinite(a[i]) or a[i] <= 0: continue day = days[i] if count_today.get(day, 0) >= p.max_per_day: continue if comp_long[i]: direction, mb = 1, p.uscitalong elif comp_short[i]: direction, mb = -1, p.uscitashort else: continue if p.exit_mode == "atr": sl_off, tp_off = p.sl_atr * a[i], p.tp_atr * a[i] else: sl_off, tp_off = p.sl_pct * c[i], p.tp_pct * c[i] if direction == 1: sl, tp = c[i] - sl_off, c[i] + tp_off else: sl, tp = c[i] + sl_off, c[i] - tp_off entries[i] = {"dir": direction, "tp": float(tp), "sl": float(sl), "max_bars": int(mb)} count_today[day] = count_today.get(day, 0) + 1 return entries # --------------------------------------------------------------------------- # Eval helpers (FULL + HOLD-OUT + fee sweep + per-year, both assets) — mirrors sk.study. # --------------------------------------------------------------------------- def _split(eq, idx, mask): e = eq[mask] if len(e) < 5: return dict(sharpe=0.0, ret=0.0, maxdd=0.0, n=int(len(e))) r = np.diff(e) / e[:-1]; r = r[np.isfinite(r)] ix = idx[mask] dt = pd.Series(ix).diff().dt.total_seconds().median() or 86400 bpy = 86400 * 365.25 / dt sh = float(np.mean(r) / np.std(r) * np.sqrt(bpy)) if len(r) and np.std(r) > 0 else 0.0 pk = np.maximum.accumulate(e); dd = float(np.max((pk - e) / pk)) return dict(sharpe=round(sh, 3), ret=round(float(e[-1] / e[0] - 1), 4), maxdd=round(dd, 4), n=int(len(e))) def study_dualtf(name, p, confirm): per_asset = {} fee_ok_all = True for a in ("BTC", "ETH"): ltf, htf = sk.frames(a) ent = dualtf_entries(ltf, htf, p, **confirm) m = backtest_signals(ltf, ent, fee_rt=FEE, leverage=1.0, asset=a, tf="230m") eq = m.equity idx = pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True)) hmask = np.asarray(idx >= HOLDOUT) full = dict(sharpe=round(m.sharpe, 3), ret=round(m.net_return, 4), maxdd=round(m.max_dd, 4), n_trades=int(m.n_trades), win_rate=round(m.win_rate, 1)) hold = _split(eq, idx, hmask) sweep = {} for f in (0.0, 0.001, 0.002, 0.003): mf = backtest_signals(ltf, ent, fee_rt=f, leverage=1.0, asset=a, tf="230m") sweep[f"{f*100:.2f}%"] = round(mf.sharpe, 3) fee_ok_all = fee_ok_all and (sweep["0.30%"] > 0) per_asset[a] = dict(full=full, hold=hold, fee_sweep=sweep, yearly={int(y): round(v, 4) for y, v in m.yearly.items()}) mf = min(per_asset[a]["full"]["sharpe"] for a in per_asset) mh = min(per_asset[a]["hold"]["sharpe"] for a in per_asset) mt = min(per_asset[a]["full"]["n_trades"] for a in per_asset) mdd = max(per_asset[a]["full"]["maxdd"] for a in per_asset) grade = "PASS" if (mt >= 20 and mf >= 0.5 and mh >= 0.2 and fee_ok_all) else \ ("WEAK" if (mt >= 20 and mf >= 0.3 and mh >= 0.0) else "FAIL") print(f"\n=== {name} -> {grade} (minFull={mf:+.2f} minHold={mh:+.2f} minTr={mt} maxDD={mdd*100:.0f}% feeOK={fee_ok_all})") for a in per_asset: pa = per_asset[a] yr = " ".join(f"{y}:{r*100:+.0f}%" for y, r in pa["yearly"].items()) print(f" {a}: FULL Sh={pa['full']['sharpe']:+.2f} ret={pa['full']['ret']*100:+.0f}% DD={pa['full']['maxdd']*100:.0f}%" f" n={pa['full']['n_trades']} wr={pa['full']['win_rate']:.0f}% | HOLD Sh={pa['hold']['sharpe']:+.2f} ret={pa['hold']['ret']*100:+.0f}% DD={pa['hold']['maxdd']*100:.0f}%") print(f" fee: " + " ".join(f"{k}={v:+.2f}" for k, v in pa["fee_sweep"].items())) print(f" yr: {yr}") return dict(grade=grade, minFull=mf, minHold=mh, minTr=mt, maxDD=mdd, fee_ok=fee_ok_all, per_asset=per_asset) def marginal_dualtf(p, confirm): import altlib as al def daily(a): ltf, htf = sk.frames(a) ent = dualtf_entries(ltf, htf, p, **confirm) m = backtest_signals(ltf, ent, fee_rt=FEE, leverage=1.0, asset=a, tf="230m") s = pd.Series(m.equity, index=pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True))) return s.resample("1D").last().ffill().pct_change().dropna() sb, se = daily("BTC"), daily("ETH") J = pd.concat({"BTC": sb, "ETH": se}, axis=1, join="inner").fillna(0.0) cand = 0.5 * J["BTC"] + 0.5 * J["ETH"] return al.marginal_vs_tp01(cand) def check_causality(p, confirm, asset="BTC", tail=150): ltf, htf = sk.frames(asset) full = dualtf_entries(ltf, htf, p, **confirm) n = len(ltf); bad = 0; checked = 0 for frac in (0.80, 0.92): cut = int(n * frac) cut_ts = int(ltf["timestamp"].iloc[cut - 1]) htf_cut = htf[htf["timestamp"] <= cut_ts].reset_index(drop=True) sub = dualtf_entries(ltf.iloc[:cut].reset_index(drop=True), htf_cut, p, **confirm) for i in range(max(0, cut - tail), cut): checked += 1 a, b = full[i], sub[i] if (a is None) != (b is None): bad += 1 elif a is not None and (a["dir"] != b["dir"] or abs(a["sl"] - b["sl"]) > 1e-6 or abs(a["tp"] - b["tp"]) > 1e-6): bad += 1 return dict(ok=bool(bad == 0), mismatches=int(bad), checked=int(checked)) if __name__ == "__main__": print("########## SKH2_DUALTF_PTN: LTF confirmation of HTF breakout ##########") # Reference: V2 winner WITHOUT LTF confirmation (mode 'none' via wide-open masks). p = winner_params() # --- Reference (no LTF confirm) using sk.run_asset directly --- print("\n--- V2 WINNER reference (no LTF confirm) ---") refF, refH, refDD, refTr = [], [], [], [] for a in ("BTC", "ETH"): r = sk.run_asset(a, p, FEE) refF.append(r["full"]["sharpe"]); refH.append(r["holdout"]["sharpe"]) refDD.append(r["full"]["maxdd"]); refTr.append(r["full"]["n_trades"]) print(f" {a}: FULL Sh={r['full']['sharpe']:+.2f} DD={r['full']['maxdd']*100:.0f}% n={r['full']['n_trades']}" f" | HOLD Sh={r['holdout']['sharpe']:+.2f}") print(f" REF minFull={min(refF):+.2f} minHold={min(refH):+.2f} maxDD={max(refDD)*100:.0f}% minTr={min(refTr)}") # --- Sweep LTF confirmation configs --- # ema_n / mom_n on 230m bars. ~6.26 bars/day. EMA 10~1.6d, 20~3.2d. mom small (1-6 bars). # Directional confirms are near-redundant with a fresh breakout (barely filter). # The real DD lever in this family: REJECT OVEREXTENDED LTF fills (late, blow-off, # mean-reversion-prone, big-stop). max_ext_atr = max allowed (close-ema)/ATR_LTF at entry. configs = { # reference directional confirm (keeps ~all trades) "mom_only_m3": dict(ema_n=20, mom_n=3, mode="mom"), # NOT-OVEREXTENDED gate: tighter max_ext -> fewer late fills -> aim lower DD "notext_e20_x4": dict(ema_n=20, mom_n=0, mode="notext", max_ext_atr=4.0), "notext_e20_x3": dict(ema_n=20, mom_n=0, mode="notext", max_ext_atr=3.0), "notext_e20_x2": dict(ema_n=20, mom_n=0, mode="notext", max_ext_atr=2.0), "notext_e20_x1_5": dict(ema_n=20, mom_n=0, mode="notext", max_ext_atr=1.5), "notext_e30_x3": dict(ema_n=30, mom_n=0, mode="notext", max_ext_atr=3.0), "notext_e30_x2": dict(ema_n=30, mom_n=0, mode="notext", max_ext_atr=2.0), "notext_e10_x2": dict(ema_n=10, mom_n=0, mode="notext", max_ext_atr=2.0), "notext_e10_x1_5": dict(ema_n=10, mom_n=0, mode="notext", max_ext_atr=1.5), "notext_e30_x1_5": dict(ema_n=30, mom_n=0, mode="notext", max_ext_atr=1.5), } results = {} for tag, cfg in configs.items(): r = study_dualtf(f"DUALTF_{tag}", p, cfg) results[tag] = (cfg, r) # --- Pick best: priority (1) DD<30, (2) earns_slot, (3) minHold high --- print("\n\n##### MARGINAL vs TP01 (configs with minTr>=20) #####") scored = [] for tag, (cfg, r) in results.items(): if r["minTr"] < 20: print(f"[{tag}] minTr={r['minTr']} <20 -> skip marginal") continue mg = marginal_dualtf(p, cfg) verdict = mg.get("marginal_verdict") robust = bool(mg.get("robust_oos")) hedge = bool(mg.get("is_hedge")) earns = (r["grade"] != "FAIL") and (verdict == "ADDS") and robust and (not hedge) w25 = mg.get("blends", {}).get("w25", {}).get("uplift_hold") beats = earns and (r["maxDD"] < 0.30) and (w25 is not None and w25 >= 0.55) and (r["minHold"] >= 0.65) scored.append((tag, cfg, r, mg, earns, beats)) print(f"[{tag}] grade={r['grade']} minFull={r['minFull']:+.2f} minHold={r['minHold']:+.2f} maxDD={r['maxDD']*100:.0f}%" f" | corr_full={mg.get('corr_full')} verdict={verdict} insample={mg.get('has_insample_edge')}" f" hedge={hedge} robust={robust} w25uplift={w25} earns_slot={earns} BEATS={beats}") if not scored: print("\nNo config with enough trades.") sys.exit(0) # Rank: beats_winner first, then DD<30 & earns, then by minHold, then by lowest DD. def rank_key(item): tag, cfg, r, mg, earns, beats = item w25 = mg.get("blends", {}).get("w25", {}).get("uplift_hold") or -9 return (beats, (r["maxDD"] < 0.30 and earns), earns, r["minHold"], -r["maxDD"]) scored.sort(key=rank_key, reverse=True) best_tag, best_cfg, best_r, best_mg, best_earns, best_beats = scored[0] # --- Causality on best --- cb = check_causality(p, best_cfg, "BTC") ce = check_causality(p, best_cfg, "ETH") caus_ok = cb["ok"] and ce["ok"] w25 = best_mg.get("blends", {}).get("w25", {}).get("uplift_hold") w50 = best_mg.get("blends", {}).get("w50", {}) fee030_min = min(best_r["per_asset"][a]["fee_sweep"]["0.30%"] for a in ("BTC", "ETH")) print("\n\n##################### BEST CONFIG #####################") print(f"BEST = DUALTF_{best_tag} cfg={best_cfg}") print(f" params = {WINNER}") print(f" grade={best_r['grade']} minFull={best_r['minFull']:+.2f} minHold={best_r['minHold']:+.2f}" f" maxDD={best_r['maxDD']*100:.1f}% minTr={best_r['minTr']}") print(f" fee@0.30% (min asset FULL Sh) = {fee030_min:+.3f} feeOK={best_r['fee_ok']}") print(f" causality: BTC={cb} ETH={ce} -> OK={caus_ok}") print(f" marginal: corr_full={best_mg.get('corr_full')} corr_hold={best_mg.get('corr_hold')}" f" verdict={best_mg.get('marginal_verdict')}") print(f" has_insample_edge={best_mg.get('has_insample_edge')} is_hedge={best_mg.get('is_hedge')}" f" robust_oos={best_mg.get('robust_oos')} multicut_persistent={best_mg.get('multicut_persistent')}") print(f" clean_year_uplift={best_mg.get('clean_year_uplift')}" f" jackknife_min_uplift={best_mg.get('jackknife_min_uplift')}" f" cand_insample_sharpe={best_mg.get('cand_insample_sharpe')}") print(f" blend w25 uplift_hold={w25} w50={w50}") print(f" earns_slot={best_earns} BEATS_WINNER={best_beats}") # Emit a compact machine-readable line for the harness. import json out = dict(family="DUALTF_PTN", best_tag=best_tag, best_cfg=best_cfg, winner_params=WINNER, grade=best_r["grade"], minFull=best_r["minFull"], minHold=best_r["minHold"], maxDD=best_r["maxDD"], minTr=best_r["minTr"], fee030_min=fee030_min, causality_ok=caus_ok, marginal_verdict=best_mg.get("marginal_verdict"), corr_full=best_mg.get("corr_full"), has_insample_edge=best_mg.get("has_insample_edge"), is_hedge=best_mg.get("is_hedge"), robust_oos=best_mg.get("robust_oos"), multicut_persistent=best_mg.get("multicut_persistent"), clean_year_uplift=best_mg.get("clean_year_uplift"), w25_uplift_hold=w25, w50=w50, earns_slot=best_earns, beats_winner=best_beats) print("\nJSON " + json.dumps(out, default=str))