"""SKH2_ENS_STRUCT — cross-definition ENSEMBLE [ENS_STRUCT].

WAVE GOAL: cut the V2 winner's STANDALONE max-DD below 30% (BTC 34% / ETH 31% is the only
unmet goal) while keeping min-asset HOLD-OUT >= ~0.70 and earns_slot True.

IDEA: the V2 winner uses ONE regime definition (Chande01 cycle band on ATR + Donchian
breakout). Its drawdowns come from that one signal source firing into the wrong tape. If we
ensemble it with STRUCTURALLY DIFFERENT regime definitions — (B) causal PERCENTILE-RANK regime
(SKH_R_PCTL) and (C) VOLATILITY-EXPANSION regime (SKH_R_EXPAND) — that disagree about WHEN to
trade, their drawdowns are imperfectly correlated. Equal-weighting the three daily-return
streams (per asset, then 50/50 across BTC+ETH) should reduce the COMBINED-equity DD below any
single member's DD, at a modest cost to full Sharpe.

The ensemble is EQUAL-WEIGHT on DAILY RETURNS (a constant-weight rebalanced book of three
sub-sleeves on the SAME asset). Standalone DD = max-DD of the COMBINED equity curve (per asset,
max over BTC & ETH). Marginal vs TP01 uses the 50/50 BTC+ETH combined daily series.

All three members are causal/leak-free (winner via sk.causality; PCTL/EXPAND via their own
truncated-prefix guards, re-run here). Equal-weighting causal streams stays causal.
"""
from __future__ import annotations

import sys
sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/skyhook")

import importlib.util
import numpy as np
import pandas as pd

import skyhooklib as sk
import altlib as al
from src.backtest.harness import backtest_signals
from src.strategies import skyhook as S
from src.strategies.skyhook import SkyhookParams

HOLDOUT = sk.HOLDOUT
FEE = sk.FEE_RT


# --- import the two structural entry builders without running their __main__ ----------------
def _load(modname, path):
    spec = importlib.util.spec_from_file_location(modname, path)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod

PCTL = _load("skr_pctl", "/opt/docker/PythagorasGoal/scripts/research/skyhook/runs/SKH_R_PCTL.py")
EXPD = _load("skr_expd", "/opt/docker/PythagorasGoal/scripts/research/skyhook/runs/SKH_R_EXPAND.py")


# --- the V2 winner from the prior wave (Chande/Donchian regime) -----------------------------
WINNER = SkyhookParams(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16,
                       vola_lo=35.0, vola_hi=95.0, vol_lo=0.0)

# pattern/exit shared knobs for the structural members. Match the WINNER's exit profile
# (sl2.5/tp7.0, asym time exits) so the difference is the REGIME, not the exit.
def member_params(**kw):
    base = dict(ptn_n=45, sl_atr=2.5, tp_atr=7.0, uscitalong=24, uscitashort=16)
    base.update(kw)
    return SkyhookParams(**base)


# ============================================================================================
# Per-asset equity helpers. We need the FULL bar-level equity curve of each member so we can
# build the COMBINED equity (for an honest combined max-DD), AND the daily-return series (for
# Sharpe split + marginal). All on the SAME 230m execution index.
# ============================================================================================
def member_equity(asset, kind, p, cfg=None):
    """Return (eq, idx) bar-level equity for a member. kind in {'winner','pctl','expand'}."""
    ltf, htf = sk.frames(asset)
    if kind == "winner":
        ent = S.skyhook_entries(ltf, htf, p)
    elif kind == "pctl":
        ent = PCTL.pctl_entries(ltf, htf, p, **cfg)
    elif kind == "expand":
        ent = EXPD.expand_entries(ltf, htf, p, **cfg)
    else:
        raise ValueError(kind)
    m = backtest_signals(ltf, ent, fee_rt=FEE, leverage=1.0, asset=asset, tf="230m")
    idx = pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True))
    return np.asarray(m.equity, float), idx, int(m.n_trades)


def eq_to_daily_ret(eq, idx):
    """bar equity -> daily simple returns (last-of-day, ffill, pct_change)."""
    s = pd.Series(eq, index=idx)
    return s.resample("1D").last().ffill().pct_change()


def combined_daily_ret(asset, members):
    """Equal-weight DAILY returns of the members on one asset -> combined daily return series.
    Each member contributes its own daily return; the equal-weight portfolio return is the mean.
    Members are aligned on the union of daily timestamps; a member that has not started yet
    (NaN) contributes 0 that day and is dropped from the active-weight denominator (outer-join
    with renormalized equal weights), so early days where only some members trade are handled."""
    drs = {}
    ntr = {}
    for name, kind, p, cfg in members:
        eq, idx, nt = member_equity(asset, kind, p, cfg)
        drs[name] = eq_to_daily_ret(eq, idx)
        ntr[name] = nt
    D = pd.concat(drs, axis=1, join="outer")
    # renormalized equal weight across the members that are ACTIVE (non-NaN) each day
    active = D.notna()
    w = active.div(active.sum(axis=1).replace(0, np.nan), axis=0)
    comb = (D.fillna(0.0) * w.fillna(0.0)).sum(axis=1)
    comb = comb.dropna()
    return comb, ntr


def combined_metrics(comb):
    """Sharpe (full + hold-out) and max-DD from a DAILY combined return series."""
    comb = comb[np.isfinite(comb.values)]
    eq = (1.0 + comb).cumprod().values
    idx = comb.index

    def _sh(r):
        r = r[np.isfinite(r)]
        return float(np.mean(r) / np.std(r) * np.sqrt(365.25)) if len(r) and np.std(r) > 0 else 0.0

    pk = np.maximum.accumulate(eq)
    dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0
    full_sh = _sh(comb.values)
    hmask = idx >= HOLDOUT
    hold_sh = _sh(comb.values[hmask]) if hmask.sum() > 5 else 0.0
    # hold-out DD on the hold-out slice of the combined equity
    eqh = eq[hmask]
    pkh = np.maximum.accumulate(eqh) if len(eqh) else np.array([1.0])
    ddh = float(np.max((pkh - eqh) / pkh)) if len(eqh) else 0.0
    yrs = {}
    for y in sorted(set(idx.year)):
        rr = comb.values[idx.year == y]
        yrs[int(y)] = round(float((1 + pd.Series(rr)).prod() - 1), 4)
    return dict(full_sharpe=round(full_sh, 3), hold_sharpe=round(hold_sh, 3),
                maxdd=round(dd, 4), hold_maxdd=round(ddh, 4), yearly=yrs)


# --- per-member standalone DD (for the correlation/decorrelation diagnostic) ----------------
def member_standalone_dd(asset, kind, p, cfg=None):
    eq, idx, nt = member_equity(asset, kind, p, cfg)
    pk = np.maximum.accumulate(eq)
    dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0
    return dd, nt


# ============================================================================================
# Fee robustness of the ENSEMBLE: re-run every member at fee f, recombine, check Sharpe>0.
# ============================================================================================
def ensemble_fee_sweep(members, fees=(0.0, 0.001, 0.002, 0.003)):
    rows = {}
    for f in fees:
        ok = True
        for asset in ("BTC", "ETH"):
            drs = {}
            for name, kind, p, cfg in members:
                ltf, htf = sk.frames(asset)
                if kind == "winner":
                    ent = S.skyhook_entries(ltf, htf, p)
                elif kind == "pctl":
                    ent = PCTL.pctl_entries(ltf, htf, p, **cfg)
                else:
                    ent = EXPD.expand_entries(ltf, htf, p, **cfg)
                m = backtest_signals(ltf, ent, fee_rt=f, leverage=1.0, asset=asset, tf="230m")
                drs[name] = eq_to_daily_ret(np.asarray(m.equity, float),
                                            pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True)))
            D = pd.concat(drs, axis=1, join="outer")
            active = D.notna()
            w = active.div(active.sum(axis=1).replace(0, np.nan), axis=0)
            comb = (D.fillna(0.0) * w.fillna(0.0)).sum(axis=1).dropna()
            sh = combined_metrics(comb)["full_sharpe"]
            rows[(f, asset)] = sh
            ok = ok and (sh > 0)
        rows[(f, "ok")] = ok
    return rows


def ensemble_daily_5050(members):
    """50/50 BTC+ETH combined daily series for marginal_vs_tp01."""
    cb, _ = combined_daily_ret("BTC", members)
    ce, _ = combined_daily_ret("ETH", members)
    J = pd.concat({"BTC": cb, "ETH": ce}, axis=1, join="inner").fillna(0.0)
    return 0.5 * J["BTC"] + 0.5 * J["ETH"]


# ============================================================================================
def run_ensemble(tag, members):
    print(f"\n========== ENSEMBLE: {tag} ==========")
    print(f"  members: {[m[0] for m in members]}")
    per = {}
    for asset in ("BTC", "ETH"):
        comb, ntr = combined_daily_ret(asset, members)
        met = combined_metrics(comb)
        per[asset] = dict(met=met, ntr=ntr)
        print(f"  {asset}: FULL Sh={met['full_sharpe']:+.2f} HOLD Sh={met['hold_sharpe']:+.2f}"
              f" maxDD={met['maxdd']*100:.0f}% holdDD={met['hold_maxdd']*100:.0f}%  ntrades={ntr}")
        print(f"       yearly: " + " ".join(f"{y}:{v*100:+.0f}%" for y, v in met['yearly'].items()))
    min_full = min(per[a]["met"]["full_sharpe"] for a in per)
    min_hold = min(per[a]["met"]["hold_sharpe"] for a in per)
    max_dd = max(per[a]["met"]["maxdd"] for a in per)
    n_trades_min = min(sum(per[a]["ntr"].values()) for a in per)
    print(f"  -> minFull={min_full:+.2f} minHold={min_hold:+.2f} maxDD={max_dd*100:.0f}%"
          f" nTrades(min,sum-of-members)={n_trades_min}")
    return dict(per=per, min_full=min_full, min_hold=min_hold, max_dd=max_dd,
                n_trades_min=n_trades_min)


if __name__ == "__main__":
    print("=== SKH2_ENS_STRUCT: cross-definition regime ensemble (DD reduction) ===")

    # --- 0) Baseline: the V2 winner ALONE (reference) -----------------------------------
    print("\n--- V2 WINNER standalone (Chande/Donchian) ---")
    w_per = {}
    for a in ("BTC", "ETH"):
        r = sk.run_asset(a, WINNER, FEE)
        w_per[a] = r
        print(f"  {a}: FULL Sh={r['full']['sharpe']:+.2f} DD={r['full']['maxdd']*100:.0f}%"
              f" n={r['full']['n_trades']} | HOLD Sh={r['holdout']['sharpe']:+.2f}")
    w_minfull = min(w_per[a]["full"]["sharpe"] for a in w_per)
    w_minhold = min(w_per[a]["holdout"]["sharpe"] for a in w_per)
    w_maxdd = max(w_per[a]["full"]["maxdd"] for a in w_per)
    print(f"  WINNER: minFull={w_minfull:+.2f} minHold={w_minhold:+.2f} maxDD={w_maxdd*100:.0f}%")

    # --- 1) Structural member configs (different regime definitions) --------------------
    pP = member_params()
    pE = member_params()
    # PCTL: expanding percentile-rank regime. From SKH_R_PCTL_final, the lower-DD / robust
    # band was the low-vola band; we use a mid band with a modest volume floor so it disagrees
    # with the winner's HIGH-vola Chande band (different regime -> decorrelated DD).
    PCTL_CFG = dict(vola_win=None, vol_win=None, vola_lo=0.0, vola_hi=0.70, vol_lo=0.0, vol_hi=1.0)
    PCTL_CFG2 = dict(vola_win=None, vol_win=None, vola_lo=0.10, vola_hi=0.80, vol_lo=0.30, vol_hi=1.0)
    # EXPAND: volatility-expansion regime (ATR above its MA + volume elevated).
    EXP_CFG = dict(w_atr=20, k_atr=1.00, w_vol=20, k_vol=1.00)
    EXP_CFG2 = dict(w_atr=20, k_atr=1.10, w_vol=20, k_vol=1.20)

    # --- standalone DD + pairwise correlation of the three regime streams (BTC) ---------
    print("\n--- standalone member DD + pairwise daily-return corr (decorrelation check) ---")
    for a in ("BTC", "ETH"):
        dW, nW = member_standalone_dd(a, "winner", WINNER)
        dP, nP = member_standalone_dd(a, "pctl", pP, PCTL_CFG)
        dE, nE = member_standalone_dd(a, "expand", pE, EXP_CFG)
        print(f"  {a} standalone DD: winner={dW*100:.0f}%(n{nW}) pctl={dP*100:.0f}%(n{nP}) expand={dE*100:.0f}%(n{nE})")
        # corr of daily returns
        rw = eq_to_daily_ret(*member_equity(a, "winner", WINNER)[:2])
        rp = eq_to_daily_ret(*member_equity(a, "pctl", pP, PCTL_CFG)[:2])
        re_ = eq_to_daily_ret(*member_equity(a, "expand", pE, EXP_CFG)[:2])
        DD = pd.concat({"W": rw, "P": rp, "E": re_}, axis=1, join="inner").fillna(0.0)
        cc = DD.corr()
        print(f"     corr W-P={cc.loc['W','P']:.2f} W-E={cc.loc['W','E']:.2f} P-E={cc.loc['P','E']:.2f}")

    # --- 2) Candidate ensembles ---------------------------------------------------------
    candidates = {
        "WPE (winner+pctlLo+expand)": [
            ("winner", "winner", WINNER, None),
            ("pctl", "pctl", pP, PCTL_CFG),
            ("expand", "expand", pE, EXP_CFG),
        ],
        "WP (winner+pctlLo)": [
            ("winner", "winner", WINNER, None),
            ("pctl", "pctl", pP, PCTL_CFG),
        ],
        "WE (winner+expand)": [
            ("winner", "winner", WINNER, None),
            ("expand", "expand", pE, EXP_CFG),
        ],
        "WPE2 (winner+pctlMid+expandStrong)": [
            ("winner", "winner", WINNER, None),
            ("pctl", "pctl", pP, PCTL_CFG2),
            ("expand", "expand", pE, EXP_CFG2),
        ],
    }

    results = {}
    for tag, members in candidates.items():
        results[tag] = (members, run_ensemble(tag, members))

    # --- 3) Pick best by: max_dd<0.30, then maximize min_hold ---------------------------
    def score(v):
        # prefer DD<0.30 AND min_hold>=0.65; among those, maximize min_hold then -DD
        ok = v["max_dd"] < 0.30 and v["min_hold"] >= 0.65
        return (1 if ok else 0, v["min_hold"], -v["max_dd"])
    best_tag = max(results, key=lambda t: score(results[t][1]))
    best_members, best_v = results[best_tag]
    print(f"\n*** BEST ENSEMBLE = {best_tag} ***")
    print(f"    minFull={best_v['min_full']:+.2f} minHold={best_v['min_hold']:+.2f}"
          f" maxDD={best_v['max_dd']*100:.0f}% nTradesMin={best_v['n_trades_min']}")

    # --- 4) Causality: winner via sk; structural members via their own guards -----------
    print("\n--- causality (all active members of best ensemble) ---")
    caus_ok = True
    kinds = {m[0]: (m[1], m[2], m[3]) for m in best_members}
    if "winner" in kinds:
        cb = sk.causality(WINNER, "BTC"); ce = sk.causality(WINNER, "ETH")
        print(f"  winner: BTC {cb} ETH {ce}")
        caus_ok = caus_ok and cb["ok"] and ce["ok"]
    if "pctl" in kinds:
        _, p, cfg = kinds["pctl"]
        cb = PCTL.check_causality(cfg, p, "BTC"); ce = PCTL.check_causality(cfg, p, "ETH")
        print(f"  pctl:   BTC {cb} ETH {ce}")
        caus_ok = caus_ok and cb["ok"] and ce["ok"]
    if "expand" in kinds:
        _, p, cfg = kinds["expand"]
        cb = EXPD.check_causality(cfg, p, "BTC"); ce = EXPD.check_causality(cfg, p, "ETH")
        print(f"  expand: BTC {cb} ETH {ce}")
        caus_ok = caus_ok and cb["ok"] and ce["ok"]
    print(f"  causality_ok (all members) = {caus_ok}")

    # --- 5) Fee sweep on the ensemble ---------------------------------------------------
    print("\n--- ensemble fee sweep (FULL Sharpe per asset) ---")
    fsw = ensemble_fee_sweep(best_members)
    for f in (0.0, 0.001, 0.002, 0.003):
        print(f"  {f*100:.2f}%RT: BTC={fsw[(f,'BTC')]:+.2f} ETH={fsw[(f,'ETH')]:+.2f} ok={fsw[(f,'ok')]}")
    fee_survives = fsw[(0.003, "ok")]
    print(f"  fee_survives 0.30%RT (both): {fee_survives}")

    # --- 6) Marginal vs TP01 on the ensemble 50/50 series -------------------------------
    print("\n--- marginal vs TP01 (best ensemble, 50/50 BTC+ETH) ---")
    cand = ensemble_daily_5050(best_members)
    marg = al.marginal_vs_tp01(cand)
    corr_full = marg.get("corr_full")
    verdict = marg.get("marginal_verdict")
    has_edge = marg.get("has_insample_edge")
    is_hedge = marg.get("is_hedge")
    robust_oos = marg.get("robust_oos")
    multicut = marg.get("multicut_persistent")
    clean_year = marg.get("clean_year_uplift")
    w25 = marg.get("blends", {}).get("w25", {})
    w50 = marg.get("blends", {}).get("w50", {})
    up25_hold = w25.get("uplift_hold")
    print(f"  corr_full={corr_full} corr_hold={marg.get('corr_hold')}")
    print(f"  marginal_verdict={verdict} robust_oos={robust_oos} multicut_persistent={multicut}")
    print(f"  has_insample_edge={has_edge} (cand_insample_sharpe={marg.get('cand_insample_sharpe')}) is_hedge={is_hedge}")
    print(f"  clean_year_uplift={clean_year} jackknife_min_uplift={marg.get('jackknife_min_uplift')}")
    print(f"  multicut_uplift={marg.get('multicut_uplift')}")
    print(f"  blend w25: uplift_hold={up25_hold} uplift_full={w25.get('uplift_full')} dd={w25.get('dd')}")
    print(f"  blend w50: full={w50.get('full')} hold={w50.get('hold')} uplift_hold={w50.get('uplift_hold')} dd={w50.get('dd')}")

    # --- 7) Grade + earns_slot + beats_winner ------------------------------------------
    n_trades_min = best_v["n_trades_min"]
    grade = "PASS" if (n_trades_min >= 20 and best_v["min_full"] >= 0.5
                       and best_v["min_hold"] >= 0.2 and fee_survives) else \
            ("WEAK" if (n_trades_min >= 20 and best_v["min_full"] >= 0.3
                        and best_v["min_hold"] >= 0.0) else "FAIL")
    earns_slot = (grade != "FAIL") and verdict == "ADDS" and robust_oos and (not is_hedge)
    beats = (earns_slot and best_v["max_dd"] < 0.30
             and (up25_hold is not None and up25_hold >= 0.55)
             and best_v["min_hold"] >= 0.65)

    print("\n=========== FINAL ===========")
    print(f"BEST CONFIG = {best_tag}")
    print(f"  members:")
    for m in best_members:
        print(f"    {m[0]}: kind={m[1]} cfg={m[3]}")
    print(f"  minFull={best_v['min_full']:+.2f}  minHold={best_v['min_hold']:+.2f}"
          f"  max_dd={best_v['max_dd']*100:.1f}%  n_trades_min={n_trades_min}")
    print(f"  fee@0.30%RT survives={fee_survives}  causality_ok={caus_ok}  grade={grade}")
    print(f"  marginal: corr_full={corr_full} verdict={verdict} insample_edge={has_edge}"
          f" is_hedge={is_hedge} robust_oos={robust_oos} multicut={multicut}")
    print(f"            clean_year_uplift={clean_year}  blend_w25_uplift_hold={up25_hold}")
    print(f"            blend_w50: full={w50.get('full')} hold={w50.get('hold')} dd={w50.get('dd')}")
    print(f"  earns_slot={earns_slot}  beats_winner={beats}")
    print(f"\n  (WINNER ref: minFull={w_minfull:+.2f} minHold={w_minhold:+.2f} maxDD={w_maxdd*100:.0f}%)")

    # machine-readable tail for the agent
    import json
    print("\nRESULT_JSON=" + json.dumps({
        "best_tag": best_tag,
        "best_config": {"members": [{"name": m[0], "kind": m[1], "cfg": m[3]} for m in best_members]},
        "min_full": best_v["min_full"], "min_hold": best_v["min_hold"],
        "max_dd": best_v["max_dd"], "n_trades_min": n_trades_min,
        "fee_survives": bool(fee_survives), "causality_ok": bool(caus_ok), "grade": grade,
        "corr_full": corr_full, "marginal_verdict": verdict, "has_insample_edge": bool(has_edge),
        "is_hedge": bool(is_hedge), "robust_oos": bool(robust_oos),
        "multicut_persistent": bool(multicut), "clean_year_uplift": clean_year,
        "blend_w25_uplift_hold": up25_hold, "earns_slot": bool(earns_slot), "beats_winner": bool(beats),
    }, default=str))
