"""altlib — SHARED HONEST EVALUATION LIBRARY for the alt-strategy fan-out (2026-06-20).

Built for the "studia altre strategie alternative su Deribit" research wave: >=100 agents,
each studying ONE distinct strategy hypothesis on the certified BTC/ETH (+ DVOL) universe.
Every agent imports THIS module so that:
  * NO look-ahead is structurally possible: a target/weight decided at close[i] is held
    during bar i+1 (the evaluator shifts it for you — you never multiply by r[i] with a
    weight that used close[i] for the *same* bar).
  * Fees are realistic Deribit (0.10% RT taker = 0.0005/side) and a fee SWEEP is built in.
  * Metrics are comparable: FULL Sharpe/CAGR/maxDD, HOLD-OUT (2025-01-01+), per-year.
  * Only certified data exists (BTC/ETH from Deribit mainnet, DVOL from Deribit). load()
    raises on anything else — a physical guardrail.

Two evaluation styles:
  1. eval_weights(df, target)  -> for CONTINUOUS-position strategies (trend, vol overlays,
     pairs, risk-parity). `target` is a per-bar position (fraction of equity, sign=dir),
     decided with data <= close[i]. VECTORIZED (numpy) -> fast even on 68k 1h bars.
  2. eval_signals(df, entries) -> for DISCRETE entry/exit strategies (breakout w/ TP-SL,
     mean-reversion bounce). Wraps the project's trade-based harness. Use on 1h/1d only
     (the Python loop is O(n*max_bars); 5m has 840k bars -> too slow on 2 CPUs).

Quick start (inside an agent script):
    import sys; sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/alt")
    import altlib as al
    rep = al.study_weights("MY-STRAT", lambda df: my_target(df), tfs=("1d","12h"))
    print(al.fmt(rep)); print(al.as_json(rep))   # human + machine
"""
from __future__ import annotations

import inspect
import json
import sys
from functools import lru_cache
from pathlib import Path

import numpy as np
import pandas as pd

# --- make `from src...` work no matter where the agent's script lives -------
_ROOT = Path(__file__).resolve().parents[3]
if str(_ROOT) not in sys.path:
    sys.path.insert(0, str(_ROOT))

from src.backtest.harness import backtest_signals, load  # noqa: E402
from src.strategies.trend_portfolio import resample_tf  # noqa: E402

HOLDOUT = pd.Timestamp("2025-01-01", tz="UTC")
FEE_SIDE = 0.0005          # 0.05%/side = 0.10% round-trip (Deribit taker)
FEE_SWEEP = (0.0, 0.0005, 0.001, 0.0015)   # per-side fee grid for robustness
CERTIFIED = ("BTC", "ETH")
DATA_DIR = _ROOT / "data" / "raw"


# ===========================================================================
# DATA (cached) — 1h base, resampled to >=4h; DVOL aligned causally.
# ===========================================================================
@lru_cache(maxsize=32)
def get(asset: str, tf: str) -> pd.DataFrame:
    """Certified OHLCV with a tz-aware 'datetime' col and RangeIndex.
    tf in {5m,15m,1h} loaded directly; {4h,6h,8h,12h,1d,2d,3d,1w} resampled from 1h.
    Resample uses the leak-free per-single-TF path (trend_portfolio.resample_tf)."""
    asset = asset.upper()
    if asset not in CERTIFIED:
        raise ValueError(f"Asset non certificato: {asset}. Universo={CERTIFIED}.")
    tf = tf.lower()
    if tf in ("5m", "15m", "1h"):
        df = load(asset, tf)
    else:
        rule = {"4h": "4h", "6h": "6h", "8h": "8h", "12h": "12h",
                "1d": "1D", "2d": "2D", "3d": "3D", "1w": "1W"}.get(tf)
        if rule is None:
            raise ValueError(f"TF non gestito: {tf}")
        df = resample_tf(load(asset, "1h"), rule)
    df = df.reset_index(drop=True)
    if "datetime" not in df.columns:
        df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True)
    return df


@lru_cache(maxsize=8)
def _dvol_raw(asset: str) -> pd.DataFrame:
    p = DATA_DIR / f"dvol_{asset.lower()}.parquet"
    if not p.exists():
        raise FileNotFoundError(f"DVOL non trovato: {p}")
    d = pd.read_parquet(p).sort_values("timestamp").reset_index(drop=True)
    return d


def dvol(df: pd.DataFrame, asset: str) -> np.ndarray:
    """Deribit DVOL (implied vol index) aligned CAUSALLY to df's bars.
    For each bar we take the most recent DVOL value timestamped at/before the bar's
    open (merge_asof backward) -> known by decision time. NaN before DVOL history
    (DVOL starts 2021-03). Returns float array len(df), in vol POINTS (e.g. 65.0)."""
    d = _dvol_raw(asset)
    left = pd.DataFrame({"timestamp": df["timestamp"].astype("int64").values})
    merged = pd.merge_asof(left, d.rename(columns={"close": "dvol"}),
                           on="timestamp", direction="backward")
    return merged["dvol"].values.astype(float)


# ===========================================================================
# INDICATORS (all causal: value at i uses data <= i)
# ===========================================================================
def simple_returns(c: np.ndarray) -> np.ndarray:
    r = np.zeros(len(c)); r[1:] = c[1:] / c[:-1] - 1.0
    return r


def log_returns(c: np.ndarray) -> np.ndarray:
    r = np.zeros(len(c)); r[1:] = np.log(c[1:] / c[:-1])
    return r


def ema(x: np.ndarray, span: int) -> np.ndarray:
    return pd.Series(x).ewm(span=span, adjust=False).mean().values


def sma(x: np.ndarray, win: int) -> np.ndarray:
    return pd.Series(x).rolling(win, min_periods=win).mean().values


def rolling_std(x: np.ndarray, win: int) -> np.ndarray:
    return pd.Series(x).rolling(win, min_periods=max(2, win // 2)).std().values


def zscore(x: np.ndarray, win: int) -> np.ndarray:
    s = pd.Series(x)
    m = s.rolling(win, min_periods=win).mean()
    sd = s.rolling(win, min_periods=win).std()
    return ((s - m) / sd.replace(0, np.nan)).values


def rsi(c: np.ndarray, win: int = 14) -> np.ndarray:
    d = np.diff(c, prepend=c[0])
    up = pd.Series(np.where(d > 0, d, 0.0)).ewm(alpha=1 / win, adjust=False).mean()
    dn = pd.Series(np.where(d < 0, -d, 0.0)).ewm(alpha=1 / win, adjust=False).mean()
    rs = up / dn.replace(0, np.nan)
    return (100 - 100 / (1 + rs)).values


def atr(df: pd.DataFrame, win: int = 14) -> np.ndarray:
    h, l, c = df["high"].values, df["low"].values, df["close"].values
    pc = np.roll(c, 1); pc[0] = c[0]
    tr = np.maximum(h - l, np.maximum(np.abs(h - pc), np.abs(l - pc)))
    return pd.Series(tr).ewm(alpha=1 / win, adjust=False).mean().values


def realized_vol(r: np.ndarray, win: int, bars_per_year: float) -> np.ndarray:
    """Annualized realized vol from returns up to i inclusive (no leakage)."""
    return pd.Series(r).rolling(win, min_periods=max(2, win // 2)).std().values * np.sqrt(bars_per_year)


def donchian(df: pd.DataFrame, win: int):
    """Upper/lower channel using bars STRICTLY before i (shifted) -> a close[i] that
    breaks the prior `win`-bar high is a real, tradeable breakout at close[i]."""
    hi = pd.Series(df["high"].values).rolling(win, min_periods=win).max().shift(1).values
    lo = pd.Series(df["low"].values).rolling(win, min_periods=win).min().shift(1).values
    return hi, lo


def bbands(c: np.ndarray, win: int = 20, k: float = 2.0):
    m = pd.Series(c).rolling(win, min_periods=win).mean()
    sd = pd.Series(c).rolling(win, min_periods=win).std()
    return (m + k * sd).values, m.values, (m - k * sd).values


def _call_target(fn, df: pd.DataFrame, asset: str):
    """Call a strategy fn as fn(df, asset) when it accepts 2 args, else fn(df).
    Lets DVOL/cross-asset strategies receive the asset cleanly (no inference hacks)."""
    try:
        n = len(inspect.signature(fn).parameters)
    except (ValueError, TypeError):
        n = 1
    return fn(df, asset) if n >= 2 else fn(df)


def bars_per_year(df: pd.DataFrame) -> float:
    dt = pd.to_datetime(df["datetime"]).diff().dt.total_seconds().median()
    return 86400 * 365.25 / dt if dt and dt > 0 else 365.25


def bars_per_day(df: pd.DataFrame) -> int:
    dt = pd.to_datetime(df["datetime"]).diff().dt.total_seconds().median()
    return max(1, round(86400 / dt))


def vol_target(target_dir: np.ndarray, df: pd.DataFrame, target_vol: float = 0.20,
               vol_win_days: int = 30, leverage_cap: float = 2.0) -> np.ndarray:
    """Scale a direction array in [-1,1] to a vol-targeted position (TP01-style).
    Causal: uses realized vol up to i. Returns position clipped to +/-leverage_cap."""
    c = df["close"].values.astype(float)
    bpd = bars_per_day(df)
    bpy = bpd * 365.25
    vol = realized_vol(simple_returns(c), max(2, vol_win_days * bpd), bpy)
    scal = np.where((vol > 0) & np.isfinite(vol), target_vol / vol, 0.0)
    tgt = np.clip(target_dir * scal, -leverage_cap, leverage_cap)
    tgt[~np.isfinite(tgt)] = 0.0
    return tgt


# ===========================================================================
# METRICS
# ===========================================================================
def _metrics_from_net(net: np.ndarray, idx: pd.DatetimeIndex) -> dict:
    net = np.nan_to_num(net, nan=0.0)
    eq = np.cumprod(1.0 + np.clip(net, -0.99, None))
    rr = net[np.isfinite(net)]
    bpy = 86400 * 365.25 / (pd.Series(idx).diff().dt.total_seconds().median() or 86400)
    sharpe = float(np.mean(rr) / np.std(rr) * np.sqrt(bpy)) if np.std(rr) > 0 else 0.0
    pk = np.maximum.accumulate(eq)
    dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0
    span_days = (idx[-1] - idx[0]).total_seconds() / 86400 if len(idx) > 1 else 1.0
    years = max(span_days / 365.25, 1e-6)
    total = eq[-1] / eq[0] if len(eq) else 1.0
    cagr = total ** (1 / years) - 1 if total > 0 else -1.0
    return dict(sharpe=round(sharpe, 3), cagr=round(cagr, 4), maxdd=round(dd, 4),
                ret=round(total - 1, 4), n=int(len(rr)))


def _yearly(net: np.ndarray, idx: pd.DatetimeIndex) -> dict:
    s = pd.Series(np.nan_to_num(net), index=idx)
    out = {}
    for y, g in s.groupby(s.index.year):
        eq = np.cumprod(1 + g.values); pk = np.maximum.accumulate(eq)
        out[int(y)] = dict(ret=round(float(eq[-1] - 1), 4),
                           dd=round(float(np.max((pk - eq) / pk)), 4))
    return out


def eval_weights(df: pd.DataFrame, target: np.ndarray, fee_side: float = FEE_SIDE) -> dict:
    """Honest backtest of a CONTINUOUS position series.
    target[i] is decided with data <= close[i]; it is HELD during bar i+1. The shift
    is done HERE -> you cannot leak by construction. Fee charged on |Δposition| turnover.
    Returns {full, holdout, yearly, time_in_market, turnover_per_year, net, idx}."""
    c = df["close"].values.astype(float)
    target = np.asarray(target, float)
    target = np.nan_to_num(target, nan=0.0)
    r = simple_returns(c)
    pos = np.zeros(len(target)); pos[1:] = target[:-1]      # held during bar t = decided at t-1
    gross = pos * r
    turn = np.abs(np.diff(pos, prepend=0.0))
    net = gross - fee_side * turn
    net[0] = 0.0
    idx = pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True))
    full = _metrics_from_net(net, idx)
    hmask = idx >= HOLDOUT
    hold = _metrics_from_net(net[hmask], idx[hmask]) if hmask.sum() > 3 else dict(sharpe=0.0, n=0)
    bpy_d = bars_per_day(df) * 365.25
    return dict(full=full, holdout=hold, yearly=_yearly(net, idx),
                time_in_market=round(float(np.mean(pos != 0)), 3),
                turnover_per_year=round(float(turn.sum() / (len(turn) / bpy_d)), 1),
                net=net, idx=idx)


def eval_signals(df: pd.DataFrame, entries: list, fee_rt: float = 2 * FEE_SIDE,
                 leverage: float = 1.0, asset: str = "", tf: str = "") -> dict:
    """Honest backtest of DISCRETE entry/exit signals (TP/SL/max_bars). Wraps the
    project trade-based harness, adds a standardized hold-out split. Use on 1h/1d."""
    m = backtest_signals(df, entries, fee_rt=fee_rt, leverage=leverage, asset=asset, tf=tf)
    idx = pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True)) if m.eq_index is not None \
        else pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True))
    eq = m.equity
    hmask = idx >= HOLDOUT
    hold = dict(sharpe=0.0, ret=0.0, n=0)
    if hmask.sum() > 3:
        he = eq[hmask]
        hr = np.diff(he) / he[:-1]
        bpy = m.bars_per_year or 365.0
        hsharpe = float(np.mean(hr) / np.std(hr) * np.sqrt(bpy)) if len(hr) and np.std(hr) > 0 else 0.0
        hold = dict(sharpe=round(hsharpe, 3), ret=round(float(he[-1] / he[0] - 1), 4), n=int(hmask.sum()))
    full = dict(sharpe=round(m.sharpe, 3), cagr=round(m.cagr, 4), maxdd=round(m.max_dd, 4),
                ret=round(m.net_return, 4), n=int(m.n_trades))
    return dict(full=full, holdout=hold, n_trades=int(m.n_trades),
                win_rate=round(m.win_rate, 1), time_in_market=round(m.time_in_market, 3),
                yearly={int(y): round(v, 4) for y, v in m.yearly.items()})


# ===========================================================================
# MARGINAL SCORING vs TP01 baseline — the lesson of the 2026-06-20 sweep.
#
# Absolute Sharpe is NOT enough to earn a portfolio slot: an overlay on TSMOM
# (DVOL gate, low-vol filter, kill-switch, ...) inherits TP01's trend Sharpe and
# scores ~1.0-1.3 while adding ZERO new return. The only question that matters for
# a NEW sleeve is: does it IMPROVE the existing TP01 portfolio out-of-sample?
# We answer it three ways: (a) blend uplift (Sharpe of TP01 + w*candidate minus
# TP01 alone, full & hold-out), (b) correlation to TP01, (c) residual alpha after
# removing the TP01 beta (the part of the candidate orthogonal to trend).
# ===========================================================================
def _sh(s) -> float:
    r = np.asarray(s.dropna().values, float)
    return float(np.mean(r) / np.std(r) * np.sqrt(365.25)) if len(r) > 2 and np.std(r) > 0 else 0.0


def _dd_ret(s) -> float:
    eq = np.cumprod(1.0 + np.asarray(s.dropna().values, float))
    pk = np.maximum.accumulate(eq)
    return float(np.max((pk - eq) / pk)) if len(eq) else 0.0


def _to_daily(s: pd.Series) -> pd.Series:
    s = s.dropna().sort_index()
    if not isinstance(s.index, pd.DatetimeIndex):
        s.index = pd.to_datetime(s.index, utc=True)
    if s.index.tz is None:
        s.index = s.index.tz_localize("UTC")
    return ((1.0 + s).resample("1D").prod() - 1.0).dropna()


@lru_cache(maxsize=2)
def tp01_baseline_daily() -> pd.Series:
    """The reference a candidate must BEAT: TP01 CANONICAL, 50/50 BTC+ETH, net daily
    returns on common dates (full Sharpe ~1.30, hold-out ~0.31). Cached."""
    from src.strategies.trend_portfolio import CANONICAL, TrendPortfolio
    tp = TrendPortfolio(**CANONICAL)
    series = {}
    for a in CERTIFIED:
        df = get(a, "1d")
        net, _ = tp.net_returns(df)
        series[a] = pd.Series(net, index=pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True)))
    J = pd.concat(series, axis=1, join="inner").fillna(0.0)
    return _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]])


def candidate_daily(target_fn, tf: str = "1d", fee_side: float = FEE_SIDE) -> pd.Series:
    """Build a candidate's 50/50 BTC+ETH net DAILY return series (same convention as
    tp01_baseline_daily) from a continuous-position target_fn. Intraday TFs are
    compounded to daily so they align with the TP01 baseline grid."""
    series = {}
    for a in CERTIFIED:
        df = get(a, tf)
        ev = eval_weights(df, _call_target(target_fn, df, a), fee_side=fee_side)
        series[a] = pd.Series(ev["net"], index=ev["idx"])
    J = pd.concat(series, axis=1, join="inner").fillna(0.0)
    return _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]])


def _uplift_series(B: pd.Series, C: pd.Series, w: float = 0.25) -> float:
    """Sharpe of the (1-w)*TP01 + w*candidate blend minus Sharpe of TP01 alone."""
    return _sh((1 - w) * B + w * C) - _sh(B)


def _null_uplift_pctl(B: pd.Series, C: pd.Series, w: float = 0.25,
                      n: int = 300, seed: int = 20260621):
    """Where does the candidate's blend-uplift sit vs the NULL of a zero-correlation
    noise asset with the SAME mean & vol? Lesson of 2026-06-21: a low-corr asset with a
    little positive drift 'adds' ~+0.03 Sharpe by pure diversification MATH — that is not
    a signal. We draw `n` iid-normal assets (same mean/std as C, independent of B => corr 0
    by construction), measure each one's uplift, and return (real_uplift, percentile of
    real vs the null). pctl >= ~0.8 => the uplift is meaningfully above diversification
    math; pctl ~0.5 => it IS diversification math. Seeded -> deterministic."""
    Bx, Cx = B.align(C, join="inner")
    bs, cs = Bx.values.astype(float), Cx.values.astype(float)
    if len(cs) < 30:
        return None, None
    base = _sh(Bx)
    real = _sh((1 - w) * Bx + w * Cx) - base
    mu, sd = float(np.nanmean(cs)), float(np.nanstd(cs))
    if sd == 0:
        return round(real, 3), None
    rng = np.random.default_rng(seed)
    draws = rng.normal(mu, sd, size=(n, len(cs)))
    blends = (1 - w) * bs[None, :] + w * draws
    m, s = blends.mean(axis=1), blends.std(axis=1)
    null = np.where(s > 0, m / s * np.sqrt(365.25), 0.0) - base
    return round(float(real), 3), round(float(np.mean(null <= real)), 3)


def marginal_vs_tp01(cand_daily: pd.Series, weights=(0.25, 0.5)) -> dict:
    """Does this candidate IMPROVE the TP01 portfolio? Returns correlation, blend uplift
    (full & hold-out, per weight), TP01-beta + residual alpha, and a verdict:
      ADDS      -> lifts the blend, PERSISTENTLY (multi-cut), beats the zero-corr noise
                   null, in BOTH TP01-up and TP01-down regimes
      HEDGE     -> low corr but only pays when TP01 is WEAK (a drawdown dampener, not a
                   standing premium): real, but price it as a hedge, not as alpha
      NOISE     -> uplift indistinguishable from a random zero-corr asset (diversification
                   math, not a signal)
      REDUNDANT -> ~identical to TP01 (corr high, ~zero uplift): a re-skin, no slot
      DILUTES   -> drags the blend down
      NEUTRAL   -> changes little either way (a weak, optional satellite at best)
    Score a NEW sleeve on THIS, not on absolute Sharpe.

    Hardened 2026-06-21 (ortho wave): the fixed-HOLDOUT uplift + drop-month jackknife was
    fooled (17/18 relative-value books 'ADDS' on a single 2025 ETH-bleed window). Three
    gates added: (1) MULTI-CUT persistence (positive uplift at several hold-out starts, not
    only 2025); (2) NOISE-NULL (uplift must beat a zero-corr random asset); (3) HEDGE vs
    alpha (a low-corr sleeve that only helps when TP01 is down is a hedge)."""
    B = tp01_baseline_daily()
    J = pd.concat({"B": B, "C": cand_daily}, axis=1, join="inner").dropna()
    if len(J) < 30:
        return dict(marginal_verdict="N/A", reason="insufficient overlap with TP01 baseline")
    if J["C"].std() == 0:
        return dict(marginal_verdict="NEUTRAL", reason="flat/zero candidate (no exposure)",
                    corr_full=None, blends={"w25": dict(uplift_full=0.0, uplift_hold=0.0)})
    JH = J[J.index >= HOLDOUT]
    has_h = len(JH) > 5
    out = {
        "n_days": int(len(J)), "n_hold_days": int(len(JH)),
        "corr_full": round(float(J["B"].corr(J["C"])), 3),
        "corr_hold": round(float(JH["B"].corr(JH["C"])), 3) if has_h else None,
        "tp01_full_sharpe": round(_sh(J["B"]), 3), "tp01_hold_sharpe": round(_sh(JH["B"]), 3) if has_h else None,
        "cand_full_sharpe": round(_sh(J["C"]), 3), "cand_hold_sharpe": round(_sh(JH["C"]), 3) if has_h else None,
    }
    blends = {}
    for w in weights:
        bf, bh = (1 - w) * J["B"] + w * J["C"], (1 - w) * JH["B"] + w * JH["C"]
        blends[f"w{int(w * 100)}"] = dict(
            full=round(_sh(bf), 3), hold=round(_sh(bh), 3) if has_h else None,
            uplift_full=round(_sh(bf) - _sh(J["B"]), 3),
            uplift_hold=round(_sh(bh) - _sh(JH["B"]), 3) if has_h else None,
            dd=round(_dd_ret(bf), 4))
    out["blends"] = blends
    b, c = J["B"].values, J["C"].values
    beta = float(np.cov(c, b)[0, 1] / np.var(b)) if np.var(b) > 0 else 0.0
    resid = c - beta * b
    out["beta_to_tp01"] = round(beta, 3)
    out["resid_sharpe_full"] = round(float(np.mean(resid) / np.std(resid) * np.sqrt(365.25)) if np.std(resid) > 0 else 0.0, 3)
    out["alpha_ann"] = round(float(np.mean(resid) * 365.25), 4)
    # OOS robustness — the marginal point-estimate can be fooled by ONE lucky month
    # (e.g. KAMA: +0.06 uplift that flips to -0.03 when its best month is dropped). Require
    # the blend uplift to be positive in the earliest CLEAN hold-out year AND to survive a
    # drop-one-month jackknife. This is lesson #2 of the 2026-06-20 sweep, in code.
    out["clean_year_uplift"] = out["jackknife_min_uplift"] = None
    robust_h = False
    if has_h:
        def _u(sub):
            return _uplift_series(sub["B"], sub["C"])
        yrs = sorted(set(JH.index.year))
        clean = JH[JH.index.year == yrs[0]]
        cu = _u(clean) if len(clean) > 20 else None
        months = sorted(set(zip(JH.index.year, JH.index.month)))
        jk = (min(_u(JH[~((JH.index.year == y) & (JH.index.month == mo))]) for y, mo in months)
              if len(months) > 1 else _u(JH))
        out["clean_year_uplift"] = round(cu, 3) if cu is not None else None
        out["jackknife_min_uplift"] = round(jk, 3) if jk is not None else None
        robust_h = bool(cu is not None and cu > 0.02 and jk is not None and jk > 0.0)

    # --- GATE 1: MULTI-CUT PERSISTENCE -------------------------------------------------
    # Uplift at the start of each year (not only the fixed HOLDOUT). A real edge adds at
    # SEVERAL cuts incl. an early one; a regime artifact only adds at the latest window.
    mc = {}
    for y in sorted(set(J.index.year))[1:]:
        sub = J[J.index >= pd.Timestamp(f"{y}-01-01", tz="UTC")]
        if len(sub) >= 120:
            mc[y] = round(_uplift_series(sub["B"], sub["C"]), 3)
    out["multicut_uplift"] = mc
    pos = [u for u in mc.values() if u > 0]
    earliest = mc[min(mc)] if mc else None
    multicut_persistent = bool(len(mc) >= 2 and len(pos) / len(mc) >= 0.6
                               and earliest is not None and earliest > 0.0)
    out["multicut_persistent"] = multicut_persistent

    # --- GATE 2: NOISE-NULL (uplift must beat a random zero-corr asset) -----------------
    JI = J[J.index < HOLDOUT]                 # in-sample part (not the lucky recent window)
    real_is, pctl_is = _null_uplift_pctl(JI["B"], JI["C"]) if len(JI) >= 60 else (None, None)
    real_f, pctl_f = _null_uplift_pctl(J["B"], J["C"])
    cand_is_sharpe = round(_sh(JI["C"]), 3) if len(JI) >= 60 else None
    out["null_pctl_insample"] = pctl_is
    out["null_pctl_full"] = pctl_f
    out["cand_insample_sharpe"] = cand_is_sharpe
    # A candidate must STAND ON ITS OWN before the hold-out: a real in-sample standalone
    # Sharpe. The ortho basket's in-sample Sharpe was 0.29 -> its only "value" was the
    # diversification math of a near-zero-Sharpe stream, dressed up by the lucky 2025 window.
    # (null_pctl_* are reported as the diversification-math context: a low-corr asset adds
    # ~+0.03 Sharpe by math, so pctl~0.5 just means "no TP01-specific timing" — true of GOOD
    # and BAD uncorrelated sleeves alike, so it can't be the gate. The in-sample edge is.)
    has_insample_edge = (cand_is_sharpe is None) or (cand_is_sharpe >= 0.5)
    out["has_insample_edge"] = bool(has_insample_edge)
    out["beats_noise_null"] = bool(has_insample_edge)   # back-compat alias for the gate

    # --- GATE 3: HEDGE vs ALPHA (does it only pay when TP01 is weak?) -------------------
    yr_sh, yr_up = [], []
    for y in sorted(set(J.index.year)):
        sub = J[J.index.year == y]
        if len(sub) >= 40:
            yr_sh.append(_sh(sub["B"])); yr_up.append(_uplift_series(sub["B"], sub["C"]))
    hedge_corr = (round(float(np.corrcoef(yr_sh, yr_up)[0, 1]), 3)
                  if len(yr_sh) >= 3 and np.std(yr_sh) > 0 and np.std(yr_up) > 0 else None)
    trail = J["B"].rolling(60, min_periods=20).sum().shift(1)
    up_seg, dn_seg = J[trail > 0], J[trail <= 0]
    u_up = _uplift_series(up_seg["B"], up_seg["C"]) if len(up_seg) > 30 else None
    u_dn = _uplift_series(dn_seg["B"], dn_seg["C"]) if len(dn_seg) > 30 else None
    out["hedge_yearly_corr"] = hedge_corr
    out["uplift_tp01_up"] = round(u_up, 3) if u_up is not None else None
    out["uplift_tp01_down"] = round(u_dn, 3) if u_dn is not None else None
    is_hedge = bool(hedge_corr is not None and hedge_corr < -0.5
                    and u_up is not None and u_up <= 0.0
                    and u_dn is not None and u_dn > 0.05)
    out["is_hedge"] = is_hedge

    # robust_oos now REQUIRES multi-cut persistence (kills the single-window winners)
    out["robust_oos"] = bool(robust_h and multicut_persistent)

    # --- VERDICT ----------------------------------------------------------------------
    up_h = blends["w25"]["uplift_hold"]
    up_f = blends["w25"]["uplift_full"]
    ch = out["corr_hold"] if out["corr_hold"] is not None else out["corr_full"]
    if out["corr_full"] > 0.9 and (up_h is None or abs(up_h) < 0.05):
        v = "REDUNDANT"
    elif up_f <= -0.10 and (up_h is None or up_h <= 0.0):
        v = "DILUTES"
    elif is_hedge:
        v = "HEDGE"
    elif not has_insample_edge:
        v = "NOISE"
    elif (up_h is not None and up_h >= 0.05 and up_f > -0.15 and ch < 0.85
          and multicut_persistent):
        v = "ADDS"
    else:
        v = "NEUTRAL"
    out["marginal_verdict"] = v
    return out


def study_marginal(name: str, target_fn, tf: str = "1d", fee_side: float = FEE_SIDE) -> dict:
    """Score a continuous candidate BOTH ways: absolute (study_weights) AND marginal vs
    TP01. The combined gate: a candidate earns a sleeve slot only if it is not FAIL on
    absolute robustness AND marginal_verdict == 'ADDS'."""
    absolute = study_weights(name, target_fn, tfs=(tf,))
    marg = marginal_vs_tp01(candidate_daily(target_fn, tf=tf, fee_side=fee_side))
    abs_grade = absolute["verdict"]["grade"]
    # ADDS already embeds multi-cut + beats-null + not-hedge; we also require robust_oos
    # (multi-cut robustness) explicitly. A HEDGE/NOISE/NEUTRAL never earns a live slot.
    earns_slot = (abs_grade != "FAIL" and marg.get("marginal_verdict") == "ADDS"
                  and marg.get("robust_oos", False)
                  and marg.get("beats_noise_null", False)
                  and not marg.get("is_hedge", False))
    return dict(name=name, tf=tf, absolute=absolute, marginal=marg,
                abs_grade=abs_grade, marginal_verdict=marg.get("marginal_verdict"),
                earns_slot=earns_slot)


def fmt_marginal(rep: dict) -> str:
    m = rep["marginal"]
    bl = m.get("blends", {})
    lines = [f"=== {rep['name']} | abs={rep['abs_grade']}  marginal={rep['marginal_verdict']}  "
             f"EARNS_SLOT={rep['earns_slot']}"]
    lines.append(f"  corr->TP01 full {m.get('corr_full')} hold {m.get('corr_hold')}  "
                 f"beta {m.get('beta_to_tp01')}  resid Sharpe {m.get('resid_sharpe_full')}  alpha/yr {m.get('alpha_ann')}")
    lines.append(f"  OOS robustness: clean-year uplift {m.get('clean_year_uplift')}  "
                 f"drop-best-month {m.get('jackknife_min_uplift')}  robust_oos={m.get('robust_oos')}")
    lines.append(f"  multi-cut persistence: {m.get('multicut_uplift')}  persistent={m.get('multicut_persistent')}")
    lines.append(f"  in-sample edge: standalone Sharpe {m.get('cand_insample_sharpe')}  "
                 f"has_insample_edge={m.get('has_insample_edge')}  "
                 f"(diversification-math null pctl in-sample {m.get('null_pctl_insample')} full {m.get('null_pctl_full')})")
    lines.append(f"  hedge check: yearly corr(TP01-Sh, uplift) {m.get('hedge_yearly_corr')}  "
                 f"uplift TP01-up {m.get('uplift_tp01_up')} / TP01-down {m.get('uplift_tp01_down')}  "
                 f"is_hedge={m.get('is_hedge')}")
    lines.append(f"  standalone: TP01 full {m.get('tp01_full_sharpe')}/hold {m.get('tp01_hold_sharpe')}  |  "
                 f"cand full {m.get('cand_full_sharpe')}/hold {m.get('cand_hold_sharpe')}")
    for w, d in bl.items():
        uh = "n/a" if d["uplift_hold"] is None else f"{d['uplift_hold']:+.3f}"
        hold = "n/a" if d["hold"] is None else f"{d['hold']}"
        lines.append(f"  blend {w}: full {d['full']} (uplift {d['uplift_full']:+.3f})  "
                     f"hold {hold} (uplift {uh})  DD {d['dd'] * 100:.1f}%")
    return "\n".join(lines)


# ===========================================================================
# HARNESS REALISM — two gates codified from the 2026-06-21 intraday wave.
#
# LESSON 1 (day-boundary): open_drive ("first 8h UTC predicts rest-of-day") scored a
# +0.23 uplift but INVERTED to -0.10 when the UTC day start was shifted 4h — a calendar-
# LABELING artifact, not an intraday effect. A real hour/session/day edge degrades
# gracefully under a boundary shift; an artifact flips sign.
#
# LESSON 2 (small-cap fills): eval_weights charges fee on EVERY |Δposition|, incl. the
# thousands of sub-dollar rebalances a vol-target overlay produces. At ~$600 real capital a
# $0.03 trade can't execute — the modeled proportional fee is a continuous-rebalancing
# fiction. eval_weights_smallcap skips changes below min_order and reports the Sharpe haircut.
# ===========================================================================
def _shift_calendar(df: pd.DataFrame, offset_hours: int) -> pd.DataFrame:
    """Relabel the clock the SIGNAL sees by +offset_hours (datetime & timestamp), leaving
    prices/returns untouched -> the signal's .dt.hour / day-grouping shifts, the backtest
    does not. (get() is cached; copy so we never mutate the shared frame.)"""
    d = df.copy()
    dt = pd.to_datetime(d["datetime"], utc=True) + pd.Timedelta(hours=offset_hours)
    d["datetime"] = dt
    if "timestamp" in d:
        d["timestamp"] = d["timestamp"].astype("int64") + int(offset_hours * 3600 * 1000)
    return d


def day_boundary_robust(target_fn, tf: str = "1h",
                        offsets=(0, 3, 6, 9, 12, 15, 18, 21), w: float = 0.25) -> dict:
    """Is a candidate's marginal uplift ROBUST to shifting the UTC day boundary? For each
    offset we relabel the calendar the signal sees, recompute its 50/50 BTC+ETH daily series
    and the blend uplift vs TP01. A datetime-independent signal is INVARIANT (spread ~0); a
    calendar signal that stays positive is ROBUST; one whose uplift flips sign is ARTIFACT-RISK
    (open_drive). Run this on ANY hour/session/day-of-week signal before believing it."""
    B = tp01_baseline_daily()
    per = {}
    for off in offsets:
        series = {}
        for a in CERTIFIED:
            df0 = get(a, tf)                                  # ORIGINAL bars/dates
            tgt = _call_target(target_fn, _shift_calendar(df0, off), a)  # signal sees shifted clock
            ev = eval_weights(df0, tgt)                        # backtest on the real calendar
            series[a] = pd.Series(ev["net"], index=ev["idx"])
        J = pd.concat(series, axis=1, join="inner").fillna(0.0)
        cand = _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]])
        JJ = pd.concat({"B": B, "C": cand}, axis=1, join="inner").dropna()
        per[int(off)] = round(_sh((1 - w) * JJ["B"] + w * JJ["C"]) - _sh(JJ["B"]), 3) if len(JJ) > 30 else None
    ups = [v for v in per.values() if v is not None]
    if not ups:
        return dict(per_offset=per, verdict="N/A", reason="no evaluable offsets")
    spread = round(max(ups) - min(ups), 3)
    calendar_sensitive = spread > 0.02
    robust = min(ups) > 0
    verdict = ("INVARIANT" if not calendar_sensitive else ("ROBUST" if robust else "ARTIFACT-RISK"))
    return dict(per_offset=per, base=per[offsets[0]], min=min(ups), max=max(ups),
                spread=spread, calendar_sensitive=calendar_sensitive,
                robust_to_boundary=robust, verdict=verdict)


def eval_weights_smallcap(df: pd.DataFrame, target, capital: float = 600.0,
                          min_order: float = 5.0, fee_side: float = FEE_SIDE) -> dict:
    """Honest net at SMALL capital. A desired position change whose notional |Δw|*capital is
    below min_order is NOT executed (held -> tracking error, no trade) — removing the
    continuous-rebalancing fiction. Returns realistic vs modeled metrics, the Sharpe haircut,
    and the number of trades that actually execute. (Applies to ANY sleeve at this capital,
    TP01 included.)"""
    c = df["close"].values.astype(float)
    tgt = np.clip(np.nan_to_num(np.asarray(target, float)), -10, 10)
    held = np.empty(len(tgt)); cur = 0.0; n_tr = 0
    for i in range(len(tgt)):
        if abs(tgt[i] - cur) * capital >= min_order:
            cur = tgt[i]; n_tr += 1
        held[i] = cur
    r = simple_returns(c)
    pos = np.zeros(len(held)); pos[1:] = held[:-1]
    turn = np.abs(np.diff(pos, prepend=0.0))
    net = pos * r - fee_side * turn; net[0] = 0.0
    idx = pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True))
    real = _metrics_from_net(net, idx)
    modeled = eval_weights(df, tgt, fee_side=fee_side)["full"]
    bpy_d = bars_per_day(df) * 365.25
    return dict(realistic=real, modeled=modeled,
                sharpe_haircut=round(modeled["sharpe"] - real["sharpe"], 3),
                n_executed_trades=int(n_tr),
                executed_turnover_per_year=round(float(turn.sum() / (len(turn) / bpy_d)), 1))


def causality_ok(target_fn, tf: str = "1h", assets=CERTIFIED,
                 tail: int = 80, tol: float = 1e-3) -> dict:
    """Online-consistency / LOOK-AHEAD guard for a continuous target_fn(df) [or (df, asset)].
    eval_weights SHIFTS the position so you cannot leak by multiplying a weight by the SAME
    bar's return — but it does NOT verify the FEATURE construction is causal: a centered
    window, a .shift(-k), or a full-sample statistic would pass eval_weights yet peek at the
    future. Here we recompute the target on a TRUNCATED prefix and require its tail to MATCH
    target(full)[:cut] (the bars a deployable signal would have emitted in real time). Any
    future-peeking diverges. Run this in every altlib-based lab (blind/ortho already do)."""
    worst = 0.0; bad = False; checked = 0
    for a in assets:
        df = get(a, tf)
        full = np.nan_to_num(np.asarray(_call_target(target_fn, df, a), float))
        n = len(df)
        for cut in (int(n * 0.80), int(n * 0.92)):
            if cut <= tail + 5 or cut >= n:
                continue
            sub = df.iloc[:cut].reset_index(drop=True)
            s = np.nan_to_num(np.asarray(_call_target(target_fn, sub, a), float))
            if len(s) != cut:
                bad = True
                continue
            d = np.abs(s[cut - tail:cut] - full[cut - tail:cut])
            worst = max(worst, float(np.max(d)) if len(d) else 0.0)
            checked += 1
    return dict(ok=bool((not bad) and worst <= tol),
                max_tail_diff=round(worst, 8), checked=checked,
                reason=("length-mismatch on prefix" if bad else None))


# ===========================================================================
# DRIVERS — run a hypothesis across both assets, several TFs, with a fee sweep.
# ===========================================================================
def _verdict(per_cell: list[dict]) -> dict:
    """A finding PASSES only if it is robust: positive FULL Sharpe AND positive HOLD-OUT
    on BOTH assets in its best TF, survives the fee sweep, and isn't a 1-cell fluke."""
    if not per_cell:
        return dict(grade="FAIL", reason="no cells")
    ok = [c for c in per_cell if c.get("full_sharpe", -9) > 0]
    best = max(per_cell, key=lambda c: c.get("min_asset_holdout_sharpe", -9))
    pass_ = (best.get("min_asset_full_sharpe", -9) >= 0.5 and
             best.get("min_asset_holdout_sharpe", -9) >= 0.2 and
             best.get("fee_survives", False))
    weak = (best.get("min_asset_full_sharpe", -9) >= 0.3 and
            best.get("min_asset_holdout_sharpe", -9) >= 0.0)
    grade = "PASS" if pass_ else ("WEAK" if weak else "FAIL")
    return dict(grade=grade, best_tf=best.get("tf"),
                best_full_sharpe=best.get("min_asset_full_sharpe"),
                best_holdout_sharpe=best.get("min_asset_holdout_sharpe"),
                n_positive_cells=len(ok), n_cells=len(per_cell))


def study_weights(name: str, target_fn, tfs=("1d", "12h"),
                  assets=CERTIFIED, fee_sweep=FEE_SWEEP) -> dict:
    """Run a CONTINUOUS-position hypothesis on each (asset,tf), report robustness.
    target_fn(df) -> per-bar position array (decided <= close[i]). Returns a dict
    ready to print/serialize, with a conservative PASS/WEAK/FAIL verdict."""
    cells = []
    for tf in tfs:
        per_asset = {}
        fee_ok_all = True
        for a in assets:
            df = get(a, tf)
            tgt = _call_target(target_fn, df, a)
            base = eval_weights(df, tgt, fee_side=FEE_SIDE)
            sweep = {f"{2*f*100:.2f}%RT": eval_weights(df, tgt, fee_side=f)["full"]["sharpe"]
                     for f in fee_sweep}
            fee_ok = sweep.get("0.20%RT", -9) > 0
            fee_ok_all = fee_ok_all and fee_ok
            per_asset[a] = dict(full=base["full"], holdout=base["holdout"],
                                tim=base["time_in_market"], turnover=base["turnover_per_year"],
                                fee_sweep=sweep, yearly=base["yearly"])
        min_full = min(per_asset[a]["full"]["sharpe"] for a in assets)
        min_hold = min(per_asset[a]["holdout"].get("sharpe", 0.0) for a in assets)
        cells.append(dict(tf=tf, per_asset=per_asset,
                          min_asset_full_sharpe=round(min_full, 3),
                          min_asset_holdout_sharpe=round(min_hold, 3),
                          full_sharpe=round(np.mean([per_asset[a]["full"]["sharpe"] for a in assets]), 3),
                          fee_survives=fee_ok_all))
    return dict(name=name, kind="weights", cells=cells, verdict=_verdict(cells))


def study_signals(name: str, entries_fn, tfs=("1d",), assets=CERTIFIED,
                  fee_sweep=FEE_SWEEP, leverage: float = 1.0) -> dict:
    """Run a DISCRETE entry/exit hypothesis on each (asset,tf). entries_fn(df) ->
    list[dict|None] len(df). Use 1h/1d TFs only (Python loop)."""
    cells = []
    for tf in tfs:
        per_asset = {}
        fee_ok_all = True
        for a in assets:
            df = get(a, tf)
            ent = _call_target(entries_fn, df, a)
            base = eval_signals(df, ent, fee_rt=2 * FEE_SIDE, leverage=leverage, asset=a, tf=tf)
            sweep = {f"{2*f*100:.2f}%RT": eval_signals(df, ent, fee_rt=2 * f, leverage=leverage)["full"]["sharpe"]
                     for f in fee_sweep}
            fee_ok = sweep.get("0.20%RT", -9) > 0
            fee_ok_all = fee_ok_all and fee_ok
            per_asset[a] = dict(full=base["full"], holdout=base["holdout"],
                                n_trades=base["n_trades"], win_rate=base["win_rate"],
                                fee_sweep=sweep, yearly=base["yearly"])
        min_full = min(per_asset[a]["full"]["sharpe"] for a in assets)
        min_hold = min(per_asset[a]["holdout"].get("sharpe", 0.0) for a in assets)
        cells.append(dict(tf=tf, per_asset=per_asset,
                          min_asset_full_sharpe=round(min_full, 3),
                          min_asset_holdout_sharpe=round(min_hold, 3),
                          full_sharpe=round(np.mean([per_asset[a]["full"]["sharpe"] for a in assets]), 3),
                          fee_survives=fee_ok_all))
    return dict(name=name, kind="signals", cells=cells, verdict=_verdict(cells))


# ===========================================================================
# OUTPUT
# ===========================================================================
def _clean(o):
    if isinstance(o, dict):
        return {k: _clean(v) for k, v in o.items() if k not in ("net", "idx", "equity")}
    if isinstance(o, (list, tuple)):
        return [_clean(x) for x in o]
    if isinstance(o, (np.floating,)):
        return round(float(o), 4)
    if isinstance(o, (np.integer,)):
        return int(o)
    return o


def as_json(rep: dict) -> str:
    return json.dumps(_clean(rep), default=str)


def fmt(rep: dict) -> str:
    v = rep["verdict"]
    lines = [f"=== {rep['name']} [{rep['kind']}] -> {v['grade']} "
             f"(best {v.get('best_tf')}: full {v.get('best_full_sharpe')}, "
             f"hold {v.get('best_holdout_sharpe')})"]
    for c in rep["cells"]:
        lines.append(f"  TF {c['tf']:>4s}: minFull={c['min_asset_full_sharpe']:+.2f} "
                     f"minHold={c['min_asset_holdout_sharpe']:+.2f} feeOK={c['fee_survives']}")
        for a, pa in c["per_asset"].items():
            yr = " ".join(f"{y}:{d['ret']*100 if isinstance(d, dict) else d*100:+.0f}%"
                          for y, d in list(pa["yearly"].items()))
            lines.append(f"      {a}: full Sh={pa['full']['sharpe']:+.2f} "
                         f"DD={pa['full']['maxdd']*100:.0f}% ret={pa['full']['ret']*100:+.0f}% "
                         f"hold Sh={pa['holdout'].get('sharpe', 0):+.2f} | {yr}")
    return "\n".join(lines)


if __name__ == "__main__":
    # smoke test: buy&hold, TSMOM trend, donchian breakout
    print("--- SMOKE TEST altlib ---")
    bh = study_weights("BUYHOLD", lambda df: np.ones(len(df)), tfs=("1d",))
    print(fmt(bh))

    def tsmom(df):
        c = df["close"].values
        bpd = bars_per_day(df)
        d = np.zeros(len(c))
        for h in (30 * bpd, 90 * bpd, 180 * bpd):
            s = np.full(len(c), np.nan); s[h:] = np.sign(c[h:] / c[:-h] - 1)
            d = d + np.nan_to_num(s)
        d = np.clip(np.sign(d), 0, None)
        return vol_target(d, df, 0.20, 30, 2.0)
    print(fmt(study_weights("TSMOM-LF", tsmom, tfs=("1d",))))

    def donch(df):
        hi, lo = donchian(df, 20)
        c = df["close"].values
        pos = np.where(c > hi, 1.0, np.nan)
        pos = np.where(c < lo, 0.0, pos)
        return pd.Series(pos).ffill().fillna(0.0).values
    print(fmt(study_weights("DONCHIAN20-LF", donch, tfs=("1d",))))
    print("\nJSON sample:", as_json(bh)[:300])
