"""altlib — SHARED HONEST EVALUATION LIBRARY for the alt-strategy fan-out (2026-06-20). Built for the "studia altre strategie alternative su Deribit" research wave: >=100 agents, each studying ONE distinct strategy hypothesis on the certified BTC/ETH (+ DVOL) universe. Every agent imports THIS module so that: * NO look-ahead is structurally possible: a target/weight decided at close[i] is held during bar i+1 (the evaluator shifts it for you — you never multiply by r[i] with a weight that used close[i] for the *same* bar). * Fees are realistic Deribit (0.10% RT taker = 0.0005/side) and a fee SWEEP is built in. * Metrics are comparable: FULL Sharpe/CAGR/maxDD, HOLD-OUT (2025-01-01+), per-year. * Only certified data exists (BTC/ETH from Deribit mainnet, DVOL from Deribit). load() raises on anything else — a physical guardrail. Two evaluation styles: 1. eval_weights(df, target) -> for CONTINUOUS-position strategies (trend, vol overlays, pairs, risk-parity). `target` is a per-bar position (fraction of equity, sign=dir), decided with data <= close[i]. VECTORIZED (numpy) -> fast even on 68k 1h bars. 2. eval_signals(df, entries) -> for DISCRETE entry/exit strategies (breakout w/ TP-SL, mean-reversion bounce). Wraps the project's trade-based harness. Use on 1h/1d only (the Python loop is O(n*max_bars); 5m has 840k bars -> too slow on 2 CPUs). Quick start (inside an agent script): import sys; sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/alt") import altlib as al rep = al.study_weights("MY-STRAT", lambda df: my_target(df), tfs=("1d","12h")) print(al.fmt(rep)); print(al.as_json(rep)) # human + machine """ from __future__ import annotations import inspect import json import sys from functools import lru_cache from pathlib import Path import numpy as np import pandas as pd # --- make `from src...` work no matter where the agent's script lives ------- _ROOT = Path(__file__).resolve().parents[3] if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) from src.backtest.harness import backtest_signals, load # noqa: E402 from src.strategies.trend_portfolio import resample_tf # noqa: E402 HOLDOUT = pd.Timestamp("2025-01-01", tz="UTC") FEE_SIDE = 0.0005 # 0.05%/side = 0.10% round-trip (Deribit taker) FEE_SWEEP = (0.0, 0.0005, 0.001, 0.0015) # per-side fee grid for robustness CERTIFIED = ("BTC", "ETH") DATA_DIR = _ROOT / "data" / "raw" # =========================================================================== # DATA (cached) — 1h base, resampled to >=4h; DVOL aligned causally. # =========================================================================== @lru_cache(maxsize=32) def get(asset: str, tf: str) -> pd.DataFrame: """Certified OHLCV with a tz-aware 'datetime' col and RangeIndex. tf in {5m,15m,1h} loaded directly; {4h,6h,8h,12h,1d,2d,3d,1w} resampled from 1h. Resample uses the leak-free per-single-TF path (trend_portfolio.resample_tf).""" asset = asset.upper() if asset not in CERTIFIED: raise ValueError(f"Asset non certificato: {asset}. Universo={CERTIFIED}.") tf = tf.lower() if tf in ("5m", "15m", "1h"): df = load(asset, tf) else: rule = {"4h": "4h", "6h": "6h", "8h": "8h", "12h": "12h", "1d": "1D", "2d": "2D", "3d": "3D", "1w": "1W"}.get(tf) if rule is None: raise ValueError(f"TF non gestito: {tf}") df = resample_tf(load(asset, "1h"), rule) df = df.reset_index(drop=True) if "datetime" not in df.columns: df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True) return df @lru_cache(maxsize=8) def _dvol_raw(asset: str) -> pd.DataFrame: p = DATA_DIR / f"dvol_{asset.lower()}.parquet" if not p.exists(): raise FileNotFoundError(f"DVOL non trovato: {p}") d = pd.read_parquet(p).sort_values("timestamp").reset_index(drop=True) return d def dvol(df: pd.DataFrame, asset: str) -> np.ndarray: """Deribit DVOL (implied vol index) aligned CAUSALLY to df's bars. For each bar we take the most recent DVOL value timestamped at/before the bar's open (merge_asof backward) -> known by decision time. NaN before DVOL history (DVOL starts 2021-03). Returns float array len(df), in vol POINTS (e.g. 65.0).""" d = _dvol_raw(asset) left = pd.DataFrame({"timestamp": df["timestamp"].astype("int64").values}) merged = pd.merge_asof(left, d.rename(columns={"close": "dvol"}), on="timestamp", direction="backward") return merged["dvol"].values.astype(float) # =========================================================================== # INDICATORS (all causal: value at i uses data <= i) # =========================================================================== def simple_returns(c: np.ndarray) -> np.ndarray: r = np.zeros(len(c)); r[1:] = c[1:] / c[:-1] - 1.0 return r def log_returns(c: np.ndarray) -> np.ndarray: r = np.zeros(len(c)); r[1:] = np.log(c[1:] / c[:-1]) return r def ema(x: np.ndarray, span: int) -> np.ndarray: return pd.Series(x).ewm(span=span, adjust=False).mean().values def sma(x: np.ndarray, win: int) -> np.ndarray: return pd.Series(x).rolling(win, min_periods=win).mean().values def rolling_std(x: np.ndarray, win: int) -> np.ndarray: return pd.Series(x).rolling(win, min_periods=max(2, win // 2)).std().values def zscore(x: np.ndarray, win: int) -> np.ndarray: s = pd.Series(x) m = s.rolling(win, min_periods=win).mean() sd = s.rolling(win, min_periods=win).std() return ((s - m) / sd.replace(0, np.nan)).values def rsi(c: np.ndarray, win: int = 14) -> np.ndarray: d = np.diff(c, prepend=c[0]) up = pd.Series(np.where(d > 0, d, 0.0)).ewm(alpha=1 / win, adjust=False).mean() dn = pd.Series(np.where(d < 0, -d, 0.0)).ewm(alpha=1 / win, adjust=False).mean() rs = up / dn.replace(0, np.nan) return (100 - 100 / (1 + rs)).values def atr(df: pd.DataFrame, win: int = 14) -> np.ndarray: h, l, c = df["high"].values, df["low"].values, df["close"].values pc = np.roll(c, 1); pc[0] = c[0] tr = np.maximum(h - l, np.maximum(np.abs(h - pc), np.abs(l - pc))) return pd.Series(tr).ewm(alpha=1 / win, adjust=False).mean().values def realized_vol(r: np.ndarray, win: int, bars_per_year: float) -> np.ndarray: """Annualized realized vol from returns up to i inclusive (no leakage).""" return pd.Series(r).rolling(win, min_periods=max(2, win // 2)).std().values * np.sqrt(bars_per_year) def donchian(df: pd.DataFrame, win: int): """Upper/lower channel using bars STRICTLY before i (shifted) -> a close[i] that breaks the prior `win`-bar high is a real, tradeable breakout at close[i].""" hi = pd.Series(df["high"].values).rolling(win, min_periods=win).max().shift(1).values lo = pd.Series(df["low"].values).rolling(win, min_periods=win).min().shift(1).values return hi, lo def bbands(c: np.ndarray, win: int = 20, k: float = 2.0): m = pd.Series(c).rolling(win, min_periods=win).mean() sd = pd.Series(c).rolling(win, min_periods=win).std() return (m + k * sd).values, m.values, (m - k * sd).values def _call_target(fn, df: pd.DataFrame, asset: str): """Call a strategy fn as fn(df, asset) when it accepts 2 args, else fn(df). Lets DVOL/cross-asset strategies receive the asset cleanly (no inference hacks).""" try: n = len(inspect.signature(fn).parameters) except (ValueError, TypeError): n = 1 return fn(df, asset) if n >= 2 else fn(df) def bars_per_year(df: pd.DataFrame) -> float: dt = pd.to_datetime(df["datetime"]).diff().dt.total_seconds().median() return 86400 * 365.25 / dt if dt and dt > 0 else 365.25 def bars_per_day(df: pd.DataFrame) -> int: dt = pd.to_datetime(df["datetime"]).diff().dt.total_seconds().median() return max(1, round(86400 / dt)) def vol_target(target_dir: np.ndarray, df: pd.DataFrame, target_vol: float = 0.20, vol_win_days: int = 30, leverage_cap: float = 2.0) -> np.ndarray: """Scale a direction array in [-1,1] to a vol-targeted position (TP01-style). Causal: uses realized vol up to i. Returns position clipped to +/-leverage_cap.""" c = df["close"].values.astype(float) bpd = bars_per_day(df) bpy = bpd * 365.25 vol = realized_vol(simple_returns(c), max(2, vol_win_days * bpd), bpy) scal = np.where((vol > 0) & np.isfinite(vol), target_vol / vol, 0.0) tgt = np.clip(target_dir * scal, -leverage_cap, leverage_cap) tgt[~np.isfinite(tgt)] = 0.0 return tgt # =========================================================================== # METRICS # =========================================================================== def _metrics_from_net(net: np.ndarray, idx: pd.DatetimeIndex) -> dict: net = np.nan_to_num(net, nan=0.0) eq = np.cumprod(1.0 + np.clip(net, -0.99, None)) rr = net[np.isfinite(net)] bpy = 86400 * 365.25 / (pd.Series(idx).diff().dt.total_seconds().median() or 86400) sharpe = float(np.mean(rr) / np.std(rr) * np.sqrt(bpy)) if np.std(rr) > 0 else 0.0 pk = np.maximum.accumulate(eq) dd = float(np.max((pk - eq) / pk)) if len(eq) else 0.0 span_days = (idx[-1] - idx[0]).total_seconds() / 86400 if len(idx) > 1 else 1.0 years = max(span_days / 365.25, 1e-6) total = eq[-1] / eq[0] if len(eq) else 1.0 cagr = total ** (1 / years) - 1 if total > 0 else -1.0 return dict(sharpe=round(sharpe, 3), cagr=round(cagr, 4), maxdd=round(dd, 4), ret=round(total - 1, 4), n=int(len(rr))) def _yearly(net: np.ndarray, idx: pd.DatetimeIndex) -> dict: s = pd.Series(np.nan_to_num(net), index=idx) out = {} for y, g in s.groupby(s.index.year): eq = np.cumprod(1 + g.values); pk = np.maximum.accumulate(eq) out[int(y)] = dict(ret=round(float(eq[-1] - 1), 4), dd=round(float(np.max((pk - eq) / pk)), 4)) return out def eval_weights(df: pd.DataFrame, target: np.ndarray, fee_side: float = FEE_SIDE) -> dict: """Honest backtest of a CONTINUOUS position series. target[i] is decided with data <= close[i]; it is HELD during bar i+1. The shift is done HERE -> you cannot leak by construction. Fee charged on |Δposition| turnover. Returns {full, holdout, yearly, time_in_market, turnover_per_year, net, idx}.""" c = df["close"].values.astype(float) target = np.asarray(target, float) target = np.nan_to_num(target, nan=0.0) r = simple_returns(c) pos = np.zeros(len(target)); pos[1:] = target[:-1] # held during bar t = decided at t-1 gross = pos * r turn = np.abs(np.diff(pos, prepend=0.0)) net = gross - fee_side * turn net[0] = 0.0 idx = pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True)) full = _metrics_from_net(net, idx) hmask = idx >= HOLDOUT hold = _metrics_from_net(net[hmask], idx[hmask]) if hmask.sum() > 3 else dict(sharpe=0.0, n=0) bpy_d = bars_per_day(df) * 365.25 return dict(full=full, holdout=hold, yearly=_yearly(net, idx), time_in_market=round(float(np.mean(pos != 0)), 3), turnover_per_year=round(float(turn.sum() / (len(turn) / bpy_d)), 1), net=net, idx=idx) def eval_signals(df: pd.DataFrame, entries: list, fee_rt: float = 2 * FEE_SIDE, leverage: float = 1.0, asset: str = "", tf: str = "") -> dict: """Honest backtest of DISCRETE entry/exit signals (TP/SL/max_bars). Wraps the project trade-based harness, adds a standardized hold-out split. Use on 1h/1d.""" m = backtest_signals(df, entries, fee_rt=fee_rt, leverage=leverage, asset=asset, tf=tf) idx = pd.DatetimeIndex(pd.to_datetime(m.eq_index, utc=True)) if m.eq_index is not None \ else pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True)) eq = m.equity hmask = idx >= HOLDOUT hold = dict(sharpe=0.0, ret=0.0, n=0) if hmask.sum() > 3: he = eq[hmask] hr = np.diff(he) / he[:-1] bpy = m.bars_per_year or 365.0 hsharpe = float(np.mean(hr) / np.std(hr) * np.sqrt(bpy)) if len(hr) and np.std(hr) > 0 else 0.0 hold = dict(sharpe=round(hsharpe, 3), ret=round(float(he[-1] / he[0] - 1), 4), n=int(hmask.sum())) full = dict(sharpe=round(m.sharpe, 3), cagr=round(m.cagr, 4), maxdd=round(m.max_dd, 4), ret=round(m.net_return, 4), n=int(m.n_trades)) return dict(full=full, holdout=hold, n_trades=int(m.n_trades), win_rate=round(m.win_rate, 1), time_in_market=round(m.time_in_market, 3), yearly={int(y): round(v, 4) for y, v in m.yearly.items()}) # =========================================================================== # MARGINAL SCORING vs TP01 baseline — the lesson of the 2026-06-20 sweep. # # Absolute Sharpe is NOT enough to earn a portfolio slot: an overlay on TSMOM # (DVOL gate, low-vol filter, kill-switch, ...) inherits TP01's trend Sharpe and # scores ~1.0-1.3 while adding ZERO new return. The only question that matters for # a NEW sleeve is: does it IMPROVE the existing TP01 portfolio out-of-sample? # We answer it three ways: (a) blend uplift (Sharpe of TP01 + w*candidate minus # TP01 alone, full & hold-out), (b) correlation to TP01, (c) residual alpha after # removing the TP01 beta (the part of the candidate orthogonal to trend). # =========================================================================== def _sh(s) -> float: r = np.asarray(s.dropna().values, float) return float(np.mean(r) / np.std(r) * np.sqrt(365.25)) if len(r) > 2 and np.std(r) > 0 else 0.0 def _dd_ret(s) -> float: eq = np.cumprod(1.0 + np.asarray(s.dropna().values, float)) pk = np.maximum.accumulate(eq) return float(np.max((pk - eq) / pk)) if len(eq) else 0.0 def _to_daily(s: pd.Series) -> pd.Series: s = s.dropna().sort_index() if not isinstance(s.index, pd.DatetimeIndex): s.index = pd.to_datetime(s.index, utc=True) if s.index.tz is None: s.index = s.index.tz_localize("UTC") return ((1.0 + s).resample("1D").prod() - 1.0).dropna() @lru_cache(maxsize=2) def tp01_baseline_daily() -> pd.Series: """The reference a candidate must BEAT: TP01 CANONICAL, 50/50 BTC+ETH, net daily returns on common dates (full Sharpe ~1.30, hold-out ~0.31). Cached.""" from src.strategies.trend_portfolio import CANONICAL, TrendPortfolio tp = TrendPortfolio(**CANONICAL) series = {} for a in CERTIFIED: df = get(a, "1d") net, _ = tp.net_returns(df) series[a] = pd.Series(net, index=pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True))) J = pd.concat(series, axis=1, join="inner").fillna(0.0) return _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]]) def candidate_daily(target_fn, tf: str = "1d", fee_side: float = FEE_SIDE) -> pd.Series: """Build a candidate's 50/50 BTC+ETH net DAILY return series (same convention as tp01_baseline_daily) from a continuous-position target_fn. Intraday TFs are compounded to daily so they align with the TP01 baseline grid.""" series = {} for a in CERTIFIED: df = get(a, tf) ev = eval_weights(df, _call_target(target_fn, df, a), fee_side=fee_side) series[a] = pd.Series(ev["net"], index=ev["idx"]) J = pd.concat(series, axis=1, join="inner").fillna(0.0) return _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]]) def _uplift_series(B: pd.Series, C: pd.Series, w: float = 0.25) -> float: """Sharpe of the (1-w)*TP01 + w*candidate blend minus Sharpe of TP01 alone.""" return _sh((1 - w) * B + w * C) - _sh(B) def _null_uplift_pctl(B: pd.Series, C: pd.Series, w: float = 0.25, n: int = 300, seed: int = 20260621): """Where does the candidate's blend-uplift sit vs the NULL of a zero-correlation noise asset with the SAME mean & vol? Lesson of 2026-06-21: a low-corr asset with a little positive drift 'adds' ~+0.03 Sharpe by pure diversification MATH — that is not a signal. We draw `n` iid-normal assets (same mean/std as C, independent of B => corr 0 by construction), measure each one's uplift, and return (real_uplift, percentile of real vs the null). pctl >= ~0.8 => the uplift is meaningfully above diversification math; pctl ~0.5 => it IS diversification math. Seeded -> deterministic.""" Bx, Cx = B.align(C, join="inner") bs, cs = Bx.values.astype(float), Cx.values.astype(float) if len(cs) < 30: return None, None base = _sh(Bx) real = _sh((1 - w) * Bx + w * Cx) - base mu, sd = float(np.nanmean(cs)), float(np.nanstd(cs)) if sd == 0: return round(real, 3), None rng = np.random.default_rng(seed) draws = rng.normal(mu, sd, size=(n, len(cs))) blends = (1 - w) * bs[None, :] + w * draws m, s = blends.mean(axis=1), blends.std(axis=1) null = np.where(s > 0, m / s * np.sqrt(365.25), 0.0) - base return round(float(real), 3), round(float(np.mean(null <= real)), 3) def marginal_vs_tp01(cand_daily: pd.Series, weights=(0.25, 0.5)) -> dict: """Does this candidate IMPROVE the TP01 portfolio? Returns correlation, blend uplift (full & hold-out, per weight), TP01-beta + residual alpha, and a verdict: ADDS -> lifts the blend, PERSISTENTLY (multi-cut), beats the zero-corr noise null, in BOTH TP01-up and TP01-down regimes HEDGE -> low corr but only pays when TP01 is WEAK (a drawdown dampener, not a standing premium): real, but price it as a hedge, not as alpha NOISE -> uplift indistinguishable from a random zero-corr asset (diversification math, not a signal) REDUNDANT -> ~identical to TP01 (corr high, ~zero uplift): a re-skin, no slot DILUTES -> drags the blend down NEUTRAL -> changes little either way (a weak, optional satellite at best) Score a NEW sleeve on THIS, not on absolute Sharpe. Hardened 2026-06-21 (ortho wave): the fixed-HOLDOUT uplift + drop-month jackknife was fooled (17/18 relative-value books 'ADDS' on a single 2025 ETH-bleed window). Three gates added: (1) MULTI-CUT persistence (positive uplift at several hold-out starts, not only 2025); (2) NOISE-NULL (uplift must beat a zero-corr random asset); (3) HEDGE vs alpha (a low-corr sleeve that only helps when TP01 is down is a hedge).""" B = tp01_baseline_daily() J = pd.concat({"B": B, "C": cand_daily}, axis=1, join="inner").dropna() if len(J) < 30: return dict(marginal_verdict="N/A", reason="insufficient overlap with TP01 baseline") if J["C"].std() == 0: return dict(marginal_verdict="NEUTRAL", reason="flat/zero candidate (no exposure)", corr_full=None, blends={"w25": dict(uplift_full=0.0, uplift_hold=0.0)}) JH = J[J.index >= HOLDOUT] has_h = len(JH) > 5 out = { "n_days": int(len(J)), "n_hold_days": int(len(JH)), "corr_full": round(float(J["B"].corr(J["C"])), 3), "corr_hold": round(float(JH["B"].corr(JH["C"])), 3) if has_h else None, "tp01_full_sharpe": round(_sh(J["B"]), 3), "tp01_hold_sharpe": round(_sh(JH["B"]), 3) if has_h else None, "cand_full_sharpe": round(_sh(J["C"]), 3), "cand_hold_sharpe": round(_sh(JH["C"]), 3) if has_h else None, } blends = {} for w in weights: bf, bh = (1 - w) * J["B"] + w * J["C"], (1 - w) * JH["B"] + w * JH["C"] blends[f"w{int(w * 100)}"] = dict( full=round(_sh(bf), 3), hold=round(_sh(bh), 3) if has_h else None, uplift_full=round(_sh(bf) - _sh(J["B"]), 3), uplift_hold=round(_sh(bh) - _sh(JH["B"]), 3) if has_h else None, dd=round(_dd_ret(bf), 4)) out["blends"] = blends b, c = J["B"].values, J["C"].values beta = float(np.cov(c, b)[0, 1] / np.var(b)) if np.var(b) > 0 else 0.0 resid = c - beta * b out["beta_to_tp01"] = round(beta, 3) out["resid_sharpe_full"] = round(float(np.mean(resid) / np.std(resid) * np.sqrt(365.25)) if np.std(resid) > 0 else 0.0, 3) out["alpha_ann"] = round(float(np.mean(resid) * 365.25), 4) # OOS robustness — the marginal point-estimate can be fooled by ONE lucky month # (e.g. KAMA: +0.06 uplift that flips to -0.03 when its best month is dropped). Require # the blend uplift to be positive in the earliest CLEAN hold-out year AND to survive a # drop-one-month jackknife. This is lesson #2 of the 2026-06-20 sweep, in code. out["clean_year_uplift"] = out["jackknife_min_uplift"] = None robust_h = False if has_h: def _u(sub): return _uplift_series(sub["B"], sub["C"]) yrs = sorted(set(JH.index.year)) clean = JH[JH.index.year == yrs[0]] cu = _u(clean) if len(clean) > 20 else None months = sorted(set(zip(JH.index.year, JH.index.month))) jk = (min(_u(JH[~((JH.index.year == y) & (JH.index.month == mo))]) for y, mo in months) if len(months) > 1 else _u(JH)) out["clean_year_uplift"] = round(cu, 3) if cu is not None else None out["jackknife_min_uplift"] = round(jk, 3) if jk is not None else None robust_h = bool(cu is not None and cu > 0.02 and jk is not None and jk > 0.0) # --- GATE 1: MULTI-CUT PERSISTENCE ------------------------------------------------- # Uplift at the start of each year (not only the fixed HOLDOUT). A real edge adds at # SEVERAL cuts incl. an early one; a regime artifact only adds at the latest window. mc = {} for y in sorted(set(J.index.year))[1:]: sub = J[J.index >= pd.Timestamp(f"{y}-01-01", tz="UTC")] if len(sub) >= 120: mc[y] = round(_uplift_series(sub["B"], sub["C"]), 3) out["multicut_uplift"] = mc pos = [u for u in mc.values() if u > 0] earliest = mc[min(mc)] if mc else None multicut_persistent = bool(len(mc) >= 2 and len(pos) / len(mc) >= 0.6 and earliest is not None and earliest > 0.0) out["multicut_persistent"] = multicut_persistent # --- GATE 2: NOISE-NULL (uplift must beat a random zero-corr asset) ----------------- JI = J[J.index < HOLDOUT] # in-sample part (not the lucky recent window) real_is, pctl_is = _null_uplift_pctl(JI["B"], JI["C"]) if len(JI) >= 60 else (None, None) real_f, pctl_f = _null_uplift_pctl(J["B"], J["C"]) cand_is_sharpe = round(_sh(JI["C"]), 3) if len(JI) >= 60 else None out["null_pctl_insample"] = pctl_is out["null_pctl_full"] = pctl_f out["cand_insample_sharpe"] = cand_is_sharpe # A candidate must STAND ON ITS OWN before the hold-out: a real in-sample standalone # Sharpe. The ortho basket's in-sample Sharpe was 0.29 -> its only "value" was the # diversification math of a near-zero-Sharpe stream, dressed up by the lucky 2025 window. # (null_pctl_* are reported as the diversification-math context: a low-corr asset adds # ~+0.03 Sharpe by math, so pctl~0.5 just means "no TP01-specific timing" — true of GOOD # and BAD uncorrelated sleeves alike, so it can't be the gate. The in-sample edge is.) has_insample_edge = (cand_is_sharpe is None) or (cand_is_sharpe >= 0.5) out["has_insample_edge"] = bool(has_insample_edge) out["beats_noise_null"] = bool(has_insample_edge) # back-compat alias for the gate # --- GATE 3: HEDGE vs ALPHA (does it only pay when TP01 is weak?) ------------------- yr_sh, yr_up = [], [] for y in sorted(set(J.index.year)): sub = J[J.index.year == y] if len(sub) >= 40: yr_sh.append(_sh(sub["B"])); yr_up.append(_uplift_series(sub["B"], sub["C"])) hedge_corr = (round(float(np.corrcoef(yr_sh, yr_up)[0, 1]), 3) if len(yr_sh) >= 3 and np.std(yr_sh) > 0 and np.std(yr_up) > 0 else None) trail = J["B"].rolling(60, min_periods=20).sum().shift(1) up_seg, dn_seg = J[trail > 0], J[trail <= 0] u_up = _uplift_series(up_seg["B"], up_seg["C"]) if len(up_seg) > 30 else None u_dn = _uplift_series(dn_seg["B"], dn_seg["C"]) if len(dn_seg) > 30 else None out["hedge_yearly_corr"] = hedge_corr out["uplift_tp01_up"] = round(u_up, 3) if u_up is not None else None out["uplift_tp01_down"] = round(u_dn, 3) if u_dn is not None else None is_hedge = bool(hedge_corr is not None and hedge_corr < -0.5 and u_up is not None and u_up <= 0.0 and u_dn is not None and u_dn > 0.05) out["is_hedge"] = is_hedge # robust_oos now REQUIRES multi-cut persistence (kills the single-window winners) out["robust_oos"] = bool(robust_h and multicut_persistent) # --- VERDICT ---------------------------------------------------------------------- up_h = blends["w25"]["uplift_hold"] up_f = blends["w25"]["uplift_full"] ch = out["corr_hold"] if out["corr_hold"] is not None else out["corr_full"] if out["corr_full"] > 0.9 and (up_h is None or abs(up_h) < 0.05): v = "REDUNDANT" elif up_f <= -0.10 and (up_h is None or up_h <= 0.0): v = "DILUTES" elif is_hedge: v = "HEDGE" elif not has_insample_edge: v = "NOISE" elif (up_h is not None and up_h >= 0.05 and up_f > -0.15 and ch < 0.85 and multicut_persistent): v = "ADDS" else: v = "NEUTRAL" out["marginal_verdict"] = v return out def study_marginal(name: str, target_fn, tf: str = "1d", fee_side: float = FEE_SIDE) -> dict: """Score a continuous candidate BOTH ways: absolute (study_weights) AND marginal vs TP01. The combined gate: a candidate earns a sleeve slot only if it is not FAIL on absolute robustness AND marginal_verdict == 'ADDS'.""" absolute = study_weights(name, target_fn, tfs=(tf,)) marg = marginal_vs_tp01(candidate_daily(target_fn, tf=tf, fee_side=fee_side)) abs_grade = absolute["verdict"]["grade"] # ADDS already embeds multi-cut + beats-null + not-hedge; we also require robust_oos # (multi-cut robustness) explicitly. A HEDGE/NOISE/NEUTRAL never earns a live slot. earns_slot = (abs_grade != "FAIL" and marg.get("marginal_verdict") == "ADDS" and marg.get("robust_oos", False) and marg.get("beats_noise_null", False) and not marg.get("is_hedge", False)) return dict(name=name, tf=tf, absolute=absolute, marginal=marg, abs_grade=abs_grade, marginal_verdict=marg.get("marginal_verdict"), earns_slot=earns_slot) def fmt_marginal(rep: dict) -> str: m = rep["marginal"] bl = m.get("blends", {}) lines = [f"=== {rep['name']} | abs={rep['abs_grade']} marginal={rep['marginal_verdict']} " f"EARNS_SLOT={rep['earns_slot']}"] lines.append(f" corr->TP01 full {m.get('corr_full')} hold {m.get('corr_hold')} " f"beta {m.get('beta_to_tp01')} resid Sharpe {m.get('resid_sharpe_full')} alpha/yr {m.get('alpha_ann')}") lines.append(f" OOS robustness: clean-year uplift {m.get('clean_year_uplift')} " f"drop-best-month {m.get('jackknife_min_uplift')} robust_oos={m.get('robust_oos')}") lines.append(f" multi-cut persistence: {m.get('multicut_uplift')} persistent={m.get('multicut_persistent')}") lines.append(f" in-sample edge: standalone Sharpe {m.get('cand_insample_sharpe')} " f"has_insample_edge={m.get('has_insample_edge')} " f"(diversification-math null pctl in-sample {m.get('null_pctl_insample')} full {m.get('null_pctl_full')})") lines.append(f" hedge check: yearly corr(TP01-Sh, uplift) {m.get('hedge_yearly_corr')} " f"uplift TP01-up {m.get('uplift_tp01_up')} / TP01-down {m.get('uplift_tp01_down')} " f"is_hedge={m.get('is_hedge')}") lines.append(f" standalone: TP01 full {m.get('tp01_full_sharpe')}/hold {m.get('tp01_hold_sharpe')} | " f"cand full {m.get('cand_full_sharpe')}/hold {m.get('cand_hold_sharpe')}") for w, d in bl.items(): uh = "n/a" if d["uplift_hold"] is None else f"{d['uplift_hold']:+.3f}" hold = "n/a" if d["hold"] is None else f"{d['hold']}" lines.append(f" blend {w}: full {d['full']} (uplift {d['uplift_full']:+.3f}) " f"hold {hold} (uplift {uh}) DD {d['dd'] * 100:.1f}%") return "\n".join(lines) # =========================================================================== # HARNESS REALISM — two gates codified from the 2026-06-21 intraday wave. # # LESSON 1 (day-boundary): open_drive ("first 8h UTC predicts rest-of-day") scored a # +0.23 uplift but INVERTED to -0.10 when the UTC day start was shifted 4h — a calendar- # LABELING artifact, not an intraday effect. A real hour/session/day edge degrades # gracefully under a boundary shift; an artifact flips sign. # # LESSON 2 (small-cap fills): eval_weights charges fee on EVERY |Δposition|, incl. the # thousands of sub-dollar rebalances a vol-target overlay produces. At ~$600 real capital a # $0.03 trade can't execute — the modeled proportional fee is a continuous-rebalancing # fiction. eval_weights_smallcap skips changes below min_order and reports the Sharpe haircut. # =========================================================================== def _shift_calendar(df: pd.DataFrame, offset_hours: int) -> pd.DataFrame: """Relabel the clock the SIGNAL sees by +offset_hours (datetime & timestamp), leaving prices/returns untouched -> the signal's .dt.hour / day-grouping shifts, the backtest does not. (get() is cached; copy so we never mutate the shared frame.)""" d = df.copy() dt = pd.to_datetime(d["datetime"], utc=True) + pd.Timedelta(hours=offset_hours) d["datetime"] = dt if "timestamp" in d: d["timestamp"] = d["timestamp"].astype("int64") + int(offset_hours * 3600 * 1000) return d def day_boundary_robust(target_fn, tf: str = "1h", offsets=(0, 3, 6, 9, 12, 15, 18, 21), w: float = 0.25) -> dict: """Is a candidate's marginal uplift ROBUST to shifting the UTC day boundary? For each offset we relabel the calendar the signal sees, recompute its 50/50 BTC+ETH daily series and the blend uplift vs TP01. A datetime-independent signal is INVARIANT (spread ~0); a calendar signal that stays positive is ROBUST; one whose uplift flips sign is ARTIFACT-RISK (open_drive). Run this on ANY hour/session/day-of-week signal before believing it.""" B = tp01_baseline_daily() per = {} for off in offsets: series = {} for a in CERTIFIED: df0 = get(a, tf) # ORIGINAL bars/dates tgt = _call_target(target_fn, _shift_calendar(df0, off), a) # signal sees shifted clock ev = eval_weights(df0, tgt) # backtest on the real calendar series[a] = pd.Series(ev["net"], index=ev["idx"]) J = pd.concat(series, axis=1, join="inner").fillna(0.0) cand = _to_daily(0.5 * J[CERTIFIED[0]] + 0.5 * J[CERTIFIED[1]]) JJ = pd.concat({"B": B, "C": cand}, axis=1, join="inner").dropna() per[int(off)] = round(_sh((1 - w) * JJ["B"] + w * JJ["C"]) - _sh(JJ["B"]), 3) if len(JJ) > 30 else None ups = [v for v in per.values() if v is not None] if not ups: return dict(per_offset=per, verdict="N/A", reason="no evaluable offsets") spread = round(max(ups) - min(ups), 3) calendar_sensitive = spread > 0.02 robust = min(ups) > 0 verdict = ("INVARIANT" if not calendar_sensitive else ("ROBUST" if robust else "ARTIFACT-RISK")) return dict(per_offset=per, base=per[offsets[0]], min=min(ups), max=max(ups), spread=spread, calendar_sensitive=calendar_sensitive, robust_to_boundary=robust, verdict=verdict) def eval_weights_smallcap(df: pd.DataFrame, target, capital: float = 600.0, min_order: float = 5.0, fee_side: float = FEE_SIDE) -> dict: """Honest net at SMALL capital. A desired position change whose notional |Δw|*capital is below min_order is NOT executed (held -> tracking error, no trade) — removing the continuous-rebalancing fiction. Returns realistic vs modeled metrics, the Sharpe haircut, and the number of trades that actually execute. (Applies to ANY sleeve at this capital, TP01 included.)""" c = df["close"].values.astype(float) tgt = np.clip(np.nan_to_num(np.asarray(target, float)), -10, 10) held = np.empty(len(tgt)); cur = 0.0; n_tr = 0 for i in range(len(tgt)): if abs(tgt[i] - cur) * capital >= min_order: cur = tgt[i]; n_tr += 1 held[i] = cur r = simple_returns(c) pos = np.zeros(len(held)); pos[1:] = held[:-1] turn = np.abs(np.diff(pos, prepend=0.0)) net = pos * r - fee_side * turn; net[0] = 0.0 idx = pd.DatetimeIndex(pd.to_datetime(df["datetime"], utc=True)) real = _metrics_from_net(net, idx) modeled = eval_weights(df, tgt, fee_side=fee_side)["full"] bpy_d = bars_per_day(df) * 365.25 return dict(realistic=real, modeled=modeled, sharpe_haircut=round(modeled["sharpe"] - real["sharpe"], 3), n_executed_trades=int(n_tr), executed_turnover_per_year=round(float(turn.sum() / (len(turn) / bpy_d)), 1)) # =========================================================================== # DRIVERS — run a hypothesis across both assets, several TFs, with a fee sweep. # =========================================================================== def _verdict(per_cell: list[dict]) -> dict: """A finding PASSES only if it is robust: positive FULL Sharpe AND positive HOLD-OUT on BOTH assets in its best TF, survives the fee sweep, and isn't a 1-cell fluke.""" if not per_cell: return dict(grade="FAIL", reason="no cells") ok = [c for c in per_cell if c.get("full_sharpe", -9) > 0] best = max(per_cell, key=lambda c: c.get("min_asset_holdout_sharpe", -9)) pass_ = (best.get("min_asset_full_sharpe", -9) >= 0.5 and best.get("min_asset_holdout_sharpe", -9) >= 0.2 and best.get("fee_survives", False)) weak = (best.get("min_asset_full_sharpe", -9) >= 0.3 and best.get("min_asset_holdout_sharpe", -9) >= 0.0) grade = "PASS" if pass_ else ("WEAK" if weak else "FAIL") return dict(grade=grade, best_tf=best.get("tf"), best_full_sharpe=best.get("min_asset_full_sharpe"), best_holdout_sharpe=best.get("min_asset_holdout_sharpe"), n_positive_cells=len(ok), n_cells=len(per_cell)) def study_weights(name: str, target_fn, tfs=("1d", "12h"), assets=CERTIFIED, fee_sweep=FEE_SWEEP) -> dict: """Run a CONTINUOUS-position hypothesis on each (asset,tf), report robustness. target_fn(df) -> per-bar position array (decided <= close[i]). Returns a dict ready to print/serialize, with a conservative PASS/WEAK/FAIL verdict.""" cells = [] for tf in tfs: per_asset = {} fee_ok_all = True for a in assets: df = get(a, tf) tgt = _call_target(target_fn, df, a) base = eval_weights(df, tgt, fee_side=FEE_SIDE) sweep = {f"{2*f*100:.2f}%RT": eval_weights(df, tgt, fee_side=f)["full"]["sharpe"] for f in fee_sweep} fee_ok = sweep.get("0.20%RT", -9) > 0 fee_ok_all = fee_ok_all and fee_ok per_asset[a] = dict(full=base["full"], holdout=base["holdout"], tim=base["time_in_market"], turnover=base["turnover_per_year"], fee_sweep=sweep, yearly=base["yearly"]) min_full = min(per_asset[a]["full"]["sharpe"] for a in assets) min_hold = min(per_asset[a]["holdout"].get("sharpe", 0.0) for a in assets) cells.append(dict(tf=tf, per_asset=per_asset, min_asset_full_sharpe=round(min_full, 3), min_asset_holdout_sharpe=round(min_hold, 3), full_sharpe=round(np.mean([per_asset[a]["full"]["sharpe"] for a in assets]), 3), fee_survives=fee_ok_all)) return dict(name=name, kind="weights", cells=cells, verdict=_verdict(cells)) def study_signals(name: str, entries_fn, tfs=("1d",), assets=CERTIFIED, fee_sweep=FEE_SWEEP, leverage: float = 1.0) -> dict: """Run a DISCRETE entry/exit hypothesis on each (asset,tf). entries_fn(df) -> list[dict|None] len(df). Use 1h/1d TFs only (Python loop).""" cells = [] for tf in tfs: per_asset = {} fee_ok_all = True for a in assets: df = get(a, tf) ent = _call_target(entries_fn, df, a) base = eval_signals(df, ent, fee_rt=2 * FEE_SIDE, leverage=leverage, asset=a, tf=tf) sweep = {f"{2*f*100:.2f}%RT": eval_signals(df, ent, fee_rt=2 * f, leverage=leverage)["full"]["sharpe"] for f in fee_sweep} fee_ok = sweep.get("0.20%RT", -9) > 0 fee_ok_all = fee_ok_all and fee_ok per_asset[a] = dict(full=base["full"], holdout=base["holdout"], n_trades=base["n_trades"], win_rate=base["win_rate"], fee_sweep=sweep, yearly=base["yearly"]) min_full = min(per_asset[a]["full"]["sharpe"] for a in assets) min_hold = min(per_asset[a]["holdout"].get("sharpe", 0.0) for a in assets) cells.append(dict(tf=tf, per_asset=per_asset, min_asset_full_sharpe=round(min_full, 3), min_asset_holdout_sharpe=round(min_hold, 3), full_sharpe=round(np.mean([per_asset[a]["full"]["sharpe"] for a in assets]), 3), fee_survives=fee_ok_all)) return dict(name=name, kind="signals", cells=cells, verdict=_verdict(cells)) # =========================================================================== # OUTPUT # =========================================================================== def _clean(o): if isinstance(o, dict): return {k: _clean(v) for k, v in o.items() if k not in ("net", "idx", "equity")} if isinstance(o, (list, tuple)): return [_clean(x) for x in o] if isinstance(o, (np.floating,)): return round(float(o), 4) if isinstance(o, (np.integer,)): return int(o) return o def as_json(rep: dict) -> str: return json.dumps(_clean(rep), default=str) def fmt(rep: dict) -> str: v = rep["verdict"] lines = [f"=== {rep['name']} [{rep['kind']}] -> {v['grade']} " f"(best {v.get('best_tf')}: full {v.get('best_full_sharpe')}, " f"hold {v.get('best_holdout_sharpe')})"] for c in rep["cells"]: lines.append(f" TF {c['tf']:>4s}: minFull={c['min_asset_full_sharpe']:+.2f} " f"minHold={c['min_asset_holdout_sharpe']:+.2f} feeOK={c['fee_survives']}") for a, pa in c["per_asset"].items(): yr = " ".join(f"{y}:{d['ret']*100 if isinstance(d, dict) else d*100:+.0f}%" for y, d in list(pa["yearly"].items())) lines.append(f" {a}: full Sh={pa['full']['sharpe']:+.2f} " f"DD={pa['full']['maxdd']*100:.0f}% ret={pa['full']['ret']*100:+.0f}% " f"hold Sh={pa['holdout'].get('sharpe', 0):+.2f} | {yr}") return "\n".join(lines) if __name__ == "__main__": # smoke test: buy&hold, TSMOM trend, donchian breakout print("--- SMOKE TEST altlib ---") bh = study_weights("BUYHOLD", lambda df: np.ones(len(df)), tfs=("1d",)) print(fmt(bh)) def tsmom(df): c = df["close"].values bpd = bars_per_day(df) d = np.zeros(len(c)) for h in (30 * bpd, 90 * bpd, 180 * bpd): s = np.full(len(c), np.nan); s[h:] = np.sign(c[h:] / c[:-h] - 1) d = d + np.nan_to_num(s) d = np.clip(np.sign(d), 0, None) return vol_target(d, df, 0.20, 30, 2.0) print(fmt(study_weights("TSMOM-LF", tsmom, tfs=("1d",)))) def donch(df): hi, lo = donchian(df, 20) c = df["close"].values pos = np.where(c > hi, 1.0, np.nan) pos = np.where(c < lo, 0.0, pos) return pd.Series(pos).ffill().fillna(0.0).values print(fmt(study_weights("DONCHIAN20-LF", donch, tfs=("1d",)))) print("\nJSON sample:", as_json(bh)[:300])