research(intraday): de-bias del lead ERM (filone B) — falso positivo + gate selection-on-holdout
L'analisi di robustezza affonda lo "earns_slot=True" di ERM: era prodotto da selezione-sull'hold-out + coda 2026 + multiple-testing non corretto. A) deflated-Sharpe FAIL: 0.00 (tutti 122 trial) / 0.16 (no-TOD) / 0.24 (solo-ERM) << 0.95 B) selezione in-sample-only -> ALTRA cella (long-flat, corr->TP01 0.53) = NEUTRAL, no slot C) ensemble del plateau (no cherry-pick) -> ADDS ma robust_oos=False -> no slot D) uplift FULL solo +0.10, negativo 2021/2022; uplift HOLD +0.30 concentrato nel 2026 => ERM SCARTATO come sleeve. Conferma ennesima del soffitto BTC/ETH-direzionale ~1.3. Lezione CODIFICATA in altlib (LESSON 4, test in tests/test_harness_realism.py): - deflated_sharpe() Bailey & Lopez de Prado, PASS >= 0.95 - select_cell_insample() scelta cella col solo Sharpe pre-HOLDOUT (no peeking) - study_family_honest() gate combinato: earns_slot[cella in-sample] AND DSR>=0.95 Regola: una strategia direzionale grid-searched si giudica con study_family_honest, non chiamando study_marginal sulla cella a max hold-out. Verificato end-to-end su ERM (earns_slot_honest=False). Chiude il punto cieco gemello di CC01. Diario aggiornato (verdetto downgrade), CLAUDE.md aggiornato. Test 119/119 verdi. Nessun impatto live (branch separato). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -29,12 +29,14 @@ from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import norm
|
||||
|
||||
# --- make `from src...` work no matter where the agent's script lives -------
|
||||
_ROOT = Path(__file__).resolve().parents[3]
|
||||
@@ -670,6 +672,94 @@ def causality_ok(target_fn, tf: str = "1h", assets=CERTIFIED,
|
||||
reason=("length-mismatch on prefix" if bad else None))
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# SELECTION-ON-HOLDOUT GATE — codified 2026-06-29 from filone B (intraday ERM).
|
||||
#
|
||||
# LESSON 3: intraday_regime.py picked its "winner" cell by MAX hold-out Sharpe over a ~60-cell
|
||||
# grid, then ran study_marginal on THAT cell -> earns_slot=True. But the slot was an artifact of
|
||||
# SELECTING THE CELL ON THE HOLD-OUT: picking the cell IN-SAMPLE-ONLY (no peeking) lands on a
|
||||
# DIFFERENT, TP01-correlated cell that scores NEUTRAL, and the standalone Sharpe deflates to
|
||||
# DSR~0.0-0.24 over the trials searched. study_marginal alone can't catch this — it judges ONE
|
||||
# stream and never sees how the cell was chosen. The fix is two-fold and lives here:
|
||||
# (1) choose the cell IN-SAMPLE-ONLY (or walk-forward) BEFORE scoring the marginal, and
|
||||
# (2) DEFLATE the standalone Sharpe for the number of cells/families searched.
|
||||
# Twin of the CC01 ("implausible Sharpe -> hidden risk") and alt-sweep ("hold-out-fitting") blind
|
||||
# spots, in its "selection-on-holdout" form.
|
||||
# ===========================================================================
|
||||
def deflated_sharpe(sr_ann, all_sr_ann, daily_ret, dpy: float = 365.25):
|
||||
"""Deflated Sharpe Ratio (Bailey & Lopez de Prado): P(true Sharpe > the MAX Sharpe expected
|
||||
under the null of N independent trials). Penalizes multiple-testing — a standalone Sharpe ~1
|
||||
over a 100+ cell grid is routinely NOT significant once deflated. sr_ann = annualized Sharpe
|
||||
of the CHOSEN config; all_sr_ann = the Sharpe of EVERY cell searched; daily_ret = the chosen
|
||||
config's daily returns (for skew/kurt/T). Returns (DSR, expected_null_max_sharpe_ann);
|
||||
PASS if DSR >= 0.95."""
|
||||
r = np.asarray(pd.Series(daily_ret).dropna().values, float)
|
||||
T = len(r)
|
||||
if T < 30 or np.std(r) == 0:
|
||||
return float("nan"), float("nan")
|
||||
sr = sr_ann / math.sqrt(dpy)
|
||||
trials = np.asarray([s / math.sqrt(dpy) for s in all_sr_ann if np.isfinite(s)], float)
|
||||
N = max(len(trials), 2)
|
||||
var_tr = float(np.var(trials, ddof=1)) if N > 1 else 0.0
|
||||
emc = 0.5772156649
|
||||
z1 = norm.ppf(1 - 1.0 / N)
|
||||
z2 = norm.ppf(1 - 1.0 / (N * math.e))
|
||||
sr0 = math.sqrt(var_tr) * ((1 - emc) * z1 + emc * z2)
|
||||
sk = float(pd.Series(r).skew())
|
||||
ku = float(pd.Series(r).kurt()) + 3.0 # pandas kurt = excess
|
||||
den = math.sqrt(max(1e-9, 1 - sk * sr + (ku - 1) / 4.0 * sr ** 2))
|
||||
dsr = float(norm.cdf((sr - sr0) * math.sqrt(T - 1) / den))
|
||||
return dsr, sr0 * math.sqrt(dpy)
|
||||
|
||||
|
||||
def select_cell_insample(factory, grid, tfs, fee_side: float = FEE_SIDE) -> dict:
|
||||
"""Pick a config WITHOUT looking at the hold-out: rank grid cells by IN-SAMPLE (pre-HOLDOUT)
|
||||
standalone Sharpe of the 50/50 candidate. `factory(tf=..., **params)` -> target_fn; each grid
|
||||
item is a dict of factory kwargs (besides tf). Returns the in-sample-best cell, all rows
|
||||
(sorted), and EVERY cell's FULL Sharpe (for deflated_sharpe). This is the honest replacement
|
||||
for picking the max-hold-out cell."""
|
||||
rows = []
|
||||
for tf in tfs:
|
||||
for params in grid:
|
||||
try:
|
||||
daily = candidate_daily(factory(tf=tf, **params), tf=tf, fee_side=fee_side)
|
||||
except Exception:
|
||||
continue
|
||||
ins = daily[daily.index < HOLDOUT]
|
||||
is_sh = _sh(ins) if len(ins) > 60 else float("nan")
|
||||
rows.append(dict(tf=tf, params=params, insample_sharpe=round(is_sh, 3),
|
||||
full_sharpe=round(_sh(daily), 3)))
|
||||
valid = [r for r in rows if np.isfinite(r["insample_sharpe"])]
|
||||
chosen = max(valid, key=lambda r: r["insample_sharpe"]) if valid else None
|
||||
return dict(chosen=chosen,
|
||||
rows=sorted(valid, key=lambda r: r["insample_sharpe"], reverse=True),
|
||||
all_full_sharpe=[r["full_sharpe"] for r in rows])
|
||||
|
||||
|
||||
def study_family_honest(name: str, factory, grid, tfs, fee_side: float = FEE_SIDE,
|
||||
dsr_min: float = 0.95) -> dict:
|
||||
"""HARDENED family gate. A grid-searched directional candidate earns a slot ONLY if, picking
|
||||
the cell IN-SAMPLE-ONLY (no hold-out peeking), it STILL earns_slot via study_marginal AND its
|
||||
standalone Sharpe survives deflation for the WHOLE grid searched. Use this INSTEAD of
|
||||
cherry-picking the max-hold cell and calling study_marginal on it."""
|
||||
sel = select_cell_insample(factory, grid, tfs, fee_side=fee_side)
|
||||
ch = sel["chosen"]
|
||||
if ch is None:
|
||||
return dict(name=name, chosen=None, earns_slot_honest=False,
|
||||
reason="no valid in-sample cell")
|
||||
fn = factory(tf=ch["tf"], **ch["params"])
|
||||
sm = study_marginal(f"{name} ISpick {ch['params']}", fn, tf=ch["tf"], fee_side=fee_side)
|
||||
daily = candidate_daily(fn, tf=ch["tf"], fee_side=fee_side)
|
||||
dsr, sr0 = deflated_sharpe(_sh(daily), sel["all_full_sharpe"], daily)
|
||||
dsr_pass = bool(np.isfinite(dsr) and dsr >= dsr_min)
|
||||
return dict(name=name, n_cells=len(sel["all_full_sharpe"]), chosen=ch, rows=sel["rows"],
|
||||
marginal=sm, earns_slot_marginal=bool(sm["earns_slot"]),
|
||||
deflated_sharpe=round(dsr, 3) if np.isfinite(dsr) else None,
|
||||
expected_null_max=round(sr0, 3) if np.isfinite(sr0) else None,
|
||||
dsr_pass=dsr_pass,
|
||||
earns_slot_honest=bool(sm["earns_slot"] and dsr_pass))
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# DRIVERS — run a hypothesis across both assets, several TFs, with a fee sweep.
|
||||
# ===========================================================================
|
||||
|
||||
Reference in New Issue
Block a user