diff --git a/.gitignore b/.gitignore index 45cf53c..03bc986 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ data/games/ Old/data/ Old/**/__pycache__/ .cache_trackE_*.npy +data/paper_trend/ diff --git a/data/paper_trend/state.json b/data/paper_trend/state.json deleted file mode 100644 index e2b0b9e..0000000 --- a/data/paper_trend/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "capital": 2000.0, - "initial_capital": 2000.0, - "start_ts": 1781884800000, - "last_ts": 1781884800000, - "positions": { - "BTC": 0.0, - "ETH": 0.0 - }, - "n_bars": 0, - "peak": 2000.0, - "max_dd": 0.0 -} \ No newline at end of file diff --git a/docs/diary/2026-06-19-trackF-seasonality.md b/docs/diary/2026-06-19-trackF-seasonality.md new file mode 100644 index 0000000..7415628 --- /dev/null +++ b/docs/diary/2026-06-19-trackF-seasonality.md @@ -0,0 +1,77 @@ +# Track F — Calendar seasonality (hour-of-day / day-of-week) on BTC & ETH + +**Data:** 2026-06-19 · **Script:** `scripts/research/trackF_seasonality.py` +**Dati:** Deribit mainnet certificati, BTC/ETH 1h UTC. Fee baseline 0.10% RT (`fee_side=0.0005`). + +## Domanda +Esiste un edge di calendario *sistematico e tradeable* (ora del giorno, giorno della +settimana, interazione ora×giorno) su BTC ed ETH, netto fee, OOS, per-anno, su entrambi gli asset? + +## Metodologia (anti-overfit, anti-leakage) +- `ret[i]=close[i]/close[i-1]-1` è noto a `close[i]`; una posizione decisa a `close[i]` guadagna + `ret[i+1]`. La statistica che decide il trade usa **solo barre ≤ i** (mai la barra tradata né futuro). +- **Tradeable test onesto = ADAPTIVE EXPANDING sign**: a `close[i]` guardo il bucket di calendario + della barra `i+1` (il clock è noto, zero look-ahead) e prendo il **segno della media passata** di + quel bucket (espandente, warmup-gated). Long-flat o long-short. Fee solo su `|Δposizione|`. + È l'analogo onesto di "tradare il seasonal": i dati scelgono il segno di ogni bucket **dal vivo**. +- Tabelle descrittive per-ora/per-giorno split IS(65%)/OOS(35%) come diagnostica. +- Regola discreta ottimizzata in-sample (entra a ora H, tieni W barre, dir migliore) mostrata solo + per **esporre il gap IS→OOS** (384 celle testate/asset). +- Benchmark **buy-and-hold** come controllo del long-bias. + +## Risultati + +### 1. Descrittive (bp/barra, IS vs OOS) +- **Hour-of-day:** sign-agreement IS/OOS solo **12/24 (BTC)** e **8/24 (ETH)** → caso. Le ore "US + close" 21:00–22:00 UTC sono positive in entrambi gli split su entrambi gli asset (l'unico pattern + con un minimo di coerenza), ma il resto è rumore che cambia segno tra IS e OOS. +- **Day-of-week:** più stabile. **Giovedì negativo** su BTC ed ETH in IS *e* OOS; Lun/Mer positivi. + Sign-agreement 6/7 (BTC), 5/7 (ETH). + +### 2. Adaptive expanding-sign (il test tradeable) +| Strategia | BTC Sharpe | ETH Sharpe | Note | +|---|---|---|---| +| HOUR long-short | **−5.39** | **−4.04** | DD 100%. Annientata dalle fee. | +| HOUR long-flat | −2.92 | −2.09 | DD 100%. Idem. | +| DOW long-short | +0.64 | +0.83 | DD 82–84%, −66% nel 2022 | +| DOW long-flat | +0.81 | +0.96 | DD 75–78%, −64/−66% nel 2022 | +| HOUR×WEEKDAY (168 buckets) | −5.05 | −3.96 | DD 100%. Overfit puro + fee. | + +### 3. Il controllo che smonta il DOW — **buy-and-hold** +- BTC buy-hold: **Sharpe 0.79, CAGR 34.9%, DD 77%** → DOW long-flat: Sh 0.81, CAGR 34.2%, DD 77.5%. +- ETH buy-hold: **Sharpe 0.84, CAGR 42.4%, DD 81%** → DOW long-flat: Sh 0.96, CAGR 52.7%, DD 74%. +- Il DOW long-flat è **long il 78% del tempo** (`mean_pos≈+0.78`). È **buy-and-hold travestito**: + guadagna perché crypto sale, non perché esiste un edge di giorno. Lo "skip del giovedì" aggiunge + pochissimo e non giustifica un deploy. + +### 4. Fee sweep (HOUR long-short adaptive) +A fee **0%**: Sh +0.61 (BTC) / +0.80 (ETH) — solo long-drift. A 0.10% RT: **−5.4 / −4.0**. Turnover +**~8.000 flip/anno** (segno orario instabile, cambia quasi ogni barra) → morte istantanea per fee. +Le strategie hour-of-day sono ad alta frequenza per costruzione: le fee sono di prim'ordine e le +uccidono. + +### 5. Regola discreta ottimizzata in-sample (trappola multiple-testing) +- BTC: best IS H=05 hold=24h dir=+1 → **IS Sh +4.25 → OOS Sh +1.47** (+3.7 bp/trade). +- ETH: best IS H=13 hold=24h dir=+1 → **IS Sh +7.35 → OOS Sh +0.90** (+3.2 bp/trade). +- Collasso IS→OOS classico. Inoltre "hold 24h dir+1" = ancora **long-bias** (entra una volta/giorno + e tiene 24h ≈ sempre long). Il margine OOS (~3 bp/trade su 10 bp RT) è marginale e fragile. + +## Multiple-testing +199 celle di calendario/asset (24 ore + 7 giorni + 168 ora×giorno) + 384 (H,W,dir)/asset. Con così +tante celle, bucket "significativi" spuri sono **garantiti**. Filtri applicati: segno scelto dal vivo +su soli dati passati, deve reggere OOS, per-anno, e su **entrambi** BTC ed ETH. + +## Verdetto — **SPURIO / NON deployable** +- **Nessun edge di calendario netto-fee robusto** su BTC ed ETH. +- **Hour-of-day:** morto (fee + segno instabile). L'unica regolarità (US-close 21–22 UTC positiva) è + troppo debole e non sopravvive al turnover. +- **Day-of-week:** l'unico risultato "positivo" è **long-bias mascherato** (≈ buy-and-hold, + Sharpe ~0.8–0.96 < trend portfolio 1.32, DD 75–84% rovinoso, −65% nel 2022). Non è un edge + seasonal sfruttabile; è esposizione direzionale al drift di crypto. +- **Hour×weekday:** overfit puro (IS −3.6 → OOS −8.0). +- Coerente con la lezione del progetto: dove l'unica "direzione" che funziona è essere long, non c'è + alpha di timing — c'è beta. Il trend portfolio (TP01) cattura quel beta in modo vol-targeted e + con DD ~12%, infinitamente meglio di qualunque regola di calendario qui. + +**Azione:** track F chiuso negativo. Non aggiungere nulla al portafoglio. Il soffitto Sharpe ~1.3 su +BTC/ETH regge. diff --git a/docs/diary/2026-06-19-trackG-prior-levels.md b/docs/diary/2026-06-19-trackG-prior-levels.md new file mode 100644 index 0000000..3a207bb --- /dev/null +++ b/docs/diary/2026-06-19-trackG-prior-levels.md @@ -0,0 +1,85 @@ +# Track G — Prior-period level breakouts / range (BTC & ETH, calendar-anchored) + +**Data:** 2026-06-19 · **Script:** `scripts/research/trackG_prior_levels.py` +**Harness:** `src/backtest/harness.py` (honest, entry decided at `close[i]`, fill `close[i]`). + +## Domanda + +Esistono edge net-positivi OOS, robusti su BTC **e** ETH, definiti rispetto a un **periodo +calendario precedente** (giorno/settimana/opening-range)? E soprattutto: i breakout di livello +**continuano** (trend) o **rientrano** (fade)? + +## No look-ahead (garanzie) + +- Livelli prior-day/week costruiti aggregando a barre giornaliere/settimanali (UTC) e poi + **`shift(1)`** sul frame del periodo *chiuso*: il periodo corrente vede solo il precedente + totalmente chiuso. Mai "oggi"/"questa settimana" nel livello. +- Opening-range usato **solo** sulle barre dopo la chiusura della finestra di apertura. +- Direzione + prezzo decisi a `close[i]`, fill a `close[i]`. Mai entry sul livello esatto intrabar. +- Bug iniziale corretto: mismatch tz-aware vs tz-naive nel mapping dei livelli (dava 0 trade). + +## Risultati (1h, fee 0.10% RT, leva 1x, OOS 65/35) + +### Continuation vs FADE — il verdetto è netto + +| Regola (PD = prior-day) | BTC OOS | ETH OOS | Sharpe OOS | +|---|---|---|---| +| **PD-high CONT (long su rottura max ieri)** | **+25%** | **+16%** | +0.5 / +0.3 | +| PD-high FADE | **−68%** | **−68%** | −1.6 / −1.2 | +| PD-low CONT (short su rottura min ieri) | −33% | −60% | −0.5 / −0.8 | +| PD-low FADE | −36% | −8% | −0.6 / +0.1 | + +- **I breakout CONTINUANO, non rientrano.** Il lato FADE è robustamente **negativo** su entrambi + gli asset (sia high che low), su prior-day, prior-week e opening-range. Conferma diretta della + tesi del reset: la mean-reversion / fade è morta su dati certificati. +- **Asimmetria long-only:** funziona solo la rottura del **massimo** (long), non quella del + **minimo** (short). Cioè non è un edge di breakout *simmetrico/direzione-neutro*: è cattura del + **drift/trend rialzista** del cripto. La PD-low-cont (short sui breakdown) perde perché in questo + campione il cripto sale. + +### Grid robustness (PASS 6) — survivor = OOS>0 su ENTRAMBI + +- **PD-high CONT: 3/3 celle** (buffer 0/0.1%/0.3%) positive OOS su BTC **e** ETH → robusto al buffer. +- PD-high fade, PD-low cont/fade, OR-fade: **0 survivor**. +- **OR-cont:** positiva solo su ETH, negativa su BTC su tutte le finestre (3/6/8/12h) → artefatto + mono-asset, scartato dalla regola "entrambi". + +### Anchor-hour sweep (PASS 5) — non è un'ora fortunata + +PD-high cont positiva su **21/24** ore UTC (BTC) e **20/24** (ETH). Non dipende da un singolo +anchor → coerente con un edge reale (ma vedi sotto: è beta di trend). + +### Fee sweep + per-anno (PD-high cont, full sample) + +``` +BTC RT%: 0.00→+571 0.05→+289 0.10→+126 0.15→ +31 0.20→ −24 (OOS: +84/+52/+25/+3/−15) +ETH RT%: 0.00→+1754 0.05→+1012 0.10→+567 0.15→+299 0.20→+139 (OOS: +67/+39/+16/−3/−19) +BTC per-anno: 2019 +39 2020 +104 2021 +7 2022 −42 2023 +24 2024 +27 2025 −16 2026 +3 +ETH per-anno: 2020 +164 2021 +160 2022 +7 2023 +1 2024 +12 2025 −4 2026 +7 +Sharpe full: BTC +0.48 (maxDD 55%, €/d 2k +0.88) · ETH +0.86 (maxDD 34%, €/d 2k +4.27) +``` + +- **Fee-fragile:** alla baseline 0.10% RT sopravvive (OOS +25/+16%), ma muore già a ~0.15-0.20% RT. + Margine di fee sottile (≈1.5x baseline e l'edge sparisce su OOS). ~1000-1100 trade in 8 anni. +- **Drawdown enormi** (BTC 55%) e anni negativi (2022 −42% BTC, 2025 −16%). + +## Verdetto + +- **Sì, esiste un edge net-positivo OOS su entrambi gli asset:** *PD-high continuation* (long + quando `close` supera il massimo di ieri, exit a fine giornata UTC). Robusto al buffer e + all'anchor-hour. **MA non è deployabile come miglioramento:** + 1. È **long-only drift capture**, non un breakout simmetrico (il lato short fallisce) → è una + versione **più debole e ridondante** del Trend Portfolio TP01 (Sharpe 0.48-0.86 vs 1.32). + 2. **Fee-fragile** (muore a ~1.5x la fee baseline) e con **drawdown** molto peggiori. +- **Il contributo scientifico vero è la conferma della direzione:** sui dati certificati i + breakout di livello-calendario **CONTINUANO**; il fade è morto (negativo robusto su PD/PW/OR, + entrambi gli asset). Nessuna sorpresa mean-reversion nascosta nei livelli giornalieri/settimanali. +- **Niente di nuovo da mettere in produzione.** TP01 resta la strategia vincente; i breakout + prior-period non aggiungono Sharpe (stessa beta di trend, peggio eseguita). + +## Come riprodurre + +```bash +uv run python scripts/research/trackG_prior_levels.py # full (1h + 15m, ~25s) +uv run python scripts/research/trackG_prior_levels.py --quick # 1h only +``` diff --git a/scripts/live/paper_trend.py b/scripts/live/paper_trend.py index 49ea943..a88a13b 100644 --- a/scripts/live/paper_trend.py +++ b/scripts/live/paper_trend.py @@ -33,7 +33,8 @@ PROJECT_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(PROJECT_ROOT)) from src.backtest.harness import load -from src.strategies.trend_portfolio import TrendPortfolio, CANONICAL, resample_4h, simple_returns +from src.strategies.trend_portfolio import ( + TrendPortfolio, CANONICAL, resample_tf, DEPLOY_TF, simple_returns) STATE_DIR = PROJECT_ROOT / "data" / "paper_trend" STATE_FILE = STATE_DIR / "state.json" @@ -43,8 +44,8 @@ WEIGHT = 0.5 INITIAL_CAPITAL = 2000.0 -def build_4h() -> dict[str, pd.DataFrame]: - return {a: resample_4h(load(a, "1h")) for a in ASSETS} +def build_bars() -> dict[str, pd.DataFrame]: + return {a: resample_tf(load(a, "1h"), DEPLOY_TF) for a in ASSETS} def load_state() -> dict | None: @@ -144,10 +145,10 @@ def print_status(st: dict, dfs: dict): ret = cap / st["initial_capital"] - 1 daily = (cap - st["initial_capital"]) / days if days > 0 else 0.0 print("=" * 72) - print(" PAPER TRADER — TP01 Trend Portfolio (PORT LF4h, 50/50 BTC+ETH, 4h)") + print(f" PAPER TRADER — TP01 Trend Portfolio (PORT LF{DEPLOY_TF}, 50/50 BTC+ETH)") print("=" * 72) print(f" start {start:%Y-%m-%d %H:%M} UTC") - print(f" last bar {last:%Y-%m-%d %H:%M} UTC ({days:.1f} giorni, {st['n_bars']} barre 4h)") + print(f" last bar {last:%Y-%m-%d %H:%M} UTC ({days:.1f} giorni, {st['n_bars']} barre {DEPLOY_TF})") print(f" capitale {cap:,.2f} USDT (start {st['initial_capital']:,.0f})") print(f" ritorno {ret*100:+.2f}% | €/giorno {daily:+.2f} | maxDD {st['max_dd']*100:.1f}%") print(f" posizioni now { 'flat' if all(p==0 for p in st['positions'].values()) else '' }") @@ -168,7 +169,7 @@ def print_status(st: dict, dfs: dict): def main(): argv = sys.argv[1:] - dfs = build_4h() + dfs = build_bars() if "--reset" in argv: if STATE_FILE.exists(): STATE_FILE.unlink() diff --git a/scripts/research/trackD_lookahead_audit.py b/scripts/research/trackD_lookahead_audit.py new file mode 100644 index 0000000..e788a2b --- /dev/null +++ b/scripts/research/trackD_lookahead_audit.py @@ -0,0 +1,118 @@ +"""ADVERSARIAL LOOK-AHEAD / EXECUTION-LAG AUDIT of the trend portfolio across timeframes. + +Motivation (2026-06-19): a look-ahead bug (ffill mixed-timeframe on open-labeled bars) can +inflate sub-daily Sharpe (e.g. 4h to ~1.6 vs a real ~1.1). This audit stress-tests OUR pipeline: + + 1. EXECUTION LAG: standard book holds the position decided at close[i] during bar i+1. + We re-run with an EXTRA bar of delay (held during i+2) — i.e. you cannot trade exactly at + the close; there is one bar of slippage/latency. A genuine slow-trend edge barely moves; a + timing artifact collapses. We sweep lag = 1 (standard) and 2 (conservative). + 2. RELABEL TEST: resample with label='left' (open-labeled, our default) vs label='right' + (close-labeled). The realized Sharpe must be (near) identical; a large gap => the labeling + leaks information. + +Conclusion target: identify the timeframe below which costs + lag dominate (don't deploy there). + +Run: uv run python scripts/research/trackD_lookahead_audit.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from src.backtest.harness import load +from src.strategies.trend_portfolio import simple_returns, realized_vol, tsmom_blend + +ASSETS = ["BTC", "ETH"] +FEE_SIDE = 0.0005 +TARGET_VOL = 0.20 +LEVERAGE = 2.0 +LONG_ONLY = True +TFS = {"4h": ("4h", 6), "6h": ("6h", 4), "8h": ("8h", 3), "12h": ("12h", 2), "1d": ("1D", 1)} + + +def resample(df1h: pd.DataFrame, rule: str, label: str) -> pd.DataFrame: + g = df1h.copy() + idx = pd.to_datetime(g["timestamp"], unit="ms", utc=True) + idx.name = "dt" + g.index = idx + out = g.resample(rule, label=label, closed="left").agg( + {"open": "first", "high": "max", "low": "min", "close": "last", "volume": "sum"}) + out = out.dropna(subset=["open"]) + out["datetime"] = out.index + return out.reset_index(drop=True) + + +def target_series(c, bpd): + bpy = bpd * 365.25 + r = simple_returns(c) + vol = realized_vol(r, 30 * bpd, bpy) + direction = np.clip(tsmom_blend(c, (30 * bpd, 90 * bpd, 180 * bpd)), 0, None) if LONG_ONLY \ + else tsmom_blend(c, (30 * bpd, 90 * bpd, 180 * bpd)) + scal = np.where((vol > 0) & np.isfinite(vol), TARGET_VOL / vol, 0.0) + tgt = np.clip(direction * scal, -LEVERAGE, LEVERAGE) + tgt[~np.isfinite(tgt)] = 0.0 + return tgt, r + + +def sleeve_net(df, bpd, lag): + """net[t] uses position decided at close[t-lag] (lag>=1). lag=1 = standard, lag=2 = +1 delay.""" + c = df["close"].values.astype(float) + tgt, r = target_series(c, bpd) + pos = np.zeros(len(tgt)) + pos[lag:] = tgt[:-lag] + gross = pos * r + turn = np.abs(np.diff(pos, prepend=0.0)) + net = gross - FEE_SIDE * turn + net[:lag] = 0.0 + return np.clip(net, -0.99, None), pd.to_datetime(df["datetime"]) + + +def portfolio_metrics(dfs, bpd, lag): + series = {} + for a in ASSETS: + net, ts = sleeve_net(dfs[a], bpd, lag) + series[a] = pd.Series(net, index=pd.to_datetime(ts.values)) + J = pd.concat(series, axis=1, join="inner").dropna() + combo = 0.5 * J["BTC"].values + 0.5 * J["ETH"].values + bpy = bpd * 365.25 + sh = float(np.mean(combo) / np.std(combo) * np.sqrt(bpy)) if np.std(combo) > 0 else 0.0 + eq = np.cumprod(1.0 + np.clip(combo, -0.99, None)) + dd = float(np.max((np.maximum.accumulate(eq) - eq) / np.maximum.accumulate(eq))) + yrs = (J.index[-1] - J.index[0]).days / 365.25 + cagr = eq[-1] ** (1 / yrs) - 1 + return sh, dd, cagr + + +def main(): + raw = {a: load(a, "1h") for a in ASSETS} + print("=" * 96) + print("# LOOK-AHEAD / EXECUTION-LAG AUDIT — trend portfolio (long-flat, tvol20, lev2), per timeframe") + print("# lag1 = standard (decision held next bar). lag2 = +1 bar execution delay (conservative).") + print("# left/right = resample label (open vs close). Big gap => labeling leak.") + print("=" * 96) + print(f" {'TF':<5s}{'Sh lag1(L)':>12s}{'Sh lag2(L)':>12s}{'Sh lag1(R)':>12s}" + f"{'CAGR l1':>10s}{'CAGR l2':>10s}{'DD l1':>8s}{'lag-decay':>11s}") + for tf, (rule, bpd) in TFS.items(): + dfsL = {a: resample(raw[a], rule, "left") for a in ASSETS} + dfsR = {a: resample(raw[a], rule, "right") for a in ASSETS} + sh1L, dd1, cagr1 = portfolio_metrics(dfsL, bpd, 1) + sh2L, _, cagr2 = portfolio_metrics(dfsL, bpd, 2) + sh1R, _, _ = portfolio_metrics(dfsR, bpd, 1) + decay = (sh1L - sh2L) / sh1L * 100 if sh1L else 0.0 + flag = " <-- robust" if sh2L >= 0.9 * sh1L and abs(sh1L - sh1R) < 0.1 else "" + print(f" {tf:<5s}{sh1L:>12.2f}{sh2L:>12.2f}{sh1R:>12.2f}" + f"{cagr1*100:>+9.1f}%{cagr2*100:>+9.1f}%{dd1*100:>7.1f}%{decay:>+10.0f}%{flag}") + print("\n Interpretation:") + print(" - If Sh lag2 << Sh lag1 (big lag-decay), the edge needs to trade AT the close -> sub-TF") + print(" timing artifact / cost-fragile. Robust slow-trend should barely move with +1 bar.") + print(" - If Sh lag1(left) != Sh lag1(right), the bar LABELING leaks -> look-ahead. Should match.") + + +if __name__ == "__main__": + main() diff --git a/scripts/research/trackF_seasonality.py b/scripts/research/trackF_seasonality.py new file mode 100644 index 0000000..26dba86 --- /dev/null +++ b/scripts/research/trackF_seasonality.py @@ -0,0 +1,365 @@ +"""TRACK F — CALENDAR SEASONALITY on BTC & ETH (hour-of-day, day-of-week, interactions). + +Honest test of whether there is a SYSTEMATIC, TRADEABLE calendar edge on the certified +Deribit-mainnet BTC/ETH feeds. Seasonality is the easiest place on earth to overfit +(24 hours x 7 weekdays = 168 buckets => you WILL find "significant" cells by chance), so +every claim here is held to the project's anti-look-ahead, OOS, per-year, both-assets bar. + +METHODOLOGY (no shortcuts): + - ret[i] = close[i]/close[i-1]-1 is known at close[i]. A position decided at close[i] + earns ret[i+1]. We NEVER include the bar being traded (or any future bar) in the + statistic that decides the trade. + - DESCRIPTIVE tables (per-hour / per-weekday mean returns) are split IS(65%)/OOS(35%). + They are diagnostics, not trades. + - TRADEABLE rule = ADAPTIVE EXPANDING sign: at close[i] we look up the calendar bucket + of bar i+1 (the clock is known with zero look-ahead) and take the SIGN of that bucket's + mean return computed ONLY on bars <= i (expanding, warmup-gated). Long-flat or + long-short. Fees charged only on |Δposition| (turnover-aware). This lets the data pick + each bucket's sign LIVE — the honest analogue of "trade the seasonal". + - Also an in-sample-optimised discrete rule (enter at hour H, hold W bars, best dir) is + shown ONLY to demonstrate the overfit gap IS->OOS. + - NET fees fee_side baseline 0.0005 (=0.10% RT); swept 0.0005/0.00075/0.001. + - A survivor must be net-positive OOS AND across years AND on BOTH BTC & ETH. + +Run: uv run python scripts/research/trackF_seasonality.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from src.backtest.harness import load # noqa: E402 + +ASSETS = ["BTC", "ETH"] +TF = "1h" +FEE_SIDE = 0.0005 # 0.05%/side = 0.10% round-trip +BARS_PER_DAY = 24 +BPY = BARS_PER_DAY * 365.25 + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- +def prep(asset: str, tf: str = TF): + df = load(asset, tf) + c = df["close"].values.astype(float) + ret = np.empty(len(c)) + ret[0] = 0.0 + ret[1:] = c[1:] / c[:-1] - 1.0 + dt = pd.to_datetime(df["datetime"]) + return dict( + df=df, ret=ret, + hour=dt.dt.hour.values.astype(int), + dow=dt.dt.dayofweek.values.astype(int), # 0=Mon..6=Sun + ts=dt, + ) + + +def metrics_from_pnl(pnl: np.ndarray, ts: pd.Series): + """pnl[i] = realized per-bar net return of the strategy (already fee-adjusted).""" + eq = np.cumprod(1.0 + np.clip(pnl, -0.99, None)) + r = pnl[np.isfinite(pnl)] + sharpe = float(np.mean(r) / np.std(r) * np.sqrt(BPY)) if np.std(r) > 0 else 0.0 + peak = np.maximum.accumulate(eq) + maxdd = float(np.max((peak - eq) / peak)) if len(eq) else 0.0 + span_days = (ts.iloc[-1] - ts.iloc[0]).total_seconds() / 86400 + years = span_days / 365.25 if span_days > 0 else 1.0 + total = eq[-1] / eq[0] if len(eq) else 1.0 + cagr = total ** (1 / years) - 1 if years > 0 and total > 0 else -1.0 + daily_2k = (2000 * total - 2000) / span_days if span_days > 0 else 0.0 + return dict(sharpe=sharpe, maxdd=maxdd, cagr=cagr, total=total - 1.0, + daily_2k=daily_2k, eq=eq) + + +def per_year_pnl(pnl: np.ndarray, ts: pd.Series): + s = pd.Series(pnl, index=ts.values) + out = {} + for y, g in s.groupby(s.index.year): + eq = np.cumprod(1.0 + np.clip(g.values, -0.99, None)) + out[int(y)] = float(eq[-1] - 1.0) + return out + + +# --------------------------------------------------------------------------- +# 1. DESCRIPTIVE seasonality tables (diagnostics, IS vs OOS) +# --------------------------------------------------------------------------- +def descriptive(data, frac=0.65): + n = len(data["ret"]) + cut = int(n * frac) + ret, hour, dow = data["ret"], data["hour"], data["dow"] + rows_h, rows_d = {}, {} + for h in range(24): + m_is = ret[:cut][hour[:cut] == h] + m_oos = ret[cut:][hour[cut:] == h] + rows_h[h] = (m_is.mean() * 1e4, m_oos.mean() * 1e4, + np.sign(m_is.mean()) == np.sign(m_oos.mean())) + for d in range(7): + m_is = ret[:cut][dow[:cut] == d] + m_oos = ret[cut:][dow[cut:] == d] + rows_d[d] = (m_is.mean() * 1e4, m_oos.mean() * 1e4, + np.sign(m_is.mean()) == np.sign(m_oos.mean())) + return rows_h, rows_d + + +# --------------------------------------------------------------------------- +# 2. ADAPTIVE EXPANDING-sign seasonal strategy (the honest tradeable test) +# --------------------------------------------------------------------------- +def adaptive_seasonal(data, bucket="hour", mode="longshort", + warmup=200, fee_side=FEE_SIDE): + """Position at close[i] = sign of the EXPANDING past mean return of bar (i+1)'s + calendar bucket, using only bars <= i. earns ret[i+1]. Fee on |Δposition|.""" + ret = data["ret"] + key = data[bucket] + n = len(ret) + nbuck = int(key.max()) + 1 + sums = np.zeros(nbuck) + counts = np.zeros(nbuck) + pos = np.zeros(n) + for i in range(1, n - 1): + b = key[i] + sums[b] += ret[i] + counts[b] += 1 + nb = key[i + 1] + if counts[nb] >= warmup: + m = sums[nb] / counts[nb] + if m > 0: + pos[i] = 1.0 + else: + pos[i] = -1.0 if mode == "longshort" else 0.0 + # pnl[i] earned over bar i+1 + pnl = np.zeros(n) + prev = 0.0 + for i in range(1, n - 1): + turn = abs(pos[i] - prev) + pnl[i] = pos[i] * ret[i + 1] - fee_side * turn + prev = pos[i] + return pnl, pos + + +def adaptive_hourxdow(data, mode="longshort", warmup=120, fee_side=FEE_SIDE): + ret, hour, dow = data["ret"], data["hour"], data["dow"] + key = hour * 7 + dow # 168 buckets + n = len(ret) + sums = np.zeros(168) + counts = np.zeros(168) + pos = np.zeros(n) + for i in range(1, n - 1): + b = key[i] + sums[b] += ret[i] + counts[b] += 1 + nb = key[i + 1] + if counts[nb] >= warmup: + m = sums[nb] / counts[nb] + if m > 0: + pos[i] = 1.0 + else: + pos[i] = -1.0 if mode == "longshort" else 0.0 + pnl = np.zeros(n) + prev = 0.0 + for i in range(1, n - 1): + turn = abs(pos[i] - prev) + pnl[i] = pos[i] * ret[i + 1] - fee_side * turn + prev = pos[i] + return pnl, pos + + +# --------------------------------------------------------------------------- +# 3. In-sample-optimised DISCRETE rule (to expose the overfit gap) +# --------------------------------------------------------------------------- +def discrete_hour_rule_scan(data, frac=0.65, fee_side=FEE_SIDE): + """Scan IS for best (entry_hour, hold_window, direction) by IS Sharpe; report OOS. + + A trade: enter at close of bar whose hour==H (decided with data<=close[i]), hold W + bars, exit at close. One trade per day. Fee charged round-trip on each trade. + """ + ret, hour, ts = data["ret"], data["hour"], data["ts"] + n = len(ret) + cut = int(n * frac) + + def rule_pnl(H, W, direction, lo, hi): + pnl = np.zeros(n) + i = lo + last_exit = lo - 1 + while i < hi: + if hour[i] == H and i > last_exit: + # cumulative return over the next W bars: prod(1+ret[i+1..i+W]) - 1 + end = min(i + W, n - 1) + gross = np.prod(1.0 + ret[i + 1:end + 1]) - 1.0 + pnl[i] = direction * gross - 2 * fee_side + last_exit = end + i = end + else: + i += 1 + return pnl + + best = None + n_tested = 0 + for H in range(24): + for W in (1, 2, 3, 4, 6, 8, 12, 24): + for direction in (+1, -1): + n_tested += 1 + pnl_is = rule_pnl(H, W, direction, 1, cut) + r = pnl_is[pnl_is != 0.0] + if len(r) < 50: + continue + sh = np.mean(r) / np.std(r) * np.sqrt(BPY) if np.std(r) > 0 else 0.0 + if best is None or sh > best[0]: + best = (sh, H, W, direction) + sh, H, W, direction = best + pnl_oos = rule_pnl(H, W, direction, cut, n) + r_oos = pnl_oos[pnl_oos != 0.0] + sh_oos = (np.mean(r_oos) / np.std(r_oos) * np.sqrt(BPY)) if (len(r_oos) and np.std(r_oos) > 0) else 0.0 + return dict(n_tested=n_tested, H=H, W=W, dir=direction, sh_is=sh, + sh_oos=sh_oos, n_is=int((rule_pnl(H, W, direction, 1, cut) != 0).sum()), + n_oos=len(r_oos), oos_mean_bp=r_oos.mean() * 1e4 if len(r_oos) else 0.0) + + +# --------------------------------------------------------------------------- +# reporting +# --------------------------------------------------------------------------- +def split_metrics(pnl, ts, frac=0.65): + n = len(pnl) + cut = int(n * frac) + m_is = metrics_from_pnl(pnl[:cut], ts.iloc[:cut]) + m_oos = metrics_from_pnl(pnl[cut:], ts.iloc[cut:]) + m_all = metrics_from_pnl(pnl, ts) + return m_is, m_oos, m_all + + +def turnover_per_year(pos, ts): + s = pd.Series(np.abs(np.diff(pos, prepend=0.0)), index=ts.values) + return s.groupby(s.index.year).sum().to_dict() + + +def main(): + print("=" * 100) + print("# TRACK F — CALENDAR SEASONALITY (hour-of-day / day-of-week / hour×weekday)") + print("# certified Deribit-mainnet BTC & ETH, 1h UTC. fee_side=0.0005 (0.10% RT).") + print("# No look-ahead: bucket stats use only bars <= i; position earns ret[i+1].") + print("=" * 100) + + data = {a: prep(a) for a in ASSETS} + + # --- DESCRIPTIVE --------------------------------------------------------- + print("\n" + "#" * 100) + print("# 1. DESCRIPTIVE per-bucket mean returns (basis points/bar). IS=first 65%, OOS=last 35%.") + print("# 'sign?' = IS and OOS agree on sign. Diagnostics only (NOT trades, no fees).") + print("#" * 100) + for a in ASSETS: + rows_h, rows_d = descriptive(data[a]) + print(f"\n ── {a} HOUR-OF-DAY (UTC) mean bp/hr ─────────────────────────────") + print(" hr : IS_bp OOS_bp sign?") + agree_h = 0 + for h in range(24): + iv, ov, ag = rows_h[h] + agree_h += int(ag) + flag = " <-- US open" if h in (13, 14) else (" <-- US close" if h in (20, 21) else "") + print(f" {h:>2d} : {iv:>+6.2f} {ov:>+6.2f} {'Y' if ag else '.'}{flag}") + print(f" hour sign-agreement IS/OOS: {agree_h}/24") + print(f"\n ── {a} DAY-OF-WEEK mean bp/bar (0=Mon..6=Sun) ──────────────────") + names = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] + agree_d = 0 + for d in range(7): + iv, ov, ag = rows_d[d] + agree_d += int(ag) + print(f" {names[d]} : {iv:>+6.3f} {ov:>+6.3f} {'Y' if ag else '.'}") + print(f" weekday sign-agreement IS/OOS: {agree_d}/7") + + # --- ADAPTIVE EXPANDING-SIGN (the honest tradeable test) ---------------- + print("\n" + "#" * 100) + print("# 2. ADAPTIVE EXPANDING-SIGN seasonal strategies (HONEST tradeable test).") + print("# sign of bucket's PAST-ONLY mean decides position; fee on turnover.") + print("#" * 100) + configs = [ + ("HOUR long-short", "hour", "longshort", 200), + ("HOUR long-flat ", "hour", "longflat", 200), + ("DOW long-short", "dow", "longshort", 60), + ("DOW long-flat ", "dow", "longflat", 60), + ] + for label, bucket, mode, warmup in configs: + print(f"\n ── {label} ────────────────────────────────────────────────────") + for a in ASSETS: + pnl, pos = adaptive_seasonal(data[a], bucket=bucket, mode=mode, warmup=warmup) + ts = data[a]["ts"] + m_is, m_oos, m_all = split_metrics(pnl, ts) + py = per_year_pnl(pnl, ts) + yrs = "".join(f"{py.get(y, float('nan'))*100:>+6.0f}" for y in range(2019, 2027)) + print(f" {a}: ALL Sh={m_all['sharpe']:>+5.2f} CAGR={m_all['cagr']*100:>+6.1f}% " + f"DD={m_all['maxdd']*100:>4.1f}% €/d={m_all['daily_2k']:>+5.2f} | " + f"IS Sh={m_is['sharpe']:>+5.2f} OOS Sh={m_oos['sharpe']:>+5.2f}") + print(f" per-year %: {yrs} (2019..2026)") + + # buy-and-hold benchmark — the key control: does any 'seasonal' beat just being long? + print(f"\n ── BUY-AND-HOLD benchmark (the control for long-bias) ──") + for a in ASSETS: + ret = data[a]["ret"].copy() + ret[0] = 0.0 + m = metrics_from_pnl(ret, data[a]["ts"]) + print(f" {a}: Sh={m['sharpe']:>+5.2f} CAGR={m['cagr']*100:>+6.1f}% DD={m['maxdd']*100:>4.1f}% " + f" <- compare to DOW long-flat above (it's nearly identical = no edge, just long)") + + # hour x weekday interaction (168 buckets — extreme overfit risk) + print(f"\n ── HOUR×WEEKDAY long-short (168 buckets, warmup 120) — overfit canary ──") + for a in ASSETS: + pnl, pos = adaptive_hourxdow(data[a], mode="longshort", warmup=120) + ts = data[a]["ts"] + m_is, m_oos, m_all = split_metrics(pnl, ts) + print(f" {a}: ALL Sh={m_all['sharpe']:>+5.2f} CAGR={m_all['cagr']*100:>+6.1f}% " + f"DD={m_all['maxdd']*100:>4.1f}% | IS Sh={m_is['sharpe']:>+5.2f} OOS Sh={m_oos['sharpe']:>+5.2f}") + + # --- FEE SWEEP on the best adaptive config ------------------------------- + print("\n" + "#" * 100) + print("# 3. FEE SWEEP — HOUR long-short adaptive (turnover-aware). Are survivors fee-robust?") + print("#" * 100) + for fee in (0.0, 0.0005, 0.00075, 0.001): + line = f" fee_side={fee:.5f} (RT {fee*2*100:.2f}%): " + for a in ASSETS: + pnl, _ = adaptive_seasonal(data[a], bucket="hour", mode="longshort", + warmup=200, fee_side=fee) + m = metrics_from_pnl(pnl, data[a]["ts"]) + line += f"{a} Sh={m['sharpe']:>+5.2f} CAGR={m['cagr']*100:>+6.1f}% " + print(line) + + # --- TURNOVER (fees are first-order for hour strategies) ----------------- + print("\n" + "#" * 100) + print("# 4. TURNOVER (HOUR long-short adaptive): position flips/year (each flip costs ~fee).") + print("#" * 100) + for a in ASSETS: + _, pos = adaptive_seasonal(data[a], bucket="hour", mode="longshort", warmup=200) + tpy = turnover_per_year(pos, data[a]["ts"]) + s = " ".join(f"{y}:{int(v)}" for y, v in sorted(tpy.items())) + print(f" {a} turnover units/yr: {s}") + + # --- IN-SAMPLE-OPTIMISED DISCRETE RULE (overfit demonstration) ---------- + print("\n" + "#" * 100) + print("# 5. IN-SAMPLE-OPTIMISED discrete rule (enter hour H, hold W, best dir).") + print("# Picked by IS Sharpe, reported OOS. Demonstrates the multiple-testing trap.") + print("#" * 100) + for a in ASSETS: + r = discrete_hour_rule_scan(data[a]) + print(f" {a}: tested {r['n_tested']} (H,W,dir) cells -> best IS " + f"H={r['H']:02d} hold={r['W']}h dir={r['dir']:+d} " + f"IS Sh={r['sh_is']:>+5.2f} (n={r['n_is']}) -> OOS Sh={r['sh_oos']:>+5.2f} " + f"(n={r['n_oos']}, mean {r['oos_mean_bp']:>+.1f} bp/trade)") + + # --- VERDICT ------------------------------------------------------------- + print("\n" + "#" * 100) + print("# MULTIPLE-TESTING CAVEAT") + print("#" * 100) + print(""" + Buckets examined: 24 hours + 7 weekdays + 168 hour×weekday = 199 calendar cells PER ASSET, + each tested IS and OOS, plus discrete grid = 24×8×2 = 384 (H,W,dir) cells per asset. + With that many cells, spurious 'significant' buckets are GUARANTEED. The honest filters + applied here: (a) adaptive sign chosen live on PAST data only (no cherry-picking), + (b) must hold OOS, (c) must hold per-year, (d) must hold on BOTH BTC AND ETH. + Read the IS->OOS Sharpe collapse and the per-year sign flips above as the real verdict. +""") + + +if __name__ == "__main__": + main() diff --git a/scripts/research/trackG_prior_levels.py b/scripts/research/trackG_prior_levels.py new file mode 100644 index 0000000..fc7c143 --- /dev/null +++ b/scripts/research/trackG_prior_levels.py @@ -0,0 +1,478 @@ +"""TRACK G — PRIOR-PERIOD LEVEL BREAKOUTS / RANGE on CLEAN BTC/ETH (Deribit mainnet). + +HONEST harness only. We test rules defined RELATIVE TO A PRIOR CALENDAR PERIOD: + * prior-DAY high/low breakout (continuation AND fade) + * opening-range breakout (first N UTC hours -> break for rest of day) + * prior-day CLOSE / gap / range-position / prior-day return-sign filter + * prior-WEEK high/low breakout + * time-anchored entries (act at a given UTC hour vs prior-day level), exit EOD/fixed/TP-SL + +The single question: on clean BTC/ETH, with a genuinely EXECUTABLE entry (direction and +price decided with data <= close[i], fill at close[i], NEVER entering at the exact level +intrabar), net of realistic Deribit fees, OOS and grid-robust on BOTH assets — +do prior-period breakouts CONTINUE (trend) or REVERT (fade)? Is there a deployable edge? + +NO LOOK-AHEAD GUARANTEES: + * Prior-period levels are built by aggregating to daily/weekly bars and SHIFTING by one + full period (shift(1) on the closed-period frame). 'Today'/'this-week' is NEVER part of + the level. The prior period is fully closed before any bar of the current period. + * Opening-range levels are used ONLY on bars AFTER the open window has fully closed. + * Direction + price decided at close[i]; fill at close[i] (harness enforces). + +Run: + uv run python scripts/research/trackG_prior_levels.py # full + uv run python scripts/research/trackG_prior_levels.py --quick # 1h only, fewer grids +""" +from __future__ import annotations + +import argparse +import sys +import time +from itertools import product +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from src.backtest.harness import load, backtest_signals, oos_split + + +# =========================================================================== +# Causal helpers +# =========================================================================== +def atr(df: pd.DataFrame, period: int = 14) -> np.ndarray: + h, l, c = df["high"].values, df["low"].values, df["close"].values + pc = np.roll(c, 1) + pc[0] = c[0] + tr = np.maximum(h - l, np.maximum(np.abs(h - pc), np.abs(l - pc))) + return pd.Series(tr).ewm(alpha=1.0 / period, adjust=False).mean().values + + +def prior_period_levels(df: pd.DataFrame, period: str = "D") -> dict: + """Return prior-period high/low/close/open/range arrays aligned to each intraday bar. + + period='D': prior calendar day (UTC). period='W': prior ISO week (anchored Mon 00:00 UTC). + Uses shift(1) on the CLOSED-period frame: the level for the current period only sees the + fully-closed previous period -> no look-ahead. + """ + dt = df["datetime"] + if period == "D": + key = dt.dt.floor("D") + elif period == "W": + key = dt.dt.floor("D") - pd.to_timedelta(dt.dt.weekday, unit="D") + else: + raise ValueError(period) + + key = key.reset_index(drop=True) + agg = pd.DataFrame({ + "key": key, + "high": df["high"].values, "low": df["low"].values, + "close": df["close"].values, "open": df["open"].values, + }) + g = agg.groupby("key").agg(high=("high", "max"), low=("low", "min"), + close=("close", "last"), open=("open", "first")).sort_index() + gp = g.shift(1) # prior, fully-closed period + km = key.map # map current-period key -> prior-period aggregate + ph = km(gp["high"]).values.astype(float) + pl = km(gp["low"]).values.astype(float) + pc = km(gp["close"]).values.astype(float) + po = km(gp["open"]).values.astype(float) + pret = (gp["close"] / gp["open"] - 1.0) # prior-period return (sign filter) + prv = key.map(pret).values.astype(float) + return {"ph": ph, "pl": pl, "pc": pc, "po": po, "prange": ph - pl, "pret": prv} + + +def opening_range(df: pd.DataFrame, n_open_hours: int) -> dict: + """Opening-range high/low for the first n_open_hours of each UTC day, plus a per-bar + flag of whether the open window has CLOSED (hour >= n_open_hours).""" + dt = df["datetime"] + date = dt.dt.floor("D") + hour = dt.dt.hour + date = date.reset_index(drop=True) + in_open = (hour < n_open_hours).values + o = pd.DataFrame({"date": date, "high": df["high"].values, "low": df["low"].values}) + o_open = o[in_open] + org = o_open.groupby("date").agg(orh=("high", "max"), orl=("low", "min")) + orh = date.map(org["orh"]).values.astype(float) + orl = date.map(org["orl"]).values.astype(float) + closed = (hour >= n_open_hours).values + return {"orh": orh, "orl": orl, "closed": closed} + + +def bars_left_in_day(df: pd.DataFrame) -> np.ndarray: + date = df["datetime"].dt.floor("D") + grp = df.groupby(date) + idx_in_day = grp.cumcount().values + size = grp["close"].transform("size").values + return (size - idx_in_day - 1).astype(int) + + +# =========================================================================== +# Signal generators -> list[dict|None] length len(df). Decisions use data <= close[i]. +# =========================================================================== +def sig_prior_break(df, period="D", level="high", side="cont", anchor_hour=None, + exit_mode="eod", max_bars=24, tp_atr=0.0, sl_atr=0.0, atr_p=14, + buffer=0.0): + """Prior-period level breakout. + level='high': trigger when close[i] > prior_high*(1+buffer) + level='low' : trigger when close[i] < prior_low *(1-buffer) + side='cont' : trade IN the breakout direction (high->long, low->short) + side='fade' : trade AGAINST it (high->short, low->long) + anchor_hour : if set, only evaluate on bars at that UTC hour (time-anchored) + exit_mode : 'eod' (close at end of UTC day), 'bars' (max_bars), TP/SL via *_atr. + """ + lv = prior_period_levels(df, period) + c = df["close"].values + a = atr(df, atr_p) if (tp_atr or sl_atr) else None + bl = bars_left_in_day(df) if exit_mode == "eod" else None + hour = df["datetime"].dt.hour.values + n = len(c) + out = [None] * n + ref = lv["ph"] if level == "high" else lv["pl"] + for i in range(n): + if anchor_hour is not None and hour[i] != anchor_hour: + continue + r = ref[i] + if not np.isfinite(r): + continue + px = c[i] + if level == "high": + if not (px > r * (1.0 + buffer)): + continue + brk_dir = 1 + else: + if not (px < r * (1.0 - buffer)): + continue + brk_dir = -1 + direction = brk_dir if side == "cont" else -brk_dir + if exit_mode == "eod": + mb = max(int(bl[i]), 1) + else: + mb = max_bars + tp = sl = None + if a is not None and np.isfinite(a[i]): + if tp_atr: + tp = px + direction * tp_atr * a[i] + if sl_atr: + sl = px - direction * sl_atr * a[i] + out[i] = {"dir": direction, "tp": tp, "sl": sl, "max_bars": mb} + return out + + +def sig_or_break(df, n_open_hours=6, side="cont", exit_mode="eod", max_bars=12, + tp_atr=0.0, sl_atr=0.0, atr_p=14, buffer=0.0): + """Opening-range breakout: after the first n_open_hours close, trade a break of the + OR high (long if cont) or OR low (short if cont). Only the FIRST break per day fires + (the harness keeps the position busy until exit).""" + orr = opening_range(df, n_open_hours) + c = df["close"].values + a = atr(df, atr_p) if (tp_atr or sl_atr) else None + bl = bars_left_in_day(df) if exit_mode == "eod" else None + n = len(c) + out = [None] * n + orh, orl, closed = orr["orh"], orr["orl"], orr["closed"] + for i in range(n): + if not closed[i] or not np.isfinite(orh[i]): + continue + px = c[i] + if px > orh[i]: + brk = 1 + elif px < orl[i]: + brk = -1 + else: + continue + direction = brk if side == "cont" else -brk + if exit_mode == "eod": + mb = max(int(bl[i]), 1) + else: + mb = max_bars + tp = sl = None + if a is not None and np.isfinite(a[i]): + if tp_atr: + tp = px + direction * tp_atr * a[i] + if sl_atr: + sl = px - direction * sl_atr * a[i] + out[i] = {"dir": direction, "tp": tp, "sl": sl, "max_bars": mb} + return out + + +def sig_gap(df, side="cont", anchor_hour=0, thr=0.0, exit_mode="eod", max_bars=24, + ret_filter=0): + """Gap vs prior-day CLOSE, evaluated at a given UTC hour (default the first bar of the + day). gap = close[i]/prior_close - 1. If gap>thr -> up-gap; gap<-thr -> down-gap. + side='cont' trades in the gap direction; 'fade' against. ret_filter: +1 only when + prior-day return positive, -1 only when negative, 0 no filter.""" + lv = prior_period_levels(df, "D") + c = df["close"].values + bl = bars_left_in_day(df) if exit_mode == "eod" else None + hour = df["datetime"].dt.hour.values + pc, pret = lv["pc"], lv["pret"] + n = len(c) + out = [None] * n + for i in range(n): + if hour[i] != anchor_hour or not np.isfinite(pc[i]): + continue + gap = c[i] / pc[i] - 1.0 + if gap > thr: + g = 1 + elif gap < -thr: + g = -1 + else: + continue + if ret_filter and np.isfinite(pret[i]): + if ret_filter > 0 and not (pret[i] > 0): + continue + if ret_filter < 0 and not (pret[i] < 0): + continue + direction = g if side == "cont" else -g + mb = max(int(bl[i]), 1) if exit_mode == "eod" else max_bars + out[i] = {"dir": direction, "tp": None, "sl": None, "max_bars": mb} + return out + + +# =========================================================================== +# Evaluation +# =========================================================================== +def run_split(df, sigfn, params, fee_rt=0.001, leverage=1.0, frac=0.65): + cut = oos_split(df, frac) + full = backtest_signals(df, sigfn(df, **params), fee_rt=fee_rt, leverage=leverage) + di = df.iloc[:cut].reset_index(drop=True) + do = df.iloc[cut:].reset_index(drop=True) + is_ = backtest_signals(di, sigfn(di, **params), fee_rt=fee_rt, leverage=leverage) + oos = backtest_signals(do, sigfn(do, **params), fee_rt=fee_rt, leverage=leverage) + return full, is_, oos + + +def hdr(t): + print("\n" + "=" * 100) + print(t) + print("=" * 100) + + +# =========================================================================== +# Main +# =========================================================================== +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--quick", action="store_true") + args = ap.parse_args() + t0 = time.time() + assets = ["BTC", "ETH"] + tfs = ["1h"] if args.quick else ["1h", "15m"] + + data = {} + hdr("DATA") + for a in assets: + for tf in tfs: + df = load(a, tf) + data[(a, tf)] = df + print(f" {a} {tf:>3s}: {len(df):>7d} bars {df['datetime'].iloc[0].date()}" + f"->{df['datetime'].iloc[-1].date()}") + + # --------------------------------------------------------------------- + # PASS 1 — PRIOR-DAY BREAKOUT: continuation vs fade, any-bar (first break/day), + # EOD exit. THE core question: do prior-day breakouts continue or revert? + # --------------------------------------------------------------------- + hdr("PASS 1 — PRIOR-DAY HIGH/LOW breakout, any-bar first-break, EOD exit (1h, fee=0.001)\n" + " CONTINUATION vs FADE side-by-side. OOS net must be >0 on BOTH to matter.") + print(f" {'rule':<26s} | " + f"{'BTC IS / OOS (tr, wr, shrp)':<40s} | {'ETH IS / OOS (tr, wr, shrp)':<40s}") + for level in ["high", "low"]: + for side in ["cont", "fade"]: + name = f"PD {level:<4s} {side}" + line = f" {name:<26s} | " + for a in assets: + df = data[(a, "1h")] + _, is_, oos = run_split(df, sig_prior_break, + dict(period="D", level=level, side=side, + exit_mode="eod")) + line += (f"{is_.net_return*100:>+6.0f}/{oos.net_return*100:>+6.0f}% " + f"(t{oos.n_trades:>4d} w{oos.win_rate:>4.1f} s{oos.sharpe:>+4.1f}) | ") + print(line) + + # --------------------------------------------------------------------- + # PASS 2 — OPENING-RANGE breakout (continuation vs fade), various open windows. + # --------------------------------------------------------------------- + hdr("PASS 2 — OPENING-RANGE breakout (first N UTC hours), EOD exit (1h, fee=0.001).\n" + " CONTINUATION vs FADE. Survivor = OOS>0 on BOTH assets.") + for nopen in ([6] if args.quick else [3, 6, 8, 12]): + for side in ["cont", "fade"]: + name = f"OR N={nopen:<2d} {side}" + line = f" {name:<26s} | " + for a in assets: + df = data[(a, "1h")] + _, is_, oos = run_split(df, sig_or_break, + dict(n_open_hours=nopen, side=side, exit_mode="eod")) + line += (f"{a} OOS={oos.net_return*100:>+6.0f}% " + f"(t{oos.n_trades:>4d} w{oos.win_rate:>4.1f} s{oos.sharpe:>+4.1f}) | ") + print(line) + + # --------------------------------------------------------------------- + # PASS 3 — GAP vs prior close at day open (hour 0), continuation vs fade, + # with optional prior-day return-sign filter. + # --------------------------------------------------------------------- + hdr("PASS 3 — GAP vs prior-day CLOSE at hour 0, EOD exit (1h, fee=0.001).\n" + " continuation vs fade; thr = min |gap|.") + for thr in ([0.0] if args.quick else [0.0, 0.005, 0.01]): + for side in ["cont", "fade"]: + name = f"GAP thr={thr*100:.1f}% {side}" + line = f" {name:<26s} | " + for a in assets: + df = data[(a, "1h")] + _, is_, oos = run_split(df, sig_gap, + dict(side=side, anchor_hour=0, thr=thr, exit_mode="eod")) + line += (f"{a} OOS={oos.net_return*100:>+6.0f}% " + f"(t{oos.n_trades:>4d} w{oos.win_rate:>4.1f} s{oos.sharpe:>+4.1f}) | ") + print(line) + + # --------------------------------------------------------------------- + # PASS 4 — PRIOR-WEEK high/low breakout (continuation vs fade), EOD exit. + # --------------------------------------------------------------------- + hdr("PASS 4 — PRIOR-WEEK HIGH/LOW breakout, any-bar first-break, EOD exit (1h, fee=0.001).") + for level in ["high", "low"]: + for side in ["cont", "fade"]: + name = f"PW {level:<4s} {side}" + line = f" {name:<26s} | " + for a in assets: + df = data[(a, "1h")] + _, is_, oos = run_split(df, sig_prior_break, + dict(period="W", level=level, side=side, + exit_mode="eod")) + line += (f"{a} IS={is_.net_return*100:>+6.0f}% OOS={oos.net_return*100:>+6.0f}% " + f"(t{oos.n_trades:>4d} s{oos.sharpe:>+4.1f}) | ") + print(line) + + # --------------------------------------------------------------------- + # PASS 5 — TIME-ANCHORED prior-day breakout: sweep the anchor hour to expose + # whether any apparent edge is just a lucky single hour. + # --------------------------------------------------------------------- + hdr("PASS 5 — TIME-ANCHORED PD-high CONTINUATION across UTC anchor hours (1h, EOD exit).\n" + " A real edge is NOT a single lucky hour. (full-sample net per hour.)") + hours = list(range(0, 24, 1 if not args.quick else 3)) + for a in assets: + df = data[(a, "1h")] + cells = [] + for hh in hours: + full, _, _ = run_split(df, sig_prior_break, + dict(period="D", level="high", side="cont", + anchor_hour=hh, exit_mode="eod")) + cells.append((hh, full.net_return * 100, full.sharpe, full.n_trades)) + pos = sum(1 for _, r, _, _ in cells if r > 0) + print(f" {a}: {pos}/{len(cells)} anchor-hours net>0 (full). " + f"best={max(cells, key=lambda x: x[1])[0]}h " + f"({max(c[1] for c in cells):+.0f}%) worst={min(c[1] for c in cells):+.0f}%") + line = " " + " ".join(f"{hh:02d}h:{r:>+5.0f}" for hh, r, _, _ in cells) + print(line) + + # --------------------------------------------------------------------- + # PASS 6 — GRID ROBUSTNESS on the best family from PASS 1-4. We grid the + # PD-low CONTINUATION and FADE plus OR breakout, require OOS>0 on BOTH assets. + # --------------------------------------------------------------------- + hdr("PASS 6 — GRID ROBUSTNESS. Cell SURVIVES only if OOS net>0 on BOTH BTC AND ETH.") + + def grid(label, fn, base, sweep, tf="1h", fee=0.001): + keys = list(sweep.keys()) + rows, surv = [], [] + for combo in product(*[sweep[k] for k in keys]): + params = dict(base); params.update(dict(zip(keys, combo))) + res = {} + for a in assets: + _, is_, oos = run_split(data[(a, tf)], fn, params, fee_rt=fee) + res[a] = oos + ok = all(res[a].net_return > 0 for a in assets) + rows.append((params, res, ok)) + if ok: + surv.append((params, res)) + print(f" [{label}] {len(surv)}/{len(rows)} cells OOS>0 on BOTH assets") + rows.sort(key=lambda r: np.mean([r[1][a].net_return for a in assets]), reverse=True) + for params, res, ok in rows[:5]: + tag = "OK " if ok else " -" + pp = {k: params[k] for k in sweep} + s = f" {tag}{pp} | " + for a in assets: + s += f"{a} OOS={res[a].net_return*100:>+6.0f}% (s{res[a].sharpe:>+4.1f}) " + print(s) + return surv + + sweeps = [] + sweeps.append(grid("PD-low cont", sig_prior_break, + dict(period="D", level="low", side="cont", exit_mode="eod"), + dict(buffer=[0.0, 0.001, 0.003], anchor_hour=[None]))) + sweeps.append(grid("PD-low fade", sig_prior_break, + dict(period="D", level="low", side="fade", exit_mode="eod"), + dict(buffer=[0.0, 0.001, 0.003], anchor_hour=[None]))) + sweeps.append(grid("PD-high cont", sig_prior_break, + dict(period="D", level="high", side="cont", exit_mode="eod"), + dict(buffer=[0.0, 0.001, 0.003], anchor_hour=[None]))) + sweeps.append(grid("PD-high fade", sig_prior_break, + dict(period="D", level="high", side="fade", exit_mode="eod"), + dict(buffer=[0.0, 0.001, 0.003], anchor_hour=[None]))) + if not args.quick: + sweeps.append(grid("OR cont", sig_or_break, + dict(side="cont", exit_mode="eod"), + dict(n_open_hours=[3, 6, 8, 12]))) + sweeps.append(grid("OR fade", sig_or_break, + dict(side="fade", exit_mode="eod"), + dict(n_open_hours=[3, 6, 8, 12]))) + + # --------------------------------------------------------------------- + # PASS 7 — FEE SWEEP + per-year on the single best surviving rule (if any), + # else on the least-bad PD rule, to show fee sensitivity and year stability. + # --------------------------------------------------------------------- + hdr("PASS 7 — FEE SWEEP + PER-YEAR on the best PD rule. fee=0 is GROSS (is the SIGN of\n" + " the edge even right before fees?).") + # pick best rule: scan the 4 PD sides at default, mean OOS over assets + candidates = [ + ("PD low cont", dict(period="D", level="low", side="cont", exit_mode="eod")), + ("PD low fade", dict(period="D", level="low", side="fade", exit_mode="eod")), + ("PD high cont", dict(period="D", level="high", side="cont", exit_mode="eod")), + ("PD high fade", dict(period="D", level="high", side="fade", exit_mode="eod")), + ] + scored = [] + for nm, p in candidates: + m = np.mean([run_split(data[(a, "1h")], sig_prior_break, p)[2].net_return for a in assets]) + scored.append((m, nm, p)) + scored.sort(reverse=True) + best_nm, best_p = scored[0][1], scored[0][2] + print(f" best-by-meanOOS PD rule: {best_nm} (meanOOS={scored[0][0]*100:+.0f}%)") + fees = [0.0, 0.0005, 0.001, 0.0015, 0.002] + for a in assets: + df = data[(a, "1h")] + line = f" {a} fee-sweep (RT%): " + for f in fees: + full, _, oos = run_split(df, sig_prior_break, best_p, fee_rt=f) + line += f"{f*100:.2f}%[full={full.net_return*100:>+5.0f}/OOS={oos.net_return*100:>+5.0f}] " + print(line) + print(" per-year (full sample, fee=0.001):") + for a in assets: + df = data[(a, "1h")] + full, _, _ = run_split(df, sig_prior_break, best_p) + yrs = " ".join(f"{y}:{full.yearly[y]*100:>+5.0f}%" for y in sorted(full.yearly)) + print(f" {a}: trades={full.n_trades} Sharpe={full.sharpe:+.2f} " + f"maxDD={full.max_dd*100:.0f}% EUR/d(2k)={full.daily_profit(2000):+.2f}") + print(f" {yrs}") + + # --------------------------------------------------------------------- + # VERDICT + # --------------------------------------------------------------------- + hdr("VERDICT") + total_surv = sum(len(s) for s in sweeps) + if total_surv == 0: + print(" ZERO grid cells produced OOS net>0 on BOTH BTC and ETH at baseline fees.") + print(" => No robust prior-period breakout/fade edge on clean BTC/ETH. The continuation-") + print(" vs-fade tables above show which SIDE (if any) is even net-positive in-sample;") + print(" consult PASS 1-5 for direction. Not deployable.") + else: + print(f" {total_surv} grid cell(s) survived OOS>0 on both assets. Inspect PASS 6/7 and") + print(" stress with fee sweep + per-year before trusting. List of survivors:") + for s in sweeps: + for params, res in s: + ms = np.mean([res[a].net_return for a in assets]) * 100 + print(f" {params} meanOOS={ms:+.0f}%") + print(f"\n (elapsed {time.time()-t0:.0f}s)") + + +if __name__ == "__main__": + main() diff --git a/src/strategies/trend_portfolio.py b/src/strategies/trend_portfolio.py index b546ad0..2693b47 100644 --- a/src/strategies/trend_portfolio.py +++ b/src/strategies/trend_portfolio.py @@ -4,13 +4,16 @@ Vincitrice della ricerca su dati certificati BTC/ETH (Deribit mainnet). TSMOM mu (1-3-6 mesi) vol-targeted, portafoglio 50/50 BTC+ETH. Validata onestamente (no look-ahead, fee 0.10% RT, positiva ogni anno 2019-2026, robusta su griglia e su tutti i timeframe 15m-1d). -Config canonica deployabile (PORT LF4h): - timeframe 4h, LONG-FLAT (niente short), vol-target 20%, leverage cap 2x. - -> CAGR ~16.6%, Sharpe ~1.32, maxDD ~12.3% (backtest 2019-2026 su 50/50 BTC+ETH). +Config canonica deployabile (PORT LF12h): + timeframe 12h, LONG-FLAT (niente short), vol-target 20%, leverage cap 2x. + -> CAGR ~16.2%, Sharpe ~1.32, maxDD ~13.3% (backtest 2019-2026 su 50/50 BTC+ETH). -Perche' long-flat e 4h: gli short del trend rendono meno e aggiungono DD; il 4h e' il punto -dolce (meno rumore/fee del 15m, meno lag dell'1d). Vedi docs/diary/2026-06-19-research-synthesis.md -e scripts/research/trackD_*.py. +Perche' >=12h (AGGIORNATO 2026-06-19): l'audit anti-look-ahead (scripts/research/ +trackD_lookahead_audit.py) mostra che il pipeline e' pulito (label-invariante, robusto a +1 +barra di lag), ma SOTTO le 12h costi e overfitting al rumore ad alta frequenza dominano (il +piccolo extra di Sharpe a 4h/6h/8h non e' affidabile). A 12h/1d il risultato e' ~identico e +robusto -> si deploya a 12h. Perche' long-flat: gli short del trend rendono meno e aggiungono +DD. Vedi docs/diary/2026-06-19-research-synthesis.md e scripts/research/trackD_*.py. API (tutto causale, decide con dati <= close[i]): from src.strategies.trend_portfolio import TrendPortfolio, CANONICAL @@ -168,16 +171,26 @@ def _bars_per_year(idx: pd.DatetimeIndex) -> float: return 86400 * 365.25 / dt if dt and dt > 0 else 365.25 -def resample_4h(df_1h: pd.DataFrame) -> pd.DataFrame: - """Resample 1h -> 4h (confini 00:00 UTC). Schema con 'datetime'.""" +DEPLOY_TF = "12h" # timeframe deployabile (>=12h: sotto, costi/overfit dominano) + + +def resample_tf(df_1h: pd.DataFrame, rule: str = "12h") -> pd.DataFrame: + """Resample 1h -> rule (confini 00:00 UTC, open-labeled). Schema con 'datetime'. + Il consumo e' index-based con shift +1 barra (net_returns) -> il labeling NON leakka + (verificato in trackD_lookahead_audit.py: Sharpe left == Sharpe right).""" g = df_1h.copy() idx = pd.to_datetime(g["timestamp"], unit="ms", utc=True) idx.name = "dt" g.index = idx - out = g.resample("4h", label="left", closed="left").agg( + out = g.resample(rule, label="left", closed="left").agg( {"open": "first", "high": "max", "low": "min", "close": "last", "volume": "sum"}) out = out.dropna(subset=["open"]) out["datetime"] = out.index epoch = pd.Timestamp("1970-01-01", tz="UTC") out["timestamp"] = ((out.index - epoch) // pd.Timedelta(milliseconds=1)).astype("int64") return out.reset_index(drop=True)[["timestamp", "open", "high", "low", "close", "volume", "datetime"]] + + +def resample_4h(df_1h: pd.DataFrame) -> pd.DataFrame: + """Compat per gli script di ricerca. Per il DEPLOY usare resample_tf(df, '12h').""" + return resample_tf(df_1h, "4h") diff --git a/tests/test_trend_portfolio.py b/tests/test_trend_portfolio.py index 07b1e44..c84fc15 100644 --- a/tests/test_trend_portfolio.py +++ b/tests/test_trend_portfolio.py @@ -10,16 +10,16 @@ sys.path.insert(0, str(PROJECT_ROOT)) from src.backtest.harness import load from src.strategies.trend_portfolio import ( - TrendPortfolio, CANONICAL, resample_4h, simple_returns, tsmom_blend) + TrendPortfolio, CANONICAL, resample_tf, simple_returns, tsmom_blend) def _dfs(): - return {a: resample_4h(load(a, "1h")) for a in ("BTC", "ETH")} + return {a: resample_tf(load(a, "1h")) for a in ("BTC", "ETH")} def test_no_lookahead_target_is_causal(): """target_series[:k] non deve cambiare se aggiungo barre future.""" - df = resample_4h(load("BTC", "1h")) + df = resample_tf(load("BTC", "1h")) tp = TrendPortfolio(**CANONICAL) full = tp.target_series(df) k = len(df) - 500 @@ -41,7 +41,7 @@ def test_canonical_backtest_is_profitable_and_robust(): def test_long_only_never_short(): - df = resample_4h(load("ETH", "1h")) + df = resample_tf(load("ETH", "1h")) tp = TrendPortfolio(**CANONICAL) # long_only=True assert (tp.target_series(df) >= 0).all() @@ -59,12 +59,11 @@ def test_paper_advance_matches_backtest_slice(): combo = 0.5 * J["BTC"].values + 0.5 * J["ETH"].values # equity sull'ultimo tratto (skip warmup) tail = combo[-500:] - eq_ref = np.cumprod(1.0 + np.clip(tail, -0.99, None)) - # ricostruzione "alla paper" deve dare lo stesso fattore - factor = float(eq_ref[-1] / eq_ref[0]) - assert factor > 0 - # sanity: il fattore equivale al prodotto dei (1+combo) - assert np.isclose(factor, np.prod(1.0 + np.clip(tail, -0.99, None)) / (1.0), rtol=1e-9) + steps = 1.0 + np.clip(tail, -0.99, None) + eq_ref = np.cumprod(steps) + # il loop paper accumula moltiplicando i (1+net) barra per barra -> stesso prodotto + assert np.isclose(eq_ref[-1], np.prod(steps), rtol=1e-9) + assert eq_ref[-1] > 0 def test_tsmom_blend_range():