#!/usr/bin/env python """r0702_expiry_calendar.py — FILONE: effetti del calendario SCADENZE Deribit (2026-07-02). Deribit: opzioni settimanali scadono ogni VENERDI' 08:00 UTC; mensili l'ultimo venerdi' del mese 08:00 UTC; trimestrali l'ultimo venerdi' di mar/giu/set/dic. Ipotesi: pinning / compressione pre-expiry, drift post-expiry (rimozione hedging dealer), pattern di vol. === GRIGLIA DICHIARATA PRIMA DI GUARDARE I DATI (nessun cherry-picking a posteriori) === Finestre evento (24h, allineate alla griglia giornaliera ancorata alle 08:00 UTC): W-2 = [-48h,-24h) W-1 = [-24h,0) W0 = [0,+24h) W+1 = [+24h,+48h) Tipi expiry: WEEKLY (ogni venerdi' 08:00), MONTHLY (ultimo venerdi' del mese), QUARTERLY (ultimo venerdi' di mar/giu/set/dic). NB: MONTHLY subset di WEEKLY, QUARTERLY subset di MONTHLY (nesting dichiarato). Asset: BTC, ETH. => 24 celle base (3 tipi x 4 finestre x 2 asset). Multiple testing: Bonferroni a 24 celle, alpha 5% due code -> |t| >= 3.09. CONFOUND STRUTTURALE dichiarato: il WEEKLY e' osservazionalmente IDENTICO al day-of-week venerdi' (ogni venerdi' e' un expiry: non esistono venerdi' di controllo) e SEA02 (day-of-week) e' gia' morto. Quindi: - CONTRASTI CHIAVE separabili: MONTHLY vs ALTRI venerdi' (controlla il day-of-week), QUARTERLY vs ALTRE monthly. Sono questi i test che possono dare un PASS. NULL (tutti e tre, un effetto vero li passa tutti): (a) PLACEBO WEEKDAY: ancora lun/mar/mer/gio 08:00 (weekly) e last-lun..last-gio del mese (monthly): il venerdi'/ultimo-venerdi' deve essere speciale. (b) ANCHOR-SHIFT: ancora a 08:00 +/-2h/+/-4h (04/06/08/10/12): un evento reale degrada gradualmente, un artefatto di etichettatura si inverte. (c) PERMUTATION: 500 calendari con stesso n eventi/anno. perm-A = giorni casuali (qualsiasi weekday); perm-B (piu' affilato) = venerdi' casuali (monthly) / ultimi-venerdi'-del-mese casuali (quarterly). Statistica prima della strategia. Regola tradabile costruita SOLO se una cella passa: |t2|>=3.09 (Bonferroni) AND placebo AND anchor-shift senza inversione AND perm pctl estremo (<=1% o >=99%). La famiglia strategica (3 tipi x 4 finestre x 2 direzioni = 24 trial) viene comunque valutata per riportare deflated_sharpe con n trial onesto. Vincoli rispettati: nessun .view("int64") su datetimes tz-aware (epoca esplicita in ms via colonna `timestamp` gia' in ms); posizioni causali (target[i] deciso a close[i], tenuto nella barra i+1 — eval_weights shifta); fee 0.10% RT. """ from __future__ import annotations import sys sys.path.insert(0, "/opt/docker/PythagorasGoal/scripts/research/alt") import numpy as np import pandas as pd from scipy import stats as sps import altlib as al RNG_SEED = 20260702 N_PERM = 500 ANCHOR_HOUR = 8 OFFSETS = {"[-48,-24)": -2, "[-24,0)": -1, "[0,+24)": 0, "[+24,+48)": +1} N_CELLS = 24 # 3 tipi x 4 finestre x 2 asset — dichiarato BONF_T = float(sps.norm.ppf(1 - 0.025 / N_CELLS)) # ~3.09 ASSETS = ("BTC", "ETH") ETYPES = ("WEEKLY", "MONTHLY", "QUARTERLY") MS_H = 3_600_000 MS_D = 24 * MS_H # =========================================================================== # CALENDARIO SCADENZE (funzione pura, nessun dato di mercato) # =========================================================================== def _utc_index(values) -> pd.DatetimeIndex: idx = pd.DatetimeIndex(values) return idx.tz_localize("UTC") if idx.tz is None else idx.tz_convert("UTC") def expiry_calendar(start: str, end: str, anchor_hour: int = ANCHOR_HOUR) -> dict: """Ancore expiry Deribit (tz UTC, ore = anchor_hour). Ritorna dict tipo->DatetimeIndex.""" days = pd.date_range(start, end, freq="D", tz="UTC") fri = days[days.weekday == 4] weekly = pd.DatetimeIndex(fri) + pd.Timedelta(hours=anchor_hour) ym = np.asarray(fri.year * 100 + fri.month) per = pd.Series(fri).groupby(ym).max() monthly = _utc_index(per.to_numpy()) + pd.Timedelta(hours=anchor_hour) quarterly = monthly[monthly.month.isin([3, 6, 9, 12])] return {"WEEKLY": weekly, "MONTHLY": monthly, "QUARTERLY": quarterly} def placebo_calendar(start: str, end: str, weekday: int, anchor_hour: int = ANCHOR_HOUR) -> dict: """Placebo: stesso costrutto ancorato a un ALTRO giorno della settimana.""" days = pd.date_range(start, end, freq="D", tz="UTC") wd = days[days.weekday == weekday] weekly = pd.DatetimeIndex(wd) + pd.Timedelta(hours=anchor_hour) ym = np.asarray(wd.year * 100 + wd.month) per = pd.Series(wd).groupby(ym).max() monthly = _utc_index(per.to_numpy()) + pd.Timedelta(hours=anchor_hour) return {"WEEKLY": weekly, "MONTHLY": monthly} # =========================================================================== # GRIGLIA GIORNALIERA ancorata (ritorno log 24h + RV) — tutta epoca ms esplicita # =========================================================================== def day_table(asset: str, anchor_hour: int = ANCHOR_HOUR) -> pd.DataFrame: """Partiziona le barre 1h in 'giorni' [anchor, anchor+24h). Ritorna per giorno: ret (somma log-ret orari = log-ret close->close della finestra), rv (std oraria annualizzata), n barre. r[i] copre ~[open_i, open_i+1h) => giorno = open in finestra.""" df = al.get(asset, "1h") ts = df["timestamp"].to_numpy(dtype=np.int64) # epoca ms esplicita c = df["close"].to_numpy(dtype=float) lr = np.zeros(len(c)) lr[1:] = np.log(c[1:] / c[:-1]) day_ms = ((ts - anchor_hour * MS_H) // MS_D) * MS_D + anchor_hour * MS_H g = pd.DataFrame({"day_ms": day_ms, "lr": lr}) agg = g.groupby("day_ms")["lr"].agg(ret="sum", rv="std", n="size") agg = agg[agg["n"] >= 20] # solo giorni ~completi agg["rv"] = agg["rv"] * np.sqrt(24 * 365.25) # RV annualizzata agg.index = pd.to_datetime(agg.index, unit="ms", utc=True) return agg _EPOCH = pd.Timestamp("1970-01-01", tz="UTC") def anchors_ms(anchors: pd.DatetimeIndex) -> np.ndarray: """Epoca ms ESPLICITA e unit-safe: in pandas 2.x un DatetimeIndex tz-aware puo' essere in unita' s/ms/ns (.asi8 cambia scala!) — la delta da EPOCH no.""" delta = pd.DatetimeIndex(anchors) - _EPOCH return np.asarray(delta // pd.Timedelta(milliseconds=1), dtype=np.int64) def window_days(agg: pd.DataFrame, anchors: pd.DatetimeIndex, offset: int) -> pd.DataFrame: """I giorni-griglia che iniziano a (anchor + offset*24h) e cadono nel campione.""" starts = pd.DatetimeIndex(pd.to_datetime( anchors_ms(anchors) + offset * MS_D, unit="ms", utc=True)) return agg.loc[agg.index.isin(starts)] def cell_stats(agg: pd.DataFrame, anchors: pd.DatetimeIndex, offset: int) -> dict: ev = window_days(agg, anchors, offset) base = agg.loc[~agg.index.isin(ev.index)] r = ev["ret"].to_numpy() if len(r) < 8: return dict(n=len(r)) mean, med = float(r.mean()), float(np.median(r)) sem = float(r.std(ddof=1) / np.sqrt(len(r))) t1 = mean / sem if sem > 0 else 0.0 # vs zero t2, p2 = sps.ttest_ind(r, base["ret"].to_numpy(), equal_var=False) # vs tutte le altre finestre rv_ev, rv_b = float(ev["rv"].mean()), float(base["rv"].mean()) trv, prv = sps.ttest_ind(ev["rv"].dropna(), base["rv"].dropna(), equal_var=False) ci = 1.96 * sem return dict(n=len(r), mean=mean, median=med, ci95=ci, t1=float(t1), t2=float(t2), p2=float(p2), base_mean=float(base["ret"].mean()), rv_ev=rv_ev, rv_base=rv_b, rv_ratio=rv_ev / rv_b if rv_b > 0 else np.nan, t_rv=float(trv)) def contrast_stats(agg: pd.DataFrame, a1: pd.DatetimeIndex, a2: pd.DatetimeIndex, offset: int) -> dict: """Welch t fra finestre-evento di due calendari (es. MONTHLY vs altri venerdi').""" e1 = window_days(agg, a1, offset)["ret"].to_numpy() e2 = window_days(agg, a2, offset)["ret"].to_numpy() if len(e1) < 8 or len(e2) < 8: return dict(n1=len(e1), n2=len(e2)) t, p = sps.ttest_ind(e1, e2, equal_var=False) return dict(n1=len(e1), n2=len(e2), m1=float(e1.mean()), m2=float(e2.mean()), diff=float(e1.mean() - e2.mean()), t=float(t), p=float(p)) # =========================================================================== # PERMUTATION NULL — 500 calendari, stesso n eventi/anno # =========================================================================== def permutation_null(agg: pd.DataFrame, anchors: pd.DatetimeIndex, offsets: dict, pool: pd.DatetimeIndex, n_perm: int = N_PERM, seed: int = RNG_SEED) -> dict: """Percentile della media reale per finestra vs n_perm calendari casuali estratti da `pool` (stesso numero di ancore per anno del calendario reale, senza rimpiazzo).""" rng = np.random.default_rng(seed) full_idx = pd.date_range(agg.index.min(), agg.index.max(), freq="24h") ret_full = agg["ret"].reindex(full_idx).to_numpy() # NaN dove giorno mancante t0 = int(anchors_ms(full_idx)[0]) def pos_of(idx: pd.DatetimeIndex) -> np.ndarray: return ((anchors_ms(idx) - t0) // MS_D).astype(np.int64) n_days = len(full_idx) a_pos = pos_of(anchors) a_pos = a_pos[(a_pos >= 2) & (a_pos < n_days - 2)] pool_pos = pos_of(pool) pool_pos = pool_pos[(pool_pos >= 2) & (pool_pos < n_days - 2)] years_a = pd.to_datetime((a_pos * MS_D + t0), unit="ms", utc=True).year years_p = pd.to_datetime((pool_pos * MS_D + t0), unit="ms", utc=True).year per_year = pd.Series(a_pos).groupby(np.asarray(years_a)).size() pool_by_year = {y: pool_pos[years_p == y] for y in per_year.index} def win_means(pos: np.ndarray) -> dict: out = {} for wname, off in offsets.items(): v = ret_full[pos + off] out[wname] = float(np.nanmean(v)) return out real = win_means(a_pos) null = {w: np.empty(n_perm) for w in offsets} for k in range(n_perm): draw = [] for y, cnt in per_year.items(): p = pool_by_year.get(y, np.array([], dtype=np.int64)) if len(p) == 0: continue take = min(cnt, len(p)) draw.append(rng.choice(p, size=take, replace=False)) pos = np.concatenate(draw) if draw else np.array([], dtype=np.int64) wm = win_means(pos) for w in offsets: null[w][k] = wm[w] return {w: dict(real=real[w], pctl=float(np.mean(null[w] <= real[w])), null_mean=float(np.nanmean(null[w])), null_sd=float(np.nanstd(null[w]))) for w in offsets} # =========================================================================== # STRATEGIA (famiglia dichiarata: 3 tipi x 4 finestre x 2 direzioni = 24 trial) # =========================================================================== def make_expiry_target(anchors: pd.DatetimeIndex, offset: int, direction: float): """target[i] = direction se la PROSSIMA barra (open+1h) cade nella finestra [anchor+offset*24h, anchor+(offset+1)*24h). Calendario noto ex-ante => causale; eval_weights shifta comunque di +1 barra (decidi a close[i], agisci in i+1).""" ws = np.sort(anchors_ms(anchors) + offset * MS_D) # start finestre, ms def target_fn(df: pd.DataFrame) -> np.ndarray: ts = df["timestamp"].to_numpy(dtype=np.int64) + MS_H # open prossima barra j = np.searchsorted(ws, ts, side="right") - 1 ok = (j >= 0) & ((ts - ws[np.clip(j, 0, len(ws) - 1)]) < MS_D) return np.where(ok, direction, 0.0) return target_fn def strategy_family(cal: dict) -> list[dict]: fam = [] for et in ETYPES: for wname, off in OFFSETS.items(): for d in (+1.0, -1.0): fam.append(dict(etype=et, window=wname, offset=off, direction=d, fn=make_expiry_target(cal[et], off, d))) return fam # =========================================================================== # MAIN # =========================================================================== def main() -> None: aggs = {a: day_table(a) for a in ASSETS} spans = {a: (aggs[a].index.min(), aggs[a].index.max()) for a in ASSETS} cal_start = min(s[0] for s in spans.values()) - pd.Timedelta(days=3) cal_end = max(s[1] for s in spans.values()) + pd.Timedelta(days=3) cal = expiry_calendar(str(cal_start.date()), str(cal_end.date())) print(f"Span dati (griglia 08:00): " + "; ".join(f"{a} {spans[a][0].date()}->{spans[a][1].date()} ({len(aggs[a])}g)" for a in ASSETS)) print(f"Eventi calendario: " + ", ".join(f"{k}={len(v)}" for k, v in cal.items())) print(f"Soglia Bonferroni (24 celle, 5% due code): |t2| >= {BONF_T:.2f}\n") # ------------------------------------------------------------------ (3) statistica print("=" * 100) print("(1) EFFETTI PER FINESTRA x TIPO-EXPIRY x ASSET — ret log 24h; t1 vs 0, t2 vs TUTTE le altre finestre") print("=" * 100) cells = {} for a in ASSETS: for et in ETYPES: for wname, off in OFFSETS.items(): st = cell_stats(aggs[a], cal[et], off) cells[(a, et, wname)] = st if st.get("n", 0) >= 8: print(f"{a} {et:9s} {wname:10s} n={st['n']:4d} " f"mean={st['mean']*100:+.3f}%±{st['ci95']*100:.3f} " f"med={st['median']*100:+.3f}% t1={st['t1']:+.2f} t2={st['t2']:+.2f} " f"| RV ev/base={st['rv_ev']:.3f}/{st['rv_base']:.3f} " f"ratio={st['rv_ratio']:.2f} tRV={st['t_rv']:+.2f}") print("-" * 100) print("\nPer-anno (mean ret % della finestra; n eventi):") for a in ASSETS: for et in ETYPES: years = sorted(set(aggs[a].index.year)) for wname, off in OFFSETS.items(): ev = window_days(aggs[a], cal[et], off) parts = [] for y in years: r = ev[ev.index.year == y]["ret"] parts.append(f"{y}:{r.mean()*100:+.2f}({len(r)})" if len(r) else f"{y}:--") print(f"{a} {et:9s} {wname:10s} " + " ".join(parts)) print() # -------------------------------------------------- contrasti chiave (separabili) print("=" * 100) print("CONTRASTI CHIAVE (separano l'expiry dal day-of-week): MONTHLY vs ALTRI venerdi'; QUARTERLY vs ALTRE monthly") print("=" * 100) other_fri = cal["WEEKLY"][~cal["WEEKLY"].isin(cal["MONTHLY"])] other_mon = cal["MONTHLY"][~cal["MONTHLY"].isin(cal["QUARTERLY"])] contrasts = {} for a in ASSETS: for label, (a1, a2) in {"MONTHLY-vs-otherFRI": (cal["MONTHLY"], other_fri), "QUARTERLY-vs-otherMON": (cal["QUARTERLY"], other_mon)}.items(): for wname, off in OFFSETS.items(): cs = contrast_stats(aggs[a], a1, a2, off) contrasts[(a, label, wname)] = cs if cs.get("n1", 0) >= 8: print(f"{a} {label:22s} {wname:10s} n={cs['n1']}/{cs['n2']} " f"m_ev={cs['m1']*100:+.3f}% m_ctrl={cs['m2']*100:+.3f}% " f"diff={cs['diff']*100:+.3f}% t={cs['t']:+.2f} p={cs['p']:.3f}") # ------------------------------------------------------------------ (4a) placebo print("\n" + "=" * 100) print("(2a) PLACEBO WEEKDAY — stesso costrutto ancorato a lun/mar/mer/gio (t2 vs base; venerdi' deve spiccare)") print("=" * 100) wd_names = {0: "MON", 1: "TUE", 2: "WED", 3: "THU", 4: "FRI(reale)"} placebo_t2 = {} for a in ASSETS: for et in ("WEEKLY", "MONTHLY"): for wname, off in OFFSETS.items(): row = {} for wd in (0, 1, 2, 3): pc = placebo_calendar(str(cal_start.date()), str(cal_end.date()), wd) st = cell_stats(aggs[a], pc[et], off) row[wd_names[wd]] = st.get("t2", np.nan) row[wd_names[4]] = cells[(a, et, wname)].get("t2", np.nan) placebo_t2[(a, et, wname)] = row print(f"{a} {et:8s} {wname:10s} " + " ".join(f"{k}:{v:+.2f}" for k, v in row.items())) # -------------------------------------------------------------- (4b) anchor-shift print("\n" + "=" * 100) print("(2b) ANCHOR-SHIFT — media evento (%) con ancora a 04/06/08/10/12 UTC (reale=08). Inversione => artefatto") print("=" * 100) shift_means = {} for a in ASSETS: tabs = {h: day_table(a, anchor_hour=h) for h in (4, 6, 8, 10, 12)} for et in ETYPES: for wname, off in OFFSETS.items(): row = {} for h in (4, 6, 8, 10, 12): calh = expiry_calendar(str(cal_start.date()), str(cal_end.date()), anchor_hour=h) st = cell_stats(tabs[h], calh[et], off) row[h] = st.get("mean", np.nan) shift_means[(a, et, wname)] = row base = row[8] vals = [row[h] for h in (4, 6, 10, 12) if np.isfinite(row.get(h, np.nan))] inverts = (np.isfinite(base) and abs(base) > 0 and any(np.sign(v) == -np.sign(base) and abs(v) > 0.5 * abs(base) for v in vals)) print(f"{a} {et:9s} {wname:10s} " + " ".join(f"{h:02d}h:{row[h]*100:+.3f}%" for h in (4, 6, 8, 10, 12)) + f" inverts={inverts}") # -------------------------------------------------------------- (4c) permutation print("\n" + "=" * 100) print(f"(2c) PERMUTATION NULL — {N_PERM} calendari, stesso n eventi/anno. pctl = quota null <= reale") print(" perm-A: giorni casuali. perm-B (affilato): venerdi' casuali (MONTHLY) / monthly casuali (QUARTERLY)") print("=" * 100) perm = {} for a in ASSETS: idx_all = aggs[a].index pool_any = idx_all pool_fri = idx_all[idx_all.weekday == 4] pool_mon_expiry = idx_all[idx_all.isin(cal["MONTHLY"])] specs = {("WEEKLY", "A"): (cal["WEEKLY"], pool_any), ("MONTHLY", "A"): (cal["MONTHLY"], pool_any), ("MONTHLY", "B"): (cal["MONTHLY"], pool_fri), ("QUARTERLY", "A"): (cal["QUARTERLY"], pool_any), ("QUARTERLY", "B"): (cal["QUARTERLY"], pool_mon_expiry)} for (et, mode), (anch, pool) in specs.items(): res = permutation_null(aggs[a], anch, OFFSETS, pool) for wname, r in res.items(): perm[(a, et, mode, wname)] = r print(f"{a} {et:9s} perm-{mode} " + " ".join(f"{w}:{r['pctl']*100:.1f}%" for w, r in res.items())) # ------------------------------------------------------- gate statistico dichiarato print("\n" + "=" * 100) print("GATE (dichiarato in testa): |t2|>=Bonferroni AND placebo AND no-inversione AND perm pctl <=1% o >=99%") print("=" * 100) survivors = [] for a in ASSETS: for et in ETYPES: for wname in OFFSETS: st = cells[(a, et, wname)] if st.get("n", 0) < 8: continue t2 = st["t2"] bonf_ok = abs(t2) >= BONF_T pt = placebo_t2.get((a, et, wname)) placebo_ok = (pt is None or abs(pt["FRI(reale)"]) > max(abs(pt[k]) for k in ("MON", "TUE", "WED", "THU"))) row = shift_means[(a, et, wname)] base = row[8] shift_ok = not (np.isfinite(base) and any( np.isfinite(row[h]) and np.sign(row[h]) == -np.sign(base) and abs(row[h]) > 0.5 * abs(base) for h in (4, 6, 10, 12))) pA = perm.get((a, et, "A", wname), {}).get("pctl", 0.5) pB = perm.get((a, et, "B", wname), {}).get("pctl", pA) perm_ok = all(p <= 0.01 or p >= 0.99 for p in (pA, pB)) ok = bonf_ok and placebo_ok and shift_ok and perm_ok flag = " <== SURVIVES" if ok else "" print(f"{a} {et:9s} {wname:10s} t2={t2:+.2f} bonf={bonf_ok} " f"placebo={placebo_ok} shift_ok={shift_ok} " f"permA={pA:.3f} permB={pB:.3f} perm_ok={perm_ok}{flag}") if ok: survivors.append((a, et, wname, st)) # ---------------------------------------------- (5) famiglia strategica + DSR onesto print("\n" + "=" * 100) print("(3) FAMIGLIA STRATEGICA (24 trial dichiarati: 3 tipi x 4 finestre x 2 dir) — Sharpe 50/50 netto fee") print(" Riportata SEMPRE per il conteggio trial/DSR; study_marginal SOLO se la statistica sopravvive.") print("=" * 100) fam = strategy_family(cal) rows = [] for f in fam: daily = al.candidate_daily(f["fn"], tf="1h") ins = daily[daily.index < al.HOLDOUT] rows.append(dict(etype=f["etype"], window=f["window"], direction=f["direction"], fn=f["fn"], daily=daily, full_sh=al._sh(daily), ins_sh=al._sh(ins), hold_sh=al._sh(daily[daily.index >= al.HOLDOUT]))) rows.sort(key=lambda r: r["ins_sh"], reverse=True) for r in rows: print(f"{r['etype']:9s} {r['window']:10s} dir={r['direction']:+.0f} " f"insample={r['ins_sh']:+.2f} full={r['full_sh']:+.2f} hold={r['hold_sh']:+.2f}") all_full = [r["full_sh"] for r in rows] best = rows[0] # scelto IN-SAMPLE-ONLY (no hold-out) dsr, sr0 = al.deflated_sharpe(best["full_sh"], all_full, best["daily"]) print(f"\nCella best IN-SAMPLE: {best['etype']} {best['window']} dir={best['direction']:+.0f} " f"(ins {best['ins_sh']:+.2f}, full {best['full_sh']:+.2f}, hold {best['hold_sh']:+.2f})") print(f"deflated_sharpe (N={len(all_full)} trial): DSR={dsr:.3f} " f"(null max atteso ~{sr0:.2f} ann.) PASS>=0.95: {dsr >= 0.95}") if survivors: print("\nStatistica SOPRAVVISSUTA -> study_marginal sulla regola piu' semplice della cella best in-sample:") rep = al.study_marginal( f"EXPIRY-{best['etype']}-{best['window']}-d{best['direction']:+.0f}", best["fn"], tf="1h") print(al.fmt_marginal(rep)) yr = al.eval_weights(al.get("BTC", "1h"), best["fn"](al.get("BTC", "1h")))["yearly"] print("Per-anno BTC:", {y: v["ret"] for y, v in yr.items()}) else: print("\nNESSUNA cella sopravvive al gate statistico -> NESSUNA regola tradabile costruita (per protocollo).") print(f"\nSopravvissuti al gate statistico: {len(survivors)}/24 celle") if __name__ == "__main__": main()