chore(reset): v2.0.0 — storico certificato Deribit mainnet, ripartenza pulita
Reset del progetto su fondamenta verificate dopo la scoperta che l'intera libreria "validata OOS" era artefatto di feed contaminato (print fantasma del feed Cerbero TESTNET + storico Binance/USDT). - Storico ricostruito da Deribit MAINNET (ccxt pubblico, tokenless) e CERTIFICATO (certify_feed.py): BTC/ETH puliti su TUTTA la storia (mediana 2-6 bps vs Coinbase USD), integrita' OHLC + coerenza resample (maxΔ 0.00) + cross-venue OK. Alt esclusi (illiquidi/divergenti: LTC/DOGE 50-82% barre flat; XRP/BNB non certificabili). - Verdetto sul feed pulito: FADE / PAIRS / XS01 / TSM01 morti (ogni portafoglio Sharpe -2.3..-3.0, DD ~40%); solo SH01 e frammenti HONEST con segnale residuo, da ri-validare in isolamento. - Cleanup "restart pulito": strategie, stack live (src/live, src/portfolio, runner/executor, yml, docker), ~100 script ricerca/gate, waste/games/ portfolios, dati non certificati + cache e 60+ diari -> archiviati in Old/ (preservati, non cancellati). Diario consolidato in un unico documento. - Skeleton ricerca tenuto: Strategy ABC + indicatori + src/fractal + src/backtest/engine + load_data; tool dati certificati (rebuild_history, certify_feed, audit_feed, multi_source_check). - Universo dati ATTIVO: solo BTC/ETH (5m/15m/1h); guardrail fisico (load_data su alt -> FileNotFoundError). Esecuzione DISABILITATA, conto flat. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
"""Fetch dati REGIME backtestabili da Deribit MAINNET (public, no-auth) -> parquet.
|
||||
|
||||
Abilita la ricerca strategie frattali x regime (ARGO-proxy). Salva in data/raw/:
|
||||
{btc,eth}_dvol.parquet : DVOL index 1h (IV 30d "VIX crypto"), storico ~2021->oggi
|
||||
{btc,eth}_funding.parquet : funding rate perp 1h, storico ~2019->oggi
|
||||
|
||||
Solo componenti ARGO con STORICO GRATUITO (DVOL, funding) -> validabili OOS. Il GEX
|
||||
per-strike resta snapshot-only (vedi analisi 2026-06-01). Run:
|
||||
uv run python scripts/analysis/regime_fetcher.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RAW = ROOT / "data" / "regime" # NON data/raw (solo OHLCV) — evita pollution discovery asset
|
||||
BASE = "https://www.deribit.com/api/v2/public/"
|
||||
|
||||
|
||||
def _get(method: str, params: dict) -> dict:
|
||||
url = BASE + method + "?" + urllib.parse.urlencode(params)
|
||||
for _ in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception:
|
||||
time.sleep(1.0)
|
||||
return {}
|
||||
|
||||
|
||||
def fetch_dvol(currency: str, start_ms: int, end_ms: int, res: int = 3600) -> pd.DataFrame:
|
||||
"""DVOL index (OHLC). Cap 1000 righe/chiamata -> chaining all'indietro."""
|
||||
rows = []
|
||||
cur_end = end_ms
|
||||
span = 1000 * res * 1000
|
||||
while cur_end > start_ms:
|
||||
cur_start = max(start_ms, cur_end - span)
|
||||
d = _get("get_volatility_index_data", {
|
||||
"currency": currency, "start_timestamp": cur_start,
|
||||
"end_timestamp": cur_end, "resolution": res})
|
||||
data = (d.get("result") or {}).get("data") or []
|
||||
if not data:
|
||||
break
|
||||
rows.extend(data)
|
||||
oldest = min(x[0] for x in data)
|
||||
if oldest >= cur_end:
|
||||
break
|
||||
cur_end = oldest - 1
|
||||
time.sleep(0.15)
|
||||
if not rows:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(rows, columns=["timestamp", "open", "high", "low", "close"])
|
||||
df = df.drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
|
||||
df["dvol"] = df["close"]
|
||||
return df
|
||||
|
||||
|
||||
def fetch_funding(instrument: str, start_ms: int, end_ms: int) -> pd.DataFrame:
|
||||
"""funding rate history perp (1h). Paginazione ~30g/chiamata."""
|
||||
rows = []
|
||||
cur_start = start_ms
|
||||
step = 30 * 24 * 3600 * 1000
|
||||
while cur_start < end_ms:
|
||||
cur_end = min(end_ms, cur_start + step)
|
||||
d = _get("get_funding_rate_history", {
|
||||
"instrument_name": instrument,
|
||||
"start_timestamp": cur_start, "end_timestamp": cur_end})
|
||||
data = d.get("result") or []
|
||||
if data:
|
||||
rows.extend(data)
|
||||
cur_start = cur_end + 1
|
||||
time.sleep(0.12)
|
||||
if not rows:
|
||||
return pd.DataFrame()
|
||||
df = pd.DataFrame(rows)
|
||||
ts_col = "timestamp" if "timestamp" in df.columns else df.columns[0]
|
||||
df = df.rename(columns={ts_col: "timestamp"})
|
||||
keep = [c for c in ("timestamp", "interest_1h", "interest_8h", "index_price", "prev_index_price") if c in df.columns]
|
||||
df = df[keep].drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
|
||||
return df
|
||||
|
||||
|
||||
def main():
|
||||
RAW.mkdir(parents=True, exist_ok=True)
|
||||
now = _get("get_time", {})
|
||||
end_ms = int(now.get("result", 0)) or int(time.time() * 1000)
|
||||
start_ms = end_ms - int(6.5 * 365 * 24 * 3600 * 1000) # ~6.5 anni
|
||||
for cur, inst in (("BTC", "BTC-PERPETUAL"), ("ETH", "ETH-PERPETUAL")):
|
||||
dv = fetch_dvol(cur, start_ms, end_ms)
|
||||
if not dv.empty:
|
||||
p = RAW / f"{cur.lower()}_dvol.parquet"
|
||||
dv.to_parquet(p)
|
||||
rng = (pd.to_datetime(dv['timestamp'].min(), unit='ms').date(),
|
||||
pd.to_datetime(dv['timestamp'].max(), unit='ms').date())
|
||||
print(f" {cur} DVOL: {len(dv)} righe {rng[0]}->{rng[1]} (ora={dv['dvol'].iloc[-1]:.1f}) -> {p.name}")
|
||||
fr = fetch_funding(inst, start_ms, end_ms)
|
||||
if not fr.empty:
|
||||
p = RAW / f"{cur.lower()}_funding.parquet"
|
||||
fr.to_parquet(p)
|
||||
rng = (pd.to_datetime(fr['timestamp'].min(), unit='ms').date(),
|
||||
pd.to_datetime(fr['timestamp'].max(), unit='ms').date())
|
||||
print(f" {cur} FUNDING: {len(fr)} righe {rng[0]}->{rng[1]} -> {p.name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user