PythagorasGoal/scripts/analysis/regime_fetcher.py

"""Fetch dati REGIME backtestabili da Deribit MAINNET (public, no-auth) -> parquet.

Abilita la ricerca strategie frattali x regime (ARGO-proxy). Salva in data/raw/:
  {btc,eth}_dvol.parquet     : DVOL index 1h (IV 30d "VIX crypto"), storico ~2021->oggi
  {btc,eth}_funding.parquet  : funding rate perp 1h, storico ~2019->oggi

Solo componenti ARGO con STORICO GRATUITO (DVOL, funding) -> validabili OOS. Il GEX
per-strike resta snapshot-only (vedi analisi 2026-06-01). Run:
  uv run python scripts/analysis/regime_fetcher.py
"""
from __future__ import annotations

import time
import urllib.request
import urllib.parse
import json
from pathlib import Path

import pandas as pd

ROOT = Path(__file__).resolve().parents[2]
RAW = ROOT / "data" / "regime"          # NON data/raw (solo OHLCV) — evita pollution discovery asset
BASE = "https://www.deribit.com/api/v2/public/"


def _get(method: str, params: dict) -> dict:
    url = BASE + method + "?" + urllib.parse.urlencode(params)
    for _ in range(4):
        try:
            with urllib.request.urlopen(url, timeout=30) as r:
                return json.loads(r.read())
        except Exception:
            time.sleep(1.0)
    return {}


def fetch_dvol(currency: str, start_ms: int, end_ms: int, res: int = 3600) -> pd.DataFrame:
    """DVOL index (OHLC). Cap 1000 righe/chiamata -> chaining all'indietro."""
    rows = []
    cur_end = end_ms
    span = 1000 * res * 1000
    while cur_end > start_ms:
        cur_start = max(start_ms, cur_end - span)
        d = _get("get_volatility_index_data", {
            "currency": currency, "start_timestamp": cur_start,
            "end_timestamp": cur_end, "resolution": res})
        data = (d.get("result") or {}).get("data") or []
        if not data:
            break
        rows.extend(data)
        oldest = min(x[0] for x in data)
        if oldest >= cur_end:
            break
        cur_end = oldest - 1
        time.sleep(0.15)
    if not rows:
        return pd.DataFrame()
    df = pd.DataFrame(rows, columns=["timestamp", "open", "high", "low", "close"])
    df = df.drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
    df["dvol"] = df["close"]
    return df


def fetch_funding(instrument: str, start_ms: int, end_ms: int) -> pd.DataFrame:
    """funding rate history perp (1h). Paginazione ~30g/chiamata."""
    rows = []
    cur_start = start_ms
    step = 30 * 24 * 3600 * 1000
    while cur_start < end_ms:
        cur_end = min(end_ms, cur_start + step)
        d = _get("get_funding_rate_history", {
            "instrument_name": instrument,
            "start_timestamp": cur_start, "end_timestamp": cur_end})
        data = d.get("result") or []
        if data:
            rows.extend(data)
        cur_start = cur_end + 1
        time.sleep(0.12)
    if not rows:
        return pd.DataFrame()
    df = pd.DataFrame(rows)
    ts_col = "timestamp" if "timestamp" in df.columns else df.columns[0]
    df = df.rename(columns={ts_col: "timestamp"})
    keep = [c for c in ("timestamp", "interest_1h", "interest_8h", "index_price", "prev_index_price") if c in df.columns]
    df = df[keep].drop_duplicates("timestamp").sort_values("timestamp").reset_index(drop=True)
    return df


def main():
    RAW.mkdir(parents=True, exist_ok=True)
    now = _get("get_time", {})
    end_ms = int(now.get("result", 0)) or int(time.time() * 1000)
    start_ms = end_ms - int(6.5 * 365 * 24 * 3600 * 1000)   # ~6.5 anni
    for cur, inst in (("BTC", "BTC-PERPETUAL"), ("ETH", "ETH-PERPETUAL")):
        dv = fetch_dvol(cur, start_ms, end_ms)
        if not dv.empty:
            p = RAW / f"{cur.lower()}_dvol.parquet"
            dv.to_parquet(p)
            rng = (pd.to_datetime(dv['timestamp'].min(), unit='ms').date(),
                   pd.to_datetime(dv['timestamp'].max(), unit='ms').date())
            print(f"  {cur} DVOL: {len(dv)} righe {rng[0]}->{rng[1]} (ora={dv['dvol'].iloc[-1]:.1f}) -> {p.name}")
        fr = fetch_funding(inst, start_ms, end_ms)
        if not fr.empty:
            p = RAW / f"{cur.lower()}_funding.parquet"
            fr.to_parquet(p)
            rng = (pd.to_datetime(fr['timestamp'].min(), unit='ms').date(),
                   pd.to_datetime(fr['timestamp'].max(), unit='ms').date())
            print(f"  {cur} FUNDING: {len(fr)} righe {rng[0]}->{rng[1]} -> {p.name}")


if __name__ == "__main__":
    main()