"""
Stage 1.5: 加新维度——资金费率 + Taker买卖比
Binance Vision 公开数据，免费
"""
import pandas as pd, numpy as np, requests, io, zipfile, time

# ── 1. K线 + Taker ──
print("📡 K线 + Taker...")
cols = ["open_time","open","high","low","close","volume",
        "close_time","quote_vol","trades","taker_buy_base","taker_buy_quote","ignore"]
kline_dfs = []
months = [f"2025-{m:02d}" for m in range(7,13)] + [f"2026-{m:02d}" for m in range(1,7)]

for m in months:
    url = f"https://data.binance.vision/data/futures/um/monthly/klines/BTCUSDT/15m/BTCUSDT-15m-{m}.zip"
    r = requests.get(url, timeout=30)
    if r.status_code != 200: continue
    with zipfile.ZipFile(io.BytesIO(r.content)) as z:
        for name in z.namelist():
            if not name.endswith(".csv"): continue
            df = pd.read_csv(z.open(name), header=None)
            df.columns = cols[:len(df.columns)]
            df = df[pd.to_numeric(df["open_time"], errors="coerce").notna()]
            df["ts"] = pd.to_datetime(df["open_time"].astype(float), unit="ms")
            for c in ["open","high","low","close","volume","taker_buy_base","taker_buy_quote"]:
                df[c] = pd.to_numeric(df[c], errors="coerce")
            df["taker_buy_pct"] = df["taker_buy_base"] / (df["volume"].replace(0, np.nan))
            kline_dfs.append(df[["ts","open","high","low","close","volume","taker_buy_pct"]])
    time.sleep(0.2)

klines = pd.concat(kline_dfs).drop_duplicates("ts").sort_values("ts").set_index("ts")
print(f"  {len(klines)} candles")

# ── 2. 资金费率 ──
print("📡 资金费率...")
fr_dfs = []
for m in months:
    url = f"https://data.binance.vision/data/futures/um/monthly/fundingRate/BTCUSDT/BTCUSDT-fundingRate-{m}.zip"
    r = requests.get(url, timeout=30)
    if r.status_code != 200: continue
    with zipfile.ZipFile(io.BytesIO(r.content)) as z:
        for name in z.namelist():
            if not name.endswith(".csv"): continue
            df = pd.read_csv(z.open(name))  # has header: calc_time,funding_interval_hours,last_funding_rate
            df["ts"] = pd.to_datetime(df["calc_time"].astype(float), unit="ms")
            df["funding_rate"] = pd.to_numeric(df["last_funding_rate"], errors="coerce")
            fr_dfs.append(df[["ts","funding_rate"]])

fr = pd.concat(fr_dfs).drop_duplicates("ts").set_index("ts").sort_index()
# Resample 到 15min，前向填充
fr = fr.resample("15min").ffill()
print(f"  {len(fr)} intervals")

# ── 3. 合并 + 因子 ──
print("🔗 合并...")
merged = klines.join(fr, how="left")
merged["funding_rate"] = merged["funding_rate"].ffill()

# 新因子
merged["fr_level"] = merged["funding_rate"]           # 资金费率水平（正=多头过热）
merged["fr_chg"] = merged["funding_rate"].diff(20)    # 费率变化方向
merged["taker_pct"] = merged["taker_buy_pct"].clip(0, 1)  # 主动买占比
merged["taker_ma"] = merged["taker_pct"].rolling(20).mean()  # 主动买均线
merged["taker_dev"] = merged["taker_pct"] - merged["taker_ma"]  # 偏离

# 原价量因子
merged["ret_5"] = merged["close"].pct_change(5)
merged["ret_20"] = merged["close"].pct_change(20)
merged["ma_dev"] = merged["close"] / merged["close"].rolling(20).mean() - 1

# 标签
merged["fwd_ret"] = merged["close"].shift(-4) / merged["close"] - 1

merged = merged.dropna()
merged.to_parquet("data/btc_multidim.parquet")

# ── 报告 ──
print(f"\n✅ {len(merged)} candles")
print(f"   {merged.index.min()} → {merged.index.max()}")
print(f"   新维度: 资金费率(水平+变化), Taker主动买卖比(偏离)")
X_new = ["ret_5","ret_20","ma_dev","fr_level","fr_chg","taker_pct","taker_dev"]
print(f"   因子: {X_new}")
print(f"   NaN: {merged.isna().sum().sum()}")