"""Beta-neutralization experiment --- separate factor alpha from market exposure.
Motivation
----------
Every long-only monetization (ew_top50 / rank_all / zpos) of the surviving
skewness signal carries full market beta: max drawdowns run -48%..-68%
because the books ride the underlying index through 2015/2018/2024. The
market cycle dwarfs the few %/yr of factor alpha, so "excess vs EW" is
dominated by beta mismatch rather than factor P&L (see docs/research_findings.md).
Two separable questions, both labelled explicitly:
Q1 feature-level (cross-sectional) neutralization
Before ranking, strip the systematic beta tilt out of the composite:
z_tilde_i = z_i - (a + b_t * beta_i) (OLS residual per date)
with beta_i = 60d trailing CAPM beta of stock i vs the equal-weight
universe. Nothing about the ranking mechanics changes; only the score
fed to it is orthogonalized. Reported as the zpos book's ex-ante beta
tilt before/after (should go ~0 by construction) and the resulting
zpos/rank_all/zls metrics.
Q2 portfolio-level hedge (market-neutrality)
Hold the long-only book and short beta_book x benchmark:
r_hedged_t = r_book_t - beta_hedge_t * r_bm_t
Two beta_hedge estimators are reported:
* ex-post : full-sample regression of the book on the benchmark
(a hindsight diagnostic = the frictionless-hedge ceiling)
* causal : 60d trailing beta, shifted so day t uses data <= t-1
(estimable in real time)
A hedged book has beta ~ 0, so IR-vs-benchmark is a category error: we
report annualized return, ann vol, absolute Sharpe and max drawdown.
Conventions (entry-only cost, halt renormalization by absolute weight mass,
last-day gross = 0) are mirrored from BaseEngine / ic_weighted_test.evaluate
and NOT re-derived. ic_weighted_test.py stays frozen: its JSONs are the
reference set of docs/research_findings.md §3/§5.
"""
import argparse
import json
import math
import os
import sys
import numpy as np
import pandas as pd
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import ic_weighted_test as base
from mfm.combine import combine_factors_icir_weight
from mfm.pipelines.csi300_screen import (
CSI300ScreenConfig,
build_rolling_weights,
load_universe,
prepare_context,
validate_and_screen,
)
BETA_WINDOW = 60 # trailing window for per-stock CAPM beta
BETA_MINP = 30
HEDGE_WINDOW = 60 # trailing window for the book's realized beta
HEDGE_MINP = 30
COMPOSITES = {
"skew": ["skewness"],
"core3": ["skewness", "vol_20d", "vol_60d"],
}
def compute_market_beta(returns, window=BETA_WINDOW, minp=BETA_MINP):
"""60d trailing CAPM beta of each stock vs the equal-weight universe."""
r_mkt = returns.mean(axis=1)
var = r_mkt.rolling(window, min_periods=minp).var()
cov = returns.rolling(window, min_periods=minp).cov(r_mkt)
return cov.div(var, axis=0)
def neutralize(z, beta, min_obs=12):
"""Per-date OLS: keep the residual of z on beta (feature-level neutrality)."""
resid = pd.DataFrame(np.nan, index=z.index, columns=z.columns)
cols = z.columns
for t in z.index:
x = beta.loc[t].reindex(cols)
y = z.loc[t].reindex(cols)
m = x.notna() & y.notna()
nm = int(m.sum())
if nm < min_obs:
continue
xm = x[m].to_numpy(dtype=float)
ym = y[m].to_numpy(dtype=float)
A = np.column_stack([np.ones(nm), xm])
coef, *_ = np.linalg.lstsq(A, ym, rcond=None)
resid.loc[t, cols[m]] = ym - A @ coef
return resid
def build_scheme_returns(comp, ret_next, rebal_dates, cfg, scheme):
"""Net daily return series for ONE scheme.
Mirrors ic_weighted_test.evaluate's inner loop verbatim for `scheme`
(zpos / rank_all), returning the (gross - cost) net Series so it can be
hedged at the portfolio level.
"""
idx = comp.index
all_cols = ret_next.columns
w_rows, costs = {}, {}
prev_support, prev_w = None, None
for T in rebal_dates:
if T not in comp.index:
continue
w = base.scheme_weights(comp.loc[T], scheme, cfg.n_hold).reindex(all_cols).fillna(0.0)
w_rows[T] = w
support = w.index[w != 0.0]
if prev_support is None:
notional = float(w.abs().sum()) # full cost on initial entry
else:
entered = support.difference(prev_support)
notional = float(w.loc[entered].abs().sum()) if len(entered) else 0.0
costs[T] = cfg.cost * notional
prev_support, prev_w = support, w
W = pd.DataFrame(w_rows).T.reindex(idx).ffill().fillna(0.0)
valid = ret_next.notna() & (W != 0.0)
w_valid = W.where(valid, 0.0)
mass = w_valid.abs().sum(axis=1)
numer = (w_valid * ret_next.fillna(0.0)).sum(axis=1)
gross = (numer / mass.where(mass != 0.0)).fillna(0.0)
gross.iloc[-1] = 0.0 # engine last-day convention
cost_s = pd.Series(0.0, index=idx)
for T, c in costs.items():
cost_s.loc[T] = c
return gross - cost_s
def exante_beta_tilt(comp, beta, rebal_dates, cfg, scheme):
"""Weighted-avg 60d beta of the book's target weights (rebalance-time)."""
tilts = []
for T in rebal_dates:
if T not in comp.index:
continue
w = base.scheme_weights(comp.loc[T], scheme, cfg.n_hold)
b = beta.loc[T]
common = w.index.intersection(b.dropna().index)
if len(common) == 0:
continue
num = float((w.loc[common] * b.loc[common]).sum())
den = float(w.loc[common].abs().sum())
if den > 0:
tilts.append(num / den)
return float(np.mean(tilts)) if tilts else float("nan")
def realized_beta(ret, bm):
"""Full-sample regression beta of a book on the benchmark (ex-post)."""
a = ret - ret.mean()
b = bm - bm.mean()
denom = float((b * b).sum())
return float((a * b).sum() / denom) if denom > 0 else 0.0
def rolling_hedge_beta(book, bm, window=HEDGE_WINDOW, minp=HEDGE_MINP):
"""Causal 60d book beta; day t uses data <= t-1 (shifted)."""
var = bm.rolling(window, min_periods=minp).var()
cov = book.rolling(window, min_periods=minp).cov(bm)
return (cov / var).shift(1)
def hedge(book, bm, beta_hedge):
"""r_book - beta_hedge * r_bm (beta_hedge: float or aligned Series)."""
bm_a = bm.reindex(book.index).fillna(0.0)
if isinstance(beta_hedge, pd.Series):
bh = beta_hedge.reindex(book.index).fillna(0.0)
else:
bh = beta_hedge
return book - bh * bm_a
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--tag", required=True)
ap.add_argument("--start", required=True)
ap.add_argument("--end", required=True)
ap.add_argument("--instruments", default="data/cn_data/instruments/csi300.txt")
ap.add_argument("--outdir", default="./output_btn")
args = ap.parse_args()
os.makedirs(args.outdir, exist_ok=True)
cfg = CSI300ScreenConfig(
start=args.start, end=args.end,
instruments_path=args.instruments, out_dir=args.outdir,
)
print(f"[1/4] Loading & validating ({args.tag})...")
prices, returns, mask, uf = load_universe(cfg)
screen = validate_and_screen(uf, prices, cfg)
print("[2/4] Rolling weights (shared OOS window)...")
roll_icir, roll_blend, oos_idx = build_rolling_weights(screen, prices.index, cfg)
ctx = prepare_context(prices, returns, mask, uf, roll_icir, roll_blend, oos_idx, cfg)
print("[3/4] Market beta (60d trailing vs EW universe)...")
beta_all = compute_market_beta(returns).loc[oos_idx]
ret_next = returns.shift(-1).loc[oos_idx]
bm = ctx.benchmark
zero = pd.Series(0.0, index=bm.index)
out = {
"tag": args.tag,
"oos_window": [str(oos_idx[0].date()), str(oos_idx[-1].date())],
"oos_days": int(len(oos_idx)),
"beta_window": BETA_WINDOW,
"hedge_window": HEDGE_WINDOW,
"composites": {},
}
print("[4/4] Neutralization & hedge...")
for cname, factors in COMPOSITES.items():
w = {
f: base.FIXED_SIGNS.get(f, 1.0 if screen.icir[f] >= 0 else -1.0)
for f in factors
}
comp_raw = combine_factors_icir_weight(ctx.norm_all, w).where(ctx.mask_oos)
comp_neu = neutralize(comp_raw, beta_all).where(ctx.mask_oos)
cell = {"weights": w, "schemes_raw": None, "schemes_neutral": None, "hedge": {}}
cell["schemes_raw"] = base.evaluate(comp_raw, ret_next, ctx.rebal_dates, bm, cfg)
cell["schemes_neutral"] = base.evaluate(comp_neu, ret_next, ctx.rebal_dates, bm, cfg)
tilt_raw = exante_beta_tilt(comp_raw, beta_all, ctx.rebal_dates, cfg, "zpos")
tilt_neu = exante_beta_tilt(comp_neu, beta_all, ctx.rebal_dates, cfg, "zpos")
cell["zpos_exante_beta_tilt"] = {"raw": tilt_raw, "neutral": tilt_neu}
book_raw = build_scheme_returns(comp_raw, ret_next, ctx.rebal_dates, cfg, "zpos")
book_neu = build_scheme_returns(comp_neu, ret_next, ctx.rebal_dates, cfg, "zpos")
for bk_name, book in (("raw", book_raw), ("neutral", book_neu)):
bep = realized_beta(book, bm)
broll = rolling_hedge_beta(book, bm)
cell["hedge"][bk_name] = {
"book_beta_expost": bep,
"unhedged": base.calc_metrics(book, zero),
"hedged_expost": base.calc_metrics(hedge(book, bm, bep), zero),
"hedged_causal": base.calc_metrics(hedge(book, bm, broll), zero),
}
out["composites"][cname] = cell
wtxt = ", ".join(f"{k2}:{v2:+.0f}" for k2, v2 in w.items())
print(f"\n=== [{cname}] w=({wtxt}) zpos ex-ante beta tilt: "
f"raw={tilt_raw:+.2f} neutral={tilt_neu:+.2f}")
print(" -- feature-level (Q1): raw vs neutralized composite --")
for name in ("ew_top50", "rank_all", "zpos", "zls"):
mr = cell["schemes_raw"][name]
mn = cell["schemes_neutral"][name]
print(f" {name:9s} raw IR={mr['information_ratio']:+.2f} "
f"ann={mr['annual_return']:+.2%} maxDD={mr['max_drawdown']:.1%}"
f" | neu IR={mn['information_ratio']:+.2f} "
f"ann={mn['annual_return']:+.2%} maxDD={mn['max_drawdown']:.1%}")
print(" -- portfolio hedge (Q2): zpos unhedged vs ex-post vs causal --")
for bk_name in ("raw", "neutral"):
h = cell["hedge"][bk_name]
u, ex, cx = h["unhedged"], h["hedged_expost"], h["hedged_causal"]
vol_u = book_raw.std() * math.sqrt(252) if bk_name == "raw" \
else book_neu.std() * math.sqrt(252)
print(f" [{bk_name}] book beta(ex-post)={h['book_beta_expost']:+.2f} "
f"unhedged vol={vol_u:.1%}")
print(f" unhedged ann={u['annual_return']:+.2%} "
f"sharpe={u['sharpe_ratio']:+.2f} maxDD={u['max_drawdown']:.1%}")
print(f" hedged_ex ann={ex['annual_return']:+.2%} "
f"sharpe={ex['sharpe_ratio']:+.2f} maxDD={ex['max_drawdown']:.1%}")
print(f" hedged_caus ann={cx['annual_return']:+.2%} "
f"sharpe={cx['sharpe_ratio']:+.2f} maxDD={cx['max_drawdown']:.1%}")
path = os.path.join(args.outdir, f"btn_{args.tag}.json")
with open(path, "w") as fh:
json.dump(out, fh, indent=1, ensure_ascii=False)
print(f"\nSaved -> {path}")
if __name__ == "__main__":
main()
倍benchmark。