""" Technical trend indicators of Neely, Rapach, Tu and Zhou (2014), monthly, for the S&P 500 and 18 commodities, with a trend uncertainty index. The 14 indicators (1 = buy, 0 = sell) MA(s,l) 1 if the s-month moving average of the price is above the l-month one, s = 1, 2, 3 and l = 9, 12 MOM(m) 1 if the price is at least as high as m months earlier, m = 9, 12 VOL(s,l) 1 if the s-month moving average of on-balance volume is above the l-month one (on-balance volume adds the month's volume when the price did not fall and subtracts it when it fell). S&P 500 only. Data S&P 500 1950:12-2011:12: the indicators published by Neely et al., written at the bottom of this file. From 2012:01: computed the same way from the month-end close and average daily volume of the S&P 500 index (^GSPC, Yahoo Finance). Over 1950:12-2011:12 these public data reproduce the paper's price rules exactly and its volume rules in 99.6% of months. Commodities World Bank Commodity Price Data (the Pink Sheet), monthly average prices in US dollars. The eight price rules only: no public volume. Each series starts in the January after the last year before 1990 in which at least half of its monthly prices were unchanged, so that eras of fixed or posted prices (gold at $35, posted oil prices, regulated US gas) are left out. Summaries for each asset and month trend_uncertainty the trend uncertainty index: the share of the asset's rules on a sell signal (0 = every rule on buy, 1 = every rule on sell). For the S&P 500 it tracks the VIX (correlation 0.62 since 1990). buy_share share of the rules on a buy signal (1 - trend_uncertainty) disagreement secondary: 1 - |2 x buy_share - 1|, 0 when every rule points the same way and 1 when they are split evenly; it peaks at turning points rather than in volatile markets The commodity series are averages across the commodities available that month. Outputs (next to this script) technical_indicators_monthly.csv one row per asset and month: all rules, trend_uncertainty, buy_share, disagreement technical_summary_monthly.csv one row per month: the three summaries for every asset, plus the commodity averages Cite: Neely, C. J., Rapach, D. E., Tu, J., and Zhou, G. (2014). Forecasting the equity risk premium: The role of technical indicators. Management Science 60(7), 1772-1791. World Bank Commodity Price Data (The Pink Sheet). Requires: Python 3.9+, pandas, openpyxl. Run: python update_technical.py """ import datetime as dt import json import re from io import BytesIO from pathlib import Path from urllib.request import Request, urlopen import numpy as np import pandas as pd HERE = Path(__file__).resolve().parent BROWSER = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/126 Safari/537.36"} PAIRS = [(1, 9), (1, 12), (2, 9), (2, 12), (3, 9), (3, 12)] PRICE_RULES = [f"MA({s},{l})" for s, l in PAIRS] + ["MOM(9)", "MOM(12)"] VOLUME_RULES = [f"VOL({s},{l})" for s, l in PAIRS] SPLICE = pd.Period("2012-01", freq="M") # first S&P 500 month computed from public data SUMMARIES = ["trend_uncertainty", "buy_share", "disagreement"] # World Bank column name -> short name used in the outputs COMMODITIES = { "Crude oil, Brent": "brent", "Crude oil, WTI": "wti", "Natural gas, US": "natgas_us", "Gold": "gold", "Silver": "silver", "Platinum": "platinum", "Copper": "copper", "Aluminum": "aluminium", "Nickel": "nickel", "Zinc": "zinc", "Lead": "lead", "Tin": "tin", "Wheat, US HRW": "wheat", "Maize": "maize", "Soybeans": "soybeans", "Coffee, Arabica": "coffee", "Cotton, A Index": "cotton", "Sugar, world": "sugar", } # ---------------------------------------------------------------- rules def _ma_rule(x: pd.Series, s: int, l: int) -> pd.Series: short = x.rolling(s, min_periods=s).mean() long_ = x.rolling(l, min_periods=l).mean() # Averages equal to within rounding are a tie, and a tie is a sell, as in the original # strict comparison (repeated prices in the commodity data make exact ties common). return (short > long_ + 1e-10 * long_.abs()).astype(float).where(long_.notna()) def price_rules(p: pd.Series) -> pd.DataFrame: out = {f"MA({s},{l})": _ma_rule(p, s, l) for s, l in PAIRS} for m in (9, 12): lag = p.shift(m) out[f"MOM({m})"] = (p - lag >= 0).astype(float).where(lag.notna()) return pd.DataFrame(out) def volume_rules(p: pd.Series, vol: pd.Series) -> pd.DataFrame: sign = np.where(p.diff() >= 0, 1.0, -1.0) step = pd.Series(sign * vol.values, index=vol.index) step.iloc[0] = 0.0 obv = step.cumsum() # on-balance volume; its starting level does not affect the rules return pd.DataFrame({f"VOL({s},{l})": _ma_rule(obv, s, l) for s, l in PAIRS}) def summarise(rules: pd.DataFrame) -> pd.DataFrame: share = rules.mean(axis=1, skipna=False) return pd.DataFrame({"trend_uncertainty": 1 - share, "buy_share": share, "disagreement": 1 - (2 * share - 1).abs()}) # ---------------------------------------------------------------- data def sp500_monthly() -> pd.DataFrame: """Month-end close and average daily volume of ^GSPC from Yahoo Finance, complete months only.""" end = int(dt.datetime.now(dt.timezone.utc).timestamp()) url = f"https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC?interval=1d&period1=-631152000&period2={end}" with urlopen(Request(url, headers=BROWSER), timeout=120) as resp: r = json.load(resp)["chart"]["result"][0] q = r["indicators"]["quote"][0] d = pd.DataFrame({"close": q["close"], "volume": q["volume"]}, index=pd.to_datetime(r["timestamp"], unit="s").normalize()).dropna() m = d.groupby(d.index.to_period("M")).agg(P=("close", "last"), vol=("volume", "mean")) if m.index[-1] >= pd.Period(dt.date.today(), freq="M"): m = m.iloc[:-1] # the current month is not over return m def world_bank_monthly() -> pd.DataFrame: """Monthly average prices from the World Bank Pink Sheet (the file's address changes each month).""" page = "https://www.worldbank.org/en/research/commodity-markets" with urlopen(Request(page, headers=BROWSER), timeout=60) as resp: html = resp.read().decode("utf-8", "ignore") link = re.search(r"https?://[^\"']*CMO-Historical-Data-Monthly\.xlsx", html) if not link: raise RuntimeError(f"Could not find the monthly data file on {page}") with urlopen(Request(link.group(0), headers=BROWSER), timeout=120) as resp: raw = pd.read_excel(BytesIO(resp.read()), sheet_name="Monthly Prices", header=None) names = raw.iloc[4].tolist() body = raw.iloc[6:].dropna(subset=[0]) idx = pd.PeriodIndex([pd.Period(str(s).replace("M", "-"), freq="M") for s in body[0]]) out = {short: pd.to_numeric(body[names.index(long_)], errors="coerce").values for long_, short in COMMODITIES.items()} return pd.DataFrame(out, index=idx) def market_price_start(s: pd.Series) -> pd.Period: """January after the last year before 1990 in which at least half of the monthly prices were unchanged from the month before: this drops eras of fixed or posted prices.""" s = s.dropna() unchanged = (s.diff() == 0).groupby(s.index.year).mean() fixed = [y for y, share in unchanged.items() if share >= 0.5 and y < 1990] return pd.Period(f"{max(fixed) + 1}-01", freq="M") if fixed else s.index[0] def paper_indicators() -> pd.DataFrame: """Neely et al. (2014) S&P 500 indicators, 1950:12-2011:12, from the block below.""" rows = {} for line in NEELY_2014.strip().splitlines(): year, *bits = line.split() for m in range(12): if bits[0][m] != "-": rows[pd.Period(f"{year}-{m + 1:02d}", freq="M")] = [float(b[m]) for b in bits] return pd.DataFrame.from_dict(rows, orient="index", columns=PRICE_RULES + VOLUME_RULES) # ---------------------------------------------------------------- build def build() -> tuple[pd.DataFrame, pd.DataFrame]: frames, summary = [], {} sp = sp500_monthly() rules = pd.concat([price_rules(sp["P"]), volume_rules(sp["P"], sp["vol"])], axis=1) paper = paper_indicators() rules = pd.concat([paper, rules.loc[SPLICE:]]).loc[pd.Period("1950-12", freq="M"):] sp_out = pd.concat([rules, summarise(rules)], axis=1) sp_out["source"] = ["neely_2014" if m < SPLICE else "yahoo_gspc" for m in sp_out.index] sp_out.insert(0, "asset", "sp500") frames.append(sp_out) summary["sp500"] = sp_out[SUMMARIES] wb = world_bank_monthly() for name in COMMODITIES.values(): p = wb[name].loc[market_price_start(wb[name]):].dropna() rules = price_rules(p).dropna() out = pd.concat([rules, summarise(rules)], axis=1) out["source"] = "world_bank" out.insert(0, "asset", name) frames.append(out) summary[name] = out[SUMMARIES] long_ = pd.concat(frames).reset_index().rename(columns={"index": "month"}) long_["month"] = long_["month"].astype(str) long_ = long_[["month", "asset"] + PRICE_RULES + VOLUME_RULES + SUMMARIES + ["source"]] wide = pd.concat({k: v for k, v in summary.items()}, axis=1) wide.columns = [f"{a}_{c}" for a, c in wide.columns] names = list(COMMODITIES.values()) wide["commodities_n"] = wide[[f"{n}_disagreement" for n in names]].notna().sum(axis=1) for c in SUMMARIES: wide[f"commodities_{c}"] = wide[[f"{n}_{c}" for n in names]].mean(axis=1) wide = wide.loc[pd.Period("1950-12", freq="M"):].sort_index() wide.index = wide.index.astype(str) wide.index.name = "month" first = [f"sp500_{c}" for c in SUMMARIES] + [f"commodities_{c}" for c in SUMMARIES] + ["commodities_n"] return long_, wide[first + [c for c in wide.columns if c not in first]] def main() -> None: long_, wide = build() long_.to_csv(HERE / "technical_indicators_monthly.csv", index=False, float_format="%.6g") wide.to_csv(HERE / "technical_summary_monthly.csv", float_format="%.6g") sp = wide["sp500_trend_uncertainty"].dropna() print(f"Wrote technical_indicators_monthly.csv ({len(long_):,} rows) and technical_summary_monthly.csv " f"({len(wide)} months, {wide.index[0]} to {wide.index[-1]})") print(f"Trend uncertainty index, {sp.index[-1]}: S&P 500 {sp.iloc[-1]:.2f}, " f"commodities {wide['commodities_trend_uncertainty'].iloc[-1]:.2f}") # Neely, Rapach, Tu and Zhou (2014), S&P 500 technical indicators from their replication # files, 1950:12-2011:12. One row per year; then one 12-character string per indicator # (January to December, "-" before the sample starts) in the order # MA(1,9) MA(1,12) MA(2,9) MA(2,12) MA(3,9) MA(3,12) MOM(9) MOM(12) # VOL(1,9) VOL(1,12) VOL(2,9) VOL(2,12) VOL(3,9) VOL(3,12). NEELY_2014 = """ 1950 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 1951 111110111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111110110000 111111111000 111111011000 111111111000 111111100000 111111111100 1952 111011111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 101011110011 001011111011 000111111001 000001111001 001011111001 000011111101 1953 110000000111 111000000001 111000000011 111000000001 111000000001 111100000001 111000100000 111110000100 100000000111 100000000011 100000000011 100000000001 110000000011 110000000001 1954 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1955 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1956 111101110000 111111110001 111111110000 111111111000 111111111000 111111111000 111111110100 111111111101 001000100001 011100100001 001100110000 001100110000 100110011000 111110010000 1957 000111100000 000011100000 000011110000 000011110000 000011110000 000001110000 000001110000 100011000000 000111110000 000111110000 000111111000 000011111000 000011111000 000011111000 1958 000111111111 000111111111 000111111111 000011111111 000111111111 000011111111 000001111111 000000011111 001111111111 000111111111 000111111111 000111111111 000111111111 000011111111 1959 111111110011 111111111111 111111111001 111111111111 111111111100 111111111111 111111111111 111111111111 111111110111 111111111111 111111111111 111111111111 111111111111 111111111111 1960 000001010011 000000010001 000000000001 000000000001 000000010001 100000000001 000001000001 110000000000 111011110011 111011111011 111001111001 111111111001 111111111101 111111111101 1961 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1962 111000000011 111000000001 111000000001 111000000001 111000000001 111100000000 111000000000 111000000000 111000000011 111100000011 111100000011 111100000011 111100000011 111100000001 1963 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 000111111111 111111110101 111111111101 111111111000 111111111111 111111111100 111111111111 1964 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1965 111110011111 111110011111 111110001111 111111011111 111111001111 111111101111 111110111111 111111111111 110100011111 111100011111 110000001111 111110001111 111100001111 111100001111 1966 110100000000 110100000000 110000000000 111100000000 111000000000 111100000000 111100000000 111101000000 110100000000 111110000000 111100000000 111110000000 111100000000 111110000000 1967 111111111111 111111111111 111111111111 011111111111 111111111111 011111111111 011111111111 001111111111 111111111111 011111111111 011111111111 011111111111 111111111111 011111111111 1968 000111111111 000111111111 100111111111 100111111111 100011111111 100011111111 010111111111 111111111111 101111111111 101111111111 100111111111 111111111111 110111111111 111111111111 1969 100110000000 101110000000 100010000000 110110000000 100000000000 110010000000 101110000000 111110000000 100100000000 111100000000 100100000000 111110000000 110000000000 111110000000 1970 000000001111 000000000011 000000001111 000000000011 000000000111 000000000011 000000000001 000000000001 001000011111 000000001111 000000001111 000000001111 000000001111 000000001111 1971 111111010001 111111111001 111111100000 111111111000 111111100000 111111111100 111111111001 111111111111 111111111000 111111111100 111111111000 111111111100 111111111100 111111111100 1972 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 011111111111 1973 100000000100 110000000000 110000000100 111000000000 111000000000 111000000000 111000000000 111000100000 110000000000 110000000000 110000000000 110000000000 110000000000 111000000000 1974 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 1975 111111110111 011111111111 011111110011 011111111111 011111111001 001111111111 000111111111 000011111111 011111111110 011111111111 011111111011 001111111111 001111111101 001111111111 1976 111111111101 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111 101000000000 101001000000 111100000000 111100000000 111100000000 111100000000 1977 000000000000 000000000000 100000000000 100000000000 100000000000 110000000000 100000000000 110000000000 000000000001 000000000000 000000000000 000000000000 000000000000 000000000000 1978 000111111000 000111111001 000011111100 000011111100 000011111100 000001111110 000010111111 000010111101 001111111011 000111111111 000111111101 000111111111 000111111110 000011111111 1979 101111111011 101111111111 011111111111 111111111111 001111111111 111111111111 101101111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1980 110011111111 110011111111 111011111111 111011111111 111001111111 111101111111 110111111111 111111111111 110111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1981 111110000000 111111000000 111110000000 111111000000 111110000000 111111100000 111111100000 111111110000 111100000000 111100000000 111100000000 111110000000 111110000000 111110000000 1982 000000011111 000000011111 000000001111 000000001111 000000001111 000000000111 000000000111 000000001111 000000001111 000000000111 000000001111 000000000111 000000000111 000000000111 1983 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111110 111111111111 111111111111 111111111111 111111111111 111111111111 1984 000000011111 100000011111 000000011111 100000001111 100000001111 100000001111 000000011111 111000011101 000001010000 000101010000 000000011000 000000001000 100000011100 100000000000 1985 111111110111 111111111111 111111111011 111111111111 111111111111 111111111111 111111111111 111111111111 111011100011 111011100011 011111100011 011111110011 011111110001 011111110001 1986 111111110110 111111111111 111111111011 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1987 111111111000 111111111000 111111111000 111111111000 111111111100 111111111100 111111111000 111111111101 111111111000 111111111101 111111111100 111111111100 111111111100 111111111110 1988 000011101111 000001101111 000001111111 000000111111 000001111111 000000011111 000000111111 000000000111 111111111111 111111111111 011111111111 111111111111 011111111111 011111111111 1989 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111011 111111111111 111111111101 111111111111 111111111111 111111111111 1990 000011100000 101011100000 100011100000 100011100000 100011110000 110011110000 111011100000 111111100000 011111000000 111111000000 111111000000 111111000000 111111100000 111111100000 1991 111111111101 111111111111 111111111111 111111111111 011111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 011111111111 011111111111 1992 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 010111111111 110111111111 111111111111 111111111111 110111111111 111111111111 1993 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111 1994 110000011100 110000011100 110000011110 110000011110 111000001111 111000001111 110100010001 110110111100 110110111111 110111111111 110011111111 110011111111 111011111111 111011111111 1995 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1996 111111011111 111111111111 111111101111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1997 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 1998 111111100111 111111100111 111111100011 111111110111 111111110011 111111110011 111111111111 111111111111 111111101111 111111111111 111111110111 111111111111 111111110011 111111111111 1999 111111110111 111111111111 111111110111 111111111111 111111111111 111111111111 111111111111 111111111111 111111100011 111111100011 111111100001 111111110011 111111110001 111111110001 2000 111111110000 111111111000 111111111000 111111111000 111111111100 111111111100 111111110100 111111111100 101000000000 101000000000 100000000000 100000000000 110000000000 110000000000 2001 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 100011000001 100010000001 010011000001 000011000001 000001100000 000001000000 2002 001000000000 000000000000 001100000000 000000000000 001100000000 000000000000 000000000000 000000000000 001000000010 000000000000 100000000001 100000000000 110000000001 100000000000 2003 000111111111 000111111111 000111111111 000011111111 000011111111 000011111111 000111111111 000000111111 000111111111 000111111111 000111111111 000011111111 100011111111 000011111111 2004 111111000111 111111111111 111111100111 111111111111 111111100011 111111110111 111111111011 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 2005 111011111111 111111111111 111111111111 111111111111 111110111111 111111111111 111111111111 111111111111 111000001010 111010101010 111000000001 111110000000 111100000010 111110000000 2006 111111111111 111111111111 111111011111 111111111111 111111011111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 2007 111111111100 111111111110 111111111110 111111111110 111111111111 111111111111 111111111111 111111111111 111111011110 111111111110 111111111110 111111111111 111111111111 111111111111 2008 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 2009 000011111111 000000111111 000001111111 000000111111 000001111111 000000111111 000000111111 000000000111 000111111111 000011111111 000011111111 000011111111 000011111111 000001111111 2010 111100001111 111110101111 111110000111 111110000111 111111000111 111111000111 111110101111 111111111111 111110101101 111111101111 111111000111 111111111111 111111100011 111111111111 2011 111111000000 111111100000 111111100000 111111100000 111111100000 111111110000 111111110000 111111110110 111110000000 111110000000 111110000000 111111000000 111111000000 111111000000 """ if __name__ == "__main__": main()