%% Technical trend indicators (Neely, Rapach, Tu and Zhou, 2014), monthly
%
% The 14 indicators for the S&P 500 and the 8 price-based ones for 18
% commodities, with the trend uncertainty index (the share of rules on a
% sell signal) and trend disagreement. Same method and output as
% update_technical.py; see that file's header for the definitions.
%
%   S&P 500      1950:12-2011:12 the indicators published by Neely et al.
%                (written at the bottom of this file); from 2012:01 computed
%                from the month-end close and average daily volume of ^GSPC
%                (Yahoo Finance).
%   Commodities  World Bank Pink Sheet monthly average prices. Each series
%                starts in the January after the last year before 1990 in
%                which at least half of its monthly prices were unchanged.
%
% Outputs: technical_indicators_monthly.csv and technical_summary_monthly.csv
%
% Cite: Neely, C. J., Rapach, D. E., Tu, J., and Zhou, G. (2014). Forecasting
% the equity risk premium: The role of technical indicators. Management
% Science 60(7), 1772-1791. World Bank Commodity Price Data (The Pink Sheet).
%
% Requires MATLAB R2019b or later.

clear; clc;
RULES = {'MA(1,9)','MA(1,12)','MA(2,9)','MA(2,12)','MA(3,9)','MA(3,12)','MOM(9)','MOM(12)', ...
         'VOL(1,9)','VOL(1,12)','VOL(2,9)','VOL(2,12)','VOL(3,9)','VOL(3,12)'};
WB = {'Crude oil, Brent','brent'; 'Crude oil, WTI','wti'; 'Natural gas, US','natgas_us';
      'Gold','gold'; 'Silver','silver'; 'Platinum','platinum';
      'Copper','copper'; 'Aluminum','aluminium'; 'Nickel','nickel'; 'Zinc','zinc'; 'Lead','lead'; 'Tin','tin';
      'Wheat, US HRW','wheat'; 'Maize','maize'; 'Soybeans','soybeans';
      'Coffee, Arabica','coffee'; 'Cotton, A Index','cotton'; 'Sugar, world','sugar'};
browser = weboptions('Timeout', 120, 'UserAgent', ...
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36');

%% S&P 500 from Yahoo Finance: month-end close, average daily volume
url = sprintf('https://query1.finance.yahoo.com/v8/finance/chart/%%5EGSPC?interval=1d&period1=-631152000&period2=%d', ...
    floor(posixtime(datetime('now', 'TimeZone', 'UTC'))));
J = webread(url, browser);
r = J.chart.result(1);
q = r.indicators.quote(1);
close = cleannum(q.close);  volume = cleannum(q.volume);
day = dateshift(datetime(r.timestamp, 'ConvertFrom', 'posixtime'), 'start', 'day');
ok = ~isnan(close) & ~isnan(volume);
day = day(ok); close = close(ok); volume = volume(ok);
mkey = year(day) * 100 + month(day);
[ym, ~, g] = unique(mkey);
P = accumarray(g, close, [], @(v) v(end));
V = accumarray(g, volume, [], @mean);
nowkey = year(datetime('today')) * 100 + month(datetime('today'));
keep = ym < nowkey;                                   % drop the month in progress
ym = ym(keep); P = P(keep); V = V(keep);

S = [price_rules(P) volume_rules(P, V)];
paper = neely_2014();                                 % [yyyymm, 14 rules], 1950:12-2011:12
spYm = [paper(:,1); ym(ym >= 201201)];
spRules = [paper(:,2:end); S(ym >= 201201, :)];
spSource = [repmat("neely_2014", size(paper,1), 1); repmat("yahoo_gspc", sum(ym >= 201201), 1)];

%% Commodities from the World Bank Pink Sheet (the file's address changes each month)
page = webread('https://www.worldbank.org/en/research/commodity-markets', browser);
link = regexp(page, 'https?://[^"'']*CMO-Historical-Data-Monthly\.xlsx', 'match', 'once');
file = [tempname '.xlsx'];
websave(file, link, browser);
C = readcell(file, 'Sheet', 'Monthly Prices');
delete(file);
names = C(5, :);
body = C(7:end, :);
body = body(cellfun(@(x) ischar(x) || isstring(x), body(:,1)), :);
wbYm = cellfun(@(s) str2double(s(1:4)) * 100 + str2double(s(6:end)), body(:,1));

%% Build both outputs
assets = ["sp500"; string(WB(:,2))];
allYm = unique([spYm; wbYm]);
allYm = allYm(allYm >= 195012);
buy = nan(numel(allYm), numel(assets));  dis = buy;  tu = buy;
L = table();                                         % long output

[share, d] = summarise(spRules);
L = [L; long_rows(spYm, "sp500", spRules, share, d, spSource)];
[~, loc] = ismember(spYm, allYm);  buy(loc, 1) = share;  dis(loc, 1) = d;  tu(loc, 1) = 1 - share;

for k = 1:size(WB, 1)
    col = find(strcmp(names, WB{k,1}), 1);
    p = cellfun(@cellnum, body(:, col));
    first = market_price_start(wbYm, p);
    sel = wbYm >= first & ~isnan(p);
    yk = wbYm(sel);  pk = p(sel);
    R = price_rules(pk);
    full = all(~isnan(R), 2);
    yk = yk(full);  R = R(full, :);
    [share, d] = summarise(R);
    L = [L; long_rows(yk, string(WB{k,2}), [R nan(size(R,1), 6)], share, d, repmat("world_bank", numel(yk), 1))];
    [~, loc] = ismember(yk, allYm);  buy(loc, k+1) = share;  dis(loc, k+1) = d;  tu(loc, k+1) = 1 - share;
end

L.Properties.VariableNames = [{'month','asset'} RULES {'trend_uncertainty','buy_share','disagreement','source'}];
writetable(L, 'technical_indicators_monthly.csv');

com = 2:numel(assets);
W = table(string(datestr(datetime(floor(allYm/100), mod(allYm,100), 1), 'yyyy-mm')), ...
    tu(:,1), buy(:,1), dis(:,1), mean(tu(:,com), 2, 'omitnan'), mean(buy(:,com), 2, 'omitnan'), ...
    mean(dis(:,com), 2, 'omitnan'), sum(~isnan(dis(:,com)), 2), ...
    'VariableNames', {'month','sp500_trend_uncertainty','sp500_buy_share','sp500_disagreement', ...
    'commodities_trend_uncertainty','commodities_buy_share','commodities_disagreement','commodities_n'});
for k = com
    W.(assets(k) + "_trend_uncertainty") = tu(:,k);
    W.(assets(k) + "_buy_share") = buy(:,k);
    W.(assets(k) + "_disagreement") = dis(:,k);
end
writetable(W, 'technical_summary_monthly.csv');
fprintf('Wrote technical_indicators_monthly.csv (%d rows) and technical_summary_monthly.csv (%d months, %s to %s)\n', ...
    height(L), height(W), W.month(1), W.month(end));

% Chart: the S&P 500 trend uncertainty index (share of its 14 rules on sell)
figure('Color', 'w');
plot(datetime(floor(allYm/100), mod(allYm,100), 1), tu(:,1), 'LineWidth', 1.2);
ylabel('S&P 500 trend uncertainty index'); grid on; ylim([0 1]);

%% ---------------------------------------------------------------- functions
function S = price_rules(p)
% Six moving-average rules and two momentum rules (1 = buy).
    pairs = [1 9; 1 12; 2 9; 2 12; 3 9; 3 12];
    S = nan(numel(p), 8);
    for i = 1:6
        S(:, i) = ma_rule(p, pairs(i,1), pairs(i,2));
    end
    m = [9 12];
    for i = 1:2
        lagged = [nan(m(i), 1); p(1:end-m(i))];
        s = double(p - lagged >= 0);  s(isnan(lagged)) = NaN;
        S(:, 6+i) = s;
    end
end

function S = volume_rules(p, v)
% Six on-balance-volume rules; OBV starts at 0 in the first month.
    sgn = [1; 2 * (diff(p) >= 0) - 1];
    step = sgn .* v;  step(1) = 0;
    obv = cumsum(step);
    pairs = [1 9; 1 12; 2 9; 2 12; 3 9; 3 12];
    S = nan(numel(p), 6);
    for i = 1:6
        S(:, i) = ma_rule(obv, pairs(i,1), pairs(i,2));
    end
end

function s = ma_rule(x, sh, lg)
    short = movmean(x, [sh-1 0], 'Endpoints', 'fill');
    long = movmean(x, [lg-1 0], 'Endpoints', 'fill');
    % Averages equal to within rounding are a tie, and a tie is a sell (strict comparison).
    s = double(short > long + 1e-10 * abs(long));  s(isnan(long)) = NaN;
end

function [share, d] = summarise(R)
    share = mean(R, 2);                       % NaN if any rule is missing
    d = 1 - abs(2 * share - 1);
end

function first = market_price_start(ym, p)
% January after the last year before 1990 in which at least half of the
% monthly prices were unchanged from the month before.
    sel = ~isnan(p);  ym = ym(sel);  p = p(sel);
    unchanged = [false; diff(p) == 0];
    yrs = floor(ym / 100);
    fixed = [];
    for y = unique(yrs)'
        if y < 1990 && mean(unchanged(yrs == y)) >= 0.5, fixed(end+1) = y; end %#ok<AGROW>
    end
    if isempty(fixed), first = ym(1); else, first = (max(fixed) + 1) * 100 + 1; end
end

function T = long_rows(ym, asset, R, share, d, source)
    month = string(datestr(datetime(floor(ym/100), mod(ym,100), 1), 'yyyy-mm'));
    tu = 1 - share;
    T = [table(month, repmat(asset, numel(ym), 1)) array2table(R) table(tu, share, d, source)];
end

function x = cleannum(c)
% Yahoo returns null as [] inside a cell array, or a plain numeric vector.
    if iscell(c), c(cellfun(@isempty, c)) = {NaN}; x = cell2mat(c); else, x = double(c); end
    x = x(:);
end

function x = cellnum(c)
    if isnumeric(c) && isscalar(c), x = double(c); else, x = NaN; end
end

function M = neely_2014()
% Neely, Rapach, Tu and Zhou (2014) S&P 500 indicators from their replication
% files, 1950:12-2011:12. One row per year: the year, then one 12-character
% string per indicator (January to December, "-" before the sample starts).
    B = {
        '1950 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1 -----------1'
        '1951 111110111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111110110000 111111111000 111111011000 111111111000 111111100000 111111111100'
        '1952 111011111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 101011110011 001011111011 000111111001 000001111001 001011111001 000011111101'
        '1953 110000000111 111000000001 111000000011 111000000001 111000000001 111100000001 111000100000 111110000100 100000000111 100000000011 100000000011 100000000001 110000000011 110000000001'
        '1954 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1955 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1956 111101110000 111111110001 111111110000 111111111000 111111111000 111111111000 111111110100 111111111101 001000100001 011100100001 001100110000 001100110000 100110011000 111110010000'
        '1957 000111100000 000011100000 000011110000 000011110000 000011110000 000001110000 000001110000 100011000000 000111110000 000111110000 000111111000 000011111000 000011111000 000011111000'
        '1958 000111111111 000111111111 000111111111 000011111111 000111111111 000011111111 000001111111 000000011111 001111111111 000111111111 000111111111 000111111111 000111111111 000011111111'
        '1959 111111110011 111111111111 111111111001 111111111111 111111111100 111111111111 111111111111 111111111111 111111110111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1960 000001010011 000000010001 000000000001 000000000001 000000010001 100000000001 000001000001 110000000000 111011110011 111011111011 111001111001 111111111001 111111111101 111111111101'
        '1961 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1962 111000000011 111000000001 111000000001 111000000001 111000000001 111100000000 111000000000 111000000000 111000000011 111100000011 111100000011 111100000011 111100000011 111100000001'
        '1963 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 000111111111 111111110101 111111111101 111111111000 111111111111 111111111100 111111111111'
        '1964 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1965 111110011111 111110011111 111110001111 111111011111 111111001111 111111101111 111110111111 111111111111 110100011111 111100011111 110000001111 111110001111 111100001111 111100001111'
        '1966 110100000000 110100000000 110000000000 111100000000 111000000000 111100000000 111100000000 111101000000 110100000000 111110000000 111100000000 111110000000 111100000000 111110000000'
        '1967 111111111111 111111111111 111111111111 011111111111 111111111111 011111111111 011111111111 001111111111 111111111111 011111111111 011111111111 011111111111 111111111111 011111111111'
        '1968 000111111111 000111111111 100111111111 100111111111 100011111111 100011111111 010111111111 111111111111 101111111111 101111111111 100111111111 111111111111 110111111111 111111111111'
        '1969 100110000000 101110000000 100010000000 110110000000 100000000000 110010000000 101110000000 111110000000 100100000000 111100000000 100100000000 111110000000 110000000000 111110000000'
        '1970 000000001111 000000000011 000000001111 000000000011 000000000111 000000000011 000000000001 000000000001 001000011111 000000001111 000000001111 000000001111 000000001111 000000001111'
        '1971 111111010001 111111111001 111111100000 111111111000 111111100000 111111111100 111111111001 111111111111 111111111000 111111111100 111111111000 111111111100 111111111100 111111111100'
        '1972 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 011111111111'
        '1973 100000000100 110000000000 110000000100 111000000000 111000000000 111000000000 111000000000 111000100000 110000000000 110000000000 110000000000 110000000000 110000000000 111000000000'
        '1974 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000'
        '1975 111111110111 011111111111 011111110011 011111111111 011111111001 001111111111 000111111111 000011111111 011111111110 011111111111 011111111011 001111111111 001111111101 001111111111'
        '1976 111111111101 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111 101000000000 101001000000 111100000000 111100000000 111100000000 111100000000'
        '1977 000000000000 000000000000 100000000000 100000000000 100000000000 110000000000 100000000000 110000000000 000000000001 000000000000 000000000000 000000000000 000000000000 000000000000'
        '1978 000111111000 000111111001 000011111100 000011111100 000011111100 000001111110 000010111111 000010111101 001111111011 000111111111 000111111101 000111111111 000111111110 000011111111'
        '1979 101111111011 101111111111 011111111111 111111111111 001111111111 111111111111 101101111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1980 110011111111 110011111111 111011111111 111011111111 111001111111 111101111111 110111111111 111111111111 110111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1981 111110000000 111111000000 111110000000 111111000000 111110000000 111111100000 111111100000 111111110000 111100000000 111100000000 111100000000 111110000000 111110000000 111110000000'
        '1982 000000011111 000000011111 000000001111 000000001111 000000001111 000000000111 000000000111 000000001111 000000001111 000000000111 000000001111 000000000111 000000000111 000000000111'
        '1983 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111110 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1984 000000011111 100000011111 000000011111 100000001111 100000001111 100000001111 000000011111 111000011101 000001010000 000101010000 000000011000 000000001000 100000011100 100000000000'
        '1985 111111110111 111111111111 111111111011 111111111111 111111111111 111111111111 111111111111 111111111111 111011100011 111011100011 011111100011 011111110011 011111110001 011111110001'
        '1986 111111110110 111111111111 111111111011 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1987 111111111000 111111111000 111111111000 111111111000 111111111100 111111111100 111111111000 111111111101 111111111000 111111111101 111111111100 111111111100 111111111100 111111111110'
        '1988 000011101111 000001101111 000001111111 000000111111 000001111111 000000011111 000000111111 000000000111 111111111111 111111111111 011111111111 111111111111 011111111111 011111111111'
        '1989 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111011 111111111111 111111111101 111111111111 111111111111 111111111111'
        '1990 000011100000 101011100000 100011100000 100011100000 100011110000 110011110000 111011100000 111111100000 011111000000 111111000000 111111000000 111111000000 111111100000 111111100000'
        '1991 111111111101 111111111111 111111111111 111111111111 011111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 011111111111 011111111111'
        '1992 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 010111111111 110111111111 111111111111 111111111111 110111111111 111111111111'
        '1993 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111101 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1994 110000011100 110000011100 110000011110 110000011110 111000001111 111000001111 110100010001 110110111100 110110111111 110111111111 110011111111 110011111111 111011111111 111011111111'
        '1995 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111 011111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1996 111111011111 111111111111 111111101111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1997 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '1998 111111100111 111111100111 111111100011 111111110111 111111110011 111111110011 111111111111 111111111111 111111101111 111111111111 111111110111 111111111111 111111110011 111111111111'
        '1999 111111110111 111111111111 111111110111 111111111111 111111111111 111111111111 111111111111 111111111111 111111100011 111111100011 111111100001 111111110011 111111110001 111111110001'
        '2000 111111110000 111111111000 111111111000 111111111000 111111111100 111111111100 111111110100 111111111100 101000000000 101000000000 100000000000 100000000000 110000000000 110000000000'
        '2001 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 100011000001 100010000001 010011000001 000011000001 000001100000 000001000000'
        '2002 001000000000 000000000000 001100000000 000000000000 001100000000 000000000000 000000000000 000000000000 001000000010 000000000000 100000000001 100000000000 110000000001 100000000000'
        '2003 000111111111 000111111111 000111111111 000011111111 000011111111 000011111111 000111111111 000000111111 000111111111 000111111111 000111111111 000011111111 100011111111 000011111111'
        '2004 111111000111 111111111111 111111100111 111111111111 111111100011 111111110111 111111111011 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111'
        '2005 111011111111 111111111111 111111111111 111111111111 111110111111 111111111111 111111111111 111111111111 111000001010 111010101010 111000000001 111110000000 111100000010 111110000000'
        '2006 111111111111 111111111111 111111011111 111111111111 111111011111 111111111111 111111111111 111111111111 111111111111 111111111111 111111111111 011111111111 111111111111 111111111111'
        '2007 111111111100 111111111110 111111111110 111111111110 111111111111 111111111111 111111111111 111111111111 111111011110 111111111110 111111111110 111111111111 111111111111 111111111111'
        '2008 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000 000000000000'
        '2009 000011111111 000000111111 000001111111 000000111111 000001111111 000000111111 000000111111 000000000111 000111111111 000011111111 000011111111 000011111111 000011111111 000001111111'
        '2010 111100001111 111110101111 111110000111 111110000111 111111000111 111111000111 111110101111 111111111111 111110101101 111111101111 111111000111 111111111111 111111100011 111111111111'
        '2011 111111000000 111111100000 111111100000 111111100000 111111100000 111111110000 111111110000 111111110110 111110000000 111110000000 111110000000 111111000000 111111000000 111111000000'
    };
    M = [];
    for i = 1:numel(B)
        parts = strsplit(strtrim(B{i}));
        yr = str2double(parts{1});
        for m = 1:12
            if parts{2}(m) ~= '-'
                M(end+1, :) = [yr*100 + m, cellfun(@(b) double(b(m) == '1'), parts(2:end))]; %#ok<AGROW>
            end
        end
    end
end