%% Job openings-to-employment ratio (JOE), United States, monthly, 1951 to date
%
%   JOE = job openings / employment, in percent
%
% Construction (follows Kothari and O'Doherty, 2023):
%   1951-01 to 2000-12  Barnichon (2010) composite Help-Wanted Index, in
%                       job-openings-rate units (openings / labour force),
%                       rescaled to openings / employment:
%                           JOE = (V/LF) x (CLF16OV / CE16OV)
%   2001-01 onwards     JOE = 100 x JTSJOL / CE16OV
%
% Sources (FRED, no API key needed): JTSJOL job openings, CE16OV employment
% level, CLF16OV labour force level, USREC NBER recession indicator.
% The 1951-2000 help-wanted series is fixed history and is included at the
% bottom of this file.
%
% Output: joe_monthly.csv in the current folder, with the same columns as
% update_joe.py. joe_lagged_pct is the latest reading public by the end of
% each month (1-month lag to 2000, 2 months from 2001). October 2025 has no
% CPS employment figure (federal government shutdown), so JOE is blank there.
% From 2001 this matches the series plotted by Kothari and O'Doherty; before
% 2001 it runs about 7% above their plotted levels (a constant ratio, so the
% help-wanted index was scaled differently there). This version is
% continuous at the 2001 join.
%
% Cite: Kothari, P., and O'Doherty, M. S. (2023). Job postings and aggregate
% stock returns. Journal of Financial Markets 64, 100804.
% Barnichon, R. (2010). Building a composite Help-Wanted Index. Economics
% Letters 109(3), 175-178.
%
% Requires MATLAB R2019b or later (uses local functions in scripts).

clear; clc;

vlf = barnichon_v_over_lf();            % 600 x 1, 1951-01 to 2000-12
[tJ, jol] = fred('JTSJOL');
[tE, emp] = fred('CE16OV');
[tL, clf] = fred('CLF16OV');
[tR, rec] = fred('USREC');

lastMonth = max(tJ(~isnan(jol)));       % JOLTS is the later-published input
month = (datetime(1951,1,1):calmonths(1):lastMonth)';
n = numel(month);
splice = datetime(2001,1,1);
early = month < splice;

V_LF = nan(n,1);  V_LF(1:numel(vlf)) = vlf;
JOL  = align(tJ, jol, month);  JOL(early) = NaN;
EMP  = align(tE, emp, month);
CLF  = align(tL, clf, month);
REC  = align(tR, rec, month);

joe = nan(n,1);
joe(early)  = V_LF(early) .* CLF(early) ./ EMP(early);
joe(~early) = 100 * JOL(~early) ./ EMP(~early);

% Publication timing: 1-month lag to 2000-12, 2-month lag from 2001-01.
lag = 1 + ~early;
idx = (1:n)' - lag;
joeLag = nan(n,1);
ok = idx >= 1;
joeLag(ok) = joe(idx(ok));

source = repmat("jolts", n, 1);  source(early) = "barnichon_hwi";

T = table(string(datestr(month,'yyyy-mm')), round(joe,4), round(joeLag,4), source, ...
    V_LF, JOL, EMP, CLF, REC, 'VariableNames', {'month','joe_pct','joe_lagged_pct', ...
    'source','v_over_lf_pct','jolts_openings_thous','employment_thous', ...
    'labour_force_thous','nber_recession'});
writetable(T, 'joe_monthly.csv');

last = find(~isnan(joe), 1, 'last');
fprintf('Wrote joe_monthly.csv: %d months, %s to %s\n', n, T.month(1), T.month(end));
fprintf('Latest JOE: %.2f%% (%s)\n', joe(last), T.month(last));
gaps = T.month(isnan(joe));
if ~isempty(gaps), fprintf('No JOE value (missing input) for: %s\n', strjoin(gaps, ', ')); end

% Chart with NBER recession shading
figure('Color','w'); hold on;
yl = [0 ceil(max(joe))+0.5];
inRec = REC == 1;
starts = find(diff([0; inRec]) == 1);  stops = find(diff([inRec; 0]) == -1);
for k = 1:numel(starts)
    x0 = month(starts(k));  x1 = month(stops(k)) + calmonths(1);
    patch([x0 x1 x1 x0], [yl(1) yl(1) yl(2) yl(2)], [0.85 0.85 0.85], 'EdgeColor','none');
end
plot(month, joe, 'k', 'LineWidth', 1.4);
ylim(yl); grid on; box on;
ylabel('Job openings-to-employment ratio (%)');
title(sprintf('JOE, %s to %s', T.month(1), T.month(last)));

%% ---------------------------------------------------------------- functions
function [t, x] = fred(id)
% Download one monthly FRED series. FRED's server stalls on MATLAB's default
% request (its user agent, gzip encoding and Connection: close), so ask for
% an uncompressed reply on a kept-alive connection.
    url = ['https://fred.stlouisfed.org/graph/fredgraph.csv?id=' id '&cosd=1948-01-01'];
    file = [tempname '.csv'];
    opts = weboptions('Timeout', 60, 'UserAgent', 'joe-update/1.0', ...
        'HeaderFields', {'Accept-Encoding', 'identity'; 'Connection', 'keep-alive'});
    websave(file, url, opts);
    opts = detectImportOptions(file);
    opts = setvartype(opts, 2, 'double');
    D = readtable(file, opts);
    delete(file);
    t = dateshift(datetime(D{:,1}), 'start', 'month');
    x = D{:,2};
end

function y = align(t, x, grid)
% Put a monthly series onto the monthly grid by exact month match.
    y = nan(size(grid));
    [hit, loc] = ismember(t, grid);
    y(loc(hit)) = x(hit);
end

function v = barnichon_v_over_lf()
% Barnichon (2010) composite Help-Wanted Index, 2017 revision, in job-openings-
% rate units: vacancies / labour force, percent. Columns: year, Jan ... Dec.
    B = [
        1951  3.73 3.72 4.09 4.05 4.09 3.88 3.85 3.88 4.01 3.97 3.92 3.85
        1952  4.09 4.14 4.01 3.97 3.88 3.64 3.48 3.73 4.16 4.20 4.36 4.05
        1953  4.18 4.34 4.44 4.14 4.12 3.83 3.53 3.46 3.38 3.05 2.75 2.33
        1954  2.37 2.24 2.16 2.09 2.05 2.05 2.02 1.98 1.92 1.98 2.14 2.11
        1955  2.38 2.38 2.64 2.70 2.88 2.85 3.07 3.18 3.14 3.31 3.36 3.16
        1956  3.53 3.53 3.27 3.29 3.33 3.29 3.14 3.12 3.12 3.24 3.20 3.09
        1957  3.29 3.07 3.05 2.83 2.88 2.83 2.83 2.61 2.51 2.20 2.07 1.98
        1958  1.98 1.81 1.72 1.63 1.66 1.77 1.85 1.85 1.77 1.87 2.00 1.96
        1959  2.14 2.26 2.40 2.50 2.61 2.70 2.70 2.57 2.51 2.42 2.53 2.40
        1960  3.07 3.07 2.96 2.85 2.81 2.74 2.64 2.61 2.50 2.40 2.35 2.24
        1961  2.24 2.20 2.24 2.26 2.35 2.37 2.44 2.53 2.59 2.72 2.81 2.79
        1962  2.92 2.87 2.87 2.87 2.96 2.87 2.87 2.79 2.74 2.70 2.70 2.70
        1963  2.74 2.74 2.77 2.72 2.66 2.68 2.75 2.72 2.70 2.79 2.74 2.88
        1964  2.88 2.83 2.87 2.98 2.98 3.09 3.18 3.12 3.16 3.16 3.31 3.29
        1965  3.33 3.46 3.49 3.53 3.66 3.70 3.70 3.83 3.97 4.09 4.36 4.44
        1966  4.47 4.58 4.84 4.64 4.68 4.71 4.71 4.64 4.55 4.55 4.53 4.47
        1967  4.51 4.47 4.36 4.38 4.34 4.31 4.22 4.33 4.27 4.34 4.31 4.33
        1968  4.47 4.38 4.47 4.55 4.53 4.51 4.64 4.73 4.83 5.07 5.05 4.95
        1969  5.16 5.12 5.16 5.20 5.21 4.99 4.95 4.84 5.12 5.03 4.92 4.73
        1970  4.51 4.46 4.20 4.07 3.83 3.75 3.61 3.57 3.53 3.27 3.25 3.25
        1971  3.12 3.22 3.22 3.20 3.24 3.38 3.31 3.33 3.29 3.33 3.38 3.42
        1972  3.57 3.64 3.70 3.73 3.79 3.83 3.90 4.03 4.09 4.27 4.34 4.70
        1973  4.84 4.79 4.81 4.71 4.77 4.77 4.84 4.73 4.68 4.73 4.68 4.47
        1974  4.46 4.34 4.36 4.51 4.44 4.31 4.14 4.05 3.86 3.59 3.31 3.03
        1975  2.85 2.79 2.70 2.70 2.68 2.88 2.92 2.88 2.96 2.96 3.07 3.07
        1976  3.12 3.29 3.31 3.27 3.35 3.36 3.31 3.31 3.29 3.33 3.38 3.53
        1977  3.64 3.62 3.72 3.81 3.90 3.92 4.03 4.10 4.09 4.27 4.38 4.51
        1978  4.53 4.57 4.70 4.95 4.90 4.88 4.88 4.95 5.01 5.16 5.10 5.12
        1979  5.07 4.94 4.97 5.16 5.12 5.03 4.97 5.03 5.14 5.21 4.88 4.83
        1980  4.70 4.58 4.53 3.99 3.66 3.72 3.72 3.73 3.90 3.90 4.07 3.92
        1981  3.81 3.86 3.85 3.83 3.77 3.83 3.79 3.75 3.57 3.35 3.33 3.18
        1982  3.16 3.03 2.94 2.81 2.77 2.63 2.50 2.42 2.29 2.29 2.29 2.38
        1983  2.42 2.42 2.50 2.55 2.77 2.75 2.92 2.96 3.01 3.31 3.31 3.44
        1984  3.53 3.68 3.68 3.83 3.86 3.94 3.99 3.86 3.96 3.97 3.92 4.07
        1985  3.96 3.97 4.07 3.96 4.05 4.05 4.01 4.03 4.09 4.01 4.09 3.97
        1986  3.96 3.94 3.90 3.92 3.85 4.07 3.90 3.94 3.97 3.96 4.12 3.90
        1987  3.85 4.07 4.23 4.20 4.29 4.29 4.29 4.47 4.46 4.49 4.55 4.34
        1988  4.23 4.29 4.40 4.36 4.46 4.33 4.40 4.40 4.22 4.44 4.29 4.38
        1989  4.23 4.23 4.12 4.33 4.14 3.96 4.03 3.99 3.97 4.07 3.90 4.03
        1990  3.94 3.77 3.75 3.70 3.61 3.57 3.57 3.42 3.29 3.12 2.90 2.90
        1991  2.70 2.61 2.55 2.53 2.53 2.57 2.48 2.44 2.46 2.35 2.38 2.40
        1992  2.27 2.37 2.48 2.38 2.46 2.42 2.40 2.46 2.38 2.44 2.51 2.51
        1993  2.46 2.55 2.53 2.53 2.63 2.55 2.64 2.70 2.64 2.77 2.79 2.87
        1994  2.95 3.08 3.12 3.12 3.24 3.16 3.33 3.18 3.22 3.42 3.37 3.50
        1995  3.37 3.38 3.30 3.34 3.11 3.16 3.32 3.29 3.29 3.34 3.22 3.53
        1996  3.32 3.24 3.33 3.25 3.12 3.43 3.21 3.22 3.39 3.22 3.49 3.45
        1997  3.41 3.55 3.51 3.47 3.27 3.58 3.45 3.51 3.69 3.62 3.85 3.59
        1998  3.74 3.84 3.81 3.83 3.84 3.77 3.78 3.84 3.66 3.63 3.83 3.75
        1999  3.99 4.05 3.90 3.86 3.93 3.75 3.82 3.84 3.71 3.87 3.84 3.91
        2000  4.06 4.13 4.06 4.08 3.82 3.74 3.83 3.60 3.72 3.70 3.56 3.55
    ];
    v = reshape(B(:, 2:13)', [], 1);
end
