Bayesian logistic regression for online recalibration and revision of risk prediction models with performance guarantees
Extracted source text ยท arXiv 2110.06866
\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsthm}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{bbm}
\usepackage{url}
\usepackage{natbib}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{xcolor}
\usepackage{bbm}
\usepackage{subcaption}
\usepackage{todonotes}
\usepackage[export]{adjustbox}
\usepackage[margin=1in]{geometry}
\usepackage{float}
\usepackage{nopageno}
%\usepackage{setspace}
%\doublespace
% \usepackage{lineno}
% \linenumbers
\usepackage{pdfpages}
\theoremstyle{plain}
\newtheorem{thm}{\protect\theoremname}
\theoremstyle{plain}
\newtheorem{lem}[thm]{\protect\lemmaname}
\theoremstyle{plain}
\newtheorem{cor}[thm]{\protect\corollaryname}
\makeatother
%\usepackage{babel}
\providecommand{\corollaryname}{Corollary}
\providecommand{\lemmaname}{Lemma}
\providecommand{\theoremname}{Theorem}
%%--- Custom commands
\newcommand{\R}{\mathbb{R}}
\newcommand{\Lbf}{L_{\mathrm{BF}}}
\newcommand{\Ldyntau}{L_{\mathrm{Dyn},\boldsymbol{\tau}}}
\newcommand{\Lq}{L_{Q}}
\newcommand{\Lqwalk}{L_{Q_{\boldsymbol{\tau}, \boldsymbol{\beta}, \nu}^{(W)}}}
\newcommand{\Llocked}{L_{\mathrm{locked}}}
\newcommand{\LMarBLR}{L_{\mathrm{MarBLR}}}
\newcommand{\Lblr}{L_{\mathrm{BLR}}}
\newcommand{\taulock}{\boldsymbol{\tau}_{\mathrm{locked}}}
\newcommand{\thetataulock}{\tilde{\theta}_{\tau_{\mathrm{locked}}}}
\newcommand{\Qtaumusigma}{Q_{\boldsymbol{\tau},\boldsymbol{\mu},\Sigma}}
\newcommand{\Qtaulocked}{Q_{\boldsymbol{\tau}_{\mathrm{locked}},\tilde{\theta}_{\tau_{\mathrm{locked}}},\epsilon^2\Sigma_{\mathrm{init}}}}
\newcommand{\Qwalk}{Q_{\boldsymbol{\tau}, \boldsymbol{\beta}, \nu}^{(W)}}
\newcommand{\Qsub}{Q^{\mathrm{sub}}}
\newcommand{\psub}{p_0^{\mathrm{sub}}}
\newcommand{\thetainit}{\theta_{\mathrm{init}}}
\newcommand{\Sigmainit}{\Sigma_{\mathrm{init}}}
\DeclareMathOperator{\KL}{KL}
\DeclareMathOperator{\E}{E}
\DeclareMathOperator{\Variance}{Var}
\DeclareMathOperator{\Bernoulli}{Bernoulli}
\DeclareMathOperator{\Tr}{Tr}
\newcommand{\red}[1]{\textcolor{red}{#1}}
%%--- (end custom commands)
\usepackage{wasysym}
%\graphicspath{{../../images/}}
\renewcommand*\familydefault{\sfdefault}
\title{Bayesian logistic regression for online recalibration and revision of risk prediction models with guarantees}
\author{Jean, Romain, Alexej, Berkman}
\begin{document}
\includepdf[pages=-]{word.pdf}
\begin{figure}[H]
\centering
\includegraphics[width=0.5\linewidth]{recalib_overview}
\vspace{-1.6in}
\caption{
Given a patient with variables $x$, the model reviser $\hat{A}_t$ wraps around an underlying ML model $\hat{f}_t$ to predict the true probability of having or developing a disease (or outcome).
The focus of this work is the design of an online model reviser.
}
\label{fig:overview}
\end{figure}
\begin{figure}[H]
\centering
\begin{subfigure}{0.45\linewidth}
\centering
\includegraphics[width=\linewidth]{bayesian_model_BLR}
\caption{Bayesian logistic revision (BLR) estimates the model revision parameters $\theta_t$ for the underlying prediction model $\hat{f}_t$ with the simplifying assumption that the data $(X_t, Y_t)$ are independently and identically distributed for a constant set of model revision parameters over all time points $t = 1,...,T$.}
\end{subfigure}
\hspace{1cm}
\begin{subfigure}{0.45\linewidth}
\centering
\includegraphics[width=\linewidth]{bayesian_model}
\caption{
MarBLR defines a prior over revision parameter sequences that change over time. It assumes that the revision parameters change with probability $\alpha$ at each time $t$, as modeled by a binary random variable $W_t$. It supposes that changes in the model revision parameters follow a Gaussian prior centered at zero.
}
\end{subfigure}
\caption{
BLR and its Markov variant MarBLR update the deployed model revision at each time point per the evolving Bayesian posterior.
Theoretical guarantees for BLR and MarBLR hold under misspecification of the logistic model and/or priors.
}
\label{fig:dynami}
\end{figure}
\begin{figure}[H]
\centering
\begin{subfigure}{\linewidth}
% \includegraphics[width=0.18\linewidth, trim=0 1cm 0 0]{simulation_recalib_drift.png}
\hspace{-0.2cm}
\includegraphics[width=0.22\linewidth, trim=0 0.3cm 0 0]{simulation_recalib_drift.png}
\hspace{-0.3cm}
\includegraphics[width=0.66\linewidth]{simulation_recalib_drift_calib}
\includegraphics[width=0.13\linewidth]{legend_methods.png}
\caption{Univariate recalibration. Calibration measured with respect to the general population.}
\label{fig:recalib_general_shift}
\end{subfigure}
\begin{subfigure}{\linewidth}
\includegraphics[width=0.22\linewidth,valign=t]{simulation_subpops_drift.png}
\hspace{-0.4cm}
\includegraphics[width=0.66\linewidth,valign=t]{simulation_subpops_drift_calib}
\hspace{-0.2cm}
\includegraphics[width=0.07\linewidth,valign=t]{legend_time.png}
\caption{Subgroup-aware recalibration. Calibration measured with respect to subgroups A (top) and B (bottom).}
\label{fig:two_pops_shift}
\end{subfigure}
\caption{
Results from online model recalibration of a fixed underlying prediction model in a patient population with patient subgroups A and B with prevalence 20\% and 80\% (Scenario 1).
Left: Estimated calibration index (ECI) at each time point. Right: Calibration curves for the original model and the revised versions from BLR and MarBLR. The ideal calibration curve is the identity function, which has an ECI of zero.
}
\label{fig:two_pops}
\end{figure}
\begin{figure}[H]
\centering
\hspace{-2.2cm}\includegraphics[width=0.5\linewidth]{simulation_linear_iid}\\
\hspace{-2.2cm}\includegraphics[width=0.5\linewidth]{simulation_linear_cycle}\\
\includegraphics[width=0.5\linewidth]{simulation_linear_decay}\includegraphics[width=0.15\linewidth]{legend_methods}
\caption{Results from online logistic revision of a fixed underlying model with respect to the forecasted score and ten patient variables (Scenario 2), in terms of the estimated calibration index (ECI, left) and AUC (right).
Data is simulated to be stationary over time after an initial shift (\texttt{Initial Shift}, top), shift in a cyclical fashion (\texttt{Cyclical}, middle), and shift such that the original model decays in performance over time (\texttt{Decay}, bottom).
All online logistic revision methods outperformed locking the original model in terms of the average ECI and AUC, with BLR and MarBLR performing the best.
Note that the revised models were worse than the original model briefly in the \texttt{Decay} setting.
}
\label{fig:dynamic_sim}
\end{figure}
\begin{figure}[H]
\centering
\begin{subfigure}{0.5\linewidth}
\includegraphics[width=0.9\linewidth]{simulation_refit_iid_good}
\caption{\texttt{Initial Shift}, \texttt{All-Refit}}
\label{fig:iid_good}
\end{subfigure}
\hspace{-0.5cm}
\begin{subfigure}{0.5\linewidth}
\includegraphics[width=0.8\linewidth]{simulation_refit_drift_good}
\caption{\texttt{Decay}, \texttt{All-Refit}}
\label{fig:shift_good}
\end{subfigure}
% \hspace{-1cm}
\begin{subfigure}{0.5\linewidth}
\includegraphics[width=0.9\linewidth]{simulation_refit_iid_bad}
\caption{\texttt{Initial Shift}, \texttt{Subset-Refit}}
\label{fig:iid_bad}
\end{subfigure}
\hspace{-0.5cm}
\begin{subfigure}{0.5\linewidth}
\includegraphics[width=0.8\linewidth]{simulation_refit_drift_bad}
\includegraphics[width=0.19\linewidth]{legend_methods}
\caption{\texttt{Decay}, \texttt{Subset-Refit}}
\label{fig:shift_bad}
\end{subfigure}
\caption{Model calibration and discrimination (left and right panels, respectively) from online ensembling of the original model with an underlying prediction model that is continually refitted over time (Scenario 3).
Data is simulated to be stationary over time after an initial shift (\texttt{Initial Shift}) and nonstationary such that the original model decays in performance over time (\texttt{Decay}).
Underlying prediction model is updated by continually refitting on all previous data (\texttt{All-Refit}) or refit on the most recent subset of data (\texttt{Subset-Refit}).
Note that \texttt{Subset-Refit} simulates a sudden drop in performance for the continually refitted model at time $t = 100$ and, consequently, across all online ensembling procedures.
BLR and MarBLR recover from this sudden performance decay and achieve better performance than locking the original model in terms of the average ECI and AUC.
}
\label{fig:refitting}
\end{figure}
\begin{figure}[H]
\centering
\begin{subfigure}{\linewidth}
\centering
% \includegraphics[width=0.29\linewidth]{../code/python/analysis_copd/_output/order_date_cts/nump_36/size_108000/seed_0/fixed/batch_10/regret_0.05/inflat_0,0.005/linear_update_0/thetas_time.png}
\includegraphics[width=0.78\linewidth]{copd_linear0_err}
\hspace{-0.2cm}
\includegraphics[width=0.1\linewidth]{legend_copd_methods.png}
\caption{Online recalibration of a fixed prediction model}
\label{fig:online_recalib_copd}
\end{subfigure}
\begin{subfigure}{\linewidth}
\centering
% \includegraphics[width=0.29\linewidth]{../code/python/analysis_copd/_output/order_date_cts/nump_36/size_108000/seed_0/fixed/batch_10/regret_0.05/inflat_0,0.005/linear_update_1/thetas_time.png}
\hspace{-1.8cm}
\includegraphics[width=0.78\linewidth]{copd_linear1_err}
\caption{Online logistic revision with respect to a fixed prediction model and patient variables}
\label{fig:online_linear_copd}
\end{subfigure}
\begin{subfigure}{\linewidth}
\centering
% \includegraphics[width=0.29\linewidth]{../code/python/analysis_copd/_output/order_date_cts/nump_36/size_108000/seed_0/combo_boxed/batch_10/regret_0.05/inflat_0,0.005/linear_update_0/thetas_time.png}
% \hspace{-0.2cm}
\hspace{-1.8cm}
\includegraphics[width=0.78\linewidth]{copd_combo_err}
\includegraphics[width=\linewidth]{copd_combo_roc}
\vspace{-0.5cm}
\caption{Online ensembling of the original and continually-refitted prediction models}
\label{fig:online_refit_copd}
\end{subfigure}
%\begin{subfigure}{\linewidth}
% \centering
%
%\caption{ROC curves when online wrapper}
%\label{fig:online_refit_copd_roc}
%\end{subfigure}
\caption{Results from online logistic recalibration and revision for a fixed COPD risk prediction model (a and b, respectively) and online reweighting for fixed and continually-refitted (evolving) COPD risk prediction models (c) using BLR and MarBLR. Calibration curves are estimated across four time periods that divide the full dataset into equal lengths. The receiver operating characteristic (ROC) curve is shown for the online ensembling approach because the ROC curves did not change significantly for (a) or (b).
}
\label{fig:copd}
\end{figure}
\pagebreak
\appendix
\section*{Appendix for ``Bayesian logistic regression for online recalibration and revision of clinical prediction models with guarantees''}
\renewcommand\thefigure{A.\arabic{figure}}
\renewcommand\thetable{A.\arabic{table}}
\begin{table}
\centering
\begin{tabular}{p{2.5in}|p{3.2in}}
Notation & Description\\
\toprule
\multicolumn{2}{c}{\textit{General terms}}\\
$T$ & Time horizon \\
$d$ & Number of variables \\
$(x_t, y_t)$ & Observed variables and outcome at time $t$ \\
$\hat{f}_t : \mathcal{X} \mapsto \mathbb{R}$ & Underlying prediction model at time $t$\\
$\hat{A}_t:\mathbb{R} \times \mathcal{X} \mapsto [0,1]$ & Model revision deployed at time $t$: A function that maps predictions from the underlying prediction model at time $t$ and patient variables to a probability \\
$\hat{\theta}_t$ & Parameters for logistic model revision at time $t$ \\
$\tau = (\tau_1, \tau_2, \cdots, \tau_s)$ & Update times for a given sequence of model revisions \\
\midrule
\multicolumn{2}{c}{\textit{Regret}}\\
$
\bigg(
\frac{1}{T}\sum_{t=1}^T
\bigg [- \log p \left (y_t, \hat{A}_t(\hat{f}_t(x_t), x_t) \right)
+ \log p \left (y_t, \hat{A}_1(\hat{f}_1(x_t), x_t) \right) \bigg ] \bigg )_+
$ &
\underline{Type I Regret}: The average increase in the negative log likelihood when using the online reviser instead of locking the original model \\
\rule{0pt}{6ex}
$
\bigg(
\frac{1}{T}\sum_{t=1}^T
\bigg [ - \log p\left (y_t, \hat{A}_t(\hat{f}_t(x_t), x_t) \right)
+ \log p\left (y_t, {A}^*_{\tau, t}(\hat{f}_t(x_t), x_t) \right) \bigg ] \bigg)_+
$ &
\underline{Type II $\tau$-Regret}: The average increase in the negative log likelihood when using the online reviser versus the oracle model reviser $\{{A}^*_{\tau, t}: t = 1,...,T \}$ with update times $\tau$
\\
\midrule
\multicolumn{2}{c}{\textit{BLR and MarBLR parameters}}\\
$N(\thetainit, \Sigmainit)$ & Gaussian prior in BLR and MarBLR for the logistic revision parameter at time $t = 1$ \\
$\alpha$ & Prior probability in MarBLR that the model revision shifts at time $t$\\
$\delta^2$ & Factor controlling the variance of the MarBLR prior over shifts in the model revision parameters
\end{tabular}
\caption{
Terminology and notation
}
\label{eq:terminology}
\end{table}
\section{Practical Implementation of BLR and MarBLR}
\label{sec_a:practical}
In this manuscript, we implement MarBLR using a Laplace approximation of the logistic posterior and perform Kalman filtering with collapsing \citep{Gordon1990-jj, West1997-fa}.
Because BLR corresponds to MarBLR with $\alpha = 0$ and $\delta^2 = 0$, we use this same procedure to perform approximate Bayesian inference.
The Kalman filtering approach is simple and computationally efficient; We describe the steps below.
We note that for the special case of BLR, one can also perform posterior inference by sampling Polya-Gamma latent variables \citep{Polson2013-cr}.
This would allow one to perform full Bayesian inference but is significantly more costly in terms of computation time.
We make predictions and update the posterior using the following recursive procedure.
The process is initialized with the Gaussian prior for $\theta_1$ with mean $\thetainit$ and posterior covariance $\Sigmainit$.
Let $D^{(t)}$ denote the observations up to and including time $t$.
\textbf{Prediction step.} At time $t$, let the approximation for $\theta_{t-1}|W_{t-1}= w_{t-1},D^{(t-1)}$ be the Gaussian distribution with mean $\hat{\theta}_{t-1, w_{t-1}}$ and covariance $\hat{\Sigma}_{t-1, w_{t-1}}$.
We also assume $\Pr(W_{t - 1} = w_{t - 1}|D^{(t-1)})$ is known.
We generate predictions at time $t$ using the posterior distribution $\theta_{t}|D^{(t - 1)}$, which is a mixture of the distributions
\begin{align}
\theta_{t}|W_t= w_t,W_{t - 1} = w_{t-1}, D^{(t - 1)}
\sim
N\left(
\hat{\theta}_{t-1,w_{t-1}},
(1 + \delta^2 w_t) \hat{\Sigma}_{t-1, w_{t-1}}
\right)
\label{eq:predict_parts}
\end{align}
%Our theorems assume $R_t = \delta^2 I$; In practice, we use $R_t = \delta^2 \hat{\Sigma}_{t-1}/\lambda_{\max}(\hat{\Sigma}_{t-1})$ to reflect different levels of uncertainty across parameters.
for $w_t, w_{t=1} \in \{0,1\}$ with weights by $\Pr(W_t = w_t | W_{t-1} = w_{t-1}) \Pr(W_{t - 1} = w_{t - 1}|D^{(t-1)})$. Recall that $\Pr(W_t = 1 | W_{t-1} = w_{t-1}) = \alpha$ in the MarBLR prior.
We predict that $Y = 1$ for a subject $x$ using the posterior mean of $\Pr(Y = 1|X = x)$.
\textbf{Update step.} Next, we observe a new batch of labeled observations and update the posterior.
That is, we must perform inference for $\theta_{t}|D^{(t)}$, which is a mixture of the distributions $\theta_{t}|W_t= w_t,W_{t-1}= w_{t-1},D^{(t)}$ with probability weights $\Pr(W_t= w_t, W_{t-1} = w_{t-1}|D^{(t)})$ for $w_t, w_{t-1} \in\{ 0,1\}$.
%Suppose the conditional distributions $\theta_{t-1}|W_{t-1}= w_{t-1},D^{(t - 1)}$ follow a Gaussian distribution with mean $\hat{\theta}_{t-1,w_{t-1}}$ and covariance $\hat{\Sigma}_{t-1, w_{t-1}}$.
Let $\tilde{\ell}_t(\theta, w) = \sum_{i=1}^n \log p\left(y_{t,i}|z_{t,i}, \theta \right) + \log p(\theta\mid w, D^{(t - 1)})$.
We approximate the distribution $\theta_{t}|W_t= w_t,W_{t - 1} = w_{t-1}, D^{(t)}$ using a Gaussian distribution with its mean computed using a Newton update
\begin{align}
\hat{\theta}_{t, w_t, w_{t-1}} = \hat{\theta}_{t-1, w_{t-1}} -
\left[\nabla_{\theta}^2 \tilde{\ell}_t \left(\hat{\theta}_{t-1, w_{t-1}}, w_{t-1}\right)\right]^{-1}
\nabla_{\theta} \tilde{\ell}_t \left(\hat{\theta}_{t-1, w_{t-1}}, w_{t-1} \right)
\end{align}
and its covariance as
$$
\hat{\Sigma}_{t ,w_t, w_{t - 1}} = \left[\nabla_{\theta}^2 \tilde{\ell}_t(\hat{\theta}_{t-1, w_{t-1}}, w_{t-1})\right]^{-1}.
$$
The probability $\Pr(W_t = w_t, W_{t-1} = w_{t-1}\mid D^{(t)})$, which is proportional to
\begin{align}
\left[
\int_{\theta_t}
p\left(y_{t,\cdot} \mid z_{t,\cdot}, \theta_t \right)
p\left(\theta_t \mid w_t, w_{t-1}, D^{(t-1)} \right)
d\theta_t
\right]
\Pr(W_t|W_{t-1})
\Pr(W_{t-1} | D^{(t-1)}),
\label{eq:prob_post}
\end{align}
is approximated using a Laplace approximation for the integral in \eqref{eq:prob_post}, i.e.
\begin{align}
2\pi^{d/2}
\left| \left\{ \nabla^2_\theta \tilde{\ell}(\hat{\theta}_{t, w_{t}, w_{t - 1}}) \right\}^{-1} \right|^{1/2}
p\left(y_{t, \cdot} \mid z_{t,\cdot}, \hat{\theta}_{t, w_{t}, w_{t - 1}} \right)
p\left(\hat{\theta}_{t, w_{t}, w_{t - 1}} \mid D^{(t-1)} \right).
\label{eq:laplace}
\end{align}
Let $\hat{q}_{w_t, w_{t-1}}$ denote the estimated probability.
Finally, we approximate the posterior distribution $\theta_{t}|W_t= w_t,D^{(t)}$ using a single Gaussian distribution by moment-matching \citep{West1997-fa, Orguner2007-st} with mean and covariance
\begin{align}
\hat{\theta}_{t, w_t} =
\frac{1}{\hat{q}_{w_t, 0} + \hat{q}_{w_t, 1}} \left(
\hat{q}_{w_t, 0} \hat{\theta}_{t, w_t, 0} + \hat{q}_{w_t, 1} \hat{\theta}_{t, w_t, 1}
\right)
\\
\hat{\Sigma}_{t, w_t} =
\frac{1}{\hat{q}_{w_t, 0} + \hat{q}_{w_t, 1}} \left(
\hat{q}_{w_t, 0}\hat{\Sigma}_{t, w_t, 0} + \hat{q}_{w_t, 1} \hat{\Sigma}_{t, w_t, 1}
\right).
\end{align}
\section{Online model revision for batched data}
In certain settings, it is more convenient and practical for the data stream to be observed in batches of size $n > 1$.
Here we discuss the necessary modifications to our framework for analyzing the performance on an online model reviser for batched data.
We denote a batch of observations as $\{(x_{t,i}, y_{t,i}): i = 1,....,n\}$ and use the notation $a_{t,\cdot}$ to denote the sequence $(a_{t,1},...,a_{t,n})$.
We extend the online model reviser to output a probability distribution over all possible outcomes for a batch of observations, i.e. $\hat{A}_{t}: \mathbb{R}^n \times \mathcal{X}^n \mapsto \Delta_{2^n}$ where $\Delta_{2^n}$ is the probability simplex over all possible outcomes $(y_{t,1},...,y_{t,n})$.
The loss of the online model reviser over the entire time period is then defined as the average negative log likelihood
\begin{align}
-\frac{1}{nT}\sum_{t = 1}^T \log p\left (y_{t,\cdot}, \hat{A}_{t}\left(\hat{f}_t(x_{t, \cdot}), x_{t,\cdot} \right ) \right ).
\label{eq:batched_loss}
\end{align}
This theoretical framework allows predictions from the online model reviser to depend on all unlabeled observations $\{x_{t,i}: i = 1,...,n\}$.
By defining regret with respect to \eqref{eq:batched_loss}, we are able to derive Type I and II regret bounds for the batched setting.
This is necessary for analyzing BLR and MarBLR because outcomes are not independent given the observations up to time $t$ in the Bayesian framework.
In particular, the outcomes are correlated because of the shared (latent) revision parameter $\theta_t$.
\begin{table}[!htbp] \centering
\caption{Descriptive statistics of variables included in the COPD risk prediction model. Continuous variables are summarized by Mean (SD). Binary/ordinal variables are summarized by number of nonzero entries (\%).}
\label{tab:copd}
\begin{tabular}{@{\extracolsep{5pt}}lr}
\\[-1.8ex]\hline
\hline \\[-1.8ex]
Variable & \\
\hline \\[-1.8ex]
Diagnosed with COPD & 2756 (2.55) \\
Age at encounter & 60.31 (18.60) \\
\multicolumn{2}{c}{\textit{Medical history}}\\
Asthma & 741 (0.69) \\
Bronchitis & 5855 (5.42) \\
COPD & 14950 (13.84) \\
Smoking & 42651 (39.49) \\
Pulmonary Function Test & 2844 (2.63) \\
Intubation & 2420 (2.24) \\
Spirometry & 1091 (1.01) \\
Bilevel positive airway pressure & 710 (0.66) \\
Acute coronary syndrome & 11008 (10.19) \\
Pneumonia &15386 (14.25) \\
Steroids & 23249 (21.53) \\
Antihypertensives & 7740 (7.17) \\
Short-acting bronchodilator & 13088 (12.12) \\
Antihistiminic & 17768 (16.45) \\
Respiratory Clearance & 2791 (2.58) \\
Upper Respiratory Infection & 1242 (1.15) \\
Antiarrythmic order & 7650 (7.08) \\
Inhaled bronchodilators & 122 (0.11) \\
Inhaled corticosteroid & 78 (0.07) \\
Long-acting bronchodilator & 91 (0.08) \\
Combination of inhaled bronchodilators & 8 (0.001) \\
\multicolumn{2}{c}{\textit{History of current emergency department visit}} \\
Pneumonia & 3503 (3.24) \\
Short-acting bronchodilator & 5982 (5.54) \\
Steroids & 4518 (4.18) \\
Antihypertensives & 694 (0.64) \\
Acute coronary syndrome & 2386 (2.21) \\
Antiarrthymic & 1665 (1.54) \\
Antihistaminic & 2624 (2.43) \\
Inhaled corticosteroid & 146 (0.14) \\
Inhaled bronchodilators & 304 (0.28) \\
Long-acting bronchodilator & 420 (0.39) \\
Asthma & 142 (0.13) \\
Upper Respiratory Infection & 238 (0.22) \\
Respiratory Clearance & 131 (0.12) \\
Combination of inhaled bronchodilators & 3 (0.003) \\
\hline \\[-1.8ex]
\end{tabular}
\end{table}
\section{Type I and II Regret bounds}
\subsection{Notation and assumptions}\label{sec:notation}
We suppose there are $n$ observations at time points $t = 1,...,T$ for some $T \ge 2$.
Consider any sequence of revision parameters $\boldsymbol{\theta} = \{\theta_{1}, \theta_{2}, \ldots, \theta_{T}\}$, where $\theta_{t}\in\mathbb{R}^{d}$ for all $t = 1, 2, \ldots, T$,
with unique values at times $\boldsymbol{\tau} = \{\tau_{1},\tau_{2},...,\tau_{s}\}$, where $\tau_{1}=1 < \tau_2 < \ldots < \tau_{s} \leq T$.
In other words, $\{\theta_{\tau_{1}},\theta_{\tau_{2}},\ldots,\theta_{\tau_{s}}\}$ denotes the sequence of values that the sequence $\boldsymbol{\theta}$ shifted over.
%Let $\tau_{s+1}=T+1$ for simplicity. %% [AG: I don't think this is ever used...]
Henceforth, we use $|\boldsymbol{\tau}|$ (rather than $s$) to indicate the number of times in $\boldsymbol{\tau}$.
For ease of notation, we use the convention $\tau_{|\tau|+1} := T + 1$. Note that the variable $\tau_{|\tau|+1}$ is not part of the sequence $\boldsymbol{\tau}$ and is used purely to simplify the notation.
We use $\taulock := \{\tau_1\} = \{1\}$ to denote the shift times in the edge case of ``locked'' sequences $\boldsymbol{\theta}$ that do not shift over time.
Let $D^{(t)}$ denote all the data observed up to time $t$.
The cumulative negative log-likelihood when using Bayesian inference at each time point is
\[
\Lbf = -\sum_{t=1}^{T}\log p\left(y_{t,\cdot}\mid z_{t,\cdot},D^{(t-1)}\right),
\]
where $p\left(y_{t,\cdot}\mid z_{t,\cdot},D^{(t-1)}\right)$ is the posterior distribution at time $t$.
The cumulative negative log-likelihoods for MarBLR and BLR are denoted by $\LMarBLR$ and $\Lblr$, respectively, and are special cases of $\Lbf$ for their specific choice of priors.
The MarBLR prior $p_0$ over $\boldsymbol{\theta}$ is defined using a Gaussian random walk with a homogeneous transition matrix as follows.
Given $\thetainit \in \R^d$, $\Sigmainit \in \R^{d\times d}$, and some shift probability $\alpha\in[0,1]$, let
\begin{equation}
\theta_{1} \sim N(\thetainit,\Sigmainit), \quad W_1 = 1,
\label{eq:MarBLR_prior_theta_1}
\end{equation}
and for $t = 2, 3, \ldots, T$ let
\begin{equation}
\begin{aligned}
\theta_{t} &=\theta_{t-1}+\beta_{t}W_{t}\\
W_{t} &\sim \Bernoulli(\alpha)\\
\beta_{t} &\sim N(0,\delta^{2}\Sigmainit).
\end{aligned}
\label{eq:MarBLR_prior_theta_t}
\end{equation}
Note that $\boldsymbol{\tau} = \left\{\tau_1, \tau_2, \ldots, \tau_{|\boldsymbol{\tau}|}\right\}$ can be regarded as the indices at which the sequence $\left\{W_1, W_2, \dots, W_T\right\}$ is $1$-valued. In particular, having $\boldsymbol{\tau} = \taulock$ implies that $W_1 = 1$ and $W_t = 0$ for all $t>1$.
The BLR prior is a special case where $\delta^2 = \alpha = 0$.
Type I regret compares BLR and MarBLR to locking the original revision parameters at its initial value $\thetainit$, i.e. $\theta_t = \thetainit$ for all $t\in\left\{1, 2, \ldots, T\right\}$.
The cumulative negative log-likelihood of the locked initial model is given by
\begin{equation*}
\Llocked = -\sum_{t=1}^{T}\log p\left(y_{t,\cdot}\mid z_{t,\cdot};\thetainit\right).
\end{equation*}
Type II $\boldsymbol{\tau}$-regret compares BLR and MarBLR to the best sequence of parameters in retrospect for update times $\boldsymbol{\tau}$, denoted $\tilde{\theta}_{\tau_{j}}$ for $j=1,...,|\tau|$.
Its cumulative negative log-likelihood is defined as
\[
\Ldyntau = -\sum_{t=1}^{T} \log p\left(y_{t,\cdot}\mid z_{t,\cdot};\tilde{\theta}_{t}\right)
\]
where $\tilde{\theta}_{\tau_{j}}$ for $j=1,...,|\tau|$ satisfy
\begin{equation}
\left.\nabla\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\log p\left(y_{t,\cdot}\mid z_{t,\cdot};\theta\right)\right|_{\theta=\tilde{\theta}_{\tau_{j}}}=0,\quad\forall j=1,\ldots, |\boldsymbol{\tau}|.
\label{eq:theta_tilde}
\end{equation}
In addition, we introduce the notion of a distribution over the sequences
$\boldsymbol{\theta}$. For such a distribution
$Q$, its expected negative log-likelihood is given by
\[
\Lq = E_{Q}\left[-\sum_{t=1}^{T}\log p\left(y_{t,\cdot}\mid z_{t,\cdot};\theta_{t}\right)\right].
\]
Given mean and variance parameters $\boldsymbol{\mu} = (\mu_{t})_{t\in\boldsymbol{\tau}}$ and $\boldsymbol{\Sigma} = (\Sigma_{t})_{t\in\boldsymbol{\tau}}$, we define $\Qtaumusigma$ to be the distribution over $\boldsymbol{\theta}$ with shift times $\boldsymbol{\tau}$ where $\theta_{\tau_{j}}$ for $j=1,...,|\boldsymbol{\tau}|$ are jointly independent and normally distributed per
\begin{equation}\label{eq:Qtaumusigma}
\theta_{\tau_{j}}\sim N\left(\mu_{\tau_{j}},\Sigma_{j}\right).
\end{equation}
Some results in the following sections rely on the assumption that there exists a constant $c>0$ such that
\begin{equation}\label{eq:second_derivative_assumption}
\left|\frac{\partial^{2}}{\partial w^{2}}\log p\left(y | z^\top \theta = w\right)\right|\le c,
\end{equation}
for all $y$ and $w$.
This always holds for logistic regression with $c \leq 1$.
\subsection{Useful Results}
Consider the prior distribution $p_{0}(\boldsymbol{\theta})$ over sequences $\boldsymbol{\theta}$.
Let $p_{0}(\boldsymbol{\tau})$ be its marginal distribution over shift times $\boldsymbol\tau$ and $p_{0}\left(\boldsymbol{\theta} \mid \boldsymbol{\tau}\right)$ be the conditional distribution over sequences $\boldsymbol{\theta}$ with shift times $\boldsymbol{\tau}$.
\begin{lem}[Variational bound]\label{lemma:variational_bound_general}
Consider any prior distribution $p_{0}$ over sequences $\boldsymbol{\theta}$.
Given any $\boldsymbol{\tau}$ and any distribution $Q$, it holds that
\begin{equation*}
\Lbf - \Lq \le \E_{\boldsymbol{\tau} \sim Q}\left[
\KL\left(Q(\boldsymbol{\theta} | \boldsymbol{\tau}) \mid\mid p_{0}\left(\boldsymbol{\theta} \mid \boldsymbol{\tau}\right)\right)
\right]
+ \KL\left(Q(\boldsymbol{\tau}) \mid\mid p_{0}\left(\boldsymbol{\tau}\right)\right).
\end{equation*}
\end{lem}
\begin{proof}
First, we can reexpress the cumulative negative log-likelihood of the Bayesian dynamical model by chaining the conditional probabilities as follows:
\begin{align*}
\Lbf &= -\sum_{t=1}^{T}\log p\left(y_{t,\cdot}\mid z_{t,\cdot},D^{(t-1)}\right) \\
&= -\log p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}\right).
\end{align*}
Similarly, the cumulative negative log-likelihood of any sequence of calibration parameters can be written as
\[
-\sum_{t=1}^{T} \log p\left(y_{t,\cdot}\mid z_{t,\cdot};\theta_{t}\right)=-\log p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}; \boldsymbol{\theta}\right).
\]
Thus, the difference in the cumulative negative log-likelihood between the Bayesian dynamical model and any sequence of parameters is given by
\begin{align*}
\Lbf - \Lq &= \E_{Q} \left[\log\frac{p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}; \boldsymbol{\theta}\right)}{p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}\right)}\right].
\end{align*}
By Bayes' Rule, the posterior distribution $p_{T}$ over $\boldsymbol{\theta}$ with respect to the Bayesian dynamical model satisfies
\[
p_{T}\left(\boldsymbol{\theta}\right) = \frac{p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}; \boldsymbol{\theta} \right) p_{0}\left(\boldsymbol{\theta}\right)}{p\left((y_{t,\cdot})_{t=1,...,T}\mid(z_{t,\cdot})_{t=1,...,T}\right)}.
\]
Thus, we have that
\begin{equation}\label{eq:Lbf-Lq-2}
\begin{aligned}
\Lbf - \Lq &= \E_{Q}\left[\log\frac{p_{T}\left(\boldsymbol{\theta}\right)}{p_{0}\left(\boldsymbol{\theta}\right)}\right]\\
&= \E_{\boldsymbol{\tau}\sim Q}\left[
\E_{\boldsymbol{\theta}\sim Q(\cdot | \boldsymbol{\tau})}\left[
\log\frac{p_{T}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)}{p_{0}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)}
\right]\right]
+\E_{\boldsymbol{\tau}\sim Q}\left[\log \frac{p_{T}(\boldsymbol{\tau})}{p_{0}(\boldsymbol{\tau})}\right].
\end{aligned}
\end{equation}
Moreover, because the KL divergence is always positive, it holds that
\begin{equation}\label{eq:KLQp0-2}
\begin{aligned}
\E_{\boldsymbol{\theta}\sim Q(\cdot | \boldsymbol{\tau})}\left[\log\frac{p_{T}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)}{p_{0}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)}\right]
&= \KL\left(Q(\cdot | \boldsymbol{\tau}) \mid\mid p_{0}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)\right) - \KL\left(Q(\cdot | \boldsymbol{\tau}) \mid\mid p_{T}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)\right)\\
&\le \KL\left(Q(\cdot | \boldsymbol{\tau}) \mid\mid p_{0}\left(\boldsymbol{\theta}\mid\boldsymbol{\tau}\right)\right).
\end{aligned}
\end{equation}
Likewise,
\begin{align}
E_{\boldsymbol{\tau}\sim Q}\left[\log \frac{p_{T}(\boldsymbol{\tau})}{p_{0}(\boldsymbol{\tau})}\right]
\le \KL\left(
Q(\boldsymbol{\tau}) \mid\mid p_{0}(\boldsymbol{\tau})
\right).
\end{align}
Finally, by combining equations (\ref{eq:Lbf-Lq-2}) and (\ref{eq:KLQp0-2}) we arrive at the conclusion of this theorem.
\end{proof}
\subsection{Type I regret results for MarBLR}
Let the distribution $p_0$ be the MarBLR prior as defined per \eqref{eq:MarBLR_prior_theta_1} and \eqref{eq:MarBLR_prior_theta_t}.
For a given $\boldsymbol{\tau}$, let $\Qwalk$ be a Gaussian random walk with expected shifts $\beta_j$ at known shift times $\tau_j$ for $j = 1,...,|\tau|$.
That is,
\begin{align}
\theta_{\tau_{j}} - \theta_{\tau_{j - 1}} \sim N\left(\beta_j, \nu^2 \Sigmainit \right).
\end{align}
and
\begin{align}
\theta_{\tau_{1}} \sim N\left(\mu_1, \epsilon_1^2 \Sigmainit \right).
\end{align}
We begin with simplifying the KL divergence term in Lemma~\ref{lemma:variational_bound_general}.
\begin{lem}\label{lemma:KL_gauss_rand_walk}
For any $\boldsymbol{\tau}$, consider the Gaussian random walk $\Qwalk$.
We have that
\begin{align}
\label{eq:lemma_KL_gauss_rand_walk}
\begin{split}
\KL\left(\Qwalk \mid\mid p_{0}\left(\boldsymbol{\theta} \mid \boldsymbol{\tau}\right)\right)
=&
\frac{1}{2}\epsilon_1^{2}d+\frac{1}{2}\left(\mu_1-\thetainit\right)^{\top}\Sigmainit^{-1}\left(\mu_1-\thetainit\right)-\frac{d|\tau|}{2}\\
& + d(|\tau| - 1) \log \frac{\delta}{\nu} - d\log \epsilon_1 \\
& + \frac{d \nu^2 (|\tau| - 1)}{2 \delta^2} + \frac{1}{2\delta^{2}}
\sum_{j=2}^{|\tau|} \beta_{j}^\top \Sigmainit^{-1} \beta_{j}.
\end{split}
\end{align}
\end{lem}
\begin{proof}
For ease of notation, let $\Theta_{J}$ be the space over sequences
$\left(\theta_{1},...,\theta_{J}\right)$.
Given the known times $\boldsymbol{\tau}$, there is a one-to-one mapping from sequences
in $\Theta_{|\boldsymbol{\tau}|}$ to sequences in $\Theta_{T}$ with unique values
at times $\boldsymbol{\tau}$.
Let $\Qsub$ be the probability distribution over $\Theta_{|\boldsymbol{\tau}|}$
as defined by $\Qwalk$. Likewise, let $\psub$
be the PDF over $\Theta_{|\boldsymbol{\tau}|}$ as defined by the conditional
prior distribution $p_{0}\left(\cdot\mid\boldsymbol{\tau}\right)$.
We have that
\begin{align}
&\KL\left(\Qwalk \mid\mid p_{0}\left(\boldsymbol{\theta} \mid \boldsymbol{\tau}\right)\right) \nonumber \\
=&\int \cdots \int \Qsub(\boldsymbol{\theta})
\log \frac{\Qsub(\boldsymbol{\theta})}{\psub(\boldsymbol{\theta})}
\mathrm{d}\boldsymbol{\theta} \nonumber \\
\begin{split}
=& \int \Qsub({\theta}_1)
\log \frac{\Qsub({\theta}_1)}{\psub({\theta}_1)}
\mathrm{d}{\theta}_1 + \sum_{j=1}^{|\tau| -1}
\int \int \Qsub({\theta}_{j}, {\theta}_{j + 1})
\log \frac{\Qsub({\theta}_{j + 1} \mid {\theta}_{j})}{\psub({\theta}_{j + 1}\mid {\theta}_{j})}
\mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j}
\end{split}
\label{eq:kl_general_2}
\end{align}
The first term in \eqref{eq:kl_general_2} is the KL divergence of two multivariate Normal distributions, $N(\mu_1, \epsilon_1^2 \Sigmainit)$ and $N(\thetainit, \Sigmainit)$, and can be shown to be equal to
\begin{equation}
\frac{1}{2}\epsilon_1^{2}d+\frac{1}{2}\left(\mu_1-\thetainit\right)^{\top}\Sigmainit^{-1}\left(\mu_1-\thetainit\right)-d\log\epsilon_1-\frac{d}{2}.
\label{eq:KL_intermed_1}
\end{equation}
Also, for $j =1,...,|\tau| -1$, we have that each summand in the second term in \eqref{eq:kl_general_2} is equal to
\begin{align}
& \int \int \Qsub({\theta}_{j}, {\theta}_{j + 1})
\log \frac{\Qsub({\theta}_{j + 1}\mid {\theta}_{j})}{\psub({\theta}_{j + 1}\mid {\theta}_{j})}
\mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j} \nonumber \\
% \begin{split}
% =&
% \frac{1}{2}\log \left |2\pi \delta^2 \Sigmainit \right | \\
% &\quad+
% \frac{1}{2\delta^{2}}
% \int \int \Qsub({\theta}_{j}, {\theta}_{j + 1})
% \left(
% {\theta}_{j + 1} - {\theta}_{j}
% \right)^\top
% \Sigmainit^{-1}
% \left(
% {\theta}_{j + 1} - {\theta}_{j}
% \right)
% \mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j}\\
% &\quad- \frac{1}{2} \log \left |2\pi e \nu^2 \Sigmainit\right |
% \end{split} \nonumber \\
= &
d \log\frac{\delta}{\nu}
+\frac{1}{2\delta^{2}}
\int \int \Qsub({\theta}_{j}, {\theta}_{j + 1})
\left(
{\theta}_{j + 1} - {\theta}_{j}
\right)^\top
\Sigmainit^{-1}
\left(
{\theta}_{j + 1} - {\theta}_{j}
\right)
\mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j}
-\frac{d}{2}.
\label{eq:qq_intermed_2}
\end{align}
By the definition of $\Qwalk$, we have that $\Delta_{j + 1} = \theta_{\tau_{j + 1}} - \theta_{\tau_j} \sim N\left (\beta_{j + 1}, \nu^2 \Sigmainit \right )$.
Thus,
\begin{align*}
\int \Qsub({\theta}_{j}, {\theta}_{j + 1})
\left(
{\theta}_{j + 1} - {\theta}_{j}
\right)^\top
\Sigmainit^{-1}
\left(
{\theta}_{j + 1} - {\theta}_{j}
\right)
\mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j}
=& d\nu^2 + \beta_{j+1}^\top \Sigmainit^{-1} \beta_{j+1}.
\end{align*}
Plugging in the above result into \eqref{eq:qq_intermed_2}, we have that
\begin{align}
\int \int \Qsub({\theta}_{j}, {\theta}_{j + 1})
\log \frac{\Qsub({\theta}_{j + 1} \mid {\theta}_{j})}{\psub({\theta}_{j + 1}\mid {\theta}_{j})}
\mathrm{d}{\theta}_{j + 1} \mathrm{d}{\theta}_{j}
= &
d \log\frac{\delta}{\nu}
+\frac{1}{2\delta^{2}}
\left(
d\nu^2 + \beta_{j+1}^\top \Sigmainit^{-1} \beta_{j+1}
\right)
-\frac{d}{2}.
\label{eq:KL_intermed_2}
\end{align}
Combining the results \eqref{eq:kl_general_2}, \eqref{eq:KL_intermed_1} and \eqref{eq:KL_intermed_2}, we attain the desired conclusion.
\end{proof}
To bound the Type I regret for MarBLR, we compare the regret via the intermediary $Q$ with marginal distribution over $\boldsymbol{\tau}$ the same as $p_0$ and the conditional distribution given $\tau$ to be $\Qwalk$ with $\boldsymbol{\beta}_j = 0$ for all $j = 2,...,|\tau|$.
That is, the regret is decomposed into
\begin{align}
(\Lbf - \Lq) + (\Lq - \Llocked).
\end{align}
We bound $\Lbf - \Lq$ by marginalizing Lemma~\ref{lemma:KL_gauss_rand_walk} over $\boldsymbol{\tau}$ as follows.
\begin{lem}\label{lemma:KL_margin_walk}
Let the distribution $p_0$ be defined as above.
Let distribution $Q$ over $\boldsymbol{\theta}$ have the same distribution over $\boldsymbol{\tau}$ as $p_0$, with $\theta_{1}$ distributed $N(\thetainit, \epsilon_{1}^2 \Sigmainit)$, and $Q(\cdot | \boldsymbol{\tau})$ be a zero-centered Gaussian random walk $\boldsymbol{\beta}_j = 0$ for all $j = 2,...,|\tau|$.
Let $\xi = \E_{p_0}|\boldsymbol{\tau}|$.
We have that
\begin{align}
\Lbf - \Lq
\le &
\frac{1}{2}\epsilon_1^{2}d -\frac{d \xi}{2} + d(\xi - 1) \log\frac{\delta}{\nu} - d\log \epsilon_1 + \frac{d \nu^2 (\xi - 1)}{2 \delta^2}.
\end{align}
\end{lem}
\begin{proof}
Taking the expectation of \eqref{eq:lemma_KL_gauss_rand_walk} from Lemma~\ref{lemma:KL_gauss_rand_walk} with respect to $\boldsymbol{\tau}$ under the additional assumptions of this Lemma, and plugging the result into Lemma~\ref{lemma:variational_bound_general} yields the desired conclusion.
\end{proof}
Next we bound $\Lq - \Llocked$.
\begin{lem}\label{lemma:Lq_minus_Llocked_general}
Assume that there is a $c>0$ that bounds the second derivative as in (\ref{eq:second_derivative_assumption}).
Assume that there is an $R$ such that $\frac{1}{n(\tau_{j+1}-\tau_{j})}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\preceq R^{2}I$ for all $j\in\{1, 2, \ldots, |\boldsymbol{\tau}|\}$.
Let $\Qwalk$ be the zero-centered Gaussian random walk with $\mu_1 = \thetainit$.
Then it holds that
\[
\Lqwalk - \Llocked \le
\frac{cnR^{2} }{2} \Tr(\Sigmainit) \left(
T \epsilon_1^{2} +
\nu^2 \sum_{j=2}^{|\tau|}(\tau_{j + 1} - \tau_{j})(j - 1)
\right ).
\]
\end{lem}
\begin{proof}
We use a Taylor expansion. For $j = 1,...,|\tau|$, there is some $\theta_{mid}$ such that
\begin{align}
\begin{split}
-\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n} \log p\left(y_{t,i}\mid z_{t,i};\theta_{\tau_j} \right)
=&-\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\thetainit\right) -\left.\nabla_{\theta}\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right|_{\theta=\thetainit}^{\top}\left(\theta_{\tau_j}-\thetainit\right)\\
& \ -\frac{1}{2}\left(\theta_{\tau_j}-\thetainit\right)^{\top}\left.\nabla_{\theta}^{2}\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right|_{\theta=\theta_{mid}}\left(\theta_{\tau_j}-\thetainit\right).
\end{split}
\label{eq:taylor_expansion_general}
\end{align}
Note that
\[
\left(\theta_{\tau_j}-\thetainit\right)^{\top}\nabla_{\theta}^{2}\log p\left(y\mid z;\theta\right)\left(\theta_{\tau_j}-\thetainit\right)=\frac{\partial^{2}}{\partial w^{2}}\log p\left(y|w\right)\left(z^{\top}\left(\theta_{\tau_j}-\thetainit\right)\right)^{2},
\]
where $w=z^{\top}\theta$ is the predicted logit. Using equation (\ref{eq:second_derivative_assumption}) it follows that
\begin{align}
\label{eq:quadratic_form_general}
& \left|\frac{1}{2}\left(\theta_{\tau_j}-\thetainit\right)^{\top}\left.\nabla_{\theta}^{2}\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right|_{\theta=\theta_{mid}}\left(\theta_{\tau_j}-\thetainit\right)\right|\\
\le&\frac{c}{2}\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}\left(z_{t,i}^{\top}\left(\theta_{\tau_j}-\thetainit\right)\right)^{2}\\
= &\frac{c}{2}\left(\theta_{\tau_j}-\thetainit\right)^\top \left(\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\theta_{\tau_j}-\thetainit\right).
\end{align}
Because the expected value of $\theta$ with respect to $Q$ is $\thetainit$, we have the following after taking the expectation of equation (\ref{eq:taylor_expansion_general}) combined with equation (\ref{eq:quadratic_form_general}):
\begin{align*}
\Lq &= \E_{Q}\left[-\sum_{t=1}^{T}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta_t\right)\right] \\
&\leq \Llocked + \sum_{j=1}^{|\tau|} \frac{c}{2} \E_{Q}\left[\left(\theta_{\tau_j}-\thetainit\right)^{\top}\left(\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\theta_{\tau_j}-\thetainit\right)\right].
\end{align*}
Assuming there exists some $R^2$ that satisfies the lemma assumptions, the following holds after taking the expectation with respect to $Q$:
\begin{align*}
\E_{Q}\left[\left(\theta_{\tau_j}-\thetainit\right)^{\top}\left(\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\theta_{\tau_j}-\thetainit\right)\right]
&\le (\tau_{j + 1} - \tau_{j})nR^{2} \E_{Q}\|\theta_{\tau_j}-\thetainit\|^{2}\\
&= (\tau_{j + 1} - \tau_{j}) nR^{2} \left(\epsilon_1^{2} + (j - 1) \nu^2 \right) \Tr(\Sigmainit).
\end{align*}
After summing over $j = 1,...,|\tau|$, we reach our desired result.
\end{proof}
We combine the two prior lemmas to obtain the following bound on the Type I error for MarBLR.
\begin{thm}[Type I regret for MarBLR]
Let $\xi = \E_{p_0} |\boldsymbol{\tau}|$ denote the expected number of shift times be denoted.
The Type I regret for MarBLR is bounded as follows:
\begin{align*}
\LMarBLR - \Llocked
&\le
\frac{d}{2} \log \left(
1 + \frac{cnR^{2} T \Tr(\Sigmainit)}{d}
\right) \\
&\quad\quad +
\frac{d \alpha (T - 1)}{2} \log \left(
1 + \frac{\delta^2 c nR^{2} T \Tr(\Sigmainit)}{2 d}
\right).
\end{align*}
\label{thrm:type_i_marblr}
\end{thm}
\begin{proof}
First, note that under the MarBLR prior $p_0$ over shift times $\boldsymbol{\tau}$ as defined previously, we have that
$$
\E_{p_0} \left[
\sum_{j=2}^{|\tau|}(\tau_{j + 1} - \tau_{j})(j - 1)
\right]
=
\E_{p_0}\left[\sum_{t=2}^T W_t (T + 1 - t)\right]
= \frac{\alpha}{2} T(T - 1),
$$
% [here is why:
% - the second sum starts from t=2 because W_1 = 1 is fixed.
% - note that the factor (T + 1 - t) comes from the fact that \tau_{|\tau|+1} = T+1.
% - we have E(sum_2^T W_t (T+1-t)) = alpha ((T-1)(T+1) - sum_2^T t) = alpha ((T-1)(T+1) - T(T+1)/2 + 1) = alpha (T(T+1) - T - 1 - T(T+1)/2 + 1) = alpha (T(T+1)/2 - T) = alpha (T(T-1)/2)
% ]
and $\xi = \E_{p_{0}}|\boldsymbol{\tau}| = \alpha (T - 1) + 1$.
Thus, summing the upper bounds from Lemmas~\ref{lemma:KL_margin_walk} and \ref{lemma:Lq_minus_Llocked_general} and taking expectations with respect to $\boldsymbol{\tau} \sim p_0$, we have that
\begin{align}
\Lbf - \Llocked
& \le
\frac{1}{2}\epsilon_1^{2}d -\frac{d \alpha (T - 1)}{2} - \frac{d}{2} + d \alpha (T - 1) \log\frac{\delta}{\nu} - d\log \epsilon_1 + \frac{d \alpha (T - 1)}{2 \delta^2}\nu^2
\\
&
+ \frac{cnR^{2} T}{2} \Tr(\Sigmainit) \left(
\epsilon_1^{2} +
\frac{\alpha}{2} (T - 1) \nu^2
\right ).
\end{align}
We minimize the upper bound by selecting
$$
\epsilon_1^2 = \frac{d}{d + cnR^{2} T \Tr(\Sigmainit)}
$$
and
$$
\nu^2 = \frac{d}{\frac{d}{\delta^2} + \frac{c}{2} nR^{2} T \Tr(\Sigmainit)}
$$
to obtain the upper bound
$$
\frac{d}{2} \log \left(
1 + \frac{cnR^{2} T \Tr(\Sigmainit)}{d}
\right)
+
\frac{d \alpha (T - 1)}{2} \log \left(
1 + \frac{\delta^2 c nR^{2} T \Tr(\Sigmainit)}{2 d}
\right).
$$
\end{proof}
\subsection{Type II $\boldsymbol{\tau}$-regret results for BLR}
Let $\thetataulock$ be the minimizer of the cumulative log-likelihood of the locked model, i.e., $\thetataulock$ satisfies that
\[
\left.\nabla\sum_{t=1}^{T}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right|_{\theta=\tilde{\theta}_{\tau_{\mathrm{locked}}}}=0.
\]
Let $\tilde{Q}$ denote the distribution $\Qtaulocked$ (defined according to section \ref{sec:notation} and equation (\ref{eq:Qtaumusigma}) with the parameters specified here). That is, we have that
\[
\tilde{Q}(\theta_1) = \Qtaulocked(\theta_1)=N\left(\tilde{\theta}_{\tau_{locked}},\epsilon^{2}\Sigmainit\right),
\]
and $\theta_t = \theta_1$ for all $t\in\left\{2, 3, \ldots, T\right\}$.
We bound the difference in the cumulative negative log-likelihood,
$\Lblr - \Ldyntau$, by breaking it into two summands
\begin{equation}\label{eq:blr_decomposition}
\Lblr - \Ldyntau = \left(\Lblr- L_{\tilde{Q}}\right) + \left(L_{\tilde{Q}} - \Ldyntau\right).
\end{equation}
We have already bounded the first summand by Lemmas~\ref{lemma:variational_bound_general} and \ref{lemma:KL_gauss_rand_walk}. We just need to bound the second summand.
\begin{lem}\label{lemma:blr}
Assume that the second derivative is bounded by a constant $c$ as shown in equation (\ref{eq:second_derivative_assumption}), and that there are $R_{\tau_1}, R_{\tau_2}, \ldots, R_{\tau_{|\boldsymbol{\tau}|}}$ such that
$$\frac{1}{n(\tau_{j+1}-\tau_{j})}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\preceq R_{j}^{2}I.$$
It holds that
\begin{align*}
L_{\tilde{Q}} - \Ldyntau
& \le \frac{cn \sum_{j=1}^{|\boldsymbol{\tau}|}R_j^2\left(\tau_{j+1}-\tau_{j}\right)}{2} \epsilon^{2}\Tr(\Sigmainit)
+ \frac{cn}{2}\sum_{j=1}^{|\boldsymbol{\tau}|}R_j^2\left(\tau_{j+1}-\tau_{j}\right)\left\Vert \tilde{\theta}_{\tau_{locked}}-\tilde{\theta}_{\tau_j}\right\Vert ^{2}.
\end{align*}
\end{lem}
\begin{proof}
Because $\tilde{\theta}_{\tau_j}$ is the minimizer of $\nabla_{\theta}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)$, per Taylor's expansion there is some $\theta_{\mathrm{mid}}$ such that
\begin{multline*}
-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right) = -\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\tilde{\theta}_{\tau_j}\right)\\
-\frac{1}{2}\left(\theta-\tilde{\theta}_{\tau_j}\right)^{\top}\left.\nabla_{\theta}^{2}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right|_{\theta=\theta_{\mathrm{mid}}}\left(\theta-\tilde{\theta}_{\tau_j}\right).
\end{multline*}
Following the same arguments as in the proof of Lemma \ref{lemma:Lq_minus_Llocked_general}, we have that
\begin{multline*}
\E_{\tilde{Q}}\left[-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta\right)\right]
\le-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\tilde{\theta}_{\tau_j}\right) \\
+\frac{c}{2} \E_{\tilde{Q}}\left[\left(\theta_1-\tilde{\theta}_{\tau_j}\right)^{\top}\left(\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\theta_1-\tilde{\theta}_{\tau_j}\right)\right].
\end{multline*}
Taking expectation with respect to $\tilde{Q}$, we note that
\begin{align*}
& \E_{\tilde{Q}}\left[\left(\theta_1-\tilde{\theta}_{\tau_j}\right)^{\top} \left(\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right) \left(\theta_1-\tilde{\theta}_{\tau_j}\right)\right]\\
& = \E_{\tilde{Q}}\left[\left(\theta_1-\thetataulock\right)^{\top} \left(\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right) \left(\theta_1-\thetataulock\right)\right]
+\left(\thetataulock-\tilde{\theta}_{\tau_j}\right)^{\top}\left(\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\thetataulock-\tilde{\theta}_{\tau_j}\right)\\
& \le \left(\tau_{j+1}-\tau_{j}\right)nR_{j}^{2}\epsilon^{2} \Tr\left(\Sigmainit\right)+\left(\tau_{j+1}-\tau_{j}\right)nR_{j}^{2}\left\Vert \thetataulock-\tilde{\theta}_{\tau_j}\right\Vert ^{2}.
\end{align*}
We arrive at our results after summing over all $j = 1,...,|\tau|$.
\end{proof}
\begin{thm}[Type II regret for BLR]
Assume that there is an $R$ such that $\frac{1}{n(\tau_{j+1}-\tau_{j})}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\preceq R^{2}I$ for all $j\in\{1, 2, \ldots, |\boldsymbol{\tau}|\}$. It holds that
\begin{align*}
\Lblr - \Ldyntau & \le\frac{1}{2}\left(\thetataulock-\thetainit\right)^{\top}\Sigmainit^{-1}\left(\thetataulock-\thetainit\right) + \frac{d}{2}\log\left(\frac{d+cnTR^{2}\Tr(\Sigmainit)}{d}\right) \\
&\quad\quad+ \frac{cnR^2}{2}\sum_{j=1}^{|\tau|}\left(\tau_{j+1}-\tau_{j}\right)\left\Vert \thetataulock-\tilde{\theta}_{\tau_j}\right\Vert^{2}.
\end{align*}
\end{thm}
\begin{proof}
To bound the first summand of decomposition (\ref{eq:blr_decomposition}), we use Lemmas~\ref{lemma:variational_bound_general} and \ref{lemma:KL_gauss_rand_walk} and the fact that $p_0(\taulock) = 1$ under BLR. We use Lemma \ref{lemma:blr} to bound the second summand of decomposition (\ref{eq:blr_decomposition}). Thus, we obtain
\begin{align*}
\Lblr - \Llocked \le \frac{1}{2}\epsilon^2 d
&+ \frac{1}{2}\left(\tilde{\theta}_{\tau_{locked}}-\thetainit\right)^{\top}\Sigma^{-1}\left(\tilde{\theta}_{\tau_{locked}}-\thetainit\right) + \frac{d}{2} - d \log(\epsilon) + \frac{cnTR^2}{2} \epsilon^2 \Tr(\Sigmainit) \\
&+ \frac{cnR}{2}\sum_{j=1}^{|\tau|}\left(\tau_{j+1}-\tau_{j}\right)\left\Vert \tilde{\theta}_{\tau_{locked}}-\tilde{\theta}_{\tau,j}\right\Vert ^{2}.
\end{align*}
Choosing $\epsilon^2 = \frac{d}{d + cnTR^2\Tr(\Sigmainit)}$ will minimize this expression, which yields the desired conclusion.
\end{proof}
\subsection{Type II $\boldsymbol{\tau}$-regret results for MarBLR}
As before, we bound the difference in the cumulative negative log-likelihood,
$\Lbf-\Ldyntau$, by breaking it into two summands
\begin{equation}
\Lbf-\Ldyntau=\left(\Lbf-L_{Q_{\boldsymbol{\tau}',\tilde{\boldsymbol{\theta}}',\boldsymbol{\epsilon}^2 \Sigmainit}}\right)+\left(L_{Q_{\boldsymbol{\tau}',\tilde{\boldsymbol{\theta}}',\boldsymbol{\epsilon}^2 \Sigmainit}}-\Ldyntau\right).
\label{eq:type2_MarBLR_decomp}
\end{equation}
Thus the proof proceeds by comparing against an intermediary distribution $Q_{\boldsymbol{\tau}', \tilde{\boldsymbol{\theta}}',\boldsymbol{\epsilon}^2 \Sigmainit}$ defined per \eqref{eq:Qtaumusigma}, where $\boldsymbol{\tau}'$ be any subsequence of $\boldsymbol{\tau}$ with $\tau'_{1}=1$, $\tilde{\boldsymbol{\theta}}' := (\tilde{\theta}_t)_{t\in\boldsymbol{\tau}'}$, and $\boldsymbol{\epsilon}^2 = \left(\epsilon_1^2, \epsilon_2^2, \ldots, \epsilon_{|\boldsymbol{\tau}'|}^2 \right)$.
This intermediary distribution is centered around a dynamic oracle that may evolve slower than than the specified update times $\boldsymbol{\tau}$.
The final Type II regret bound will depend on $\boldsymbol{\tau}'$.
Optimizing our choice of $\boldsymbol{\tau}'$ can lead to tighter Type Ii regret bounds, particularly when $\alpha$ in the MarBLR prior is small and $|\tau|$ is large.
We use the following lemma to bound the first summand of \eqref{eq:type2_MarBLR_decomp}.
\begin{lem}\label{lemma:KL_Q_p0}
Consider the distribution $Q_{\boldsymbol{\tau},\boldsymbol{\mu}, \boldsymbol{\epsilon}^2 \Sigmainit}$ as defined above, and the MarBLR prior $p_0$ as defined per \eqref{eq:MarBLR_prior_theta_1} and \eqref{eq:MarBLR_prior_theta_t}.
For any $\boldsymbol{\tau}$, $\boldsymbol{\mu}$ and $\boldsymbol{\epsilon}^2$, we have that
\begin{align*}
& KL\left(Q_{\boldsymbol{\tau},\boldsymbol{\mu}, \boldsymbol{\epsilon}^2 \Sigmainit} \mid\mid p_{0}\left(\boldsymbol{\theta}\mid\tau\right)\right) \\
& = \frac{d}{2}\epsilon_1^2 + \frac{1}{2} \left(\mu_1 - \thetainit\right)^\top \Sigmainit^{-1} \left(\mu_1 - \thetainit\right)
-d\log \epsilon_1 - \frac{d}{2}|\boldsymbol{\tau}| + (|\boldsymbol{\tau}| - 1) d \log \delta \\
& + \sum_{t=2}^{|\boldsymbol{\tau}|} \left[ \frac{1}{2\delta^2} \left(d\left(\epsilon_{t-1}^2 + \epsilon_{t}^2\right) + \left(\mu_t - \mu_{t-1}\right)^\top \Sigmainit^{-1} \left(\mu_t - \mu_{t-1}\right)\right) - d\log \epsilon_t\right].
\end{align*}
\end{lem}
\begin{proof}
We define $\Theta_{J}$ and $\psub$ as in Lemma~\ref{lemma:KL_gauss_rand_walk}.
We define $\Qsub$ as the distribution over $\Theta_{J}$ as defined by $Q_{\boldsymbol{\tau},\boldsymbol{\mu}, \boldsymbol{\epsilon}^2 \Sigmainit}$.
We have that
\begin{align}
KL\left(Q_{\tau,(\mu_{t})_{t\in\tau}}\mid\mid p_{0}\left(\boldsymbol{\theta}\mid\tau\right)\right) &= KL\left(\Qsub\mid\mid \psub\right) \nonumber \\
& =\int\cdots\int \Qsub(\boldsymbol{\theta}) \sum_{t=1}^{|\boldsymbol{\tau}|}\log\frac{\Qsub\left(\theta_{t}\right)}{\psub\left(\theta_{t}\mid\theta_{t-1}\right)}d\theta_{|\boldsymbol{\tau}|}\cdots d\theta_{1}, \label{eq:kl_integral}
\end{align}
because $\theta_{t}$ in $\Qsub$
are jointly independent and $\theta_{t}$ in $\psub$ only depend
on $\theta_{t-1}$.
As such,
\begin{align}
KL\left(\Qsub\mid\mid \psub\right) & =\int \Qsub\left(\theta_{1}\right)\log\frac{\Qsub\left(\theta_{1}\right)}{\psub\left(\theta_{1}\right)}d\theta_{1} \label{eq:kl_summand_1}\\
&\quad +\sum_{t=2}^{|\boldsymbol{\tau}|}\int\int \Qsub\left(\theta_{t-1},\theta_{t}\right)\log\frac{\Qsub\left(\theta_{t}\right)}{\psub\left(\theta_{t}\mid\theta_{t-1}\right)}d\theta_{t}d\theta_{t-1}. \label{eq:kl_summand_2}
\end{align}
The first term \eqref{eq:kl_summand_1} is the KL divergence of two multivariate Normal distributions, $N(\mu_1, \epsilon_1^2 \Sigmainit)$ and $N(\thetainit, \Sigmainit)$, and can be shown to be equal to
\begin{equation}
\int \Qsub\left(\theta_{1}\right)\log\frac{\Qsub\left(\theta_{1}\right)}{\psub\left(\theta_{1}\right)}d\theta_{1}
= \frac{1}{2}\epsilon_1^{2}d+\frac{1}{2}\left(\mu_1-\thetainit\right)^{\top}\Sigmainit^{-1}\left(\mu_1-\thetainit\right)-d\log\epsilon_1-\frac{d}{2}.
\label{eq:KL_intermed_qsub_1}
\end{equation}
Next each term in the summation of \eqref{eq:kl_summand_2} is equal to
\begin{align}
& \int \int \Qsub({\theta}_{t-1}, {\theta}_{t})
\log \frac{\Qsub({\theta}_{t})}{\psub({\theta}_{t}\mid {\theta}_{t-1})}
\mathrm{d}{\theta}_{t} \mathrm{d}{\theta}_{t-1} \nonumber \\
= &
d \log\frac{\delta}{\epsilon_t}
+\frac{1}{2\delta^{2}}
\int \int \Qsub({\theta}_{t-1}, {\theta}_{t})
\left(
{\theta}_{t} - {\theta}_{t-1}
\right)^\top
\Sigmainit^{-1}
\left(
{\theta}_{t} - {\theta}_{t-1}
\right)
\mathrm{d}{\theta}_{t} \mathrm{d}{\theta}_{t-1}
-\frac{d}{2}.
\label{eq:qq_intermed_qsub}
\end{align}
We note that under $\Qsub$ it holds that
\begin{equation*}
(\theta_{t}-\theta_{t-1}) \sim N\left(\mu_{t}-\mu_{t-1}, (\epsilon_{t-1}^{2}+\epsilon_{t}^{2})\Sigmainit\right).
\end{equation*}
Therefore, \eqref{eq:qq_intermed_qsub} simplifies to
\begin{align}
& \int \int \Qsub({\theta}_{t-1}, {\theta}_{t})
\log \frac{\Qsub({\theta}_{t})}{\psub({\theta}_{t}\mid {\theta}_{t-1})}
\mathrm{d}{\theta}_{t} \mathrm{d}{\theta}_{t-1} \nonumber \\
= & d \log\frac{\delta}{\epsilon_t}
+\frac{1}{2\delta^{2}}
\left(
d\left(\epsilon_{t-1}^{2}+\epsilon_{t}^{2}\right) + (\mu_{t}-\mu_{t-1})^\top \Sigmainit^{-1} (\mu_{t}-\mu_{t-1})
\right)
-\frac{d}{2}.
\label{eq:KL_intermed_qsub_2}
\end{align}
Plugging \eqref{eq:KL_intermed_qsub_1} and \eqref{eq:KL_intermed_qsub_2} into \eqref{eq:kl_summand_1} and \eqref{eq:kl_summand_2} gives us the desired result.
\end{proof}
Next we need to bound the second summand of \eqref{eq:type2_MarBLR_decomp}.
\begin{lem}\label{lemma:Lq_minus_Ldyn}
Suppose there is a constant $c$ that bounds the second derivative as in \eqref{eq:second_derivative_assumption}.
Assume that there is an $R$ such that $\frac{1}{n(\tau_{j+1}-\tau_{j})}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\preceq R^{2}I$ for all $j\in\{1, 2, \ldots, |\boldsymbol{\tau}|\}$.
Then it holds that
\begin{align*}
L_{Q_{\boldsymbol{\tau}',\tilde{\boldsymbol{\theta}}',\boldsymbol{\epsilon}^2 \Sigmainit}}-\Ldyntau
& \le \frac{1}{2} c nR^{2} \sum_{j=1}^{|\boldsymbol{\tau}|} (\tau_{j + 1} - \tau_{j})
\left( \epsilon_{k(j)}^2 \Tr(\Sigmainit) + \left\| \tilde{\theta}_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2} \right)
\end{align*}
where $k(j) := \max \{k : \tau'_k \leq \tau_j\}$.
\end{lem}
\begin{proof}
For the ease of notation denote $\tilde{Q} := Q_{\boldsymbol{\tau}',\tilde{\boldsymbol{\theta}}',\boldsymbol{\epsilon}^2 \Sigmainit}$.
It holds that
\begin{align*}
&L_{\tilde{Q}}-\Ldyntau \\
&= \sum_{j=1}^{|\boldsymbol{\tau}|}\left(E_{\tilde{Q}}\left[\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}-\log p\left(y_{t,i}\mid z_{t,i};\theta_{t}\right)+\log p\left(y_{t,i}\mid z_{t,i};\tilde{\theta}_{\tau_{j}}\right)\right]\right)
\end{align*}
Recall that for any sequence $\boldsymbol{\theta}$ drawn from $\tilde{Q}$, for any $j = 1, \ldots, |\boldsymbol{\tau}|$, the parameters $\theta_{t}$ are constant over $t=\tau'_{j}, \ldots, \tau'_{j+1}-1$.
Taking a Taylor expansion, there exists some $\theta_{mid}$ such that
\begin{align}
\begin{split}
-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i} \middle| z_{t,i};\theta\right)
&=-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i} \middle| z_{t,i};\tilde{\theta}_{\tau_{j}}\right) \\
&\quad\quad -\left.\nabla_{\theta}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\middle| z_{t,i};\theta\right)\right|_{\theta=\tilde{\theta}_{\tau_{j}}}^{\top}\left(\theta-\tilde{\theta}_{\tau_{j}}\right) \\
&\quad\quad -\frac{1}{2}\left(\theta-\tilde{\theta}_{\tau_{j}}\right)^{\top}\left.\nabla_{\theta}^{2}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\middle| z_{t,i};\theta\right)\right|_{\theta=\theta_{mid}}\left(\theta-\tilde{\theta}_{\tau_{j}}\right). \label{eq:MarBLR_taylor}
\end{split}
\end{align}
Since $\boldsymbol{\tau}'$ is a subsequence of $\boldsymbol{\tau}$, for $\boldsymbol{\theta} \sim \tilde{Q}$ we have that $\theta_t = \theta_{\tau'_{k(j)}}$ for all $t=\tau_{j},....,\tau_{j+1}-1$, where $k(j) := \max \{k : \tau'_k \leq \tau_j\}$. Thus, we can use the above decomposition to evaluate \eqref{eq:MarBLR_taylor} with $\theta_t = \theta_{\tau'_{k_j}}$ in place of $\theta$.
By the definition of $\tilde{\theta}_{\tau_{j}}$, the gradient in the expression above is zero, so the second term is equal to zero. Because we assumed the second derivative was bounded by $c$ as in \eqref{eq:second_derivative_assumption}, the expression simplifies to the bound
\begin{align*}
-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\theta_t\right) & \le-\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}\log p\left(y_{t,i}\mid z_{t,i};\tilde{\theta}_{\tau_{j}}\right) \\
&\quad + \frac{c}{2}\left(\theta_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}}\right)^\top \left(\sum_{t=\tau_j}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\right)\left(\theta_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}}\right).
\end{align*}
Assuming there exists some $R^2$ that satisfies the lemma assumptions, it follows that
\begin{align*}
&\E_{\tilde{Q}}\left[\left( \theta_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right)^{\top} \left(\sum_{t=\tau_j}^{\tau_{j + 1} - 1}\sum_{i=1}^{n} z_{t,i} z_{t,i}^{\top}\right)\left( \theta_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right)\right] \\
&\le (\tau_{j + 1} - \tau_{j})nR^{2} \E_{\tilde{Q}}\left\| \theta_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2} \\
&= (\tau_{j + 1} - \tau_{j})nR^{2} \left( \epsilon_{k(j)}^2 \Tr(\Sigmainit) + \left\| \tilde{\theta}_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2} \right).
\end{align*}
We finish the proof by summing over $j$.
\end{proof}
We combine the results to get the following bound.
\begin{thm}[Type II regret for MarBLR] \label{thm:type2_regret_MarBLR}
Suppose there is a constant $c$ that bounds the second derivative as in \eqref{eq:second_derivative_assumption}.
Assume that there is an $R$ such that $\frac{1}{n(\tau_{j+1}-\tau_{j})}\sum_{t=\tau_{j}}^{\tau_{j+1}-1}\sum_{i=1}^{n}z_{t,i}z_{t,i}^{\top}\preceq R^{2}I$ for all $j\in\{1, 2, \ldots, |\boldsymbol{\tau}|\}$.
Let $\boldsymbol{\tau}'$ be any subsequence of the sequence of shift times $\boldsymbol{\tau}$.
Then it holds that
\begin{align*}
&\LMarBLR-\Ldyntau \\
&\leq \frac{1}{2} \left(\tilde{\theta}_1 - \thetainit\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_1 - \thetainit\right)
+ \frac{d}{2} \log\left(1 + \frac{1}{\delta^2} + \frac{cnR^2 \Tr(\Sigmainit)\left( \tau'_2 - \tau'_1 \right)}{d}\right) \\
& + \frac{1}{2} \sum_{t=2}^{|\boldsymbol{\tau}'|} \left[ \frac{1}{\delta^2} \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)
+ d \log\left(\frac{2}{\delta^2} + \frac{cnR^2 \Tr(\Sigmainit) \left(\tau'_{j+1} - \tau'_j\right)}{d}\right) \right] \\
& - \log p_0(\boldsymbol{\tau}') + (|\boldsymbol{\tau}'| - 1) d \log \delta
+ \frac{1}{2} c nR^{2} \sum_{j=1}^{|\boldsymbol{\tau}|} (\tau_{j + 1} - \tau_{j})
\left\| \tilde{\theta}_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2}.
\end{align*}
\end{thm}
\begin{proof}
By combining Lemmas \ref{lemma:variational_bound_general}, \ref{lemma:KL_Q_p0} and \ref{lemma:Lq_minus_Ldyn} we obtain the following upper bound
\begin{equation}
\begin{aligned}
\Lbf-\Ldyntau
&\leq
\frac{d}{2}\epsilon_1^2 + \frac{1}{2} \left(\tilde{\theta}_1 - \thetainit\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_1 - \thetainit\right)
-d\log \epsilon_1 - \frac{d}{2}|\boldsymbol{\tau}'| + (|\boldsymbol{\tau}'| - 1) d \log \delta \\
& + \sum_{t=2}^{|\boldsymbol{\tau}'|} \left[ \frac{1}{2\delta^2} \left(d\left(\epsilon_{t-1}^2 + \epsilon_{t}^2\right) + \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)\right) - d\log \epsilon_t\right] \\
& - \log p_0(\boldsymbol{\tau}')
+ \frac{1}{2} c nR^{2} \sum_{j=1}^{|\boldsymbol{\tau}|} (\tau_{j + 1} - \tau_{j})
\left( \epsilon_{k(j)}^2 \Tr(\Sigmainit) + \left\| \tilde{\theta}_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2} \right).
\end{aligned}
\label{eq:upper_bound_MarBLR_intermed}
\end{equation}
We minimize the upper bound with respect to $\left(\epsilon_j\right)_{j=1,2,\ldots,|\boldsymbol{\tau}'|}$.
For $j=1$, $\epsilon_{1}$ only contributes to the above bound through the terms
\begin{equation}
\frac{1}{2}\left( d + \frac{d}{\delta^2} + cnR^2 \Tr(\Sigmainit)\left( \tau'_2 - \tau'_1 \right) \right) \epsilon_1^2 - d \log \epsilon_1.
\label{eq:eps_1_term}
\end{equation}
For $j=2,...,|\tau'|-1$, $\epsilon_{j}$ only contributes to the bound through the terms
\begin{equation}
\frac{1}{2} \left[ \frac{2d}{\delta^2} + cnR^2 \Tr(\Sigmainit) \left(\tau'_{j+1} - \tau'_j\right) \right] \epsilon_j^2 - d \log \epsilon_j.
\label{eq:eps_j_term}
\end{equation}
For $j=|\boldsymbol{\tau}'|$, $\epsilon_{|\boldsymbol{\tau}'|}$ only contributes to the bound through the terms
\begin{equation}
\frac{1}{2} \left[ \frac{d}{\delta^2} + cnR^2 \Tr(\Sigmainit) \left(\tau'_{|\boldsymbol{\tau}'|+1} - \tau'_{|\boldsymbol{\tau}'|}\right) \right] \epsilon_{|\boldsymbol{\tau}'|}^2 - d \log \epsilon_{|\boldsymbol{\tau}'|}.
\label{eq:eps_tau_term}
\end{equation}
It follows that the upper bound is minimized for
\begin{align}
\epsilon_1^2 &= \frac{d}{d + \frac{d}{\delta^2} + cnR^2 \Tr(\Sigmainit)\left( \tau'_2 - \tau'_1 \right)}, \label{eq:min_eps_1} \\
\epsilon_j^2 &= \frac{d}{\frac{2d}{\delta^2} + cnR^2 \Tr(\Sigmainit) \left(\tau'_{j+1} - \tau'_j\right) }, \quad\forall j \in \left\{ 2, 3, \ldots, |\boldsymbol{\tau}'|-1 \right\}, \label{eq:min_eps_j} \\
\epsilon_{|\boldsymbol{\tau}'|} &= \frac{d}{\frac{d}{\delta^2} + cnR^2 \Tr(\Sigmainit) \left(\tau'_{|\boldsymbol{\tau}'|+1} - \tau'_{|\boldsymbol{\tau}'|}\right) }. \label{eq:min_eps_tau}
\end{align}
Note that the upper bound \eqref{eq:upper_bound_MarBLR_intermed} is a sum of the terms \eqref{eq:eps_1_term}, \eqref{eq:eps_j_term} repeated once for each $j \in \left\{ 2, 3, \ldots, |\boldsymbol{\tau}'|-1 \right\}$, \eqref{eq:eps_tau_term}, and the following remaining terms
\begin{align*}
&\frac{1}{2} \left(\tilde{\theta}_1 - \thetainit\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_1 - \thetainit\right)
- \frac{d}{2}|\boldsymbol{\tau}'| + (|\boldsymbol{\tau}'| - 1) d \log \delta - \log p_0(\boldsymbol{\tau}') \\
& + \sum_{t=2}^{|\boldsymbol{\tau}'|} \frac{1}{2\delta^2} \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)^\top \Sigmainit^{-1} \left(\tilde{\theta}_{\tau'_t} - \tilde{\theta}_{\tau'_{t-1}}\right)
+ \frac{1}{2} c nR^{2} \sum_{j=1}^{|\boldsymbol{\tau}|} (\tau_{j + 1} - \tau_{j})
\left\| \tilde{\theta}_{\tau'_{k(j)}}-\tilde{\theta}_{\tau_{j}} \right\|^{2}.
\end{align*}
Plugging in \eqref{eq:min_eps_1}, \eqref{eq:min_eps_j} and \eqref{eq:min_eps_tau}, we get the desired bound.
\end{proof}
\begin{figure}
\centering
\begin{subfigure}{0.49\linewidth}
\begin{center}
\includegraphics[width=0.6\linewidth]{simulation_refit_thetas_iid_good}
\end{center}
\caption{\texttt{Initial Shift}, \texttt{All-Refit}}
\label{fig:iid_good}
\end{subfigure}
\begin{subfigure}{0.49\linewidth}
\includegraphics[width=0.6\linewidth]{simulation_refit_thetas_drift_good}
\caption{\texttt{Decay}, \texttt{All-Refit}}
\label{fig:shift_good}
\end{subfigure}
\begin{subfigure}{0.49\linewidth}
\begin{center}
\includegraphics[width=0.6\linewidth]{simulation_refit_thetas_iid_bad}
\end{center}
\caption{\texttt{Initial Shift}, \texttt{Subset-Refit}}
\label{fig:iid_bad}
\end{subfigure}
\begin{subfigure}{0.49\linewidth}
\includegraphics[width=0.6\linewidth]{simulation_refit_thetas_drift_bad}
\includegraphics[width=0.35\linewidth]{legend_thetas}
\caption{\texttt{Decay}, \texttt{Subset-Refit}}
\label{fig:shift_bad}
\end{subfigure}
\caption{Evolution of the estimated intercepts and coefficients by BLR and MarBLR when combining the original model with an evolving prediction model (Scenario 3).
Data is simulated to be stationary over time after an initial shift (\texttt{Initial Shift}) and nonstationary such that the original model decays in performance over time (\texttt{Decay}).
Underlying prediction model is updated by continually refitting on all previous data (\texttt{All-Refit}) or refit on the most recent subset of data (\texttt{Subset-Refit}).
}
\label{fig:refitting_thetas}
\end{figure}
\begin{figure}
\centering
\begin{subfigure}{\linewidth}
\centering
\includegraphics[width=0.5\linewidth]{copd_linear0_thetas.png}
\caption{Online recalibration of a fixed prediction model}
\label{fig:online_recalib_copd}
\end{subfigure}
\begin{subfigure}{\linewidth}
\centering
\includegraphics[width=0.5\linewidth]{copd_linear1_thetas.png}
\caption{Online logistic revision with respect to a fixed prediction model and patient variables}
\label{fig:online_linear_copd}
\end{subfigure}
\begin{subfigure}{\linewidth}
\centering
\includegraphics[width=0.5\linewidth]{copd_combo_thetas.png}
\caption{Online ensembling of the original and continually-refitted prediction models}
\label{fig:online_refit_copd}
\end{subfigure}
\caption{Evolution of the estimated intercepts and coefficients for online recalibration and revision of a fixed COPD risk prediction model (a and b, respectively) and online reweighting for fixed and continually-refitted (evolving) COPD risk prediction models using BLR and MarBLR.
}
\label{fig:copd_thetas}
\end{figure}
\pagebreak
\bibliographystyle{plainnat}
\bibliography{main}
\end{document}
\begin{thebibliography}{4}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Gordon and Smith(1990)]{Gordon1990-jj}
K~Gordon and A~F~M Smith.
\newblock Modeling and monitoring biomedical time series.
\newblock \emph{J. Am. Stat. Assoc.}, 85\penalty0 (410):\penalty0 328--337,
June 1990.
\newblock URL \url{https://doi.org/10.1080/01621459.1990.10476205}.
\bibitem[Orguner and Dem{\i}rekler(2007)]{Orguner2007-st}
U~Orguner and M~Dem{\i}rekler.
\newblock Analysis of single gaussian approximation of gaussian mixtures in
bayesian filtering applied to mixed multiple-model estimation.
\newblock \emph{Int. J. Control}, 80\penalty0 (6):\penalty0 952--967, June
2007.
\newblock URL \url{https://doi.org/10.1080/00207170701261952}.
\bibitem[Polson et~al.(2013)Polson, Scott, and Windle]{Polson2013-cr}
Nicholas~G Polson, James~G Scott, and Jesse Windle.
\newblock Bayesian inference for logistic models using {P{\'o}lya--Gamma}
latent variables.
\newblock \emph{J. Am. Stat. Assoc.}, 108\penalty0 (504):\penalty0 1339--1349,
December 2013.
\newblock URL \url{https://doi.org/10.1080/01621459.2013.829001}.
\bibitem[West and Harrison(1997)]{West1997-fa}
Mike West and Jeff Harrison.
\newblock \emph{Bayesian Forecasting and Dynamic Models}.
\newblock Springer, New York, NY, 1997.
\newblock URL \url{https://link.springer.com/book/10.1007%2Fb98971}.
\end{thebibliography}