Event-Triggered Time-Varying Bayesian Optimization
Extracted source text · arXiv 2208.10790
\documentclass{article}
% if you need to pass options to natbib, use, e.g.:
% \PassOptionsToPackage{numbers, compress}{natbib}
% before loading neurips_2022
% ready for submission
% \usepackage{neurips_2022}
% to compile a preprint version, e.g., for submission to arXiv, add add the
% [preprint] option:
\usepackage[preprint,nonatbib]{neurips_2022}
% to compile a camera-ready version, add the [final] option, e.g.:
% \usepackage[final]{neurips_2022}
% to avoid loading the natbib package, add option nonatbib:
% \usepackage[nonatbib]{neurips_2022}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{url}
\usepackage{hyperref} % hyperlinks
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{xcolor} % colors
\usepackage[round]{natbib}
\bibliographystyle{plainnat}
\input{preamble.tex}
\input{macros.tex}
\input{glossary.tex}
% \nofiles
\title{Event-triggered Time-Varying Bayesian Optimization}
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to LaTeX to determine where to break the
% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
% authors names on the first line, and the last on the second line, try using
% \AND instead of \And before the third author name.
\author{%
Paul Brunzema\textsuperscript{1}, Alexander von Rohr\textsuperscript{1,2}, Friedrich Solowjow\textsuperscript{1}, Sebastian Trimpe\textsuperscript{1} \\\\
\textsuperscript{1}Institute for Data Science in Mechanical Engineering, RWTH Aachen University, Germany\\
\textsuperscript{2}IAV GmbH Ingenieurgesellschaft Auto und Verkehr, Germany\\
\texttt{\{firstname.surname\}@dsme.rwth-aachen.de} \\
% examples of more authors
% \And
% _ \\
% Institute for Data Science in Mechanical Engineering\\
% RWTH Aachen University\\
% Aachen, Germany\\
% \texttt{email} \\
% \AND
% Coauthor \\
% Institute for Data Science in Mechanical Engineering\\
% RWTH Aachen University\\
% Aachen, Germany\\
% \texttt{email} \\
% \And
% Coauthor \\
% Institute for Data Science in Mechanical Engineering\\
% RWTH Aachen University\\
% Aachen, Germany\\
% \texttt{email} \\
% \And
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
}
\begin{document}
\maketitle
\input{moinmatter/abstract}
\input{moinmatter/introduction.tex}
\input{moinmatter/problem_formulation}
\input{moinmatter/related_work.tex}
\input{moinmatter/method.tex}
\input{moinmatter/theoretical_analysis.tex}
\input{moinmatter/empirical_analysis.tex}
\input{moinmatter/conclussion.tex}
\bibliography{example_paper}
% \input{backmatter/checklist}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \setcounter{page}{11}
\input{backmatter/appendix}
\end{document}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{xspace}
\usepackage{glossaries}
\usepackage{siunitx}
\usepackage{booktabs}
\usepackage{paralist}
\usepackage{placeins}
\usepackage[inline]{enumitem}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{bm}
\usepackage{mathtools}
\usepackage{wrapfig}
% \mathtoolsset{showonlyrefs=true}
% \usepackage[noend]{algpseudocode}
% \addtolength{\textfloatsep}{-0.25in} % adjust the spacing between caption and floating
\usepackage{parskip}
\usepackage{comment}
\usepackage{tikz}
\usetikzlibrary{arrows.meta, backgrounds}
% COLORS
\definecolor{RWTHBlue}{rgb}{0, 0.32941176470588235, 0.6235294117647059}
\definecolor{RWTHBordeaux}{rgb}{0.6313725490196078, 0.06274509803921569, 0.20784313725490197}
\definecolor{RWTHRed}{rgb}{0.8,0.027450980392156862,0.11764705882352941}
\definecolor{RWTHOrange}{rgb}{0.9647058823529412, 0.6588235294117647, 0}
\definecolor{RWTHGreen}{rgb}{0.3411764705882353, 0.6705882352941176, 0.15294117647058825}
% TIKZ
\tikzset{%
>={Latex[width=2mm,length=2mm]},
% Specifications for style of nodes:
base/.style = {rectangle, rounded corners, draw=black,
minimum width=1cm, minimum height=0.8cm,
text centered},
trigger/.style = {base, fill=RWTHBlue!80, text=white, minimum width=2cm,},
standard/.style = {base, fill=RWTHBlue!10,},
basetext/.style = {text centered, minimum height=0.1cm, opacity=0,text opacity=1, },
betweennode/.style = {text centered, minimum height=0.01cm, opacity=0.7,text opacity=1},
}
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}
\newtheorem{proposition}{Proposition}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\theoremstyle{remark}
\newtheorem*{remark}{Remark}
\newcommand{\swname}[1]{\texttt{#1}}
\newcommand{\ie}{i\/.\/e\/.\/,~}
\newcommand{\eg}{e\/.\/g\/.\/,~}
\newcommand{\cf}{cf\/.\/~}
\newcommand{\iid}{i\/.i\/.d\/.\/~}
\newcommand{\fig}{Fig\/.\/~}
\newcommand{\sect}{Sec\/.\/~}
\newcommand{\defn}{Def\/.\/~}
\newcommand{\algo}{Algorithm~}
\newcommand{\theo}{Theorem~}
\newcommand{\R}{\mathbb{R}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\X}{\mathbb{X}}
\newcommand{\argmax}{\arg\max}
\DeclareMathOperator{\EX}{\mathbb{E}}
\newcommand{\algname}{ET-GP-UCB\xspace}
\newcommand{\GP}{\mathcal{GP}}
\newcommand{\optvar}{\bm{x}}
\newcommand{\stopT}{\tau}
\newcommand{\triggerT}{{t'}}
\newcommand{\triggerDelta}{\delta_{\mathrm{B}}}
\newcommand{\bigO}{\tilde{\mathcal{O}}}
\newlist{mylist}{enumerate*}{1}
\setlist[mylist]{label=(\roman*), font=\itshape}
\newcommand*{\QEDA}{\null\nobreak\hfill\ensuremath{\square}}%
\newcommand{\fakepar}[1]{\vspace{1mm}\noindent\textbf{#1:}}
% \makeglossaries
\newacronym{tvbo}{TVBO}{time-varying Bayesian optimization}
\newacronym{gp}{GP}{Gaussian process}
\newacronym{rkhs}{RKHS}{Reproducing Kernel Hilbert Space}
\newacronym{bo}{BO}{Bayesian optimization}
\newacronym{se}{SE}{squared-exponential}
\glsdisablehyper
\begin{abstract}
%-------------
We consider the problem of sequentially optimizing a time-varying objective function using \gls{tvbo}.
Here, the key challenge is the exploration-exploitation trade-off under time variations.
% Real world problem/motivation/gap in knowledge
Current approaches to \gls{tvbo} require prior knowledge of a constant rate of change.
However, in practice, the rate of change is usually unknown.
% Key focus / contribution /
We propose an event-triggered algorithm, \algname, that treats the optimization problem as static until it detects changes in the objective function online and then resets the dataset.
This allows the algorithm to adapt to realized temporal changes without the need for prior knowledge.
% Key steps of approach / findings
The event-trigger is based on probabilistic uniform error bounds used in Gaussian process regression.
% Performance / application / limitation
We provide regret bounds for ET-GP-UCB and show in numerical experiments that it outperforms state-of-the-art algorithms on synthetic and real-world data.
Furthermore, these results demonstrate that ET-GP-UCB is readily applicable to various settings without tuning hyperparameters.
\end{abstract}
\glsresetall
\section{Introduction}
%
\begin{wrapfigure}{r}{0pt}
\input{figures/tikz_scripts/algorithm_block_diagram.tex}
\end{wrapfigure}
%
Over the last two decades, \gls{bo} has emerged as a powerful method for sequential decision making and design of experiments under uncertainty \citep{Garnett2022}.
These problems are typically formalized as optimization problems of a static unknown objective function.
At the core of \gls{bo} is the \emph{exploration-exploitation trade-off}, where the decision maker weights the potential benefit of a new unexplored query against the known reward of the currently assumed optimum.
The performance of \gls{bo} algorithms is typically measured in terms of regret, \ie the difference between the actual decision taken and the (unknown) optimal one.
If one assumes that the objective function does not change over time, there exist algorithms that achieve a desirable sub-linear regret (see \citet{Garnett2022} for an overview), meaning they will efficiently converge to the globally optimal solution.
In the real world, however, objective functions are often time-varying.
A time-varying objective function alters the nature of the exploration-exploitation trade-off.
The optimal decision and the information the decision maker gains from each query are now affected by time and any collected dataset becomes stale.
Generally, in a setting of ongoing changes, we cannot expect to find the optimal query and conversely no algorithm can achieve sub-linear regret \citep[Proposition~1]{Besbes2015}.
Instead, our goal is to design algorithms with an explicit relation between regret and the rate of change $\varepsilon$.
Intuitively, for larger $\varepsilon$, finding the optimal query in each time step is more challenging since old data becomes stale faster.
In their seminal work, \citet{Bogunovic2016} proposed two algorithms for the \gls{tvbo} problem;
however, the proposed algorithms and their subsequent variants rely on prior knowledge of the rate of change $\varepsilon$ as a hyperparameter, which can be hard to obtain.
Moreover, in these algorithms, $\varepsilon$ is specified at the start and assumed to remain constant over time, which is often not the case in practice.
To address these limitations, we propose the novel concept of event-triggered \gls{tvbo} (\fig\ref{fig:alg_overview}).
We model the objective function as time-invariant until an event-trigger detects a significant deviation of the observed data from the current model.
The algorithm then reacts to this significant change by resetting the dataset.
This way, the time-varying nature of the problem is only considered in the algorithm's exploration-exploitation trade-off \emph{when necessary}.
%
The event-trigger replaces (often inaccessible) prior knowledge of a rate of change and \algname exhibits desirable theoretical properties as well as strong empirical performance for different time variations, including gradual, sudden, and no change.
%
In summary, this paper makes the following contributions:
\begin{compactenum}[\itshape(i)]
\item The novel concept of using an event-trigger in combination with \gls{bo}.
\item An event-triggerd \gls{bo} algorithm for \gls{tvbo} called Event-Triggered GP-UCB (\algname) to optimize time-varying objective functions.
It does not rely on an estimate of the rate of change and adapts to changes in the objective functions.
\item Bayesian regret bounds for \algname.
\item Empirical evaluations of \algname on synthetic data and real-world benchmarks.
\end{compactenum}
\centering
\begin{tikzpicture}[node distance=1.5cm, background rectangle/.style={fill=white!15, rounded corners}, show background rectangle, every node/.style={fill=white, font=\small}, align=center] % add draw, to rectangle/.style to show boundary
% Specification of nodes (position, etc.)
\node[thick] (dataSet) [standard] {\textbf{Data Set} \\ $\mathcal{D}_t$};
\node[thick] (acqFunc) [standard, right of=dataSet, xshift=1.1cm] {\textbf{Acquisition Function} \\ $\argmax_{\optvar \in \X} \alpha(\optvar \mid \mathcal{D}_t)$};
\node[ultra thick] (trigger) [trigger, below of=acqFunc,yshift=-0.3cm] {\textbf{Event-}\\ \textbf{Trigger}};
\node[thick] (xt) [basetext, right of=acqFunc,xshift=0.7cm] {$\optvar_t$};
\node[thick] (objective) [standard, right of=xt, xshift=-0.1cm] {\textbf{Objective} \\ $f_t(\optvar)$};
\node (newdata) [basetext, right of=trigger,xshift=0.7cm] {$(\optvar_t, y_t)$};
\node[thick] (yt) [basetext, below of=objective,yshift=-0.3cm] {$\vphantom{\}}y_t$};
\node[thick] (dtplusone) [basetext, below of=dataSet,yshift=-0.3cm] {$\mathcal{D}_{t+1}$};
% Specification of lines between nodes specified above
% with additional nodes for description
\draw[->, line width=0.4mm] (dataSet) -- (acqFunc);
\draw[-, line width=0.4mm] (acqFunc) -- (xt);
\draw[->, line width=0.4mm] (xt) -- (objective);
\draw[->, line width=0.4mm] (xt) -- (newdata);
\draw[->, line width=0.4mm] (yt) -- (newdata);
\draw[->, line width=0.4mm] (objective) -- (yt);
\draw[->, line width=0.4mm] (newdata) -- (trigger);
\path[->, line width=0.4mm] (dataSet.south east) edge [out=-45, in=90] (trigger.north);
\draw[->, line width=0.4mm] (trigger) -- node [basetext, midway, xshift=0.1cm, yshift=-0.18cm] {$\gamma_{\text{reset}}$} (dtplusone);
\draw[dashed,->, line width=0.4mm] (dtplusone) -- node [betweennode, midway, xshift=-0.1cm, yshift=-0.1cm] {next time step} (dataSet);
% \node[above,font=\bfseries, opacity=0, text opacity=1] at (current bounding box.north) {Event-Triggered TVBO (ET-GP-UCB)};
\end{tikzpicture}%
\caption{\textit{Illustration of our proposed concept.}
After the standard \gls{bo} steps of optimizing the acquisition function and obtaining an observation, an \textit{event-trigger} decides whether to reset the dataset ($\gamma_{\mathrm{reset}} = 1$) or augment it with the observered data point in the usual way ($\gamma_{\mathrm{reset}} = 0$). This event-trigger allows our algorithm \algname to be adaptive to changes in the objective function without relying on prior knowledge of the rate of change.}
\label{fig:alg_overview}
\section{Problem Setting}
We aim to sequentially optimize an unknown time-varying objective function $f_t \colon \X \to \R$ on a compact set $\X \subset \R^d$ as
\begin{equation}
\optvar_t^* = \argmax_{\optvar \in \X} f_t(\optvar) \label{eq:optimization_problem}
\end{equation}
over the discrete time steps $t \in \mathbb{I}_{1:T}$ up to the time horizon $T$.
At each time step, an algorithm chooses a query $\optvar_t \in \X$ and obtains noisy observations following the standard assumption in \gls{bo}. % zeroth-order feedback.
\begin{assumption}\label{ass:noisy_obs}
Observations $y_t = f_t(\optvar_t) + w_t$ are perturbed by independent and identically distributed (i.i.d.) Gaussian noise with a zero mean and a noise variance $\sigma_n^2$ as $w_t \sim \mathcal{N}(0, \sigma_n^2)$.
\end{assumption}
Without any assumptions on the temporal change, the problem is intractable.
Therefore, we need to impose some structure and choose a popular Markov chain model in \gls{tvbo} proposed by \citet{Bogunovic2016} and used in various applications \citep{Su2018,Koenig2021}.
\begin{assumption}[\citet{Bogunovic2016}]\label{ass:markov_model}
Let $\GP\left(\mu, k(\optvar, \optvar')\right)$ be a \gls{gp} with mean $\mu \in \R^d$ and kernel $k \colon \X \times \X \to \R$.
Given \iid samples $g_1, g_2, \dots$ with $g_i \sim \GP\left(0, k(\optvar, \optvar')\right)$, where $k(\optvar, \optvar')$ is a stationary kernel with $k(\optvar, \optvar') \leq 1$ for all $\optvar, \optvar' \in \R$, and a rate of change $\varepsilon \in [0, 1]$, the time-varying objective function $f_t(\optvar)$ follows the Markov chain:
\begin{equation}
f_t(\optvar) = \begin{cases}
g_t(\optvar)\, , & t = 1 \\
\sqrt{1-\varepsilon} f_{t-1} (\optvar) + \sqrt{\varepsilon} g_t(\optvar)\, , & t \geq 2.
\end{cases}
\end{equation}
\end{assumption}
\begin{remark}\label{rem:constant_prior}
Due to Assumption~\ref{ass:markov_model}, any choice of $\varepsilon \in [0,1]$ yields that $f_t \sim \GP\left(0, k\right)$ for all $t\in \mathbb{I}_{1:T}$.
\end{remark}
We further use standard smoothness assumptions on the kernel choice introduced in \citet{Srinivas2010} as well as \citet{Bogunovic2016}.
\begin{assumption}[\citet{Bogunovic2016}]\label{ass:smoothness_assumption}
The kernel $k(\optvar, \optvar')$ is such that, given $f\sim \GP\left(0, k(\optvar, \optvar')\right)$, $L, L_f \geq 1$, and $a_0, b_0, a_1, b_1 \geq 0$, the following holds:
\begin{equation}
\mathbb{P}\left\{\sup_{\optvar\in \X}\left\lvert f\right\rvert > L_f \right\} \leq a_0 e^{- \left(L_f/b_0\right)^2} \, \text{,} \quad \mathbb{P}\left\{\sup_{\optvar\in \X}\left\lvert \partial f / \partial x_j \right\rvert > L, j=1, ..., d \right\} \leq a_1 e^{- \left(L/b_1\right)^2}.
\end{equation}%
\end{assumption}
This assumption limits the choice of kernel to those that are at least four times differentiable \citep[Theorem~5]{Ghosal2006}, such as a squared exponential kernel.
The performance of an algorithm optimizing \eqref{eq:optimization_problem} is measured in terms of the regret $R_T$.
\begin{definition}%[Regret]
Let $\optvar_t^*$ be the maximizer of $f_t(\optvar)$, and let $\optvar_t \in \X$ be the query at time step $t$. Then, the instantaneous regret at $t$ is $r_t = f_t(\optvar_t^*) - f_t(\optvar_t)$, and the regret after $T$ time steps is $R_T = \sum_{t=1}^T r_t$.\label{def:regret}
\end{definition}
In the time-invariant setting ($\varepsilon=0$), algorithms that exhibit sub-linear regret, \ie $\lim_{T \to \infty} R_T / T = 0$, with regret of order $\bigO(\sqrt{T})$\footnotemark are achievable such as GP-UCB \citep[][Theorem~2]{Srinivas2010}.
\footnotetext{As in \citet{Bogunovic2016} and \citet{Srinivas2010}, we denote asymptotics up to log factors as $\bigO(\cdot)$.}
For a time-varying objective function following Assumption~\ref{ass:markov_model}, \citet[Theorem~4.1]{Bogunovic2016} shows that the best possible expected regret of an algorithm is linear as $\bigO(T \varepsilon)$.
\fakepar{Problem Statement}
We consider the time-varying optimization problem \eqref{eq:optimization_problem} under Assumptions~\ref{ass:noisy_obs}--\ref{ass:smoothness_assumption}.
Critically, and in contrast to the state-of-the-art in \gls{tvbo}, we assume the rate of change $\varepsilon$ to be \emph{unknown}.
A reasonable goal for an algorithm in this setting are regret bounds that are no more than linear in $T$ and explicitly dependent on the true $\varepsilon$.
\section{Related Work and Background} \label{sec:related_work}
Our work builds on prior work in \gls{bo} and \gls{tvbo}. This section gives an overview of related work in these two fields.
\paragraph{Bayesian Optimization}
Our event-trigger design and algorithm extends prior work by \citet{Srinivas2010} on the \gls{bo} algorithm GP-UCB and transfers some of their concepts to the time-varying setting.
\citet{Srinivas2010} proved sub-linear regret bounds when using an upper confidence bound (UCB) acquisition function of the form $\mu_{\mathcal{D}_{t}}(\optvar) + \sqrt{\beta}_t \sigma_{\mathcal{D}_{t}}(\optvar)$, with standard posterior updates of the \gls{gp} model, given a dataset $\mathcal{D}_t\coloneqq\left\{(\optvar_i,y_i)\right\}_{i=1}^{t-1}$, as
\begin{align}
\mu_{\mathcal{D}_{t}}(\optvar) &= \bm{k}_t(\optvar)^T \left( \bm{K}_t + \sigma_n^2 \bm{I} \right)^{-1} \bm{y}_t \label{eq:mean}\\
\sigma_{\mathcal{D}_{t}}^2(\optvar) &= k(\optvar,\optvar) - \bm{k}_t(\optvar)^T \left( \bm{K}_t + \sigma_n^2 \bm{I} \right)^{-1} \bm{k}_t(\optvar) \label{eq:var}
\end{align}
where $\bm{K}_t = [k(\optvar_i, \optvar_j)]_{i, j = 1}^{t-1}$ is the Gram matrix, $\bm{k}_t(\optvar)= [k(\optvar_i, \optvar)]_{i=1}^{t-1}$, and noisy measurements $\bm{y}_t=[y_1, \dots, y_{t-1}]^T$ follow Assumption~\ref{ass:noisy_obs}.
Our algorithm uses these standard posterior updates only as long as all the measurements are still compliant with the current posterior model; it resets the dataset if this is no longer the case.
Consequently, in contrast to GP-UCB, our algorithm can adapt to changes in the objective function by discarding stale data.
In the regret bounds of \citet{Srinivas2010}, the mutual information gain
$I(\bm{f}_M, \bm{y}_M) = \frac{1}{2} \ln \det (\bm{I} + \sigma_n^2 \bm{K}_M)$
over $M$ points with $\bm{f}_M = [f_1(\optvar_1), \dots, f_M(\optvar_M)]$ takes an important role. The maximum information gain one can obtain using a time-invariant \gls{gp} model over any set of points is $\gamma_M \coloneqq \max_{\optvar_1, \dots, \optvar_M} I(\bm{f}_M, \bm{y}_M).$
Like \citet{Srinivas2010}, we will leverage this in our theoretical analysis of \algname.
For a detailed analysis of other \gls{bo} algorithms, we refer to \citet{Garnett2022}.
\paragraph{Time-Varying Bayesian Optimization}
\gls{tvbo} can be considered a special case of contextual BO \citep{Krause2011}, with the key distinction that time is strictly increasing; therefore, it is not possible to choose samples in the whole context domain.
\gls{tvbo} was first explicitly discussed by \citet{Bogunovic2016}, where they presented two time-varying UCB variants.
They propose that changes in the objective function can be modeled as in Assumption~\ref{ass:markov_model}.
The corresponding \gls{gp} surrogate model explicitly models these time variations in \eqref{eq:mean} and \eqref{eq:var} as
\begin{align}
\tilde{\mu}_{\mathcal{D}_{t}}(\optvar) &= \tilde{\bm{k}}_t(\optvar)^T \left( \tilde{\bm{K}}_t + \sigma_n^2 \bm{I} \right)^{-1} \bm{y}_t \label{eq:mean_time}\\
\tilde{\sigma}_{\mathcal{D}_{t}}^2(\optvar) &= k(\optvar,\optvar) - \tilde{\bm{k}}_t(\optvar)^T \left( \tilde{\bm{K}}_t + \sigma_n^2 \bm{I} \right)^{-1}\tilde{\bm{k}}_t(\optvar)
\label{eq:var_time}
\end{align}
where $\tilde{\bm{K}}_t = \bm{K}_t \circ \bm{K}^{\text{time}}_t$ with $\bm{K}^{\text{time}}_t = [(1-\varepsilon)^{|i - j|/2}]_{i,j=1}^{t-1}$, and $\tilde{\bm{k}}_t(\optvar) = \bm{k}_t(\optvar) \circ \bm{k}^{\text{time}}_t$ with $\bm{k}^{\text{time}}_t = [(1-\varepsilon)^{|t+1 - i|/2}]_{i=1}^{t-1}$, and $\circ$ is the Hadamard product.
These updates ensure that the posterior distribution follows the same dynamics as $f_t$ (\cf Assumption~\ref{ass:markov_model}).
We will utilize this in our proof of Bayesian regret bounds for \algname.
Since the introduction of this setting in the seminal paper by \citet{Bogunovic2016}, their algorithm TV-GP-UCB with posterior updates as in \eqref{eq:mean_time} and \eqref{eq:var_time} has been used in various applications, such as controller learning \citep{Su2018}, safe adaptive control \citep{Koenig2021}, and hyperparameter optimization for reinforcement learning \citep{ParkerHolder2020, ParkerHolder2021}.
However, it relies on a priori estimation of the rate of change $\varepsilon$ and assumes $\varepsilon$ stays constant over time.
In contrast, the event-trigger in \algname does not need an estimate of the rate of change and enables our algorithm to adapt to changes in the objective function, \ie it can cope with a time-varying $\varepsilon$ as well as sudden changes.
%
We note that, if the rate of change is known beforehand and the structural assumptions are satisfied, then the TV-GP-UCB model is an efficient choice since these properties are explicitly embedded into the algorithm.
\citet{Bogunovic2016} also introduced R-GP-UCB, which resets the posterior periodically to account for data becoming uninformative over time.
In contrast, \algname resets the posterior only if an obtained measurement is no longer consistent with the current dataset (\cf Sec.~\ref{sec:method}).
As in R-GP-UCB, our algorithm uses time-invariant posterior updates within time intervals, but our time intervals do not have to be specified a priori based on $\varepsilon$; rather, they are determined online by the event-trigger.
Under frequentist regularity assumption, \citet{Zhou2021} also propose an algorithm with a constant reset time, BR-GP-UCB\footnote{\citet{Zhou2021} also call their algorithm R-GP-UCB.
We label it BR-GP-UCB to avoid confusions with R-GP-UCB of \citet{Bogunovic2016}.}, as well as a sliding window algorithm SW-GP-UCB.
%In both papers, the authors assume that the sum of changes in the objective function between time steps under the RKHS norm is bounded by a fixed \textit{variational budget}.
The crucial difference to our problem setting is their assumption of a \emph{fixed variational budget}, \ie $\sum_{t=1}^T ||f_{t+1} - f_t||_\mathcal{H} < B_T$, where $\mathcal{H}$ is a \gls{rkhs} and $B_T$ a scalar.
Clearly, the above assumption means that the change in the objective function will converge to zero as $T \rightarrow \infty$.
Leveraging these same assumptions, \citet{Deng2022} propose an algorithm, W-GP-UCB, which utilizes a similar weighted \gls{gp} model to TV-GP-UCB.
In this more restrictive setting of a fixed variational budget, sub-linear regret is possible (\cf \citet{Besbes2015}).
In contrast, we are interested in the setting given by Assumption~\ref{ass:markov_model}, which corresponds to a variational budget that increases linearly in time; hence, the best possible regret is also linear \citep[Proposition~1]{Besbes2015}.
Nevertheless, in Appendix.~\ref{sec:dengetal}, we empirically compare ET-GP-UCB against the algorithms of \citet{Zhou2021} and \citet{Deng2022} and display improved performance on their benchmarks.
\paragraph{Event-Triggered Learning}
The idea of resetting a dataset only if an important change occurs as detected by a trigger-event is similar to the concepts of event-triggered learning \citep{Solowjow2020} and event-triggered online learning \citep{Umlauft2019}.
Also in the multi-armed bandit setting, \cite{wei2021non} discuss the idea of including tests into optimization algorithms to detect and adapt to non-stationarity.
All concepts aim to be efficient with the available data, only updating a model when necessary or only adding new data which is relevant.
Our approach also aims for data efficiency; as long as we have a good model of our objective function, we want to use already-available data to minimize regret.
However, if the event-trigger detects a significant model mismatch, we reset the dataset and thereby delete stale data.
While the mentioned approaches consider different problem settings, this one is the first to introduce event-triggers to \gls{tvbo}.
\section{Event-Trigger Design for TVBO}\label{sec:method}
In this section, we design the event-trigger that is at the heart of \algname in \fig\ref{fig:alg_overview} and allows us to adapt to changes \emph{when necessary}.
In essence, we build on state-of-the-art uniform error bounds for \gls{gp} regression to decide if new data is consistent with the posterior.
When new data deviates significantly, we reset the dataset and thus forget old data.
While conceptually straightforward, this idea constitutes a new approach compared to previous work in \gls{tvbo}, as the resulting algorithm is independent of the rate of change; thus, standard analysis cannot be applied.
Therefore, in the next section, we prove new regret bounds for \algname.
First, we introduce the core component of our general event-triggered \gls{tvbo} framework, the event-trigger:
\begin{definition}[Event-trigger in \gls{tvbo}]\label{def:event-trigger_framework}
Given a test function $\psi_t$ and a threshold function $\kappa_t$, both of which can depend on the current dataset $\mathcal{D}_t$ and the latest query location and measurement pair $(\optvar_t, y_t)$, the event-trigger at time $t$ is
\begin{equation}
\gamma_{\mathrm{reset}} = 1 \Leftrightarrow \psi_t\left(\mathcal{D}_t, (\optvar_t, y_t)\right) > \kappa_t\left(\mathcal{D}_t, (\optvar_t, y_t)\right) \label{eq:event-trigger_framework}
\end{equation}
where $\gamma_{\mathrm{reset}}$ is the binary indicator for whether to reset the dataset $(\gamma_{\mathrm{reset}}=1)$ or not $(\gamma_{\mathrm{reset}}=0)$.
\end{definition}
We base our event-trigger on uniform error bounds used in \gls{gp} regression, specifically on the Bayesian error bound by \citet[Lemma~5.5]{Srinivas2010}.
Other design options would also be possible, \eg likelihood-based event-triggers.
In the following, we will denote $\triggerT \in \mathbb{N}$ as the time since the previous reset.
We have $\triggerT=t$ if no reset has yet occurred.
\begin{lemma}[{\citet[Lemma~5.5]{Srinivas2010}}] \label{lem:srinivas}
Pick $\delta \in (0, 1)$ and set $\rho_{\triggerT} = 2\ln{\frac{\pi_{\triggerT}}{\delta}}$, where $\sum_{\triggerT \geq 1} \pi_{\triggerT}^{-1} = 1$, $\pi_{\triggerT} >0$. Let $f\sim \mathcal{GP}(0,k(\optvar, \optvar'))$. Then, $| f(\optvar_t) - \mu_{\mathcal{D}_t}(\optvar_t) | \leq \sqrt{\rho_{\triggerT}} \sigma_{\mathcal{D}_t} (\optvar_t)$
holds for all $\triggerT \geq 1$ with probability at least $1 - \delta$.
\end{lemma}
\begin{remark} \label{rem:pi_t}
For $\sum_{\triggerT \geq 1} \pi_{\triggerT}^{-1} = 1$ to hold, one can choose $\pi_{\triggerT} = \pi^2 \triggerT^2 / 6$ as in \citet{Srinivas2010}.
\end{remark}
The result gives a high probability bound for all time steps on the absolute deviation between the posterior mean and a time-invariant objective function at the query location $\optvar_t$ based on the posterior variance and parameter $\rho_{\triggerT}$.
If a time-varying objective function satisfies this bound for all time steps, the time variations are negligible with high probability (\ie $\varepsilon \to 0$).
On the other hand, if the bound in Lemma~\ref{lem:srinivas} is violated at a chosen query location $\optvar_t$, the objective function has likely changed significantly.
This is what we want to detect with our event-trigger, as the algorithm can then react to this significant change by resetting the dataset.
We want to explicitly evaluate the trigger event at every time step.
While we cannot directly evaluate $\left\lvert f_t(\optvar_t) - \mu_{\mathcal{D}_t}(\optvar_t) \right\rvert$ in Lemma~\ref{lem:srinivas} (\cf Assumption~\ref{ass:noisy_obs}), we can evaluate $\left\lvert y_t - \mu_{\mathcal{D}_t}(\optvar_t) \right\rvert$.
Thus, we adapt Lemma~\ref{lem:srinivas} to design a probabilistic bound which includes only $y_t$ instead of $f_t(\optvar_t)$.
For this, we first bound the noise in probability. All proofs of the following Lemmas are in Appendix~\ref{sec:lemma3}.
\begin{lemma} \label{lem:bounded_noise}
Pick $\delta \in (0,1)$ and set $\bar{w}_{\triggerT} = \sqrt{2\sigma_n^2 \ln \frac{\pi_{\triggerT}}{\delta}}$, where $\sum_{{\triggerT} \geq 1} \pi_{\triggerT}^{-1} = 1$, $\pi_{\triggerT} >0$. Then, the Gaussian noise sequence in Assumption~\ref{ass:noisy_obs}, obtained since the previous reset, satisfies ${|w_{\triggerT}| \leq \bar{w}_{\triggerT}}$ for all ${\triggerT} \geq 1$ with probability at least $1 - \delta$.
\end{lemma}
With Lemma~\ref{lem:bounded_noise}, we can now use Lemma~\ref{lem:srinivas} to build a probabilistic error bound that can act as the test function and the corresponding threshold functions of our event-trigger.
\begin{lemma}\label{lem:trigger_bound}
Let $f_t(\optvar)$ follow Assumption~\ref{ass:markov_model} with $\varepsilon =0$. Pick $\triggerDelta \in (0, 1)$ and set $\rho_{\triggerT} = 2\ln{\frac{2\pi_{\triggerT}}{\triggerDelta}}$, where $\sum_{{\triggerT} \geq 1} \pi_{\triggerT}^{-1} = 1$, $\pi_{\triggerT} >0$. Also set $\bar{w}_{\triggerT} = \sqrt{2\sigma_n^2 \ln \frac{2\pi_{\triggerT}}{\triggerDelta}}$. Then, observations $y_t$ under Assumption~\ref{ass:noisy_obs} satisfy $| y_t - \mu_{\mathcal{D}_t}(\optvar_t) | \leq \sqrt{\rho_{\triggerT}} \sigma_{\mathcal{D}_t} (\optvar_t) + \bar{w}_{\triggerT}$
for all $\triggerT \geq 1$ with probability at least $1 - \triggerDelta$.
\end{lemma}
We now define our event-trigger for \algname following Def.~\ref{def:event-trigger_framework}.
As the test and threshold function, we choose
\begin{align}
\psi_t(\optvar_t) &\coloneqq \psi_t\left(\mathcal{D}_t, (\optvar_t, y_t)\right) = \left\lvert y_t - \mu_{\mathcal{D}_t}(\optvar_t) \right\rvert \label{eq:test_function}\\
\kappa_t(\optvar_t) &\coloneqq \kappa_t\left(\mathcal{D}_t, (\optvar_t, y_t)\right) = \sqrt{\rho_{\triggerT}} \sigma_{\mathcal{D}_t} (\optvar_t) + \bar{w}_{\triggerT},\label{eq:threshold_function}
\end{align}
respectively, as specified in Lemma~\ref{lem:trigger_bound}.
%
The resulting algorithm \algname is then as follows: We perform standard \gls{bo} using an UCB acquisition function as long as the event-trigger evaluates to $\gamma_{\mathrm{reset}}=0$.
When the event-trigger is activated the objective function $f_t$ has likely changed significantly, and old data has become uninformative.
\algname then resets the dataset and restarts the optimization.
This is summarized in Algorithm~\ref{alg:algorithm} in Appendix~\ref{sec:algo}.
The single design parameter of the event-trigger is $\triggerDelta$ in Lemma~\ref{lem:trigger_bound}, which has a well-understood meaning as defining the tightness of the probabilistic uniform error bound.
Consequently, we do not need any a priori knowledge of the rate of change $\varepsilon$.
%
Our event-trigger only bounds the change in the objective function at the current query location and cannot detect changes anywhere else.
Even with the local evaluation and the fact that we do not know the rate of change a priori \algname provable performs well. We show regret bounds for \algname in the next section.
\section{Theoretical Analysis} \label{sec:theory}
In this section, we provide our main theoretical result: novel Bayesian regret bounds for our method \algname.
%
For the subsequent theoretical analysis, we first define the time at which a reset occurs.% in \defn\ref{def:reset_time}.
\begin{definition}\label{def:reset_time}
The random variable corresponding to the activation time of the event-trigger following \defn\ref{def:event-trigger_framework} is denoted by $\stopT \coloneqq \min\{\triggerT\in \N: \psi_t(\optvar_t) > \kappa_t(\optvar_t)\}$.
\end{definition}
The random variable $\stopT$ is a stopping time, as $\psi_t(\optvar_t)$ is a discrete-time stochastic process and the trigger-event is completely determined by the information known up to $\triggerT$.
The stochastic process depends recursively on the acquisition function, which makes an analytical study of the stopping time intractable; even for simple stochastic processes, calculating expected values of stopping times is typically intractable.
However, since we know that $\stopT$ is a non-negative random variable, we can obtain the following bound on this stopping time in probability using Markov's inequality.
\begin{lemma} \label{lem:markov_bound}
Set $\bar \stopT = \EX[\stopT]/\delta$ and pick ${\delta \in (0,1)}$. Then $\stopT$ satisfies $ \mathbb{P}\{\stopT < \bar \stopT\} \geq 1- \delta$.
\end{lemma}
This bound will be finite as long as $\EX[\stopT]$ is finite. This is the case as long as the UCB acquisition function selects a point where the changes will let the event-trigger fire at some point; either due to changes in the objective or due to false positives.
Given the bound on $\stopT$ from Lemma~\ref{lem:markov_bound} (other bounds on the tail probability would also be possible), we can formulate Bayesian regret bounds for our algorithm in the theoretical setting of Assumption~\ref{ass:markov_model}. We later show empirically that our algorithm performs well even when Assumption~\ref{ass:markov_model} is violated.
\subsection{Regret Bounds}
Next, we present regret bounds for our proposed event-triggered algorithm \algname.
\begin{theorem}\label{theorem:regret_bounds}
Let the domain $\X \subset [0, r]^d \subset \R^d$ be convex and compact with $d\in \mathbb{N}_+$ and let $f_t$ follow Assumption~\ref{ass:markov_model} with a kernel $k(\optvar,\optvar')$ such that Assumption~\ref{ass:smoothness_assumption} is satisfied. Pick $\delta \in (0,1)$ and set $\beta_t = 2 \ln\left(5 \pi^2 t^2 / (3\delta)\right) + 2 d \ln{\left(t^2 d b_1 r \sqrt{\ln (5 d a_1 \pi^2 t^2 / (3\delta))}\right)}$.
Then, after running \algname with $\triggerDelta \geq \delta/5$ (Lemma~\ref{lem:trigger_bound}) for $T$ time steps,
\begin{align}
R_T \leq \sqrt{C_1 T \beta_T \left( \frac{T}{\bar{\stopT}} + 1\right)\gamma_{\bar{\stopT}}} + 2 + T\phi_T(\varepsilon, \delta, \bar{\stopT})\label{eq:regret}
\end{align}
is satisfied with probability at least $1 - \delta$, where ${C_1 = 72/\ln(1+\sigma_n^{-2})}$ and we defined
\begin{equation}
\phi_T(\varepsilon, \delta, \bar{\stopT}) \coloneqq 3 \sqrt{\beta_T (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon}
+ (\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \varepsilon \left( b_0 \sqrt{\ln\frac{5 a_0 \pi^2 T^2}{2\delta}} + \bar{w}_{\bar{\stopT}} \right)\label{eq:bound2}
\end{equation}
with $\bar{w}_{\bar{\stopT}} = \sqrt{2\sigma_n^2 \ln(5 \pi^2 \bar{\stopT}^2 /(6 \delta))}$.
\end{theorem}
%
The proof of \theo\ref{theorem:regret_bounds} is in Appendix~\ref{sec:proof}, but we outline the main challenges and ideas next.
Building on arguments from \citet{Srinivas2010} and \citet{Bogunovic2016}, we extend the analysis of R-GP-UCB to our setting in which $\varepsilon$ is unknown to the algorithm and resets are not periodic but determined by the stopping time of our event-trigger.
We focus on bounding the difference between the model used in the algorithm (time-invariant model, see \eqref{eq:mean} and \eqref{eq:var}) and the time-varying model (\eqref{eq:mean_time} and \eqref{eq:var_time}).
In contrast to R-GP-UCB, we utilize the explicit evaluation of the bound in Lemma~\ref{lem:trigger_bound} through our event trigger at every time step to bound the model mismatch in the mean at the query locations $\{\optvar_t\}_{t=1}^T$.
%
The obtained bounds are, as desired, linear in $T$ and jointly depend on $\varepsilon$.
\begin{wrapfigure}{r}{0.4\textwidth}
\centering
\includegraphics[]{figures/pdf_plots/stopping_times_NIPS.pdf}%
\caption{Distribution of stopping times from Definition~\ref{def:reset_time} for $\varepsilon \in \{0.03, 0.1\}$ (orange and violet, respectively) using \algname.
The vertical solid lines show the constant periodic reset time by R-GP-UCB for same $\varepsilon$ ($\varepsilon=0.03$ on the right, $\varepsilon=0.1$ on the left).
}
\label{fig:stopping_time}
\end{wrapfigure}%
%
Furthermore, for a non-changing objective function (\ie $\varepsilon=0$), our bounds recover the known result of a sub-linear rate of the regret bound of standard GP-UCB as $\bigO(\sqrt{T})$ in \citet[Theorem~2]{Srinivas2010}.
% %%%%% FOR FORMATTING %%%%%%%%%
\begin{figure*}[t]
\centering
\includegraphics[]{figures/pdf_plots/within_model_NIPS_shade.pdf}%
\caption{\textit{Within-model comparisons.} Top row: performance of the different algorithms in terms of normalized regret.
The shaded areas show one standard deviation of the respective algorithms.
Bottom row: reset timings of R-GP-UCB and \algname.
% whats in it
In (a) to (c), $\varepsilon$ is fixed for all time steps, and in (d), $\varepsilon$ is misspecified for TV-GP-UCB and R-GP-UCB.
% what to see
In (a) to (c), TV-GP-UCB is the baseline with full information, showing the best possible performance given a true constant $\varepsilon$.
We compare \algname to R-GP-UCB, the latter of which uses the true $\varepsilon$ to determine its periodic reset time.
For \algname, we use the same parameter $\triggerDelta$ in the event-trigger for the test cases (a) to (d).
In all cases, \algname outperforms R-GP-UCB.
It also outperforms TV-GP-UCB if $\varepsilon$ is misspecified, as shown in (d), since \algname does not rely on prior information on $\varepsilon$.}
\label{fig:within_model}
\end{figure*}
\subsection{Intuition on the Stopping Time}\label{sec:intuition}
%
Before discussing the empirical performance of \algname, we illustrate the behavior of the stopping time. % and show that it is well-behaved.
We run Monte Carlo simulations with \algname on objective functions following Assumption~\ref{ass:markov_model} with a fixed set of parameters (all parameters are in Appendix~\ref{sec:hypers}).
\fig\ref{fig:stopping_time} shows the obtained stopping times.
We can observe that \algname utilizes the whole distribution of $\stopT$ (histograms) compared to R-GP-UCB with the constant reset time (vertical solid lines) as $N_{\text{const}}=\lceil \min\{T, 12 \varepsilon^{-1/4} \}\rceil$ \citep[Proposition~4.1]{Bogunovic2016}.
In the following experiments, we will showcase that, by utilizing this whole distribution, \algname can react to the realized change in the objective function and yields lower empirical regret than periodic resetting.
\section{Empirical Experiments} \label{sec:empirical_experiments}
Using both synthetic data and a standard \gls{bo} benchmark, we compare \algname to the state-of-the-art algorithms R-GP-UCB, which resets periodically, and TV-GP-UCB, which gradually forgets past data points.
We further compare these time-varying algorithms to the time-invariant GP-UCB as a baseline.
In our experiments, we find that:
\begin{compactenum}[\itshape(i)]
\item The adaptive resetting of \algname always outperforms periodic resetting.
\item \algname outperforms TV-GP-UCB on synthetic data if the rate of change is misspecified.
\item \algname outperforms periodic resetting and TV-GP-UCB on real-world data.
\item Empirically, \algname is especially beneficial compared to other baselines if changes in the objective function are very pronounced.
\end{compactenum}
We reiterate that \algname does not rely on any information regarding the rate of change.
To highlight the generalization capabilities of \algname to different scenarios, we use the same parameter for the event-trigger ($\triggerDelta=0.1$) in all of the experiments.
As in \citet{Bogunovic2016}, and similarly to \citet{Srinivas2010}, we utilize an approximation of $\beta_t$ in Theorem~\ref{theorem:regret_bounds} which ensures logarithmic scaling in $t$ as $\beta_t=\mathcal{O}(d \ln (t))$, where $\beta_t= c_1 \ln(c_2 t)$.
This allows for a direct comparison to \citet{Bogunovic2016}, as they suggest to set $c_1=0.8$ and $c_2=4$ for a suitable exploration-exploitation trade off.
The experiments in \sect\ref{sec:within-model} and \sect\ref{sec:temperature-data} are conducted using BoTorch \citep{Balandat2020} and GPyTorch \citep{Gardner2018}.
\subsection{Within-Model Comparison}\label{sec:within-model}
We consider the same two-dimensional setting for synthetic data as in \citet{Bogunovic2016}.
We choose the compact set to be $\X = [0, 1]^2$ and generate objective functions according to Assumption~\ref{ass:markov_model} with a known $\varepsilon$, a squared exponential kernel with fixed lengthscales $l=0.2$, and a noise variance of $\sigma_n^2=0.05$.
This setting is often referred to as \textit{within-model comparison} \citep{Hennig2012}, in which all the hyperparameters are known to the algorithm; such comparisons are used to evaluate an algorithm's performance given all assumptions are fulfilled.
Here, TV-GP-UCB acts as the reference case with full information.
It shows the best possible performance an UCB algorithm can obtain in this setting as it takes Assumption~\ref{ass:markov_model} explicitly into account by using the time-varying posterior updates in \eqref{eq:mean_time} and \eqref{eq:var_time}.
R-GP-UCB is also parameterized using the true $\varepsilon$ to obtain its periodic reset time as $N_{\text{const}}=\lceil \min\{T, 12 \varepsilon^{-1/4} \}\rceil$ \citep{Bogunovic2016}.
In contrast, \algname does not depend on $\varepsilon$. Hence, it only utilizes the information about the lengthscales of the kernel.
We still expect \algname to outperform R-GP-UCB, as it can vary its reset timing (\cf \fig\ref{fig:stopping_time}) to adapt to the change in the objective function.
%%%%% FORMATTING! %%%%%
\begin{figure*}[t]
\centering
\includegraphics[]{figures/pdf_plots/temperatures_NIPS_shade.pdf}%
\caption{
% Subplot (a)
\textit{Real-world temperature data.} Subfigure (a) shows the temperature data consisting of sensor measurements from the two test days used in \citet{Bogunovic2016}.
% Subplot (b)
In (b), the performance of the algorithms on these two days (top) and the resets of R-GP-UCB and \algname (bottom) are displayed.
Here, \algname outperforms TV-GP-UCB as well as R-GP-UCB and GP-UCB.
% Subplot (c)
Subfigure (c) shows the performance on other test days.
Also for these days, ET-GP-UCB shows a performance improvement compared to all other algorithms.
% Concluding remarks
For these experiments on real-world data, we use the same parameter for the event-trigger as in all previous experiments whereas TV-GP-UCB and R-GP-UCB relied on estimating $\varepsilon$ from data.}
\label{fig:temperature_results}
\end{figure*}
%%%%%
In the top figures of \fig\ref{fig:within_model}~(a) to (c), we show the normalized regret for different ${\varepsilon}$.
In all cases, \algname outperforms R-GP-UCB without relying on prior knowledge on the rate of change.
The bottom figures of \fig\ref{fig:within_model}~(a) to (c) highlight the adaptive nature of \algname, as the resets do not always occur at the same time, but depend on the realized change in the objective function.
This becomes even more apparent looking at \fig\ref{fig:within_model}~(d), where the rate of change for TV-GP-UCB and R-GP-UCB is misspecified.
\algname does not rely on specifying a rate of change a-priori and we do not change any parameters for this experiment. %between experiments
We observe that \algname resets to the realized changes resulting in the same performance as in \fig\ref{fig:within_model}~(c). With this, it outperforms TV-GP-UCB and R-GP-UCB which slowly diverge as shown by the increase in normalized regret over time.
\subsection{Real-World Temperature Data}\label{sec:temperature-data}
To benchmark the algorithms on real-world data, we use the temperature dataset collected from $46$ sensors deployed at Intel Research Berkeley over eight days at $10$ minute intervals.
This dataset was also used as a benchmark in previous work on GP-UCB and \gls{tvbo} \citep{Srinivas2010,Krause2011,Bogunovic2016}.
The objective is to activate the sensor with the highest temperature at each time step resulting in a spatio-temporal monitoring problem.
In our \gls{tvbo} setting, $f_t$ consists of all sensor measurements at time step $t$, and an algorithm can query only a single sensor every $10$ minutes.
Hence, the regret following Def.~\ref{def:regret} is the difference between the maximum temperature and the recorded temperature by the activated sensor.
For each experiment, two test days are selected, while the preceding days serve as the training set.
Using this training set, all data is normalized to have a mean of zero and a standard deviation of one.
The empirical covariance matrix is computed based on the training set and employed as the kernel for all algorithms.
The sensor data of the days 7 and 8 is shown in \fig\ref{fig:temperature_results}~(a) and an overview of the remaining days is given in Appendix~\ref{sec:train_days}.
%
For TV-GP-UCB, we set $\varepsilon=0.03$ and the periodic reset time of R-GP-UCB to $N_{\text{const}}=15$ according to \citet{Bogunovic2016}.
For \algname, we use the same parameter for the event-trigger as in all the previous experiments.
\fig\ref{fig:temperature_results}~(b) presents the normalized regret (top) for each algorithm over $20$ runs and the resets of R-GP-UCB and \algname (bottom).
Consistent with the experiments on synthetic data, we observe that adaptive resetting outperforms periodic resetting.
Furthermore, \algname outperforms TV-GP-UCB.
It highlights that using a constant rate of change in the \gls{gp} model as in TV-GP-UCB can be problematic for real-world problems.
Here, being adaptive to changes as \algname is beneficial and yields lower regret.
\algname also outperforms the competing algorithms on test days other than the ones used in \citet{Bogunovic2016}, as shown in \fig\ref{fig:temperature_results}~(c).
Considering the resets of \algname, it appears that especially when changes in the objective function are more distinct (\eg changes between night and day), the proposed event-trigger framework is empirically beneficial compared to using an estimated constant rate of change.
\subsection{Policy Search in Time-Varying Environments}
\begin{wrapfigure}{r}{0.52\textwidth}
\vspace{-0.7cm}
\centering
\includegraphics[]{figures/pdf_plots/LQR_NIPS_small_shade.pdf}%
\caption{\textit{Policy search}.
Normalized regret (top) and resets (bottom) for the policy search example.
At $t=100$, a sudden change in the objective function occurs.
\algname is able to detect and react to this change outperforming all competing baselines.
}
\label{fig:lqr_small}
\end{wrapfigure}%
To further investigate this claim, we compared all algorithms on a four-dimensional policy search benchmark of a cart-pole system also used in \citet{Brunzema2022}.
The objective function in this example is quadratic in the optimization variables.
Given the system dynamics of the cart-pole system, it is possible to calculate the optimal policy and thus calculate the regret following \defn\ref{def:regret}.
To induce a sudden change in the objective function, we increase the friction in the bearing at $t=100$ by a factor of $2.5$.
We expect the event-trigger of \algname to detect this change in the objective function and converge to the new optimum after resetting the dataset and thus outperform the algorithms based on the assumption of a constant rate of change.
%
In contrast to previous examples, we also perform online hyperparameter optimization of the spatial lengthscales using Gamma priors to guide the search (all parameters are listed in Appendix~\ref{sec:hypers}).
For TV-GP-UCB, we choose $\varepsilon=0.03$ and we pick the constant reset timing of R-GP-UCB according to \citet[Corollary~4.1]{Bogunovic2016}.
In this example, we also compare \algname against UI-TVBO \citep{Brunzema2022} as it was specifically designed for policy search in changing environments.
We set its constant rate of change also to $\hat{\sigma}_w^2=\varepsilon=0.03$ (\cf \citet[Sec.~V]{Brunzema2022}).
Specifying a constant $\varepsilon$ (or $\hat{\sigma}_w^2$ in the case of UI-TVBO) for objective functions that only encounter one sudden change can be problematic.
For \algname, we again choose the same parameter $\triggerDelta = 0.1$ as in all previous examples.
The results over $50$ independent runs are displayed in \fig\ref{fig:lqr_small}.
We can observe that \algname is the best performing algorithm since it can detect the sudden change in the objective function and converge to the new optimal solution.
We can further see that TV-GP-UCB and R-GP-UCB, which assume a constant and smooth change in the objective function, struggle to cope with such sudden changes resulting in significantly increased regret.
While UI-TVBO can also recover from the sudden change, \algname still outperforms this baseline.
The example also highlights that a constant reset time that is too short can yield divergent behavior.
\section{Conclusion and Outlook}
This paper presents a novel concept based on the idea of an event-trigger to optimize time-varying objective functions with \gls{tvbo}.
Based on this concept, we developed the algorithm \algname, which can adapt to the realized changes in the objective function and, crucially, does not depend on any prior knowledge of these temporal changes.
For \algname, we provided novel regret bounds based on the event-trigger and observed that adaptive resetting outperformed periodic resetting in all empirical experiments.
\algname is competitive with the state-of-the-art on real-world benchmarks without relying on an estimation of the rate of change, proving itself to be especially beneficial when changes in the objective function are more pronounced.
Promising future research directions are the design of different event-triggers and the consideration of other changes to the dataset, rather than performing a full reset once the event-trigger is activated; this could help to retain valuable information and thus reduce regret especially for higher dimensional problems.
Also of interest is the study of ET-GP-UCB under frequentist regularity assumptions with a fixed variational budget.
\onecolumn
\appendix
\setcounter{secnumdepth}{2}
Following is the technical appendix for the paper of \textit{Event-Triggered Time-Varying Bayesian Optimization}. This includes:
\begin{compactitem}[leftmargin=1in]
\item[\textbf{In \ref{sec:algo}:}] The algorithm \algname in pseudo code
\item[\textbf{In \ref{sec:dengetal}:}] A empirical comparison to the finite variational budget setting
\item[\textbf{In \ref{sec:sensitivity}:}] A sensitivity analysis of the event-trigger hyperparameter $\triggerDelta$ of \algname
\item[\textbf{In \ref{sec:lemma3}:}] The proofs of Lemma~\ref{lem:bounded_noise}, Lemma~\ref{lem:trigger_bound}, and Lemma~\ref{lem:markov_bound}
\item[\textbf{In \ref{sec:proof}:}] The proof of Theorem~\ref{theorem:regret_bounds}
\item[\textbf{In \ref{sec:train_days}:}] The normalized temperature data for all days used
\item[\textbf{In \ref{sec:hypers}:}] An overview of the hyperparameters used in all experiments
\end{compactitem}
Note that all citations here are in the bibliography of the main document, and similarly for many of the cross-references.
\input{backmatter/algorithm}
\input{backmatter/dengetal.tex}
\input{backmatter/sensitivity_delta.tex}
\input{backmatter/proof_of_lemma3.tex}
\input{backmatter/proof_of_theorem1.tex}
\input{backmatter/overview_training_data.tex}
\input{backmatter/hyperparameter.tex}
\section{Algorithm \algname in Pseudo Code}\label{sec:algo}
\begin{algorithm}[h]
\caption{Event-triggered GP-UCB (\algname).}
\label{alg:algorithm}
\begin{algorithmic}[1]
\STATE \textbf{Define:} $\mathcal{GP}(0, k)$, $\X \subset \R^d$, $\triggerDelta \in (0,1)$, $\mathcal{D}_1=\emptyset$
\FOR{$t = 1, 2, \dots, T$}
\STATE Update GP model with $\mathcal{D}_t$
\STATE Set $\beta_t$ according to Theorem~\ref{theorem:regret_bounds}
\STATE Choose $\optvar_t = \argmax_{\optvar \in \X} \mu_{\mathcal{D}_t}(\optvar) + \sqrt{\beta_t}\sigma_{\mathcal{D}_t}(\optvar)$
\STATE Sample $y_{t} = f_{t}(\optvar_{t}) + w_t$
\STATE $\gamma_{\mathrm{reset}} \gets$ \textsc{Event-Trigger}($\mathcal{D}_{t}$, $(y_{t}, \optvar_{t}), \triggerDelta$) \hfill see \eqref{eq:event-trigger_framework}, \eqref{eq:test_function}, \eqref{eq:threshold_function}
\IF{$\gamma_{\mathrm{reset}}$}
\STATE Reset dataset $\mathcal{D}_{t+1}=\{(y_{t}, \optvar_{t})\}$
\ELSE
\STATE Update dataset $\mathcal{D}_{t+1} = \mathcal{D}_{t} \cup \{(y_{t}, \optvar_{t})\}$
\ENDIF
\ENDFOR
\end{algorithmic}
\end{algorithm}
\section{Comparisons to Algorithms in the Finite Variational Budget Setting} \label{sec:dengetal}
In this section of the appendix, we compare ET-GP-UCB to W-GP-UCB \citep{Deng2022}, and BR-GP-UCB and SW-GP-UCB \citep{Zhou2021}.
As discussed in Sec.~\ref{sec:related_work}, their underlying assumptions differ from ours in that they assume a fixed variational budget.
Therefore, a theoretical comparison is not reasonable, but an empirical comparison, especially on real-world data, is interesting.
As the baseline that does not consider time-variations, we compare also against IGP-UCB \citep{Chowdhury2017}, which is an improved form of the agnostic GP-UCB of \citet{Srinivas2010}.
For the comparison, we use the code base submitted by \citet{Deng2022} in the supplementary material of \href{https://proceedings.mlr.press/v151/deng22b}{https://proceedings.mlr.press/v151/deng22b}.
Their experiments consist of:
\begin{compactitem}
\item[(a)] an one-dimensional experiment with sudden changes at time steps $t=100$ and $t=200$,
\item[(b)] an one-dimensional experiment with a slowly changing objective function,
\item[(c)] an experiment on real-world stock market data.\footnote{Note that stock market data does not fit the fixed variational budget setting as stock prices constantly change. However, since this example was used in prior work, we treat it as a benchmark but wanted to highlight that \textit{ongoing change} is the more suitable assumption.}
\end{compactitem}
We refer to \citet{Deng2022} for the details on how the objective function for (a) and (b) is composed.
%
We implemented our algorithm ET-GP-UCB in their \textsc{Matlab} code base, added random search as a baseline.
While going through the implementation details, we noted that in the stock market data benchmark, the data was not normalized and values of the objective function ranged from $19$ to $250$.
This is problematic as all algorithms use a prior mean of zero and a prior variance of one.
Thus, all observations will have a very small likelihood under the given model.
Secondly, the noise variance was set to $300$, which seems high for stock data where observation noise should be low.
The high noise variance combined with the prior signal variance of one also explains the very high standard deviations of all the algorithms in \citet[Figure~1~(c)]{Deng2022}.
Overall, the stock market data has a very low marginal likelihood for the model chosen in \cite{Deng2022}.
We, therefore, decided to improve the model and add a preprocessing step to the stock market data example by normalizing the data to a mean of zero and a variance of one to fit the prior model assumptions of all algorithms.
Next, we chose a noise variance of $0.01$ to account for smaller short term fluctuations in stock prices.
We also changed the noise variance of the synthetic experiments (a) and (b) to $0.1$.
It was previously set to $1$ (same as the signal variance), amounting to a 'signal to noise ratio' of $1$, which is very low for practical optimization problems.
We then re-ran their experiments using $20$ runs for the synthetic experiments and $5$ runs for the stock market data as in \citet{Deng2022}.
For ET-GP-UCB, we use the same parameters for the event-trigger as in all previous experiments.
Note that all other time-varying algorithms rely on specifying the amount of change $B_T$ a priori, which is hard to estimate in practice.
The results are shown in Fig.~\ref{fig:deng_refactored} and regret at the last time step is listed in Table~\ref{tab:results_deng}.
%
\begin{figure*}[t]
\centering
\includegraphics[]{figures/pdf_plots/deng_refactored.pdf}%
\caption{Performance using the refactored \gls{gp} prior in the experiments of \citet{Deng2022}.}
\label{fig:deng_refactored}
\end{figure*}
We can see that ET-GP-UCB can outperform all baselines in experiments Fig.~\ref{fig:deng_refactored}~(a) to (c).
Especially in Fig.~\ref{fig:deng_refactored}~(a), ET-GP-UCB can detect the sudden changes in the objective function and then quickly find the new optimum due to the simple one-dimensional objective function.
In \fig\ref{fig:deng_refactored}~(b), ET-GP-UCB is the best performing algorithm, but W-GP-UCB is competitive.
Considering the good performance of IGP-UCB, we can also conclude that the changes in the objective function over the time horizon are relatively small.
In \fig\ref{fig:deng_refactored}~(c), ET-GP-UCB is again the best-performing algorithm.
We also observe that in Fig.~\ref{fig:deng_refactored}~(c) \textit{all} algorithms significantly improve in performance and yield a smaller standard deviation compared to \citet[Figure~1~(c)]{Deng2022} because the GP prior model is more suitable for modeling the objective function.
After approx. $500$ time steps ET-GP-UCB, W-GP-UCB, SW-GP-UCB, and IGP-UCB (see Table~\ref{tab:results_deng}), no longer accumulate regret.
Looking at the objective function, this is expected as the optimal bandit arm no longer changes and the optimal values do not change significantly.
\begin{table}[h]
\caption{Comparison on experiments in \citet{Deng2022}.}
\label{tab:results_deng}
\begin{center}
\begin{tabular}{l r@{}l r@{}l r@{}l}
\toprule
&\multicolumn{2}{c}{(a) Abruptly-change} & \multicolumn{2}{c}{(b) Slowly-change} & \multicolumn{2}{c}{(c) Stock market} \\
\textbf{Algorithm} & \multicolumn{2}{c}{\textbf{Regret} $(\mu \pm \sigma)$} & \multicolumn{2}{c}{\textbf{Regret} $(\mu \pm \sigma)$} & \multicolumn{2}{c}{\textbf{Regret} $(\mu \pm \sigma)$} \\
\midrule
IGP-UCB & $\quad704.6$&$\pm 238.9$ & $\quad136.0$&$\pm 32.2$ & $11\,462.3$&$\pm 56.0$ \\
BR-GP-UCB & $312.5$&$\pm 4.3$ & $153.9$&$\pm 11.0$ & $45\,362.2$&$\pm 430.1$\\
SW-GP-UCB & $233.2$&$\pm 9.1$ & $111.6$&$\pm 15.7$ & $17\,854.9$&$\pm 27.9$ \\
W-GP-UCB & $298.8$&$\pm 27.3$ & $197.0$&$\pm 29.7$ & $11\,461.3$&$\pm 32.9$ \\
ET-GP-UCB (ours) & $\bm{55.8}$&$\pm 2.7$ & $\bm{86.3}$&$\pm 42.8$ & $\bm{2\,483.8}$&$\pm 39.8$ \\
\bottomrule
\end{tabular}
\end{center}
\end{table}
\subsection*{Results with the Original Gaussian Process Prior}
Below in \fig\ref{fig:deng_original} are the results using the original \gls{gp} prior from \cite{Deng2022}.
%
We can observe that for \fig\ref{fig:deng_original}~(a), ET-GP-UCB can reliably detect the first change in the objective function at $t=100$.
However, due to the low signal-to-noise ratio, ET-GP-UCB can only detect the second change approx. $50\%$ of the time.
It is still able to outperform periodic resetting as well as the sliding window approach.
In \fig\ref{fig:deng_original}~(b), ET-GP-UCB shows a similar performance as W-GP-UCB.
For the stock market data, we observe the discussed large standard deviations.
All algorithms outperform the random baseline.
However, we can observe that the ordering of algorithms has changed compared to \citet[Figure~1~(c)]{Deng2022} even though we used their implementations and only added ET-GP-UCB and the random baseline to the run script.
Further changing the random seed again changed this ordering of the algorithms which indicates that the prior model assumptions yield random-like behavior as the prior does not allow the \gls{gp} model to make meaningful decision.
\begin{figure*}[t]
\centering
\includegraphics[]{figures/pdf_plots/deng_original.pdf}%
\caption{Performance using the original \gls{gp} prior for the experiments of \citet{Deng2022}.}
\label{fig:deng_original}
\end{figure*}
\section{Sensitivity of ET-GP-UCB to the Event-Trigger Hyperparameter $\delta_{\mathrm{B}}$} \label{sec:sensitivity}
Table~\ref{tab:sensitivity_delta} shows the sensitivity of ET-GP-UCB to the event-trigger parameter $\delta_\mathrm{B}$ in Lemma~\ref{lem:trigger_bound}.
As $\delta_\mathrm{B}$ is increased, the average number of resets increases as the probabilistic uniform error bound is softened.
We observe, that for different rate of changes $\varepsilon$ different $\delta_\mathrm{B}$ may be optimal to obtain the lowest possible empirical regret.
ET-GP-UCB displays a low sensitivity to $\delta_\mathrm{B}$ highlighting that our algorithm is readily applicable without the need for excessive hyperparameter tuning.
In practice, we found $\delta_\mathrm{B}=0.1$ to yield good empirical performance in all of our experiments.
\begin{table}[h]
\caption{Sensitivity of ET-GP-UCB to $\delta_{\mathrm{B}}$ in Lemma~\ref{lem:trigger_bound}.}
\label{tab:sensitivity_delta}
\begin{center}
\begin{small}
\begin{tabular}{c c c c}
\toprule
\textbf{Rate of Change} $\varepsilon$ & \textbf{\algname Hyperparameter} $\delta_{\mathrm{B}}$ & \textbf{Avg. No. of Resets} & \textbf{Avg. Regret} $R_T$ \\
\midrule
&$0.005$ & $2.66$ & $191.39$ \\
&$0.01$ & $2.96$ & $193.05$\\
$0.01$ &$0.05$ & $3.20$ & $200.16$ \\
&$0.1$ & $3.38$ & $200.33$ \\
&$0.5$ & $3.98$ & $196.03$ \\
\midrule
&$0.005$ & $6.42$ & $276.84$ \\
&$0.01$ & $6.82$ & $273.37$\\
$0.03$ &$0.05$ & $7.72$ & $269.01$ \\
&$0.1$ & $8.04$ & $271.59$ \\
&$0.5$ & $10.32$ & $280.05$ \\
\midrule
&$0.005$ & $9.60$ & $331.69$ \\
&$0.01$ & $9.96$ & $328.74$\\
$0.05 $ &$0.05$ & $11.40$ & $329.47$ \\
&$0.1$ & $11.88$ & $332.04$ \\
&$0.5 $ & $14.44$ & $334.80$ \\
\bottomrule
\end{tabular}
\end{small}
\end{center}
\vskip -0.1in
\end{table}
\section{Proof of Lemma~\ref{lem:bounded_noise}, Lemma~\ref{lem:trigger_bound}, and Lemma~\ref{lem:markov_bound}} \label{sec:lemma3}
\begin{proof}[Proof of Lemma~\ref{lem:bounded_noise}]
Since $w_{\triggerT} \sim \mathcal{N}(0, \sigma_n^2)$, we can use a standard bound on the tail probability of the normal distribution $\mathbb{P}\left\{|r| > s\right\} \leq \exp \left\{ - s^2/(2 \sigma^2)\right\}$ with $r\sim\mathcal{N}(0, \sigma^2)$ to obtain $\mathbb{P}\left\{|w_{\triggerT}| \leq \bar{w}_{\triggerT}\right\} \geq 1 - \exp \left\{- \bar{w}_{\triggerT}^2 / (2 \sigma_n^2)\right\}$.
Solving for $\bar{w}_{t}$ with $\delta/\pi_{\triggerT}$ using the same arguments as in the proof of \citet[Lemma~5.5]{Srinivas2010} and taking the union bound over all time steps yields the results.
\end{proof}
\begin{proof}[Proof of Lemma~\ref{lem:trigger_bound}]
From Lemma~\ref{lem:srinivas} with $\triggerDelta/2$ and incorporating the noise, we have
\begin{align}
| f_t(\optvar_t) + w_{\triggerT} - \mu_{\mathcal{D}_t}(\optvar_t) | &\leq \sqrt{\rho_{\triggerT}} \sigma_{\mathcal{D}_t} (\optvar_t) + |w_{\triggerT}| \nonumber \\
| y_t - \mu_{\mathcal{D}_t}(\optvar_t) | &\leq \sqrt{\rho_{\triggerT}} \sigma_{\mathcal{D}_t} (\optvar_t) + |w_{\triggerT}|.
\end{align}
with $\rho_{\triggerT} = 2\ln{ \frac{2\pi_{\triggerT}}{\triggerDelta}}$. Using Lemma~\ref{lem:bounded_noise} with $\triggerDelta/2$ and taking the union bound yields the result.
\end{proof}
\begin{proof}[Proof of Lemma~\ref{lem:markov_bound}]
We can use Markov's inequality to prove the claim, as $\stopT$ is a non-negative random variable.
\end{proof}
\section{Proof of Theorem~\ref{theorem:regret_bounds}} \label{sec:proof}
The proof of Theorem~\ref{theorem:regret_bounds} uses similar concepts as \citet{Bogunovic2016} and \citet{Srinivas2010} with the key distinction, that we will explicitly include the event-trigger and its stopping time into our proof.
As in \citet{Bogunovic2016} and \citet{Srinivas2010}, we first fix a discretization $\mathbb{X}_t \subset \mathbb{X} \subseteq [0, r]^d$ of size $(\vartheta_t)^d$ satisfying
\begin{equation}
\| \optvar -[\optvar]_t \| \leq\, r d / \vartheta_t, \quad \forall \optvar\in \mathbb{X}_t
\end{equation}
where $[\optvar]_t$ denotes the closest point in $\mathbb{X}_t$ to $\optvar$. Now fix a constant $\delta > 0$ and an increasing sequence of positive constants $\{\pi_t \}_{t=1}^\infty$ satisfying $\sum_{t \geq 1} \pi_t^{-1} = 1$ (\cf Remark~\ref{rem:pi_t}).
Next, we introduce three Lemmas needed to bound the regret.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% START LEMMAS %%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{lemma}[{\citet[][Proof of Theorem 4.2]{Bogunovic2016}}] \label{lem:xt_bound}
Pick $\delta \in (0,1)$ and set $\beta_t = 2 \ln \frac{\pi_t}{\delta}$, where $\sum_{t \geq 1} \pi_t^{-1} = 1$, $\pi_t >0$. Then we have
\begin{equation}
\mathbb{P}\left\{| f_t(\optvar_t) - \tilde\mu_{\mathcal{D}_t}(\optvar_t) | \leq\, \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t} (\optvar_t), \quad\forall t\geq1 \right\} \geq 1 - \delta.
\end{equation}
\end{lemma}
\begin{proof}
\citet[][Appendix C.1]{Bogunovic2016}.
\end{proof}
\begin{lemma} \label{lem:bounded_y}
Pick $\delta \in (0,1)$ and set $\bar{y}_{t} = b_0 \sqrt{\ln\frac{2 a_0 \pi_t}{\delta}} + \sqrt{2\sigma_n^2 \ln \frac{2\pi_t}{\delta}}$, where $\sum_{t \geq 1} \pi_t^{-1} = 1$, $\pi_t >0$. Then we have
\begin{equation}
\mathbb{P}\left\{|y_t| \leq\, \bar{y}_{t}, \, \forall t \geq 1\right\} \geq 1 - \delta.
\end{equation}
\end{lemma}
\begin{proof}
We have that $|y_t| \leq\, |f_t(\optvar_t)| + |w_t|$ given Assumption~\ref{ass:noisy_obs}. We can use Lemma~\ref{lem:bounded_noise} (with $t$ instead of $t'$) with $\delta/2$ to bound $|w_t|$ for all time steps. Combined with Assumption~\ref{ass:smoothness_assumption} we have
\begin{equation}
\mathbb{P}\left\{|y_t| \leq\, L_f + \bar{w}_t \right\} \geq 1 - a_0 \exp \{ - L_f^2/ b_0^2\} - \delta/2.
\end{equation}
Solving the remaining term with $\delta / 2$ and using the union bound, we get
$
\bar{y}_{t} = L_f + \bar{w}_t = b_0 \sqrt{\ln\frac{2 a_0 \pi_t}{\delta}} + \bar{w}_t.
$
\end{proof}
\begin{lemma}[{\citet[][Proof of Theorem 4.1]{Bogunovic2016}}] \label{lem:disc_bound}
Pick $\delta \in (0, 1)$ and set
\[
\beta_t = 2 \ln\left(\frac{2 \pi_t}{\delta}\right) + 2 d \ln{\left(r d t^2 b_1 \sqrt{\ln(2 a_1 d \pi_t/ \delta)}\right)},
\]
where $\sum_{t \geq 1} \pi_t^{-1} = 1$, $\pi_t >0$. Let $\vartheta_t = r d t^2 b_1 \sqrt{\ln(2 a_1 d \pi_t/ \delta)}$ and let $[\optvar^*]_t$ be the closest point in $\mathbb{X}_t$ to $\optvar^*$. Then we have
\begin{equation}
\mathbb{P}\left\{| f_t(\optvar_t^*) - \tilde\mu_{\mathcal{D}_t}([\optvar_t^*]_t) | \leq\, \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t} ([\optvar_t^*]_t) + \frac{1}{t^2}, \,\,\forall t\geq 1 \right\} \geq 1 - \delta.
\end{equation}
\end{lemma}
\begin{proof}
In \citet[][Appendix C.1]{Bogunovic2016} using Assumption~\ref{ass:smoothness_assumption} on the kernel function.
\end{proof}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% END LEMMAS %%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
For \theo\ref{theorem:regret_bounds}, we condition on five high probability events \ie Lemma~\ref{lem:bounded_noise}, Lemma~\ref{lem:markov_bound}, Lemma~\ref{lem:xt_bound}, Lemma~\ref{lem:bounded_y}, and Lemma~\ref{lem:disc_bound} each with with $\delta/5$.
We will explicitly highlight the usage of each Lemma. We can bound the immediate regret at a time step $t$ as
\begin{align}
r_t &= f_t(\optvar_t^*) - f_t(\optvar_t) \\
&\leq\, \tilde\mu_{\mathcal{D}_t}([\optvar_t^*]_t) + \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t}([\optvar_t^*]_t) + \frac{1}{t^2} - \tilde\mu_{\mathcal{D}_t}(\optvar_t) + \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t}(\optvar_t). \qquad \text{(Lemma~\ref{lem:disc_bound}, Lemma~\ref{lem:xt_bound})}
\end{align}
This is identical to \citet[][Proof of Theorem 4.2]{Bogunovic2016} for R-GP-UCB.
We now account for model mismatch between using the time-invariant model in the algorithm and the ``true" time-varying posterior as
\begin{align}
\tilde\mu_{\mathcal{D}_t}(\optvar) &\leq\, \mu_{\mathcal{D}_t}(\optvar) + \Delta_{\mathcal{D}_t}^{(\mu)}(\optvar)\label{eq:delta_mu}\qquad \text{with} \quad \Delta_{\mathcal{D}_t}^{(\mu)}(\optvar) \coloneqq |\tilde\mu_{\mathcal{D}_t}(\optvar) - \mu_{\mathcal{D}_t}(\optvar)| \\
\tilde\sigma_{\mathcal{D}_t}(\optvar) &\leq\, \sigma_{\mathcal{D}_t}(\optvar) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar) \label{eq:delta_sigma}\qquad \text{with} \quad \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar) \coloneqq |\tilde\sigma_{\mathcal{D}_t}(\optvar) - \sigma_{\mathcal{D}_t}(\optvar)|.
\end{align}
We can thus further bound the regret as
\begin{align}
r_t \leq\,\, & \mu_{\mathcal{D}_t}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\mu)}([\optvar_t^*]_t) + \sqrt{\beta_t} \left(\sigma_{\mathcal{D}_t}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}([\optvar_t^*]_t)\right)\\
&- \mu_{\mathcal{D}_t}(\optvar_t) + \Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) + \sqrt{\beta_t} \left(\sigma_{\mathcal{D}_t}(\optvar_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar_t)\right)+ \frac{1}{t^2}.
\end{align}
Per definition of Algorithm~\ref{alg:algorithm}, we have that through the optimization of its UCB-type acquisition function $\mu_{\mathcal{D}_t}([\optvar_t^*]_t) + \sqrt{\beta_t}\sigma_{\mathcal{D}_t}([\optvar_t^*]_t) \leq\, \mu_{\mathcal{D}_t}(\optvar_t) + \sqrt{\beta_t}\sigma_{\mathcal{D}_t}(\optvar_t)$, and, therefore,
\begin{align}
r_t \leq\, & 2 \sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + \frac{1}{t^2} + \Delta_{\mathcal{D}_t}^{(\mu)}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) + \sqrt{\beta_t} \left( \Delta_{\mathcal{D}_t}^{(\sigma)}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar_t) \right). \label{eq:unbounded_regret}
\end{align}
What is left to bound is the model mismatch.
First, let's consider the variance.
Here, we use the bounds on the variance as in \citet[][Proof of Theorem 4.2 (R-GP-UCB)]{Bogunovic2016}\footnote{While working on this proof, we noticed a minor mistake in \citet[Theorem~4.2]{Bogunovic2016}. There, it should be $2\cdot T\psi_T(N,\epsilon)$ in Eq.~(17) as model differences in mean and variance have to be bounded twice as it is the case for the variance in our proof (\cf \citet[Proof of Theorem~4.2]{Bogunovic2016}, Eq.~(72)). This mistake does not weaken the significant contributions of \cite{Bogunovic2016}, but this information might be helpful for future work on R-GP-UCB.} using \citet[][Lemma~D.1.]{Bogunovic2016} with $|\mathcal{D}_t| < \bar{\stopT}$ (Lemma~\ref{lem:markov_bound} and the query location and measurement pair at time $t$ is part of the new dataset) as
\begin{equation}
\Delta_{\mathcal{D}_t}^{(\sigma)}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar_t) \leq\, 2 \sup_{\optvar \in \mathbb{X}} \{|\tilde\sigma_{\mathcal{D}_t}(\optvar) - \sigma_{\mathcal{D}_t}(\optvar)|\} \leq\, 2 \sqrt{(3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon}. \label{eq:bounds_variance}
\end{equation}
Note, that other bounds on the model mismatch are possible \eg using the fact that the output of the kernel function is bounded by one, but these approaches would loose the desirable explicit dependency of the bound on $\varepsilon$.
Through this explicit dependency we can evaluate our algorithms performance for different $\varepsilon$.
The same argument applies to the bound on model mismatch in the mean.
To bound $\Delta_{\mathcal{D}_t}^{(\mu)}([\optvar_t^*]_t)$ in \eqref{eq:unbounded_regret} we again use \citet[Lemma~D.1.]{Bogunovic2016} with $\bar{\stopT}$ as
\begin{equation}
\Delta_{\mathcal{D}_t}^{(\mu)}([\optvar_t^*]_t) \leq\, \sup_{\optvar\in \mathbb{X}} \{|\tilde\mu_{\mathcal{D}_t}(\optvar) - \mu_{\mathcal{D}_t}(\optvar)|\} \leq\, (\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \bar{y}_{t} \varepsilon \qquad \text{($\bar{y}_{t}$ from Lemma~\ref{lem:bounded_y})}.
\end{equation}
Now, instead of using the same coarse bound for $\Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t)$ we can use our event-trigger. For this, we write
\begin{alignat}{2}
\Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) = |\tilde\mu_{\mathcal{D}_t}(\optvar_t) - \mu_{\mathcal{D}_t}(\optvar_t)| &= |\tilde\mu_{\mathcal{D}_t}(\optvar_t) - y_t &&+ y_t - \mu_{\mathcal{D}_t}(\optvar_t)| \\
&= |\tilde\mu_{\mathcal{D}_t}(\optvar_t) - f_t(\optvar_t) - w_t &&+ y_t - \mu_{\mathcal{D}_t}(\optvar_t)| \\
&\leq\, \underbrace{|\tilde\mu_{\mathcal{D}_t}(\optvar_t) - f_t(\optvar_t)|}_{\text{see Lemma~\ref{lem:xt_bound}}} + |w_{t}| &&+ |y_t - \mu_{\mathcal{D}_t}(\optvar_t)| \\
&\leq\, \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t}(\optvar_t) + |w_{t}| &&+ |y_t - \mu_{\mathcal{D}_t}(\optvar_t)|
\end{alignat}
To bound $|y_t - \mu_{\mathcal{D}_t}(\optvar_t)|$, which is the test function $\psi(\optvar_t)$ in \eqref{eq:test_function} of our event trigger, we can use the corresponding threshold function in \eqref{eq:threshold_function}. Note, that the bound on the test function does not hold in probability, since in each iteration, the event-trigger is explicitly evaluated. If the event-trigger is activated \ie if the test function exceeds the threshold function, the dataset is reset to again satisfying the bound. Further note that the query location and measurement pair activating the trigger is part of the \textit{new} dataset, not the old dataset (see Algorithm \ref{alg:algorithm})! Therefore, we obtain
\begin{align}
\Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) &\leq\, \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t}(\optvar_t) + |w_{t}| + \underbrace{|y_t - \mu_{\mathcal{D}_t}(\optvar_t)|}_{\psi(\optvar_t) \text{ see \eqref{eq:test_function}}} \\
&\leq\, \sqrt{\beta_t} \tilde\sigma_{\mathcal{D}_t}(\optvar_t) + 2\bar{w}_t + \sqrt{\rho_\triggerT} \sigma_{\mathcal{D}_t}(\optvar_t) \quad \text{(see \eqref{eq:threshold_function} and Lemma~\ref{lem:bounded_noise} but with $t$)}
\end{align}
and since $\beta_t \geq \rho_\triggerT$ for all $t$ as we require $\triggerDelta \geq \delta / 5$, we can write
\begin{align}
\Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t)&\leq\, \sqrt{\beta_t} \left(\sigma_{\mathcal{D}_t}(\optvar_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar) + \sigma_{\mathcal{D}_t}(\optvar_t) \right) + 2 \bar{w}_t \\
&\leq\, 2\sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + \sqrt{\beta_t}\underbrace{\Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar_t)}_{\text{see \eqref{eq:bounds_variance}}} + 2 \bar{w}_t\\
&\leq\, 2\sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + \sqrt{\beta_t (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \varepsilon} + 2 \bar{w}_t. \label{eq:noise_weg}
\end{align}
Since $\beta_t = 2 \ln\left( 10 \pi_t / \delta \right) + g(\cdot)$ where $g(\cdot) \geq 0$ and $\bar{w}_t = \sqrt{2 \sigma_n^2\ln\left( 5 \pi_t / \delta \right)}$, we have with the monotonicity of the logarithm that $\bar{w}_t \leq \sqrt{\beta_t}\sigma_n$ for all $t$.
Furthermore, we can state that the posterior standard deviation is always lower bounded by the standard deviation of the noise as $\sigma_n \leq \sigma_{\mathcal{D}_t}(\optvar_t)$ for all $\optvar_t \in \mathbb{X}_t$ and all $t$.
This allows to upper bound the noise bound using the posterior standard deviation to further bound \eqref{eq:noise_weg} as
\begin{equation}
\Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) \leq\, 4\sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + \sqrt{\beta_t (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \varepsilon}.
\end{equation}
Here, we can also directly observe the cost not knowing $\varepsilon$ but relying on the trigger instead.
We obtain an additional $4\sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t)$ term which in the end will result in a higher $C_1$ constant compared to R-GP-UCB in \cite[Theorem~4.2]{Bogunovic2016}.
To summarize, we yield the bound on the model mismatch at each time step as
\begin{align}
\Delta_{\mathcal{D}_t}^{(\mu)}([\optvar_t^*]_t)& + \Delta_{\mathcal{D}_t}^{(\mu)}(\optvar_t) + \sqrt{\beta_t}\left( \Delta_{\mathcal{D}_t}^{(\sigma)}([\optvar_t^*]_t) + \Delta_{\mathcal{D}_t}^{(\sigma)}(\optvar_t) \right) \\
\leq\, & (\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \bar{y}_{t} \varepsilon + 4\sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + 3\sqrt{\beta_t (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon} \eqqcolon \phi_t(\varepsilon,\delta,\bar{\stopT})
\end{align}
For completeness, we follow the step in \citet[Proof of Theorem~2]{Srinivas2010} and \citet[Proof of Theorem~4.2 (R-GP-UCB)]{Bogunovic2016} to bound the regret following \defn\ref{def:regret} as
\begin{equation}
R_T \coloneqq \sum_{t=1}^T r_t \leq\, \sum_{t=1}^T 6 \sqrt{\beta_t} \sigma_{\mathcal{D}_t}(\optvar_t) + \frac{1}{t^2} + \phi_t(\varepsilon, \delta, \bar{\stopT}) \label{eq:regret1}.
\end{equation}
With $\sum_{t=1}^\infty 1/t^2 = \pi^2/6 \leq\, 2$ and using Cauchy-Schwarz with $\left(\sum_{t=1}^T \optvar_t\right)^2 \leq\, T \sum_{t=1}^T \optvar_t^2$ we obtain
\begin{equation}
R_T \leq\, \sqrt{T \sum_{t=1}^T 36 \beta_t \sigma_{\mathcal{D}_t}^2(\optvar_t)} + 2 + \sum_{t=1}^T \left((\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \bar{y}_{t} \varepsilon + 3\sqrt{\beta_t (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon} \right). \label{eq:regret2}
\end{equation}
Furthermore, using the same arguments as in \citet[Proof of Theorem~4.2 (R-GP-UCB)]{Bogunovic2016} and \citet[][Proof of Lemma~5.3 (GP-UCB)]{Srinivas2010} on the information gain within a block of size $M$ is $I(y_M; f_M) = \frac{1}{2}\sum_{t=1}^M \ln\left(1 + \sigma_n^2 \sigma_{\mathcal{D}_t}^2(\optvar_t) \right) \leq\, \gamma_M$ with $\gamma_M = \max_{\optvar_1, \dots \optvar_\stopT} I(y_M; f_M)$ yields:
\begin{align}
\sum_{t=1}^T 36 \beta_t \sigma_{\mathcal{D}_t}^2(\optvar_t) &\leq\, 36 \beta_T \sigma_n^2 \sum_{t=1}^T \left(\sigma_n^{-2} \sigma_{\mathcal{D}_t}^2(\optvar_t) \right) \\
&\leq\, 36 \beta_T \sigma_n^2 C_2 \sum_{t=1}^T \ln \left(1+\sigma_n^{-2} \sigma_{\mathcal{D}_t}^2(\optvar_t) \right) \\
\intertext{with $C_2 = \sigma_n^{-2} / \ln(1 + \sigma_n^{-2}) \geq 1$, since $s^2 \leq\, C_2 \ln(1 + s^2)$ for $s \in [0, \sigma_n^2]$, and $\sigma_n^{-2}\sigma_{\mathcal{D}_t}^2(\optvar_t) \leq\, \sigma_n^{-2}k(\optvar_t,\optvar_t)\leq\, \sigma_n^{-2}$ due to the bounded kernel function in Assumption~\ref{ass:markov_model} (also see \citet[][Proof of Lemma~5.4]{Srinivas2010}, we get}
&\leq\, 72 \beta_T \sigma_n^2 C_2 \frac{T}{\bar \stopT} I(y_{\bar \stopT}; f_{\bar \stopT}) \qquad \text{($\frac{T}{\bar \stopT}$ blocks of $I(y_{\bar \stopT}; f_{\bar \stopT})$ in $T$)} \\
& \leq\, 72 \beta_T \sigma_n^2 C_2 \frac{T}{\bar \stopT} \gamma_{\bar \stopT} %\qquad \text{(maximum information gain $\gamma_{\bar \stopT}$ as upper bound)}
\end{align}
We assumed that $\frac{T}{\bar \stopT}$ is an integer as the number of block in $T$ and we can upper this with $\big(\frac{T}{\bar \stopT} + 1 \big)$. Therefore, we have
\begin{equation}
R_T \leq\, \sqrt{C_1 T \beta_T \left( \frac{T}{\bar{\stopT}} + 1\right)\gamma_{\bar{\stopT}}} + 2 + \sum_{t=1}^T\left( (\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \bar{y}_{t} \varepsilon + 3\sqrt{\beta_t (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon} \right) \label{eq:regret3}
\end{equation}
with $C_1 = 72/\ln(1+\sigma_n^{-2})$. Further bounding the sum over model mismatch with $\beta_t \leq\, \beta_T$, $\bar{w}_t \leq\, \bar{w}_T$, and $\bar{y}_t \leq\, \bar{y}_T$ for all $t$ yields
\begin{align}
R_T &\leq\, \sqrt{C_1 T \beta_T \left( \frac{T}{\bar{\stopT}} + 1\right)\gamma_{\bar{\stopT}}} + 2 + T \left(\underbrace{(\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT}^3 \bar{y}_{T} \varepsilon + 3 \sqrt{\beta_T (3\sigma_n^{-2} + \sigma_n^{-4}) \bar{\stopT} ^3 \varepsilon}}_{\phi_T(\varepsilon, \delta, \bar{\stopT}) \text{ in Theorem~\ref{theorem:regret_bounds}}}\right). \label{eq:model_mismatch3}
\end{align}
\QEDA
\section{Overview of the Temperature Data} \label{sec:train_days}
%
In \fig\ref{fig:temps}, we show the normalized temperature data used for the empirical kernel in the results in \fig\ref{fig:temperature_results}. The full pre-processing of the temperature data is described in the corresponding jupyter-notebook in the provided code base.
\begin{figure}[h]
\centering
\includegraphics[]{figures/pdf_plots/temperature_training_data.pdf}
\caption{Normalized temperature data used for the results in \fig\ref{fig:temperature_results}. Each color resembles the measurement data from one sensor.}
\label{fig:temps}
\end{figure}
\section{Hyperparameters} \label{sec:hypers}
To ensure the reproducibility of our results, we give a list of all hyperparameters used in any simulation performed for this paper. All experiments in Sec.~\ref{sec:empirical_experiments} were conducted on a 2021 MacBook Pro with an Apple M1 Pro chip and 16GB RAM. The Monte Carlo simulations in Sec.~\ref{sec:intuition} were conducted over 3 days on a compute cluster. The temperature data set is publicly available at \url{http://db.csail.mit.edu/labdata/labdata.html}. The code will be published upon acceptance and is also part of the supplementary material of the submission.
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters of the Monte Carlo Simulation for \algname (\cf \fig\ref{fig:stopping_time}).}
\begin{tabular}{l c}
\toprule
\textbf{Hyperparameters} & \textbf{Monte Carlo Simulation}\\
\midrule
dimension $d$ & $2$\\
compact set $\X$ & $[0, 1]^2$ \\
lengthscales $l$ & $0.2$\\
time horizon $T$ & $200$\\
noise variance $\sigma_n^2$ & $0.02$\\
number of i.i.d. runs & $30\, 000$\\
event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $0.1$\\
rate of change $\varepsilon$ & $\{ 0.03, 0.1 \}$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\label{tab:mc}
\end{table}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% WM %%%%%%%%%%%%%
\begin{table}[h]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the within-model comparisons (\cf \fig\ref{fig:within_model}).}
\begin{tabular}{c l c c}
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Comparisons (a) to (c)} & \textbf{Comparison (d)}\\
\midrule
&dimension $d$ & $2$ & $2$\\
&compact set $\X$ & $[0, 1]^2$ & $[0, 1]^2$\\
&lengthscales $l$ & $0.2$ & $0.2$\\
all &time horizon $T$ & $400$ & $400$\\
&noise variance $\sigma_n^2$ & $0.02$ & $0.02$\\
&$\beta_t$ approximation parameters & $c_1=0.4, c_2=4$ & $c_1=0.4, c_2=4$\\
&number of i.i.d. runs & $50$ & $50$\\
\midrule
TV-GP-UCB &rate of change $\varepsilon$ & $\{0.01, 0.03, 0.05\}$ & $0.001$\\
\midrule
R-GP-UCB & reset time $N_{\text{const}}$ & $\{38, 29, 26\}$ & $68$\\
\midrule
\algname & event-trigger parameter $\triggerDelta$ & $0.1$ & $0.1$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\end{table}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% TEMP %%%%%%%%%%%%%
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the temperature dataset (\cf \fig\ref{fig:temperature_results}).}
\begin{tabular}{c l c }
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Temperature Dataset}\\
\midrule
&dimension $d$ & $1$ \\
&compact set $\X$ & 46 arms \\
&kernel $k$ & empirical kernel \\
all &time horizon $T$ & $286$\\
&noise variance $\sigma_n^2$ & $0.01$\\
&$\beta_t$ approximation parameters & $c_1=0.8, c_2=0.4$ \\
&number of i.i.d. runs & $50$\\
\midrule
TV-GP-UCB &rate of change $\varepsilon$ (see \citet{Bogunovic2016}) & $0.03$ \\
\midrule
R-GP-UCB & reset time $N_{\text{const}}$ (see \citet{Bogunovic2016}) & $15$\\
\midrule
\algname & event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $0.1$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% LQR %%%%%%%%%%%%%
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the policy search experiment (\cf \fig\ref{fig:lqr_small}).}
\begin{tabular}{c l c }
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Policy search}\\
\midrule
&dimension $d$ & $4$ \\
& & lower bound to $[-3, -6, -50, -4]$ \\
&compact set $\X$ & upper bound to $[-2, -4, -25, -2]$ \\
& & scaling factors $[1 / 8, 1 / 4, 3, 1 / 8]$ \\
&lengthscales $l$ & Gamma prior $\mathcal{G}(15,10/3)$ \\
all &time horizon $T$ & $300$\\
&noise variance $\sigma_n^2$ & $0.02$\\
&$\beta_t$ approximation parameters & $c_1=0.8, c_2=4$ \\
&number of i.i.d. runs & $50$\\
\midrule
TV-GP-UCB &rate of change $\varepsilon$ & $0.03$ \\
\midrule
R-GP-UCB & reset time $N_{\text{const}}$ induced by rate of change $\varepsilon$ & $29$\\
\midrule
UI-TVBO &rate of change $\hat{\sigma}_w^2$ & $0.03$ \\
\midrule
\algname & event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $0.1$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\label{tab:lqr}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Deng et al. %%%%%%%%%%%%%
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the synthetic experiments in \citet{Deng2022}.}
\begin{tabular}{c l c }
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Synthetic Experiments}\\
\midrule
&dimension $d$ & $1$ \\
&compact set $\X$ & $[0, 1]^2$ \\
&kernel $k$ & fixed, $l = 0.2$ \\
&time horizon $T$ & $500$\\
&noise variance $\sigma_n^2$ & $\{0.01, 1\}$\\
&number of i.i.d. runs & $20$\\
\midrule
\algname & event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $0.1$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\label{tab:deng_hypers}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Deng et al. %%%%%%%%%%%%%
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the stock market data experiment in \citet{Deng2022}.}
\begin{tabular}{c l c }
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Stock Market Data}\\
\midrule
&dimension $d$ & $1$ \\
&compact set $\X$ & $48$ arms \\
&kernel $k$ & empirical kernel \\
&time horizon $T$ & $823$\\
&noise variance $\sigma_n^2$ & $\{0.01, 300 \}$\\
&number of i.i.d. runs & $20$\\
\midrule
\algname & event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $0.1$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\label{tab:deng_hypers2}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Sensitivity %%%%%%%%%%%%%
\begin{table}[h!]
\begin{center}
% \begin{sc}
\caption{Hyperparameters for the sensitivity analysis (\cf Table~\ref{tab:sensitivity_delta}).}
\begin{tabular}{c l c }
\toprule
\textbf{Algorithm} & \textbf{Hyperparameters} & \textbf{Sensitivity Analysis}\\
\midrule
&dimension $d$ & $2$ \\
&compact set $\X$ & $[0, 1]^2$ \\
&kernel $k$ & fixed, $l = 0.2$ \\
all &time horizon $T$ & $400$\\
&noise variance $\sigma_n^2$ & $0.02$\\
&$\beta_t$ approximation parameters & $c_1=0.4, c_2=4$ \\
&number of i.i.d. runs & $50$\\
\midrule
\algname & event-trigger parameter in Lemma~\ref{lem:trigger_bound} $\triggerDelta$ & $\{0.005, 0.01, 0.05, 0.1, 0.5\}$\\
\bottomrule
\end{tabular}
% \end{sc}
\end{center}
\label{tab:sensitivity_hypers}
\end{table}
\begin{thebibliography}{20}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Balandat et~al.(2020)Balandat, Karrer, Jiang, Daulton, Letham, Wilson,
and Bakshy]{Balandat2020}
Maximilian Balandat, Brian Karrer, Daniel~R. Jiang, Samuel Daulton, Benjamin
Letham, Andrew~Gordon Wilson, and Eytan Bakshy.
\newblock {BoTorch}: A framework for efficient {Monte}-{Carlo} {Bayesian}
optimization.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~33, 2020.
\bibitem[Besbes et~al.(2015)Besbes, Gur, and Zeevi]{Besbes2015}
Omar Besbes, Yonatan Gur, and Assaf Zeevi.
\newblock Non-stationary stochastic optimization.
\newblock \emph{Operations Research}, 63\penalty0 (5):\penalty0 1227–1244, 10
2015.
\newblock ISSN 1526-5463.
\bibitem[Bogunovic et~al.(2016)Bogunovic, Scarlett, and Cevher]{Bogunovic2016}
Ilija Bogunovic, Jonathan Scarlett, and Volkan Cevher.
\newblock Time-varying {Gaussian} process bandit optimization.
\newblock In \emph{Proceedings of The 19th International Conference on
Artificial Intelligence and Statistics}, volume~51, pages 314--323. PMLR, 05
2016.
\bibitem[Brunzema et~al.(2022)Brunzema, von Rohr, and Trimpe]{Brunzema2022}
Paul Brunzema, Alexander von Rohr, and Sebastian Trimpe.
\newblock On controller tuning with time-varying {Bayesian} optimization.
\newblock In \emph{61th IEEE Conference on Decision and Control}, 2022.
\bibitem[Chowdhury and Gopalan(2017)]{Chowdhury2017}
Sayak~Ray Chowdhury and Aditya Gopalan.
\newblock On kernelized multi-armed bandits.
\newblock In \emph{International Conference on Machine Learning}, pages
844--853. PMLR, 2017.
\bibitem[Deng et~al.(2022)Deng, Zhou, Kim, Tewari, Gupta, and Shroff]{Deng2022}
Yuntian Deng, Xingyu Zhou, Baekjin Kim, Ambuj Tewari, Abhishek Gupta, and Ness
Shroff.
\newblock Weighted {Gaussian} process bandits for non-stationary environments.
\newblock In \emph{Proceedings of The 25th International Conference on
Artificial Intelligence and Statistics}, volume 151, pages 6909--6932. PMLR,
28--30 Mar 2022.
\bibitem[Gardner et~al.(2018)Gardner, Pleiss, Weinberger, Bindel, and
Wilson]{Gardner2018}
Jacob Gardner, Geoff Pleiss, Kilian~Q Weinberger, David Bindel, and Andrew~G
Wilson.
\newblock {GPyTorch}: Blackbox matrix-matrix {Gaussian} process inference with
{GPU} acceleration.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~31. Curran Associates, Inc., 2018.
\bibitem[Garnett(2022)]{Garnett2022}
Roman Garnett.
\newblock \emph{{Bayesian} Optimization}.
\newblock Cambridge University Press, 2022.
\bibitem[Ghosal and Roy(2006)]{Ghosal2006}
Subhashis Ghosal and Anindya Roy.
\newblock {Posterior consistency of {Gaussian} process prior for nonparametric
binary regression}.
\newblock \emph{The Annals of Statistics}, 34\penalty0 (5):\penalty0 2413 --
2429, 2006.
\bibitem[Hennig and Schuler(2012)]{Hennig2012}
P.~Hennig and CJ. Schuler.
\newblock Entropy search for information-efficient global optimization.
\newblock \emph{Journal of Machine Learning Research}, 13:\penalty0 1809--1837,
June 2012.
\bibitem[Krause and Ong(2011)]{Krause2011}
Andreas Krause and Cheng Ong.
\newblock Contextual {Gaussian} process bandit optimization.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~24. Curran Associates, Inc., 2011.
\bibitem[König et~al.(2021)König, Turchetta, Lygeros, Rupenyan, and
Krause]{Koenig2021}
Christopher König, Matteo Turchetta, John Lygeros, Alisa Rupenyan, and Andreas
Krause.
\newblock Safe and efficient model-free adaptive control via {Bayesian}
optimization.
\newblock In \emph{IEEE International Conference on Robotics and Automation},
pages 9782--9788, 2021.
\bibitem[Parker-Holder et~al.(2020)Parker-Holder, Nguyen, and
Roberts]{ParkerHolder2020}
Jack Parker-Holder, Vu~Nguyen, and Stephen~J Roberts.
\newblock Provably efficient online hyperparameter optimization with
population-based bandits.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~33, pages 17200--17211. Curran Associates, Inc., 2020.
\bibitem[Parker-Holder et~al.(2021)Parker-Holder, Nguyen, Desai, and
Roberts]{ParkerHolder2021}
Jack Parker-Holder, Vu~Nguyen, Shaan Desai, and S~Roberts.
\newblock Tuning mixed input hyperparameters on the fly for efficient
population based {AutoRL}.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~34, pages 15513--15528. Curran Associates, Inc., 2021.
\bibitem[Solowjow and Trimpe(2020)]{Solowjow2020}
Friedrich Solowjow and Sebastian Trimpe.
\newblock Event-triggered learning.
\newblock \emph{Automatica}, 117, 2020.
\bibitem[Srinivas et~al.(2010)Srinivas, Krause, Kakade, and
Seeger]{Srinivas2010}
Niranjan Srinivas, Andreas Krause, Sham Kakade, and Matthias Seeger.
\newblock Gaussian process optimization in the bandit setting: No regret and
experimental design.
\newblock In \emph{Proceedings of the 27th International Conference on
International Conference on Machine Learning}, page 1015–1022. Omnipress,
2010.
\bibitem[Su et~al.(2018)Su, Wu, Cheng, and Chen]{Su2018}
Jie Su, Junfeng Wu, Peng Cheng, and Jiming Chen.
\newblock Autonomous vehicle control through the dynamics and controller
learning.
\newblock \emph{IEEE Transactions on Vehicular Technology}, 67\penalty0
(7):\penalty0 5650--5657, 2018.
\bibitem[Umlauft and Hirche(2019)]{Umlauft2019}
Jonas Umlauft and Sandra Hirche.
\newblock Feedback linearization based on {Gaussian} processes with
event-triggered online learning.
\newblock \emph{IEEE Transactions on Automatic Control}, 65\penalty0
(10):\penalty0 4154--4169, 2019.
\bibitem[Wei and Luo(2021)]{wei2021non}
Chen-Yu Wei and Haipeng Luo.
\newblock Non-stationary reinforcement learning without prior knowledge: An
optimal black-box approach.
\newblock In \emph{Conference on Learning Theory}, pages 4300--4354. PMLR,
2021.
\bibitem[Zhou and Shroff(2021)]{Zhou2021}
Xingyu Zhou and Ness Shroff.
\newblock No-regret algorithms for time-varying {Bayesian} optimization.
\newblock In \emph{Annual Conference on Information Sciences and Systems},
pages 1--6. IEEE, 2021.
\end{thebibliography}